diff --git a/App/backend/local-api-contracts/src/memory-l3-world-model.ts b/App/backend/local-api-contracts/src/memory-l3-world-model.ts index 269d5f023..bca1e15f9 100644 --- a/App/backend/local-api-contracts/src/memory-l3-world-model.ts +++ b/App/backend/local-api-contracts/src/memory-l3-world-model.ts @@ -83,6 +83,51 @@ export const L3WorldModelBoundaryResponseSchema = z.object({ }).strict(); export type L3WorldModelBoundaryResponse = z.infer; +const AbsolutePathSchema = z.string().min(1).max(4096) + .refine((value) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value), "path must be absolute"); + +/** Full list of folders the user picked in the desktop app; each push replaces the previous one. */ +export const L3UserProjectsReplaceRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + projects: z.array(z.object({ + rootPath: AbsolutePathSchema, + name: z.string().min(1).max(512) + }).strict()).max(500), + standaloneRoots: z.array(AbsolutePathSchema).max(20).default([]) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3UserProjectsReplaceRequest = z.input; + +export const L3UserProjectsReplaceResponseSchema = z.object({ + count: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3UserProjectsReplaceResponse = z.infer; + +/** Puts the user's new-L3 batches that ended in dead letter back in the queue. */ +export const L3FailedBatchesRetryRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3FailedBatchesRetryRequest = z.input; + +export const L3FailedBatchesRetryResponseSchema = z.object({ + requeued: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3FailedBatchesRetryResponse = z.infer; + +/** End of a hosted import: freeze what it added to these scanned Sessions into L3 batches. */ +export const L3ScanSessionsFreezeRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + sessionIds: z.array(NonEmptyStringSchema).max(10_000) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3ScanSessionsFreezeRequest = z.input; + +export const L3ScanSessionsFreezeResponseSchema = z.object({ + batches: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3ScanSessionsFreezeResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/App/backend/local-api-contracts/src/memory-runtime.ts b/App/backend/local-api-contracts/src/memory-runtime.ts index ec491f2eb..9a429bb25 100644 --- a/App/backend/local-api-contracts/src/memory-runtime.ts +++ b/App/backend/local-api-contracts/src/memory-runtime.ts @@ -56,6 +56,7 @@ export const JobTypeSchema = z.enum([ "l2_induction", "l3_abstraction", "l3_world_model_update", + "l3_unit_update", "project_environment_profile", "skill_crystallization", "skill_trial_resolve", diff --git a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts index 4cad6d205..93521e935 100644 --- a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts +++ b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts @@ -11,6 +11,7 @@ import { CloseSessionOutputSchema, CompleteTurnOutputSchema, SourceTurnCompleteOutputSchema, + L3ScanSessionsFreezeResponseSchema, DeleteMemoryOutputSchema, EmbeddingInferenceOutputSchema, DeletePanelTaskOutputSchema, @@ -32,6 +33,7 @@ import { RetryMemoryProcessingOutputSchema, WorkerRunOutputSchema } from "@memmy/local-api-contracts"; +import { randomUUID } from "node:crypto"; import { z, type ZodType } from "zod"; import { MemoryLayerError, MemoryLayerNetworkError } from "./errors.js"; import { buildMemoryLayerUrl, MEMORY_LAYER_PATHS } from "./memory-layer-endpoints.js"; @@ -206,6 +208,12 @@ export function createHttpMemoryClient( return request("POST", "completeSourceTurn", SourceTurnCompleteOutputSchema, { body: input, context }); }, + async freezeScanSessions(source, sessionIds, context) { + // The L3 envelope resolves the owner the same way the source turns that opened these Sessions did. + const body = { requestId: randomUUID(), adapterId: `agent-source:${source}`, source, sessionIds }; + return request("POST", "freezeScanSessions", L3ScanSessionsFreezeResponseSchema, { body, context }); + }, + async search(input, context) { return request("POST", "search", SearchOutputSchema, { body: input, context }); }, diff --git a/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts b/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts index fde606a37..7ea8f13c0 100644 --- a/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts +++ b/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts @@ -18,6 +18,7 @@ export const MEMORY_LAYER_PATHS = Object.freeze({ startTurn: "/api/v1/turns/start", completeTurn: "/api/v1/turns/:turnId/complete", completeSourceTurn: "/api/v1/source-turns/complete", + freezeScanSessions: "/api/v1/l3/scan-sessions/freeze", search: "/api/v1/memory/search", embeddingInference: "/api/v1/models/embedding/infer", addMemory: "/api/v1/memory/add", diff --git a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts index 77d9a06cc..d13a49925 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts @@ -45,6 +45,20 @@ describe("HttpMemoryClient", () => { expect(JSON.parse(String(calls[0]?.init?.body))).toEqual(input); expect(new Headers(calls[0]?.init?.headers).get("x-memmy-user-id")).toBe("fixture-user"); }); + it("hands scanned Sessions to L3 inside a full request envelope for their source", async () => { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const client = createHttpMemoryClient({ baseUrl: "http://memory.test", token: "fixture-token", timeoutMs: 500, maxRetries: 0 }, { + fetchImpl: (async (url, init) => { + calls.push({ url: String(url), init }); + return new Response(JSON.stringify({ batches: 2, serverTime: "2099-01-01T00:00:00.000Z" }), { status: 200, headers: { "content-type": "application/json" } }); + }) as typeof fetch + }); + expect(await client.freezeScanSessions("codex", ["s-1", "s-2"])).toEqual({ batches: 2, serverTime: "2099-01-01T00:00:00.000Z" }); + expect(calls[0]?.url).toBe("http://memory.test/api/v1/l3/scan-sessions/freeze"); + expect(JSON.parse(String(calls[0]?.init?.body))).toEqual({ + requestId: expect.stringMatching(/^[0-9a-f-]{36}$/), adapterId: "agent-source:codex", source: "codex", sessionIds: ["s-1", "s-2"] + }); + }); it("only defines path templates for the final memory HTTP APIs", () => { expect(Object.values(MEMORY_LAYER_PATHS)).toEqual([ "/api/v1/agent-sources", @@ -63,6 +77,7 @@ describe("HttpMemoryClient", () => { "/api/v1/turns/start", "/api/v1/turns/:turnId/complete", "/api/v1/source-turns/complete", + "/api/v1/l3/scan-sessions/freeze", "/api/v1/memory/search", "/api/v1/models/embedding/infer", "/api/v1/memory/add", diff --git a/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts index e68074f33..e1c729582 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts @@ -86,6 +86,7 @@ describe("createMockMemoryClient", () => { "completeSourceTurn", "deleteMemory", "enqueueImportSummaries", + "freezeScanSessions", "getMemory", "getMemoryProcessingStatus", "getMemoryTokenBudget", diff --git a/App/backend/src/adapters/outbound/memory-client/types.ts b/App/backend/src/adapters/outbound/memory-client/types.ts index 1bcf3e6ca..f2d5e118a 100644 --- a/App/backend/src/adapters/outbound/memory-client/types.ts +++ b/App/backend/src/adapters/outbound/memory-client/types.ts @@ -84,6 +84,8 @@ export interface MemoryClient { completeTurn(input: CompleteTurnInput & { turnId: string }, context?: MemoryRequestContext): Promise; completeSourceTurn(input: SourceTurnCompleteInput, context?: MemoryRequestContext): Promise; + /** Freeze what an import added to these scanned Sessions into L3 batches. */ + freezeScanSessions(source: string, sessionIds: string[], context?: MemoryRequestContext): Promise<{ batches: number }>; search(input: SearchInput, context?: MemoryRequestContext): Promise; embeddingInference?(input: EmbeddingInferenceInput, options?: { signal?: AbortSignal }): Promise; diff --git a/App/backend/src/services/ingestion-service.ts b/App/backend/src/services/ingestion-service.ts index 68c3dfd23..38518cc33 100644 --- a/App/backend/src/services/ingestion-service.ts +++ b/App/backend/src/services/ingestion-service.ts @@ -56,6 +56,8 @@ export interface IngestionStats { dedupedMemories: number; failedMemories: number; memoryIds: string[]; + /** Memory Sessions this import added turns to; frozen into L3 when the import ends. */ + sessionIds?: string[]; /** * Subset of `memoryIds` written through the legacy add-memory path, which still needs a * summary job. A native turn is summarized inside the Memory capture transaction, so its @@ -77,7 +79,7 @@ export interface IngestionItemSkip { /** Contract for create ingestion service options. */ export interface CreateIngestionServiceOptions { - memoryClient: Pick; + memoryClient: Pick & Partial>; agentSourceRepository: Pick; memoryAddAnalytics?: Pick< MemoryDesktopAddAnalytics, @@ -117,20 +119,30 @@ export function createIngestionService(options: CreateIngestionServiceOptions): }; const conversations = groupByConversation(messages, stats)[Symbol.asyncIterator](); - while (true) { - if (ctx.signal?.aborted) { - break; + try { + while (true) { + if (ctx.signal?.aborted) { + break; + } + + const next = await conversations.next(); + if (next.done) { + break; + } + + stats.conversations += 1; + await processConversation(options, next.value, ctx, stats); } - - const next = await conversations.next(); - if (next.done) { - break; + } finally { + // Imported history never ends its Sessions, so hand what this import added to L3 now, also + // when it stopped early. Freezing only writes rows in Memory; a failure must not fail the import. + const sessionIds = [...new Set(stats.sessionIds ?? [])]; + if (sessionIds.length > 0 && options.memoryClient.freezeScanSessions) { + await options.memoryClient.freezeScanSessions(ctx.sourceId, sessionIds).catch((error: unknown) => { + console.warn(`[agent-source] l3.freeze_failed ${error instanceof Error ? error.message : String(error)}`); + }); } - - stats.conversations += 1; - await processConversation(options, next.value, ctx, stats); } - return stats; } }; @@ -333,6 +345,8 @@ async function processNativeConversation( ...(legacyImportTurnId ? { legacyImportTurnId } : {}), ...(ctx.scanMode === "initial_subset" || ctx.scanMode === "full" ? { captureLegacyHistory: true } : {}) }); + // Stored and already-stored turns both count: a re-import must freeze what a failed freeze left. + if (result.result?.sessionId) (stats.sessionIds ??= []).push(result.result.sessionId); if (result.status === "pending" || result.status === "conflict") { failed = true; stats.failed += turn.messages.length; diff --git a/App/backend/src/services/tests/ingestion-service.test.ts b/App/backend/src/services/tests/ingestion-service.test.ts index 38e912fda..dadd5bb52 100644 --- a/App/backend/src/services/tests/ingestion-service.test.ts +++ b/App/backend/src/services/tests/ingestion-service.test.ts @@ -47,6 +47,41 @@ describe("native Codex ingestion", () => { expect(markSeen).toHaveBeenCalledTimes(2); }); + it("freezes the sessions an import touched once it ends, and a failed freeze does not fail the import", async () => { + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } }); + const freezeScanSessions = vi.fn().mockResolvedValue({ batches: 1 }); + await createService({ completeSourceTurn, freezeScanSessions }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(freezeScanSessions).toHaveBeenCalledWith("codex", ["s-1"]); + + const failing = vi.fn().mockRejectedValue(new Error("memory unavailable")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const stats = await createService({ completeSourceTurn, freezeScanSessions: failing }) + .ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(stats.memoryIds).toEqual(["l1-native"]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("l3.freeze_failed")); + warn.mockRestore(); + + // Importing the same history again returns "existing"; the freeze is asked again so it can catch up. + const again = vi.fn().mockResolvedValue({ batches: 1 }); + const existing = vi.fn().mockResolvedValue({ status: "existing", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } }); + await createService({ completeSourceTurn: existing, freezeScanSessions: again }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(again).toHaveBeenCalledWith("codex", ["s-1"]); + }); + + it("still freezes what an import added when the import stops with an error", async () => { + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } }); + const freezeScanSessions = vi.fn().mockResolvedValue({ batches: 1 }); + async function* brokenSource() { + yield* nativeMessages(); + // A later conversation fails to read after the first one was stored. + yield { ...nativeMessages()[0]!, conversationId: "next-conversation", messageId: "next-0" }; + throw new Error("source read failed"); + } + await expect(createService({ completeSourceTurn, freezeScanSessions }).ingest(brokenSource(), { sourceId: "codex" })) + .rejects.toThrow("source read failed"); + expect(freezeScanSessions).toHaveBeenCalledWith("codex", ["s-1"]); + }); + it("emits add analytics only when a native scan turn is newly stored", async () => { const events: Array<{ name: string; payload: Record }> = []; const analytics = { diff --git a/App/backend/src/tests/support/mock-memory-client.ts b/App/backend/src/tests/support/mock-memory-client.ts index 2b24f9841..55935a44f 100644 --- a/App/backend/src/tests/support/mock-memory-client.ts +++ b/App/backend/src/tests/support/mock-memory-client.ts @@ -141,6 +141,11 @@ export function createMockMemoryClient(options: CreateMockMemoryClientOptions = }; }, + async freezeScanSessions() { + failIfNeeded(); + return { batches: 0 }; + }, + async completeSourceTurn(input) { failIfNeeded(); const l1MemoryId = randomUUID(); diff --git a/App/memmy-agent/src/core/agent-runtime/loop.ts b/App/memmy-agent/src/core/agent-runtime/loop.ts index 4fa9ffc31..59bfc71d5 100644 --- a/App/memmy-agent/src/core/agent-runtime/loop.ts +++ b/App/memmy-agent/src/core/agent-runtime/loop.ts @@ -1295,6 +1295,13 @@ export class AgentLoop { return initialization; } + /** Long-running service only: keep Memory's copy of the desktop project list current. */ + startL3UserProjectSync(): void { + const projectStore = this.projectStore; + if (!projectStore) return; + this.memmyMemoryIntegration.startL3UserProjectSync?.(() => projectStore.snapshot()); + } + async closeRuntimeTools(): Promise { this.messageLoopReady = false; this.runtimeToolsClosed = true; diff --git a/App/memmy-agent/src/entrypoints/cli/commands.ts b/App/memmy-agent/src/entrypoints/cli/commands.ts index 513e5c4a5..0e57cbff4 100644 --- a/App/memmy-agent/src/entrypoints/cli/commands.ts +++ b/App/memmy-agent/src/entrypoints/cli/commands.ts @@ -1161,6 +1161,7 @@ export async function gateway({ cronService: cron, projectStore, }); + loop.startL3UserProjectSync?.(); reportStartupStage("agent-loop-created"); if (loop.sessions instanceof SessionManager) { loop.guiTranscriptMirror = new GuiTranscriptMirror(loop.sessions, canonicalWorkspace); diff --git a/App/memmy-agent/src/memmy-memory/client.ts b/App/memmy-agent/src/memmy-memory/client.ts index bb1a6ef95..14f35a5f9 100644 --- a/App/memmy-agent/src/memmy-memory/client.ts +++ b/App/memmy-agent/src/memmy-memory/client.ts @@ -1,11 +1,14 @@ import type { MemmyMemoryConnection, MemmyMemoryRequestEnvelope, JsonRecord } from "./types.js"; import { normalizeTimeZoneOffset } from "../utils/time-zone.js"; import { + L3UserProjectsReplaceResponseSchema, L3WorldModelBoundaryResponseSchema, L3WorldModelTraceHeadResponseSchema, MemoryHealthSnapshotSchema, SessionL3WorldModelContextResponseSchema, l3WorldModelGetTransport, + type L3UserProjectsReplaceRequest, + type L3UserProjectsReplaceResponse, type L3WorldModelBoundaryRequest, type L3WorldModelBoundaryResponse, type L3WorldModelRequestEnvelope, @@ -179,6 +182,13 @@ export class MemmyMemoryClient { return SessionL3WorldModelContextResponseSchema.parse(value); } + async replaceL3UserProjects( + request: L3UserProjectsReplaceRequest, + options: MemoryRequestOptions = {} + ): Promise { + return L3UserProjectsReplaceResponseSchema.parse(await this.post("/api/v1/l3/user-projects", request, options)); + } + } function safeJsonParse(text: string): any { diff --git a/App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts b/App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts new file mode 100644 index 000000000..fd97cb2c3 --- /dev/null +++ b/App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts @@ -0,0 +1,82 @@ +import { randomUUID } from "node:crypto"; +import type { L3UserProjectsReplaceRequest } from "@memmy/local-api-contracts"; +import type { WebuiProjectRegistrySnapshot } from "../entrypoints/frontend-bridge/projects.js"; +import type { MemmyMemoryClient } from "./client.js"; + +export const L3_USER_PROJECT_SYNC_INTERVAL_MS = 60_000; +/** An unchanged list is still pushed this often, so Memory gets it back after its data was cleared. */ +export const L3_USER_PROJECT_RESYNC_MS = 10 * 60_000; + +type Envelope = Omit; + +/** + * Keeps Memory's copy of the user's selected project folders current. It reads + * the registry on a timer and pushes the full list only when it differs from + * the last successful push, so startup, edits made by any process and a Memory + * service that was not reachable yet are all covered by the same loop. An + * unchanged list is pushed again every ten minutes in case Memory lost it. + */ +export class L3UserProjectSync { + private lastPushedKey: string | null = null; + private lastPushedAt = 0; + private inFlight: Promise | null = null; + private timer: ReturnType | null = null; + + constructor(private readonly options: { + client: Pick; + envelope: Envelope; + readProjects: () => WebuiProjectRegistrySnapshot; + standaloneRoots: () => string[]; + intervalMs?: number; + now?: () => number; + }) {} + + start(): void { + if (this.timer) return; + void this.syncNow(); + this.timer = setInterval(() => void this.syncNow(), this.options.intervalMs ?? L3_USER_PROJECT_SYNC_INTERVAL_MS); + this.timer.unref?.(); + } + + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + /** Pushes when the list changed or the last push failed; never throws. A corrupt registry is not pushed. */ + syncNow(): Promise { + if (this.inFlight) return this.inFlight; + this.inFlight = this.pushIfChanged().finally(() => { + this.inFlight = null; + }); + return this.inFlight; + } + + private async pushIfChanged(): Promise { + let snapshot: WebuiProjectRegistrySnapshot; + try { + snapshot = this.options.readProjects(); + } catch { + return false; + } + if (snapshot.state !== "ready") return false; + const projects = snapshot.projects.map((project) => ({ rootPath: project.rootPath, name: project.name })); + const standaloneRoots = this.options.standaloneRoots(); + const key = JSON.stringify({ projects, standaloneRoots }); + const now = (this.options.now ?? Date.now)(); + if (key === this.lastPushedKey && now - this.lastPushedAt < L3_USER_PROJECT_RESYNC_MS) return true; + try { + await this.options.client.replaceL3UserProjects({ + ...this.options.envelope, + requestId: randomUUID(), + projects, + standaloneRoots + }); + this.lastPushedKey = key; + this.lastPushedAt = now; + return true; + } catch { + return false; + } + } +} diff --git a/App/memmy-agent/src/memmy-memory/register.ts b/App/memmy-agent/src/memmy-memory/register.ts index 97913f2ce..d84b07af4 100644 --- a/App/memmy-agent/src/memmy-memory/register.ts +++ b/App/memmy-agent/src/memmy-memory/register.ts @@ -10,7 +10,10 @@ import { MemmyMemoryClient } from "./client.js"; import { resolveMemmyMemoryConfig } from "./config.js"; import { discoverMemmyMemoryConnection } from "./discovery.js"; import { MemmyMemoryHook } from "./hook.js"; +import { L3UserProjectSync } from "./l3-user-project-sync.js"; import type { MemmyMemoryInstallOptions } from "./types.js"; +import { getStandaloneTaskWorkspacesDir } from "../config/paths.js"; +import type { WebuiProjectRegistrySnapshot } from "../entrypoints/frontend-bridge/projects.js"; export type MemmyMemoryIntegration = { enabled: boolean; @@ -18,6 +21,8 @@ export type MemmyMemoryIntegration = { hook?: MemmyMemoryHook; dispose?: () => Promise; closeSession?: (sessionKey: string, reason?: string) => Promise; + /** Starts pushing the desktop project registry to Memory for L3 project recognition. */ + startL3UserProjectSync?: (readProjects: () => WebuiProjectRegistrySnapshot) => void; }; export { @@ -53,11 +58,33 @@ export function createMemmyMemoryIntegration( void hook.initialize().catch((error) => { hook.lastError = error instanceof Error ? error.message : String(error); }); + let projectSync: L3UserProjectSync | null = null; return { enabled: true, client, hook, - dispose: () => hook.dispose(), + dispose: async () => { + projectSync?.stop(); + await hook.dispose(); + }, + startL3UserProjectSync: (readProjects) => { + if (projectSync) return; + projectSync = new L3UserProjectSync({ + client, + envelope: { + adapterId: "memmy-agent", + source: "memmy-agent", + namespace: { + source: "memmy-agent", + profileId: "default", + ...(resolved.userId ? { userId: resolved.userId } : {}), + }, + }, + readProjects, + standaloneRoots: () => [getStandaloneTaskWorkspacesDir()], + }); + projectSync.start(); + }, closeSession: (sessionKey, reason = "deleted") => hook.sessionEnd(new AgentHookContext({ sessionKey, reason, diff --git a/App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts b/App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts new file mode 100644 index 000000000..571fd9d36 --- /dev/null +++ b/App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; +import type { WebuiProjectRegistrySnapshot } from "../../src/entrypoints/frontend-bridge/projects.js"; +import { L3UserProjectSync } from "../../src/memmy-memory/l3-user-project-sync.js"; + +const envelope = { + adapterId: "memmy-agent", + source: "memmy-agent", + namespace: { source: "memmy-agent", profileId: "default", userId: "u1" }, +}; + +function project(rootPath: string, name: string) { + return { id: name, name, rootPath, pinned: false, createdAt: "2026-10-24T00:00:00.000Z" }; +} + +function setup(initial: WebuiProjectRegistrySnapshot, now: () => number = () => 0) { + let snapshot = initial; + const replaceL3UserProjects = vi.fn().mockResolvedValue({ count: 0, serverTime: "2026-10-24T00:00:00.000Z" }); + const sync = new L3UserProjectSync({ + client: { replaceL3UserProjects }, + envelope, + readProjects: () => snapshot, + standaloneRoots: () => ["/Users/me/Memmy"], + now, + }); + return { sync, replaceL3UserProjects, set: (next: WebuiProjectRegistrySnapshot) => { snapshot = next; } }; +} + +describe("L3UserProjectSync", () => { + it("pushes the active project list once and again only after it changes", async () => { + const { sync, replaceL3UserProjects, set } = setup({ state: "ready", projects: [project("/work/a", "A")] }); + expect(await sync.syncNow()).toBe(true); + expect(await sync.syncNow()).toBe(true); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(1); + expect(replaceL3UserProjects.mock.calls[0]![0]).toMatchObject({ + ...envelope, + projects: [{ rootPath: "/work/a", name: "A" }], + standaloneRoots: ["/Users/me/Memmy"], + }); + set({ state: "ready", projects: [project("/work/a", "A2")] }); + await sync.syncNow(); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(2); + expect(replaceL3UserProjects.mock.calls[1]![0].projects).toEqual([{ rootPath: "/work/a", name: "A2" }]); + }); + + it("pushes an unchanged list again after ten minutes, in case Memory lost it", async () => { + let clock = 0; + const { sync, replaceL3UserProjects } = setup({ state: "ready", projects: [project("/work/a", "A")] }, () => clock); + await sync.syncNow(); + clock = 9 * 60_000; + await sync.syncNow(); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(1); + clock = 10 * 60_000; + await sync.syncNow(); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(2); + }); + + it("does not push a corrupt registry, so a failed load never wipes Memory's list", async () => { + const { sync, replaceL3UserProjects } = setup({ state: "corrupt", projects: [] }); + expect(await sync.syncNow()).toBe(false); + expect(replaceL3UserProjects).not.toHaveBeenCalled(); + }); + + it("retries on the next run after Memory was unreachable", async () => { + const { sync, replaceL3UserProjects } = setup({ state: "ready", projects: [project("/work/a", "A")] }); + replaceL3UserProjects.mockRejectedValueOnce(new Error("connect ECONNREFUSED")); + expect(await sync.syncNow()).toBe(false); + expect(await sync.syncNow()).toBe(true); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(2); + }); + + it("runs one push at a time", async () => { + const { sync, replaceL3UserProjects } = setup({ state: "ready", projects: [project("/work/a", "A")] }); + const [first, second] = [sync.syncNow(), sync.syncNow()]; + expect(first).toBe(second); + await first; + expect(replaceL3UserProjects).toHaveBeenCalledTimes(1); + }); +}); diff --git a/Memory/AgentSourceCore/src/memory-token-budget.ts b/Memory/AgentSourceCore/src/memory-token-budget.ts index 46247125f..77beb5ee5 100644 --- a/Memory/AgentSourceCore/src/memory-token-budget.ts +++ b/Memory/AgentSourceCore/src/memory-token-budget.ts @@ -22,6 +22,7 @@ export const MEMORY_WORKER_JOB_TYPES = [ "l2_induction", "l3_abstraction", "l3_world_model_update", + "l3_unit_update", "project_environment_profile", "skill_crystallization", "skill_cluster_assign", diff --git a/Memory/src/agent-source/runtime.ts b/Memory/src/agent-source/runtime.ts index 86f53d6db..499898380 100644 --- a/Memory/src/agent-source/runtime.ts +++ b/Memory/src/agent-source/runtime.ts @@ -1086,6 +1086,7 @@ async function stageStandaloneSource( return { adapter, stored, sourceId, mode, staged, scanErrorCount, errors }; } +/** Ingests a source's staged turns; the Sessions it touched go to L3 even when the pass stops early. */ async function ingestStagedMessages( service: MemoryService, store: MemoryAgentSourceScanStore, @@ -1100,6 +1101,38 @@ async function ingestStagedMessages( "trackAddStarted" | "trackAddSucceeded" | "trackAddFailed" >, scanMode?: MemoryDesktopAddScanMode +): ReturnType { + const scannedSessionIds: string[] = []; + try { + return await ingestStagedTurns( + service, store, checkpoints, sourceId, signal, checkpoint, onProgress, scannedSessionIds, scheduleWorker, memoryAddAnalytics, scanMode + ); + } finally { + // Frozen batches are new queued jobs; wake the worker in case its drain already finished. + try { + if ((service.freezeScannedL3Sessions?.(scannedSessionIds) ?? 0) > 0) scheduleWorker?.(); + } catch { + // A failed freeze must not hide the scan's own result; the idle sweep hands these Sessions over later. + } + } +} + +async function ingestStagedTurns( + service: MemoryService, + store: MemoryAgentSourceScanStore, + checkpoints: SourceCheckpoints, + sourceId: string, + signal: AbortSignal, + checkpoint: () => Promise, + onProgress: (progress: ScanProgress) => void, + /** Sessions this pass added turns to; the caller freezes them into L3 when the pass ends. */ + scannedSessionIds: string[], + scheduleWorker?: () => void, + memoryAddAnalytics?: Pick< + MemoryDesktopAddAnalytics, + "trackAddStarted" | "trackAddSucceeded" | "trackAddFailed" + >, + scanMode?: MemoryDesktopAddScanMode ): Promise<{ written: number; skipped: number; messageCount: number; errors: string[]; errorCount: number; latestSeenAt: string | null; retryFrom: string | null; hasUncommittedSkips: boolean }> { let written = 0; let messageCount = 0; @@ -1252,6 +1285,7 @@ async function ingestStagedMessages( }); reportedAdd = true; } + if (result.result?.sessionId) scannedSessionIds.push(result.result.sessionId); if (ids.length === 0) store.saveResult({ sourceId, itemKey, conversationId: turn.conversationId }); for (const memoryId of ids) store.saveResult({ sourceId, itemKey, conversationId: turn.conversationId, memoryId }); checkpoints.markMessages(turn.messages); diff --git a/Memory/src/algorithm/plugin-algorithms.ts b/Memory/src/algorithm/plugin-algorithms.ts index 1b7f2390c..582547187 100644 --- a/Memory/src/algorithm/plugin-algorithms.ts +++ b/Memory/src/algorithm/plugin-algorithms.ts @@ -13,6 +13,7 @@ import { stableHash } from "../utils/id.js"; import { matchToolResultIndices } from "../utils/tool-call-pairing.js"; import { formatZonedTime } from "../utils/time.js"; import { isObservationMemory, isObservationMemoryProcessed } from "../service/observation-memory-state.js"; +import { isL3FieldSnapshotMemory, l3FieldSnapshotLegacyFields } from "../service/l3-field/snapshot-memory.js"; import { renderL3WorldModelFields, type L3WorldModelFields @@ -3442,6 +3443,31 @@ export function skillMetaFromMemory(memory: MemoryRow): SkillMemoryMeta | null { export function worldModelMetaFromMemory(memory: MemoryRow): WorldModelMemoryMeta | null { if (memory.memoryLayer !== "L3") return null; + const snapshotFields = l3FieldSnapshotLegacyFields(memory); + if (snapshotFields) { + // New L3 snapshots are shown on the world model page in the schema-2 shape; they are injected + // at session start only and never enter retrieval. + const internal = memory.properties.internal_info as Record; + const field = internal.l3_field as Record; + const project = field.kind === "project"; + return { + id: memory.id, + memory, + title: stringField(internal, "title") ?? (project ? "项目场域认知" : "本机场域认知"), + domainKey: `l3_field:${field.unit_key as string}`, + domainTags: [project ? "project" : "machine"], + policyIds: [], + confidence: 0, + cohesion: 1, + admission: "strict", + structure: { environment: [], inference: [], constraints: [] }, + summary: renderL3WorldModelFields(snapshotFields), + body: memory.memoryValue, + vec: memoryVector(memory, "vec"), + schemaVersion: 2, + fields: snapshotFields + }; + } const wm = getInternal>(memory, "world_model"); if (!wm) return null; const v2Fields = l3WorldModelV2Fields(memory, wm); @@ -5232,6 +5258,7 @@ function candidateFromMemory( } ): RankedMemoryCandidate | null { if (memory.status === "deleted" || memory.status === "archived") return null; + if (isL3FieldSnapshotMemory(memory)) return null; const trace = memory.memoryLayer === "L1" ? traceMetaFromMemory(memory) : null; const policy = memory.memoryLayer === "L2" ? policyMetaFromMemory(memory) : null; const skill = memory.memoryLayer === "Skill" ? skillMetaFromMemory(memory) : null; diff --git a/Memory/src/config/index.ts b/Memory/src/config/index.ts index 95d15466e..006255db2 100644 --- a/Memory/src/config/index.ts +++ b/Memory/src/config/index.ts @@ -251,6 +251,11 @@ export interface AlgorithmConfig { // saw). Off: after a successful judgment only the small fields stay. turnJudgeKeepInput: boolean; }; + /** Itemized machine/project L3; on by default (local SQLite only, see isL3FieldCognitionActive). */ + l3FieldCognition: { + enabled: boolean; + inactiveRetentionDays: number; + }; retrieval: { tier1TopK: number; tier2TopK: number; @@ -513,6 +518,10 @@ export const DEFAULT_MEMMY_CONFIG: MemmyConfig = { turnJudge: true, turnJudgeKeepInput: false }, + l3FieldCognition: { + enabled: true, + inactiveRetentionDays: 15 + }, retrieval: { tier1TopK: 3, tier2TopK: 5, @@ -659,6 +668,9 @@ function configFromEnv(): Record { skillInjectionMode: process.env.MEMMY_SKILL_INJECTION_MODE, skillSummaryChars: numberEnv("MEMMY_SKILL_SUMMARY_CHARS"), skillFullMaxChars: numberEnv("MEMMY_SKILL_FULL_MAX_CHARS") + }), + l3FieldCognition: compactRecord({ + enabled: booleanEnv("MEMMY_L3_FIELD_COGNITION") }) }) }); @@ -1167,6 +1179,7 @@ function normalizeAlgorithm(input: Record): AlgorithmConfig { const l3 = asRecord(input.l3Abstraction); const skill = asRecord(input.skill); const session = asRecord(input.session); + const l3FieldCognition = asRecord(input.l3FieldCognition); const retrieval = asRecord(input.retrieval); return { enableMemoryAdd: booleanValue(input.enableMemoryAdd, DEFAULT_MEMMY_CONFIG.algorithm.enableMemoryAdd), @@ -1326,6 +1339,13 @@ function normalizeAlgorithm(input: Record): AlgorithmConfig { turnJudge: booleanValue(session.turnJudge, DEFAULT_MEMMY_CONFIG.algorithm.session.turnJudge), turnJudgeKeepInput: booleanValue(session.turnJudgeKeepInput, DEFAULT_MEMMY_CONFIG.algorithm.session.turnJudgeKeepInput) }, + l3FieldCognition: { + enabled: booleanValue(l3FieldCognition.enabled, DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition.enabled), + inactiveRetentionDays: numberValue( + l3FieldCognition.inactiveRetentionDays, + DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition.inactiveRetentionDays + ) + }, retrieval: { tier1TopK: numberValue(retrieval.tier1TopK, DEFAULT_MEMMY_CONFIG.algorithm.retrieval.tier1TopK), tier2TopK: numberValue(retrieval.tier2TopK, DEFAULT_MEMMY_CONFIG.algorithm.retrieval.tier2TopK), diff --git a/Memory/src/contracts/memory-l3-world-model.ts b/Memory/src/contracts/memory-l3-world-model.ts index 269d5f023..bca1e15f9 100644 --- a/Memory/src/contracts/memory-l3-world-model.ts +++ b/Memory/src/contracts/memory-l3-world-model.ts @@ -83,6 +83,51 @@ export const L3WorldModelBoundaryResponseSchema = z.object({ }).strict(); export type L3WorldModelBoundaryResponse = z.infer; +const AbsolutePathSchema = z.string().min(1).max(4096) + .refine((value) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value), "path must be absolute"); + +/** Full list of folders the user picked in the desktop app; each push replaces the previous one. */ +export const L3UserProjectsReplaceRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + projects: z.array(z.object({ + rootPath: AbsolutePathSchema, + name: z.string().min(1).max(512) + }).strict()).max(500), + standaloneRoots: z.array(AbsolutePathSchema).max(20).default([]) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3UserProjectsReplaceRequest = z.input; + +export const L3UserProjectsReplaceResponseSchema = z.object({ + count: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3UserProjectsReplaceResponse = z.infer; + +/** Puts the user's new-L3 batches that ended in dead letter back in the queue. */ +export const L3FailedBatchesRetryRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3FailedBatchesRetryRequest = z.input; + +export const L3FailedBatchesRetryResponseSchema = z.object({ + requeued: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3FailedBatchesRetryResponse = z.infer; + +/** End of a hosted import: freeze what it added to these scanned Sessions into L3 batches. */ +export const L3ScanSessionsFreezeRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + sessionIds: z.array(NonEmptyStringSchema).max(10_000) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3ScanSessionsFreezeRequest = z.input; + +export const L3ScanSessionsFreezeResponseSchema = z.object({ + batches: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3ScanSessionsFreezeResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/Memory/src/contracts/memory-runtime.ts b/Memory/src/contracts/memory-runtime.ts index 9364bfe4a..661adcd68 100644 --- a/Memory/src/contracts/memory-runtime.ts +++ b/Memory/src/contracts/memory-runtime.ts @@ -57,6 +57,7 @@ export const JobTypeSchema = z.enum([ "l2_induction", "l3_abstraction", "l3_world_model_update", + "l3_unit_update", "project_environment_profile", "skill_crystallization", "skill_cluster_assign", diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index d1cd76533..54ccf1746 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -8,6 +8,9 @@ import { } from "../agent-source/runtime.js"; import { EmbeddingInferenceInputSchema, + L3UserProjectsReplaceRequestSchema, + L3FailedBatchesRetryRequestSchema, + L3ScanSessionsFreezeRequestSchema, L3WorldModelBoundaryRequestSchema, L3WorldModelRequestEnvelopeSchema, OpenSessionInputSchema @@ -77,6 +80,9 @@ export const API_ROUTES = [ "GET /api/v1/sessions/:sessionId/l3-world-model-trace-head", "POST /api/v1/sessions/:sessionId/l3-world-model-boundary", "GET /api/v1/l3-world-model/sessions/:sessionId/context", + "POST /api/v1/l3/user-projects", + "POST /api/v1/l3/failed-batches/retry", + "POST /api/v1/l3/scan-sessions/freeze", "POST /api/v1/turns/start", "POST /api/v1/turns/:turnId/complete", "POST /api/v1/source-turns/complete", @@ -688,6 +694,52 @@ async function routeRequest( return result; } + if (method === "POST" && path === "/api/v1/l3/user-projects") { + requireMemoryWrite(principal); + const parsed = L3UserProjectsReplaceRequestSchema.safeParse( + strictEnvelopeWithPrincipal(asObject(body, "l3.user-projects"), principal) + ); + if (!parsed.success) { + throw new MemoryServiceError( + "invalid_argument", + parsed.error.issues.map((issue) => `${issue.path.join(".") || "request"}: ${issue.message}`).join("; ") + ); + } + return service.replaceL3UserProjects(parsed.data); + } + + if (method === "POST" && path === "/api/v1/l3/failed-batches/retry") { + requireMemoryWrite(principal); + const parsed = L3FailedBatchesRetryRequestSchema.safeParse( + strictEnvelopeWithPrincipal(asObject(body, "l3.failed-batches.retry"), principal) + ); + if (!parsed.success) { + throw new MemoryServiceError( + "invalid_argument", + parsed.error.issues.map((issue) => `${issue.path.join(".") || "request"}: ${issue.message}`).join("; ") + ); + } + const retried = service.retryFailedL3Batches(parsed.data); + if (retried.requeued > 0) autoWorker.schedule(); + return retried; + } + + if (method === "POST" && path === "/api/v1/l3/scan-sessions/freeze") { + requireMemoryWrite(principal); + const parsed = L3ScanSessionsFreezeRequestSchema.safeParse( + strictEnvelopeWithPrincipal(asObject(body, "l3.scan-sessions.freeze"), principal) + ); + if (!parsed.success) { + throw new MemoryServiceError( + "invalid_argument", + parsed.error.issues.map((issue) => `${issue.path.join(".") || "request"}: ${issue.message}`).join("; ") + ); + } + const frozen = service.freezeScanSessions(parsed.data); + if (frozen.batches > 0) autoWorker.schedule(); + return frozen; + } + const l3Context = match(path, /^\/api\/v1\/l3-world-model\/sessions\/([^/]+)\/context$/); if (method === "GET" && l3Context) { requireMemoryRead(principal); diff --git a/Memory/src/service/embedding/embedding-job-processor.ts b/Memory/src/service/embedding/embedding-job-processor.ts index 7f750584a..63c4d3328 100644 --- a/Memory/src/service/embedding/embedding-job-processor.ts +++ b/Memory/src/service/embedding/embedding-job-processor.ts @@ -58,6 +58,7 @@ import { updateMemoryVectorField } from "./embedding-pipeline.js"; import { canonicalWorkMemoryText } from "../work-memory/work-memory-pipeline.js"; +import { isL3FieldSnapshotMemory } from "../l3-field/snapshot-memory.js"; import { isObservationMemory, isObservationMemoryProcessed, observationJobPayload } from "../observation-memory-state.js"; type TraceMeta = NonNullable>; @@ -258,6 +259,7 @@ export class EmbeddingJobProcessor { throw new Error(`embedding target not found: ${job.targetMemoryId ?? "unknown"}`); } if (!processingJobMatchesMemory(job, memory)) return null; + if (isL3FieldSnapshotMemory(memory)) return null; if (isObservationMemory(memory)) { if (!isObservationMemoryProcessed(memory)) return null; diff --git a/Memory/src/service/evolution/evolution-logging.ts b/Memory/src/service/evolution/evolution-logging.ts index 1292292e4..49bac5423 100644 --- a/Memory/src/service/evolution/evolution-logging.ts +++ b/Memory/src/service/evolution/evolution-logging.ts @@ -21,7 +21,7 @@ export function evolutionJobLogFields(job: EvolutionJobRecord): Record = {} ): void { diff --git a/Memory/src/service/evolution/l3-world-model-pipeline.ts b/Memory/src/service/evolution/l3-world-model-pipeline.ts index 92b8b8de5..7ca782f18 100644 --- a/Memory/src/service/evolution/l3-world-model-pipeline.ts +++ b/Memory/src/service/evolution/l3-world-model-pipeline.ts @@ -199,7 +199,8 @@ function strictJobPayload(job: EvolutionJobRecord): { batchId: string; targetField: L3WorldModelTargetField; } { - const keys = Object.keys(job.payload).sort(); + // runAfter is the queue's own retry time, not part of the job's evidence. + const keys = Object.keys(job.payload).filter((key) => key !== "runAfter").sort(); if (keys.join(",") !== "batchId,targetField") { throw new L3WorldModelTerminalEvidenceError("invalid L3 World Model job payload keys"); } diff --git a/Memory/src/service/l3-field/evidence-fragments.ts b/Memory/src/service/l3-field/evidence-fragments.ts new file mode 100644 index 000000000..355071e80 --- /dev/null +++ b/Memory/src/service/l3-field/evidence-fragments.ts @@ -0,0 +1,402 @@ +import { existsSync } from "node:fs"; +import path from "node:path"; +import { sha256Hex } from "../../contracts/index.js"; +import type { RawTurnRecord } from "../../storage/repositories.js"; +import { matchToolResultIndices } from "../../utils/tool-call-pairing.js"; +import type { EvidenceFragment, ResolvedProject } from "./types.js"; + +const TARGET_KEYS = ["file_path", "filePath", "path", "notebook_path"]; +const WORKDIR_KEYS = ["workdir", "cwd", "directory"]; +const PATCH_KEYS = ["input", "patch"]; +const PATCH_FILE_LINE = /^\*\*\* (?:Update|Add|Delete) File: (.+)$|^\*\*\* Move to: (.+)$/gm; +const COMMAND_KEYS = ["command", "cmd"]; +const USER_TEXT_LIMIT = 4_000; +const CLIP_HEAD_CHARS = 300; +const CLIP_TAIL_CHARS = 300; +const ERROR_LINE_CHARS = 200; +const ERROR_LINE = /error|fail|exception|denied|not found|traceback|refused|cannot|no such|fatal/i; +const COMMAND_SEPARATORS = new Set(["&&", "||", ";", "|"]); +/** A redirection token, with its target attached (`2>/dev/null`) or as the next token (`>`). */ +const REDIRECT = /^\d*(?:>>?|<)|^&>/; +const REDIRECT_ONLY = /^(?:\d*(?:>>?|<)|&>)$/; + +/** + * Cuts one raw turn into evidence fragments: the user's words, and each tool + * call together with its result. Every fragment carries the project units its + * own paths resolve to, so content about repository A never borrows evidence + * from a call that touched repository B. + */ +export function buildEvidenceFragments(input: { + rawTurn: RawTurnRecord; + l1MemoryId: string; + sessionCwd: string | null; + sessionProjectRoot: string | null; + resolve: (target: string, role: "session_dir" | "tool_path") => ResolvedProject | null; + unitKeyOf: (project: ResolvedProject) => string; +}): EvidenceFragment[] { + const { rawTurn } = input; + const sessionUnits = (): string[] => { + if (!input.sessionCwd) return []; + const project = input.resolve(input.sessionCwd, "session_dir"); + return project ? [input.unitKeyOf(project)] : []; + }; + const rootOf = new Map(); + const unitsForPaths = (paths: string[]): string[] => { + if (paths.length === 0) return sessionUnits(); + const keys = new Set(); + for (const target of paths) { + const project = input.resolve(target, "tool_path"); + if (!project) continue; + const key = input.unitKeyOf(project); + rootOf.set(key, project.checkoutRoot ?? null); + keys.add(key); + } + return mostSpecificUnits([...keys], rootOf); + }; + const fragmentId = (kind: string, index: number) => + sha256Hex(`${rawTurn.id}:${kind}:${index}`).slice(0, 16); + + const calls: EvidenceFragment[] = []; + rawTurn.toolCalls.forEach((call, index) => { + if (!isRecord(call)) return; + const paths = extractToolPaths(call, input.sessionCwd); + calls.push({ + id: fragmentId("call", index), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: typeof call.name === "string" ? call.name : null, + input: clipToolInput(call.input), + output: toolOutputText(call), + isError: isErrorRecord(call), + paths, + candidateUnitKeys: unitsForPaths(paths), + at: typeof call.endedAt === "string" ? call.endedAt : rawTurn.createdAt + }); + }); + // The user's words belong to the projects this turn actually worked on; only a turn that touched + // no project falls back to the session folder (a session opened in a big parent folder). + const touched = mostSpecificUnits([...new Set(calls.flatMap((call) => call.candidateUnitKeys))], rootOf); + + const fragments: EvidenceFragment[] = []; + const userText = rawTurn.userText?.trim(); + if (userText) { + fragments.push({ + id: fragmentId("user", 0), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: null, + input: null, + output: userText.slice(0, USER_TEXT_LIMIT), + isError: false, + paths: [], + candidateUnitKeys: touched.length > 0 ? touched : sessionUnits(), + at: rawTurn.createdAt + }); + } + fragments.push(...calls); + + const matched = new Set( + matchToolResultIndices(rawTurn.toolCalls, rawTurn.toolResults).filter((index): index is number => index !== undefined) + ); + rawTurn.toolResults.forEach((result, index) => { + if (matched.has(index) || !isRecord(result)) return; + const name = [result.name, result.toolName, result.tool_name].find((value) => typeof value === "string"); + fragments.push({ + id: fragmentId("result", index), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: typeof name === "string" ? name : null, + input: null, + output: toolOutputText(result), + isError: isErrorRecord(result), + paths: [], + candidateUnitKeys: sessionUnits(), + at: rawTurn.createdAt + }); + }); + return fragments; +} + +/** + * Paths a tool call actually touched: file arguments, files named in a patch, `cd` + * and `git -C` targets, and absolute path arguments of commands. Relative paths + * resolve against the call's own workdir/cwd, else the session folder. Each shell + * subcommand that names no target of its own counts as touching the directory it ran in. + */ +export function extractToolPaths(call: Record, sessionCwd: string | null): string[] { + const rawArgs = call.input ?? call.arguments ?? call.args; + const args = parseArguments(rawArgs); + if (!args) { + if (typeof rawArgs !== "string") return []; + // A code-mode snippet can hold a patch next to other calls; keep the paths of both. + return [...new Set([...patchPaths(rawArgs, sessionCwd), ...codeModePaths(rawArgs, sessionCwd)])]; + } + const explicitWorkdir = WORKDIR_KEYS.map((key) => stringValue(args[key])).find(Boolean); + const workdir = explicitWorkdir ? resolveAgainst(explicitWorkdir, sessionCwd) : sessionCwd; + const found: string[] = []; + const add = (value: string, base: string | null) => { + const resolved = resolveAgainst(value, base); + if (resolved && !found.includes(resolved)) found.push(resolved); + }; + for (const key of TARGET_KEYS) { + const value = stringValue(args[key]); + if (value) add(value, workdir); + } + for (const key of PATCH_KEYS) { + const value = stringValue(args[key]); + if (value) for (const target of patchPaths(value, workdir)) add(target, null); + } + for (const key of COMMAND_KEYS) { + const raw = args[key]; + const command = typeof raw === "string" ? raw : Array.isArray(raw) ? raw.filter((part) => typeof part === "string").join(" ") : null; + if (command) for (const target of commandPaths(command, workdir)) add(target, null); + } + if (found.length === 0 && explicitWorkdir && workdir) found.push(workdir); + return found; +} + +/** Files an apply_patch style patch adds, updates, deletes or moves to. */ +function patchPaths(text: string, base: string | null): string[] { + if (!text.includes("*** Begin Patch")) return []; + const found: string[] = []; + for (const match of text.matchAll(PATCH_FILE_LINE)) { + const resolved = resolveAgainst((match[1] ?? match[2]!).trim(), base); + if (resolved && !found.includes(resolved)) found.push(resolved); + } + return found; +} + +/** + * Codex "code mode" passes a JavaScript snippet such as + * `tools.exec_command({cmd: "...", workdir: "..."})` instead of JSON; read the + * string literals of the known argument keys out of it. + */ +function codeModePaths(code: string, sessionCwd: string | null): string[] { + const literal = /\b(cmd|command|workdir|cwd|file_path|filePath|path)\s*:\s*("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)/g; + const values: Record = {}; + for (const match of code.matchAll(literal)) { + const value = unquoteJsString(match[2]!); + if (value !== null) (values[match[1]!] ??= []).push(value); + } + const found: string[] = []; + const add = (target: string | null) => { + if (target && !found.includes(target)) found.push(target); + }; + // A snippet can hold several calls and literals are not tied to the call they belong to, + // so every workdir counts here (unlike a single call, where a named target wins). + const workdirs = [...(values.workdir ?? []), ...(values.cwd ?? [])]; + for (const dir of workdirs) add(resolveAgainst(dir, sessionCwd)); + const base = workdirs[0] ? resolveAgainst(workdirs[0], sessionCwd) : sessionCwd; + for (const key of ["file_path", "filePath", "path"]) { + for (const value of values[key] ?? []) add(resolveAgainst(value, base)); + } + for (const command of [...(values.cmd ?? []), ...(values.command ?? [])]) { + for (const target of commandPaths(command, base)) add(target); + } + return found; +} + +function unquoteJsString(quoted: string): string | null { + const body = quoted.slice(1, -1); + if (quoted.startsWith("\"")) { + try { + return JSON.parse(quoted) as string; + } catch { + return null; + } + } + return body.replace(/\\(.)/g, (_, char: string) => (char === "n" ? "\n" : char === "t" ? "\t" : char)); +} + +/** Keeps head and tail plus any error-looking line from the middle, so a mid-output error is not lost. */ +export function clipToolOutput(text: string): string { + if (text.length <= CLIP_HEAD_CHARS + CLIP_TAIL_CHARS) return text; + const head = text.slice(0, CLIP_HEAD_CHARS); + const tail = text.slice(-CLIP_TAIL_CHARS); + const middle = text.slice(CLIP_HEAD_CHARS, text.length - CLIP_TAIL_CHARS); + const errorLines = middle.split(/\r?\n/).filter((line) => ERROR_LINE.test(line)).slice(0, 20) + .map((line) => (line.length > ERROR_LINE_CHARS ? `${line.slice(0, ERROR_LINE_CHARS)}…` : line)); + return [head, "[已截断]", ...errorLines, ...(errorLines.length ? ["[已截断]"] : []), tail].join("\n"); +} + +/** A long tool input (a patch, a written file) is clipped like output; its paths were read before clipping. */ +function clipToolInput(value: unknown): unknown { + if (value === undefined || value === null) return null; + const text = typeof value === "string" ? value : JSON.stringify(value); + return text.length <= CLIP_HEAD_CHARS + CLIP_TAIL_CHARS ? value : clipToolOutput(text); +} + +/** + * Paths each subcommand works on: `cd` and `git -C` targets, absolute path arguments, and relative + * ones (`api/AGENTS.md`, or a bare name that exists in the current directory). + * A subcommand naming none works in the current directory, so that counts instead. Where + * output is redirected to (`> /tmp/log`) is not what the command works on. + */ +function commandPaths(command: string, startDir: string | null): string[] { + const found: string[] = []; + let cwd = startDir; + let segment: string[] = []; + const flush = () => { + if (segment.length === 0) return; + const [executable, ...rest] = segment; + if (executable === "cd" && rest[0]) { + const target = resolveAgainst(rest[0], cwd); + if (target) { + found.push(target); + cwd = target; + } + } else { + const targets: string[] = []; + for (let i = 0; i < rest.length; i += 1) { + const token = rest[i]!; + if (REDIRECT.test(token)) { + if (REDIRECT_ONLY.test(token)) i += 1; + continue; + } + if (executable === "git" && token === "-C" && rest[i + 1]) { + const target = resolveAgainst(rest[i + 1]!, cwd); + if (target) targets.push(target); + i += 1; + continue; + } + if (isAbsoluteAny(token) && !token.includes("://")) targets.push(token); + else { + const relative = relativePathArgument(token, cwd); + if (relative) targets.push(relative); + } + } + if (targets.length === 0 && cwd) targets.push(cwd); + found.push(...targets); + } + segment = []; + }; + for (const token of shellTokens(command)) { + if (COMMAND_SEPARATORS.has(token)) flush(); + else segment.push(token); + } + flush(); + return found; +} + +/** Minimal shell word splitting: quotes, backslash escapes and the &&, ||, ;, | separators. */ +function shellTokens(command: string): string[] { + const tokens: string[] = []; + let current = ""; + let hasToken = false; + let quote: "'" | "\"" | null = null; + for (let i = 0; i < command.length; i += 1) { + const char = command[i]!; + if (quote) { + if (char === quote) quote = null; + else if (char === "\\" && quote === "\"" && i + 1 < command.length) current += command[++i]; + else current += char; + continue; + } + if (char === "'" || char === "\"") { + quote = char; + hasToken = true; + continue; + } + if (char === "\\" && i + 1 < command.length) { + current += command[++i]; + hasToken = true; + continue; + } + const pair = command.slice(i, i + 2); + if (pair === "&&" || pair === "||" || char === ";" || char === "|") { + if (hasToken) tokens.push(current); + tokens.push(pair === "&&" || pair === "||" ? pair : char); + current = ""; + hasToken = false; + if (pair === "&&" || pair === "||") i += 1; + continue; + } + if (/\s/.test(char)) { + if (hasToken) tokens.push(current); + current = ""; + hasToken = false; + continue; + } + current += char; + hasToken = true; + } + if (hasToken) tokens.push(current); + return tokens; +} + +function parseArguments(value: unknown): Record | null { + if (isRecord(value)) return value; + if (typeof value !== "string") return null; + try { + const parsed = JSON.parse(value) as unknown; + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } +} + +/** + * Work on a project inside a bigger folder belongs to that project: when a fragment touches both a + * folder and a project nested in it, only the nested one stays a candidate. + */ +function mostSpecificUnits(keys: string[], rootOf: ReadonlyMap): string[] { + const inside = (child: string, parent: string) => + child !== parent && [parent, parent + "/", parent + "\\"].some((prefix) => prefix !== parent && child.startsWith(prefix)); + return keys.filter((key) => { + const root = rootOf.get(key); + return !root || !keys.some((other) => { + const otherRoot = rootOf.get(other); + return Boolean(otherRoot) && inside(otherRoot!, root); + }); + }); +} + +/** Wildcards, patterns and expressions are not paths even when they contain a slash. */ +const NOT_A_PATH = /[*?[\]{}|()<>$`'"=!\s]/; + +/** A relative argument is a path when it has a slash, or when it names something in the current directory. */ +function relativePathArgument(token: string, cwd: string | null): string | null { + if (!cwd || token.startsWith("-") || token.includes("://") || NOT_A_PATH.test(token)) return null; + const target = resolveAgainst(token, cwd); + if (!target) return null; + return token.includes("/") || existsSync(target) ? target : null; +} + +function resolveAgainst(value: string, base: string | null): string | null { + const trimmed = value.trim(); + if (!trimmed || trimmed.includes("://")) return null; + if (isAbsoluteAny(trimmed)) return trimmed; + if (!base) return null; + return isWindowsPath(base) ? path.win32.resolve(base, trimmed) : path.posix.resolve(base, trimmed); +} + +function isAbsoluteAny(value: string): boolean { + return value.startsWith("/") || isWindowsPath(value); +} + +function isWindowsPath(value: string): boolean { + return /^[A-Za-z]:[\\/]/.test(value); +} + +function toolOutputText(record: Record): string | null { + const error = typeof record.error === "string" ? record.error : null; + const output = record.output ?? record.result ?? record.content; + const text = typeof output === "string" ? output : output === undefined || output === null ? null : JSON.stringify(output); + const combined = [text, error && !text?.includes(error) ? error : null].filter(Boolean).join("\n"); + return combined ? clipToolOutput(combined) : null; +} + +function isErrorRecord(record: Record): boolean { + return record.success === false || + (typeof record.error === "string" && record.error.trim().length > 0) || + record.status === "failed" || + record.status === "error"; +} + +function stringValue(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} diff --git a/Memory/src/service/l3-field/flag.ts b/Memory/src/service/l3-field/flag.ts new file mode 100644 index 000000000..68ef60772 --- /dev/null +++ b/Memory/src/service/l3-field/flag.ts @@ -0,0 +1,13 @@ +import type { MemmyConfig } from "../../config/index.js"; +import type { StorageBackendCapabilities } from "../../storage/backend.js"; + +/** The new L3 only runs on local SQLite storage, where the Memory service can see the workspace. */ +export function isL3FieldCognitionActive( + config: Pick, + capabilities: Pick, + mode: "local" | "cloud" | "dev" +): boolean { + return config.algorithm.l3FieldCognition.enabled && + mode !== "cloud" && + capabilities.backendId === "sqlite-local"; +} diff --git a/Memory/src/service/l3-field/git-identity.ts b/Memory/src/service/l3-field/git-identity.ts new file mode 100644 index 000000000..ed54b76f2 --- /dev/null +++ b/Memory/src/service/l3-field/git-identity.ts @@ -0,0 +1,124 @@ +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import path from "node:path"; + +export interface GitIdentity { + /** Working-tree root of this checkout (main clone or linked worktree). */ + repoRoot: string; + /** Git directory shared by every worktree of the clone. */ + commonDir: string; + /** Normalized origin URL, or `gitdir:` when there is no usable origin. */ + identity: string; +} + +/** + * Finds the repository containing `startDir` by reading files only (no git + * process). include/includeIf/insteadOf are not resolved; when origin cannot + * be read the clone is identified by its common git directory, so worktrees + * of one clone still match but separate clones do not. + */ +export function readGitIdentity(startDir: string): GitIdentity | null { + let current = startDir; + for (;;) { + const located = locateGitDir(current); + if (located) { + const commonDir = safeRealpath(readCommonDir(located.gitDir)); + const origin = readOriginUrl(commonDir); + const normalized = origin ? normalizeRemoteUrl(origin) : null; + return { + repoRoot: current, + commonDir, + identity: normalized ?? `gitdir:${commonDir}` + }; + } + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} + +/** Maps ssh, scp-like and http(s) remotes to `host[:port]/path`; local remotes return null. */ +export function normalizeRemoteUrl(url: string): string | null { + const trimmed = url.trim(); + if (!trimmed || trimmed.startsWith("/") || /^[A-Za-z]:[\\/]/.test(trimmed) || trimmed.startsWith("file:")) { + return null; + } + let host: string; + let port = ""; + let repoPath: string; + const scpLike = /^(?:[^@/]+@)?([^:/]+):(?!\/\/)(.+)$/.exec(trimmed); + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) { + let parsed: URL; + try { + parsed = new URL(trimmed); + } catch { + return null; + } + if (!parsed.hostname) return null; + host = parsed.hostname; + port = parsed.port; + repoPath = decodeURIComponent(parsed.pathname); + } else if (scpLike) { + host = scpLike[1]!; + repoPath = scpLike[2]!; + } else { + return null; + } + const cleanedPath = repoPath + .replace(/^\/+/, "") + .replace(/\/+$/, "") + .replace(/\.git$/i, "") + .toLowerCase(); + if (!cleanedPath) return null; + return `${host.toLowerCase()}${port ? `:${port}` : ""}/${cleanedPath}`; +} + +function locateGitDir(dir: string): { gitDir: string } | null { + const dotGit = path.join(dir, ".git"); + if (!existsSync(dotGit)) return null; + try { + if (statSync(dotGit).isDirectory()) return { gitDir: dotGit }; + const match = /^gitdir:\s*(.+)$/m.exec(readFileSync(dotGit, "utf8")); + if (!match) return null; + return { gitDir: path.resolve(dir, match[1]!.trim()) }; + } catch { + return null; + } +} + +function readCommonDir(gitDir: string): string { + try { + const relative = readFileSync(path.join(gitDir, "commondir"), "utf8").trim(); + return relative ? path.resolve(gitDir, relative) : gitDir; + } catch { + return gitDir; + } +} + +function readOriginUrl(commonDir: string): string | null { + let text: string; + try { + text = readFileSync(path.join(commonDir, "config"), "utf8"); + } catch { + return null; + } + let inOrigin = false; + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (line.startsWith("[")) { + inOrigin = /^\[remote\s+"origin"\]$/i.test(line); + continue; + } + if (!inOrigin) continue; + const match = /^url\s*=\s*(.+)$/i.exec(line); + if (match) return match[1]!.trim().replace(/^"(.*)"$/, "$1"); + } + return null; +} + +function safeRealpath(target: string): string { + try { + return realpathSync(target); + } catch { + return target; + } +} diff --git a/Memory/src/service/l3-field/l3-field-context.ts b/Memory/src/service/l3-field/l3-field-context.ts new file mode 100644 index 000000000..ab632e052 --- /dev/null +++ b/Memory/src/service/l3-field/l3-field-context.ts @@ -0,0 +1,107 @@ +import type { SessionL3WorldModelContextResponse } from "../../contracts/index.js"; +import type { Repositories, SessionRecord } from "../../storage/repositories.js"; +import { nowIso } from "../../utils/time.js"; +import { findSessionUnits, resolveSessionUnits, type L3FieldEnvironment } from "./session-units.js"; +import type { L3ItemRecord, L3UnitRecord, ProjectIdentitySource } from "./types.js"; + +const SOURCE_LABEL: Record = { + user_selected: "用户选定", + git: "git 仓库", + folder: "文件夹" +}; + +/** + * Session-start context for the new L3: this machine (profile and facts) and, + * when the session folder is a project, that project (profile, facts and + * constraints). The response keeps the schema-2 shape hooks already consume. + * With `readOnly` (memory writes are off) nothing is created or rewritten: units and snapshots + * that do not exist yet are simply not shown. + */ +export function loadL3FieldContext( + repos: Repositories, + env: L3FieldEnvironment, + session: SessionRecord, + options: { readOnly?: boolean } = {} +): SessionL3WorldModelContextResponse { + const units = options.readOnly ? findSessionUnits(repos, env, session) : (() => { + const resolved = resolveSessionUnits(repos, env, session); + return { machine: resolved.machine, project: resolved.project?.unit }; + })(); + // A unit whose snapshot the user archived is not injected, like an archived old-L3 world model. + const archived = (unit: L3UnitRecord) => + Boolean(unit.snapshotMemoryId && repos.memories.get(unit.snapshotMemoryId)?.status === "archived"); + const machine = units.machine && !archived(units.machine) ? units.machine : null; + const projectUnit = units.project && !archived(units.project) ? units.project : null; + const machineItems = machine ? repos.l3Field.activeItems(machine.unitKey) : []; + const projectItems = projectUnit ? repos.l3Field.activeItems(projectUnit.unitKey) : []; + + const machineFacts = bulletList(machineItems, "env_fact"); + const projectFacts = bulletList(projectItems, "env_fact"); + const constraints = bulletList(projectItems, "project_constraint"); + const machineProfile = trimmed(machine?.profileText); + const projectProfile = trimmed(projectUnit?.profileText); + const hasContent = Boolean(machineProfile || machineFacts || projectProfile || projectFacts || constraints); + // Clients treat projectId as the session's own project id, so it keeps that meaning. + const projectId = session.projectId ?? null; + if (!hasContent) return emptyResponse(projectId); + + const sections = [ + projectUnit + ? `当前项目：${projectUnit.displayName ?? projectUnit.projectIdentity}（${SOURCE_LABEL[projectUnit.identitySource ?? "folder"]}）` + : "当前项目：无", + section("本机环境画像", machineProfile), + section("本机环境事实", machineFacts), + section("项目环境画像", projectProfile), + section("项目环境事实", projectFacts), + section("项目约束", constraints) + ].filter(Boolean); + + // The version hooks cache on belongs to the project snapshot when there is a project; machine + // changes bump every project snapshot on the same machine, so this version still moves. + const snapshotUnit = (projectUnit ?? units.machine)!; + const stored = snapshotUnit.snapshotMemoryId ? repos.memories.get(snapshotUnit.snapshotMemoryId) : undefined; + const snapshot = stored ?? (options.readOnly ? undefined : repos.l3Field.refreshSnapshot(snapshotUnit.unitKey)); + + return { + schemaVersion: 2, + projectId, + memoryId: snapshot?.id ?? null, + memoryVersion: snapshot?.version ?? null, + renderedContext: sections.join("\n\n"), + sourceMemoryIds: [...new Set([...machineItems, ...projectItems].flatMap((item) => item.evidence.l1MemoryIds))], + generalRulesAndSafetyConstraints: null, + projectEnvironmentProfile: [machineProfile, projectProfile].filter(Boolean).join("\n\n") || null, + projectContract: constraints, + domainKnowledge: [machineFacts, projectFacts].filter(Boolean).join("\n") || null, + serverTime: nowIso() + }; +} + +function bulletList(items: readonly L3ItemRecord[], kind: L3ItemRecord["kind"]): string | null { + const lines = items.filter((item) => item.kind === kind).map((item) => `- ${item.content}`); + return lines.length ? lines.join("\n") : null; +} + +function section(title: string, body: string | null): string | null { + return body ? `## ${title}\n${body}` : null; +} + +function trimmed(value: string | null | undefined): string | null { + return value?.trim() ? value.trim() : null; +} + +function emptyResponse(projectId: string | null): SessionL3WorldModelContextResponse { + return { + schemaVersion: 2, + projectId, + memoryId: null, + memoryVersion: null, + renderedContext: "", + sourceMemoryIds: [], + generalRulesAndSafetyConstraints: null, + projectEnvironmentProfile: null, + projectContract: null, + domainKnowledge: null, + serverTime: nowIso() + }; +} diff --git a/Memory/src/service/l3-field/machine-profile.ts b/Memory/src/service/l3-field/machine-profile.ts new file mode 100644 index 000000000..0699ada69 --- /dev/null +++ b/Memory/src/service/l3-field/machine-profile.ts @@ -0,0 +1,164 @@ +import { execFile } from "node:child_process"; +import { arch, platform, release, tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { sha256Hex } from "../../contracts/index.js"; + +const execFileAsync = promisify(execFile); + +/** Always listed in the profile, as installed or not found. */ +export const MACHINE_TOOLS = ["node", "python3", "git", "npm", "brew"] as const; +/** Listed only when installed. */ +export const OPTIONAL_MACHINE_TOOLS = ["pnpm", "yarn", "uv", "conda", "docker", "java", "go", "rustc"] as const; +export type MachineTool = typeof MACHINE_TOOLS[number] | typeof OPTIONAL_MACHINE_TOOLS[number]; +const ALL_TOOLS: readonly MachineTool[] = [...MACHINE_TOOLS, ...OPTIONAL_MACHINE_TOOLS]; + +const TOOL_ARGS: Record = { + node: ["--version"], + python3: ["--version"], + git: ["--version"], + npm: ["--version"], + brew: ["--version"], + pnpm: ["--version"], + yarn: ["--version"], + uv: ["--version"], + conda: ["--version"], + docker: ["--version"], + java: ["-version"], + go: ["version"], + rustc: ["--version"] +}; +const PROBE_TIMEOUT_MS = 2_000; +// One login shell runs every probe; shell start-up files (nvm, conda, pyenv) can take a few seconds. +const LOGIN_SHELL_TIMEOUT_MS = 8_000; +const SECTION_MARKER = "__memmy_probe__"; + +export interface MachineProbe { + os: string; + osVersion: string; + arch: string; + shell: string | null; + tools: Record; +} + +export type VersionRunner = (executable: string, args: string[]) => Promise; +/** Runs a script in the user's login shell and returns its stdout, or null. */ +export type LoginShellRunner = (shell: string, script: string) => Promise; + +/** + * Probes this computer only by running `--version` style commands; never reads conversation content. + * Versions are read through the user's login shell first, because an app started from the Dock does not + * see the PATH set up in shell start-up files (nvm, conda, pyenv); tools it cannot see are run directly. + */ +export async function probeMachine(options: { + runVersion?: VersionRunner; + runLoginShell?: LoginShellRunner; + env?: NodeJS.ProcessEnv; + os?: NodeJS.Platform; +} = {}): Promise { + const runVersion = options.runVersion ?? defaultVersionRunner; + const runLoginShell = options.runLoginShell ?? defaultLoginShellRunner; + const env = options.env ?? process.env; + const os = options.os ?? platform(); + const fromShell = env.SHELL && os !== "win32" + ? parseShellProbe(await runLoginShell(env.SHELL, shellProbeScript()).catch(() => null)) + : new Map(); + const entries = await Promise.all(ALL_TOOLS.map(async (tool) => { + const viaShell = fromShell.get(tool); + if (viaShell) return [tool, viaShell] as const; + const output = await runVersion(tool, TOOL_ARGS[tool]).catch(() => null); + return [tool, output ? versionNumber(output) : null] as const; + })); + // os.release() is the Darwin kernel version on macOS; the product version is what users recognize. + const productVersion = os === "darwin" + ? await runVersion("sw_vers", ["-productVersion"]).catch(() => null) + : null; + return { + os, + osVersion: (productVersion ? versionNumber(productVersion) : null) ?? release(), + arch: arch(), + shell: env.SHELL ? path.basename(env.SHELL) : null, + tools: Object.fromEntries(entries) as Record + }; +} + +/** Fixed-format machine profile text; the same probe always renders the same text. */ +export function renderMachineProfile(probe: MachineProbe): string { + const lines = [ + `- 系统：${osLabel(probe.os)} ${probe.osVersion}（${probe.arch}）`, + ...(probe.shell ? [`- shell：${probe.shell}`] : []) + ]; + const installed = ALL_TOOLS + .filter((tool) => probe.tools[tool]) + .map((tool) => `${tool} ${probe.tools[tool]}`); + if (installed.length) lines.push(`- 已安装：${installed.join("，")}`); + const missing = MACHINE_TOOLS.filter((tool) => !probe.tools[tool]); + if (missing.length) lines.push(`- 未检测到：${missing.join("，")}`); + return lines.join("\n"); +} + +export function machineProfileFingerprint(probe: MachineProbe): string { + return sha256Hex(renderMachineProfile(probe)); +} + +function osLabel(os: string): string { + if (os === "darwin") return "macOS"; + if (os === "win32") return "Windows"; + if (os === "linux") return "Linux"; + return os; +} + +/** Keeps the version number only, e.g. `git version 2.39.5 (Apple Git-154)` -> `2.39.5`; output without one is not a version. */ +function versionNumber(output: string): string | null { + const version = /v?(\d+(?:\.\d+)+)/.exec(output); + return version ? version[1]! : null; +} + +/** Prints a marker line per tool, then its version; tools the shell cannot find print nothing. */ +function shellProbeScript(): string { + return ALL_TOOLS.map((tool) => + `printf '\\n${SECTION_MARKER}${tool}\\n'; command -v ${tool} >/dev/null 2>&1 && ${tool} ${TOOL_ARGS[tool].join(" ")} 2>&1 | head -n 3` + ).join("; "); +} + +function parseShellProbe(output: string | null): Map { + const found = new Map(); + if (!output) return found; + // Anything a start-up file prints before the first marker is ignored. + for (const section of output.split(SECTION_MARKER).slice(1)) { + const newline = section.indexOf("\n"); + const tool = (newline < 0 ? section : section.slice(0, newline)).trim() as MachineTool; + const version = newline < 0 ? null : versionNumber(section.slice(newline + 1)); + if (ALL_TOOLS.includes(tool) && version) found.set(tool, version); + } + return found; +} + +async function defaultLoginShellRunner(shell: string, script: string): Promise { + try { + const result = await execFileAsync(shell, ["-ilc", script], { + cwd: tmpdir(), + timeout: LOGIN_SHELL_TIMEOUT_MS, + maxBuffer: 64 * 1024, + windowsHide: true + }); + return result.stdout || null; + } catch { + return null; + } +} + +async function defaultVersionRunner(executable: string, args: string[]): Promise { + try { + const result = await execFileAsync(executable, args, { + cwd: tmpdir(), + timeout: PROBE_TIMEOUT_MS, + maxBuffer: 4096, + shell: false, + windowsHide: true + }); + return `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim() || null; + } catch { + return null; + } +} diff --git a/Memory/src/service/l3-field/project-resolver.ts b/Memory/src/service/l3-field/project-resolver.ts new file mode 100644 index 000000000..8a3e9f671 --- /dev/null +++ b/Memory/src/service/l3-field/project-resolver.ts @@ -0,0 +1,201 @@ +import { existsSync, realpathSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { readGitIdentity } from "./git-identity.js"; +import type { L3UserProject, ResolvedProject } from "./types.js"; + +export interface ResolverContext { + homeDir: string; + /** Folders the user picked in the desktop app. */ + userProjects: readonly L3UserProject[]; + /** Roots of throwaway task folders created by Memmy itself (default `~/Memmy`). */ + standaloneRoots: readonly string[]; + /** System temporary directories; defaults to the usual macOS/Linux locations. */ + tempRoots?: readonly string[]; + /** Remote workspaces are never resolved: the Memory service cannot see their files. */ + workspaceIsLocal: boolean; +} + +export interface ResolveProjectInput { + path: string; + /** A session folder may be a plain-folder project; a tool path only joins a project it lands in. */ + role: "session_dir" | "tool_path"; + sessionCwd: string | null; + sessionProjectRoot: string | null; +} + +type PathApi = Pick; + +export function defaultTempRoots(): string[] { + return [tmpdir(), "/tmp", "/private/tmp", "/var/folders", "/private/var/folders"]; +} + +/** True when `child` is `parent` or inside it, compared by path components (case-insensitive on Windows). */ +export function isPathWithin(parent: string, child: string, api: PathApi = path): boolean { + const caseInsensitive = api.sep === "\\"; + const from = api.resolve(caseInsensitive ? parent.toLowerCase() : parent); + const to = api.resolve(caseInsensitive ? child.toLowerCase() : child); + const relative = api.relative(from, to); + return relative === "" || (!relative.startsWith("..") && !api.isAbsolute(relative)); +} + +/** + * Decides which project a path belongs to: the nearer of a user-selected + * folder and a git repository, otherwise (for a session folder) the folder + * itself. Temporary and container folders never become projects. + */ +export function resolveProject(input: ResolveProjectInput, ctx: ResolverContext): ResolvedProject | null { + if (!ctx.workspaceIsLocal || !path.isAbsolute(input.path)) return null; + const target = lenientRealpath(input.path); + const home = lenientRealpath(ctx.homeDir); + if (isThrowawayLocation(target, home, ctx)) return null; + + const startDir = existingDirectory(target); + const selected = nearestUserProject(target, ctx.userProjects); + // A folder the user chose is a project wherever it is; otherwise tool install folders never are. + if (!selected && isToolLocation(target, home)) return null; + // A dependency that is its own git clone belongs to the project that contains it. + const gitStart = startDir ? outsidePackageDirectory(startDir) : null; + const gitRaw = gitStart ? readGitIdentity(gitStart) : null; + const git = gitRaw && isUsableRoot(gitRaw.repoRoot, home, ctx) && !isToolLocation(gitRaw.repoRoot, home) ? gitRaw : null; + + if (selected && isUsableRoot(selected.root, home, ctx)) { + const gitIsNearer = git && git.repoRoot.length > selected.root.length; + if (!gitIsNearer) { + const sameRoot = git?.repoRoot === selected.root; + return { + identity: sameRoot ? git!.identity : `dir:${selected.root}`, + checkoutRoot: selected.root, + source: "user_selected", + displayName: selected.name + }; + } + } + if (git) { + return { + identity: git.identity, + checkoutRoot: git.repoRoot, + source: "git", + // Named after the remote repository so every checkout shows the same name. + displayName: git.identity.startsWith("gitdir:") + ? path.basename(git.repoRoot) + : git.identity.split("/").at(-1)! + }; + } + + if (input.role === "session_dir") { + const root = startDir ?? path.dirname(target); + if (!isUsableRoot(root, home, ctx) || hasPackageDirectory(root)) return null; + return { identity: `dir:${root}`, checkoutRoot: root, source: "folder", displayName: path.basename(root) }; + } + const sessionFolder = input.sessionProjectRoot ?? input.sessionCwd; + if (sessionFolder && isPathWithin(lenientRealpath(sessionFolder), target)) { + return resolveProject({ ...input, path: sessionFolder, role: "session_dir" }, ctx); + } + return null; +} + +function nearestUserProject( + target: string, + projects: readonly L3UserProject[] +): { root: string; name: string } | null { + let best: { root: string; name: string } | null = null; + for (const project of projects) { + const root = lenientRealpath(project.rootPath); + if (isPathWithin(root, target) && (!best || root.length > best.root.length)) { + best = { root, name: project.name }; + } + } + return best; +} + +/** Inside a temp, Codex dated, Claude scratch, `~/.codex/memories*` or Memmy task folder. */ +function isThrowawayLocation(target: string, home: string, ctx: ResolverContext): boolean { + const roots = [ + ...(ctx.tempRoots ?? defaultTempRoots()), + ...ctx.standaloneRoots, + path.join(home, "Library", "Application Support", "Claude", "scratch-workspaces") + ].map(lenientRealpath); + if (roots.some((root) => isPathWithin(root, target))) return true; + const relativeToHome = isPathWithin(home, target) ? path.relative(home, target).split(path.sep) : []; + if (relativeToHome[0] === ".codex" && relativeToHome[1]?.startsWith("memories")) return true; + if (relativeToHome[0] === "Documents" && relativeToHome[1] === "Codex" && /^\d{4}-\d{2}-\d{2}$/.test(relativeToHome[2] ?? "")) { + return true; + } + return false; +} + +// Temp folders (/var/folders, /tmp) are handled as throwaway locations; /etc resolves to /private/etc on macOS. +const SYSTEM_ROOTS = [ + "/usr", "/bin", "/sbin", "/etc", "/private/etc", "/System", "/Library", "/Applications", + "/opt/homebrew", "/opt/local", "/nix" +]; +const WINDOWS_TOOL_DIRS = /\\(program files( \(x86\))?|programdata|appdata)(\\|$)/i; +const PACKAGE_DIRS = new Set(["node_modules", "site-packages", "dist-packages", "vendor", "bower_components"]); + +/** + * Where tools and the system live, not projects: system and package-manager folders, Windows + * program folders, ~/Library, and hidden folders right under home (~/.nvm, ~/.oh-my-zsh, ~/.pyenv, ...). + * Many of these are git clones, so the git check alone would turn them into projects. + */ +function isToolLocation(target: string, home: string): boolean { + if (WINDOWS_TOOL_DIRS.test(target)) return true; + if (isPathWithin(home, target)) { + const first = path.relative(home, target).split(path.sep)[0] ?? ""; + return first.startsWith(".") || first === "Library"; + } + return SYSTEM_ROOTS.some((root) => isPathWithin(root, target, path.posix)); +} + +/** A root inside a dependency folder is a downloaded package, not the user's project. */ +function hasPackageDirectory(root: string): boolean { + return root.split(/[\\/]/).some((part) => PACKAGE_DIRS.has(part)); +} + +/** The directory just above the first dependency folder on the path, or the path itself. */ +function outsidePackageDirectory(target: string): string { + const parts = target.split(path.sep); + const index = parts.findIndex((part) => PACKAGE_DIRS.has(part)); + return index > 0 ? parts.slice(0, index).join(path.sep) || path.sep : target; +} + +/** A project root may not be the home folder, a top-level container folder, a filesystem root or a throwaway location. */ +function isUsableRoot(root: string, home: string, ctx: ResolverContext): boolean { + if (path.dirname(root) === root) return false; + const containers = [home, ...["Desktop", "Documents", "Downloads"].map((name) => path.join(home, name))]; + if (containers.includes(root)) return false; + return !isThrowawayLocation(root, home, ctx); +} + +/** realpath that tolerates a missing tail: resolves the nearest existing ancestor and re-appends the rest. */ +function lenientRealpath(target: string): string { + const absolute = path.resolve(target); + const missing: string[] = []; + let current = absolute; + for (;;) { + try { + const real = realpathSync(current); + return missing.length ? path.join(real, ...missing.reverse()) : real; + } catch { + const parent = path.dirname(current); + if (parent === current) return absolute; + missing.push(path.basename(current)); + current = parent; + } + } +} + +/** The path itself when it is a directory, otherwise its nearest existing parent directory. */ +function existingDirectory(target: string): string | null { + let current = target; + for (;;) { + try { + if (existsSync(current) && statSync(current).isDirectory()) return current; + } catch { + // fall through to the parent + } + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} diff --git a/Memory/src/service/l3-field/session-units.ts b/Memory/src/service/l3-field/session-units.ts new file mode 100644 index 000000000..2b6c243aa --- /dev/null +++ b/Memory/src/service/l3-field/session-units.ts @@ -0,0 +1,93 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { isLocalWorkspaceUri } from "../../contracts/index.js"; +import type { Repositories, SessionRecord } from "../../storage/repositories.js"; +import { l3UnitKey } from "../../storage/l3-field-repository.js"; +import { resolveProject, type ResolverContext } from "./project-resolver.js"; +import type { L3UnitRecord, ResolvedProject } from "./types.js"; + +/** What the new L3 needs to know about the computer the Memory service runs on. */ +export interface L3FieldEnvironment { + /** Stable id of this installation (the workspace host id); machine units are keyed by it. */ + machineId: string; + homeDir: string; + /** Folders never treated as projects; defaults to the system temp folders. Tests override it. */ + tempRoots?: readonly string[]; +} + +export interface SessionUnits { + machine: L3UnitRecord; + project: { resolved: ResolvedProject; unit: L3UnitRecord } | null; + sessionCwd: string | null; + workspaceIsLocal: boolean; +} + +/** The session's working directory: the v2 workspace URI when present, else the legacy workspace path. */ +export function sessionWorkingDirectory(session: SessionRecord): { cwd: string | null; isLocal: boolean } { + const uri = typeof session.meta.workspace_uri === "string" ? session.meta.workspace_uri : null; + if (uri) { + if (!isLocalWorkspaceUri(uri)) return { cwd: null, isLocal: false }; + try { + return { cwd: fileURLToPath(uri), isLocal: true }; + } catch { + return { cwd: null, isLocal: false }; + } + } + const legacy = session.workspacePath; + return legacy && path.isAbsolute(legacy) ? { cwd: legacy, isLocal: true } : { cwd: null, isLocal: true }; +} + +export function resolverContextFor( + repos: Repositories, + env: L3FieldEnvironment, + userId: string, + workspaceIsLocal: boolean +): ResolverContext { + const standalone = repos.l3Field.getStandaloneRoots(userId); + return { + homeDir: env.homeDir, + userProjects: repos.l3Field.listUserProjects(userId), + standaloneRoots: standalone.length ? standalone : [path.join(env.homeDir, "Memmy")], + tempRoots: env.tempRoots, + workspaceIsLocal + }; +} + +/** Machine unit plus the project unit of the session folder (if it is a project); creates units on first use. */ +export function resolveSessionUnits( + repos: Repositories, + env: L3FieldEnvironment, + session: SessionRecord +): SessionUnits { + const { cwd, isLocal } = sessionWorkingDirectory(session); + const machine = repos.l3Field.ensureMachineUnit({ userId: session.userId, machineId: env.machineId }); + if (!cwd) return { machine, project: null, sessionCwd: null, workspaceIsLocal: isLocal }; + const resolved = resolveProject( + { path: cwd, role: "session_dir", sessionCwd: cwd, sessionProjectRoot: null }, + resolverContextFor(repos, env, session.userId, isLocal) + ); + if (!resolved) return { machine, project: null, sessionCwd: cwd, workspaceIsLocal: isLocal }; + const unit = repos.l3Field.ensureProjectUnit({ userId: session.userId, machineId: env.machineId, project: resolved }); + return { machine, project: { resolved, unit }, sessionCwd: cwd, workspaceIsLocal: isLocal }; +} + +/** + * The session's units as they already exist, creating nothing: for reads while memory writes are + * off. A unit that was never created has nothing to show yet. + */ +export function findSessionUnits( + repos: Repositories, + env: L3FieldEnvironment, + session: SessionRecord +): { machine: L3UnitRecord | undefined; project: L3UnitRecord | undefined } { + const machine = repos.l3Field.getUnit(l3UnitKey({ userId: session.userId, machineId: env.machineId, kind: "machine", identity: null })); + const { cwd, isLocal } = sessionWorkingDirectory(session); + const resolved = cwd ? resolveProject( + { path: cwd, role: "session_dir", sessionCwd: cwd, sessionProjectRoot: null }, + resolverContextFor(repos, env, session.userId, isLocal) + ) : null; + const project = resolved + ? repos.l3Field.getUnit(l3UnitKey({ userId: session.userId, machineId: env.machineId, kind: "project", identity: resolved.identity })) + : undefined; + return { machine, project }; +} diff --git a/Memory/src/service/l3-field/snapshot-memory.ts b/Memory/src/service/l3-field/snapshot-memory.ts new file mode 100644 index 000000000..d580b1e69 --- /dev/null +++ b/Memory/src/service/l3-field/snapshot-memory.ts @@ -0,0 +1,37 @@ +import type { L3WorldModelFields } from "../../contracts/index.js"; +import type { MemoryRow } from "../../types.js"; + +type SnapshotMemory = Pick; + +function l3FieldInfo(memory: SnapshotMemory): Record | null { + if (memory.memoryLayer !== "L3") return null; + const field = memory.properties.internal_info.l3_field; + return field && typeof field === "object" && !Array.isArray(field) ? field as Record : null; +} + +/** Unit key of an L3 field cognition snapshot memory, or null for any other memory. */ +export function l3FieldSnapshotUnitKey(memory: SnapshotMemory): string | null { + const unitKey = l3FieldInfo(memory)?.unit_key; + return typeof unitKey === "string" && unitKey ? unitKey : null; +} + +export function isL3FieldSnapshotMemory(memory: SnapshotMemory): boolean { + return l3FieldSnapshotUnitKey(memory) !== null; +} + +/** The schema-2 fields a snapshot carries for pages that only understand the old L3 shape. */ +export function l3FieldSnapshotLegacyFields(memory: SnapshotMemory): L3WorldModelFields | null { + if (!l3FieldSnapshotUnitKey(memory)) return null; + const legacy = l3FieldInfo(memory)?.legacy_fields; + if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) return null; + const value = (key: string): string | null => { + const field = (legacy as Record)[key]; + return typeof field === "string" ? field : null; + }; + return { + generalRulesAndSafetyConstraints: value("general_rules_and_safety_constraints"), + projectEnvironmentProfile: value("project_environment_profile"), + projectContract: value("project_contract"), + domainKnowledge: value("domain_knowledge") + }; +} diff --git a/Memory/src/service/l3-field/text-units.ts b/Memory/src/service/l3-field/text-units.ts new file mode 100644 index 000000000..e1acd7db9 --- /dev/null +++ b/Memory/src/service/l3-field/text-units.ts @@ -0,0 +1,16 @@ +const CJK_CHARACTER = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu; +const WORD_CHARACTER = /[\p{L}\p{N}]/u; + +/** + * Length of an L3 item in "units": each CJK character counts as one, every + * other whitespace-separated word counts as one, punctuation-only tokens + * count as zero. + */ +export function countL3Units(text: string): number { + const cjkCount = text.match(CJK_CHARACTER)?.length ?? 0; + const words = text + .replace(CJK_CHARACTER, " ") + .split(/\s+/u) + .filter((token) => WORD_CHARACTER.test(token)); + return cjkCount + words.length; +} diff --git a/Memory/src/service/l3-field/types.ts b/Memory/src/service/l3-field/types.ts new file mode 100644 index 000000000..ac4a34f9f --- /dev/null +++ b/Memory/src/service/l3-field/types.ts @@ -0,0 +1,91 @@ +/** Shared types for L3 field cognition: machine/project units holding itemized facts. */ + +export type L3UnitKind = "machine" | "project"; +export type L3ItemKind = "env_fact" | "project_constraint"; +export type L3ItemStatus = "active" | "inactive"; +export type ProjectIdentitySource = "user_selected" | "git" | "folder"; +export type L3UnitBatchStatus = "queued" | "applied" | "failed"; + +export interface ResolvedProject { + /** Stable identity shared by every checkout of the same project. */ + identity: string; + /** Root directory of the checkout the evidence came from. */ + checkoutRoot: string; + source: ProjectIdentitySource; + displayName: string; +} + +export interface EvidenceFragment { + id: string; + rawTurnId: string; + l1MemoryId: string; + toolName: string | null; + input: unknown; + output: string | null; + isError: boolean; + paths: string[]; + candidateUnitKeys: string[]; + at: string; +} + +export interface L3ItemEvidence { + fragmentIds: string[]; + l1MemoryIds: string[]; +} + +export interface L3UnitRecord { + unitKey: string; + userId: string; + machineId: string; + kind: L3UnitKind; + projectIdentity?: string; + displayName?: string; + identitySource?: ProjectIdentitySource; + checkoutRoot?: string; + profileText?: string; + profileFingerprint?: string; + snapshotMemoryId?: string; + deletedAt?: string; + nextSeq: number; + createdAt: string; + updatedAt: string; +} + +export interface L3ItemRecord { + id: string; + unitKey: string; + kind: L3ItemKind; + content: string; + evidence: L3ItemEvidence; + evidenceAt: string; + lastConfirmedAt: string; + status: L3ItemStatus; + replacedBy?: string; + /** When the evidence that removed it happened; only for a delete or replace op. */ + removedEvidenceAt?: string; + inactiveAt?: string; + batchId?: string; + createdAt: string; + updatedAt: string; +} + +export interface L3UserProject { + rootPath: string; + name: string; +} + +export interface L3UnitBatchRecord { + id: string; + machineUnitKey: string; + scopeSeq: number; + userId: string; + sessionId: string; + trigger: string; + startTraceSeq: number; + endTraceSeq: number; + l1MemoryIds: string[]; + rawTurnIds: string[]; + status: L3UnitBatchStatus; + createdAt: string; + updatedAt: string; +} diff --git a/Memory/src/service/l3-field/unit-ops.ts b/Memory/src/service/l3-field/unit-ops.ts new file mode 100644 index 000000000..04f4c3be2 --- /dev/null +++ b/Memory/src/service/l3-field/unit-ops.ts @@ -0,0 +1,149 @@ +import type { EvidenceFragment, L3ItemKind, L3ItemRecord, L3UnitRecord } from "./types.js"; + +/** One model call: the machine plus at most one project, with the evidence that may support them. */ +export interface L3UpdateGroup { + machine: L3UnitRecord; + project: L3UnitRecord | null; + fragments: EvidenceFragment[]; + l1: Array<{ id: string; title: string; summary: string; at: string }>; +} + +/** A validated operation; every op names the unit it changes. */ +export type L3UnitOp = + | { op: "add"; unitKey: string; kind: L3ItemKind; content: string; evidence: string[] } + | { op: "replace"; unitKey: string; kind: L3ItemKind; ids: string[]; content: string; evidence: string[] } + | { op: "delete"; unitKey: string; id: string; evidence: string[] } + | { op: "confirm"; unitKey: string; id: string; evidence: string[] }; + +interface RawOp { + op: "add" | "replace" | "delete" | "confirm"; + unit?: string; + unit_key?: string; + kind?: string; + content?: string; + ids?: string[]; + id?: string; + evidence: string[]; +} + +const MIN_REPOSITORY_NAME_LENGTH = 4; + +/** JSON shape only; semantic checks run in validateL3UnitOps so the model can be told what was wrong. */ +export function parseL3UnitOutput(value: unknown): { reason: string; ops: RawOp[] } { + if (!isRecord(value) || !Array.isArray(value.ops)) throw new TypeError("expected {reason, ops[]}"); + const ops = value.ops.map((op, index): RawOp => { + if (!isRecord(op) || !["add", "replace", "delete", "confirm"].includes(op.op as string)) { + throw new TypeError(`ops[${index}]: unknown op`); + } + if (!isStringArray(op.evidence)) throw new TypeError(`ops[${index}]: evidence must be a string array`); + if (op.ids !== undefined && !isStringArray(op.ids)) throw new TypeError(`ops[${index}]: ids must be a string array`); + for (const key of ["unit", "unit_key", "kind", "content", "id"]) { + if (op[key] !== undefined && typeof op[key] !== "string") throw new TypeError(`ops[${index}]: ${key} must be a string`); + } + return op as unknown as RawOp; + }); + return { reason: typeof value.reason === "string" ? value.reason : "", ops }; +} + +/** Checks every op against the group; returns the ops that pass and one line per problem. */ +export function validateL3UnitOps( + rawOps: readonly RawOp[], + group: L3UpdateGroup, + activeItems: ReadonlyMap, + /** Items written earlier in this batch: shown to the model, but changes to them are dropped, not errors. */ + addedInBatch: ReadonlySet = new Set() +): { ops: L3UnitOp[]; problems: string[] } { + const fragments = new Map(group.fragments.map((fragment) => [fragment.id, fragment])); + const groupUnits = new Set([group.machine.unitKey, ...(group.project ? [group.project.unitKey] : [])]); + const usedIds = new Set(); + const ops: L3UnitOp[] = []; + const problems: string[] = []; + + rawOps.forEach((raw, index) => { + const fail = (message: string) => problems.push(`ops[${index}]: ${message}`); + const targets = raw.op === "replace" ? raw.ids ?? [] : raw.op === "add" ? [] : [raw.id ?? ""]; + if (targets.some((id) => addedInBatch.has(id))) return; + const evidence = [...new Set(raw.evidence)]; + if (evidence.length === 0) return fail("evidence is empty"); + const unknown = evidence.filter((id) => !fragments.has(id)); + if (unknown.length) return fail(`evidence ${unknown.join(", ")} is not a fragment of this input`); + + if (raw.op === "add" || raw.op === "replace") { + const content = raw.content?.trim() ?? ""; + if (!content) return fail("content is empty"); + let unitKey: string; + let kind: L3ItemKind; + let ids: string[] = []; + if (raw.op === "add") { + if (raw.unit === "machine") { + unitKey = group.machine.unitKey; + } else if (raw.unit === "project") { + if (!group.project) return fail("no project is listed, so unit must be machine"); + if (raw.unit_key && raw.unit_key !== group.project.unitKey) return fail(`unit_key ${raw.unit_key} is not a listed project`); + unitKey = group.project.unitKey; + } else { + return fail(`unit must be "machine" or "project"`); + } + if (raw.kind !== "env_fact" && raw.kind !== "project_constraint") return fail("kind must be env_fact or project_constraint"); + kind = raw.kind; + } else { + ids = [...new Set(raw.ids ?? [])]; + if (ids.length === 0) return fail("replace needs ids"); + const items = ids.map((id) => activeItems.get(id)); + const missing = ids.filter((_, i) => !items[i] || !groupUnits.has(items[i]!.unitKey)); + if (missing.length) return fail(`id ${missing.join(", ")} is not an active item of this group`); + const reused = ids.filter((id) => usedIds.has(id)); + if (reused.length) return fail(`id ${reused.join(", ")} is already changed by another op`); + unitKey = items[0]!.unitKey; + kind = items[0]!.kind; + if (items.some((item) => item!.unitKey !== unitKey || item!.kind !== kind)) { + return fail("replaced ids must share one unit and kind"); + } + } + const isMachine = unitKey === group.machine.unitKey; + if (isMachine && kind !== "env_fact") return fail("machine items must be env_fact"); + if (kind === "env_fact" && !evidence.some((id) => provesFact(fragments.get(id)!, unitKey, isMachine))) { + return fail("env_fact needs a cited tool output or error that touched this unit"); + } + if (isMachine && group.project && mentionsProject(content, group.project)) { + return fail("a machine item must not mention a project, repository or folder"); + } + ids.forEach((id) => usedIds.add(id)); + ops.push(raw.op === "add" + ? { op: "add", unitKey, kind, content, evidence } + : { op: "replace", unitKey, kind, ids, content, evidence }); + return; + } + + const id = raw.id ?? ""; + const item = activeItems.get(id); + if (!item || !groupUnits.has(item.unitKey)) return fail(`id ${id || "(missing)"} is not an active item of this group`); + if (usedIds.has(id)) return fail(`id ${id} is already changed by another op`); + usedIds.add(id); + ops.push({ op: raw.op, unitKey: item.unitKey, id, evidence }); + }); + return { ops, problems }; +} + +/** A tool call with output or an error, whose own paths belong to the unit (any call counts for the machine). */ +function provesFact(fragment: EvidenceFragment, unitKey: string, isMachine: boolean): boolean { + if (!fragment.toolName || !(fragment.isError || fragment.output?.trim())) return false; + return isMachine || fragment.candidateUnitKeys.includes(unitKey); +} + +function mentionsProject(content: string, project: L3UnitRecord): boolean { + const lower = content.toLowerCase(); + if (project.checkoutRoot && lower.includes(project.checkoutRoot.toLowerCase())) return true; + const name = project.displayName?.trim().toLowerCase(); + if (!name || name.length < MIN_REPOSITORY_NAME_LENGTH) return false; + const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`(^|[^\\p{L}\\p{N}_-])${escaped}($|[^\\p{L}\\p{N}_-])`, "u").test(lower); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} diff --git a/Memory/src/service/l3-field/unit-update-pipeline.ts b/Memory/src/service/l3-field/unit-update-pipeline.ts new file mode 100644 index 000000000..89a58a67b --- /dev/null +++ b/Memory/src/service/l3-field/unit-update-pipeline.ts @@ -0,0 +1,429 @@ +import { canonicalJson, type JsonValue } from "../../contracts/index.js"; +import { createMemoryLogger } from "../../logging/logger.js"; +import { languageSteeringLine, steeredPromptLanguage, traceMetaFromMemory } from "../../algorithm/plugin-algorithms.js"; +import type { MemoryLanguage } from "../../config/index.js"; +import type { LlmClient } from "../../model/types.js"; +import { l3UnitKey } from "../../storage/l3-field-repository.js"; +import type { EvolutionJobRecord, Repositories } from "../../storage/repositories.js"; +import { nowIso } from "../../utils/time.js"; +import { evolutionJobLogFields, logEvolutionDecision } from "../evolution/evolution-logging.js"; +import { L3WorldModelTerminalEvidenceError } from "../evolution/l3-world-model-pipeline.js"; +import { storedGeneratedTitle } from "../read-model/display-fields.js"; +import { completeStrictJson } from "../l3-world-model/strict-json-completion.js"; +import { classifyProcessingError } from "../worker/job-handlers.js"; +import { buildEvidenceFragments } from "./evidence-fragments.js"; +import { resolveProject } from "./project-resolver.js"; +import { resolverContextFor, sessionWorkingDirectory, type L3FieldEnvironment } from "./session-units.js"; +import { countL3Units } from "./text-units.js"; +import type { EvidenceFragment, L3ItemKind, L3ItemRecord, L3UnitBatchRecord, L3UnitRecord, ResolvedProject } from "./types.js"; +import { L3_UNIT_UPDATE_PROMPT, L3_UNIT_UPDATE_SCHEMA, l3UnitRetryNote } from "./unit-update-prompt.js"; +import { parseL3UnitOutput, validateL3UnitOps, type L3UnitOp, type L3UpdateGroup } from "./unit-ops.js"; + +/** The evidence's turn and L1 still exist (neither deleted nor redacted since the input was built). */ +function evidenceStillThere(repos: Repositories, fragment: EvidenceFragment): boolean { + const rawTurn = repos.runtime.getRawTurn(fragment.rawTurnId); + const memory = repos.memories.get(fragment.l1MemoryId); + return Boolean(rawTurn && !rawTurn.deletedAt && !rawTurn.redactedAt && + memory && !memory.deletedAt && memory.status !== "deleted"); +} + +/** The run lost its job's lease to a newer run; it must leave the job and the batch alone. */ +export class L3LeaseLostError extends Error { + constructor(jobId: string) { + super(`l3 unit job ${jobId} was leased again; this run's result is dropped`); + this.name = "L3LeaseLostError"; + } +} + +interface PendingMachineItem { + id: string; + kind: L3ItemKind; + content: string; +} + +const l3FieldLogger = createMemoryLogger("l3-field"); +const MAX_INPUT_CHARS = 65_536; +/** After the model rejects an input as too long for its context, the group is asked once more with this much. */ +const FALLBACK_INPUT_CHARS = 16_384; +/** The output is a short list of ops; a smaller budget leaves room for the input in the context window. */ +const MAX_OUTPUT_TOKENS = 16_384; +const MAX_USER_PROJECTS = 50; +const MAX_ITEMS_PER_KIND = 10; +const OVER_LENGTH_UNITS = 150; +const L1_SUMMARY_CHARS = 600; + +/** + * Turns one frozen batch into item operations. Each group (the machine plus at most one project) + * gets one model call; nothing is written until every group has a valid answer, and then the + * whole batch is written in one transaction. + */ +export class L3UnitUpdatePipeline { + constructor(private readonly deps: { + repos: Repositories; + llm: LlmClient; + language?: MemoryLanguage; + environment: () => L3FieldEnvironment; + }) {} + + async process(job: EvolutionJobRecord): Promise { + const batchId = typeof job.payload.batchId === "string" ? job.payload.batchId : ""; + const batch = this.deps.repos.l3Field.getBatch(batchId); + if (!batch) throw new L3WorldModelTerminalEvidenceError(`missing l3 unit batch: ${batchId}`); + // A job recovered after a restart may run again for a batch that was already written. + if (batch.status === "applied") return; + if ( + batch.status !== "queued" || + job.userId !== batch.userId || + job.sessionId !== batch.sessionId || + job.scopeKey !== batch.machineUnitKey || + job.scopeSeq !== batch.scopeSeq + ) { + throw new L3WorldModelTerminalEvidenceError("l3 unit job does not match its batch"); + } + const machine = this.deps.repos.l3Field.getUnit(batch.machineUnitKey); + const session = this.deps.repos.runtime.getSession(batch.sessionId); + if (!machine || !session || session.userId !== batch.userId) { + throw new L3WorldModelTerminalEvidenceError("l3 unit batch scope mismatch"); + } + + const groups = this.buildGroups(batch, machine, session); + const decided: Array<{ group: L3UpdateGroup; ops: L3UnitOp[] }> = []; + // Machine items earlier groups of this batch decided to write; later groups see them so they + // do not write the same fact again in their own words. + const addedToMachine: PendingMachineItem[] = []; + for (const group of groups) { + const ops = await this.decide(job, group, addedToMachine); + decided.push({ group, ops }); + for (const op of ops) { + if ((op.op === "add" || op.op === "replace") && op.unitKey === group.machine.unitKey) { + addedToMachine.push({ id: `added-in-batch-${addedToMachine.length + 1}`, kind: op.kind, content: op.content }); + } + } + } + this.apply(job, batch, decided); + } + + private buildGroups( + batch: L3UnitBatchRecord, + machine: L3UnitRecord, + session: NonNullable> + ): L3UpdateGroup[] { + const repos = this.deps.repos; + const env = { ...this.deps.environment(), machineId: machine.machineId }; + const { cwd, isLocal } = sessionWorkingDirectory(session); + const ctx = resolverContextFor(repos, env, batch.userId, isLocal); + const sessionProject = cwd + ? resolveProject({ path: cwd, role: "session_dir", sessionCwd: cwd, sessionProjectRoot: null }, ctx) + : null; + const projects = new Map(); + const unitKeyOf = (project: ResolvedProject) => { + const key = l3UnitKey({ userId: batch.userId, machineId: machine.machineId, kind: "project", identity: project.identity }); + if (!projects.has(key)) projects.set(key, project); + return key; + }; + + const traces = repos.l3WorldModels.listInputTracesInRange(batch.sessionId, batch.startTraceSeq - 1, batch.endTraceSeq); + const l1ByRawTurn = new Map(); + for (const trace of traces) { + if (!batch.l1MemoryIds.includes(trace.l1MemoryId) || l1ByRawTurn.has(trace.rawTurnId)) continue; + const memory = repos.memories.get(trace.l1MemoryId); + if (memory && !memory.deletedAt && memory.status !== "deleted") l1ByRawTurn.set(trace.rawTurnId, memory.id); + } + + const fragments: EvidenceFragment[] = []; + for (const rawTurnId of batch.rawTurnIds) { + const rawTurn = repos.runtime.getRawTurn(rawTurnId); + const l1MemoryId = l1ByRawTurn.get(rawTurnId); + if (!rawTurn || !l1MemoryId || rawTurn.deletedAt || rawTurn.redactedAt) continue; + if (rawTurn.userId !== batch.userId || rawTurn.sessionId !== batch.sessionId) { + throw new L3WorldModelTerminalEvidenceError(`raw turn scope mismatch: ${rawTurnId}`); + } + fragments.push(...buildEvidenceFragments({ + rawTurn, + l1MemoryId, + sessionCwd: cwd, + sessionProjectRoot: sessionProject?.checkoutRoot ?? null, + resolve: (target, role) => resolveProject({ + path: target, + role, + sessionCwd: cwd, + sessionProjectRoot: sessionProject?.checkoutRoot ?? null + }, ctx), + unitKeyOf + })); + } + + const l1Entries = new Map([...new Set(l1ByRawTurn.values())].map((id) => { + const memory = repos.memories.get(id)!; + const summary = traceMetaFromMemory(memory)?.summary || memory.memoryValue; + return [id, { + id, + title: storedGeneratedTitle(memory) ?? "", + summary: [...summary].slice(0, L1_SUMMARY_CHARS).join(""), + at: memory.createdAt + }] as const; + })); + const groupOf = (picked: EvidenceFragment[], project: L3UnitRecord | null): L3UpdateGroup => ({ + machine, + project, + fragments: picked, + l1: [...new Set(picked.map((fragment) => fragment.l1MemoryId))].map((id) => l1Entries.get(id)!) + }); + + const groups: L3UpdateGroup[] = []; + for (const [unitKey, resolved] of projects) { + const picked = fragments.filter((fragment) => fragment.candidateUnitKeys.includes(unitKey)); + if (picked.length === 0) continue; + const unit = repos.l3Field.ensureProjectUnit({ userId: batch.userId, machineId: machine.machineId, project: resolved }); + groups.push(groupOf(picked, unit)); + } + const machineOnly = fragments.filter((fragment) => fragment.candidateUnitKeys.length === 0); + if (machineOnly.length > 0) groups.push(groupOf(machineOnly, null)); + return groups; + } + + private async decide( + job: EvolutionJobRecord, + group: L3UpdateGroup, + addedToMachine: readonly PendingMachineItem[] + ): Promise { + const repos = this.deps.repos; + const machineItems = repos.l3Field.activeItems(group.machine.unitKey); + // Removed on evidence newer than some of this group's: a batch processed late (rerun from dead + // letter, history scanned late) or spanning a removal must not bring them back on older evidence. + const oldestEvidenceAt = group.fragments.map((fragment) => fragment.at).sort()[0]; + const removedLater = oldestEvidenceAt + ? repos.l3Field.itemsRemovedAfter([group.machine.unitKey, ...(group.project ? [group.project.unitKey] : [])], oldestEvidenceAt) + : []; + const projectItems = group.project ? repos.l3Field.activeItems(group.project.unitKey) : []; + const activeItems = new Map([...machineItems, ...projectItems].map((item) => [item.id, item])); + let input = this.buildInput(job, group, machineItems, projectItems, addedToMachine, removedLater, MAX_INPUT_CHARS); + const addedIds = new Set(addedToMachine.map((item) => item.id)); + const language = activeItems.size === 0 + ? steeredPromptLanguage(this.deps.language, group.fragments.filter((f) => !f.toolName).map((f) => f.output)) + : undefined; + const systemPrompt = language ? `${L3_UNIT_UPDATE_PROMPT}\n\n${languageSteeringLine(language)}` : L3_UNIT_UPDATE_PROMPT; + const ask = (dynamicInput: JsonValue) => completeStrictJson({ + llm: this.deps.llm, + operation: "l3_field.unit_update", + systemPrompt, + dynamicInput, + expectedSchema: L3_UNIT_UPDATE_SCHEMA as unknown as JsonValue, + validate: parseL3UnitOutput, + maxTokens: MAX_OUTPUT_TOKENS + }); + + let first; + try { + first = await ask(input); + } catch (error) { + // Resending the same input would fail the same way; shrink it once instead. + if (classifyProcessingError(error).code !== "model_input_too_long") throw error; + logEvolutionDecision(job, "l3_unit_update", "input_too_long_shrunk", { maxChars: FALLBACK_INPUT_CHARS }); + input = this.buildInput(job, group, machineItems, projectItems, addedToMachine, removedLater, FALLBACK_INPUT_CHARS); + first = await ask(input); + } + const checked = validateL3UnitOps(first.ops, group, activeItems, addedIds); + if (checked.problems.length === 0) return checked.ops; + // Semantic problems get one regeneration with the reasons; a second failure fails the batch. + const second = await ask({ + ...input, + previous_output: first as unknown as JsonValue, + retry: l3UnitRetryNote(checked.problems) + }); + const rechecked = validateL3UnitOps(second.ops, group, activeItems, addedIds); + if (rechecked.problems.length > 0) { + throw new Error(`l3 unit output rejected twice: ${rechecked.problems.join("; ")}`); + } + return rechecked.ops; + } + + private buildInput( + job: EvolutionJobRecord, + group: L3UpdateGroup, + machineItems: readonly L3ItemRecord[], + projectItems: readonly L3ItemRecord[], + addedToMachine: readonly PendingMachineItem[], + removedLater: readonly L3ItemRecord[], + maxChars: number + ): Record { + const repos = this.deps.repos; + const session = repos.runtime.getSession(job.sessionId!)!; + const { cwd } = sessionWorkingDirectory(session); + const items = (list: readonly L3ItemRecord[]) => list.map((item) => ({ + id: item.id, + kind: item.kind, + content: item.content, + last_confirmed_at: item.lastConfirmedAt + })); + const evidence = group.fragments.map((fragment) => ({ + id: fragment.id, + type: fragment.toolName ? "tool" : "user_text", + ...(fragment.toolName + ? { tool: fragment.toolName, input: (fragment.input ?? null) as JsonValue, is_error: fragment.isError } + : {}), + ...(fragment.toolName ? { output: fragment.output } : { text: fragment.output }), + paths: fragment.paths, + candidate_units: ["machine", ...fragment.candidateUnitKeys], + at: fragment.at + })); + const input: Record = { + batch_l1: group.l1, + evidence, + machine: { + profile: group.machine.profileText ?? null, + items: [ + ...items(machineItems), + ...addedToMachine.map((item) => ({ id: item.id, kind: item.kind, content: item.content, added_in_this_batch: true })) + ] + }, + projects: group.project + ? [{ + unit_key: group.project.unitKey, + name: group.project.displayName ?? null, + source: group.project.identitySource ?? null, + root: group.project.checkoutRoot ?? null, + profile: group.project.profileText ?? null, + items: items(projectItems) + }] + : [], + // Only when some of this group's evidence is older than a removal: what was removed, and when. + ...(removedLater.length > 0 + ? { removed_by_newer_evidence: removedLater.map((item) => ({ + kind: item.kind, content: item.content, removed_at: item.removedEvidenceAt ?? null + })) } + : {}), + session: { cwd, current_project: group.project?.displayName ?? null }, + user_selected_projects: repos.l3Field.listUserProjects(job.userId) + .slice(0, MAX_USER_PROJECTS) + .map((project) => ({ name: project.name, root: project.rootPath })) + }; + // When the input is too long the oldest tool calls go first, the user's own words last; + // L1, items and profiles are kept. + const charsBefore = canonicalJson(input).length; + const evidenceBefore = evidence.length; + let droppedTool = 0; + let droppedUserText = 0; + while (canonicalJson(input).length > maxChars && evidence.length > 1) { + const oldestTool = evidence.findIndex((fragment) => fragment.type === "tool"); + const [removed] = evidence.splice(oldestTool >= 0 ? oldestTool : 0, 1); + if (removed?.type === "tool") droppedTool += 1; + else droppedUserText += 1; + } + if (droppedTool + droppedUserText > 0) { + l3FieldLogger.debug("input.truncated", { + ...evolutionJobLogFields(job), + unitKey: group.project?.unitKey ?? group.machine.unitKey, + maxChars, + charsBefore, + charsAfter: canonicalJson(input).length, + evidenceBefore, + evidenceAfter: evidence.length, + droppedToolFragments: droppedTool, + droppedUserTextFragments: droppedUserText + }); + } + return input; + } + + private apply(job: EvolutionJobRecord, batch: L3UnitBatchRecord, decided: Array<{ group: L3UpdateGroup; ops: L3UnitOp[] }>): void { + const repos = this.deps.repos; + // Two groups changing the same machine item: the first group's op wins. + const claimedMachineIds = new Set(); + const plan = decided.map(({ group, ops }) => ({ + group, + ops: ops.filter((op) => { + if (op.unitKey !== group.machine.unitKey || op.op === "add") return true; + const ids = op.op === "replace" ? op.ids : [op.id]; + if (ids.some((id) => claimedMachineIds.has(id))) { + logEvolutionDecision(job, "l3_unit_update", "machine_item_conflict_dropped", { ids }); + return false; + } + ids.forEach((id) => claimedMachineIds.add(id)); + return true; + }) + })); + + repos.transaction(() => { + // A run that outlived its lease (the job was leased again since) writes nothing. + if (job.leaseToken && !repos.runtime.isJobLeaseHeld(job)) throw new L3LeaseLostError(job.id); + const status = repos.l3Field.getBatch(batch.id)?.status; + if (status === "applied") return; + // A run that outlived its lease can come back after a newer run of the same job dead-lettered + // the batch, and later batches may have changed the items since: its result is dropped (nothing + // is written) and the batch waits in dead letter for its rerun. + if (status !== "queued") throw new L3WorldModelTerminalEvidenceError(`l3 unit batch is ${status ?? "missing"}; result dropped`); + const at = nowIso(); + const touched = new Set(); + for (const { group, ops } of plan) { + const fragments = new Map(group.fragments.map((fragment) => [fragment.id, fragment])); + for (const op of ops) { + const unit = repos.l3Field.getUnit(op.unitKey); + if (!unit) continue; + const cited = op.evidence.map((id) => fragments.get(id)!); + // The user may have deleted a turn or its L1 while the model was answering: an op that + // cites it is dropped, as it would have been had the deletion come before. + if (cited.some((fragment) => !evidenceStillThere(repos, fragment))) { + logEvolutionDecision(job, "l3_unit_update", "deleted_evidence_op_dropped", { op: op.op }); + continue; + } + const evidenceAt = cited.map((fragment) => fragment.at).sort().at(-1) ?? at; + // A unit reset must not be refilled by what was said before the reset, even when that + // conversation is frozen later (a task closed late, or a batch rerun from dead letter). + if (unit.deletedAt && (batch.createdAt <= unit.deletedAt || evidenceAt <= unit.deletedAt)) continue; + // Older evidence must not overturn an item that newer evidence wrote or confirmed (a batch + // rerun from dead letter is applied after the batches that came after it), and an item + // already gone is not changed again. + if (op.op === "replace" || op.op === "delete") { + const targets = (op.op === "replace" ? op.ids : [op.id]).map((id) => repos.l3Field.getItem(id)); + if (targets.some((item) => !item || item.status !== "active" || item.lastConfirmedAt > evidenceAt)) { + logEvolutionDecision(job, "l3_unit_update", "stale_op_dropped", { op: op.op, evidenceAt }); + continue; + } + } + const evidence = { + fragmentIds: op.evidence, + l1MemoryIds: [...new Set(cited.map((fragment) => fragment.l1MemoryId))] + }; + if (op.op === "confirm") { + repos.l3Field.confirmItem(op.id, evidenceAt, at); + } else if (op.op === "delete") { + repos.l3Field.inactivateItem(op.id, { removedEvidenceAt: evidenceAt, at }); + } else { + const created = repos.l3Field.insertItem({ + unitKey: op.unitKey, kind: op.kind, content: op.content, evidence, evidenceAt, batchId: batch.id, at + }); + if (op.op === "replace") { + for (const id of op.ids) repos.l3Field.inactivateItem(id, { replacedBy: created.id, removedEvidenceAt: evidenceAt, at }); + } + if (countL3Units(op.content) > OVER_LENGTH_UNITS) { + logEvolutionDecision(job, "l3_unit_update", "item_over_length", { itemId: created.id }); + } + } + touched.add(op.unitKey); + } + } + for (const unitKey of touched) this.enforceLimit(job, unitKey, at); + repos.l3Field.setBatchStatus(batch.id, "applied", at); + // Refreshing the machine also refreshes every project snapshot on it. + if (touched.has(batch.machineUnitKey)) repos.l3Field.refreshSnapshot(batch.machineUnitKey, at); + for (const unitKey of touched) { + if (unitKey !== batch.machineUnitKey) repos.l3Field.refreshSnapshot(unitKey, at); + } + }); + } + + /** Over the limit, the items confirmed longest ago go first. */ + private enforceLimit(job: EvolutionJobRecord, unitKey: string, at: string): void { + const items = this.deps.repos.l3Field.activeItems(unitKey); + for (const kind of ["env_fact", "project_constraint"] as L3ItemKind[]) { + const ofKind = items + .filter((item) => item.kind === kind) + .sort((a, b) => a.lastConfirmedAt.localeCompare(b.lastConfirmedAt) || a.createdAt.localeCompare(b.createdAt)); + for (const item of ofKind.slice(0, Math.max(0, ofKind.length - MAX_ITEMS_PER_KIND))) { + this.deps.repos.l3Field.inactivateItem(item.id, { at }); + logEvolutionDecision(job, "l3_unit_update", "item_evicted_over_limit", { itemId: item.id, unitKey }); + } + } + } +} diff --git a/Memory/src/service/l3-field/unit-update-prompt.ts b/Memory/src/service/l3-field/unit-update-prompt.ts new file mode 100644 index 000000000..ebfdd4e51 --- /dev/null +++ b/Memory/src/service/l3-field/unit-update-prompt.ts @@ -0,0 +1,141 @@ +/** System prompt of the new L3 item update; approved draft: memmy-流程页面/L3提示词草稿.md (2026-10-24, polished 2026-10-25). */ +export const L3_UNIT_UPDATE_PROMPT = `You maintain "field cognition" for one computer and the projects on it: short, itemized notes about +what this computer and these projects are like, so that a future Agent session here starts out knowing them. + +The input contains: +- batch_l1: summaries of the turns that just happened, oldest first. Read them to understand what happened. +- evidence: raw fragments of those turns (the user's own words, and each tool call with its output). + Every item you write must cite them. Something that appears only in batch_l1 and not in evidence + cannot be written. +- machine: this computer's profile (read-only) and its current items. +- projects: each project touched by this batch, with its profile (read-only) and current items. +- removed_by_newer_evidence (only sometimes): items another conversation removed or replaced, with when that + evidence happened (removed_at). Do not write their content back on evidence older than removed_at; only + evidence after it can bring it back. +- session: the session's working folder and its current project, if any. +- user_selected_projects: folders the user has chosen as projects in the desktop app (read-only context). + +Field cognition has three parts: the environment profile, env_fact items and project_constraint items. +The profile is produced by code from scanning the computer and the project; it is given to you read-only. +You maintain only the two kinds of items. + +The most important test: write only what would still hold if a different person worked on this computer +or this project tomorrow. Field cognition records how the computer and the project themselves behave, not +what changes with the person working on them (their preferences, habits, current plans or mistakes), and +not what belongs to the Agent doing the work. + +1. env_fact — something an Agent will run into when working here, proven by a tool output or error. + Write it as "condition -> observable result". + Machine example: "the external drive /Volumes/data is mounted read-only -> write outputs elsewhere". + Project example: "git clone over SSH fails for this repo -> use the https remote". + Write a fact only if a future Agent here would act better for knowing it and could not easily find it + out on its own: it differs from what a competent Agent would expect. One occurrence is enough when a + tool output or error shows it. + Do NOT write: + - how a language, library, tool, model or online service behaves wherever it is used (general + knowledge), and results that only confirm normal, expected behavior. Ask: would the same thing + happen on any other computer with the same software and settings? If yes, it is not a fact about + this computer or project; + - limits of the Agent's own runtime: its sandbox, its permission or approval mode, the shell + environment it runs commands in, and how its own tools behave. Another Agent or a person working on + this computer would not meet them. When evidence from different Agents disagrees about the same + behavior, it is one of these: do not write it, and delete an existing item about it; + - what comes from whoever is working rather than from the environment: their own mistakes, choices + and plans; + - temporary states: what one ordinary action would change — something not installed yet, a file + that happened to be missing, the current branch — or what will be different next time; + - what is learned by reading content (the code, a document, a web page) rather than met when working here; + - guessed causes no output shows, advice without an observed result, or what the profile already + covers (versions, installed tools, OS, shell, stack, folder layout). + +2. project_constraint — a rule of this project that the user expects to hold for future tasks here. + Test: "next time someone does a different task in this project, must they still follow it?" + Examples: "Reports follow main on GitHub; pull before editing." "In the review folder, each paper + gets its own subfolder." + These are lasting rules: write them. + - How work in this project is done: how it is organized, what is authoritative, which conventions it + follows (what things are called, what is in or out of scope, which methods are used), and how + results are delivered. + - A requirement the user repeats across different tasks in this project. + Do NOT write: + - what only the current task or its deliverable needs, including the current plan of work; + - preferences of the person, or ways of working that are not specific to this project; + - anything the Agent decided on its own; + - what the work itself produces: its findings, conclusions, explanations or current state, even when + the user stated them. A rule changes only when the user changes it; what the work produces changes + as the work goes on. + If you cannot tell whether a requirement is meant to last, check the user's own words in evidence; + if it is still unclear, do not write it. + +Where an item goes: +- Machine items are only about this computer itself: its system, hardware, settings, and what is + installed on it or reachable from it. A machine item must not mention a specific project, repository + or folder. Machine items are env_fact only. +- A website, service, data source or tool that only some work uses is not part of this computer: its + behavior belongs to the project that uses it. +- project_constraint always goes to a project. An env_fact about a project goes to the project whose + files the evidence actually touched, which may differ from the session folder. +- Each evidence fragment lists its candidate_units; the target unit must be one of them. +- If something belongs neither to this computer nor to a listed project, leave it out; never put it on + the machine instead. + +Operations: +- add: a new item. +- replace: newer evidence changes or corrects existing items. The new content replaces them; list every + replaced id. All replaced ids must be the same unit and kind. Use this also to merge duplicates. +- delete: newer evidence shows an existing item is no longer true, and nothing replaces it. +- confirm: the batch shows an existing item is still true. It only refreshes when the item was last seen. +An empty list is the normal result: output only what changes. When new evidence conflicts with an +existing item, the newer evidence wins: replace or delete the old item. Replace an item only when its +meaning materially changes, never just to reword it; items this batch does not bear on stay as they are. +Two items that would lead an Agent to the same action are duplicates: keep the more precise one. + +Size: +- Each item is one or two sentences, about 60 characters (or 30 Chinese characters) at most. +- Each unit keeps at most 10 env_fact items, and a project at most 10 project_constraint items. + When an add would exceed that, first merge similar items with replace. If code still finds too many, + it drops the ones confirmed longest ago. + +Evidence: every operation carries "evidence", ids of fragments from this input that support it; use only +ids that appear in the input. An env_fact add or replace must cite at least one fragment containing a +tool output or error. + +Before you output, check every add and replace once more: +1. Would it still hold if a different person worked here tomorrow? +2. Is it about this computer or this project in particular — not true everywhere, and not about the + Agent itself? +3. Would it still hold after one ordinary action (installing, creating, switching)? +4. Is it in the right unit? +Drop any op that fails one of these. + +Return exactly one valid JSON object, no Markdown. Give a short "reason" (not stored) first, saying what +in this batch is worth keeping and what you left out: +{"reason":"...","ops":[ ... ]} +where each op is one of: +{"op":"add","unit":"machine","kind":"env_fact","content":"...","evidence":["f..."]} +{"op":"add","unit":"project","unit_key":"","kind":"env_fact"|"project_constraint","content":"...","evidence":["f..."]} +{"op":"replace","ids":["", ...],"content":"...","evidence":["f..."]} +{"op":"delete","id":"","evidence":["f..."]} +{"op":"confirm","id":"","evidence":["f..."]} +"unit_key" may be omitted only when exactly one project is listed. + +Write item content in the language of the existing items. When there are none, follow the interface language when one is pinned; otherwise use the dominant language of the user's own words in evidence. Do not translate content merely because this instruction is written in English.`; + +/** Added to the input when the previous output failed validation; the model regenerates once. */ +export function l3UnitRetryNote(problems: readonly string[]): string { + return [ + "Your previous output was rejected:", + ...problems.map((problem) => `- ${problem}`), + "Return the whole JSON again with these problems fixed. Drop an op if it cannot be fixed." + ].join("\n"); +} + +export const L3_UNIT_UPDATE_SCHEMA = { + reason: "string", + ops: [ + { op: "add", unit: "machine|project", unit_key: "string (optional)", kind: "env_fact|project_constraint", content: "string", evidence: ["fragment id"] }, + { op: "replace", ids: ["item id"], content: "string", evidence: ["fragment id"] }, + { op: "delete", id: "item id", evidence: ["fragment id"] }, + { op: "confirm", id: "item id", evidence: ["fragment id"] } + ] +} as const; diff --git a/Memory/src/service/l3-world-model/strict-json-completion.ts b/Memory/src/service/l3-world-model/strict-json-completion.ts index 3142b2302..bbbd854d5 100644 --- a/Memory/src/service/l3-world-model/strict-json-completion.ts +++ b/Memory/src/service/l3-world-model/strict-json-completion.ts @@ -18,6 +18,8 @@ export interface StrictJsonCompletionInput { dynamicInput: JsonValue; expectedSchema: JsonValue; validate(value: unknown): T; + /** Output budget; defaults to L3_WORLD_MODEL_MAX_OUTPUT_TOKENS. */ + maxTokens?: number; } /** Runs one strict JSON completion and at most one model-based schema repair. */ @@ -26,7 +28,7 @@ export async function completeStrictJson(input: StrictJsonCompletionInput) { role: "system", content: input.systemPrompt }, { role: "user", content: canonicalJson(input.dynamicInput) } ]; - const candidate = await input.llm.complete(messages, completionOptions(input.operation)); + const candidate = await input.llm.complete(messages, completionOptions(input.operation, input.maxTokens)); try { return parseAndValidate(candidate, input.validate); } catch (error) { @@ -41,16 +43,16 @@ export async function completeStrictJson(input: StrictJsonCompletionInput) validation_error: validationErrorMessage(error) }) } - ], completionOptions(`${input.operation}.repair`)); + ], completionOptions(`${input.operation}.repair`, input.maxTokens)); return parseAndValidate(repaired, input.validate); } } -function completionOptions(operation: string) { +function completionOptions(operation: string, maxTokens = L3_WORLD_MODEL_MAX_OUTPUT_TOKENS) { return { operation, temperature: 0, - maxTokens: L3_WORLD_MODEL_MAX_OUTPUT_TOKENS, + maxTokens, jsonMode: true } as const; } diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index 2f012b2a7..b2b177da4 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -1,11 +1,27 @@ import { EmbeddingMaintenance } from "./embedding/embedding-maintenance.js"; import { embeddingRetryTargetKindForMemory, embeddingRetryVectorFieldForMemory } from "./embedding/embedding-pipeline.js"; +import { homedir } from "node:os"; import { assertJsonValue, canonicalJson, + deriveWorkspaceHostId, isLocalWorkspaceUri, - sha256Hex + sha256Hex, + type L3FailedBatchesRetryRequest, + type L3FailedBatchesRetryResponse, + type L3ScanSessionsFreezeRequest, + type L3ScanSessionsFreezeResponse, + type L3UserProjectsReplaceRequest, + type L3UserProjectsReplaceResponse } from "../contracts/index.js"; +import { getOrCreateInstallationId } from "../cli/analytics.js"; +import { isL3FieldCognitionActive } from "./l3-field/flag.js"; +import { loadL3FieldContext } from "./l3-field/l3-field-context.js"; +import type { L3FieldEnvironment } from "./l3-field/session-units.js"; +import { l3FieldSnapshotUnitKey } from "./l3-field/snapshot-memory.js"; +import { resolveSessionUnits } from "./l3-field/session-units.js"; +import { machineProfileFingerprint, probeMachine, renderMachineProfile, type MachineProbe } from "./l3-field/machine-profile.js"; +import { L3UnitUpdatePipeline } from "./l3-field/unit-update-pipeline.js"; import { languageSteeringLine, skillOutputLanguageFor, @@ -232,6 +248,10 @@ export interface MemoryServiceOptions { db?: MemoryDb; backend?: StorageBackend; mode?: "local" | "cloud" | "dev"; + /** Machine identity and home folder for the new L3; defaults to this installation. Tests override it. */ + l3FieldEnvironment?: () => L3FieldEnvironment; + /** Probes this computer for the new L3 machine profile; tests override it. */ + l3MachineProbe?: () => Promise; configPath?: string; configLoader?: (configPath?: string) => { config: MemmyConfig; @@ -299,6 +319,8 @@ export class MemoryService { private readonly importJobs: ImportJobProcessor; private readonly l3WorldModelContextReadModel: L3WorldModelContextReadModel; private readonly projectEnvironment: ProjectEnvironmentService; + private readonly l3UnitUpdate: L3UnitUpdatePipeline; + private readonly l3MachineProfileRefreshedAt = new Map(); private readonly panelReadModel: PanelReadModel; private readonly retrieval: RetrievalService; private readonly sessionTurns: SessionTurnService; @@ -391,7 +413,8 @@ export class MemoryService { get llm() { return serviceOwner.llm; }, get embedder() { return serviceOwner.embedder; }, get embedAfterCapture() { return serviceOwner.config.algorithm.capture.embedAfterCapture; }, - nowIso + nowIso, + l3FieldActive: () => serviceOwner.l3FieldActive() }); const projectEnvironmentOwner = this; this.projectEnvironment = new ProjectEnvironmentService({ @@ -399,6 +422,15 @@ export class MemoryService { get llm() { return projectEnvironmentOwner.skillLlm; }, get language() { return projectEnvironmentOwner.config.language; } }); + this.l3UnitUpdate = new L3UnitUpdatePipeline({ + repos: this.repos, + get llm() { return projectEnvironmentOwner.skillLlm; }, + get language() { return projectEnvironmentOwner.config.language; }, + environment: () => projectEnvironmentOwner.l3FieldEnvironment() + }); + this.repos.l3WorldModels.routeBatchesToFieldCognition( + () => projectEnvironmentOwner.l3FieldActive() ? projectEnvironmentOwner.l3FieldEnvironment().machineId : null + ); const episodeTitleOwner = this; this.episodeTitle = new EpisodeTitleService({ repos: this.repos, @@ -429,6 +461,7 @@ export class MemoryService { abstractL3: (job) => this.evolutionJobs.abstractL3(job), updateL3WorldModel: (job) => this.evolutionJobs.updateL3WorldModel(job), updateProjectEnvironment: (job) => this.projectEnvironment.processProfileJob(job), + updateL3Unit: (job) => this.l3UnitUpdate.process(job), crystallizeSkill: (job) => this.evolutionJobs.crystallizeSkill(job), assignSkillCluster: (job) => this.evolutionJobs.assignSkillCluster(job), evolveSkillCluster: (job) => this.evolutionJobs.evolveSkillCluster(job), @@ -633,6 +666,13 @@ export class MemoryService { runWorkerNoWrite: this.runWorkerNoWrite.bind(this), restartFailedProcessing: this.restartFailedProcessing.bind(this), enqueueObservationBackfill: (limit, at) => this.observationMemory.backfill(limit, at), + l3FieldActive: () => this.l3FieldActive(), + requeueL3DeadLettersOnce: (at) => this.requeueL3DeadLettersOnce(at), + armPendingL3Flushes: () => this.armPendingL3Flushes(), + evolutionModelConfigured: () => workerRunnerOwner.skillLlm.isConfigured(), + purgeL3InactiveItems: (at) => this.repos.l3Field.purgeInactive( + new Date(Date.parse(at) - this.config.algorithm.l3FieldCognition.inactiveRetentionDays * 86_400_000).toISOString() + ), previewPolicyEvidenceReconciliation: this.evolutionJobs.previewPolicyEvidenceReconciliation.bind(this.evolutionJobs), reconcileOrphanedPolicies: this.evolutionJobs.reconcileOrphanedPolicies.bind(this.evolutionJobs), enqueueJob: this.workerHandlers.enqueueJob, @@ -775,6 +815,7 @@ export class MemoryService { maybeCreateDecisionRepair: this.feedbackExperience.maybeCreateDecisionRepair.bind(this.feedbackExperience), memoryAddEnabled: this.memoryAddEnabled.bind(this), memorySearchEnabled: this.memorySearchEnabled.bind(this), + l3FieldActive: () => this.l3FieldActive(), observeToolNoWrite: this.observeToolNoWrite.bind(this), openSessionNoWrite: this.openSessionNoWrite.bind(this), pendingTrialsForFeedback: this.skillTrials.pendingTrialsForFeedback.bind(this.skillTrials), @@ -1103,6 +1144,58 @@ export class MemoryService { this.storageCapabilities().backendId === "sqlite-local"; } + private l3FieldActive(): boolean { + return isL3FieldCognitionActive(this.config, this.storageCapabilities(), this.mode); + } + + private l3FieldEnvironmentCache?: L3FieldEnvironment; + + /** New L3 profiles at session start and compaction: rescan the session's project, re-probe the machine. */ + private requestL3FieldProfiles(session: SessionRecord): void { + if (!this.memoryAddEnabled()) return; + const units = resolveSessionUnits(this.repos, this.l3FieldEnvironment(), session); + if (units.project?.unit.checkoutRoot) { + this.projectEnvironment.requestL3UnitProfile({ + userId: session.userId, + unitKey: units.project.unit.unitKey, + checkoutRoot: units.project.unit.checkoutRoot + }); + } + void this.refreshL3MachineProfile(units.machine.unitKey); + } + + /** At most once per ten minutes per machine; the profile is rewritten only when the probe changed. */ + private async refreshL3MachineProfile(unitKey: string): Promise { + const now = Date.now(); + if (now - (this.l3MachineProfileRefreshedAt.get(unitKey) ?? 0) < 10 * 60_000) return; + this.l3MachineProfileRefreshedAt.set(unitKey, now); + try { + // A reset (snapshot deleted) while probing makes this probe stale; it must not bring the profile back. + const resetBefore = this.repos.l3Field.getUnit(unitKey)?.deletedAt; + const probe = await (this.options.l3MachineProbe ?? probeMachine)(); + // The service may have stopped, or memory writes been turned off, while the probe ran. + if (this.closing || !this.memoryAddEnabled()) return; + const fingerprint = machineProfileFingerprint(probe); + this.repos.transaction(() => { + const unit = this.repos.l3Field.getUnit(unitKey); + if (!unit || unit.deletedAt !== resetBefore || unit.profileFingerprint === fingerprint) return; + this.repos.l3Field.setProfile(unitKey, { text: renderMachineProfile(probe), fingerprint }); + this.repos.l3Field.refreshSnapshot(unitKey); + }); + } catch { + // A failed probe leaves the old profile; the next session start tries again. + this.l3MachineProfileRefreshedAt.delete(unitKey); + } + } + + private l3FieldEnvironment(): L3FieldEnvironment { + this.l3FieldEnvironmentCache ??= this.options.l3FieldEnvironment?.() ?? { + machineId: deriveWorkspaceHostId(getOrCreateInstallationId({ env: process.env })), + homeDir: homedir() + }; + return this.l3FieldEnvironmentCache; + } + private memorySearchEnabled(): boolean { return this.config.algorithm.enableMemorySearch; } @@ -1249,6 +1342,7 @@ export class MemoryService { const changed = stableStringify(previousConfig) !== stableStringify(nextConfig); const requiresRestart = stableStringify(previousConfig.storage) !== stableStringify(nextConfig.storage); const reloadedAt = nowIso(); + const l3FieldWasActive = this.l3FieldActive(); this.config = nextConfig; this.tokenBudgetLedger.setLimits(this.config.tokenBudget); @@ -1261,6 +1355,8 @@ export class MemoryService { for (const { before, after } of this.repos.runtime.requeueFailedEpisodeAssignments(reloadedAt)) this.workerHandlers.appendJobChange(after, "queued", before); this.capturePendingListener?.(); } + // Turned on without a restart: flushes that ran while it was off did not freeze, so hand those turns over now. + if (!l3FieldWasActive) this.armPendingL3Flushes(); this.workerRunner.notifyWorkAvailable(); serviceLogger.info("config.reloaded", { changed, @@ -1433,7 +1529,10 @@ export class MemoryService { serverTime: string; } { const response = this.sessionTurns.openSession(this.withTimeZone(request)); - if (this.projectEnvironmentScanEnabled() && response.projectId) { + if (this.l3FieldActive()) { + // With memory writes off the session is not stored, and profiles are writes anyway. + if (this.memoryAddEnabled()) this.requestL3FieldProfiles(this.requireSession(response.sessionId)); + } else if (this.projectEnvironmentScanEnabled() && response.projectId) { const session = this.requireSession(response.sessionId); const scope = this.repos.l3WorldModels.getScope(session.userId, response.projectId); if (scope?.workspaceUri && isLocalWorkspaceUri(scope.workspaceUri)) { @@ -1476,6 +1575,94 @@ export class MemoryService { return this.repos.l3WorldModels.traceHead(sessionId); } + /** Stores the desktop app's selected project folders; kept regardless of the L3 field cognition flag. */ + replaceL3UserProjects(request: L3UserProjectsReplaceRequest): L3UserProjectsReplaceResponse { + const userId = normalizeNamespace(request.namespace).userId; + const at = nowIso(); + this.repos.transaction(() => { + this.repos.l3Field.replaceUserProjects(userId, request.projects, at); + this.repos.l3Field.setStandaloneRoots(userId, request.standaloneRoots ?? [], at); + }); + return { count: request.projects.length, serverTime: at }; + } + + /** + * End of an agent-source scan pass: freeze what the pass added to each touched Session into new-L3 + * batches, through the same entry Hook Session boundaries use. Only database rows are written (no + * model call), so the scan never waits on L3; a pass with nothing new freezes nothing. With the new + * L3 off, scans stay out of L3 as before. + */ + freezeScannedL3Sessions(sessionIds: readonly string[]): number { + if (!this.memoryAddEnabled() || !this.l3FieldActive()) return 0; + let batches = 0; + for (const sessionId of new Set(sessionIds)) { + const session = this.repos.runtime.getSession(sessionId); + if (!session || session.meta.l3_world_model_protocol_version !== 2) continue; + batches += this.repos.l3WorldModels.freezeBatches({ sessionId, trigger: "scan_pass" }).batchIds.length; + } + return batches; + } + + /** + * A scan or hosted import finished: freeze the scanned Sessions it touched. Capturing a scanned turn + * keeps the conversation's earlier owner even after an account switch, so the same rule applies + * here: the caller's own Sessions, and scanned Sessions of the caller's source. + */ + freezeScanSessions(request: L3ScanSessionsFreezeRequest): L3ScanSessionsFreezeResponse { + const namespace = normalizeNamespace(request.namespace); + const owned = request.sessionIds.filter((id) => { + const session = this.repos.runtime.getSession(id); + if (!session) return false; + return session.userId === namespace.userId || + (session.meta.work_memory_disabled === true && session.source === namespace.source); + }); + return { batches: this.freezeScannedL3Sessions(owned), serverTime: nowIso() }; + } + + /** Rerun entry for the new L3 only; dead letters of the old pipeline are not rerun. */ + retryFailedL3Batches(request: L3FailedBatchesRetryRequest): L3FailedBatchesRetryResponse { + this.assertMemoryAddEnabled(); + const userId = normalizeNamespace(request.namespace).userId; + const at = nowIso(); + const before = new Map(this.repos.runtime.listJobs("dead_letter", 10_000, userId).map((job) => [job.id, job])); + const requeued = this.repos.l3Field.requeueDeadLetterBatches(userId, at); + for (const id of requeued) { + const after = this.repos.runtime.getJob(id); + if (after) this.workerHandlers.appendJobChange(after, "queued", before.get(id)); + } + return { requeued: requeued.length, serverTime: at }; + } + + /** + * Worker startup (and after an import): turns not yet handed to the new L3 get someone responsible + * for them, also when the process stopped before a scan's hand-off or the data came from a backup. + * An open Session gets its idle flush armed; a closed one, which will not close again, is frozen + * now. Nothing while the new L3 is off. + */ + armPendingL3Flushes(): number { + if (!this.l3FieldActive() || !this.memoryAddEnabled()) return 0; + const pending = this.repos.l3WorldModels.sessionsWithUnfrozenTurns(); + this.repos.transaction(() => { + for (const { sessionId, open, lastActivityAt } of pending) { + if (open) this.workMemory.armIdleFlush(sessionId, lastActivityAt); + else this.repos.l3WorldModels.freezeBatches({ sessionId, trigger: "session_close" }); + } + }); + return pending.length; + } + + /** Worker startup: one automatic rerun for each dead-lettered new-L3 batch; nothing while the new L3 is off. */ + private requeueL3DeadLettersOnce(at: string): number { + if (!this.l3FieldActive()) return 0; + const before = new Map(this.repos.runtime.listJobs("dead_letter", 10_000).map((job) => [job.id, job])); + const requeued = this.repos.l3Field.requeueDeadLetterBatchesOnce(at); + for (const id of requeued) { + const after = this.repos.runtime.getJob(id); + if (after) this.workerHandlers.appendJobChange(after, "queued", before.get(id)); + } + return requeued.length; + } + l3WorldModelBoundary( sessionId: string, request: L3WorldModelBoundaryRequest @@ -1498,7 +1685,9 @@ export class MemoryService { if (!result.throughTraceSeq) { throw new MemoryServiceError("conflict", "through L1 memory was not registered"); } - if ( + if (request.trigger === "token_compaction" && this.l3FieldActive()) { + this.requestL3FieldProfiles(session); + } else if ( request.trigger === "token_compaction" && this.projectEnvironmentScanEnabled() && session.projectId @@ -1528,6 +1717,9 @@ export class MemoryService { if (session.status !== "open") { throw new MemoryServiceError("conflict", "l3_world_model_session_not_open"); } + if (this.l3FieldActive()) { + return loadL3FieldContext(this.repos, this.l3FieldEnvironment(), session, { readOnly: !this.memoryAddEnabled() }); + } return this.l3WorldModelContextReadModel.load(session); } @@ -2596,6 +2788,9 @@ export class MemoryService { conflictStrategy: request.conflictStrategy ?? "skip" }); this.sessionTurns.migrateLegacyCaptures(); + // Imported Sessions with turns not yet handed to the new L3 (including work the import had to + // drop for a unit kept here) get their idle flush armed. + this.armPendingL3Flushes(); const importedVectors = importedMemoryVectors(request.bundle.tables.memory_vectors); this.repos.vectors.importRows(importedVectors); result.inserted.memory_vectors = importedVectors.length; @@ -2652,7 +2847,14 @@ export class MemoryService { this.assertMemoryInScope(memory, request.namespace); const kind = kindFromMemory(memory); const at = nowIso(); - const archived = this.repos.memories.archive(memory.id, at); + const archived = this.repos.transaction(() => { + const row = this.repos.memories.archive(memory.id, at); + // A project's session context includes the machine, so archiving the machine's new-L3 + // snapshot moves the project versions too, or clients keep using their cached context. + const l3Unit = row ? l3FieldSnapshotUnitKey(memory) : null; + if (l3Unit) this.repos.l3Field.refreshProjectContexts(l3Unit, at); + return row; + }); if (!archived) { throw new MemoryServiceError("not_found", `memory not found: ${id}`); } @@ -2777,9 +2979,16 @@ export class MemoryService { } const kind = kindFromMemory(memory); const at = nowIso(); - const deleted = strictV2WorldModel - ? this.repos.l3WorldModels.deleteScopeMemory(memory.id, at)?.deleted - : this.repos.memories.softDelete(memory.id, at); + const deletedL3Unit = l3FieldSnapshotUnitKey(memory); + const deleted = this.repos.transaction(() => { + const removed = strictV2WorldModel + ? this.repos.l3WorldModels.deleteScopeMemory(memory.id, at)?.deleted + : this.repos.memories.softDelete(memory.id, at); + // Deleting a new-L3 snapshot resets its unit: what it knew goes, later sessions start it afresh. + // Same transaction, so a failure never leaves the unit's items behind a deleted snapshot. + if (removed && deletedL3Unit) this.repos.l3Field.markUnitDeleted(deletedL3Unit, at); + return removed; + }); if (!deleted) { throw new MemoryServiceError("not_found", `memory not found: ${id}`); } diff --git a/Memory/src/service/project-environment/local-scanner.ts b/Memory/src/service/project-environment/local-scanner.ts index 36e9c10bf..0f145d134 100644 --- a/Memory/src/service/project-environment/local-scanner.ts +++ b/Memory/src/service/project-environment/local-scanner.ts @@ -54,7 +54,11 @@ interface InventorySnapshot { omittedCount: number; } -export async function scanLocalProject(workspaceUri: WorkspaceUri): Promise { +/** `withRuntimeProbes: false` skips running the declared runtimes (the new L3 keeps them in the machine profile). */ +export async function scanLocalProject( + workspaceUri: WorkspaceUri, + options: { withRuntimeProbes?: boolean } = {} +): Promise { const root = await resolveLocalWorkspaceRoot(workspaceUri); let stable = await scanInventory(root); let next = await scanInventory(root); @@ -70,7 +74,7 @@ export async function scanLocalProject(workspaceUri: WorkspaceUri): Promise; try { - output = await completeStrictJson({ - llm: this.deps.llm, - operation: derived.projectKind === "code" - ? "project_environment_code_profile" - : "project_environment_folder_profile", - systemPrompt, - dynamicInput: profileDynamicInput(derived, currentProfile), - expectedSchema: { - op: "noop | create | update", - profile: "complete final profile; empty only for noop or update-clear" - }, - validate: (value) => validateProjectEnvironmentProfileOutput( - value, - currentProfile, - evidenceSupportsProfile - ) - }); + output = await summarizeProjectProfile(this.deps, derived, currentProfile); } catch (error) { const latest = this.deps.repos.projectEnvironments.getState(payload.userId, payload.projectId); const latestProfile = this.deps.repos.l3WorldModels.fields( @@ -131,6 +109,37 @@ export class ProjectEnvironmentProfilePipeline { } } +/** One model call that turns scan evidence into the project environment profile (shared by old and new L3). */ +export async function summarizeProjectProfile( + deps: { llm: LlmClient; language?: MemoryLanguage }, + derived: ProjectEnvironmentDerivedEvidence, + currentProfile: string | null +): Promise<{ op: "noop" | "create" | "update"; profile: string }> { + const evidenceSupportsProfile = projectEnvironmentEvidenceSupportsProfile(derived); + const basePrompt = derived.projectKind === "code" ? CODE_PROFILE_PROMPT : FOLDER_PROFILE_PROMPT; + const language = currentProfile?.trim() + ? undefined + : steeredPromptLanguage(deps.language, [derived.compactFileTree]); + const systemPrompt = language ? `${basePrompt}\n\n${languageSteeringLine(language)}` : basePrompt; + return completeStrictJson({ + llm: deps.llm, + operation: derived.projectKind === "code" + ? "project_environment_code_profile" + : "project_environment_folder_profile", + systemPrompt, + dynamicInput: profileDynamicInput(derived, currentProfile), + expectedSchema: { + op: "noop | create | update", + profile: "complete final profile; empty only for noop or update-clear" + }, + validate: (value) => validateProjectEnvironmentProfileOutput( + value, + currentProfile, + evidenceSupportsProfile + ) + }); +} + export function projectEnvironmentProfileJobPayload(value: Record): { userId: string; projectId: string; diff --git a/Memory/src/service/project-environment/project-environment-service.ts b/Memory/src/service/project-environment/project-environment-service.ts index b4ff673a5..800d897de 100644 --- a/Memory/src/service/project-environment/project-environment-service.ts +++ b/Memory/src/service/project-environment/project-environment-service.ts @@ -1,3 +1,4 @@ +import { pathToFileURL } from "node:url"; import type { MemoryLanguage } from "../../config/index.js"; import type { LlmClient } from "../../model/types.js"; import type { @@ -5,9 +6,15 @@ import type { Repositories, SessionRecord } from "../../storage/repositories.js"; +import { newId } from "../../utils/id.js"; +import { nowIso } from "../../utils/time.js"; import { scanLocalProject } from "./local-scanner.js"; import { parseDeterministicProjectFacts } from "./manifest-parsers.js"; -import { ProjectEnvironmentProfilePipeline, projectEnvironmentProfileJobPayload } from "./profile-pipeline.js"; +import { + ProjectEnvironmentProfilePipeline, + projectEnvironmentProfileJobPayload, + summarizeProjectProfile +} from "./profile-pipeline.js"; import { classifyProjectInventory } from "./project-classifier.js"; import { buildCompactFileTree, projectFingerprint } from "./scan-policy.js"; import type { ProjectEnvironmentDerivedEvidence } from "./types.js"; @@ -18,6 +25,10 @@ interface ProjectEnvironmentServiceDeps { readonly language?: MemoryLanguage; } +const L3_FIELD_MODE = "l3_field"; +/** A project is scanned for its new-L3 profile at most this often; each session open asks for one. */ +const L3_UNIT_PROFILE_INTERVAL_MS = 10 * 60_000; + export class ProjectEnvironmentService { private readonly profilePipeline: ProjectEnvironmentProfilePipeline; @@ -61,32 +72,58 @@ export class ProjectEnvironmentService { }); } + /** + * New L3: queue a profile scan of one checkout of a project unit. At most one + * such job is pending per unit, and a unit scanned in the last ten minutes is + * not scanned again; such a request is a no-op. + */ + requestL3UnitProfile(input: { userId: string; unitKey: string; checkoutRoot: string; at?: string }): boolean { + const at = input.at ?? nowIso(); + const recent = this.deps.repos.db.prepare( + `SELECT 1 FROM evolution_jobs + WHERE job_type = 'project_environment_profile' + AND json_extract(payload_json, '$.mode') = ? + AND json_extract(payload_json, '$.unitKey') = ? + AND (status IN ('queued', 'leased', 'failed') OR (status = 'succeeded' AND updated_at >= ?)) + LIMIT 1` + ).get(L3_FIELD_MODE, input.unitKey, new Date(Date.parse(at) - L3_UNIT_PROFILE_INTERVAL_MS).toISOString()); + if (recent) return false; + const scanId = newId("l3scan"); + // The immutable dedupe index never frees a key, so each request carries its own scan id. + const dedupeKey = ["project_environment_profile", L3_FIELD_MODE, input.unitKey, scanId].join(":"); + this.deps.repos.l3WorldModels.insertImmutableJob({ + id: newId("job"), + jobType: "project_environment_profile", + status: "queued", + dedupeKey, + userId: input.userId, + payload: { + mode: L3_FIELD_MODE, + userId: input.userId, + unitKey: input.unitKey, + checkoutRoot: input.checkoutRoot, + scanId + }, + attempts: 0, + maxAttempts: 3, + createdAt: at, + updatedAt: at + }); + return true; + } + async processProfileJob(job: EvolutionJobRecord): Promise { + if (job.payload.mode === L3_FIELD_MODE) { + await this.processL3UnitProfileJob(job); + return; + } const payload = projectEnvironmentProfileJobPayload(job.payload); if (job.userId !== payload.userId) throw new Error("project_environment_job_owner_mismatch"); if (!this.deps.repos.projectEnvironments.beginScan(payload.userId, payload.projectId, payload.scanId)) return; try { const scope = this.deps.repos.l3WorldModels.getScope(payload.userId, payload.projectId); if (!scope?.workspaceUri) throw new Error("project_environment_workspace_uri_missing"); - const scan = await scanLocalProject(scope.workspaceUri); - const classification = classifyProjectInventory(scan.entries); - const facts = parseDeterministicProjectFacts({ - entries: scan.entries, - textFiles: scan.textFiles, - runtimeProbes: scan.runtimeProbes - }); - const derived: ProjectEnvironmentDerivedEvidence = { - projectKind: classification.kind, - compactFileTree: buildCompactFileTree(scan.entries), - omittedCount: scan.omittedCount, - deterministicFacts: facts, - fingerprint: projectFingerprint({ - kind: classification.kind, - entries: scan.entries, - omittedCount: scan.omittedCount, - deterministicFacts: facts - }) - }; + const derived = await deriveScanEvidence(scope.workspaceUri); const state = this.deps.repos.projectEnvironments.getState(payload.userId, payload.projectId); if (!state || state.currentScanId !== payload.scanId) return; const currentProfile = this.deps.repos.l3WorldModels.fields( @@ -123,4 +160,66 @@ export class ProjectEnvironmentService { throw error; } } + + /** + * New L3 project profile. Only what the project itself declares goes in: + * runtime versions probed on this computer belong to the machine profile. + * The result is written only if the unit still exists and nobody changed + * its profile while the model was running. + */ + private async processL3UnitProfileJob(job: EvolutionJobRecord): Promise { + const userId = typeof job.payload.userId === "string" ? job.payload.userId : ""; + const unitKey = typeof job.payload.unitKey === "string" ? job.payload.unitKey : ""; + const checkoutRoot = typeof job.payload.checkoutRoot === "string" ? job.payload.checkoutRoot : ""; + if (!userId || !unitKey || !checkoutRoot || job.userId !== userId) { + throw new Error("invalid l3 field profile job payload"); + } + const unit = this.deps.repos.l3Field.getUnit(unitKey); + if (!unit || unit.userId !== userId || unit.kind !== "project") return; + // A job queued before the unit was reset (deleted) must not bring old results back. + if (unit.deletedAt && job.createdAt <= unit.deletedAt) return; + // Runtime versions probed on this machine belong to the machine profile, so they are neither + // shown to the model nor part of the fingerprint: upgrading node does not re-profile every project. + const derived = await deriveScanEvidence(pathToFileURL(checkoutRoot).href, { withRuntimeProbes: false }); + // Unchanged since the last summary, including one that found nothing worth a profile. + if (unit.profileFingerprint === derived.fingerprint) return; + const currentProfile = unit.profileText ?? null; + const output = await summarizeProjectProfile(this.deps, derived, currentProfile); + const latest = this.deps.repos.l3Field.getUnit(unitKey); + if ( + !latest || + (latest.deletedAt && job.createdAt <= latest.deletedAt) || + (latest.profileText ?? null) !== currentProfile + ) return; + const nextProfile = output.op === "noop" ? currentProfile : (output.profile || null); + this.deps.repos.transaction(() => { + this.deps.repos.l3Field.setProfile(unitKey, { text: nextProfile, fingerprint: derived.fingerprint }); + if (nextProfile !== currentProfile) this.deps.repos.l3Field.refreshSnapshot(unitKey); + }); + } +} + +async function deriveScanEvidence( + workspaceUri: string, + options: { withRuntimeProbes?: boolean } = {} +): Promise { + const scan = await scanLocalProject(workspaceUri, options); + const classification = classifyProjectInventory(scan.entries); + const facts = parseDeterministicProjectFacts({ + entries: scan.entries, + textFiles: scan.textFiles, + runtimeProbes: scan.runtimeProbes + }); + return { + projectKind: classification.kind, + compactFileTree: buildCompactFileTree(scan.entries), + omittedCount: scan.omittedCount, + deterministicFacts: facts, + fingerprint: projectFingerprint({ + kind: classification.kind, + entries: scan.entries, + omittedCount: scan.omittedCount, + deterministicFacts: facts + }) + }; } diff --git a/Memory/src/service/session/session-turn-service.ts b/Memory/src/service/session/session-turn-service.ts index bc96e978a..df43d88f0 100644 --- a/Memory/src/service/session/session-turn-service.ts +++ b/Memory/src/service/session/session-turn-service.ts @@ -1919,7 +1919,11 @@ export class SessionTurnService { const mapped = this.deps.repos.runtime.getSession(mappedSessionId); if (!mapped) return { status: "pending", reason: "source_session_missing" }; if (!inScope(mapped)) throw new MemoryServiceError("forbidden", "source_session_scope_conflict"); - if (mapped.status === "open") return mapped; + if (mapped.status === "open") { + const live = this.hookTakeoverOfScannedSession(request, mapped, inScope); + if (live) return live; + return mapped; + } if (!canStartAfter(mapped)) return { status: "pending", reason: "source_session_closed" }; } if (request.sessionId) { @@ -1946,9 +1950,37 @@ export class SessionTurnService { meta: { conversationId: identity.conversationId, source_namespace_key: scope.namespaceKey }, timeZone: request.timeZone }, { createNew: true, at: identity.startedAt }); + if (request.channel === "agent_source_scan" && this.deps.l3FieldActive()) { + // With the new L3 on, a newly scanned conversation feeds it like a Hook Session, but without + // Work Memory: a scan replays finished history, so there is no live task to extract working + // requirements from. With it off, scans stay out of L3 as before. Sessions opened earlier + // keep their protocol. + this.deps.repos.runtime.updateSessionMeta(opened.sessionId, { + l3_world_model_protocol_version: 2, + work_memory_disabled: true + }, identity.startedAt); + } return this.deps.repos.runtime.getSession(opened.sessionId)!; } + /** + * A conversation first seen by a scan, then continued live under a Hook: the Hook's own Session + * takes over, so its turns, compaction, close and Work Memory all act on one Session. The scan's + * Session is closed, which freezes what it still holds into L3. + */ + private hookTakeoverOfScannedSession( + request: SourceTurnCompleteRequest, + mapped: SessionRecord, + inScope: (candidate: SessionRecord) => boolean + ): SessionRecord | undefined { + if (request.channel !== "hook" || !request.sessionId || request.sessionId === mapped.id) return undefined; + if (mapped.meta.work_memory_disabled !== true) return undefined; + const live = this.deps.repos.runtime.getSession(request.sessionId); + if (!live || live.status !== "open" || !inScope(live)) return undefined; + this.closeSession(mapped.id); + return live; + } + completeTurn( turnId: string, request: TurnCompleteRequest & Record, diff --git a/Memory/src/service/work-memory/work-memory-pipeline.ts b/Memory/src/service/work-memory/work-memory-pipeline.ts index 8382f10a6..29cd15ed4 100644 --- a/Memory/src/service/work-memory/work-memory-pipeline.ts +++ b/Memory/src/service/work-memory/work-memory-pipeline.ts @@ -56,6 +56,8 @@ export interface WorkMemoryPipelineDependencies { embedder: Embedder; embedAfterCapture: boolean; nowIso: () => string; + /** With the new L3 on, the idle flush also hands a quiet Session's turns to L3. */ + l3FieldActive?: () => boolean; } const WORK_MEMORY_SCHEMA_VERSION = 1; @@ -112,7 +114,7 @@ export class WorkMemoryPipeline { at = this.deps.nowIso() ): WorkMemoryExtractionResult { const session = this.deps.repos.runtime.getSession(sessionId); - if (!session || session.meta.l3_world_model_protocol_version !== 2) { + if (!session || session.meta.l3_world_model_protocol_version !== 2 || session.meta.work_memory_disabled === true) { return { windows: 0, enqueued: 0, cursorAdvancedTo: 0 }; } // The first Work Memory pass of a Session starts from a clean slate: the L3 @@ -197,6 +199,8 @@ export class WorkMemoryPipeline { ): EvolutionJobRecord | undefined { const session = this.deps.repos.runtime.getSession(sessionId); if (!session || session.meta.l3_world_model_protocol_version !== 2) return undefined; + // A scanned Session has no Work Memory, but with the new L3 on the same flush hands its turns to L3. + if (session.meta.work_memory_disabled === true && this.deps.l3FieldActive?.() !== true) return undefined; return this.deps.repos.runtime.armWorkMemoryIdleFlush({ sessionId, userId: session.userId, @@ -218,9 +222,10 @@ export class WorkMemoryPipeline { if (!sessionId) throw new Error(`work memory idle flush job has no session: ${job.id}`); const empty: WorkMemoryExtractionResult = { windows: 0, enqueued: 0, cursorAdvancedTo: 0 }; const session = this.deps.repos.runtime.getSession(sessionId); - if (!session || session.status !== "open" || session.meta.l3_world_model_protocol_version !== 2) { - return empty; - } + if (!session || session.status !== "open" || session.meta.l3_world_model_protocol_version !== 2) return empty; + const workMemory = session.meta.work_memory_disabled !== true; + const l3Field = this.deps.l3FieldActive?.() === true; + if (!workMemory && !l3Field) return empty; const at = this.deps.nowIso(); const lastActivityAt = this.deps.repos.l3WorldModels.latestInputTraceCreatedAt(sessionId) ?? (typeof job.payload.lastActivityAt === "string" ? job.payload.lastActivityAt : at); @@ -231,7 +236,12 @@ export class WorkMemoryPipeline { return empty; } const throughTraceSeq = this.deps.repos.l3WorldModels.maxInputTraceSeq(sessionId); - return this.deps.repos.transaction(() => this.extractUnextracted(sessionId, throughTraceSeq, at)); + return this.deps.repos.transaction(() => { + // Every captured turn ends up here once its Session goes quiet, so a conversation that just + // stops (no close, no later turn) still reaches the new L3; what is already frozen is skipped. + if (l3Field) this.deps.repos.l3WorldModels.freezeBatches({ sessionId, trigger: "session_idle", at }); + return workMemory ? this.extractUnextracted(sessionId, throughTraceSeq, at) : empty; + }); } async extract(job: EvolutionJobRecord): Promise { diff --git a/Memory/src/service/worker/fair-worker-queue.ts b/Memory/src/service/worker/fair-worker-queue.ts index 7a9fbd616..011846e4d 100644 --- a/Memory/src/service/worker/fair-worker-queue.ts +++ b/Memory/src/service/worker/fair-worker-queue.ts @@ -64,12 +64,20 @@ export function releaseFairWorkerResources(scan: FairWorkerScan, resources: read if (lanes.has("interactive")) scan.continuation.interactiveDenied = false; } -function dependencyKey(job: EvolutionJobRecord): string | undefined { +/** Job types that run one at a time, in order, per scope_key (see scopedJobReadySql). */ +const SCOPE_ORDERED_JOB_TYPES = ["l3_world_model_update", "l3_unit_update", "work_memory_extract"]; + +function dependencyKey(job: EvolutionJobRecord, repos: Pick): string | undefined { if (job.sessionId && (["episode_assignment", "episode_finalize", "feedback_experience"].includes(job.jobType) || (job.jobType === "trace_summary" && job.payload.turnJudge === true))) { return JSON.stringify(["dependency", "session", job.sessionId]); } - if (["l3_world_model_update", "work_memory_extract"].includes(job.jobType) && job.scopeKey) { + // A new-L3 batch waits for its L1 summaries first: their completion releases it, then its scope order. + if (job.jobType === "l3_unit_update" && typeof job.payload.batchId === "string") { + const l1MemoryId = repos.runtime.unsummarizedL3BatchL1(job.payload.batchId); + if (l1MemoryId) return JSON.stringify(["dependency", "memory", l1MemoryId]); + } + if (SCOPE_ORDERED_JOB_TYPES.includes(job.jobType) && job.scopeKey) { return JSON.stringify(["dependency", "scope", job.jobType, job.scopeKey]); } if (job.jobType === "reward" && typeof job.payload.l1MemoryId === "string") { @@ -82,8 +90,8 @@ function dependencyKey(job: EvolutionJobRecord): string | undefined { export function releaseFairWorkerDependencies(scan: FairWorkerScan, work: FairClaimedWork): void { const keys: string[] = []; if (work.queue === "main") { - if (["l3_world_model_update", "work_memory_extract"].includes(work.job.jobType)) { - const key = dependencyKey(work.job); if (key) keys.push(key); + if (SCOPE_ORDERED_JOB_TYPES.includes(work.job.jobType) && work.job.scopeKey) { + keys.push(JSON.stringify(["dependency", "scope", work.job.jobType, work.job.scopeKey])); } if (work.job.sessionId) keys.push(JSON.stringify(["dependency", "session", work.job.sessionId])); if (work.job.targetMemoryId) keys.push(JSON.stringify(["dependency", "memory", work.job.targetMemoryId])); @@ -346,7 +354,7 @@ function claimAdmittedWorkerBatch( const eligible = !blocked && repos.runtime.scheduling.isEligibleCandidate(chosen, input.at, request); if (!blocked && !eligible && scan && chosen.queue === "main") { const job = repos.runtime.getJob(chosen.id); - const key = job && dependencyKey(job); + const key = job && dependencyKey(job, repos); if (key && repos.runtime.scheduling.isPendingCandidate(chosen)) { scan.waiting.park(chosen, key); admissionBlocked = true; } diff --git a/Memory/src/service/worker/job-handlers.ts b/Memory/src/service/worker/job-handlers.ts index 5f82a5b0a..a498fd83c 100644 --- a/Memory/src/service/worker/job-handlers.ts +++ b/Memory/src/service/worker/job-handlers.ts @@ -25,6 +25,7 @@ import { import { episodeTitleIsCurrent } from "../episode-title/episode-title-service.js"; import { memoryHasImportPipeline } from "../import/import-job-processor.js"; import { observationMemoryState } from "../observation-memory-state.js"; +import { isTerminalL3WorldModelError } from "../evolution/l3-world-model-pipeline.js"; import { isNeutralFeedback, turnJudgeJobFlags } from "../evolution/turn-judge.js"; import { namespaceForMemory, @@ -68,6 +69,7 @@ export interface WorkerJobProcessors { abstractL3(job: EvolutionJobRecord): MaybePromise; updateL3WorldModel(job: EvolutionJobRecord): MaybePromise; updateProjectEnvironment(job: EvolutionJobRecord): MaybePromise; + updateL3Unit(job: EvolutionJobRecord): MaybePromise; crystallizeSkill(job: EvolutionJobRecord): MaybePromise; assignSkillCluster(job: EvolutionJobRecord): MaybePromise; evolveSkillCluster(job: EvolutionJobRecord): MaybePromise; @@ -96,7 +98,7 @@ export interface WorkerJobProcessors { } export interface WorkerJobHandlerDeps { - repos: Pick; + repos: Pick; capture: { synthReflection: boolean }; reward: { feedbackWindowSec: number }; nowIso(): string; @@ -270,6 +272,10 @@ export async function processJob( case "project_environment_profile": await deps.processors.evolution.updateProjectEnvironment(job); return; + case "l3_unit_update": + // A terminal error goes straight to dead letter in failLeasedWorkerJob. + await deps.processors.evolution.updateL3Unit(job); + return; case "skill_crystallization": await deps.processors.evolution.crystallizeSkill(job); return; @@ -381,7 +387,15 @@ export function finalizeClosedEpisode( trigger: ClosedEpisodeTrigger ): EvolutionJobRecord[] { const current = deps.repos.runtime.getEpisode(episode.id) ?? episode; - if (current.status !== "closed" || current.l1MemoryIds.length === 0) return []; + if (current.status !== "closed") return []; + // Every way a task closes ends up here, so this is where its turns go to the new L3. + deps.repos.l3WorldModels.freezeClosedTask({ + sessionId: current.sessionId, + episodeId: current.id, + trigger: trigger === "idle_timeout" ? "episode_idle_close" : trigger === "session_closed" ? "session_close" : "new_task", + at + }); + if (current.l1MemoryIds.length === 0) return []; if (episodeHasPendingCaptureDecision(deps, current) || deps.repos.runtime.hasUnassignedTurns(current.sessionId)) return []; // Titling is independent of reward and reflection, so it must be queued before // the mutually exclusive branches below can return. @@ -568,6 +582,7 @@ export function workerJobCanRunInParallel(job: EvolutionJobRecord): boolean { job.jobType === "observation_extract" || job.jobType === "embedding" || job.jobType === "l3_world_model_update" || + job.jobType === "l3_unit_update" || job.jobType === "project_environment_profile"; } @@ -657,6 +672,60 @@ export function classifyProcessingError(error: unknown): { return { code: "processing_failed", retryAction: "retry" }; } +/** L3 jobs whose failures are retried later instead of right away. */ +export const L3_RETRY_JOB_TYPES: readonly string[] = ["l3_world_model_update", "l3_unit_update", "project_environment_profile"]; +/** Waits before retries 1-5 of a temporary failure; the sixth run that fails goes to dead letter. */ +export const L3_RETRY_DELAYS_MINUTES = [1, 5, 15, 30, 60] as const; +/** Used-up quota (40309) or no connection is tried again this often, without using up an attempt. */ +export const L3_SUSPEND_MINUTES = 30; + +export type L3FailureHandling = + | { kind: "retry_later"; delayMinutes: number; maxAttempts: number } + | { kind: "suspend" } + | { kind: "dead_letter" } + | { kind: "default" }; + +/** Connection errors that mean the model endpoint cannot be reached at all, as when the computer is offline. */ +const CANNOT_CONNECT_CODES = new Set([ + "ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED", "ENETUNREACH", "EHOSTUNREACH", "ENETDOWN", "UND_ERR_CONNECT_TIMEOUT" +]); + +/** + * How a failed L3 job goes back into the queue. Used-up quota or no connection waits without using + * an attempt, so going offline loses nothing; any other configuration problem (a bad key or endpoint) + * is left to the queue, which holds the job until the configuration changes. Temporary model trouble + * retries five times; any other failure (a rejected answer, a bug) would most likely fail the same + * way, so it gets one more try. Then the job goes to dead letter, never to "failed": a failed L3 + * job left in the queue would hold back every later batch of the same scope until a restart. + */ +export function l3FailureHandling(job: EvolutionJobRecord, error: unknown): L3FailureHandling { + if (!L3_RETRY_JOB_TYPES.includes(job.jobType) || isTerminalL3WorldModelError(error)) return { kind: "default" }; + if (cannotConnect(error)) return { kind: "suspend" }; + const message = error instanceof Error ? error.message : String(error); + // What the model clients really throw (model/http.ts, model/llm.ts): an empty body or missing + // output, and fetch's TypeError or the timeout's AbortError. Checked before configuration errors, + // because an empty reply also says "expected JSON". + const emptyReply = /received an empty response|response missing |unexpected end of json input/i + .test(message); + const network = error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError" || + (error instanceof TypeError && /fetch failed|network|socket/i.test(message))); + const classification = classifyProcessingError(error); + if (!emptyReply && !network && classification.retryAction === "open_settings") { + return classification.code === "40309" ? { kind: "suspend" } : { kind: "default" }; + } + const temporary = emptyReply || network || classification.code === "transient_provider_error"; + const delay = (temporary ? L3_RETRY_DELAYS_MINUTES : L3_RETRY_DELAYS_MINUTES.slice(0, 1))[job.failureCount ?? 0]; + if (delay === undefined) return { kind: "dead_letter" }; + return { kind: "retry_later", delayMinutes: delay, maxAttempts: L3_RETRY_DELAYS_MINUTES.length + 1 }; +} + +/** fetch's "fetch failed" whose cause says the host was never reached (no cause: same, nothing was sent). */ +function cannotConnect(error: unknown): boolean { + if (!(error instanceof TypeError) || !/fetch failed/i.test(error.message)) return false; + const code = (error.cause as { code?: unknown } | undefined)?.code; + return typeof code !== "string" || CANNOT_CONNECT_CODES.has(code); +} + export function evolutionJobDedupeKey(input: Pick): string | undefined { const payload = input.payload ?? {}; const payloadString = (key: string): string | undefined => { diff --git a/Memory/src/service/worker/job-policy-registry.ts b/Memory/src/service/worker/job-policy-registry.ts index 7788b84ac..a27f77d3e 100644 --- a/Memory/src/service/worker/job-policy-registry.ts +++ b/Memory/src/service/worker/job-policy-registry.ts @@ -85,6 +85,8 @@ const DEFINITIONS: Readonly> = Object.fr l3_abstraction: definition("evolution", "database", 60, [], "user_project"), l3_world_model_update: definition("evolution", "evolution_llm", 60, ["evolution"], "user_project"), project_environment_profile: definition("maintenance", "evolution_llm", 55, ["evolution"], "user_project"), + // A new-L3 batch runs in order per machine unit through its own scope_key, not per project. + l3_unit_update: definition("evolution", "evolution_llm", 60, ["evolution"], "none"), skill_crystallization: definition("evolution", "evolution_llm", 70, ["evolution"]), skill_cluster_assign: definition("evolution", "evolution_llm", 72, ["embedding", "evolution"], "user_project"), skill_batch_evolve: definition("evolution", "evolution_llm", 73, ["evolution"], "user_project"), diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index e8e490375..1ba6bf824 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -44,6 +44,10 @@ import { memoryNeedsImportSummary, summaryProcessingJobType } from "../import/im import { classifyProcessingError, type EnqueueJobInput, + L3_RETRY_JOB_TYPES, + L3_SUSPEND_MINUTES, + l3FailureHandling, + type L3FailureHandling, processingJobMatchesMemory, processingStageForJob, sanitizeProcessingError, @@ -62,6 +66,7 @@ import { resolveWorkerLeaseOptions, type WorkerLeaseOptions } from "./worker-lea import { WorkerConcurrencyController, workerCapacityKey, type PreparedWorkerExecution, type WorkerPermit } from "./worker-concurrency.js"; import { workerWorkRequirements, workerEmbeddingGroupKey } from "./worker-work-requirements.js"; import { observeWorker, type WorkerObserver, type WorkerMetricOutcome } from "./worker-observability.js"; +import { isL3FieldSnapshotMemory } from "../l3-field/snapshot-memory.js"; export const SUMMARY_WORKER_CONCURRENCY = 4; export const EMBEDDING_RETRY_LEASE_MS = 5 * 60_000; @@ -175,6 +180,16 @@ export interface WorkerRunnerDeps { runWorkerNoWrite: (request: RequestEnvelope) => Promise; restartFailedProcessing: (at: string, limit: number) => number; enqueueObservationBackfill?: (limit: number, at: string) => number; + /** Drops new-L3 items that have been inactive longer than the retention window. */ + purgeL3InactiveItems?: (at: string) => number; + /** Arms the idle flush of every Session holding turns not yet handed to the new L3. */ + armPendingL3Flushes?: () => number; + /** Gives each dead-lettered new-L3 batch one automatic rerun. */ + requeueL3DeadLettersOnce?: (at: string) => number; + /** Whether the new L3 is on; while off its jobs stay queued and are not leased. */ + l3FieldActive?: () => boolean; + /** Without an evolution model, L3 jobs stay queued (not leased, no attempt used) until one is configured. */ + evolutionModelConfigured?: () => boolean; previewPolicyEvidenceReconciliation: (limit: number) => PolicyEvidencePreflightReport; reconcileOrphanedPolicies: (at: string, limit: number) => number; enqueueJob: (input: EnqueueJobInput) => EvolutionJobRecord; @@ -236,12 +251,14 @@ export class WorkerRunner { } this.refreshConfigurationBlockedJobs(); const reconcileAt = this.deps.memoryBudgetNextReconcileAtMs?.(); - const heldSummaryJobs = this.summaryJobsHeld(); + // Wake only for jobs leasing would actually take, or a held job's runAfter spins the worker. + const heldJobs = this.excludedLeaseJobTypes(); if (this.deps.memoryBudgetPaused()) { const allowed = allowedMemoryBudgetJobTypes(this.deps.memoryBudgetModelSources()) - .filter((jobType) => !heldSummaryJobs?.includes(jobType)); + .filter((jobType) => !heldJobs?.includes(jobType)); const allowedAt = this.deps.repos.runtime.nextWorkerRunAt({ jobTypes: allowed, + ...(heldJobs ? { excludedJobTypes: heldJobs } : {}), includeEmbeddingRetries: !this.deps.memoryBudgetJobConsumes("embedding") }); const midnight = this.deps.memoryBudgetNextWakeAtMs(); @@ -249,7 +266,7 @@ export class WorkerRunner { return times.length > 0 ? Math.min(...times) : undefined; } const scheduled = this.deps.repos.runtime.nextWorkerRunAt( - heldSummaryJobs ? { excludedJobTypes: heldSummaryJobs } : undefined + heldJobs ? { excludedJobTypes: heldJobs } : undefined ); const times = [scheduled, reconcileAt].filter((time): time is number => Number.isFinite(time)); return times.length > 0 ? Math.min(...times) : undefined; @@ -432,6 +449,7 @@ export class WorkerRunner { }, limit); for (const memory of retrievalMemories) { this.deps.repos.memories.reindexFts(memory); + if (isL3FieldSnapshotMemory(memory)) continue; if (!this.deps.capture.embedAfterCapture || retrievalDocumentIsCurrent(memory)) continue; if (this.deps.repos.runtime.hasPendingJob(memory.id, "embedding")) continue; const sourceHash = retrievalDocumentSourceHash(memory); @@ -451,6 +469,9 @@ export class WorkerRunner { enqueuedRetrievalReindexes += 1; } + this.deps.purgeL3InactiveItems?.(at); + this.deps.requeueL3DeadLettersOnce?.(at); + this.deps.armPendingL3Flushes?.(); this.refreshObservationModelConfiguration(); return { requeuedJobs: interruptedJobs.length + failedJobs.length, @@ -554,7 +575,7 @@ export class WorkerRunner { const comparison = this.schedulingShadow.sample({ at: this.deps.nowIso(), limit: normalizedLimit, targetMemoryIds, laneWeights: options.laneWeights, priorityCohortOnly: request.priorityCohortOnly, - allowedJobTypes: allowedWhenPaused, excludedJobTypes: this.summaryJobsHeld(), + allowedJobTypes: allowedWhenPaused, excludedJobTypes: this.excludedLeaseJobTypes(), includeRetries: !(this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding")) }); if (comparison) { @@ -641,7 +662,7 @@ export class WorkerRunner { let previous: { key: string; permit: WorkerPermit; group: symbol; count: number } | undefined; const keyOf = (work: FairClaimedWork) => `${work.queue}:${work.queue === "main" ? work.job.id : work.retry.id}`; const paused = this.deps.memoryBudgetPaused(); - const filters = JSON.stringify([retryOnly, paused, this.deps.memoryBudgetModelSources(), this.summaryJobsHeld(), + const filters = JSON.stringify([retryOnly, paused, this.deps.memoryBudgetModelSources(), this.excludedLeaseJobTypes(), this.deps.configurationFingerprint?.()]); if (!scan.more) scanDenied = scan.waiting.size > 0; const revision = this.deps.repos.runtime.scheduling.getDeadlineRevision(); @@ -686,7 +707,7 @@ export class WorkerRunner { at: this.deps.nowIso(), limit: capacity, targetMemoryIds: request.targetMemoryIds, priorityCohortOnly: request.priorityCohortOnly, allowedJobTypes: retryOnly ? [] : paused ? allowedMemoryBudgetJobTypes(this.deps.memoryBudgetModelSources()) : undefined, - excludedJobTypes: this.summaryJobsHeld(), + excludedJobTypes: this.excludedLeaseJobTypes(), includeRetries: !(paused && this.deps.memoryBudgetJobConsumes("embedding")), owner: this.leaseOwner, retryOwner: this.deps.embeddingRetryWorkerId, leaseSeconds: WORKER_LEASE_SECONDS, @@ -872,7 +893,7 @@ export class WorkerRunner { const scheduled = this.deps.repos.runtime.nextWorkerRunAt({ futureAfterMs: now, targetMemoryIds, jobTypes: retryOnly ? [] : paused ? allowedMemoryBudgetJobTypes(this.deps.memoryBudgetModelSources()) : undefined, - excludedJobTypes: this.summaryJobsHeld(), + excludedJobTypes: this.excludedLeaseJobTypes(), includeEmbeddingRetries: !(paused && this.deps.memoryBudgetJobConsumes("embedding")) }); return scheduled; @@ -1071,12 +1092,14 @@ export class WorkerRunner { this.observeWork({ queue: "main", job }, "finished", "released"); return { succeeded: 0, failed: 0, ref: { ...jobToRef(deferred), status: "queued" } }; } + const l3Handling = l3FailureHandling(job, error); + if (l3Handling.kind === "retry_later" || l3Handling.kind === "suspend") return this.rescheduleFailedL3Job(job, error, l3Handling); const errorMessage = processingStageForJob(job.jobType) ? sanitizeProcessingError(error) : error instanceof Error ? error.message : String(error); const classification = classifyProcessingError(error); const configurationBlocked = classification.retryAction === "open_settings" && classification.code !== "40309"; - const forceDeadLetter = classification.code === "40309" + const forceDeadLetter = l3Handling.kind === "dead_letter" || classification.code === "40309" || classification.retryAction === "none" || error instanceof UnknownJobPolicyError || isTerminalL3WorldModelError(error) @@ -1121,6 +1144,36 @@ export class WorkerRunner { }; } + /** Temporary trouble waits and retries; a configuration problem waits for a fix without using an attempt. */ + private rescheduleFailedL3Job( + job: EvolutionJobRecord, + error: unknown, + handling: Extract + ): WorkerJobRunResult { + const at = this.deps.nowIso(); + const minutes = handling.kind === "retry_later" ? handling.delayMinutes : L3_SUSPEND_MINUTES; + const rescheduled = this.deps.repos.runtime.rescheduleFailedJob(job.id, { + runAfter: new Date(Date.parse(at) + minutes * 60_000).toISOString(), + error: sanitizeProcessingError(error), + refundAttempt: handling.kind === "suspend", + minMaxAttempts: handling.kind === "retry_later" ? handling.maxAttempts : undefined, + at + }, job); + if (!rescheduled) return this.lostLeaseResult(job); + this.deps.appendJobChange(rescheduled, "queued", job); + workerLogger.warn(handling.kind === "suspend" ? "job.suspended" : "job.retry_scheduled", { + ...workerJobLogFields(rescheduled), + retryInMinutes: minutes, + ...memoryErrorFields(error) + }); + this.observeWork({ queue: "main", job }, "finished", handling.kind === "suspend" ? "released" : "failed"); + return { + succeeded: 0, + failed: handling.kind === "suspend" ? 0 : 1, + ref: { ...jobToRef(rescheduled), status: "queued" } + }; + } + markProcessingJobLeased(job: EvolutionJobRecord): void { if (!this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso())) throw new JobLeaseLostError(job.id); if (!job.targetMemoryId) return; @@ -1402,6 +1455,20 @@ export class WorkerRunner { return { succeeded: 0, failed: 0, item: null }; } + private excludedLeaseJobTypes(): readonly string[] | undefined { + const held = [ + ...(this.summaryJobsHeld() ?? []), + ...(this.deps.evolutionModelConfigured?.() === false ? L3_RETRY_JOB_TYPES : []) + ]; + // One L3 runs at a time; the other's jobs wait in the queue for the switch to turn back. + // Excluding l3_unit_update also holds new-L3 profile jobs, and excluding l3_world_model_update + // holds the old project profile jobs (see mainQueueEligibility). + const inactiveL3 = this.deps.l3FieldActive === undefined ? [] + : this.deps.l3FieldActive() ? ["l3_world_model_update"] : ["l3_unit_update"]; + const excluded = [...new Set([...held, ...inactiveL3])]; + return excluded.length > 0 ? excluded : undefined; + } + private summaryJobsHeld(): readonly string[] | undefined { return this.deps.summaryModelConfigured() ? undefined : SUMMARY_JOBS_REQUIRING_MODEL; } diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts new file mode 100644 index 000000000..b9829d7c4 --- /dev/null +++ b/Memory/src/storage/l3-field-repository.ts @@ -0,0 +1,605 @@ +import type Database from "better-sqlite3"; +import { canonicalJson, sha256Hex } from "../contracts/index.js"; +import type { + L3ItemEvidence, + L3ItemKind, + L3ItemRecord, + L3UnitBatchRecord, + L3UnitBatchStatus, + L3UnitKind, + L3UnitRecord, + L3UserProject, + ProjectIdentitySource, + ResolvedProject +} from "../service/l3-field/types.js"; +import type { MemoryRow } from "../types.js"; +import { newId } from "../utils/id.js"; +import { parseJson, toJson } from "../utils/json.js"; +import { nowIso } from "../utils/time.js"; +import type { MemoryRepository } from "./repositories.js"; + +const SNAPSHOT_SOURCE = "worker.l3_field.v1"; + +interface SqlUnitRow { + unit_key: string; + user_id: string; + machine_id: string; + kind: L3UnitKind; + project_identity: string | null; + display_name: string | null; + identity_source: ProjectIdentitySource | null; + checkout_root: string | null; + profile_text: string | null; + profile_fingerprint: string | null; + snapshot_memory_id: string | null; + deleted_at: string | null; + next_seq: number; + created_at: string; + updated_at: string; +} + +interface SqlItemRow { + id: string; + unit_key: string; + kind: L3ItemKind; + content: string; + evidence_json: string; + evidence_at: string; + last_confirmed_at: string; + status: "active" | "inactive"; + replaced_by: string | null; + removed_evidence_at: string | null; + inactive_at: string | null; + batch_id: string | null; + created_at: string; + updated_at: string; +} + +export function l3UnitKey(input: { + userId: string; + machineId: string; + kind: L3UnitKind; + identity: string | null; +}): string { + return `l3u:${sha256Hex(canonicalJson({ + userId: input.userId, + machineId: input.machineId, + kind: input.kind, + identity: input.identity + }))}`; +} + +export function l3UnitSnapshotMemoryKey(unitKey: string): string { + return `world_model:l3_field:${unitKey}`; +} + +export { isL3FieldSnapshotMemory, l3FieldSnapshotUnitKey } from "../service/l3-field/snapshot-memory.js"; + +/** Renders one unit (profile plus active items) as the text stored in its snapshot memory. */ +export function renderL3UnitSnapshot(unit: L3UnitRecord, items: readonly L3ItemRecord[]): string { + const facts = items.filter((item) => item.kind === "env_fact"); + const constraints = items.filter((item) => item.kind === "project_constraint"); + const sections: string[] = []; + if (unit.profileText?.trim()) sections.push(`## 环境画像\n${unit.profileText.trim()}`); + if (facts.length) sections.push(`## 环境事实\n${facts.map((item) => `- ${item.content}`).join("\n")}`); + if (constraints.length) sections.push(`## 项目约束\n${constraints.map((item) => `- ${item.content}`).join("\n")}`); + return sections.join("\n\n"); +} + +export class L3FieldRepository { + constructor( + private readonly db: Database.Database, + private readonly memories: MemoryRepository + ) {} + + getUnit(unitKey: string): L3UnitRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_units WHERE unit_key = ?`).get(unitKey) as SqlUnitRow | undefined; + return row ? unitFromSql(row) : undefined; + } + + ensureMachineUnit(input: { userId: string; machineId: string; at?: string }): L3UnitRecord { + const unitKey = l3UnitKey({ userId: input.userId, machineId: input.machineId, kind: "machine", identity: null }); + const at = input.at ?? nowIso(); + this.db.prepare( + `INSERT INTO l3_units (unit_key, user_id, machine_id, kind, created_at, updated_at) + VALUES (?, ?, ?, 'machine', ?, ?) + ON CONFLICT(unit_key) DO NOTHING` + ).run(unitKey, input.userId, input.machineId, at, at); + return this.getUnit(unitKey)!; + } + + ensureProjectUnit(input: { + userId: string; + machineId: string; + project: ResolvedProject; + at?: string; + }): L3UnitRecord { + const unitKey = l3UnitKey({ + userId: input.userId, + machineId: input.machineId, + kind: "project", + identity: input.project.identity + }); + const at = input.at ?? nowIso(); + this.db.prepare( + `INSERT INTO l3_units ( + unit_key, user_id, machine_id, kind, project_identity, display_name, + identity_source, checkout_root, created_at, updated_at + ) VALUES (?, ?, ?, 'project', ?, ?, ?, ?, ?, ?) + ON CONFLICT(unit_key) DO UPDATE SET + display_name = excluded.display_name, + identity_source = excluded.identity_source, + checkout_root = excluded.checkout_root, + updated_at = CASE + WHEN l3_units.display_name IS excluded.display_name + AND l3_units.identity_source IS excluded.identity_source + AND l3_units.checkout_root IS excluded.checkout_root + THEN l3_units.updated_at ELSE excluded.updated_at END` + ).run( + unitKey, + input.userId, + input.machineId, + input.project.identity, + input.project.displayName, + input.project.source, + input.project.checkoutRoot, + at, + at + ); + return this.getUnit(unitKey)!; + } + + activeItems(unitKey: string): L3ItemRecord[] { + return (this.db.prepare( + `SELECT * FROM l3_items WHERE unit_key = ? AND status = 'active' + ORDER BY created_at ASC, id ASC` + ).all(unitKey) as SqlItemRow[]).map(itemFromSql); + } + + /** + * Items of these units that a delete or replace removed on evidence newer than `evidenceAt`, + * newest last. Empty when all of a batch's evidence is newer (the usual case); a batch rerun late, + * history scanned late, or a batch spanning a removal sees what was removed. + */ + itemsRemovedAfter(unitKeys: readonly string[], evidenceAt: string, limit = 20): L3ItemRecord[] { + if (unitKeys.length === 0) return []; + return (this.db.prepare( + `SELECT * FROM l3_items + WHERE unit_key IN (${unitKeys.map(() => "?").join(", ")}) + AND status = 'inactive' AND removed_evidence_at > ? + ORDER BY removed_evidence_at DESC, id DESC + LIMIT ?` + ).all(...unitKeys, evidenceAt, limit) as SqlItemRow[]).map(itemFromSql).reverse(); + } + + getItem(id: string): L3ItemRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_items WHERE id = ?`).get(id) as SqlItemRow | undefined; + return row ? itemFromSql(row) : undefined; + } + + insertItem(input: { + unitKey: string; + kind: L3ItemKind; + content: string; + evidence: L3ItemEvidence; + evidenceAt: string; + batchId?: string; + at?: string; + }): L3ItemRecord { + const id = newId("l3item"); + const at = input.at ?? nowIso(); + this.db.prepare( + `INSERT INTO l3_items ( + id, unit_key, kind, content, evidence_json, evidence_at, last_confirmed_at, + status, batch_id, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?)` + ).run( + id, + input.unitKey, + input.kind, + input.content, + toJson(input.evidence), + input.evidenceAt, + input.evidenceAt, + input.batchId ?? null, + at, + at + ); + return this.getItem(id)!; + } + + /** + * `removedEvidenceAt` is when the evidence that removed it happened (a delete or replace op); + * evictions and unit resets leave it empty. + */ + inactivateItem(id: string, input: { replacedBy?: string; removedEvidenceAt?: string; at?: string }): void { + const at = input.at ?? nowIso(); + this.db.prepare( + `UPDATE l3_items + SET status = 'inactive', replaced_by = ?, removed_evidence_at = ?, inactive_at = ?, updated_at = ? + WHERE id = ? AND status = 'active'` + ).run(input.replacedBy ?? null, input.removedEvidenceAt ?? null, at, at, id); + } + + /** Moves the last-confirmed time forward only; an older confirmation never rewinds it. */ + confirmItem(id: string, confirmedAt: string, at = nowIso()): void { + this.db.prepare( + `UPDATE l3_items + SET last_confirmed_at = ?, updated_at = ? + WHERE id = ? AND status = 'active' AND last_confirmed_at < ?` + ).run(confirmedAt, at, id, confirmedAt); + } + + /** + * Creates or updates the snapshot memory for one unit. A change to a machine + * unit also bumps the snapshot version of every project on the same machine, + * because session context renders the machine together with the project. + */ + refreshSnapshot(unitKey: string, at = nowIso()): MemoryRow { + const unit = this.getUnit(unitKey); + if (!unit) throw new Error(`L3 unit not found: ${unitKey}`); + const before = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; + const saved = this.writeSnapshot(unit, at); + const changed = saved.id !== before?.id || saved.version !== before?.version; + if (unit.kind === "machine" && changed) this.bumpProjectSnapshots(unit, at); + return saved; + } + + /** After a machine unit's snapshot changed outside a refresh (archived), its projects get new versions. */ + refreshProjectContexts(unitKey: string, at = nowIso()): void { + const unit = this.getUnit(unitKey); + if (unit?.kind === "machine") this.bumpProjectSnapshots(unit, at); + } + + /** + * Every project on the machine with a current snapshot gets a new version. A project reset + * earlier keeps its `deleted_at` (it guards old batches) but counts again once rebuilt. + */ + private bumpProjectSnapshots(machine: L3UnitRecord, at: string): void { + const projects = (this.db.prepare( + `SELECT * FROM l3_units + WHERE user_id = ? AND machine_id = ? AND kind = 'project' AND snapshot_memory_id IS NOT NULL` + ).all(machine.userId, machine.machineId) as SqlUnitRow[]).map(unitFromSql); + for (const project of projects) this.writeSnapshot(project, at, true); + } + + /** + * Deleting a unit's L3 resets it rather than disabling it: items become + * inactive, the profile and snapshot link are cleared, and `deleted_at` + * records when, so work queued before the reset is discarded while later + * conversations build the unit up again. + */ + markUnitDeleted(unitKey: string, at = nowIso()): void { + this.db.transaction(() => { + this.db.prepare( + `UPDATE l3_units + SET deleted_at = ?, snapshot_memory_id = NULL, profile_text = NULL, profile_fingerprint = NULL, updated_at = ? + WHERE unit_key = ?` + ).run(at, at, unitKey); + this.db.prepare( + `UPDATE l3_items + SET status = 'inactive', inactive_at = ?, updated_at = ? + WHERE unit_key = ? AND status = 'active'` + ).run(at, at, unitKey); + // Project context includes the machine, so its cached versions must move when the machine resets. + const unit = this.getUnit(unitKey); + if (unit?.kind === "machine") this.bumpProjectSnapshots(unit, at); + })(); + } + + /** Deletes items that became inactive before `before`; active items are never touched. */ + purgeInactive(before: string): number { + return this.db.prepare( + `DELETE FROM l3_items WHERE status = 'inactive' AND inactive_at < ?` + ).run(before).changes; + } + + listUserProjects(userId: string): L3UserProject[] { + return (this.db.prepare( + `SELECT root_path, name FROM l3_user_projects WHERE user_id = ? ORDER BY root_path ASC` + ).all(userId) as Array<{ root_path: string; name: string }>) + .map((row) => ({ rootPath: row.root_path, name: row.name })); + } + + /** Replaces the whole list; the latest push wins. */ + replaceUserProjects(userId: string, projects: readonly L3UserProject[], at = nowIso()): void { + this.db.transaction(() => { + this.db.prepare(`DELETE FROM l3_user_projects WHERE user_id = ?`).run(userId); + const insert = this.db.prepare( + `INSERT INTO l3_user_projects (user_id, root_path, name, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(user_id, root_path) DO UPDATE SET name = excluded.name, updated_at = excluded.updated_at` + ); + for (const project of projects) insert.run(userId, project.rootPath, project.name, at); + })(); + } + + /** Profile text comes only from code probes (machine) or scan summaries (project). */ + setProfile(unitKey: string, profile: { text: string | null; fingerprint: string }, at = nowIso()): void { + this.db.prepare( + `UPDATE l3_units SET profile_text = ?, profile_fingerprint = ?, updated_at = ? WHERE unit_key = ?` + ).run(profile.text, profile.fingerprint, at, unitKey); + } + + /** + * Freezes one chunk of a session's input traces as a batch of the machine unit, claiming the + * unit's next sequence number. The caller queues the matching l3_unit_update job. + */ + freezeUnitBatch(input: { + userId: string; + machineId: string; + sessionId: string; + trigger: string; + startTraceSeq: number; + endTraceSeq: number; + l1MemoryIds: readonly string[]; + rawTurnIds: readonly string[]; + at?: string; + }): L3UnitBatchRecord { + const at = input.at ?? nowIso(); + const machine = this.ensureMachineUnit({ userId: input.userId, machineId: input.machineId, at }); + const claimed = this.db.prepare( + `UPDATE l3_units SET next_seq = next_seq + 1, updated_at = ? WHERE unit_key = ? + RETURNING next_seq - 1 AS scope_seq` + ).get(at, machine.unitKey) as { scope_seq: number }; + const id = newId("l3batch"); + this.db.prepare( + `INSERT INTO l3_unit_batches ( + id, machine_unit_key, scope_seq, user_id, session_id, trigger, start_trace_seq, end_trace_seq, + l1_memory_ids_json, raw_turn_ids_json, status, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', ?, ?)` + ).run( + id, + machine.unitKey, + claimed.scope_seq, + input.userId, + input.sessionId, + input.trigger, + input.startTraceSeq, + input.endTraceSeq, + toJson([...input.l1MemoryIds]), + toJson([...input.rawTurnIds]), + at, + at + ); + return this.getBatch(id)!; + } + + getBatch(id: string): L3UnitBatchRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_unit_batches WHERE id = ?`).get(id) as SqlBatchRow | undefined; + return row ? batchFromSql(row) : undefined; + } + + /** + * Rerun entry: the user's batches whose job ended in dead letter go back in the queue. The original + * job is reset in place (a copy would hit the dedupe and sequence indexes); applied batches are left alone. + * Returns the ids of the jobs put back. + */ + requeueDeadLetterBatches(userId: string, at = nowIso()): string[] { + return this.requeueDeadLetters(`AND j.user_id = ?`, [userId], at, false); + } + + /** + * Worker startup: every user's dead-lettered batches get one automatic rerun each, so a batch that + * failed through a long outage is not lost; a batch that dead-letters again stays for the manual rerun. + */ + requeueDeadLetterBatchesOnce(at = nowIso()): string[] { + return this.requeueDeadLetters(`AND json_extract(j.payload_json, '$.autoRequeued') IS NULL`, [], at, true); + } + + private requeueDeadLetters(filter: string, params: string[], at: string, markAuto: boolean): string[] { + return this.db.transaction(() => { + const jobs = this.db.prepare( + `SELECT j.id, b.id AS batch_id FROM evolution_jobs j + JOIN l3_unit_batches b ON b.id = CAST(json_extract(j.payload_json, '$.batchId') AS TEXT) + WHERE j.job_type = 'l3_unit_update' AND j.status = 'dead_letter' AND b.status != 'applied' ${filter} + ORDER BY j.scope_key, j.scope_seq` + ).all(...params) as Array<{ id: string; batch_id: string }>; + for (const job of jobs) { + this.db.prepare(`UPDATE l3_unit_batches SET status = 'queued', updated_at = ? WHERE id = ?`).run(at, job.batch_id); + this.db.prepare( + `UPDATE evolution_jobs SET status = 'queued', attempts = 0, failure_count = 0, leased_until = NULL, + lease_owner = NULL, lease_token = NULL, available_at = NULL, blocked_reason = NULL, last_error = NULL, + payload_json = ${markAuto ? "json_set(json_remove(payload_json, '$.runAfter'), '$.autoRequeued', 1)" : "json_remove(payload_json, '$.runAfter')"}, + updated_at = ? + WHERE id = ?` + ).run(at, job.id); + } + return jobs.map((job) => job.id); + })(); + } + + setBatchStatus(id: string, status: L3UnitBatchStatus, at = nowIso()): void { + this.db.prepare(`UPDATE l3_unit_batches SET status = ?, updated_at = ? WHERE id = ?`).run(status, at, id); + } + + /** Throwaway task roots Memmy itself creates (they are never projects); stored with the project list. */ + getStandaloneRoots(userId: string): string[] { + const row = this.db.prepare(`SELECT value_json FROM runtime_kv WHERE key = ?`) + .get(standaloneRootsKey(userId)) as { value_json: string } | undefined; + const value = row ? parseJson(row.value_json, []) : []; + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; + } + + setStandaloneRoots(userId: string, roots: readonly string[], at = nowIso()): void { + this.db.prepare( + `INSERT INTO runtime_kv (key, value_json, updated_at) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at` + ).run(standaloneRootsKey(userId), toJson([...roots]), at); + } + + /** `force` writes a new version even when nothing shown changed (a project carrying a machine change). */ + private writeSnapshot(unit: L3UnitRecord, at: string, force = false): MemoryRow { + const items = this.activeItems(unit.unitKey); + const memoryValue = renderL3UnitSnapshot(unit, items); + const existing = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; + const title = unit.kind === "machine" ? "本机场域认知" : `项目场域认知：${unit.displayName ?? unit.projectIdentity}`; + const tags = ["world_model", "l3_field", `scope:${unit.kind}`]; + const info: Record = { l3_field_unit_key: unit.unitKey }; + // Rewriting a snapshot keeps the user's archive; only the user unarchives it. + const status = existing?.status === "archived" ? "archived" as const : "activated" as const; + const bullets = (kind: L3ItemKind) => { + const lines = items.filter((item) => item.kind === kind).map((item) => `- ${item.content}`); + return lines.length ? lines.join("\n") : null; + }; + // Shape the desktop "world model" page already understands (it only reads schema-2 fields). + const legacyFields = { + general_rules_and_safety_constraints: null, + project_environment_profile: unit.profileText?.trim() ? unit.profileText.trim() : null, + project_contract: bullets("project_constraint"), + domain_knowledge: bullets("env_fact") + }; + const memory: MemoryRow = { + id: existing?.id ?? newId("memory"), + timeline: existing?.timeline ?? at, + userId: unit.userId, + memoryType: "LongTermMemory", + status, + visibility: "private", + memoryKey: l3UnitSnapshotMemoryKey(unit.unitKey), + memoryValue, + tags, + info, + properties: { + memory_type: "LongTermMemory", + status, + tags, + info: { ...info }, + internal_info: { + memory_layer: "L3", + memory_kind: "world_model", + schema_version: 3, + source: SNAPSHOT_SOURCE, + title, + summary: [...memoryValue.replace(/\s+/gu, " ").trim()].slice(0, 240).join(""), + body: memoryValue, + l3_field: { + unit_key: unit.unitKey, + kind: unit.kind, + display_name: unit.displayName ?? null, + identity_source: unit.identitySource ?? null, + legacy_fields: legacyFields + } + } + }, + memoryLayer: "L3", + contentHash: sha256Hex(memoryValue), + version: existing?.version ?? 1, + createdAt: existing?.createdAt ?? at, + updatedAt: at, + deletedAt: null + }; + // Nothing shown changed (a confirm only moves times): keep the version, so clients do not refetch + // and the change log does not fill with empty updates. + if ( + existing && !force && !existing.deletedAt && existing.status === status && + existing.contentHash === memory.contentHash && + toJson(existing.properties.internal_info) === toJson(memory.properties.internal_info) + ) return existing; + const saved = existing ? this.memories.update(memory) : this.memories.insert(memory); + if (!existing) { + this.db.prepare( + `UPDATE l3_units SET snapshot_memory_id = ?, updated_at = ? WHERE unit_key = ?` + ).run(saved.id, at, unit.unitKey); + } + this.db.prepare( + `INSERT INTO memory_change_log ( + memory_id, namespace_id, kind, op, entity_id, user_id, + change_type, version, before_json, after_json, source, created_at + ) VALUES (?, ?, 'world_model', ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + saved.id, + unit.unitKey, + existing ? "updated" : "created", + saved.id, + unit.userId, + existing ? "l3_field_snapshot_update" : "l3_field_snapshot_create", + saved.version, + existing ? toJson(existing) : null, + toJson(saved), + SNAPSHOT_SOURCE, + at + ); + return saved; + } +} + +function standaloneRootsKey(userId: string): string { + return `l3_field.standalone_roots:${userId}`; +} + +interface SqlBatchRow { + id: string; + machine_unit_key: string; + scope_seq: number; + user_id: string; + session_id: string; + trigger: string; + start_trace_seq: number; + end_trace_seq: number; + l1_memory_ids_json: string; + raw_turn_ids_json: string; + status: L3UnitBatchStatus; + created_at: string; + updated_at: string; +} + +function batchFromSql(row: SqlBatchRow): L3UnitBatchRecord { + return { + id: row.id, + machineUnitKey: row.machine_unit_key, + scopeSeq: Number(row.scope_seq), + userId: row.user_id, + sessionId: row.session_id, + trigger: row.trigger, + startTraceSeq: Number(row.start_trace_seq), + endTraceSeq: Number(row.end_trace_seq), + l1MemoryIds: parseJson(row.l1_memory_ids_json, []), + rawTurnIds: parseJson(row.raw_turn_ids_json, []), + status: row.status, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} + +function unitFromSql(row: SqlUnitRow): L3UnitRecord { + return { + unitKey: row.unit_key, + userId: row.user_id, + machineId: row.machine_id, + kind: row.kind, + projectIdentity: row.project_identity ?? undefined, + displayName: row.display_name ?? undefined, + identitySource: row.identity_source ?? undefined, + checkoutRoot: row.checkout_root ?? undefined, + profileText: row.profile_text ?? undefined, + profileFingerprint: row.profile_fingerprint ?? undefined, + snapshotMemoryId: row.snapshot_memory_id ?? undefined, + deletedAt: row.deleted_at ?? undefined, + nextSeq: row.next_seq, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} + +function itemFromSql(row: SqlItemRow): L3ItemRecord { + const evidence = parseJson>(row.evidence_json, {}); + return { + id: row.id, + unitKey: row.unit_key, + kind: row.kind, + content: row.content, + evidence: { + fragmentIds: Array.isArray(evidence.fragmentIds) ? evidence.fragmentIds : [], + l1MemoryIds: Array.isArray(evidence.l1MemoryIds) ? evidence.l1MemoryIds : [] + }, + evidenceAt: row.evidence_at, + lastConfirmedAt: row.last_confirmed_at, + status: row.status, + replacedBy: row.replaced_by ?? undefined, + removedEvidenceAt: row.removed_evidence_at ?? undefined, + inactiveAt: row.inactive_at ?? undefined, + batchId: row.batch_id ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} diff --git a/Memory/src/storage/queue-eligibility.ts b/Memory/src/storage/queue-eligibility.ts index bb9f1a1c3..d91c79000 100644 --- a/Memory/src/storage/queue-eligibility.ts +++ b/Memory/src/storage/queue-eligibility.ts @@ -51,29 +51,7 @@ export function mainQueueEligibility(at: string, options: QueueEligibilityOption json_extract(payload_json, '$.runAfter') IS NULL OR CAST(json_extract(payload_json, '$.runAfter') AS TEXT) <= ? ) - AND ${includeDependencyBlocked ? "1=1" : `( - job_type NOT IN ('l3_world_model_update', 'work_memory_extract') - OR ( - scope_key IS NOT NULL - AND scope_seq IS NOT NULL - AND NOT EXISTS ( - SELECT 1 - FROM evolution_jobs AS leased_l3_job - WHERE leased_l3_job.job_type = evolution_jobs.job_type - AND leased_l3_job.scope_key = evolution_jobs.scope_key - AND leased_l3_job.status = 'leased' - AND leased_l3_job.id <> evolution_jobs.id - ) - AND NOT EXISTS ( - SELECT 1 - FROM evolution_jobs AS earlier_l3_job - WHERE earlier_l3_job.job_type = evolution_jobs.job_type - AND earlier_l3_job.scope_key = evolution_jobs.scope_key - AND earlier_l3_job.scope_seq < evolution_jobs.scope_seq - AND earlier_l3_job.status IN ('queued', 'leased', 'failed') - ) - ) - )`} + AND ${includeDependencyBlocked ? "1=1" : scopedJobReadySql()} AND ${includeDependencyBlocked ? "1=1" : `NOT ( job_type = 'feedback_experience' AND EXISTS ( @@ -131,11 +109,75 @@ export function mainQueueEligibility(at: string, options: QueueEligibilityOption ))`} ${allowedJobTypes ? `AND job_type IN (${allowedJobTypes.map(() => "?").join(", ")})` : ""} ${excludedJobTypes?.length ? `AND job_type NOT IN (${excludedJobTypes.map(() => "?").join(", ")})` : ""} + ${l3ProfileHeldSql(excludedJobTypes)} ${targetFilter}`, - params: [at, at, at, ...(allowedJobTypes ?? []), ...(excludedJobTypes ?? []), ...(targetMemoryIds ?? [])] + params: [at, at, at, ...(includeDependencyBlocked ? [] : [at]), + ...(allowedJobTypes ?? []), ...(excludedJobTypes ?? []), ...(targetMemoryIds ?? [])] }; } +/** + * Profile jobs of the L3 that is not running wait with its update jobs: excluding l3_unit_update + * (the new L3 is off) holds new-L3 profiles, excluding l3_world_model_update (the new L3 is on) + * holds the old project profiles. + */ +export function l3ProfileHeldSql(excludedJobTypes: readonly string[] | undefined): string { + const held = [ + ...(excludedJobTypes?.includes("l3_unit_update") + ? ["NOT (job_type = 'project_environment_profile' AND json_extract(payload_json, '$.mode') IS 'l3_field')"] : []), + ...(excludedJobTypes?.includes("l3_world_model_update") + ? ["NOT (job_type = 'project_environment_profile' AND json_extract(payload_json, '$.mode') IS NOT 'l3_field')"] : []) + ]; + return held.map((condition) => `AND ${condition}`).join(" "); +} + +/** + * Whether a job is not held back by an earlier job of its scope (L3 and Work Memory run in order) + * or, for a new-L3 batch, by L1 summaries still being written. Leasing and the worker's next + * wake-up both use it; a wake-up time for a job that cannot be leased would spin the worker. + * Its one parameter is the current time: a lease that ran out no longer holds the scope, so an + * earlier batch put back in the queue is not stuck behind a later batch whose run was lost. + */ +export function scopedJobReadySql(): string { + return `( + job_type NOT IN ('l3_world_model_update', 'l3_unit_update', 'work_memory_extract') + OR ( + scope_key IS NOT NULL + AND scope_seq IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM evolution_jobs AS leased_l3_job + WHERE leased_l3_job.job_type = evolution_jobs.job_type + AND leased_l3_job.scope_key = evolution_jobs.scope_key + AND leased_l3_job.status = 'leased' + AND leased_l3_job.leased_until > ? + AND leased_l3_job.id <> evolution_jobs.id + ) + AND NOT EXISTS ( + SELECT 1 + FROM evolution_jobs AS earlier_l3_job + WHERE earlier_l3_job.job_type = evolution_jobs.job_type + AND earlier_l3_job.scope_key = evolution_jobs.scope_key + AND earlier_l3_job.scope_seq < evolution_jobs.scope_seq + AND earlier_l3_job.status IN ('queued', 'leased', 'failed') + ) + ) + ) + AND NOT ( + job_type = 'l3_unit_update' + AND EXISTS ( + SELECT 1 + FROM l3_unit_batches AS unit_batch, json_each(unit_batch.l1_memory_ids_json) AS batch_l1 + JOIN memory_processing_state AS l1_state ON l1_state.memory_id = batch_l1.value + JOIN memories AS batch_memory ON batch_memory.id = l1_state.memory_id + WHERE unit_batch.id = CAST(json_extract(evolution_jobs.payload_json, '$.batchId') AS TEXT) + AND l1_state.state IN ('summary_pending', 'summarizing') + -- A deleted L1 is skipped by the pipeline, so it must not hold the batch back. + AND batch_memory.deleted_at IS NULL AND batch_memory.status != 'deleted' + ) + )`; +} + export function retryQueueEligibility(now: number, targetMemoryIds?: readonly string[]): { sql: string; params: Array; } { diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index d62a20140..8d5d534db 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -59,8 +59,16 @@ import { type VectorSearchCandidate } from "./sqlite-vec-store.js"; import { QueueSchedulingStore, type SchedulingMetadata } from "./queue-scheduling-store.js"; -import { mainQueueEligibility, retryQueueEligibility, type QueueEligibilityOptions } from "./queue-eligibility.js"; +import { + l3ProfileHeldSql, + mainQueueEligibility, + retryQueueEligibility, + scopedJobReadySql, + type QueueEligibilityOptions +} from "./queue-eligibility.js"; import { compactHistoryChangePayload, compactHistoryChangeRow } from "./history-change-snapshot.js"; +import { L3FieldRepository } from "./l3-field-repository.js"; +import { isL3FieldSnapshotMemory } from "../service/l3-field/snapshot-memory.js"; type SqlValue = string | number | Buffer | null; const BUNDLE_TABLES = [ @@ -86,6 +94,10 @@ const BUNDLE_TABLES = [ "api_logs", "memory_change_log", "l3_world_model_project_environment_state", + "l3_units", + "l3_unit_batches", + "l3_items", + "l3_user_projects", "evolution_jobs", "embedding_retry_queue", "memory_processing_state", @@ -353,7 +365,11 @@ export type L3WorldModelBatchTrigger = | "token_compaction" | "token_compaction_attempt" | "session_close" - | "episode_idle_close"; + | "episode_idle_close" + /** A finished agent-source scan pass; new L3 only (scans stay out of the old pipeline). */ + | "scan_pass" + /** A Session that went quiet without a close (its idle flush); new L3 only. */ + | "session_idle"; export type L3WorldModelTargetField = Exclude; @@ -1374,7 +1390,7 @@ export class MemoryRepository { reindexFts(memory: MemoryRow): void { try { this.db.prepare(`DELETE FROM memories_fts WHERE id = ?`).run(memory.id); - if (!memory.deletedAt && memory.status !== "deleted") { + if (!memory.deletedAt && memory.status !== "deleted" && !isL3FieldSnapshotMemory(memory)) { this.db .prepare(`INSERT INTO memories_fts (id, identifier, memory_value, tags) VALUES (?, ?, ?, ?)`) .run(memory.id, memory.id, retrievalDocumentForMemory(memory), memory.tags.join(" ")); @@ -2755,6 +2771,18 @@ export class RuntimeRepository { .all(rawTurnId, rawTurnId) as Array<{ id: string }>).map(r => r.id); } + /** An L1 of a new-L3 batch whose summary is still being written (the batch waits for it), if any. */ + unsummarizedL3BatchL1(batchId: string): string | undefined { + return this.db.prepare( + `SELECT batch_l1.value FROM l3_unit_batches AS unit_batch, json_each(unit_batch.l1_memory_ids_json) AS batch_l1 + JOIN memory_processing_state AS l1_state ON l1_state.memory_id = batch_l1.value + JOIN memories AS batch_memory ON batch_memory.id = l1_state.memory_id + WHERE unit_batch.id = ? AND l1_state.state IN ('summary_pending', 'summarizing') + AND batch_memory.deleted_at IS NULL AND batch_memory.status != 'deleted' + LIMIT 1` + ).pluck().get(batchId) as string | undefined; + } + hasUnassignedTurns(sessionId: string): boolean { return Boolean(this.db.prepare(`SELECT 1 FROM raw_turns WHERE session_id = ? AND deleted_at IS NULL AND redacted_at IS NULL AND (episode_id IS NULL OR json_extract(message_payload_json, '$.episode_assignment.status') IN ('pending', 'selected', 'applying')) LIMIT 1`).get(sessionId)); @@ -3726,8 +3754,9 @@ export class RuntimeRepository { : `AND job_type IN (${jobTypes.map(() => "?").join(", ")})` : ""; const excluded = options?.excludedJobTypes?.length ? options.excludedJobTypes : undefined; + // Same rule as leasing: excluding an L3's update jobs also keeps its profile jobs out. const excludeFilter = excluded - ? `AND job_type NOT IN (${excluded.map(() => "?").join(", ")})` + ? `AND job_type NOT IN (${excluded.map(() => "?").join(", ")}) ${l3ProfileHeldSql(excluded)}` : ""; const jobTypeParams = [ ...(jobTypes && jobTypes.length > 0 ? jobTypes : []), @@ -3739,17 +3768,18 @@ export class RuntimeRepository { const retryTargetFilter = targets ? `AND target_id IN (${targets.map(() => "?").join(", ")})` : ""; const future = options?.futureAfterMs; const futureIso = future === undefined ? undefined : new Date(future).toISOString(); - const mainParams = [...jobTypeParams, ...(targets ?? []), ...(futureIso ? [futureIso] : [])]; + // A job its scope still holds back is not a reason to wake, neither by its runAfter nor by its lease running out. + const mainParams = [nowIso(), ...jobTypeParams, ...(targets ?? []), ...(futureIso ? [futureIso] : [])]; const queuedJob = this.db .prepare(`SELECT ${MAIN_DEADLINE} AS run_after FROM evolution_jobs INDEXED BY worker_main_deadline - WHERE ${MAIN_TIMED} ${jobTypeFilter} ${excludeFilter} ${targetFilter} + WHERE ${MAIN_TIMED} AND ${scopedJobReadySql()} ${jobTypeFilter} ${excludeFilter} ${targetFilter} ${futureIso ? `AND ${MAIN_DEADLINE} > ?` : ""} ORDER BY ${MAIN_DEADLINE} ASC LIMIT 1`) .get(...mainParams) as { run_after: string } | undefined; const leasedJob = this.db .prepare(`SELECT leased_until FROM evolution_jobs INDEXED BY worker_main_lease_deadline - WHERE ${MAIN_LEASED} ${jobTypeFilter} ${excludeFilter} ${targetFilter} + WHERE ${MAIN_LEASED} AND ${scopedJobReadySql()} ${jobTypeFilter} ${excludeFilter} ${targetFilter} ${futureIso ? "AND leased_until > ?" : ""} ORDER BY leased_until ASC LIMIT 1`) .get(...mainParams) as { leased_until: string } | undefined; @@ -4032,6 +4062,33 @@ export class RuntimeRepository { return result.changes > 0 ? this.getJob(id) : undefined; } + /** + * Puts a failed job this run still holds back in the queue to run after `runAfter`. A refunded + * attempt means the run did not count as a failure (waiting for a configuration fix); + * `minMaxAttempts` raises the failure limit. Returns undefined when the run lost its lease. + */ + rescheduleFailedJob(id: string, input: { + runAfter: string; + error: string; + refundAttempt?: boolean; + minMaxAttempts?: number; + at?: string; + }, claim?: EvolutionJobRecord): EvolutionJobRecord | undefined { + const at = input.at ?? nowIso(); + const where = this.jobClaimWhere(claim, at); + const result = this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, + lease_owner = NULL, lease_token = NULL, available_at = ?, blocked_reason = NULL, last_error = ?, + failure_count = failure_count + CASE WHEN ? THEN 0 ELSE 1 END, + attempts = CASE WHEN ? THEN MAX(0, attempts - 1) ELSE attempts END, + max_attempts = MAX(max_attempts, ?), updated_at = ?, + payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE id = ? AND ${where.sql}`).run( + input.runAfter, input.error, input.refundAttempt ? 1 : 0, input.refundAttempt ? 1 : 0, + input.minMaxAttempts ?? 0, at, input.runAfter, id, ...where.params + ); + return result.changes > 0 ? this.getJob(id) : undefined; + } + /** Waiting for a dependency is not a model failure and does not consume an attempt. */ deferProcessingJob(id: string, runAfter: string, at = nowIso(), claim?: EvolutionJobRecord): EvolutionJobRecord | undefined { const where = this.jobClaimWhere(claim, at); @@ -4268,6 +4325,14 @@ export class RuntimeRepository { updateL3WorldModelBatchTerminalOutcome(this.db, batchId, at); } } + if (status === "dead_letter" && row.job_type === "l3_unit_update") { + const payload = parseJson>(row.payload_json, {}); + if (typeof payload.batchId === "string") { + this.db.prepare( + `UPDATE l3_unit_batches SET status = 'failed', updated_at = ? WHERE id = ? AND status = 'queued'` + ).run(at, payload.batchId); + } + } if (status === "dead_letter" && row.job_type === "project_environment_profile") { const payload = parseJson>(row.payload_json, {}); const userId = typeof payload.userId === "string" ? payload.userId : undefined; @@ -5369,6 +5434,10 @@ export class RuntimeRepository { }> }; this.db.transaction(() => { + const keptL3Units = this.existingL3Units(tables, conflictStrategy); + // Unfinished work of a kept unit that this database does not have: its Sessions' traces must be + // frozen again here, so their cursors are moved back below the first trace no batch here covers. + const droppedL3Work: Array<{ sessionId: string; startTraceSeq: number }> = []; const hadRuntimeData = Boolean(this.db.prepare(`SELECT EXISTS( SELECT 1 FROM memories UNION ALL SELECT 1 FROM sessions UNION ALL SELECT 1 FROM raw_turns UNION ALL SELECT 1 FROM source_turn_captures @@ -5378,6 +5447,21 @@ export class RuntimeRepository { for (const row of rows) { const defaults = applyBundleDefaults(table, deserializeBundleRow(row)); const normalized = table === "memory_change_log" ? compactHistoryChangeRow(defaults) : defaults; + const l3UnitKey = keptL3Units.size > 0 ? bundleRowL3UnitKey(table, normalized) : null; + if (l3UnitKey && keptL3Units.has(l3UnitKey)) { + if (table === "l3_unit_batches" && normalized.status !== "applied" && typeof normalized.id === "string" && + !this.rowExists("l3_unit_batches", ["id"], [normalized.id]) && + typeof normalized.session_id === "string" && typeof normalized.start_trace_seq === "number" && + typeof normalized.end_trace_seq === "number") { + const unfrozen = this.firstUnfrozenL3Trace(normalized.session_id, normalized.start_trace_seq, normalized.end_trace_seq); + if (unfrozen !== null) droppedL3Work.push({ sessionId: normalized.session_id, startTraceSeq: unfrozen }); + } + if (table === "l3_units") { + result.conflicts.push({ table, primaryKey: "unit_key", sourceId: l3UnitKey, targetId: l3UnitKey, action: "skipped" }); + } + result.skipped[table] = (result.skipped[table] ?? 0) + 1; + continue; + } const identity = bundleIdentity(table, normalized); if (identity) { recordMigrationMap(result.migrationMap, table, identity.sourceId, identity.sourceId); @@ -5435,12 +5519,35 @@ export class RuntimeRepository { } } } + const rewind = this.db.prepare( + `UPDATE l3_world_model_session_cursors SET last_scheduled_seq = MIN(last_scheduled_seq, ?) WHERE session_id = ?` + ); + for (const work of droppedL3Work) rewind.run(work.startTraceSeq - 1, work.sessionId); })(); this.scheduleLogTablesPrune(); this.scheduleHistoryLogCompaction(true); return result; } + /** + * Row-by-row conflict handling cannot keep an L3 unit consistent (an old item could stay active + * next to its imported successor), so a unit that already exists here is kept whole: the bundle's + * rows for it (unit, items, batches, jobs, snapshot) are all skipped, as re-importing one's own + * backup skips every other existing row. With conflictStrategy "error" the import fails instead. + */ + private existingL3Units(tables: Record, conflictStrategy: "skip" | "replace" | "error"): Set { + const rows = Array.isArray(tables.l3_units) ? tables.l3_units as Array> : []; + const exists = this.db.prepare(`SELECT 1 FROM l3_units WHERE unit_key = ?`).pluck(); + const kept = new Set(); + for (const row of rows) { + const unitKey = deserializeBundleRow(row).unit_key; + if (typeof unitKey !== "string" || !exists.get(unitKey)) continue; + if (conflictStrategy === "error") throw new Error(`import conflict: l3 unit already exists: ${unitKey}`); + kept.add(unitKey); + } + return kept; + } + private mergeSourceTurnActivation( row: Record, hadRuntimeData: boolean @@ -5465,6 +5572,28 @@ export class RuntimeRepository { return existing ? "replaced" : "inserted"; } + + /** + * The first trace in [start, end] that no batch of this database froze, or null when all were + * (an earlier import of the same bundle refroze them). A batch counts only if it froze the turn now + * at that position: a replace import can put another turn there, which a batch reading its frozen + * ids would leave out. + */ + private firstUnfrozenL3Trace(sessionId: string, start: number, end: number): number | null { + const covered = this.db.prepare( + `SELECT EXISTS( + SELECT 1 FROM l3_world_model_input_traces t JOIN l3_unit_batches b + ON b.session_id = t.session_id AND b.start_trace_seq <= t.trace_seq AND b.end_trace_seq >= t.trace_seq + WHERE t.session_id = ? AND t.trace_seq = ? + AND EXISTS(SELECT 1 FROM json_each(b.l1_memory_ids_json) WHERE value = t.l1_memory_id) + AND EXISTS(SELECT 1 FROM json_each(b.raw_turn_ids_json) WHERE value = t.raw_turn_id) + )` + ).pluck(); + for (let seq = start; seq <= end; seq += 1) { + if (!covered.get(sessionId, seq)) return seq; + } + return null; + } private rowExists(table: BundleTableName, columns: string[], values: Array): boolean { const tableColumns = this.tableColumns(table); if (columns.length === 0 || columns.some((column) => !tableColumns.includes(column))) { @@ -5567,11 +5696,45 @@ export class L3WorldModelScopeWorkspaceConflictError extends Error { } export class L3WorldModelRepository { + /** When it returns a machine id, frozen batches go to the new L3 (field cognition) instead. */ + private fieldCognitionMachineId?: () => string | null; + constructor( private readonly db: Database.Database, - private readonly memories: MemoryRepository + private readonly memories: MemoryRepository, + private readonly l3Field?: L3FieldRepository ) {} + routeBatchesToFieldCognition(machineId: () => string | null): void { + this.fieldCognitionMachineId = machineId; + } + + /** v2 Sessions holding captured turns that were never frozen into a batch, with their last activity. */ + sessionsWithUnfrozenTurns(): Array<{ sessionId: string; open: boolean; lastActivityAt: string }> { + return (this.db.prepare( + `SELECT s.id AS sessionId, s.status = 'open' AS open, t.last_at AS lastActivityAt FROM sessions AS s + JOIN (SELECT session_id, MAX(trace_seq) AS last_seq, MAX(created_at) AS last_at + FROM l3_world_model_input_traces GROUP BY session_id) AS t + ON t.session_id = s.id + LEFT JOIN l3_world_model_session_cursors AS c ON c.session_id = s.id + WHERE json_extract(s.meta_json, '$.l3_world_model_protocol_version') = 2 + AND t.last_seq > COALESCE(c.last_scheduled_seq, 0)` + ).all() as Array<{ sessionId: string; open: number; lastActivityAt: string }>) + .map((row) => ({ ...row, open: row.open === 1 })); + } + + /** + * New L3 only: hand a closed task's turns to L3. Called wherever a task closes, so a conversation that + * simply stops (idle, or the user ended the topic) still reaches L3; what is already frozen is skipped. + * With the switch off nothing changes. + */ + freezeClosedTask(input: { sessionId: string; episodeId: string; trigger: L3WorldModelBatchTrigger; at?: string }): void { + if (!this.l3Field || !this.fieldCognitionMachineId?.()) return; + const row = this.db.prepare(`SELECT * FROM sessions WHERE id = ?`).get(input.sessionId) as SqlSessionRow | undefined; + if (!row || sessionFromSql(row).meta.l3_world_model_protocol_version !== 2) return; + this.freezeBatches({ sessionId: input.sessionId, trigger: input.trigger, episodeId: input.episodeId, at: input.at }); + } + getScope(userId: string, projectId?: string | null): L3WorldModelScopeRecord | undefined { const row = this.db.prepare( `SELECT * FROM l3_world_model_scopes @@ -5956,13 +6119,20 @@ export class L3WorldModelRepository { } insertImmutableJob(job: EvolutionJobRecord): EvolutionJobRecord { - if (job.jobType !== "l3_world_model_update" && job.jobType !== "project_environment_profile") { + if ( + job.jobType !== "l3_world_model_update" && + job.jobType !== "project_environment_profile" && + job.jobType !== "l3_unit_update" + ) { throw new TypeError("immutable L3 job must use an L3 World Model job type"); } if (!job.dedupeKey) { throw new TypeError("immutable L3 job requires dedupeKey"); } - if (job.jobType === "l3_world_model_update" && (!job.scopeKey || job.scopeSeq === undefined)) { + if ( + (job.jobType === "l3_world_model_update" || job.jobType === "l3_unit_update") && + (!job.scopeKey || job.scopeSeq === undefined) + ) { throw new TypeError("L3 field update job requires scopeKey and scopeSeq"); } if (job.jobType === "project_environment_profile" && (job.scopeKey || job.scopeSeq !== undefined)) { @@ -5995,6 +6165,43 @@ export class L3WorldModelRepository { return job; } + /** One new-L3 batch per chunk, run in order per machine through the l3_unit_update FIFO. */ + private freezeL3UnitBatch( + session: SessionRecord, + machineId: string, + trigger: L3WorldModelBatchTrigger, + chunk: readonly L3WorldModelInputTraceRecord[], + at: string + ): string { + const batch = this.l3Field!.freezeUnitBatch({ + userId: session.userId, + machineId, + sessionId: session.id, + trigger, + startTraceSeq: chunk[0]!.traceSeq, + endTraceSeq: chunk.at(-1)!.traceSeq, + l1MemoryIds: chunk.map((trace) => trace.l1MemoryId), + rawTurnIds: [...new Set(chunk.map((trace) => trace.rawTurnId))], + at + }); + this.insertImmutableJob({ + id: newId("job"), + jobType: "l3_unit_update", + status: "queued", + dedupeKey: `l3_unit_update:${batch.id}`, + userId: session.userId, + sessionId: session.id, + scopeKey: batch.machineUnitKey, + scopeSeq: batch.scopeSeq, + payload: { batchId: batch.id }, + attempts: 0, + maxAttempts: 3, + createdAt: at, + updatedAt: at + }); + return batch.id; + } + private freezeBatchesInTransaction(input: { sessionId: string; trigger: L3WorldModelBatchTrigger; @@ -6003,6 +6210,11 @@ export class L3WorldModelRepository { at?: string; }): FreezeL3WorldModelBatchesResult { const session = this.requireV2Session(input.sessionId); + // Scanned history only ever goes to the new L3. While it is off (switched off after the scan), the + // turns wait without moving the cursor, so they reach the new L3 once it is on again. + if (session.meta.work_memory_disabled === true && !(this.l3Field && this.fieldCognitionMachineId?.())) { + return { scheduled: false, batchIds: [], targetCount: 0 }; + } const at = input.at ?? nowIso(); this.db.prepare( `INSERT INTO l3_world_model_session_cursors (session_id, last_scheduled_seq, updated_at) @@ -6059,6 +6271,21 @@ export class L3WorldModelRepository { } const chunks = splitL3TracesByRawTurn(traces, 20); + const fieldMachineId = this.l3Field ? this.fieldCognitionMachineId?.() ?? null : null; + if (fieldMachineId) { + const batchIds = chunks.map((chunk) => this.freezeL3UnitBatch(session, fieldMachineId, input.trigger, chunk, at)); + this.db.prepare( + `UPDATE l3_world_model_session_cursors + SET last_scheduled_seq = ?, updated_at = ? WHERE session_id = ?` + ).run(endTraceSeq, at, session.id); + return { + scheduled: true, + throughL1MemoryId: traces.at(-1)?.l1MemoryId, + throughTraceSeq: endTraceSeq, + batchIds, + targetCount: batchIds.length + }; + } const scope = this.ensureScope(session.userId, session.projectId, at); const batchIds: string[] = []; let targetCount = 0; @@ -6531,6 +6758,7 @@ export class Repositories { readonly processing: MemoryProcessingRepository; readonly runtime: RuntimeRepository; readonly l3WorldModels: L3WorldModelRepository; + readonly l3Field: L3FieldRepository; readonly projectEnvironments: ProjectEnvironmentRepository; readonly vectors: SqliteVecStore; @@ -6541,7 +6769,8 @@ export class Repositories { this.userMemories = new UserMemoryRepository(db); this.processing = new MemoryProcessingRepository(db); this.runtime = new RuntimeRepository(db); - this.l3WorldModels = new L3WorldModelRepository(db, this.memories); + this.l3Field = new L3FieldRepository(db, this.memories); + this.l3WorldModels = new L3WorldModelRepository(db, this.memories, this.l3Field); this.projectEnvironments = new ProjectEnvironmentRepository(db, this.l3WorldModels, this.runtime); } @@ -8442,6 +8671,31 @@ interface BundleIdentity { values: Array; } +/** The new-L3 unit a bundle row belongs to, for the tables that hold one unit's state. */ +function bundleRowL3UnitKey(table: string, row: Record): string | null { + const record = (value: unknown): Record => { + const parsed = typeof value === "string" ? parseJson(value, {}) : value; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : {}; + }; + const text = (value: unknown) => (typeof value === "string" && value ? value : null); + switch (table) { + case "l3_units": + case "l3_items": + return text(row.unit_key); + case "l3_unit_batches": + return text(row.machine_unit_key); + case "evolution_jobs": { + if (row.job_type === "l3_unit_update") return text(row.scope_key); + const payload = record(row.payload_json); + return row.job_type === "project_environment_profile" && payload.mode === "l3_field" ? text(payload.unitKey) : null; + } + case "memories": + return text(record(record(record(row.properties_json).internal_info).l3_field).unit_key); + default: + return null; + } +} + function bundleIdentity( table: BundleTableName, row: Record @@ -8455,7 +8709,9 @@ function bundleIdentity( l3_world_model_input_traces: ["session_id", "trace_seq"], l3_world_model_evidence_batches: ["id"], l3_world_model_batch_targets: ["batch_id", "target_field"], - l3_world_model_project_environment_state: ["user_id", "project_id"] + l3_world_model_project_environment_state: ["user_id", "project_id"], + l3_units: ["unit_key"], + l3_user_projects: ["user_id", "root_path"] }; const newColumns = newTableIdentityColumns[table]; if (newColumns) { @@ -8548,9 +8804,15 @@ function normalizeRedactedL3WorldModelBundle( .filter((row) => terminalBatchIds.has(String(row.batch_id))); tables.evolution_jobs = (tables.evolution_jobs ?? []).filter((row) => { const jobType = row.job_type; - if (jobType !== "l3_world_model_update" && jobType !== "project_environment_profile") return true; + if ( + jobType !== "l3_world_model_update" && + jobType !== "project_environment_profile" && + jobType !== "l3_unit_update" + ) return true; return row.status === "succeeded" || row.status === "dead_letter"; }); + // Unfinished unit batches depend on raw text that a redacted bundle no longer carries. + tables.l3_unit_batches = (tables.l3_unit_batches ?? []).filter((row) => row.status === "applied"); tables.l3_world_model_project_environment_state = ( tables.l3_world_model_project_environment_state ?? [] ).map((row) => ({ @@ -8608,6 +8870,13 @@ function deserializeBundleRow(row: Record): Record): Record { + if (table === "evolution_jobs" || table === "embedding_retry_queue") { + // A queue position belongs to the database it was taken in: an imported row is numbered on + // insert, after what is already queued here, instead of colliding with (or, on replace, + // displacing) a local row that holds the same number. + const { enqueue_seq: _sourcePosition, ...rest } = row; + return rest; + } if (table === "sessions" && row.last_seen_at === undefined) { return { ...row, @@ -8739,7 +9008,7 @@ function evolutionJobPrioritySql(): string { WHEN job_type = 'l2_association' THEN 40 WHEN job_type = 'l2_induction' THEN 50 WHEN job_type = 'project_environment_profile' THEN 55 - WHEN job_type IN ('l3_abstraction', 'l3_world_model_update') THEN 60 + WHEN job_type IN ('l3_abstraction', 'l3_world_model_update', 'l3_unit_update') THEN 60 WHEN job_type = 'skill_crystallization' THEN 70 WHEN job_type = 'skill_cluster_assign' THEN 72 WHEN job_type = 'skill_batch_evolve' THEN 73 diff --git a/Memory/src/storage/schema.ts b/Memory/src/storage/schema.ts index 429f00f3b..bf33ee929 100644 --- a/Memory/src/storage/schema.ts +++ b/Memory/src/storage/schema.ts @@ -607,6 +607,9 @@ const statements = [ ON evolution_jobs (dedupe_key) WHERE dedupe_key IS NOT NULL AND job_type IN ('l3_world_model_update', 'project_environment_profile')`, + `CREATE UNIQUE INDEX IF NOT EXISTS uq_evolution_jobs_l3_unit_dedupe + ON evolution_jobs (dedupe_key) + WHERE dedupe_key IS NOT NULL AND job_type = 'l3_unit_update'`, `CREATE UNIQUE INDEX IF NOT EXISTS uq_evolution_jobs_scope_seq ON evolution_jobs (scope_key, scope_seq) WHERE scope_key IS NOT NULL`, @@ -701,6 +704,76 @@ const statements = [ sequence INTEGER PRIMARY KEY, event_id TEXT NOT NULL UNIQUE, payload_json TEXT NOT NULL CHECK (json_valid(payload_json)) + )`, + + // L3 field cognition (itemized machine/project units). These tables are added + // without bumping SCHEMA_VERSION: they are new, empty and ignored by older builds. + `CREATE TABLE IF NOT EXISTS l3_units ( + unit_key TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + machine_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('machine', 'project')), + project_identity TEXT, + display_name TEXT, + identity_source TEXT CHECK (identity_source IS NULL OR identity_source IN ('user_selected', 'git', 'folder')), + checkout_root TEXT, + profile_text TEXT, + profile_fingerprint TEXT, + snapshot_memory_id TEXT, + deleted_at TEXT, + next_seq INTEGER NOT NULL DEFAULT 1 CHECK (next_seq >= 1), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + CHECK ((kind = 'machine') = (project_identity IS NULL)) + )`, + `CREATE INDEX IF NOT EXISTS idx_l3_units_user_machine + ON l3_units (user_id, machine_id, kind)`, + `CREATE TABLE IF NOT EXISTS l3_items ( + id TEXT PRIMARY KEY, + unit_key TEXT NOT NULL REFERENCES l3_units(unit_key) ON DELETE CASCADE, + kind TEXT NOT NULL CHECK (kind IN ('env_fact', 'project_constraint')), + content TEXT NOT NULL, + evidence_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(evidence_json)), + evidence_at TEXT NOT NULL, + last_confirmed_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive')), + replaced_by TEXT, + removed_evidence_at TEXT, + inactive_at TEXT, + batch_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + CHECK ((status = 'inactive') = (inactive_at IS NOT NULL)) + )`, + `CREATE INDEX IF NOT EXISTS idx_l3_items_unit_status_kind + ON l3_items (unit_key, status, kind)`, + `CREATE INDEX IF NOT EXISTS idx_l3_items_inactive + ON l3_items (inactive_at) + WHERE status = 'inactive'`, + `CREATE TABLE IF NOT EXISTS l3_unit_batches ( + id TEXT PRIMARY KEY, + machine_unit_key TEXT NOT NULL, + scope_seq INTEGER NOT NULL CHECK (scope_seq >= 1), + user_id TEXT NOT NULL, + session_id TEXT NOT NULL, + trigger TEXT NOT NULL, + start_trace_seq INTEGER NOT NULL, + end_trace_seq INTEGER NOT NULL, + l1_memory_ids_json TEXT NOT NULL CHECK (json_valid(l1_memory_ids_json)), + raw_turn_ids_json TEXT NOT NULL CHECK (json_valid(raw_turn_ids_json)), + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'applied', 'failed')), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (machine_unit_key, scope_seq) + )`, + `CREATE INDEX IF NOT EXISTS idx_l3_unit_batches_session + ON l3_unit_batches (session_id, end_trace_seq)`, + `CREATE TABLE IF NOT EXISTS l3_user_projects ( + user_id TEXT NOT NULL, + root_path TEXT NOT NULL, + name TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (user_id, root_path) )` ]; @@ -784,6 +857,8 @@ export function migrate(db: Database.Database): void { for (const statement of statements) { db.prepare(statement).run(); } + // l3_items created by an earlier build of the new L3 lacks this column. + addColumnIfMissing(db, "l3_items", "removed_evidence_at", "TEXT"); if (version > 0 && version < 5) { addColumnIfMissing(db, "recall_events", "query_id", "TEXT"); addColumnIfMissing(db, "recall_events", "user_memory_candidate_ids_json", "TEXT NOT NULL DEFAULT '[]'"); diff --git a/Memory/src/types.ts b/Memory/src/types.ts index 086b3fadf..70fcd80bb 100644 --- a/Memory/src/types.ts +++ b/Memory/src/types.ts @@ -84,6 +84,7 @@ export type JobType = | "l2_induction" | "l3_abstraction" | "l3_world_model_update" + | "l3_unit_update" | "project_environment_profile" | "skill_crystallization" | "skill_cluster_assign" diff --git a/Memory/tests/config.test.ts b/Memory/tests/config.test.ts index 0991b5454..6a4fee472 100644 --- a/Memory/tests/config.test.ts +++ b/Memory/tests/config.test.ts @@ -21,6 +21,19 @@ afterEach(() => { }); describe("memmy memory config", () => { + it("turns the new L3 on by default, and off from the config file or MEMMY_L3_FIELD_COGNITION=0", () => { + const root = tempRoot(), configPath = join(root, "config.yaml"); + envBackup.MEMMY_L3_FIELD_COGNITION = process.env.MEMMY_L3_FIELD_COGNITION; + delete process.env.MEMMY_L3_FIELD_COGNITION; + writeFileSync(configPath, YAML.stringify({ memmyMemory: {} })); + expect(loadMemmyConfig(configPath).config.algorithm.l3FieldCognition.enabled).toBe(true); + writeFileSync(configPath, YAML.stringify({ memmyMemory: { algorithm: { l3FieldCognition: { enabled: false } } } })); + expect(loadMemmyConfig(configPath).config.algorithm.l3FieldCognition.enabled).toBe(false); + writeFileSync(configPath, YAML.stringify({ memmyMemory: {} })); + process.env.MEMMY_L3_FIELD_COGNITION = "0"; + expect(loadMemmyConfig(configPath).config.algorithm.l3FieldCognition.enabled).toBe(false); + }); + it.each([ ["ja-jp", "ja-JP"], ["fr-fr", "fr-FR"], ["zh-hant-tw", "zh-Hant-TW"], ["en-gb", "en-GB"], ["sr-latn-rs", "sr-Latn-RS"], ["en-us-u-ca-gregory", "en-US-u-ca-gregory"], diff --git a/Memory/tests/contract/l3-field-user-projects.test.ts b/Memory/tests/contract/l3-field-user-projects.test.ts new file mode 100644 index 000000000..5a687e764 --- /dev/null +++ b/Memory/tests/contract/l3-field-user-projects.test.ts @@ -0,0 +1,71 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { createMemoryHttpServer } from "../../src/index.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); + +async function withServer(run: (post: (body: unknown) => Promise, repos: Repositories) => Promise): Promise { + const { service, db } = createTestService(); + const server = createMemoryHttpServer({ + service, + startAgentSourceAutomation: false, + auth: { mode: "local", allowAnonymous: true } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const post = (body: unknown) => fetch(`http://127.0.0.1:${address.port}/api/v1/l3/user-projects`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }); + try { + return await run(post, new Repositories(db.db)); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +} + +const envelope = () => ({ + requestId: randomUUID(), + adapterId: "memmy-agent", + source: "memmy-agent", + namespace: { source: "memmy-agent", profileId: "default", userId: "desk-user" } +}); + +describe("L3 user projects endpoint", () => { + it("replaces the whole list and the standalone roots for the user", async () => { + await withServer(async (post, repos) => { + const first = await post({ ...envelope(), projects: [ + { rootPath: "/work/a", name: "A" }, { rootPath: "/work/b", name: "B" } + ], standaloneRoots: ["/Users/me/Memmy"] }); + expect(first.status).toBe(200); + expect(await first.json()).toMatchObject({ count: 2 }); + const second = await post({ ...envelope(), projects: [{ rootPath: "/work/b", name: "B2" }] }); + expect(second.status).toBe(200); + expect(repos.l3Field.listUserProjects("desk-user")).toEqual([{ rootPath: "/work/b", name: "B2" }]); + expect(repos.l3Field.getStandaloneRoots("desk-user")).toEqual([]); + }); + }); + + it("rejects relative project paths", async () => { + await withServer(async (post, repos) => { + const response = await post({ ...envelope(), projects: [{ rootPath: "work/a", name: "A" }] }); + expect(response.status).toBe(400); + expect(repos.l3Field.listUserProjects("desk-user")).toEqual([]); + }); + }); +}); + +describe("L3 user projects contract copies", () => { + it("keeps the Memory and App contract files identical", () => { + const memoryCopy = fileURLToPath(new URL("../../src/contracts/memory-l3-world-model.ts", import.meta.url)); + const appCopy = fileURLToPath(new URL("../../../App/backend/local-api-contracts/src/memory-l3-world-model.ts", import.meta.url)); + expect(readFileSync(appCopy, "utf8")).toBe(readFileSync(memoryCopy, "utf8")); + }); +}); diff --git a/Memory/tests/contract/memory-rest-service.test.ts b/Memory/tests/contract/memory-rest-service.test.ts index 594a56deb..5e172d473 100644 --- a/Memory/tests/contract/memory-rest-service.test.ts +++ b/Memory/tests/contract/memory-rest-service.test.ts @@ -742,7 +742,8 @@ describe("MemoryService / REST contract", () => { }); it("auto-drains L3 World Model jobs frozen by a REST session.close with no open episode", async () => { - const { db, service } = createTestService(); + // L3 jobs wait in the queue while no evolution model is configured, so configure one. + const { db, service } = createTestService({ skillLlm: createDeterministicSummaryLlm("unused") }); const opened = service.openSession({ l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", @@ -801,7 +802,8 @@ describe("MemoryService / REST contract", () => { }); expect(l3Jobs()).toHaveLength(1); - await waitFor(() => l3Jobs().every((job) => job.status !== "queued" && job.attempts > 0)); + // The job ran once; a failed L3 run goes back to the queue to retry later. + await waitFor(() => l3Jobs().every((job) => job.attempts > 0 && job.status !== "leased")); }); db.close(); diff --git a/Memory/tests/contract/source-turn-http.test.ts b/Memory/tests/contract/source-turn-http.test.ts index 0864a21ea..8ad147012 100644 --- a/Memory/tests/contract/source-turn-http.test.ts +++ b/Memory/tests/contract/source-turn-http.test.ts @@ -1,11 +1,12 @@ import { LEGACY_TEST_CONFIG } from "../fixtures/memory-service-fixture.js"; import { afterEach, describe, expect, it } from "vitest"; import { buildSourceTurnRequest } from "@memmy/agent-source-core"; -import { createMemoryHttpServer } from "../../src/index.js"; +import { join } from "node:path"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb, createMemoryHttpServer } from "../../src/index.js"; import { Repositories } from "../../src/storage/repositories.js"; import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; -const { cleanup, createTestService } = createMemoryServiceFixture(); +const { cleanup, createTestService, createTestMemoryService, createTestRoot } = createMemoryServiceFixture(); afterEach(cleanup); describe("source turn HTTP contract", () => { @@ -41,6 +42,41 @@ describe("source turn HTTP contract", () => { } finally { await new Promise((resolve) => server.close(() => resolve())); } }); + it("freezes scanned Sessions through the request the desktop backend sends", async () => { + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = true; + const db = new MemoryDb({ path: join(createTestRoot(), "memory.sqlite") }); + const service = createTestMemoryService({ + db, + mode: "dev", + config, + l3FieldEnvironment: () => ({ machineId: "6".repeat(64), homeDir: "/nonexistent-home", tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const server = createMemoryHttpServer({ service, startAgentSourceAutomation: false, auth: { mode: "local", allowAnonymous: true } }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const post = (path: string, body: unknown) => fetch(`http://127.0.0.1:${address.port}${path}`, { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) + }); + try { + const turn = await post("/api/v1/source-turns/complete", buildSourceTurnRequest({ source: "codex", conversationId: "scanned-conversation", + turnId: "scanned-turn", startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", sequence: 1, + completionEvidence: "final_answer:scanned-turn", query: "Rename the export helper.", answer: "Renamed.", status: "succeeded", + toolCalls: [], toolResults: [] }, "agent_source_scan")); + expect(turn.status).toBe(200); + const { result } = await turn.json() as { result: { sessionId: string } }; + const frozen = await post("/api/v1/l3/scan-sessions/freeze", { + requestId: crypto.randomUUID(), adapterId: "agent-source:codex", source: "codex", sessionIds: [result.sessionId] + }); + expect(frozen.status).toBe(200); + // The worker's startup pass may have handed the session over first; either way it is frozen once. + expect(await frozen.json()).toMatchObject({ batches: expect.any(Number) }); + expect(db.db.prepare(`SELECT COUNT(*) AS n FROM l3_unit_batches WHERE session_id = ?`).get(result.sessionId)).toEqual({ n: 1 }); + } finally { await new Promise((resolve) => server.close(() => resolve())); } + }); + it.each(["hook", "agent_source_scan", "concurrent"] as const)("reuses the installed Hook owner after account switch when %s arrives first", async (channel) => { const { service, db } = createTestService({ config: { ...LEGACY_TEST_CONFIG, userId: "configured-owner" } }); const opened = service.openSession({ diff --git a/Memory/tests/fixtures/memory-service-fixture.ts b/Memory/tests/fixtures/memory-service-fixture.ts index 6e25e36d7..cc4fd8878 100644 --- a/Memory/tests/fixtures/memory-service-fixture.ts +++ b/Memory/tests/fixtures/memory-service-fixture.ts @@ -12,8 +12,10 @@ import type { ActualModelContext } from "../../src/contracts/index.js"; // Suites for legacy capture/retrieval behavior keep that mode explicit as the // production default changes. Enabled/default-on suites use production config. +// That includes the old L3 World Model: the new L3 replaces it when on. export const LEGACY_TEST_CONFIG = { ...DEFAULT_MEMMY_CONFIG, algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, - session: { ...DEFAULT_MEMMY_CONFIG.algorithm.session, turnJudge: false } } }; + session: { ...DEFAULT_MEMMY_CONFIG.algorithm.session, turnJudge: false }, + l3FieldCognition: { ...DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition, enabled: false } } }; export function createMemoryServiceFixture(): { cleanup: () => Promise; @@ -57,6 +59,12 @@ export function createMemoryServiceFixture(): { // Legacy fixture suites opt out explicitly; turn-judge suites pass their // enabled configuration, and default-on coverage passes DEFAULT_MEMMY_CONFIG. config: options.config ?? LEGACY_TEST_CONFIG, + // The new L3 must not probe the real computer or read its installation id from tests; + // the probe never settles, so it starts no process and writes nothing. + l3FieldEnvironment: options.l3FieldEnvironment ?? (() => ({ + machineId: "f".repeat(64), homeDir: createTestRoot("memmy-l3-home-"), tempRoots: [] + })), + l3MachineProbe: options.l3MachineProbe ?? (() => new Promise(() => undefined)), fetchAppMemoryBudget: options.fetchAppMemoryBudget ?? (async () => null), skillLlm: options.skillLlm ?? options.llm, embedder: options.embedder ?? createCapturingEmbedder([]) diff --git a/Memory/tests/repository/l3-field-repository.test.ts b/Memory/tests/repository/l3-field-repository.test.ts new file mode 100644 index 000000000..188a74363 --- /dev/null +++ b/Memory/tests/repository/l3-field-repository.test.ts @@ -0,0 +1,266 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import YAML from "yaml"; +import { MemoryDb, SCHEMA_VERSION } from "../../src/index.js"; +import { DEFAULT_MEMMY_CONFIG, loadMemmyConfig } from "../../src/config/index.js"; +import { isL3FieldCognitionActive } from "../../src/service/l3-field/flag.js"; +import type { ResolvedProject } from "../../src/service/l3-field/types.js"; +import { Repositories } from "../../src/storage/repositories.js"; + +const NEW_TABLES = ["l3_units", "l3_items", "l3_unit_batches", "l3_user_projects"]; +const USER = "l3-field-user"; +const MACHINE = "a".repeat(64); +const PROJECT: ResolvedProject = { + identity: "github.com/example/repo", + checkoutRoot: "/work/repo", + source: "git", + displayName: "repo" +}; + +const opened: MemoryDb[] = []; +const folders: string[] = []; + +function openRepos(): { db: MemoryDb; repos: Repositories } { + const db = new MemoryDb({ path: ":memory:" }); + opened.push(db); + return { db, repos: new Repositories(db.db) }; +} + +afterEach(() => { + for (const db of opened.splice(0)) db.close(); + for (const folder of folders.splice(0)) rmSync(folder, { recursive: true, force: true }); +}); + +function tableNames(db: MemoryDb): string[] { + return (db.db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table'`).pluck().all() as string[]); +} + +describe("L3 field cognition storage", () => { + it("creates the new tables without changing the schema version, keeping the old L3 tables", () => { + const { db } = openRepos(); + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + const names = tableNames(db); + for (const table of NEW_TABLES) expect(names).toContain(table); + expect(names).toContain("l3_world_model_scopes"); + }); + + it("reopens an existing database without touching old L3 rows", () => { + const folder = mkdtempSync(join(tmpdir(), "l3-field-reopen-")); + folders.push(folder); + const path = join(folder, "memory.sqlite"); + const first = new MemoryDb({ path }); + first.db.exec(`DROP TABLE l3_user_projects; DROP TABLE l3_items; DROP TABLE l3_unit_batches; DROP TABLE l3_units;`); + first.db.prepare( + `INSERT INTO l3_world_model_scopes (scope_key, user_id, project_id, memory_id, next_scope_seq, updated_at) + VALUES ('legacy-scope', ?, NULL, NULL, 3, '2026-10-01T00:00:00.000Z')` + ).run(USER); + first.close(); + + const reopened = new MemoryDb({ path }); + opened.push(reopened); + expect(reopened.schemaVersion().version).toBe(SCHEMA_VERSION); + for (const table of NEW_TABLES) expect(tableNames(reopened)).toContain(table); + expect(reopened.db.prepare(`SELECT next_scope_seq FROM l3_world_model_scopes WHERE scope_key = 'legacy-scope'`).get()) + .toEqual({ next_scope_seq: 3 }); + }); + + it("keys machine and project units by user, machine and identity", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + expect(repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }).unitKey).toBe(machine.unitKey); + expect(repos.l3Field.ensureMachineUnit({ userId: "other-user", machineId: MACHINE }).unitKey).not.toBe(machine.unitKey); + const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + expect(project.kind).toBe("project"); + expect(project.projectIdentity).toBe(PROJECT.identity); + expect(project.unitKey).not.toBe(machine.unitKey); + expect(repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }).unitKey) + .toBe(project.unitKey); + }); + + it("refreshes a unit snapshot only when it changed, and bumps project snapshots when the machine unit changes", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + repos.l3Field.insertItem({ + unitKey: project.unitKey, + kind: "project_constraint", + content: "报告以 GitHub 上的 main 为准", + evidence: { fragmentIds: ["f1"], l1MemoryIds: ["l1"] }, + evidenceAt: "2026-10-20T00:00:00.000Z" + }); + const first = repos.l3Field.refreshSnapshot(project.unitKey); + expect(first.memoryLayer).toBe("L3"); + expect(first.properties.internal_info.schema_version).toBe(3); + expect(first.memoryValue).toContain("报告以 GitHub 上的 main 为准"); + const again = repos.l3Field.refreshSnapshot(project.unitKey); + expect(again.id).toBe(first.id); + expect(again.version).toBe(first.version); + repos.l3Field.refreshSnapshot(machine.unitKey); + const machineVersion = repos.memories.get(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId!)!.version; + // A machine that did not change leaves the project snapshots alone. + repos.l3Field.refreshSnapshot(machine.unitKey); + expect(repos.memories.get(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId!)!.version).toBe(machineVersion); + expect(repos.memories.get(first.id)!.version).toBe(again.version + 1); + + repos.l3Field.insertItem({ + unitKey: machine.unitKey, + kind: "env_fact", + content: "这台电脑的 zsh 会展开 = 开头的参数", + evidence: { fragmentIds: ["f2"], l1MemoryIds: ["l2"] }, + evidenceAt: "2026-10-21T00:00:00.000Z" + }); + repos.l3Field.refreshSnapshot(machine.unitKey); + expect(repos.memories.get(first.id)!.version).toBe(again.version + 2); + }); + + it("replaces items by marking the old one inactive and pointing at its successor", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const old = repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content: "旧事实", + evidence: { fragmentIds: [], l1MemoryIds: ["l1"] }, evidenceAt: "2026-10-01T00:00:00.000Z" + }); + const next = repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content: "新事实", + evidence: { fragmentIds: [], l1MemoryIds: ["l2"] }, evidenceAt: "2026-10-02T00:00:00.000Z" + }); + repos.l3Field.inactivateItem(old.id, { replacedBy: next.id, at: "2026-10-02T00:00:00.000Z" }); + expect(repos.l3Field.activeItems(machine.unitKey).map((item) => item.id)).toEqual([next.id]); + expect(repos.l3Field.getItem(old.id)).toMatchObject({ status: "inactive", replacedBy: next.id }); + }); + + it("purges only items that have been inactive longer than the cutoff", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const add = (content: string) => repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content, + evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" + }); + const keep = add("仍然有效"); + const oldInactive = add("失效 16 天"); + const recentInactive = add("失效 14 天"); + repos.l3Field.inactivateItem(oldInactive.id, { at: "2026-10-08T00:00:00.000Z" }); + repos.l3Field.inactivateItem(recentInactive.id, { at: "2026-10-10T00:00:00.000Z" }); + expect(repos.l3Field.purgeInactive("2026-10-09T00:00:00.000Z")).toBe(1); + expect(repos.l3Field.getItem(oldInactive.id)).toBeUndefined(); + expect(repos.l3Field.getItem(recentInactive.id)?.status).toBe("inactive"); + expect(repos.l3Field.getItem(keep.id)?.status).toBe("active"); + }); + + it("marks a deleted unit and inactivates its items", () => { + const { repos } = openRepos(); + const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + repos.l3Field.insertItem({ + unitKey: project.unitKey, kind: "project_constraint", content: "规矩", + evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-10-01T00:00:00.000Z" + }); + repos.l3Field.markUnitDeleted(project.unitKey, "2026-10-05T00:00:00.000Z"); + expect(repos.l3Field.getUnit(project.unitKey)?.deletedAt).toBe("2026-10-05T00:00:00.000Z"); + expect(repos.l3Field.activeItems(project.unitKey)).toEqual([]); + }); + + it("replaces the whole user project list", () => { + const { repos } = openRepos(); + repos.l3Field.replaceUserProjects(USER, [{ rootPath: "/a", name: "A" }, { rootPath: "/b", name: "B" }]); + repos.l3Field.replaceUserProjects(USER, [{ rootPath: "/b", name: "B2" }]); + expect(repos.l3Field.listUserProjects(USER)).toEqual([{ rootPath: "/b", name: "B2" }]); + repos.l3Field.replaceUserProjects(USER, []); + expect(repos.l3Field.listUserProjects(USER)).toEqual([]); + }); + + it("round-trips the new tables through export and import into an empty database", () => { + const source = openRepos().repos; + const project = source.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + source.l3Field.insertItem({ + unitKey: project.unitKey, kind: "project_constraint", content: "规矩", + evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-10-01T00:00:00.000Z" + }); + source.l3Field.replaceUserProjects(USER, [{ rootPath: "/work/repo", name: "repo" }]); + const tables = source.runtime.exportBundleTables(true); + + const target = openRepos().repos; + target.runtime.importBundleTables(tables); + expect(target.l3Field.getUnit(project.unitKey)?.projectIdentity).toBe(PROJECT.identity); + expect(target.l3Field.activeItems(project.unitKey).map((item) => item.content)).toEqual(["规矩"]); + expect(target.l3Field.listUserProjects(USER)).toEqual([{ rootPath: "/work/repo", name: "repo" }]); + }); + + it("keeps a unit that already exists whole when importing, and fails only when asked to on conflict", () => { + const source = openRepos().repos; + const sourceMachine = source.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + source.l3Field.insertItem({ unitKey: sourceMachine.unitKey, kind: "env_fact", content: "旧备份里的事实", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-10-01T00:00:00.000Z" }); + source.l3Field.refreshSnapshot(sourceMachine.unitKey); + source.l3Field.replaceUserProjects(USER, [{ rootPath: "/work/repo", name: "repo" }]); + const tables = source.runtime.exportBundleTables(true); + + const target = openRepos().repos; + const machine = target.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + target.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机现在的事实", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-10-02T00:00:00.000Z" }); + const snapshot = target.l3Field.refreshSnapshot(machine.unitKey); + expect(() => target.runtime.importBundleTables(tables, { conflictStrategy: "error" })).toThrow(/l3 unit already exists/); + + const imported = target.runtime.importBundleTables(tables); + expect(imported.conflicts).toContainEqual(expect.objectContaining({ table: "l3_units", action: "skipped" })); + expect(target.l3Field.activeItems(machine.unitKey).map((item) => item.content)).toEqual(["本机现在的事实"]); + expect(target.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId).toBe(snapshot.id); + const snapshots = target.db.prepare( + `SELECT COUNT(*) AS n FROM memories WHERE json_extract(properties_json, '$.internal_info.l3_field.unit_key') = ?` + ).get(machine.unitKey) as { n: number }; + expect(snapshots.n).toBe(1); + expect(target.l3Field.listUserProjects(USER)).toEqual([{ rootPath: "/work/repo", name: "repo" }]); + }); + + it("drops unfinished unit batches and jobs from a redacted export", () => { + const { repos, db } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const insertBatch = (id: string, status: string) => db.db.prepare( + `INSERT INTO l3_unit_batches (id, machine_unit_key, scope_seq, user_id, session_id, trigger, + start_trace_seq, end_trace_seq, l1_memory_ids_json, raw_turn_ids_json, status, created_at, updated_at) + VALUES (?, ?, ?, ?, 's', 'session_close', 1, 1, '[]', '[]', ?, '2026-10-01T00:00:00.000Z', '2026-10-01T00:00:00.000Z')` + ).run(id, machine.unitKey, id === "done" ? 1 : 2, USER, status); + insertBatch("done", "applied"); + insertBatch("pending", "queued"); + const redacted = repos.runtime.exportBundleTables(false); + expect((redacted.l3_unit_batches ?? []).map((row) => row.id)).toEqual(["done"]); + }); + + it("empties the new tables when all memory data is cleared", () => { + const { repos, db } = openRepos(); + repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + repos.l3Field.replaceUserProjects(USER, [{ rootPath: "/a", name: "A" }]); + repos.clearAllMemoryData(); + for (const table of NEW_TABLES) { + expect(db.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 }); + } + }); +}); + +describe("L3 field cognition flag", () => { + it("is on by default with a 15-day retention, on local sqlite storage only", () => { + expect(DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition).toEqual({ enabled: true, inactiveRetentionDays: 15 }); + expect(isL3FieldCognitionActive(DEFAULT_MEMMY_CONFIG, { backendId: "sqlite-local" }, "local")).toBe(true); + expect(isL3FieldCognitionActive(DEFAULT_MEMMY_CONFIG, { backendId: "sqlite-local" }, "cloud")).toBe(false); + }); + + it("is active only when enabled on local sqlite storage", () => { + const enabled = { + ...DEFAULT_MEMMY_CONFIG, + algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, l3FieldCognition: { enabled: true, inactiveRetentionDays: 15 } } + }; + expect(isL3FieldCognitionActive(enabled, { backendId: "sqlite-local" }, "local")).toBe(true); + expect(isL3FieldCognitionActive(enabled, { backendId: "sqlite-local" }, "cloud")).toBe(false); + expect(isL3FieldCognitionActive(enabled, { backendId: "openmem-cloud-rest" }, "local")).toBe(false); + }); + + it("reads the switch from memmyMemory.algorithm in the config file", () => { + const folder = mkdtempSync(join(tmpdir(), "l3-field-config-")); + folders.push(folder); + const file = join(folder, "config.yaml"); + writeFileSync(file, YAML.stringify({ memmyMemory: { algorithm: { l3FieldCognition: { enabled: true } } } })); + expect(loadMemmyConfig(file).config.algorithm.l3FieldCognition).toEqual({ enabled: true, inactiveRetentionDays: 15 }); + }); +}); diff --git a/Memory/tests/service/bundle/bundle.test.ts b/Memory/tests/service/bundle/bundle.test.ts index 82f6493dd..132140109 100644 --- a/Memory/tests/service/bundle/bundle.test.ts +++ b/Memory/tests/service/bundle/bundle.test.ts @@ -79,6 +79,29 @@ describe("MemoryService / bundle", () => { ).get(opened.sessionId)).toEqual({ last_scheduled_seq: 2 }); }); + it("imports queued jobs into a database that already has its own, keeping every local job", async () => { + const turn = async (service: ReturnType["service"], key: string) => { + const opened = service.openSession({ namespace: { source: "codex", profileId: "default", sessionKey: key, userId: "u1" } }); + await service.completeTurn(`${key}-turn`, { sessionId: opened.sessionId, query: "hello", answer: "hi", status: "succeeded" }); + }; + const source = createTestService(); + await turn(source.service, "source"); + const bundle = source.service.exportBundle({ includeRawText: true }); + const imported = (bundle.tables.evolution_jobs as Array>).map((row) => row.id); + expect(imported.length).toBeGreaterThan(0); + + for (const conflictStrategy of ["skip", "replace"] as const) { + const target = createTestService(); + await turn(target.service, `target-${conflictStrategy}`); + const jobs = () => target.db.db.prepare(`SELECT id, enqueue_seq FROM evolution_jobs`).all() as Array<{ id: string; enqueue_seq: number }>; + const local = jobs().map((job) => job.id); + expect(target.service.importBundle({ bundle, conflictStrategy }).ok).toBe(true); + const after = jobs(); + expect(after.map((job) => job.id).sort()).toEqual([...local, ...imported].sort()); + expect(new Set(after.map((job) => job.enqueue_seq)).size).toBe(after.length); + } + }); + it("exports bundles across namespaces", async () => { const { db, service } = createTestService(); const namespaceA = { diff --git a/Memory/tests/service/evolution/l3-world-model.test.ts b/Memory/tests/service/evolution/l3-world-model.test.ts index f69af61d4..179f5be2d 100644 --- a/Memory/tests/service/evolution/l3-world-model.test.ts +++ b/Memory/tests/service/evolution/l3-world-model.test.ts @@ -165,6 +165,20 @@ describe("L3 World Model trace field pipeline", () => { db.close(); }); + it("accepts a job that the queue rescheduled with runAfter", async () => { + const llm = fieldLlm(); + const { db, service } = createTestService({ skillLlm: llm }); + const opened = openProject(service, "l3-run-after-user", "l3-run-after-session"); + await service.completeTurn("l3-run-after-turn", { + sessionId: opened.sessionId, query: "这个项目必须先运行测试", answer: "好", status: "succeeded" + }); + service.closeSession(opened.sessionId); + const repos = new Repositories(db.db); + const job = repos.runtime.listJobs("queued", 100).find((candidate) => candidate.jobType === "l3_world_model_update")!; + await expect(new L3WorldModelTraceFieldPipeline({ repos, skillLlm: llm }) + .updateField({ ...job, payload: { ...job.payload, runAfter: "2026-10-24T00:00:00.000Z" } })).resolves.toBeUndefined(); + }); + it("removes reasoning from no-project generation and repair input", async () => { const complete = vi.fn() .mockResolvedValueOnce("{}") diff --git a/Memory/tests/service/l3-field/ended-task-l3.test.ts b/Memory/tests/service/l3-field/ended-task-l3.test.ts new file mode 100644 index 000000000..fd3c37995 --- /dev/null +++ b/Memory/tests/service/l3-field/ended-task-l3.test.ts @@ -0,0 +1,240 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { createMemoryServiceFixture, setRawTurnActivityAt, settleCaptureAssignments } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +function setup(enabled = true) { + const root = fixture.createTestRoot("ended-task-l3-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + // Without a model here, a task is ended by the user's explicit command rather than turn.judge.v1. + config.algorithm.session.turnJudge = false; + // What a later reloadConfig() reads, to turn the switch without a restart. + const reload = { config }; + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, configLoader: () => ({ config: reload.config }), + l3FieldEnvironment: () => ({ machineId: "7".repeat(64), homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const open = (key: string) => service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, workspaceHostId: "c".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: key, userId: "u1" } + }).sessionId; + const batches = (sessionId: string) => db.db.prepare( + `SELECT trigger FROM l3_unit_batches WHERE session_id = ? ORDER BY scope_seq` + ).all(sessionId).map((row) => (row as { trigger: string }).trigger); + const legacyBatches = (sessionId: string) => (db.db.prepare( + `SELECT COUNT(*) AS n FROM l3_world_model_evidence_batches WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; + const flush = (sessionId: string) => db.db.prepare( + `SELECT status, json_extract(payload_json, '$.runAfter') AS run_after FROM evolution_jobs + WHERE job_type = 'work_memory_idle_flush' AND session_id = ?` + ).get(sessionId) as { status: string; run_after: string } | undefined; + /** Turn the new L3 on or off as a config reload does, without restarting the service. */ + const switchTo = (on: boolean) => { + reload.config = structuredClone(config); + reload.config.algorithm.l3FieldCognition.enabled = on; + service.reloadConfig({ restartFailedProcessing: false }); + }; + return { db, service, open, batches, legacyBatches, flush, switchTo }; +} + +/** A conversation that goes quiet for over two hours, then a turn somewhere else runs the idle sweep. */ +async function goIdle(env: ReturnType) { + const quiet = env.open("quiet"); + const turn = await env.service.completeTurn("quiet-turn", { + sessionId: quiet, query: "以后改完都跑 npm test", answer: "好", status: "succeeded" + }); + await env.service.runWorkerOnce(20); + setRawTurnActivityAt(env.db, turn.rawTurnId, new Date(Date.now() - 2 * 60 * 60 * 1000 - 5_000).toISOString()); + const other = env.open("other"); + await env.service.startTurn({ turnId: "other-turn", sessionId: other, query: "Start a separate task" }); + await env.service.completeTurn("other-turn", { sessionId: other, query: "Start a separate task", answer: "Started." }); + await env.service.runWorkerOnce(20); + return quiet; +} + +describe("a task that ends without a session close still reaches the new L3", () => { + it("hands a conversation that went idle to L3 when the idle sweep closes its task", async () => { + const env = setup(); + const quiet = await goIdle(env); + expect(env.batches(quiet)).toEqual(["episode_idle_close"]); + }); + + it("hands a task to L3 when the user ends the topic", async () => { + const env = setup(); + const session = env.open("ended"); + await env.service.completeTurn("work-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(env.service); + await env.service.completeTurn("end-turn", { sessionId: session, query: "结束会话", answer: "好的。" }); + await settleCaptureAssignments(env.service); + expect(env.batches(session)).toEqual(["new_task"]); + }); + + it("does not cut into the task in progress when a closed task is finalized again later", async () => { + const env = setup(); + const session = env.open("ongoing"); + const ended = await env.service.completeTurn("old-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(env.service); + await env.service.completeTurn("end-turn", { sessionId: session, query: "结束会话", answer: "好的。" }); + await settleCaptureAssignments(env.service); + await env.service.completeTurn("new-turn", { sessionId: session, query: "Start the next task", answer: "Started." }); + await settleCaptureAssignments(env.service); + const handlers = (env.service as unknown as { workerHandlers: { finalizeClosedEpisode: (e: unknown, at: string, t: string) => unknown } }) + .workerHandlers; + const closed = (env.service as unknown as { repos: { runtime: { getEpisode(id: string): unknown } } }).repos.runtime.getEpisode(ended.episodeId!); + handlers.finalizeClosedEpisode(closed, new Date().toISOString(), "episode_rewarded"); + expect(env.batches(session)).toEqual(["new_task"]); + }); + + it("hands a conversation that just stops, with no close and no later turn, to L3 once it has been quiet", async () => { + const env = setup(); + const session = env.open("stopped"); + await env.service.completeTurn("stopped-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(env.service); + expect(env.flush(session)).toMatchObject({ status: "queued" }); + expect(env.batches(session)).toEqual([]); + // Two hours later with nothing else happening anywhere, the session's own flush comes due. + const quietSince = new Date(Date.now() - 2 * 60 * 60 * 1000 - 5_000).toISOString(); + env.db.db.prepare(`UPDATE l3_world_model_input_traces SET created_at = ? WHERE session_id = ?`).run(quietSince, session); + env.db.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(quietSince); + await env.service.runWorkerOnce(50); + expect(env.batches(session)).toEqual(["session_idle"]); + }); + + it("keeps the flush queued when it comes due before the session has been quiet long enough", async () => { + const env = setup(); + const session = env.open("early"); + await env.service.completeTurn("early-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(env.service); + // Due now, but the last turn is recent (as when a turn lands after the flush was leased): it re-arms itself. + env.db.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(new Date(Date.now() - 1_000).toISOString()); + await env.service.runWorkerOnce(50); + const flush = env.flush(session)!; + expect(flush.status).toBe("queued"); + expect(Date.parse(flush.run_after)).toBeGreaterThan(Date.now()); + expect(env.batches(session)).toEqual([]); + }); + + it("hands over turns whose flush ran while the switch was off once it is turned on without a restart", async () => { + const env = setup(); + const session = env.open("toggled"); + await env.service.completeTurn("toggled-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(env.service); + const quietSince = new Date(Date.now() - 2 * 60 * 60 * 1000 - 5_000).toISOString(); + env.db.db.prepare(`UPDATE l3_world_model_input_traces SET created_at = ? WHERE session_id = ?`).run(quietSince, session); + env.db.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(quietSince); + env.switchTo(false); + await env.service.runWorkerOnce(50); + expect(env.flush(session)!.status).toBe("succeeded"); + expect(env.batches(session)).toEqual([]); + env.switchTo(true); + expect(env.flush(session)!.status).toBe("queued"); + await env.service.runWorkerOnce(50); + expect(env.batches(session)).toEqual(["session_idle"]); + }); + + it("re-freezes imported turns whose unfinished batch an import had to drop for a machine kept here", async () => { + const source = setup(); + const session = source.open("exported"); + await source.service.completeTurn("exported-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(source.service); + source.service.closeSession(session); + expect(source.batches(session)).toEqual(["session_close"]); + const bundle = source.service.exportBundle({ includeRawText: true }); + + const target = setup(); + target.open("local"); + await target.service.completeTurn("local-turn", { sessionId: target.open("local"), query: "看看", answer: "好" }); + await settleCaptureAssignments(target.service); + expect(target.service.importBundle({ bundle }).ok).toBe(true); + // The exported batch was dropped with the machine unit kept here; the closed session's turns are + // frozen afresh into this database's own queue. + expect(target.batches(session)).toEqual(["session_close"]); + const job = target.db.db.prepare( + `SELECT j.status FROM evolution_jobs j JOIN l3_unit_batches b ON b.id = json_extract(j.payload_json, '$.batchId') + WHERE b.session_id = ?` + ).get(session) as { status: string }; + expect(job.status).toBe("queued"); + // Importing the same bundle again finds those turns already frozen here and leaves them alone. + expect(target.service.importBundle({ bundle }).ok).toBe(true); + expect(target.batches(session)).toEqual(["session_close"]); + }); + + it("re-freezes the turns a replace import adds after what this database already froze", async () => { + const source = setup(); + const session = source.open("growing"); + await source.service.completeTurn("first-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(source.service); + const target = setup(); + await target.service.completeTurn("local-turn", { sessionId: target.open("local"), query: "看看", answer: "好" }); + await settleCaptureAssignments(target.service); + expect(target.service.importBundle({ bundle: source.service.exportBundle({ includeRawText: true }) }).ok).toBe(true); + const frozen = (sessionId: string) => target.db.db.prepare( + `SELECT start_trace_seq AS s, end_trace_seq AS e FROM l3_unit_batches WHERE session_id = ? ORDER BY scope_seq` + ).all(sessionId) as Array<{ s: number; e: number }>; + // The imported session goes quiet here and its idle flush freezes turn 1. + (target.service as unknown as { repos: { l3WorldModels: { freezeBatches(input: { sessionId: string; trigger: "session_idle" }): unknown } } }) + .repos.l3WorldModels.freezeBatches({ sessionId: session, trigger: "session_idle" }); + expect(frozen(session)).toEqual([{ s: 1, e: 1 }]); + // The source goes on: a second turn, then a close freezes [1,2] there; that bundle replaces this one. + await source.service.completeTurn("second-turn", { sessionId: session, query: "提交信息用中文", answer: "好" }); + await settleCaptureAssignments(source.service); + source.service.closeSession(session); + expect(target.service.importBundle({ + bundle: source.service.exportBundle({ includeRawText: true }), conflictStrategy: "replace" + }).ok).toBe(true); + expect(frozen(session).at(-1)!.e).toBe(2); + }); + + it("re-freezes a turn a replace import put at a position this database froze with a different turn", async () => { + const source = setup(); + const session = source.open("forked"); + await source.service.completeTurn("shared-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + await settleCaptureAssignments(source.service); + const target = setup(); + await target.service.completeTurn("local-turn", { sessionId: target.open("local"), query: "看看", answer: "好" }); + await settleCaptureAssignments(target.service); + expect(target.service.importBundle({ bundle: source.service.exportBundle({ includeRawText: true }) }).ok).toBe(true); + // Both databases go on in the same session with a different second turn; this one freezes [1,2] here. + await target.service.completeTurn("target-turn", { sessionId: session, query: "提交信息用英文", answer: "好" }); + await settleCaptureAssignments(target.service); + (target.service as unknown as { repos: { l3WorldModels: { freezeBatches(input: { sessionId: string; trigger: "session_idle" }): unknown } } }) + .repos.l3WorldModels.freezeBatches({ sessionId: session, trigger: "session_idle" }); + await source.service.completeTurn("source-turn", { sessionId: session, query: "提交信息用中文", answer: "好" }); + await settleCaptureAssignments(source.service); + source.service.closeSession(session); + expect(target.service.importBundle({ + bundle: source.service.exportBundle({ includeRawText: true }), conflictStrategy: "replace" + }).ok).toBe(true); + // Trace 2 is now the source's turn: some batch here must have frozen that turn, not just its position. + const trace = target.db.db.prepare( + `SELECT l1_memory_id AS id FROM l3_world_model_input_traces WHERE session_id = ? AND trace_seq = 2` + ).get(session) as { id: string }; + const owners = target.db.db.prepare( + `SELECT COUNT(*) AS n FROM l3_unit_batches b, json_each(b.l1_memory_ids_json) j + WHERE b.session_id = ? AND b.start_trace_seq <= 2 AND b.end_trace_seq >= 2 AND j.value = ?` + ).get(session, trace.id) as { n: number }; + expect(owners.n).toBe(1); + }); + + it("changes nothing with the switch off", async () => { + const env = setup(false); + const quiet = await goIdle(env); + expect(env.batches(quiet)).toEqual([]); + expect(env.legacyBatches(quiet)).toBe(0); + }); +}); diff --git a/Memory/tests/service/l3-field/evidence-fragments.test.ts b/Memory/tests/service/l3-field/evidence-fragments.test.ts new file mode 100644 index 000000000..f0407e2c5 --- /dev/null +++ b/Memory/tests/service/l3-field/evidence-fragments.test.ts @@ -0,0 +1,233 @@ +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildEvidenceFragments, + clipToolOutput, + extractToolPaths +} from "../../../src/service/l3-field/evidence-fragments.js"; +import type { ResolvedProject } from "../../../src/service/l3-field/types.js"; +import type { RawTurnRecord } from "../../../src/storage/repositories.js"; + +function rawTurn(toolCalls: unknown[], toolResults: unknown[] = [], userText = "帮我看一下"): RawTurnRecord { + return { + id: "raw-1", + sessionId: "s", + episodeId: "e", + turnId: "t", + userId: "u", + userText, + assistantText: "好的", + toolCalls, + toolResults, + sourceMemoryIds: [], + usage: {}, + status: "succeeded", + createdAt: "2026-10-24T01:00:00.000Z" + }; +} + +/** Projects are the first two path components under /work, e.g. /work/repo-a. */ +function resolve(target: string): ResolvedProject | null { + const match = /^\/work\/([^/]+)/.exec(target); + if (!match) return null; + return { identity: `id:${match[1]}`, checkoutRoot: `/work/${match[1]}`, source: "git", displayName: match[1]! }; +} +const unitKeyOf = (project: ResolvedProject) => `unit:${project.identity}`; + +function build(turn: RawTurnRecord, sessionCwd: string | null = "/work/repo-a") { + return buildEvidenceFragments({ rawTurn: turn, l1MemoryId: "l1-1", sessionCwd, sessionProjectRoot: sessionCwd, resolve, unitKeyOf }); +} + +describe("extractToolPaths", () => { + it("reads Claude Code file tools and resolves relative paths against the working directory", () => { + expect(extractToolPaths({ name: "Read", input: { file_path: "/work/repo-a/src/a.ts" } }, null)).toEqual(["/work/repo-a/src/a.ts"]); + expect(extractToolPaths({ name: "Edit", input: { file_path: "src/a.ts" } }, "/work/repo-a")).toEqual(["/work/repo-a/src/a.ts"]); + expect(extractToolPaths({ name: "NotebookEdit", input: { notebook_path: "/work/repo-a/n.ipynb" } }, null)).toEqual(["/work/repo-a/n.ipynb"]); + }); + + it("parses Codex arguments given as a JSON string, including workdir", () => { + const call = { name: "exec_command", input: JSON.stringify({ cmd: "npm test", workdir: "/work/repo-b" }) }; + expect(extractToolPaths(call, "/work/repo-a")).toEqual(["/work/repo-b"]); + }); + + it("reads cmd and workdir out of a Codex code-mode snippet", () => { + const code = [ + "const r = await Promise.allSettled([", + " tools.exec_command({cmd: \"sed -n 1,20p src/a.ts\", workdir: \"/work/repo-b\"}),", + " tools.exec_command({cmd: 'cat /work/repo-c/x.md'}),", + "]);" + ].join("\n"); + expect(extractToolPaths({ name: "exec", input: code }, "/work/repo-a")).toEqual([ + "/work/repo-b", "/work/repo-b/src/a.ts", "/work/repo-c/x.md" + ]); + }); + + it("reads relative paths, and bare names that exist, against the working directory", () => { + const big = mkdtempSync(join(tmpdir(), "l3-big-")); + mkdirSync(join(big, "api")); + const command = "cat api/AGENTS.md; rg -n 'listen|PORT' api --glob '!node_modules/**'; ls"; + expect(extractToolPaths({ name: "exec_command", input: { cmd: command, workdir: big } }, big)).toEqual([ + join(big, "api/AGENTS.md"), join(big, "api"), big + ]); + rmSync(big, { recursive: true, force: true }); + }); + + it("counts the workdir only when the call names no target of its own", () => { + expect(extractToolPaths({ name: "exec_command", input: { cmd: "git -C /work/repo-b status", workdir: "/work/repo-a" } }, "/work/repo-a")) + .toEqual(["/work/repo-b"]); + expect(extractToolPaths({ name: "exec_command", input: { cmd: "npm test", workdir: "/work/repo-b" } }, "/work/repo-a")) + .toEqual(["/work/repo-b"]); + }); + + it("judges each subcommand on its own and ignores where output is redirected", () => { + const run = (cmd: string) => extractToolPaths({ name: "exec_command", input: { cmd, workdir: "/work/repo-a" } }, "/work/repo-a"); + expect(run("git -C /work/repo-b status && npm test")).toEqual(["/work/repo-b", "/work/repo-a"]); + expect(run("npm test > /tmp/test.log 2>&1")).toEqual(["/work/repo-a"]); + expect(run("npm test 2>/dev/null | tail -5")).toEqual(["/work/repo-a"]); + }); + + it("keeps the other calls of a code-mode snippet that also holds a patch", () => { + const code = [ + "await tools.exec_command({cmd: \"npm test\", workdir: \"/work/repo-a\"});", + "await tools.apply_patch(`*** Begin Patch", + "*** Update File: /work/repo-b/file.ts", + "@@", + "-a", + "+b", + "*** End Patch`);" + ].join("\n"); + expect(extractToolPaths({ name: "exec", input: code }, "/work/repo-a").sort()).toEqual(["/work/repo-a", "/work/repo-b/file.ts"]); + }); + + it("reads the files a patch touches, relative ones against the workdir", () => { + const patch = "*** Begin Patch\n*** Update File: /work/repo-b/file.ts\n@@\n-a\n+b\n*** Add File: docs/new.md\n+x\n*** End Patch"; + expect(extractToolPaths({ name: "apply_patch", input: patch }, "/work/repo-a")).toEqual([ + "/work/repo-b/file.ts", "/work/repo-a/docs/new.md" + ]); + expect(extractToolPaths({ name: "apply_patch", input: { input: patch, workdir: "/work/repo-c" } }, "/work/repo-a")).toEqual([ + "/work/repo-b/file.ts", "/work/repo-c/docs/new.md" + ]); + }); + + it("skips arguments that are broken JSON", () => { + expect(extractToolPaths({ name: "exec_command", input: "{not json" }, "/work/repo-a")).toEqual([]); + }); + + it("takes cd and git -C targets and absolute path arguments from commands, but not URLs or the executable", () => { + const command = `cd /work/repo-a && git -C "/work/repo b" status && /usr/bin/env cat /work/repo-c/README.md && curl https://example.com/x`; + expect(extractToolPaths({ name: "Bash", input: { command } }, null)).toEqual([ + "/work/repo-a", "/work/repo b", "/work/repo-c/README.md" + ]); + }); + + it("follows a relative cd from the current directory", () => { + expect(extractToolPaths({ name: "Bash", input: { command: "cd packages/app && npm test" } }, "/work/mono")) + .toEqual(["/work/mono/packages/app"]); + }); + + it("recognizes Windows drive paths", () => { + expect(extractToolPaths({ name: "Read", input: { file_path: "C:\\work\\repo\\a.ts" } }, null)).toEqual(["C:\\work\\repo\\a.ts"]); + }); +}); + +describe("clipToolOutput", () => { + it("keeps head, tail and the error lines in between, and marks the cut", () => { + const middle = Array.from({ length: 400 }, (_, i) => `line ${i}`); + middle[200] = "Error: ENOENT: no such file or directory"; + const clipped = clipToolOutput(middle.join("\n")); + expect(clipped).toContain("line 0"); + expect(clipped).toContain("line 399"); + expect(clipped).toContain("Error: ENOENT: no such file or directory"); + expect(clipped).toContain("[已截断]"); + }); + + it("returns short output unchanged", () => { + expect(clipToolOutput("ok")).toBe("ok"); + }); + + it("keeps only the start of a very long error line", () => { + const middle = Array.from({ length: 400 }, (_, i) => `line ${i}`); + middle[200] = `{"error":"${"x".repeat(50_000)}"}`; + expect(clipToolOutput(middle.join("\n")).length).toBeLessThan(1_000); + }); +}); + +describe("buildEvidenceFragments", () => { + it("makes one fragment for the user text and one per tool call with its own candidates", () => { + const fragments = build(rawTurn([ + { id: "c1", name: "Edit", input: { file_path: "/work/repo-a/a.ts" }, output: "done", success: true }, + { id: "c2", name: "Bash", input: { command: "cd /work/repo-b && npm test" }, output: "1 failed", success: false } + ])); + expect(fragments.map((fragment) => [fragment.toolName, fragment.candidateUnitKeys, fragment.isError])).toEqual([ + [null, ["unit:id:repo-a", "unit:id:repo-b"], false], + ["Edit", ["unit:id:repo-a"], false], + ["Bash", ["unit:id:repo-b"], true] + ]); + expect(fragments[0]!.output).toBe("帮我看一下"); + expect(new Set(fragments.map((fragment) => fragment.id)).size).toBe(3); + expect(fragments.every((fragment) => fragment.l1MemoryId === "l1-1" && fragment.rawTurnId === "raw-1")).toBe(true); + }); + + it("gives the user's words to the repository the turn worked on, not the big folder the session opened in", () => { + const resolveBig = (target: string) => target.startsWith("/big/report") + ? { identity: "id:report", checkoutRoot: "/big/report", source: "git" as const, displayName: "report" } + : target.startsWith("/big") + ? { identity: "dir:/big", checkoutRoot: "/big", source: "folder" as const, displayName: "big" } + : null; + const turn = (calls: unknown[]) => buildEvidenceFragments({ + rawTurn: rawTurn(calls, [], "以后报告都以 main 为准"), l1MemoryId: "l1-1", + sessionCwd: "/big", sessionProjectRoot: "/big", resolve: resolveBig, unitKeyOf + }); + expect(turn([{ name: "Bash", input: { command: "git -C /big/report pull" }, output: "ok" }])[0]!.candidateUnitKeys) + .toEqual(["unit:id:report"]); + expect(turn([])[0]!.candidateUnitKeys).toEqual(["unit:dir:/big"]); + }); + + it("clips a long tool input but still reads its paths", () => { + const [, write] = build(rawTurn([ + { id: "c1", name: "Write", input: { file_path: "/work/repo-a/big.md", content: "y".repeat(20_000) }, output: "ok" } + ])); + expect(write!.candidateUnitKeys).toEqual(["unit:id:repo-a"]); + expect(JSON.stringify(write!.input).length).toBeLessThan(1_000); + }); + + it("keeps only the nested project when a call touches it and the big folder around it", () => { + const resolveBig = (target: string) => target.startsWith("/big/report") + ? { identity: "id:report", checkoutRoot: "/big/report", source: "git" as const, displayName: "report" } + : target.startsWith("/big") + ? { identity: "dir:/big", checkoutRoot: "/big", source: "folder" as const, displayName: "big" } + : null; + const fragments = buildEvidenceFragments({ + rawTurn: rawTurn([{ name: "exec_command", input: { cmd: "pwd; printf '\\n--- rules ---\\n'; cat /big/report/AGENTS.md", workdir: "/big" }, output: "ok" }], [], "看看 report 的规矩"), + l1MemoryId: "l1-1", sessionCwd: "/big", sessionProjectRoot: "/big", resolve: resolveBig, unitKeyOf + }); + expect(fragments.map((fragment) => fragment.candidateUnitKeys)).toEqual([["unit:id:report"], ["unit:id:report"]]); + }); + + it("gives stable fragment ids for the same raw turn", () => { + const turn = rawTurn([{ id: "c1", name: "Read", input: { file_path: "/work/repo-a/a.ts" }, output: "x" }]); + expect(build(turn).map((fragment) => fragment.id)).toEqual(build(turn).map((fragment) => fragment.id)); + }); + + it("falls back to the session folder when a tool call has no path, and to nothing outside any project", () => { + expect(build(rawTurn([{ name: "WebSearch", input: { query: "x" }, output: "y" }]))[1]!.candidateUnitKeys) + .toEqual(["unit:id:repo-a"]); + expect(build(rawTurn([{ name: "WebSearch", input: { query: "x" }, output: "y" }]), "/Users/me")[1]!.candidateUnitKeys) + .toEqual([]); + }); + + it("keeps a tool result that matches no call as its own fragment", () => { + const fragments = build(rawTurn( + [{ id: "c1", name: "Read", input: { file_path: "/work/repo-a/a.ts" }, output: "x" }], + [{ id: "c1", output: "x" }, { id: "orphan", name: "Bash", output: "permission denied", success: false }] + )); + expect(fragments.at(-1)).toMatchObject({ toolName: "Bash", output: "permission denied", isError: true }); + }); + + it("treats an explicit error message as an error even without success=false", () => { + const fragments = build(rawTurn([{ id: "c1", name: "Bash", input: { command: "ls" }, error: "exit code 1" }])); + expect(fragments[1]).toMatchObject({ isError: true, output: "exit code 1" }); + }); +}); diff --git a/Memory/tests/service/l3-field/l3-field-context.test.ts b/Memory/tests/service/l3-field/l3-field-context.test.ts new file mode 100644 index 000000000..70ff62400 --- /dev/null +++ b/Memory/tests/service/l3-field/l3-field-context.test.ts @@ -0,0 +1,270 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const MACHINE_ID = "c".repeat(64); + +function setup(enabled = true, l3MachineProbe: () => Promise | Promise = () => new Promise(() => undefined)) { + const root = fixture.createTestRoot("l3-field-context-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + const service = fixture.createTestMemoryService({ + db, + mode: "dev", + config, + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }), + l3MachineProbe: l3MachineProbe as never + }); + const repos = new Repositories(db.db); + return { db, service, repos, home, project }; +} + +function open(service: ReturnType["service"], sessionKey: string, workspace?: string) { + const namespace = { source: "codex", profileId: "default", sessionKey, userId: "u1" }; + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + ...(workspace ? { workspaceUri: pathToFileURL(workspace).href, workspaceHostId: "b".repeat(64) } : {}), + namespace + }); + const request = { + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", + adapterId: "codex-memory", + source: "codex", + namespace: opened.projectId ? { ...namespace, projectId: opened.projectId } : namespace + }; + return { opened, request, namespace: request.namespace }; +} + +function evidence() { + return { fragmentIds: ["f1"], l1MemoryIds: ["memory_l1"] }; +} + +describe("new L3 session context", () => { + it("reads without writing while memory writes are off, showing only what already exists", () => { + const { db, service, repos, home, project } = setup(); + const { opened, request } = open(service, "read-only", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "pnpm -> 未安装", evidence: evidence(), evidenceAt: new Date().toISOString() }); + repos.db.prepare(`DELETE FROM l3_units WHERE kind = 'project'`).run(); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.enableMemoryAdd = false; + const readOnly = fixture.createTestMemoryService({ + db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const counts = () => repos.db.prepare( + `SELECT (SELECT COUNT(*) FROM l3_units) AS units, (SELECT COUNT(*) FROM memories) AS memories, + (SELECT COUNT(*) FROM evolution_jobs) AS jobs` + ).get(); + const before = counts(); + const context = readOnly.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toContain("pnpm -> 未安装"); + expect(counts()).toEqual(before); + }); + + it("does not store a machine probe that finishes after the service stopped", async () => { + let finish!: (probe: unknown) => void; + const { service, repos } = setup(true, () => new Promise((resolve) => { finish = resolve; })); + open(service, "stopping"); + await service.stop(); + finish({ os: "darwin", osVersion: "15", arch: "arm64", shell: "/bin/zsh", tools: {} }); + await new Promise((resolve) => setTimeout(resolve, 10)); + const machine = repos.l3Field.getUnit(repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }).unitKey)!; + expect(machine.profileFingerprint ?? null).toBeNull(); + }); + + it("is empty for a session without a folder until the machine knows something", () => { + const { service, repos } = setup(); + const { opened, request } = open(service, "machine-only"); + expect(service.l3WorldModelContext(opened.sessionId, request)).toMatchObject({ renderedContext: "", memoryId: null }); + + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const context = service.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toBe("当前项目：无\n\n## 本机环境事实\n- 本机用 pnpm 9"); + expect(context.memoryId).toBe(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId); + expect(context.projectContract).toBeNull(); + expect(context.domainKnowledge).toBe("- 本机用 pnpm 9"); + }); + + it("names the project and its source, and shows machine and project knowledge together", () => { + const { service, repos, project } = setup(); + const { opened, request } = open(service, "with-project", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + service.l3WorldModelContext(opened.sessionId, request); + const unit = repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project'`).get() as { unit_key: string }; + repos.l3Field.insertItem({ unitKey: unit.unit_key, kind: "project_constraint", content: "改完跑 npm test", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(unit.unit_key); + + const context = service.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toBe( + "当前项目：demo（文件夹）\n\n## 本机环境事实\n- 本机用 pnpm 9\n\n## 项目约束\n- 改完跑 npm test" + ); + expect(context.projectId).toBe(opened.projectId); + expect(context.memoryId).toBe(repos.l3Field.getUnit(unit.unit_key)!.snapshotMemoryId); + expect(context.sourceMemoryIds).toEqual(["memory_l1"]); + }); + + it("shows a snapshot on the world model page and keeps it out of full-text search", () => { + const { service, repos, db } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { request } = open(service, "panel"); + const detail = service.getMemory(snapshot.id, request); + expect(JSON.stringify(detail)).toContain("\"domainKnowledge\":\"- 本机用 pnpm 9\""); + const fts = db.db.prepare(`SELECT COUNT(*) AS n FROM memories_fts WHERE id = ?`).get(snapshot.id) as { n: number }; + expect(fts.n).toBe(0); + }); + + it("deleting a snapshot resets its unit, and the next session starts it afresh", () => { + const { service, repos } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const item = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { opened, request } = open(service, "delete"); + + service.deleteMemory(snapshot.id, { namespace: request.namespace, reason: "user" }); + expect(repos.l3Field.getItem(item.id)!.status).toBe("inactive"); + expect(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId).toBeUndefined(); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).toBe(""); + + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 bun", evidence: evidence(), evidenceAt: "2026-10-03T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const context = service.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toContain("本机用 bun"); + expect(context.memoryId).not.toBe(snapshot.id); + }); + + it("does not inject a unit whose snapshot the user archived, and keeps it archived when it is rewritten", () => { + const { service, repos } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { opened, request } = open(service, "archive"); + service.archiveMemory(snapshot.id, { namespace: request.namespace, reason: "user" }); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).toBe(""); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 bun", evidence: evidence(), evidenceAt: "2026-10-02T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + expect(repos.memories.get(snapshot.id)!.status).toBe("archived"); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).toBe(""); + }); + + it("moves a project's context version when the machine snapshot is archived, so clients drop the cached machine part", () => { + const { service, repos, project } = setup(); + const { opened, request } = open(service, "archive-machine", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + service.l3WorldModelContext(opened.sessionId, request); + const unit = repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project'`).get() as { unit_key: string }; + repos.l3Field.insertItem({ unitKey: unit.unit_key, kind: "project_constraint", content: "改完跑 npm test", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const machineSnapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + repos.l3Field.refreshSnapshot(unit.unit_key); + const before = service.l3WorldModelContext(opened.sessionId, request); + expect(before.renderedContext).toContain("本机用 pnpm 9"); + + service.archiveMemory(machineSnapshot.id, { namespace: request.namespace, reason: "user" }); + const after = service.l3WorldModelContext(opened.sessionId, request); + expect(after.renderedContext).not.toContain("本机用 pnpm 9"); + expect(after.renderedContext).toContain("改完跑 npm test"); + expect(after.memoryVersion).not.toBe(before.memoryVersion); + }); + + it("deletes a snapshot and resets its unit together, or neither", () => { + const { service, repos } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const item = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { request } = open(service, "atomic-delete"); + const internal = (service as unknown as { repos: Repositories }).repos; + const reset = vi.spyOn(internal.l3Field, "markUnitDeleted").mockImplementation(() => { throw new Error("disk full"); }); + expect(() => service.deleteMemory(snapshot.id, { namespace: request.namespace, reason: "user" })).toThrow("disk full"); + reset.mockRestore(); + expect(repos.memories.get(snapshot.id)!.deletedAt).toBeFalsy(); + expect(repos.l3Field.getItem(item.id)!.status).toBe("active"); + }); + + it("drops a machine probe that was running when the user deleted the machine snapshot", async () => { + let finishProbe: (probe: unknown) => void = () => undefined; + const probe = () => new Promise((resolve) => { finishProbe = resolve; }); + const { service, repos } = setup(true, probe); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { request } = open(service, "probe-race"); + service.deleteMemory(snapshot.id, { namespace: request.namespace, reason: "user" }); + finishProbe({ os: { name: "macOS", version: "26.0" }, arch: "arm64", shell: "zsh", tools: [] }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(repos.l3Field.getUnit(machine.unitKey)!.profileText).toBeFalsy(); + expect(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId).toBeUndefined(); + }); + + it("moves the project version when the machine is reset, and keeps refreshing a project rebuilt after a reset", () => { + const { service, repos, project } = setup(); + const { opened, request } = open(service, "versions", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + service.l3WorldModelContext(opened.sessionId, request); + const unitKey = (repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project'`).get() as { unit_key: string }).unit_key; + repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "改完跑测试", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const before = service.l3WorldModelContext(opened.sessionId, request); + + repos.l3Field.markUnitDeleted(machine.unitKey); + const afterMachineReset = service.l3WorldModelContext(opened.sessionId, request); + expect(afterMachineReset.memoryVersion).not.toBe(before.memoryVersion); + expect(afterMachineReset.renderedContext).not.toContain("pnpm"); + + repos.l3Field.markUnitDeleted(unitKey); + repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "新规则", evidence: evidence(), evidenceAt: "2026-10-02T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(unitKey); + const rebuilt = service.l3WorldModelContext(opened.sessionId, request); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 bun", evidence: evidence(), evidenceAt: "2026-10-03T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const afterMachineChange = service.l3WorldModelContext(opened.sessionId, request); + expect(afterMachineChange.memoryVersion).not.toBe(rebuilt.memoryVersion); + expect(afterMachineChange.renderedContext).toContain("本机用 bun"); + }); + + it("on startup drops items inactive for over 15 days and never queues embeddings for snapshots", () => { + const { service, repos, db } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const old = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "旧", evidence: evidence(), evidenceAt: "2026-01-01T00:00:00.000Z" }); + const recent = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "新", evidence: evidence(), evidenceAt: "2026-01-01T00:00:00.000Z" }); + repos.l3Field.inactivateItem(old.id, { at: new Date(Date.now() - 16 * 86_400_000).toISOString() }); + repos.l3Field.inactivateItem(recent.id, { at: new Date(Date.now() - 14 * 86_400_000).toISOString() }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + + service.reconcileWorkerStartup(); + expect(repos.l3Field.getItem(old.id)).toBeUndefined(); + expect(repos.l3Field.getItem(recent.id)).toBeDefined(); + const jobs = db.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'embedding' AND target_memory_id = ?` + ).get(snapshot.id) as { n: number }; + expect(jobs.n).toBe(0); + }); + + it("keeps the old L3 when the switch is off", () => { + const { service, repos } = setup(false); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const { opened, request } = open(service, "flag-off"); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).not.toContain("pnpm"); + }); +}); diff --git a/Memory/tests/service/l3-field/l3-retry.test.ts b/Memory/tests/service/l3-field/l3-retry.test.ts new file mode 100644 index 000000000..8c9c313f5 --- /dev/null +++ b/Memory/tests/service/l3-field/l3-retry.test.ts @@ -0,0 +1,369 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMemoryHttpServer, DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { ModelHttpError } from "../../../src/model/http.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { l3FailureHandling } from "../../../src/service/worker/job-handlers.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture, settleCaptureAssignments } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const MACHINE_ID = "9".repeat(64); +const networkError = () => new Error("fetch failed: network connect ECONNRESET"); +/** Reached the host, then lost the connection: retried, unlike being offline. */ +const networkErrorAfterConnect = () => new TypeError("fetch failed", { cause: { code: "ECONNRESET" } }); + +function job(over: Partial = {}): EvolutionJobRecord { + return { id: "j", jobType: "l3_unit_update", status: "leased", userId: "u", payload: {}, attempts: 1, failureCount: 0, maxAttempts: 3, + createdAt: "", updatedAt: "", ...over } as EvolutionJobRecord; +} + +describe("L3 failure handling", () => { + it("waits 1, 5, 15, 30 and 60 minutes after temporary failures, then sends the sixth failure to dead letter", () => { + expect([0, 1, 2, 3, 4].map((failureCount) => l3FailureHandling(job({ failureCount }), networkError()))) + .toEqual([1, 5, 15, 30, 60].map((delayMinutes) => ({ kind: "retry_later", delayMinutes, maxAttempts: 6 }))); + expect(l3FailureHandling(job({ failureCount: 5 }), networkError())).toEqual({ kind: "dead_letter" }); + }); + + it("recognizes the errors the model clients really throw", () => { + const aborted = Object.assign(new Error("This operation was aborted"), { name: "AbortError" }); + expect(l3FailureHandling(job(), aborted).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new TypeError("fetch failed", { cause: { code: "ECONNRESET" } })).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("openai_responses response missing output text")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("bedrock response missing output.message.content")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("host response missing text/content/output")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("deepseek HTTP 200: expected JSON but received an empty response")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("p HTTP 200: expected JSON but received text/html; check the configured model endpoint")).kind) + .toBe("default"); + expect(l3FailureHandling(job(), new TypeError("Cannot read properties of undefined (reading 'x')")).kind).toBe("retry_later"); + }); + + it("suspends without using an attempt when the model endpoint cannot be reached, as when offline", () => { + expect(l3FailureHandling(job(), new TypeError("fetch failed")).kind).toBe("suspend"); + expect(l3FailureHandling(job(), new TypeError("fetch failed", { cause: { code: "ENOTFOUND" } })).kind).toBe("suspend"); + expect(l3FailureHandling(job({ failureCount: 5 }), new TypeError("fetch failed", { cause: { code: "ECONNREFUSED" } })).kind) + .toBe("suspend"); + }); + + it("retries empty replies, timeouts, rate limits and rejected answers later, and leaves other job types alone", () => { + const rejected = new Error("l3 unit output rejected twice: ops[0]: content is empty"); + expect(l3FailureHandling(job({ failureCount: 0 }), rejected)).toEqual({ kind: "retry_later", delayMinutes: 1, maxAttempts: 6 }); + // Likely to fail the same way again: one more try only. + expect(l3FailureHandling(job({ failureCount: 1 }), rejected)).toEqual({ kind: "dead_letter" }); + expect(l3FailureHandling(job(), new Error("openai_compatible response missing choices[0].message.content")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new TypeError("invalid JSON: Unexpected end of JSON input")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("request timed out after 180000ms")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new ModelHttpError("too many", "p", 429, undefined, "rate limit")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("l3 unit output rejected twice: ops[0]: content is empty")).kind).toBe("retry_later"); + expect(l3FailureHandling(job({ jobType: "reward" }), networkError()).kind).toBe("default"); + }); + + it("suspends on used-up quota (40309) and leaves a bad key to the queue's configuration hold", () => { + expect(l3FailureHandling(job(), new ModelHttpError("quota", "p", 403, "40309", "")).kind).toBe("suspend"); + expect(l3FailureHandling(job(), new Error("401 unauthorized: invalid api key")).kind).toBe("default"); + }); +}); + +describe("L3 retries through the job queue", () => { + function setup(complete: LlmClient["complete"], configured = true) { + const root = fixture.createTestRoot("l3-retry-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = true; + const skillLlm = testLlm(complete, configured); + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, skillLlm, + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const repos = new Repositories(db.db); + const sessions: string[] = []; + const freeze = async (key: string) => { + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: key, userId: "u1" } + }); + await service.completeTurn(`${key}-turn`, { + sessionId: opened.sessionId, query: "以后改完都跑 npm test", answer: "好", status: "succeeded", + toolCalls: [{ id: "c", name: "exec_command", input: { cmd: "npm test", workdir: project }, success: false }], + toolResults: [{ id: "c", name: "exec_command", output: "vitest: command not found", exitCode: 127 }] + }); + service.closeSession(opened.sessionId); + // The close freezes the session once its turns are assigned to a task. + await settleCaptureAssignments(service); + sessions.push(opened.sessionId); + // This test is about the L3 job, not summaries: mark the L1 summaries done so the job can be leased. + repos.db.prepare(`UPDATE memory_processing_state SET state = 'ready'`).run(); + repos.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE job_type IN ('trace_summary', 'episode_title')`).run(); + }; + const unitJobs = () => repos.db.prepare( + `SELECT id, status, attempts, max_attempts, scope_seq, json_extract(payload_json, '$.runAfter') AS run_after, + json_extract(payload_json, '$.batchId') AS batch_id + FROM evolution_jobs WHERE job_type = 'l3_unit_update' ORDER BY scope_seq` + ).all() as Array<{ id: string; status: string; attempts: number; max_attempts: number; scope_seq: number; run_after: string | null; batch_id: string }>; + const runNow = async () => { + repos.db.prepare( + `UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', '2000-01-01T00:00:00.000Z'), + available_at = NULL + WHERE job_type = 'l3_unit_update' AND status = 'queued' AND json_extract(payload_json, '$.runAfter') IS NOT NULL` + ).run(); + await service.runWorkerOnce(50); + }; + return { service, repos, freeze, unitJobs, runNow }; + } + + it("retries a temporary failure later, dead-letters the sixth failure, lets the next batch run, and reruns on request", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw networkError(); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, repos, freeze, unitJobs, runNow } = setup(complete); + await freeze("first"); + await service.runWorkerOnce(50); + let [first] = unitJobs(); + expect(first).toMatchObject({ status: "queued", attempts: 1, max_attempts: 6 }); + expect(Date.parse(first!.run_after!) - Date.now()).toBeGreaterThan(50_000); + + for (let run = 2; run <= 6; run += 1) await runNow(); + [first] = unitJobs(); + expect(first).toMatchObject({ status: "dead_letter", attempts: 6 }); + expect(repos.l3Field.getBatch(first!.batch_id)!.status).toBe("failed"); + + failing = false; + await freeze("second"); + await service.runWorkerOnce(50); + expect(unitJobs()[1]).toMatchObject({ status: "succeeded" }); + + expect(service.retryFailedL3Batches({ + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", + adapterId: "codex-memory", + source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "x", userId: "u1" } + })) + .toMatchObject({ requeued: 1 }); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0, run_after: null }); + await service.runWorkerOnce(50); + expect(unitJobs()[0]).toMatchObject({ status: "succeeded" }); + expect(repos.l3Field.getBatch(first!.batch_id)!.status).toBe("applied"); + }); + + it("gives a dead-lettered batch one automatic rerun when the worker starts, and only one", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw networkErrorAfterConnect(); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, repos, freeze, unitJobs, runNow } = setup(complete); + await freeze("restart"); + await service.runWorkerOnce(50); + for (let run = 2; run <= 6; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + + service.reconcileWorkerStartup(); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0 }); + expect(repos.l3Field.getBatch(unitJobs()[0]!.batch_id)!.status).toBe("queued"); + await runNow(); + for (let run = 2; run <= 6; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + + service.reconcileWorkerStartup(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + failing = false; + }); + + it("wakes the worker when the rerun endpoint puts batches back", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw networkError(); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, freeze, unitJobs, runNow } = setup(complete); + await freeze("http-rerun"); + await service.runWorkerOnce(50); + for (let run = 2; run <= 6; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + failing = false; + + // The startup fallback stays out of the window, so only the endpoint can wake the worker. + const server = createMemoryHttpServer({ + service, startAgentSourceAutomation: false, auth: { mode: "local", allowAnonymous: true }, workerStartupFallbackMs: 60_000 + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const response = await fetch(`http://127.0.0.1:${address.port}/api/v1/l3/failed-batches/retry`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", adapterId: "codex-memory", source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "x", userId: "u1" } + }) + }); + expect(await response.json()).toMatchObject({ requeued: 1 }); + await vi.waitFor(() => expect(unitJobs()[0]).toMatchObject({ status: "succeeded" }), { timeout: 5000 }); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it("does not let a rejected answer hold back later batches until a restart", async () => { + let bad = true; + const complete = vi.fn(async () => + JSON.stringify(bad ? { reason: "bad", ops: [{ op: "confirm", id: "missing", evidence: [] }] } : { reason: "ok", ops: [] })); + const { service, freeze, unitJobs, runNow } = setup(complete); + await freeze("rejected"); + await freeze("after"); + await service.runWorkerOnce(50); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 1, max_attempts: 6 }); + await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter", attempts: 2 }); + bad = false; + await runNow(); + expect(unitJobs()[1]).toMatchObject({ status: "succeeded" }); + }); + + it("waits out an offline spell without using attempts, then runs the batch", async () => { + let offline = true; + const complete = vi.fn(async () => { + if (offline) throw new TypeError("fetch failed", { cause: { code: "ENOTFOUND" } }); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, freeze, unitJobs, runNow } = setup(complete); + await freeze("offline"); + await service.runWorkerOnce(50); + const [suspended] = unitJobs(); + expect(suspended).toMatchObject({ status: "queued", attempts: 0 }); + expect(Date.parse(suspended!.run_after!) - Date.now()).toBeGreaterThan(29 * 60_000); + for (let run = 0; run < 10; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0 }); + offline = false; + await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "succeeded" }); + }); + + it("does not wake the worker for a batch whose wait is over but an earlier batch still holds it back", () => { + const db = new MemoryDb({ path: ":memory:" }); + const repos = new Repositories(db.db); + const at = new Date().toISOString(); + for (const [id, seq] of [["first", 1], ["second", 2]] as const) { + repos.l3WorldModels.insertImmutableJob({ + id, jobType: "l3_unit_update", status: "queued", dedupeKey: `k-${id}`, userId: "u", sessionId: "s", + scopeKey: "machine", scopeSeq: seq, payload: { batchId: `b-${id}` }, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at + }); + } + // The second batch failed and its wait is over; the first was put back from dead letter and now waits 30 minutes. + db.db.prepare(`UPDATE evolution_jobs SET status = 'leased', attempts = 1`).run(); + repos.runtime.rescheduleFailedJob("second", { runAfter: new Date(Date.now() - 1000).toISOString(), error: "x" }); + repos.runtime.rescheduleFailedJob("first", { runAfter: new Date(Date.now() + 30 * 60_000).toISOString(), error: "offline" }); + expect(repos.runtime.leaseQueuedJobs(10, 60)).toEqual([]); + expect(repos.runtime.nextWorkerRunAt()! - Date.now()).toBeGreaterThan(29 * 60_000); + db.close(); + }); + + it("runs every ready batch of a machine in one worker pass, one after another", async () => { + const complete = vi.fn(async () => JSON.stringify({ reason: "ok", ops: [] })); + const { service, freeze, unitJobs } = setup(complete); + await freeze("one"); + await freeze("two"); + await freeze("three"); + await service.runWorkerOnce(50); + expect(unitJobs().map((job) => job.status)).toEqual(["succeeded", "succeeded", "succeeded"]); + }); + + it("reruns an earlier batch even when a later batch's run was lost with its lease", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw new Error("l3 unit output rejected twice: ops[0]: content is empty"); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, repos, freeze, unitJobs, runNow } = setup(complete); + await freeze("earlier"); + await service.runWorkerOnce(50); + await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + failing = false; + await freeze("later"); + // The later batch's run stopped (the computer slept) and its lease ran out while it was still marked leased. + const past = new Date(Date.now() - 60_000).toISOString(); + repos.db.prepare(`UPDATE evolution_jobs SET status = 'leased', leased_until = ?, lease_owner = 'gone', lease_token = 'gone' + WHERE id = ?`).run(past, unitJobs()[1]!.id); + expect(service.retryFailedL3Batches({ + requestId: "1d9b6f8e-6c39-4c7a-9a43-7a3f1b2f5d10", adapterId: "codex-memory", source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "x", userId: "u1" } + })).toMatchObject({ requeued: 1 }); + await service.runWorkerOnce(50); + expect(unitJobs().map((job) => job.status)).toEqual(["succeeded", "succeeded"]); + }); + + it("holds a batch that failed on a bad key until the configuration changes, without counting a failure", async () => { + const complete = vi.fn(async () => { throw new Error("401 unauthorized: invalid api key"); }); + const { service, repos, freeze, unitJobs } = setup(complete); + await freeze("bad-key"); + await service.runWorkerOnce(50); + const held = repos.db.prepare(`SELECT status, failure_count, blocked_reason FROM evolution_jobs WHERE id = ?`) + .get(unitJobs()[0]!.id); + expect(held).toEqual({ status: "failed", failure_count: 0, blocked_reason: "configuration" }); + }); + + it("holds the old L3's queued jobs while the new L3 is on, without waking the worker for them", async () => { + const complete = vi.fn(async () => JSON.stringify({ reason: "ok", ops: [] })); + const { service, repos, freeze } = setup(complete); + await freeze("old-held"); + await service.runWorkerOnce(50); + const now = new Date().toISOString(); + repos.db.prepare( + `INSERT INTO evolution_jobs (id, job_type, status, user_id, scope_key, scope_seq, payload_json, attempts, max_attempts, created_at, updated_at) + VALUES ('old-update', 'l3_world_model_update', 'queued', 'u1', 'old-scope', 1, + json_object('batchId', 'x', 'targetField', 'project_contract', 'runAfter', '2000-01-01T00:00:00.000Z'), 0, 3, ?, ?), + ('old-profile', 'project_environment_profile', 'queued', 'u1', NULL, NULL, + json_object('userId', 'u1', 'projectId', 'p', 'runAfter', '2000-01-01T00:00:00.000Z'), 0, 3, ?, ?)` + ).run(now, now, now, now); + repos.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE id NOT IN ('old-update', 'old-profile')`).run(); + expect(service.nextWorkerRunAt() ?? Number.POSITIVE_INFINITY).toBeGreaterThan(Date.parse(now)); + await service.runWorkerOnce(20); + const statuses = repos.db.prepare(`SELECT id, status FROM evolution_jobs WHERE id IN ('old-update', 'old-profile') ORDER BY id`).all(); + expect(statuses).toEqual([{ id: "old-profile", status: "queued" }, { id: "old-update", status: "queued" }]); + }); + + it("suspends on a quota error without using an attempt", async () => { + const complete = vi.fn(async () => { throw new ModelHttpError("quota", "p", 403, "40309", ""); }); + const { service, freeze, unitJobs } = setup(complete); + await freeze("quota"); + await service.runWorkerOnce(50); + const [suspended] = unitJobs(); + expect(suspended).toMatchObject({ status: "queued", attempts: 0 }); + expect(Date.parse(suspended!.run_after!) - Date.now()).toBeGreaterThan(29 * 60_000); + }); + + it("leaves L3 jobs in the queue while no evolution model is configured", async () => { + const complete = vi.fn(); + const { service, freeze, unitJobs } = setup(complete, false); + await freeze("unconfigured"); + await service.runWorkerOnce(50); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0 }); + expect(complete).not.toHaveBeenCalled(); + }); +}); + +function testLlm(complete: LlmClient["complete"], configured: boolean): LlmClient { + return { + config: { + provider: "host", endpoint: "http://localhost/unused", model: "l3-retry-test", apiKey: "", + temperature: 0, maxTokens: 4096, timeoutMs: 30_000, maxRetries: 0, malformedRetries: 0, enableThinking: false + }, + isConfigured: () => configured, + complete, + completeJson: vi.fn(), + status: () => ({ provider: "host", model: "l3-retry-test", configured, remote: false }) + }; +} diff --git a/Memory/tests/service/l3-field/machine-profile.test.ts b/Memory/tests/service/l3-field/machine-profile.test.ts new file mode 100644 index 000000000..02e5276fd --- /dev/null +++ b/Memory/tests/service/l3-field/machine-profile.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { + machineProfileFingerprint, + probeMachine, + renderMachineProfile, + type LoginShellRunner, + type VersionRunner +} from "../../../src/service/l3-field/machine-profile.js"; + +const outputs: Record = { + node: "v25.1.0", + python3: "Python 3.11.5", + git: "git version 2.39.5 (Apple Git-154)", + npm: "11.6.2" +}; +const runner: VersionRunner = async (executable) => outputs[executable] ?? null; +const noShell: LoginShellRunner = async () => null; + +describe("machine profile", () => { + it("keeps only version numbers and marks tools that were not found", async () => { + const probe = await probeMachine({ runVersion: runner, runLoginShell: noShell, env: { SHELL: "/bin/zsh" } }); + expect(probe.shell).toBe("zsh"); + expect(probe.tools).toMatchObject({ node: "25.1.0", python3: "3.11.5", git: "2.39.5", npm: "11.6.2", brew: null, pnpm: null }); + const text = renderMachineProfile(probe); + expect(text).toContain("shell：zsh"); + expect(text).toContain("node 25.1.0，python3 3.11.5，git 2.39.5，npm 11.6.2"); + expect(text).toContain("未检测到：brew"); + }); + + it("reports the macOS product version rather than the Darwin kernel version", async () => { + const withSwVers: VersionRunner = async (executable) => (executable === "sw_vers" ? "26.1\n" : outputs[executable] ?? null); + const probe = await probeMachine({ runVersion: withSwVers, runLoginShell: noShell, env: {}, os: "darwin" }); + expect(probe.osVersion).toBe("26.1"); + expect(renderMachineProfile(probe)).toContain("系统：macOS 26.1"); + }); + + it("renders the same probe to the same text and fingerprint", async () => { + const a = await probeMachine({ runVersion: runner, runLoginShell: noShell, env: { SHELL: "/bin/zsh" } }); + const b = await probeMachine({ runVersion: runner, runLoginShell: noShell, env: { SHELL: "/bin/zsh" } }); + expect(renderMachineProfile(a)).toBe(renderMachineProfile(b)); + expect(machineProfileFingerprint(a)).toBe(machineProfileFingerprint(b)); + }); + + it("treats a failing or timed-out probe as not installed", async () => { + const failing: VersionRunner = async () => { + throw new Error("timeout"); + }; + const probe = await probeMachine({ runVersion: failing, runLoginShell: noShell, env: {} }); + expect(Object.values(probe.tools).every((value) => value === null)).toBe(true); + expect(probe.shell).toBeNull(); + }); + + it("prefers versions seen by the login shell and falls back to running tools directly", async () => { + const shellOutput = [ + "Welcome banner 1.2.3 from .zshrc", + "__memmy_probe__node", "v22.11.0", + "__memmy_probe__python3", "Python 3.12.4", + "__memmy_probe__conda", "conda 24.9.2", + "__memmy_probe__java", "The operation couldn't be completed. Unable to locate a Java Runtime.", + "__memmy_probe__go", "" + ].join("\n"); + let script = ""; + const shell: LoginShellRunner = async (_shell, body) => { + script = body; + return shellOutput; + }; + const probe = await probeMachine({ runVersion: runner, runLoginShell: shell, env: { SHELL: "/bin/zsh" }, os: "linux" }); + expect(script).toContain("command -v rustc"); + expect(probe.tools).toMatchObject({ node: "22.11.0", python3: "3.12.4", conda: "24.9.2", git: "2.39.5", java: null, go: null }); + const text = renderMachineProfile(probe); + expect(text).toContain("node 22.11.0，python3 3.12.4，git 2.39.5，npm 11.6.2，conda 24.9.2"); + expect(text).toContain("未检测到：brew"); + expect(text).not.toContain("java"); + }); + + it("does not use a login shell on Windows", async () => { + let called = false; + const shell: LoginShellRunner = async () => { + called = true; + return null; + }; + await probeMachine({ runVersion: runner, runLoginShell: shell, env: { SHELL: "bash" }, os: "win32" }); + expect(called).toBe(false); + }); +}); diff --git a/Memory/tests/service/l3-field/project-resolver.test.ts b/Memory/tests/service/l3-field/project-resolver.test.ts new file mode 100644 index 000000000..9d7fd30af --- /dev/null +++ b/Memory/tests/service/l3-field/project-resolver.test.ts @@ -0,0 +1,242 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path, { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { normalizeRemoteUrl, readGitIdentity } from "../../../src/service/l3-field/git-identity.js"; +import { + isPathWithin, + resolveProject, + type ResolverContext +} from "../../../src/service/l3-field/project-resolver.js"; + +let home: string; +let ctx: ResolverContext; + +beforeEach(() => { + home = realpathSync(mkdtempSync(join(tmpdir(), "l3-resolver-home-"))); + ctx = { + homeDir: home, + userProjects: [], + standaloneRoots: [join(home, "Memmy")], + tempRoots: [join(home, "tmp")], + workspaceIsLocal: true + }; +}); + +afterEach(() => { + rmSync(home, { recursive: true, force: true }); +}); + +function dir(...parts: string[]): string { + const target = join(home, ...parts); + mkdirSync(target, { recursive: true }); + return target; +} + +function gitRepo(root: string, origin?: string): void { + mkdirSync(join(root, ".git"), { recursive: true }); + const remote = origin ? `[remote "origin"]\n\turl = ${origin}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n` : ""; + writeFileSync(join(root, ".git", "config"), `[core]\n\tbare = false\n${remote}`); +} + +function worktree(mainRoot: string, worktreeRoot: string, name: string): void { + const gitDir = join(mainRoot, ".git", "worktrees", name); + mkdirSync(gitDir, { recursive: true }); + writeFileSync(join(gitDir, "commondir"), "../..\n"); + mkdirSync(worktreeRoot, { recursive: true }); + writeFileSync(join(worktreeRoot, ".git"), `gitdir: ${path.relative(worktreeRoot, gitDir)}\n`); +} + +const asSession = (p: string) => ({ path: p, role: "session_dir" as const, sessionCwd: p, sessionProjectRoot: null }); + +describe("normalizeRemoteUrl", () => { + it("maps ssh, scp-like and https forms of the same repository to one identity", () => { + const expected = "github.com/memtensor/memmy-agent"; + expect(normalizeRemoteUrl("git@github.com:MemTensor/memmy-agent.git")).toBe(expected); + expect(normalizeRemoteUrl("ssh://git@GitHub.com/MemTensor/memmy-agent.git")).toBe(expected); + expect(normalizeRemoteUrl("https://github.com/MemTensor/memmy-agent")).toBe(expected); + expect(normalizeRemoteUrl("https://user@github.com/MemTensor/memmy-agent.git/")).toBe(expected); + }); + + it("keeps a non-default port and ignores local remotes", () => { + expect(normalizeRemoteUrl("ssh://git@git.example.test:2222/team/repo.git")).toBe("git.example.test:2222/team/repo"); + expect(normalizeRemoteUrl("/srv/git/repo.git")).toBeNull(); + expect(normalizeRemoteUrl("file:///srv/git/repo.git")).toBeNull(); + }); +}); + +describe("readGitIdentity", () => { + it("gives a worktree, its main clone and a subdirectory the same identity", () => { + const main = dir("code", "repo"); + gitRepo(main, "git@github.com:Org/Repo.git"); + const sub = dir("code", "repo", "src", "deep"); + const wt = join(home, "code", "repo-wt"); + worktree(main, wt, "repo-wt"); + const a = readGitIdentity(main)!; + expect(readGitIdentity(sub)!.identity).toBe(a.identity); + expect(readGitIdentity(sub)!.repoRoot).toBe(main); + expect(readGitIdentity(wt)!.identity).toBe(a.identity); + expect(readGitIdentity(wt)!.repoRoot).toBe(wt); + expect(a.identity).toBe("github.com/org/repo"); + }); + + it("falls back to the common git directory when there is no origin or the config is broken", () => { + const repo = dir("code", "local-only"); + gitRepo(repo); + expect(readGitIdentity(repo)!.identity).toBe(`gitdir:${join(repo, ".git")}`); + writeFileSync(join(repo, ".git", "config"), "\u0000not a config"); + expect(readGitIdentity(repo)!.identity).toBe(`gitdir:${join(repo, ".git")}`); + }); + + it("returns null outside any repository", () => { + expect(readGitIdentity(dir("plain"))).toBeNull(); + }); +}); + +describe("resolveProject", () => { + it("resolves a git repository by its origin, named after the remote repository", () => { + const repo = dir("code", "memmy-agent-feature-x"); + gitRepo(repo, "https://github.com/MemTensor/memmy-agent.git"); + expect(resolveProject(asSession(join(repo, "Memory")), ctx)).toEqual({ + identity: "github.com/memtensor/memmy-agent", + checkoutRoot: repo, + source: "git", + displayName: "memmy-agent" + }); + }); + + it("treats a plain session folder as a folder project", () => { + const folder = dir("Desktop", "reviews"); + expect(resolveProject(asSession(folder), ctx)).toEqual({ + identity: `dir:${folder}`, + checkoutRoot: folder, + source: "folder", + displayName: "reviews" + }); + }); + + it("does not treat temporary, Codex dated, Claude scratch or Memmy task folders as projects", () => { + expect(resolveProject(asSession(dir("tmp", "x")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir("Documents", "Codex", "2026-10-22", "na")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir("Library", "Application Support", "Claude", "scratch-workspaces", "a", "b")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir(".codex", "memories_v2")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir("Memmy", "20261022-101010")), ctx)).toBeNull(); + }); + + it("does not treat tool installs as projects even when they are git clones", () => { + const nvm = dir(".nvm"); + gitRepo(nvm, "https://github.com/nvm-sh/nvm.git"); + const toolPath = (p: string) => ({ path: p, role: "tool_path" as const, sessionCwd: null, sessionProjectRoot: null }); + expect(resolveProject(toolPath(join(nvm, "nvm.sh")), ctx)).toBeNull(); + expect(resolveProject(asSession(nvm), ctx)).toBeNull(); + expect(resolveProject(toolPath("/opt/homebrew/bin/git"), ctx)).toBeNull(); + expect(resolveProject(toolPath("/usr/local/Homebrew/Library/Homebrew/brew.sh"), ctx)).toBeNull(); + expect(resolveProject(toolPath("C:\\Program Files\\Git\\bin\\git.exe"), ctx)).toBeNull(); + + const app = dir("code", "app"); + gitRepo(app, "git@github.com:me/app.git"); + const pkg = dir("code", "app", "node_modules", "left-pad"); + gitRepo(pkg, "https://github.com/x/left-pad.git"); + expect(resolveProject(toolPath(join(pkg, "index.js")), ctx)).toMatchObject({ identity: "github.com/me/app" }); + }); + + it("still takes a folder the user selected inside a hidden folder", () => { + const notes = dir(".config", "notes"); + ctx.userProjects = [{ rootPath: notes, name: "notes" }]; + expect(resolveProject(asSession(notes), ctx)).toMatchObject({ source: "user_selected", checkoutRoot: notes }); + }); + + it("does not treat the home directory, Desktop, Documents or Downloads themselves as projects", () => { + for (const container of [home, dir("Desktop"), dir("Documents"), dir("Downloads")]) { + expect(resolveProject(asSession(container), ctx)).toBeNull(); + } + }); + + it("ignores a git repository rooted at the home directory and falls back to the folder", () => { + gitRepo(home, "git@github.com:me/dotfiles.git"); + const notes = dir("notes"); + expect(resolveProject(asSession(notes), ctx)).toMatchObject({ source: "folder", checkoutRoot: notes }); + expect(resolveProject(asSession(home), ctx)).toBeNull(); + }); + + it("prefers the nearer root when a user-selected folder and a repository overlap", () => { + const repo = dir("code", "mono"); + gitRepo(repo, "git@github.com:org/mono.git"); + const selectedInside = dir("code", "mono", "packages", "app"); + const outer = dir("work"); + const repoInside = dir("work", "lib"); + gitRepo(repoInside, "git@github.com:org/lib.git"); + const withSelections: ResolverContext = { + ...ctx, + userProjects: [{ rootPath: selectedInside, name: "App" }, { rootPath: outer, name: "Work" }] + }; + expect(resolveProject(asSession(join(selectedInside, "src")), withSelections)).toMatchObject({ + source: "user_selected", checkoutRoot: selectedInside, displayName: "App" + }); + expect(resolveProject(asSession(repoInside), withSelections)).toMatchObject({ + source: "git", identity: "github.com/org/lib" + }); + }); + + it("counts a user-selected folder that is also the repository root once, with the git identity", () => { + const repo = dir("code", "same"); + gitRepo(repo, "git@github.com:org/same.git"); + const withSelection: ResolverContext = { ...ctx, userProjects: [{ rootPath: repo, name: "我的项目" }] }; + expect(resolveProject(asSession(repo), withSelection)).toEqual({ + identity: "github.com/org/same", + checkoutRoot: repo, + source: "user_selected", + displayName: "我的项目" + }); + }); + + it("compares by path components, so /repo does not contain /repo2", () => { + const repo = dir("code", "repo"); + gitRepo(repo, "git@github.com:org/repo.git"); + const sibling = dir("code", "repo2"); + expect(resolveProject(asSession(sibling), ctx)).toMatchObject({ source: "folder", checkoutRoot: sibling }); + }); + + it("resolves a tool path that does not exist yet through its nearest existing parent", () => { + const repo = dir("code", "repo"); + gitRepo(repo, "git@github.com:org/repo.git"); + expect(resolveProject({ + path: join(repo, "src", "new-dir", "new-file.ts"), role: "tool_path", sessionCwd: repo, sessionProjectRoot: repo + }, ctx)).toMatchObject({ identity: "github.com/org/repo" }); + }); + + it("falls back to the session project for a tool path under the session folder and ignores unrelated tool paths", () => { + const folder = dir("Desktop", "reviews"); + const file = join(folder, "paper-1", "notes.md"); + expect(resolveProject({ path: file, role: "tool_path", sessionCwd: folder, sessionProjectRoot: folder }, ctx)) + .toMatchObject({ source: "folder", checkoutRoot: folder }); + expect(resolveProject({ path: join(dir("elsewhere"), "x.txt"), role: "tool_path", sessionCwd: folder, sessionProjectRoot: folder }, ctx)) + .toBeNull(); + }); + + it("resolves a symlinked path to the same project as its target", () => { + const repo = dir("code", "real"); + gitRepo(repo, "git@github.com:org/real.git"); + const link = join(home, "link-to-real"); + symlinkSync(repo, link); + expect(resolveProject(asSession(link), ctx)).toMatchObject({ identity: "github.com/org/real", checkoutRoot: repo }); + }); + + it("returns null for a remote workspace", () => { + expect(resolveProject(asSession(dir("Desktop", "x")), { ...ctx, workspaceIsLocal: false })).toBeNull(); + }); +}); + +describe("isPathWithin", () => { + it("compares Windows paths by drive and components", () => { + expect(isPathWithin("C:\\work\\repo", "C:\\work\\repo\\src\\a.ts", path.win32)).toBe(true); + expect(isPathWithin("C:\\work\\repo", "c:\\work\\repo\\src", path.win32)).toBe(true); + expect(isPathWithin("C:\\work\\repo", "C:\\work\\repo2\\a.ts", path.win32)).toBe(false); + expect(isPathWithin("C:\\work\\repo", "D:\\work\\repo\\a.ts", path.win32)).toBe(false); + }); + + it("compares POSIX paths by components", () => { + expect(isPathWithin("/work/repo", "/work/repo")).toBe(true); + expect(isPathWithin("/work/repo", "/work/repo2")).toBe(false); + }); +}); diff --git a/Memory/tests/service/l3-field/scan-l3.test.ts b/Memory/tests/service/l3-field/scan-l3.test.ts new file mode 100644 index 000000000..067f1b29e --- /dev/null +++ b/Memory/tests/service/l3-field/scan-l3.test.ts @@ -0,0 +1,197 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import type { SourceTurnCompleteRequest } from "../../../src/types.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +function setup(enabled = true) { + const root = fixture.createTestRoot("scan-l3-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: "8".repeat(64), homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + return { db, service, repos: new Repositories(db.db), project }; +} + +function sourceTurn( + channel: SourceTurnCompleteRequest["channel"], + conversationId: string, + index: number, + workspacePath?: string +): SourceTurnCompleteRequest { + const minute = String(index).padStart(2, "0"); + return { + namespace: { source: "codex", profileId: "default", userId: "scan-user" }, + source: "codex", + channel, + ...(workspacePath ? { workspacePath } : {}), + sourceTurn: { + source: "codex", profileId: "default", conversationId, turnId: `${conversationId}-turn-${index}`, + startedAt: `2099-01-01T10:${minute}:00.000Z`, completedAt: `2099-01-01T10:${minute}:30.000Z`, + sequence: index, completionEvidence: `task_complete:${index}` + }, + query: `Run the project tests, step ${index}.`, + answer: "The tests ran and one failed because vitest was missing.", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: workspacePath } }], + toolResults: [{ id: `call-${index}`, output: "sh: vitest: command not found" }] + }; +} + +function sessionOf(repos: Repositories, conversationId: string) { + const row = repos.db.prepare(`SELECT id FROM sessions WHERE conversation_id = ? OR host_session_key = ?`) + .get(conversationId, conversationId) as { id: string }; + return repos.runtime.getSession(row.id)!; +} + +describe("scanned sessions enter L3", () => { + it("opens a scanned conversation on the L3 v2 protocol with Work Memory off, but leaves Hook sessions alone", async () => { + const { service, repos, project } = setup(); + expect((await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project))).status).toBe("stored"); + expect((await service.completeSourceTurn(sourceTurn("hook", "hooked", 1, project))).status).toBe("stored"); + expect(sessionOf(repos, "scanned").meta).toMatchObject({ l3_world_model_protocol_version: 2, work_memory_disabled: true }); + expect(sessionOf(repos, "hooked").meta.l3_world_model_protocol_version).toBeUndefined(); + }); + + it("never extracts Work Memory for a scanned session, on idle, new task or close", async () => { + const { service, repos, project } = setup(); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 2, project)); + // Its idle flush (armed for the new L3) comes due and runs: it hands turns to L3, not to Work Memory. + const quietSince = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString(); + repos.db.prepare(`UPDATE l3_world_model_input_traces SET created_at = ?`).run(quietSince); + repos.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(quietSince); + await service.runWorkerOnce(50); + service.closeSession(sessionOf(repos, "scanned").id); + const workMemoryJobs = repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_extract'` + ).get() as { n: number }; + expect(workMemoryJobs.n).toBe(0); + expect(repos.db.prepare(`SELECT trigger FROM l3_unit_batches`).all()).toEqual([{ trigger: "session_idle" }]); + }); + + it("freezes what a scan pass added into new-L3 batches, and nothing when the pass added nothing", async () => { + const { service, repos, project } = setup(); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 2, project)); + const session = sessionOf(repos, "scanned"); + expect(service.freezeScannedL3Sessions([session.id, session.id])).toBe(1); + const batches = repos.db.prepare(`SELECT trigger, l1_memory_ids_json FROM l3_unit_batches`).all() as Array<{ trigger: string; l1_memory_ids_json: string }>; + expect(batches).toHaveLength(1); + expect(batches[0]!.trigger).toBe("scan_pass"); + expect(JSON.parse(batches[0]!.l1_memory_ids_json).length).toBeGreaterThanOrEqual(2); + expect(repos.runtime.listJobs("queued", 100).filter((job) => job.jobType === "l3_unit_update")).toHaveLength(1); + + expect(service.freezeScannedL3Sessions([session.id])).toBe(0); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 3, project)); + expect(service.freezeScannedL3Sessions([session.id])).toBe(1); + }); + + it("lets a live Hook take over a conversation a scan saw first", async () => { + const { service, repos, project } = setup(); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "resumed", 1, project)); + const scanned = sessionOf(repos, "resumed"); + const hook = service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + namespace: { source: "codex", profileId: "default", userId: "scan-user", sessionKey: "codex-memory-resumed" } + }); + const result = await service.completeSourceTurn({ ...sourceTurn("hook", "resumed", 2, project), sessionId: hook.sessionId }); + expect(result.status).toBe("stored"); + expect(result.result?.sessionId).toBe(hook.sessionId); + expect(repos.runtime.getSession(scanned.id)!.status).toBe("closed"); + expect(repos.runtime.getSession(hook.sessionId)!.meta.work_memory_disabled).toBeUndefined(); + const batches = repos.db.prepare(`SELECT session_id FROM l3_unit_batches`).all() as Array<{ session_id: string }>; + expect(batches.map((batch) => batch.session_id)).toEqual([scanned.id]); + const idleFlush = repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush' AND session_id = ?` + ).get(hook.sessionId) as { n: number }; + expect(idleFlush.n).toBe(1); + // Later turns follow the Hook Session. + expect((await service.completeSourceTurn({ ...sourceTurn("hook", "resumed", 3, project), sessionId: hook.sessionId })).result?.sessionId) + .toBe(hook.sessionId); + }); + + it("lets a scan freeze the scanned sessions of its source, also after an account switch, but not another source's", async () => { + const { service, repos, project } = setup(); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "imported", 1, project)); + const session = sessionOf(repos, "imported"); + const envelope = (userId: string, source: string) => ({ + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", adapterId: "memmy-desktop", source, + namespace: { source, profileId: "default", userId }, sessionIds: [session.id] + }); + expect(service.freezeScanSessions(envelope("someone-else", "claude_code")).batches).toBe(0); + // Capture kept the conversation's earlier owner after the switch; the hand-off follows it. + expect(service.freezeScanSessions(envelope("switched-account", "codex")).batches).toBe(1); + }); + + it("arms the idle flush of a scanned session only while the new L3 is on", async () => { + const flushOf = (repos: Repositories, sessionId: string) => repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush' AND session_id = ? AND status = 'queued'` + ).get(sessionId) as { n: number }; + const on = setup(); + await on.service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, on.project)); + expect(flushOf(on.repos, sessionOf(on.repos, "scanned").id)).toEqual({ n: 1 }); + const off = setup(false); + await off.service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, off.project)); + expect(off.repos.db.prepare(`SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush'`).get()).toEqual({ n: 0 }); + }); + + it("re-arms the idle flush on worker start for sessions whose turns never reached L3, as after a crash mid-scan", async () => { + const { service, repos, project } = setup(); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "crashed", 1, project)); + const sessionId = sessionOf(repos, "crashed").id; + // The process stopped before the scan's hand-off and the flush job was lost with an old build. + repos.db.prepare(`DELETE FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush'`).run(); + service.reconcileWorkerStartup(); + expect(repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush' AND session_id = ? AND status = 'queued'` + ).get(sessionId)).toEqual({ n: 1 }); + expect(repos.db.prepare(`SELECT COUNT(*) AS n FROM l3_unit_batches`).get()).toEqual({ n: 0 }); + }); + + it("keeps a session scanned while the switch was on out of the old L3 after it is switched off", async () => { + const on = setup(); + await on.service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, on.project)); + const sessionId = sessionOf(on.repos, "scanned").id; + const reopen = (enabled: boolean) => { + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + return fixture.createTestMemoryService({ + db: on.db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: "8".repeat(64), homeDir: join(on.project, "..", ".."), tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + }; + reopen(false).closeSession(sessionId); + const count = (table: string) => (on.repos.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; + expect([count("l3_world_model_evidence_batches"), count("l3_unit_batches")]).toEqual([0, 0]); + expect(on.repos.runtime.listJobs("queued", 100).filter((job) => job.jobType === "l3_world_model_update")).toEqual([]); + // Switched on again, the waiting turns still reach the new L3. + expect(reopen(true).freezeScannedL3Sessions([sessionId])).toBe(1); + }); + + it("keeps scans out of L3 when the switch is off, as before", async () => { + const { service, repos, project } = setup(false); + await service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)); + const session = sessionOf(repos, "scanned"); + expect(session.meta.l3_world_model_protocol_version).toBeUndefined(); + expect(session.meta.work_memory_disabled).toBeUndefined(); + expect(service.freezeScannedL3Sessions([session.id])).toBe(0); + service.closeSession(session.id); + const count = (table: string) => (repos.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; + expect([count("l3_world_model_input_traces"), count("l3_world_model_evidence_batches"), count("l3_unit_batches")]).toEqual([0, 0, 0]); + expect(repos.runtime.listJobs("queued", 100).filter((job) => job.jobType.startsWith("l3_"))).toEqual([]); + }); +}); diff --git a/Memory/tests/service/l3-field/text-units.test.ts b/Memory/tests/service/l3-field/text-units.test.ts new file mode 100644 index 000000000..0d2ae03fd --- /dev/null +++ b/Memory/tests/service/l3-field/text-units.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { countL3Units } from "../../../src/service/l3-field/text-units.js"; + +describe("countL3Units", () => { + it("counts CJK characters one by one", () => { + expect(countL3Units("这个仓库用 SSH 拉代码会失败")).toBe(12); + }); + + it("counts other text by whitespace-separated words", () => { + expect(countL3Units("pytest is the test runner here")).toBe(6); + }); + + it("counts mixed text by both rules and ignores punctuation-only tokens", () => { + expect(countL3Units("用 npm ci 安装依赖 , 不用 npm install")).toBe(11); + }); + + it("returns zero for blank text", () => { + expect(countL3Units(" \n ")).toBe(0); + }); +}); diff --git a/Memory/tests/service/l3-field/unit-profile.test.ts b/Memory/tests/service/l3-field/unit-profile.test.ts new file mode 100644 index 000000000..7067ce027 --- /dev/null +++ b/Memory/tests/service/l3-field/unit-profile.test.ts @@ -0,0 +1,137 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/index.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { ProjectEnvironmentService } from "../../../src/service/project-environment/project-environment-service.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; + +const cleanups: Array<() => void> = []; +afterEach(() => { + for (const cleanup of cleanups.splice(0)) cleanup(); +}); + +function setup(reply = '{"op":"create","profile":"Node 项目，测试用 vitest"}') { + const db = new MemoryDb({ path: ":memory:" }); + const root = realpathSync(mkdtempSync(join(tmpdir(), "l3-unit-profile-"))); + cleanups.push(() => { + db.close(); + rmSync(root, { recursive: true, force: true }); + }); + const project = join(root, "repo"); + mkdirSync(project); + writeFileSync(join(project, "package.json"), JSON.stringify({ name: "demo", engines: { node: ">=20" }, scripts: { test: "vitest run" } })); + writeFileSync(join(project, "index.ts"), "export const a = 1;\n"); + const repos = new Repositories(db.db); + const complete = vi.fn().mockResolvedValue(reply); + const service = new ProjectEnvironmentService({ repos, llm: testLlm(complete) }); + const unit = repos.l3Field.ensureProjectUnit({ + userId: "u1", + machineId: "m".repeat(64), + project: { identity: "dir:repo", checkoutRoot: project, source: "folder", displayName: "repo" } + }); + const pendingJob = (): EvolutionJobRecord => { + const row = db.db.prepare( + `SELECT id, user_id, payload_json, created_at FROM evolution_jobs + WHERE job_type = 'project_environment_profile' AND status = 'queued' ORDER BY created_at DESC LIMIT 1` + ).get() as { id: string; user_id: string; payload_json: string; created_at: string }; + return { + id: row.id, + jobType: "project_environment_profile", + userId: row.user_id, + payload: JSON.parse(row.payload_json), + createdAt: row.created_at + } as EvolutionJobRecord; + }; + return { db, repos, service, complete, unit, project, pendingJob }; +} + +describe("L3 field cognition project profile", () => { + it("queues at most one pending profile job per unit, and scans a unit at most once in ten minutes", () => { + const { service, unit, project, db } = setup(); + const request = () => service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + expect(request()).toBe(true); + expect(request()).toBe(false); + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 9 * 60_000).toISOString()); + expect(request()).toBe(false); + db.db.prepare(`UPDATE evolution_jobs SET updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 11 * 60_000).toISOString()); + expect(request()).toBe(true); + }); + + it("writes the summary into the unit, refreshes its snapshot, and leaves machine runtime versions out", async () => { + const { service, repos, complete, unit, project, pendingJob } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + const updated = repos.l3Field.getUnit(unit.unitKey)!; + expect(updated.profileText).toBe("Node 项目，测试用 vitest"); + expect(updated.profileFingerprint).toBeTruthy(); + expect(repos.memories.get(updated.snapshotMemoryId!)!.memoryValue).toContain("Node 项目，测试用 vitest"); + const sent = JSON.stringify(complete.mock.calls); + expect(sent).toContain("runtime_probes\\\":[]"); + expect(sent).toContain(">=20"); + expect(repos.l3WorldModels.getScope("u1", "dir:repo")).toBeUndefined(); + }); + + it("skips the model when the scan is unchanged", async () => { + const { service, complete, unit, project, pendingJob, db } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + // Past the ten-minute gap, so the next session open scans again. + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 11 * 60_000).toISOString()); + expect(service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project })).toBe(true); + await service.processProfileJob(pendingJob()); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("does not ask the model again for an unchanged project that had nothing worth a profile (an empty folder)", async () => { + const { service, complete, unit, project, pendingJob, db } = setup('{"op":"noop","profile":""}'); + rmSync(join(project, "package.json")); + rmSync(join(project, "index.ts")); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 11 * 60_000).toISOString()); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("drops a job queued before the unit was deleted, but serves jobs queued after it", async () => { + const { service, repos, complete, unit, project, pendingJob, db } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project, at: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.markUnitDeleted(unit.unitKey, "2026-10-02T00:00:00.000Z"); + await service.processProfileJob(pendingJob()); + expect(complete).not.toHaveBeenCalled(); + expect(repos.l3Field.getUnit(unit.unitKey)!.profileText).toBeUndefined(); + + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE job_type = 'project_environment_profile'`).run(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project, at: "2026-10-03T00:00:00.000Z" }); + await service.processProfileJob(pendingJob()); + expect(repos.l3Field.getUnit(unit.unitKey)!.profileText).toBe("Node 项目，测试用 vitest"); + }); +}); + +function testLlm(complete: LlmClient["complete"]): LlmClient { + return { + config: { + provider: "host", + endpoint: "http://localhost/unused", + model: "l3-unit-profile-test", + apiKey: "", + temperature: 0, + maxTokens: 4096, + timeoutMs: 30_000, + maxRetries: 0, + malformedRetries: 0, + enableThinking: false + }, + isConfigured: () => true, + complete, + completeJson: vi.fn(), + status: () => ({ provider: "host", model: "l3-unit-profile-test", configured: true, remote: false }) + }; +} diff --git a/Memory/tests/service/l3-field/unit-update.test.ts b/Memory/tests/service/l3-field/unit-update.test.ts new file mode 100644 index 000000000..8899c2502 --- /dev/null +++ b/Memory/tests/service/l3-field/unit-update.test.ts @@ -0,0 +1,662 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { ModelHttpError } from "../../../src/model/http.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { L3UnitUpdatePipeline } from "../../../src/service/l3-field/unit-update-pipeline.js"; +import { validateL3UnitOps, type L3UpdateGroup } from "../../../src/service/l3-field/unit-ops.js"; +import type { EvidenceFragment, L3ItemRecord, L3UnitRecord } from "../../../src/service/l3-field/types.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const MACHINE_ID = "d".repeat(64); +type Reply = (input: ModelInput, call: number) => unknown; + +interface ModelInput { + evidence: Array<{ id: string; type: string; candidate_units: string[]; paths: string[] }>; + machine: { items: Array<{ id: string; content: string }> }; + projects: Array<{ unit_key: string; name: string; items: Array<{ id: string }> }>; + retry?: string; +} + +function setup(options: { enabled?: boolean; reply?: Reply } = {}) { + const root = fixture.createTestRoot("l3-unit-update-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + const other = join(home, "code", "other-repo"); + mkdirSync(project, { recursive: true }); + mkdirSync(other, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = options.enabled ?? true; + const environment = () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }); + const service = fixture.createTestMemoryService({ + db, + mode: "dev", + config, + l3FieldEnvironment: environment, + l3MachineProbe: () => new Promise(() => undefined) + }); + const repos = new Repositories(db.db); + const inputs: ModelInput[] = []; + const reply = options.reply ?? defaultReply; + const complete = vi.fn(async (messages) => { + const input = JSON.parse(messages[1]!.content) as ModelInput; + inputs.push(input); + return JSON.stringify(reply(input, inputs.length)); + }); + const pipeline = new L3UnitUpdatePipeline({ repos, llm: testLlm(complete), environment }); + return { db, service, repos, pipeline, complete, inputs, home, project, other }; +} + +/** A tool error becomes a project fact; the user's words become a project rule. */ +const defaultReply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool"); + const user = input.evidence.find((fragment) => fragment.type === "user_text"); + const ops = []; + if (input.projects.length && tool) { + ops.push({ op: "add", unit: "project", kind: "env_fact", content: "npm test 需要先 npm ci -> 否则找不到 vitest", evidence: [tool.id] }); + } + if (input.projects.length && user) { + ops.push({ op: "add", unit: "project", kind: "project_constraint", content: "改完代码要跑 npm test", evidence: [user.id] }); + } + return { reason: "test", ops }; +}; + +async function captureTurn( + service: ReturnType["service"], + sessionKey: string, + cwd: string, + turns: Array<{ query: string; workdir?: string }> = [{ query: "以后改完代码都要跑 npm test" }] +) { + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(cwd).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey, userId: "u1" } + }); + for (const [index, turn] of turns.entries()) { + await service.completeTurn(`${sessionKey}-turn-${index}`, { + sessionId: opened.sessionId, + query: turn.query, + answer: "好的", + status: "succeeded", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: turn.workdir ?? cwd }, success: false }], + toolResults: [{ id: `call-${index}`, name: "exec_command", output: "sh: vitest: command not found", exitCode: 127 }] + }); + } + service.closeSession(opened.sessionId); + return opened; +} + +function queuedUnitJob(repos: Repositories): EvolutionJobRecord { + const job = repos.runtime.listJobs("queued", 100).find((candidate) => candidate.jobType === "l3_unit_update"); + if (!job) throw new Error("expected a queued l3_unit_update job"); + return job; +} + +function projectUnitKey(repos: Repositories, name = "demo"): string { + return (repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project' AND display_name = ?`).get(name) as { unit_key: string }).unit_key; +} + +describe("new L3 update pipeline", () => { + it("freezes a session into a new-L3 batch whose job writes items the next session sees", async () => { + const { service, repos, pipeline, project } = setup(); + await captureTurn(service, "e2e", project); + expect(repos.runtime.listJobs("queued", 100).some((job) => job.jobType === "l3_world_model_update")).toBe(false); + const job = queuedUnitJob(repos); + expect(job.scopeKey).toBe(repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }).unitKey); + + await pipeline.process(job); + const items = repos.l3Field.activeItems(projectUnitKey(repos)); + expect(items.map((item) => [item.kind, item.content]).sort()).toEqual([ + ["env_fact", "npm test 需要先 npm ci -> 否则找不到 vitest"], + ["project_constraint", "改完代码要跑 npm test"] + ]); + expect(repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + + const next = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: "e2e-next", userId: "u1" } + }); + const context = service.l3WorldModelContext(next.sessionId, { + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", + adapterId: "codex-memory", + source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "e2e-next", userId: "u1", projectId: next.projectId! } + }); + expect(context.renderedContext).toContain("当前项目：demo（文件夹）"); + expect(context.renderedContext).toContain("## 项目约束\n- 改完代码要跑 npm test"); + }); + + it("keeps the old batches and jobs when the switch is off", async () => { + const { service, repos, project } = setup({ enabled: false }); + await captureTurn(service, "flag-off", project); + expect(repos.runtime.listJobs("queued", 100).some((job) => job.jobType === "l3_world_model_update")).toBe(true); + expect(repos.runtime.listJobs("queued", 100).some((job) => job.jobType === "l3_unit_update")).toBe(false); + expect((repos.db.prepare(`SELECT COUNT(*) AS n FROM l3_unit_batches`).get() as { n: number }).n).toBe(0); + }); + + it("writes a batch only once even if its job runs again", async () => { + const { service, repos, pipeline, project, complete } = setup(); + await captureTurn(service, "twice", project); + const job = queuedUnitJob(repos); + await pipeline.process(job); + await pipeline.process(job); + expect(repos.l3Field.activeItems(projectUnitKey(repos))).toHaveLength(2); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("regenerates once with the reasons, and fails without writing when rejected twice", async () => { + const userFact: Reply = (input) => ({ + reason: "bad", + ops: [{ op: "add", unit: "project", kind: "env_fact", content: "没有工具输出的事实", evidence: [input.evidence.find((f) => f.type === "user_text")!.id] }] + }); + const fixed = setup({ reply: (input, call) => (call === 1 ? userFact(input, call) : defaultReply(input, call)) }); + await captureTurn(fixed.service, "retry-ok", fixed.project); + await fixed.pipeline.process(queuedUnitJob(fixed.repos)); + expect(fixed.inputs[1]!.retry).toContain("env_fact needs a cited tool output or error"); + expect(fixed.repos.l3Field.activeItems(projectUnitKey(fixed.repos))).toHaveLength(2); + + const broken = setup({ reply: userFact }); + await captureTurn(broken.service, "retry-bad", broken.project); + const job = queuedUnitJob(broken.repos); + await expect(broken.pipeline.process(job)).rejects.toThrow("rejected twice"); + expect(broken.repos.l3Field.activeItems(projectUnitKey(broken.repos))).toHaveLength(0); + expect(broken.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("queued"); + }); + + it("drops results for a unit reset after the batch was frozen", async () => { + const { service, repos, pipeline, project } = setup(); + await captureTurn(service, "reset", project); + const job = queuedUnitJob(repos); + repos.l3Field.markUnitDeleted(projectUnitKey(repos), new Date(Date.now() + 1000).toISOString()); + await pipeline.process(job); + expect(repos.l3Field.activeItems(projectUnitKey(repos))).toHaveLength(0); + expect(repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + }); + + it("does not let what was said before a unit reset refill it, even when frozen after the reset", async () => { + const { service, repos, pipeline, project } = setup(); + await captureTurn(service, "late-freeze", project); + const job = queuedUnitJob(repos); + repos.l3Field.markUnitDeleted(projectUnitKey(repos), new Date(Date.now() + 1000).toISOString()); + repos.db.prepare(`UPDATE l3_unit_batches SET created_at = ?`).run(new Date(Date.now() + 2000).toISOString()); + await pipeline.process(job); + expect(repos.l3Field.activeItems(projectUnitKey(repos))).toHaveLength(0); + }); + + it("does not let older evidence replace or delete an item newer evidence wrote, as a late rerun would", async () => { + const reply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + const [first, second] = input.projects[0]!.items; + return { reason: "old", ops: [ + { op: "replace", ids: [first!.id], content: "Node 18", evidence: [tool.id] }, + { op: "delete", id: second!.id, evidence: [tool.id] } + ] }; + }; + const { service, repos, pipeline, project } = setup({ reply }); + await captureTurn(service, "rerun", project, [{ query: "看看" }]); + const job = queuedUnitJob(repos); + const unitKey = projectUnitKey(repos); + const newer = (content: string) => repos.l3Field.insertItem({ + unitKey, kind: "env_fact", content, evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2099-01-01T00:00:00.000Z" + }); + const kept = [newer("Node 20"), newer("用 pnpm")]; + await pipeline.process(job); + expect(kept.map((item) => repos.l3Field.getItem(item.id)!.status)).toEqual(["active", "active"]); + expect(repos.l3Field.activeItems(unitKey).map((item) => item.content)).not.toContain("Node 18"); + }); + + it("drops the result of a run that outlived its lease once another run dead-lettered the batch", async () => { + let db: ReturnType["db"] | undefined; + const reply: Reply = (input, call) => { + // While this run waits on the model, a newer run of the same job fails and dead-letters the batch. + if (call === 1) db!.db.prepare(`UPDATE l3_unit_batches SET status = 'failed'`).run(); + return defaultReply(input, call); + }; + const env = setup({ reply }); + db = env.db; + await captureTurn(env.service, "late-run", env.project); + const job = queuedUnitJob(env.repos); + await expect(env.pipeline.process(job)).rejects.toThrow(/result dropped/); + expect(env.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("failed"); + expect(env.repos.l3Field.activeItems(projectUnitKey(env.repos))).toEqual([]); + }); + + it("lets only the run that holds the lease write: a run that outlived it leaves job and batch alone", async () => { + let db: ReturnType["db"] | undefined; + const reply: Reply = (input, call) => { + // While this run waits on the model its lease runs out, the job is leased again and later put back + // in the queue by a rerun: the batch is "queued" again, but this run no longer holds the job. + if (call === 1) db!.db.prepare(`UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, + lease_owner = NULL, lease_token = NULL WHERE job_type = 'l3_unit_update'`).run(); + return defaultReply(input, call); + }; + const env = setup({ reply }); + db = env.db; + await captureTurn(env.service, "lease-lost", env.project); + env.db.db.prepare(`UPDATE memory_processing_state SET state = 'ready'`).run(); + const [job] = env.repos.runtime.leaseQueuedJobs(10, 60, undefined, false, ["l3_unit_update"]); + expect(typeof job!.leaseToken).toBe("string"); + await expect(env.pipeline.process(job!)).rejects.toThrow(/leased again/); + expect(env.repos.l3Field.getBatch(job!.payload.batchId as string)!.status).toBe("queued"); + expect(env.repos.l3Field.activeItems(projectUnitKey(env.repos))).toEqual([]); + expect(env.repos.runtime.getJob(job!.id)!.status).toBe("queued"); + }); + + it("drops ops citing a turn whose L1 the user deleted while the model was answering", async () => { + let db: ReturnType["db"] | undefined; + const reply: Reply = (input, call) => { + if (call === 1) db!.db.prepare(`UPDATE memories SET deleted_at = ?, status = 'deleted' WHERE memory_layer = 'L1'`).run(new Date().toISOString()); + return defaultReply(input, call); + }; + const env = setup({ reply }); + db = env.db; + await captureTurn(env.service, "deleted-l1", env.project); + const job = queuedUnitJob(env.repos); + await env.pipeline.process(job); + expect(env.repos.l3Field.activeItems(projectUnitKey(env.repos))).toEqual([]); + expect(env.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + }); + + it("does not replace or delete an item that is already gone when the batch is written", async () => { + let gone: string | undefined; + let db: ReturnType["db"] | undefined; + const reply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + const [first, second] = input.projects[0]!.items; + // Another writer removes the first item while the model is answering. + db!.db.prepare(`UPDATE l3_items SET status = 'inactive', inactive_at = ? WHERE id = ?`).run(new Date().toISOString(), first!.id); + gone = first!.id; + return { reason: "x", ops: [ + { op: "replace", ids: [first!.id], content: "改写后的规则", evidence: [tool.id] }, + { op: "delete", id: second!.id, evidence: [tool.id] } + ] }; + }; + const env = setup({ reply }); + db = env.db; + await captureTurn(env.service, "gone", env.project, [{ query: "看看" }]); + const job = queuedUnitJob(env.repos); + const unitKey = projectUnitKey(env.repos); + for (const content of ["规则一", "规则二"]) { + env.repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content, evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" }); + } + await env.pipeline.process(job); + expect(env.repos.l3Field.activeItems(unitKey).map((item) => item.content)).toEqual([]); + expect(env.repos.l3Field.getItem(gone!)!.replacedBy).toBeUndefined(); + }); + + it("shows a batch what was removed on evidence newer than some of its own: late or spanning yes, behind a backlog no", async () => { + const late = setup(); + await captureTurn(late.service, "late", late.project); + const job = queuedUnitJob(late.repos); + const unitKey = projectUnitKey(late.repos); + const removed = late.repos.l3Field.insertItem({ + unitKey, kind: "project_constraint", content: "提交信息用英文", evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, + evidenceAt: "2099-01-01T00:00:00.000Z" + }); + // A newer conversation, applied while this batch sat in dead letter (or before this old history was + // scanned), deleted the rule. + late.repos.l3Field.inactivateItem(removed.id, { removedEvidenceAt: new Date(Date.now() + 60_000).toISOString() }); + await late.pipeline.process(job); + expect(late.inputs[0]).toMatchObject({ removed_by_newer_evidence: [{ kind: "project_constraint", content: "提交信息用英文" }] }); + + // Batches in order behind a backlog: the rule was removed on older evidence than this batch's, + // so this batch is free to bring it back. + const inOrder = setup(); + await captureTurn(inOrder.service, "in-order", inOrder.project); + const olderRemoval = inOrder.repos.l3Field.insertItem({ + unitKey: projectUnitKey(inOrder.repos), kind: "project_constraint", content: "提交信息用中文", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" + }); + inOrder.repos.l3Field.inactivateItem(olderRemoval.id, { removedEvidenceAt: "2026-01-02T00:00:00.000Z", at: new Date(Date.now() + 60_000).toISOString() }); + await inOrder.pipeline.process(queuedUnitJob(inOrder.repos)); + expect(inOrder.inputs[0]).not.toHaveProperty("removed_by_newer_evidence"); + + // One batch spanning the removal: its older turn must not bring the rule back, so it is shown. + const spanning = setup(); + await captureTurn(spanning.service, "spanning", spanning.project, [{ query: "提交信息用英文" }, { query: "再跑一次测试" }]); + const turns = spanning.db.db.prepare(`SELECT id FROM raw_turns ORDER BY created_at, id`).all() as Array<{ id: string }>; + spanning.db.db.prepare(`UPDATE raw_turns SET created_at = '2026-01-01T00:00:00.000Z' WHERE id = ?`).run(turns[0]!.id); + const removedBetween = spanning.repos.l3Field.insertItem({ + unitKey: projectUnitKey(spanning.repos), kind: "project_constraint", content: "提交信息用英文", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2025-12-01T00:00:00.000Z" + }); + spanning.repos.l3Field.inactivateItem(removedBetween.id, { removedEvidenceAt: "2026-06-01T00:00:00.000Z" }); + await spanning.pipeline.process(queuedUnitJob(spanning.repos)); + expect(spanning.inputs[0]).toMatchObject({ + removed_by_newer_evidence: [{ content: "提交信息用英文", removed_at: "2026-06-01T00:00:00.000Z" }] + }); + }); + + it("evicts the item confirmed longest ago when a kind goes over 10", async () => { + const { service, repos, pipeline, project } = setup(); + await captureTurn(service, "evict-seed", project, [{ query: "看看" }]); + const seedJob = queuedUnitJob(repos); + const unitKey = projectUnitKey(repos); + for (let i = 0; i < 10; i += 1) { + repos.l3Field.insertItem({ + unitKey, kind: "project_constraint", content: `规则 ${i}`, + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: `2026-01-${String(10 + i)}T00:00:00.000Z` + }); + } + await pipeline.process(seedJob); + const constraints = repos.l3Field.activeItems(unitKey).filter((item) => item.kind === "project_constraint"); + expect(constraints).toHaveLength(10); + expect(constraints.map((item) => item.content)).not.toContain("规则 0"); + }); + + it("keeps only the first group's change when two groups change the same machine item", async () => { + const reply: Reply = (input) => { + const machineItem = input.machine.items[0]!; + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + return input.projects[0]?.name === "demo" + ? { reason: "a", ops: [{ op: "replace", ids: [machineItem.id], content: "本机 zsh 展开 = 参数 -> 要加引号", evidence: [tool.id] }] } + : { reason: "b", ops: [{ op: "delete", id: machineItem.id, evidence: [tool.id] }] }; + }; + const { service, repos, pipeline, project, other } = setup({ reply }); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const old = repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content: "旧的本机事实", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" + }); + await captureTurn(service, "conflict", project, [{ query: "跑一下", workdir: project }, { query: "那边也跑", workdir: other }]); + await pipeline.process(queuedUnitJob(repos)); + const replaced = repos.l3Field.getItem(old.id)!; + expect(replaced.status).toBe("inactive"); + expect(replaced.replacedBy).toBeTruthy(); + expect(repos.l3Field.getItem(replaced.replacedBy!)!.content).toBe("本机 zsh 展开 = 参数 -> 要加引号"); + }); + + it("marks a batch done when the model changes nothing, and merges several items with one replace", async () => { + const empty = setup({ reply: () => ({ reason: "nothing worth keeping", ops: [] }) }); + await captureTurn(empty.service, "empty", empty.project); + const job = queuedUnitJob(empty.repos); + await empty.pipeline.process(job); + expect(empty.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + + const merge = setup({ + reply: (input) => ({ + reason: "merge", + ops: [{ op: "replace", ids: input.projects[0]!.items.map((entry) => entry.id), content: "合并后的规则", evidence: [input.evidence[0]!.id] }] + }) + }); + await captureTurn(merge.service, "merge", merge.project); + const unitKey = projectUnitKey(merge.repos); + const a = merge.repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "规则 a", evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" }); + const b = merge.repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "规则 b", evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" }); + await merge.pipeline.process(queuedUnitJob(merge.repos)); + const active = merge.repos.l3Field.activeItems(unitKey); + expect(active.map((entry) => entry.content)).toEqual(["合并后的规则"]); + expect(merge.repos.l3Field.getItem(a.id)!.replacedBy).toBe(active[0]!.id); + expect(merge.repos.l3Field.getItem(b.id)!.replacedBy).toBe(active[0]!.id); + }); + + it("writes the probed machine profile at session start", async () => { + const root = fixture.createTestRoot("l3-machine-profile-"); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = true; + const probe = vi.fn(async () => ({ + os: "darwin", osVersion: "26.1", arch: "arm64", shell: "zsh", + tools: { node: "22.11.0", python3: null, git: "2.50.1", npm: null, brew: null, pnpm: null, yarn: null, uv: null, conda: null, docker: null, java: null, go: null, rustc: null } + })); + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: join(root, "home"), tempRoots: [] }), + l3MachineProbe: probe + }); + service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + namespace: { source: "codex", profileId: "default", sessionKey: "probe-1", userId: "u1" } + }); + service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + namespace: { source: "codex", profileId: "default", sessionKey: "probe-2", userId: "u1" } + }); + await vi.waitFor(() => { + const machine = new Repositories(db.db).l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + expect(machine.profileText).toContain("node 22.11.0，git 2.50.1"); + }); + expect(probe).toHaveBeenCalledTimes(1); + }); + + it("shows later groups the machine items earlier groups of the batch added, so they are not written twice", async () => { + const reply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + const added = (input.machine.items as Array<{ id: string; added_in_this_batch?: boolean }>) + .find((entry) => entry.added_in_this_batch); + return added + ? { reason: "already there", ops: [{ op: "confirm", id: added.id, evidence: [tool.id] }] } + : { reason: "new", ops: [{ op: "add", unit: "machine", kind: "env_fact", content: "本机 git 会打印 xcrun_db 警告 -> 结果仍正常", evidence: [tool.id] }] }; + }; + const { service, repos, pipeline, project, other, complete, inputs } = setup({ reply }); + await captureTurn(service, "dedupe", project, [{ query: "跑一下", workdir: project }, { query: "那边也跑", workdir: other }]); + await pipeline.process(queuedUnitJob(repos)); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + expect(repos.l3Field.activeItems(machine.unitKey).map((entry) => entry.content)).toEqual(["本机 git 会打印 xcrun_db 警告 -> 结果仍正常"]); + expect(complete).toHaveBeenCalledTimes(2); + expect(JSON.stringify(inputs[1]!.machine.items)).toContain("\"added_in_this_batch\":true"); + }); + + it("drops old tool calls before the user's own words when the input is too long", async () => { + const { service, repos, pipeline, project, inputs } = setup({ reply: () => ({ reason: "none", ops: [] }) }); + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: "long", userId: "u1" } + }); + for (let index = 0; index < 20; index += 1) { + await service.completeTurn(`long-turn-${index}`, { + sessionId: opened.sessionId, + query: index === 0 ? "这个项目以后提交信息都用中文写" : `继续第 ${index} 步`, + answer: "好的", + status: "succeeded", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: project }, success: false }], + toolResults: [{ id: `call-${index}`, name: "exec_command", output: Array.from({ length: 50 }, () => `error: ${"y".repeat(400)}`).join("\n"), exitCode: 1 }] + }); + } + service.closeSession(opened.sessionId); + const job = repos.runtime.listJobs("queued", 500).find((candidate) => candidate.jobType === "l3_unit_update")!; + vi.stubEnv("MEMMY_LOG_LEVEL", "debug"); + const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + try { + await pipeline.process(job); + } finally { + vi.unstubAllEnvs(); + } + const truncated = stdout.mock.calls.map(([line]) => String(line)).filter((line) => line.includes("input.truncated")); + stdout.mockRestore(); + const evidence = inputs[0]!.evidence as Array<{ type: string; text?: string }>; + expect(evidence.filter((fragment) => fragment.type === "tool").length).toBeLessThan(20); + expect(evidence.some((fragment) => fragment.text === "这个项目以后提交信息都用中文写")).toBe(true); + expect(truncated).toHaveLength(1); + expect(JSON.parse(truncated[0]!).level).toBe("debug"); + expect(truncated[0]).toContain("maxChars=65536"); + expect(truncated[0]).toContain("droppedUserTextFragments=0"); + }); + + it("asks once more with a smaller input when the model rejects the input as too long", async () => { + const sizes: number[] = []; + const reply: Reply = (input) => { + const size = JSON.stringify(input).length; + sizes.push(size); + if (size > 20_000) { + throw new ModelHttpError("model request failed", "host", 400, undefined, "This model's maximum context length is 8192 tokens"); + } + return { reason: "none", ops: [] }; + }; + const { service, repos, pipeline, project, complete } = setup({ reply }); + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: "too-long", userId: "u1" } + }); + for (let index = 0; index < 20; index += 1) { + await service.completeTurn(`too-long-turn-${index}`, { + sessionId: opened.sessionId, + query: `第 ${index} 步`, + answer: "好的", + status: "succeeded", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: project }, success: false }], + toolResults: [{ id: `call-${index}`, name: "exec_command", output: Array.from({ length: 50 }, () => `error: ${"y".repeat(400)}`).join("\n"), exitCode: 1 }] + }); + } + service.closeSession(opened.sessionId); + const job = repos.runtime.listJobs("queued", 500).find((candidate) => candidate.jobType === "l3_unit_update")!; + await pipeline.process(job); + expect(sizes).toHaveLength(2); + expect(sizes[1]!).toBeLessThan(20_000); + expect(repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + expect(complete.mock.calls.every(([, options]) => options?.maxTokens === 16_384)).toBe(true); + }); + + it("does not let a deleted L1 whose summary never finished hold its batch back", async () => { + const { service, repos, project } = setup(); + await captureTurn(service, "deleted-l1", project); + const job = queuedUnitJob(repos); + const l1 = repos.l3Field.getBatch(job.payload.batchId as string)!.l1MemoryIds[0]!; + repos.db.prepare( + `INSERT INTO memory_processing_state (memory_id, state, updated_at) VALUES (?, 'summary_pending', ?) + ON CONFLICT(memory_id) DO UPDATE SET state = 'summary_pending'` + ).run(l1, new Date().toISOString()); + repos.memories.softDelete(l1, new Date().toISOString()); + const leased = repos.runtime.leaseQueuedJobs(50, 60, undefined, false, ["l3_unit_update"]); + expect(leased.map((entry) => entry.id)).toContain(job.id); + }); + + it("does not wake the worker for new-L3 jobs it would not lease while the switch is off", async () => { + const { service, repos, project } = setup({ enabled: false }); + await captureTurn(service, "wake-off", project); + const now = new Date().toISOString(); + repos.db.prepare( + `INSERT INTO evolution_jobs (id, job_type, status, user_id, payload_json, attempts, max_attempts, created_at, updated_at) + VALUES ('held-unit', 'l3_unit_update', 'queued', 'u1', json_object('batchId', 'x', 'runAfter', '2000-01-01T00:00:00.000Z'), 0, 6, ?, ?), + ('held-profile', 'project_environment_profile', 'queued', 'u1', + json_object('mode', 'l3_field', 'runAfter', '2000-01-01T00:00:00.000Z'), 0, 3, ?, ?)` + ).run(now, now, now, now); + repos.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE id NOT IN ('held-unit', 'held-profile')`).run(); + // Their runAfter is long past; other wake-ups (budget reconcile) may still be scheduled ahead. + expect(service.nextWorkerRunAt() ?? Number.POSITIVE_INFINITY).toBeGreaterThan(Date.parse(now)); + }); + + it("does not lease a batch while its L1 summary is pending, and leaves jobs queued while the switch is off", async () => { + const { service, repos, project } = setup(); + await captureTurn(service, "lease", project); + const job = queuedUnitJob(repos); + const l1 = (repos.l3Field.getBatch(job.payload.batchId as string)!.l1MemoryIds)[0]!; + const now = new Date().toISOString(); + repos.db.prepare( + `INSERT INTO memory_processing_state (memory_id, state, updated_at) VALUES (?, 'summary_pending', ?) + ON CONFLICT(memory_id) DO UPDATE SET state = 'summary_pending'` + ).run(l1, now); + const leasedIds = () => repos.runtime.leaseQueuedJobs(50, 60, undefined, false, ["l3_unit_update"]).map((leased) => leased.id); + expect(leasedIds()).not.toContain(job.id); + + repos.db.prepare(`UPDATE memory_processing_state SET state = 'ready' WHERE memory_id = ?`).run(l1); + const offLeased = repos.runtime.leaseQueuedJobs(50, 60, undefined, false, undefined, ["l3_unit_update"]); + expect(offLeased.some((leased) => leased.jobType === "l3_unit_update")).toBe(false); + expect(offLeased.some((leased) => leased.payload.mode === "l3_field")).toBe(false); + expect(leasedIds()).toContain(job.id); + }); +}); + +describe("new L3 op validation", () => { + const machine = unit("machine", "m"); + const demo = unit("project", "p", { displayName: "demo-app", checkoutRoot: "/Users/x/code/demo-app" }); + const fragment = (id: string, over: Partial = {}): EvidenceFragment => ({ + id, rawTurnId: "r", l1MemoryId: "l1", toolName: "exec_command", input: null, output: "error: boom", + isError: true, paths: [], candidateUnitKeys: ["p"], at: "2026-10-01T00:00:00.000Z", ...over + }); + const group: L3UpdateGroup = { + machine, + project: demo, + fragments: [ + fragment("tool-p"), + fragment("tool-other", { candidateUnitKeys: ["other"] }), + fragment("user", { toolName: null, output: "以后都这样", isError: false }) + ], + l1: [] + }; + const items = new Map([ + ["mi", item("mi", "m", "env_fact")], + ["pi", item("pi", "p", "project_constraint")], + ["pf", item("pf", "p", "env_fact")], + ["xi", item("xi", "elsewhere", "env_fact")] + ]); + const problems = (ops: unknown[]) => validateL3UnitOps(ops as never, group, items).problems; + + it("rejects ids of other units, inactive ids and an id changed twice", () => { + expect(problems([{ op: "delete", id: "xi", evidence: ["tool-p"] }])[0]).toContain("not an active item"); + expect(problems([{ op: "confirm", id: "gone", evidence: ["tool-p"] }])[0]).toContain("not an active item"); + expect(problems([ + { op: "confirm", id: "pi", evidence: ["user"] }, + { op: "delete", id: "pi", evidence: ["user"] } + ])[0]).toContain("already changed"); + }); + + it("rejects replace across units or kinds, and evidence from outside the input", () => { + expect(problems([{ op: "replace", ids: ["mi", "pf"], content: "x", evidence: ["tool-p"] }])[0]).toContain("one unit and kind"); + expect(problems([{ op: "replace", ids: ["pi", "pf"], content: "x", evidence: ["tool-p"] }])[0]).toContain("one unit and kind"); + expect(problems([{ op: "confirm", id: "pi", evidence: ["nope"] }])[0]).toContain("not a fragment"); + }); + + it("needs a tool output from this project for a project fact", () => { + expect(problems([{ op: "add", unit: "project", kind: "env_fact", content: "x", evidence: ["tool-other"] }])[0]).toContain("touched this unit"); + expect(problems([{ op: "add", unit: "project", kind: "env_fact", content: "x", evidence: ["user"] }])[0]).toContain("touched this unit"); + expect(problems([{ op: "add", unit: "project", kind: "env_fact", content: "x", evidence: ["tool-p"] }])).toEqual([]); + expect(problems([{ op: "add", unit: "machine", kind: "env_fact", content: "brew 无写权限", evidence: ["tool-other"] }])).toEqual([]); + }); + + it("keeps machine items about the machine only", () => { + expect(problems([{ op: "add", unit: "machine", kind: "project_constraint", content: "x", evidence: ["user"] }])[0]).toContain("env_fact"); + expect(problems([{ op: "add", unit: "machine", kind: "env_fact", content: "/Users/x/code/demo-app 里 npm 报错", evidence: ["tool-p"] }])[0]).toContain("must not mention"); + expect(problems([{ op: "add", unit: "machine", kind: "env_fact", content: "demo-app 的测试很慢", evidence: ["tool-p"] }])[0]).toContain("must not mention"); + const goGroup = { ...group, project: { ...demo, displayName: "go", checkoutRoot: "/Users/x/go" } }; + expect(validateL3UnitOps([{ op: "add", unit: "machine", kind: "env_fact", content: "go 1.22 编译报错", evidence: ["tool-p"] }] as never, goGroup, items).problems).toEqual([]); + }); + + it("asks for unit machine when no project is listed, and rejects an unknown unit_key", () => { + const noProject = { ...group, project: null }; + expect(validateL3UnitOps([{ op: "add", unit: "project", kind: "project_constraint", content: "x", evidence: ["user"] }] as never, noProject, items).problems[0]).toContain("unit must be machine"); + expect(problems([{ op: "add", unit: "project", unit_key: "zzz", kind: "project_constraint", content: "x", evidence: ["user"] }])[0]).toContain("not a listed project"); + }); +}); + +function unit(kind: "machine" | "project", unitKey: string, over: Partial = {}): L3UnitRecord { + return { unitKey, userId: "u1", machineId: MACHINE_ID, kind, nextSeq: 1, createdAt: "", updatedAt: "", ...over } as L3UnitRecord; +} + +function item(id: string, unitKey: string, kind: L3ItemRecord["kind"]): L3ItemRecord { + return { + id, unitKey, kind, content: id, evidence: { fragmentIds: [], l1MemoryIds: [] }, + evidenceAt: "", lastConfirmedAt: "", status: "active", createdAt: "", updatedAt: "" + }; +} + +function testLlm(complete: LlmClient["complete"]): LlmClient { + return { + config: { + provider: "host", endpoint: "http://localhost/unused", model: "l3-unit-update-test", apiKey: "", + temperature: 0, maxTokens: 4096, timeoutMs: 30_000, maxRetries: 0, malformedRetries: 0, enableThinking: false + }, + isConfigured: () => true, + complete, + completeJson: vi.fn(), + status: () => ({ provider: "host", model: "l3-unit-update-test", configured: true, remote: false }) + }; +} diff --git a/Memory/tests/service/project-environment/local-scanner.test.ts b/Memory/tests/service/project-environment/local-scanner.test.ts index 6736cce8f..74b2e0b48 100644 --- a/Memory/tests/service/project-environment/local-scanner.test.ts +++ b/Memory/tests/service/project-environment/local-scanner.test.ts @@ -39,6 +39,14 @@ describe("local project scanner", () => { ); }); + it("can skip running the declared runtimes", async () => { + const { root, uri } = await fixture(); + await writeFile(join(root, "package.json"), JSON.stringify({ engines: { node: ">=22" } })); + const result = await scanLocalProject(uri, { withRuntimeProbes: false }); + expect(result.runtimeProbes).toEqual([]); + expect(result.entries.map((entry) => entry.relativePath)).toEqual(["package.json"]); + }); + it("excludes gitignored, fixed, sensitive, binary, and symlink paths from the inventory", async () => { const { root, uri } = await fixture(); await mkdir(join(root, "node_modules")); diff --git a/Memory/tests/service/session/turn-judge-routing.test.ts b/Memory/tests/service/session/turn-judge-routing.test.ts index ca3453905..06698e80c 100644 --- a/Memory/tests/service/session/turn-judge-routing.test.ts +++ b/Memory/tests/service/session/turn-judge-routing.test.ts @@ -77,6 +77,11 @@ async function settle(service: ReturnType["service"], } } +/** These cases check the old L3 World Model's evidence, which the new L3 replaces when it is on. */ +function withOldL3(config: ReturnType): ReturnType { + return { ...config, algorithm: { ...config.algorithm, l3FieldCognition: { ...config.algorithm.l3FieldCognition, enabled: false } } }; +} + describe("turn.judge.v1 routing (switch on)", () => { it("does not call a model at the start of a turn and reports the relation as pending", async () => { const { llm, judgeCalls, otherCalls } = createTurnJudgeLlm(); @@ -123,7 +128,7 @@ describe("turn.judge.v1 routing (switch on)", () => { it.each(["positive", "negative"])("closes new_task and freezes its %s feedback in L3 evidence", async (polarity) => { const { llm } = createTurnJudgeLlm({ "第二轮": [judged("new_task", polarity, l1For("第二轮：换个事，写求职信"))] }); const l3Evidence = captureL3Evidence(llm); - const { service, db } = createTestService({ llm, config: turnJudgeConfig() }); + const { service, db } = createTestService({ llm, config: withOldL3(turnJudgeConfig()) }); const session = service.openSession({ l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", @@ -191,7 +196,7 @@ describe("turn.judge.v1 routing (switch on)", () => { "C task": [judged("follow_up", "positive", l1For("C task"))] }); const l3Evidence = captureL3Evidence(llm); - const { service, db } = createTestService({ llm, config: turnJudgeConfig() }); + const { service, db } = createTestService({ llm, config: withOldL3(turnJudgeConfig()) }); const session = service.openSession({ l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", namespace: { source: "codex", profileId: "default", userId: "reopen-backlog", sessionKey: "reopen-backlog" } }); const historical = await service.completeTurn("historical", { sessionId: session.sessionId, query: "Historical task", answer: "Historical output" });