From bc82954b5f5ca38dde65027116d3b521e23ade1e Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:23:51 +0200 Subject: [PATCH 1/9] feat(memory): add l3 field cognition tables and flag Add the storage base for itemized machine/project L3, behind algorithm.l3FieldCognition (off by default, local SQLite only): - four new tables (l3_units, l3_items, l3_unit_batches, l3_user_projects) created alongside the existing ones without bumping SCHEMA_VERSION; old L3 tables and rows are untouched - L3FieldRepository for units, items, snapshot memories, unit deletion, inactive-item purge and the user project list; an item removed by a delete or replace keeps when that evidence happened - register the tables for bundle export/import and clear-all; a unit that already exists is kept whole on import (its unit, items, batches, jobs and snapshot from the bundle are skipped; conflictStrategy "error" fails instead; its unfinished batches this database lacks move their sessions' cursors back so those turns are frozen again), and unfinished unit batches are dropped from redacted exports - countL3Units for the per-item length limit Co-Authored-By: Claude Opus 5.5 --- Memory/src/config/index.ts | 20 + Memory/src/service/l3-field/flag.ts | 13 + Memory/src/service/l3-field/text-units.ts | 16 + Memory/src/service/l3-field/types.ts | 75 ++++ Memory/src/storage/l3-field-repository.ts | 382 ++++++++++++++++++ Memory/src/storage/repositories.ts | 108 ++++- Memory/src/storage/schema.ts | 72 ++++ .../repository/l3-field-repository.test.ts | 260 ++++++++++++ .../tests/service/l3-field/text-units.test.ts | 20 + 9 files changed, 964 insertions(+), 2 deletions(-) create mode 100644 Memory/src/service/l3-field/flag.ts create mode 100644 Memory/src/service/l3-field/text-units.ts create mode 100644 Memory/src/service/l3-field/types.ts create mode 100644 Memory/src/storage/l3-field-repository.ts create mode 100644 Memory/tests/repository/l3-field-repository.test.ts create mode 100644 Memory/tests/service/l3-field/text-units.test.ts diff --git a/Memory/src/config/index.ts b/Memory/src/config/index.ts index afefd30dd..6c8f8f4ed 100644 --- a/Memory/src/config/index.ts +++ b/Memory/src/config/index.ts @@ -240,6 +240,11 @@ export interface AlgorithmConfig { followUpMode: "merge_follow_ups" | "episode_per_turn"; mergeMaxGapMs: number; }; + /** Itemized machine/project L3; off by default while it is being tested. */ + l3FieldCognition: { + enabled: boolean; + inactiveRetentionDays: number; + }; retrieval: { tier1TopK: number; tier2TopK: number; @@ -499,6 +504,10 @@ export const DEFAULT_MEMMY_CONFIG: MemmyConfig = { followUpMode: "merge_follow_ups", mergeMaxGapMs: 2 * 60 * 60 * 1000 }, + l3FieldCognition: { + enabled: false, + inactiveRetentionDays: 15 + }, retrieval: { tier1TopK: 3, tier2TopK: 5, @@ -644,6 +653,9 @@ function configFromEnv(): Record { skillInjectionMode: process.env.MEMMY_SKILL_INJECTION_MODE, skillSummaryChars: numberEnv("MEMMY_SKILL_SUMMARY_CHARS"), skillFullMaxChars: numberEnv("MEMMY_SKILL_FULL_MAX_CHARS") + }), + l3FieldCognition: compactRecord({ + enabled: booleanEnv("MEMMY_L3_FIELD_COGNITION") }) }) }); @@ -1148,6 +1160,7 @@ function normalizeAlgorithm(input: Record): AlgorithmConfig { const l3 = asRecord(input.l3Abstraction); const skill = asRecord(input.skill); const session = asRecord(input.session); + const l3FieldCognition = asRecord(input.l3FieldCognition); const retrieval = asRecord(input.retrieval); return { enableMemoryAdd: booleanValue(input.enableMemoryAdd, DEFAULT_MEMMY_CONFIG.algorithm.enableMemoryAdd), @@ -1304,6 +1317,13 @@ function normalizeAlgorithm(input: Record): AlgorithmConfig { followUpMode: "merge_follow_ups", mergeMaxGapMs: numberValue(session.mergeMaxGapMs, DEFAULT_MEMMY_CONFIG.algorithm.session.mergeMaxGapMs) }, + l3FieldCognition: { + enabled: booleanValue(l3FieldCognition.enabled, DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition.enabled), + inactiveRetentionDays: numberValue( + l3FieldCognition.inactiveRetentionDays, + DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition.inactiveRetentionDays + ) + }, retrieval: { tier1TopK: numberValue(retrieval.tier1TopK, DEFAULT_MEMMY_CONFIG.algorithm.retrieval.tier1TopK), tier2TopK: numberValue(retrieval.tier2TopK, DEFAULT_MEMMY_CONFIG.algorithm.retrieval.tier2TopK), diff --git a/Memory/src/service/l3-field/flag.ts b/Memory/src/service/l3-field/flag.ts new file mode 100644 index 000000000..68ef60772 --- /dev/null +++ b/Memory/src/service/l3-field/flag.ts @@ -0,0 +1,13 @@ +import type { MemmyConfig } from "../../config/index.js"; +import type { StorageBackendCapabilities } from "../../storage/backend.js"; + +/** The new L3 only runs on local SQLite storage, where the Memory service can see the workspace. */ +export function isL3FieldCognitionActive( + config: Pick, + capabilities: Pick, + mode: "local" | "cloud" | "dev" +): boolean { + return config.algorithm.l3FieldCognition.enabled && + mode !== "cloud" && + capabilities.backendId === "sqlite-local"; +} diff --git a/Memory/src/service/l3-field/text-units.ts b/Memory/src/service/l3-field/text-units.ts new file mode 100644 index 000000000..e1acd7db9 --- /dev/null +++ b/Memory/src/service/l3-field/text-units.ts @@ -0,0 +1,16 @@ +const CJK_CHARACTER = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu; +const WORD_CHARACTER = /[\p{L}\p{N}]/u; + +/** + * Length of an L3 item in "units": each CJK character counts as one, every + * other whitespace-separated word counts as one, punctuation-only tokens + * count as zero. + */ +export function countL3Units(text: string): number { + const cjkCount = text.match(CJK_CHARACTER)?.length ?? 0; + const words = text + .replace(CJK_CHARACTER, " ") + .split(/\s+/u) + .filter((token) => WORD_CHARACTER.test(token)); + return cjkCount + words.length; +} diff --git a/Memory/src/service/l3-field/types.ts b/Memory/src/service/l3-field/types.ts new file mode 100644 index 000000000..5a8937685 --- /dev/null +++ b/Memory/src/service/l3-field/types.ts @@ -0,0 +1,75 @@ +/** Shared types for L3 field cognition: machine/project units holding itemized facts. */ + +export type L3UnitKind = "machine" | "project"; +export type L3ItemKind = "env_fact" | "project_constraint"; +export type L3ItemStatus = "active" | "inactive"; +export type ProjectIdentitySource = "user_selected" | "git" | "folder"; +export type L3UnitBatchStatus = "queued" | "applied" | "failed"; + +export interface ResolvedProject { + /** Stable identity shared by every checkout of the same project. */ + identity: string; + /** Root directory of the checkout the evidence came from. */ + checkoutRoot: string; + source: ProjectIdentitySource; + displayName: string; +} + +export interface EvidenceFragment { + id: string; + rawTurnId: string; + l1MemoryId: string; + toolName: string | null; + input: unknown; + output: string | null; + isError: boolean; + paths: string[]; + candidateUnitKeys: string[]; + at: string; +} + +export interface L3ItemEvidence { + fragmentIds: string[]; + l1MemoryIds: string[]; +} + +export interface L3UnitRecord { + unitKey: string; + userId: string; + machineId: string; + kind: L3UnitKind; + projectIdentity?: string; + displayName?: string; + identitySource?: ProjectIdentitySource; + checkoutRoot?: string; + profileText?: string; + profileFingerprint?: string; + snapshotMemoryId?: string; + deletedAt?: string; + nextSeq: number; + createdAt: string; + updatedAt: string; +} + +export interface L3ItemRecord { + id: string; + unitKey: string; + kind: L3ItemKind; + content: string; + evidence: L3ItemEvidence; + evidenceAt: string; + lastConfirmedAt: string; + status: L3ItemStatus; + replacedBy?: string; + /** When the evidence that removed it happened; only for a delete or replace op. */ + removedEvidenceAt?: string; + inactiveAt?: string; + batchId?: string; + createdAt: string; + updatedAt: string; +} + +export interface L3UserProject { + rootPath: string; + name: string; +} diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts new file mode 100644 index 000000000..a235260b0 --- /dev/null +++ b/Memory/src/storage/l3-field-repository.ts @@ -0,0 +1,382 @@ +import type Database from "better-sqlite3"; +import { canonicalJson, sha256Hex } from "../contracts/index.js"; +import type { + L3ItemEvidence, + L3ItemKind, + L3ItemRecord, + L3UnitKind, + L3UnitRecord, + L3UserProject, + ProjectIdentitySource, + ResolvedProject +} from "../service/l3-field/types.js"; +import type { MemoryRow } from "../types.js"; +import { newId } from "../utils/id.js"; +import { parseJson, toJson } from "../utils/json.js"; +import { nowIso } from "../utils/time.js"; +import type { MemoryRepository } from "./repositories.js"; + +const SNAPSHOT_SOURCE = "worker.l3_field.v1"; + +interface SqlUnitRow { + unit_key: string; + user_id: string; + machine_id: string; + kind: L3UnitKind; + project_identity: string | null; + display_name: string | null; + identity_source: ProjectIdentitySource | null; + checkout_root: string | null; + profile_text: string | null; + profile_fingerprint: string | null; + snapshot_memory_id: string | null; + deleted_at: string | null; + next_seq: number; + created_at: string; + updated_at: string; +} + +interface SqlItemRow { + id: string; + unit_key: string; + kind: L3ItemKind; + content: string; + evidence_json: string; + evidence_at: string; + last_confirmed_at: string; + status: "active" | "inactive"; + replaced_by: string | null; + removed_evidence_at: string | null; + inactive_at: string | null; + batch_id: string | null; + created_at: string; + updated_at: string; +} + +export function l3UnitKey(input: { + userId: string; + machineId: string; + kind: L3UnitKind; + identity: string | null; +}): string { + return `l3u:${sha256Hex(canonicalJson({ + userId: input.userId, + machineId: input.machineId, + kind: input.kind, + identity: input.identity + }))}`; +} + +export function l3UnitSnapshotMemoryKey(unitKey: string): string { + return `world_model:l3_field:${unitKey}`; +} + +/** Renders one unit (profile plus active items) as the text stored in its snapshot memory. */ +export function renderL3UnitSnapshot(unit: L3UnitRecord, items: readonly L3ItemRecord[]): string { + const facts = items.filter((item) => item.kind === "env_fact"); + const constraints = items.filter((item) => item.kind === "project_constraint"); + const sections: string[] = []; + if (unit.profileText?.trim()) sections.push(`## 环境画像\n${unit.profileText.trim()}`); + if (facts.length) sections.push(`## 环境事实\n${facts.map((item) => `- ${item.content}`).join("\n")}`); + if (constraints.length) sections.push(`## 项目约束\n${constraints.map((item) => `- ${item.content}`).join("\n")}`); + return sections.join("\n\n"); +} + +export class L3FieldRepository { + constructor( + private readonly db: Database.Database, + private readonly memories: MemoryRepository + ) {} + + getUnit(unitKey: string): L3UnitRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_units WHERE unit_key = ?`).get(unitKey) as SqlUnitRow | undefined; + return row ? unitFromSql(row) : undefined; + } + + ensureMachineUnit(input: { userId: string; machineId: string; at?: string }): L3UnitRecord { + const unitKey = l3UnitKey({ userId: input.userId, machineId: input.machineId, kind: "machine", identity: null }); + const at = input.at ?? nowIso(); + this.db.prepare( + `INSERT INTO l3_units (unit_key, user_id, machine_id, kind, created_at, updated_at) + VALUES (?, ?, ?, 'machine', ?, ?) + ON CONFLICT(unit_key) DO NOTHING` + ).run(unitKey, input.userId, input.machineId, at, at); + return this.getUnit(unitKey)!; + } + + ensureProjectUnit(input: { + userId: string; + machineId: string; + project: ResolvedProject; + at?: string; + }): L3UnitRecord { + const unitKey = l3UnitKey({ + userId: input.userId, + machineId: input.machineId, + kind: "project", + identity: input.project.identity + }); + const at = input.at ?? nowIso(); + this.db.prepare( + `INSERT INTO l3_units ( + unit_key, user_id, machine_id, kind, project_identity, display_name, + identity_source, checkout_root, created_at, updated_at + ) VALUES (?, ?, ?, 'project', ?, ?, ?, ?, ?, ?) + ON CONFLICT(unit_key) DO UPDATE SET + display_name = excluded.display_name, + identity_source = excluded.identity_source, + checkout_root = excluded.checkout_root, + updated_at = CASE + WHEN l3_units.display_name IS excluded.display_name + AND l3_units.identity_source IS excluded.identity_source + AND l3_units.checkout_root IS excluded.checkout_root + THEN l3_units.updated_at ELSE excluded.updated_at END` + ).run( + unitKey, + input.userId, + input.machineId, + input.project.identity, + input.project.displayName, + input.project.source, + input.project.checkoutRoot, + at, + at + ); + return this.getUnit(unitKey)!; + } + + activeItems(unitKey: string): L3ItemRecord[] { + return (this.db.prepare( + `SELECT * FROM l3_items WHERE unit_key = ? AND status = 'active' + ORDER BY created_at ASC, id ASC` + ).all(unitKey) as SqlItemRow[]).map(itemFromSql); + } + + getItem(id: string): L3ItemRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_items WHERE id = ?`).get(id) as SqlItemRow | undefined; + return row ? itemFromSql(row) : undefined; + } + + insertItem(input: { + unitKey: string; + kind: L3ItemKind; + content: string; + evidence: L3ItemEvidence; + evidenceAt: string; + batchId?: string; + at?: string; + }): L3ItemRecord { + const id = newId("l3item"); + const at = input.at ?? nowIso(); + this.db.prepare( + `INSERT INTO l3_items ( + id, unit_key, kind, content, evidence_json, evidence_at, last_confirmed_at, + status, batch_id, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?)` + ).run( + id, + input.unitKey, + input.kind, + input.content, + toJson(input.evidence), + input.evidenceAt, + input.evidenceAt, + input.batchId ?? null, + at, + at + ); + return this.getItem(id)!; + } + + /** + * `removedEvidenceAt` is when the evidence that removed it happened (a delete or replace op); + * evictions and unit resets leave it empty. + */ + inactivateItem(id: string, input: { replacedBy?: string; removedEvidenceAt?: string; at?: string }): void { + const at = input.at ?? nowIso(); + this.db.prepare( + `UPDATE l3_items + SET status = 'inactive', replaced_by = ?, removed_evidence_at = ?, inactive_at = ?, updated_at = ? + WHERE id = ? AND status = 'active'` + ).run(input.replacedBy ?? null, input.removedEvidenceAt ?? null, at, at, id); + } + + /** Moves the last-confirmed time forward only; an older confirmation never rewinds it. */ + confirmItem(id: string, confirmedAt: string, at = nowIso()): void { + this.db.prepare( + `UPDATE l3_items + SET last_confirmed_at = ?, updated_at = ? + WHERE id = ? AND status = 'active' AND last_confirmed_at < ?` + ).run(confirmedAt, at, id, confirmedAt); + } + + /** + * Creates or updates the snapshot memory for one unit. A change to a machine + * unit also bumps the snapshot version of every project on the same machine, + * because session context renders the machine together with the project. + */ + refreshSnapshot(unitKey: string, at = nowIso()): MemoryRow { + const unit = this.getUnit(unitKey); + if (!unit) throw new Error(`L3 unit not found: ${unitKey}`); + const saved = this.writeSnapshot(unit, at); + if (unit.kind === "machine") { + const projects = (this.db.prepare( + `SELECT * FROM l3_units + WHERE user_id = ? AND machine_id = ? AND kind = 'project' + AND snapshot_memory_id IS NOT NULL AND deleted_at IS NULL` + ).all(unit.userId, unit.machineId) as SqlUnitRow[]).map(unitFromSql); + for (const project of projects) this.writeSnapshot(project, at); + } + return saved; + } + + markUnitDeleted(unitKey: string, at = nowIso()): void { + this.db.transaction(() => { + this.db.prepare( + `UPDATE l3_units SET deleted_at = COALESCE(deleted_at, ?), updated_at = ? WHERE unit_key = ?` + ).run(at, at, unitKey); + this.db.prepare( + `UPDATE l3_items + SET status = 'inactive', inactive_at = ?, updated_at = ? + WHERE unit_key = ? AND status = 'active'` + ).run(at, at, unitKey); + })(); + } + + /** Deletes items that became inactive before `before`; active items are never touched. */ + purgeInactive(before: string): number { + return this.db.prepare( + `DELETE FROM l3_items WHERE status = 'inactive' AND inactive_at < ?` + ).run(before).changes; + } + + listUserProjects(userId: string): L3UserProject[] { + return (this.db.prepare( + `SELECT root_path, name FROM l3_user_projects WHERE user_id = ? ORDER BY root_path ASC` + ).all(userId) as Array<{ root_path: string; name: string }>) + .map((row) => ({ rootPath: row.root_path, name: row.name })); + } + + /** Replaces the whole list; the latest push wins. */ + replaceUserProjects(userId: string, projects: readonly L3UserProject[], at = nowIso()): void { + this.db.transaction(() => { + this.db.prepare(`DELETE FROM l3_user_projects WHERE user_id = ?`).run(userId); + const insert = this.db.prepare( + `INSERT INTO l3_user_projects (user_id, root_path, name, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(user_id, root_path) DO UPDATE SET name = excluded.name, updated_at = excluded.updated_at` + ); + for (const project of projects) insert.run(userId, project.rootPath, project.name, at); + })(); + } + + private writeSnapshot(unit: L3UnitRecord, at: string): MemoryRow { + const memoryValue = renderL3UnitSnapshot(unit, this.activeItems(unit.unitKey)); + const existing = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; + const title = unit.kind === "machine" ? "本机场域认知" : `项目场域认知：${unit.displayName ?? unit.projectIdentity}`; + const tags = ["world_model", "l3_field", `scope:${unit.kind}`]; + const info: Record = { l3_field_unit_key: unit.unitKey }; + const status = unit.deletedAt ? "deleted" as const : "activated" as const; + const memory: MemoryRow = { + id: existing?.id ?? newId("memory"), + timeline: existing?.timeline ?? at, + userId: unit.userId, + memoryType: "LongTermMemory", + status, + visibility: "private", + memoryKey: l3UnitSnapshotMemoryKey(unit.unitKey), + memoryValue, + tags, + info, + properties: { + memory_type: "LongTermMemory", + status, + tags, + info: { ...info }, + internal_info: { + memory_layer: "L3", + memory_kind: "world_model", + schema_version: 3, + source: SNAPSHOT_SOURCE, + title, + summary: [...memoryValue.replace(/\s+/gu, " ").trim()].slice(0, 240).join(""), + body: memoryValue, + l3_field: { unit_key: unit.unitKey, kind: unit.kind } + } + }, + memoryLayer: "L3", + contentHash: sha256Hex(memoryValue), + version: existing?.version ?? 1, + createdAt: existing?.createdAt ?? at, + updatedAt: at, + deletedAt: unit.deletedAt ?? null + }; + const saved = existing ? this.memories.update(memory) : this.memories.insert(memory); + if (!existing) { + this.db.prepare( + `UPDATE l3_units SET snapshot_memory_id = ?, updated_at = ? WHERE unit_key = ?` + ).run(saved.id, at, unit.unitKey); + } + this.db.prepare( + `INSERT INTO memory_change_log ( + memory_id, namespace_id, kind, op, entity_id, user_id, + change_type, version, before_json, after_json, source, created_at + ) VALUES (?, ?, 'world_model', ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + saved.id, + unit.unitKey, + existing ? "updated" : "created", + saved.id, + unit.userId, + existing ? "l3_field_snapshot_update" : "l3_field_snapshot_create", + saved.version, + existing ? toJson(existing) : null, + toJson(saved), + SNAPSHOT_SOURCE, + at + ); + return saved; + } +} + +function unitFromSql(row: SqlUnitRow): L3UnitRecord { + return { + unitKey: row.unit_key, + userId: row.user_id, + machineId: row.machine_id, + kind: row.kind, + projectIdentity: row.project_identity ?? undefined, + displayName: row.display_name ?? undefined, + identitySource: row.identity_source ?? undefined, + checkoutRoot: row.checkout_root ?? undefined, + profileText: row.profile_text ?? undefined, + profileFingerprint: row.profile_fingerprint ?? undefined, + snapshotMemoryId: row.snapshot_memory_id ?? undefined, + deletedAt: row.deleted_at ?? undefined, + nextSeq: row.next_seq, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} + +function itemFromSql(row: SqlItemRow): L3ItemRecord { + const evidence = parseJson>(row.evidence_json, {}); + return { + id: row.id, + unitKey: row.unit_key, + kind: row.kind, + content: row.content, + evidence: { + fragmentIds: Array.isArray(evidence.fragmentIds) ? evidence.fragmentIds : [], + l1MemoryIds: Array.isArray(evidence.l1MemoryIds) ? evidence.l1MemoryIds : [] + }, + evidenceAt: row.evidence_at, + lastConfirmedAt: row.last_confirmed_at, + status: row.status, + replacedBy: row.replaced_by ?? undefined, + removedEvidenceAt: row.removed_evidence_at ?? undefined, + inactiveAt: row.inactive_at ?? undefined, + batchId: row.batch_id ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index b33b216f8..6794c77d7 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -55,6 +55,7 @@ import { type VectorSearchCandidate } from "./sqlite-vec-store.js"; import { compactHistoryChangePayload, compactHistoryChangeRow } from "./history-change-snapshot.js"; +import { L3FieldRepository } from "./l3-field-repository.js"; type SqlValue = string | number | Buffer | null; const BUNDLE_TABLES = [ @@ -80,6 +81,10 @@ const BUNDLE_TABLES = [ "api_logs", "memory_change_log", "l3_world_model_project_environment_state", + "l3_units", + "l3_unit_batches", + "l3_items", + "l3_user_projects", "evolution_jobs", "embedding_retry_queue", "memory_processing_state", @@ -4857,6 +4862,10 @@ export class RuntimeRepository { }> }; this.db.transaction(() => { + const keptL3Units = this.existingL3Units(tables, conflictStrategy); + // Unfinished work of a kept unit that this database does not have: its Sessions' traces must be + // frozen again here, so their cursors are moved back below the first trace no batch here covers. + const droppedL3Work: Array<{ sessionId: string; startTraceSeq: number }> = []; const hadRuntimeData = Boolean(this.db.prepare(`SELECT EXISTS( SELECT 1 FROM memories UNION ALL SELECT 1 FROM sessions UNION ALL SELECT 1 FROM raw_turns UNION ALL SELECT 1 FROM source_turn_captures @@ -4866,6 +4875,21 @@ export class RuntimeRepository { for (const row of rows) { const defaults = applyBundleDefaults(table, deserializeBundleRow(row)); const normalized = table === "memory_change_log" ? compactHistoryChangeRow(defaults) : defaults; + const l3UnitKey = keptL3Units.size > 0 ? bundleRowL3UnitKey(table, normalized) : null; + if (l3UnitKey && keptL3Units.has(l3UnitKey)) { + if (table === "l3_unit_batches" && normalized.status !== "applied" && typeof normalized.id === "string" && + !this.rowExists("l3_unit_batches", ["id"], [normalized.id]) && + typeof normalized.session_id === "string" && typeof normalized.start_trace_seq === "number" && + typeof normalized.end_trace_seq === "number") { + const unfrozen = this.firstUnfrozenL3Trace(normalized.session_id, normalized.start_trace_seq, normalized.end_trace_seq); + if (unfrozen !== null) droppedL3Work.push({ sessionId: normalized.session_id, startTraceSeq: unfrozen }); + } + if (table === "l3_units") { + result.conflicts.push({ table, primaryKey: "unit_key", sourceId: l3UnitKey, targetId: l3UnitKey, action: "skipped" }); + } + result.skipped[table] = (result.skipped[table] ?? 0) + 1; + continue; + } const identity = bundleIdentity(table, normalized); if (identity) { recordMigrationMap(result.migrationMap, table, identity.sourceId, identity.sourceId); @@ -4923,12 +4947,35 @@ export class RuntimeRepository { } } } + const rewind = this.db.prepare( + `UPDATE l3_world_model_session_cursors SET last_scheduled_seq = MIN(last_scheduled_seq, ?) WHERE session_id = ?` + ); + for (const work of droppedL3Work) rewind.run(work.startTraceSeq - 1, work.sessionId); })(); this.scheduleLogTablesPrune(); this.scheduleHistoryLogCompaction(true); return result; } + /** + * Row-by-row conflict handling cannot keep an L3 unit consistent (an old item could stay active + * next to its imported successor), so a unit that already exists here is kept whole: the bundle's + * rows for it (unit, items, batches, jobs, snapshot) are all skipped, as re-importing one's own + * backup skips every other existing row. With conflictStrategy "error" the import fails instead. + */ + private existingL3Units(tables: Record, conflictStrategy: "skip" | "replace" | "error"): Set { + const rows = Array.isArray(tables.l3_units) ? tables.l3_units as Array> : []; + const exists = this.db.prepare(`SELECT 1 FROM l3_units WHERE unit_key = ?`).pluck(); + const kept = new Set(); + for (const row of rows) { + const unitKey = deserializeBundleRow(row).unit_key; + if (typeof unitKey !== "string" || !exists.get(unitKey)) continue; + if (conflictStrategy === "error") throw new Error(`import conflict: l3 unit already exists: ${unitKey}`); + kept.add(unitKey); + } + return kept; + } + private mergeSourceTurnActivation( row: Record, hadRuntimeData: boolean @@ -4953,6 +5000,28 @@ export class RuntimeRepository { return existing ? "replaced" : "inserted"; } + + /** + * The first trace in [start, end] that no batch of this database froze, or null when all were + * (an earlier import of the same bundle refroze them). A batch counts only if it froze the turn now + * at that position: a replace import can put another turn there, which a batch reading its frozen + * ids would leave out. + */ + private firstUnfrozenL3Trace(sessionId: string, start: number, end: number): number | null { + const covered = this.db.prepare( + `SELECT EXISTS( + SELECT 1 FROM l3_world_model_input_traces t JOIN l3_unit_batches b + ON b.session_id = t.session_id AND b.start_trace_seq <= t.trace_seq AND b.end_trace_seq >= t.trace_seq + WHERE t.session_id = ? AND t.trace_seq = ? + AND EXISTS(SELECT 1 FROM json_each(b.l1_memory_ids_json) WHERE value = t.l1_memory_id) + AND EXISTS(SELECT 1 FROM json_each(b.raw_turn_ids_json) WHERE value = t.raw_turn_id) + )` + ).pluck(); + for (let seq = start; seq <= end; seq += 1) { + if (!covered.get(sessionId, seq)) return seq; + } + return null; + } private rowExists(table: BundleTableName, columns: string[], values: Array): boolean { const tableColumns = this.tableColumns(table); if (columns.length === 0 || columns.some((column) => !tableColumns.includes(column))) { @@ -6017,6 +6086,7 @@ export class Repositories { readonly processing: MemoryProcessingRepository; readonly runtime: RuntimeRepository; readonly l3WorldModels: L3WorldModelRepository; + readonly l3Field: L3FieldRepository; readonly projectEnvironments: ProjectEnvironmentRepository; readonly vectors: SqliteVecStore; @@ -6028,6 +6098,7 @@ export class Repositories { this.processing = new MemoryProcessingRepository(db); this.runtime = new RuntimeRepository(db); this.l3WorldModels = new L3WorldModelRepository(db, this.memories); + this.l3Field = new L3FieldRepository(db, this.memories); this.projectEnvironments = new ProjectEnvironmentRepository(db, this.l3WorldModels, this.runtime); } @@ -7892,6 +7963,31 @@ interface BundleIdentity { values: Array; } +/** The new-L3 unit a bundle row belongs to, for the tables that hold one unit's state. */ +function bundleRowL3UnitKey(table: string, row: Record): string | null { + const record = (value: unknown): Record => { + const parsed = typeof value === "string" ? parseJson(value, {}) : value; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : {}; + }; + const text = (value: unknown) => (typeof value === "string" && value ? value : null); + switch (table) { + case "l3_units": + case "l3_items": + return text(row.unit_key); + case "l3_unit_batches": + return text(row.machine_unit_key); + case "evolution_jobs": { + if (row.job_type === "l3_unit_update") return text(row.scope_key); + const payload = record(row.payload_json); + return row.job_type === "project_environment_profile" && payload.mode === "l3_field" ? text(payload.unitKey) : null; + } + case "memories": + return text(record(record(record(row.properties_json).internal_info).l3_field).unit_key); + default: + return null; + } +} + function bundleIdentity( table: BundleTableName, row: Record @@ -7905,7 +8001,9 @@ function bundleIdentity( l3_world_model_input_traces: ["session_id", "trace_seq"], l3_world_model_evidence_batches: ["id"], l3_world_model_batch_targets: ["batch_id", "target_field"], - l3_world_model_project_environment_state: ["user_id", "project_id"] + l3_world_model_project_environment_state: ["user_id", "project_id"], + l3_units: ["unit_key"], + l3_user_projects: ["user_id", "root_path"] }; const newColumns = newTableIdentityColumns[table]; if (newColumns) { @@ -7995,9 +8093,15 @@ function normalizeRedactedL3WorldModelBundle( .filter((row) => terminalBatchIds.has(String(row.batch_id))); tables.evolution_jobs = (tables.evolution_jobs ?? []).filter((row) => { const jobType = row.job_type; - if (jobType !== "l3_world_model_update" && jobType !== "project_environment_profile") return true; + if ( + jobType !== "l3_world_model_update" && + jobType !== "project_environment_profile" && + jobType !== "l3_unit_update" + ) return true; return row.status === "succeeded" || row.status === "dead_letter"; }); + // Unfinished unit batches depend on raw text that a redacted bundle no longer carries. + tables.l3_unit_batches = (tables.l3_unit_batches ?? []).filter((row) => row.status === "applied"); tables.l3_world_model_project_environment_state = ( tables.l3_world_model_project_environment_state ?? [] ).map((row) => ({ diff --git a/Memory/src/storage/schema.ts b/Memory/src/storage/schema.ts index 3cb76e0c4..5e07b3ae6 100644 --- a/Memory/src/storage/schema.ts +++ b/Memory/src/storage/schema.ts @@ -685,6 +685,76 @@ const statements = [ sequence INTEGER PRIMARY KEY, event_id TEXT NOT NULL UNIQUE, payload_json TEXT NOT NULL CHECK (json_valid(payload_json)) + )`, + + // L3 field cognition (itemized machine/project units). These tables are added + // without bumping SCHEMA_VERSION: they are new, empty and ignored by older builds. + `CREATE TABLE IF NOT EXISTS l3_units ( + unit_key TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + machine_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('machine', 'project')), + project_identity TEXT, + display_name TEXT, + identity_source TEXT CHECK (identity_source IS NULL OR identity_source IN ('user_selected', 'git', 'folder')), + checkout_root TEXT, + profile_text TEXT, + profile_fingerprint TEXT, + snapshot_memory_id TEXT, + deleted_at TEXT, + next_seq INTEGER NOT NULL DEFAULT 1 CHECK (next_seq >= 1), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + CHECK ((kind = 'machine') = (project_identity IS NULL)) + )`, + `CREATE INDEX IF NOT EXISTS idx_l3_units_user_machine + ON l3_units (user_id, machine_id, kind)`, + `CREATE TABLE IF NOT EXISTS l3_items ( + id TEXT PRIMARY KEY, + unit_key TEXT NOT NULL REFERENCES l3_units(unit_key) ON DELETE CASCADE, + kind TEXT NOT NULL CHECK (kind IN ('env_fact', 'project_constraint')), + content TEXT NOT NULL, + evidence_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(evidence_json)), + evidence_at TEXT NOT NULL, + last_confirmed_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive')), + replaced_by TEXT, + removed_evidence_at TEXT, + inactive_at TEXT, + batch_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + CHECK ((status = 'inactive') = (inactive_at IS NOT NULL)) + )`, + `CREATE INDEX IF NOT EXISTS idx_l3_items_unit_status_kind + ON l3_items (unit_key, status, kind)`, + `CREATE INDEX IF NOT EXISTS idx_l3_items_inactive + ON l3_items (inactive_at) + WHERE status = 'inactive'`, + `CREATE TABLE IF NOT EXISTS l3_unit_batches ( + id TEXT PRIMARY KEY, + machine_unit_key TEXT NOT NULL, + scope_seq INTEGER NOT NULL CHECK (scope_seq >= 1), + user_id TEXT NOT NULL, + session_id TEXT NOT NULL, + trigger TEXT NOT NULL, + start_trace_seq INTEGER NOT NULL, + end_trace_seq INTEGER NOT NULL, + l1_memory_ids_json TEXT NOT NULL CHECK (json_valid(l1_memory_ids_json)), + raw_turn_ids_json TEXT NOT NULL CHECK (json_valid(raw_turn_ids_json)), + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'applied', 'failed')), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (machine_unit_key, scope_seq) + )`, + `CREATE INDEX IF NOT EXISTS idx_l3_unit_batches_session + ON l3_unit_batches (session_id, end_trace_seq)`, + `CREATE TABLE IF NOT EXISTS l3_user_projects ( + user_id TEXT NOT NULL, + root_path TEXT NOT NULL, + name TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (user_id, root_path) )` ]; @@ -720,6 +790,8 @@ export function migrate(db: Database.Database): void { for (const statement of statements) { db.prepare(statement).run(); } + // l3_items created by an earlier build of the new L3 lacks this column. + addColumnIfMissing(db, "l3_items", "removed_evidence_at", "TEXT"); if (version > 0 && version < 5) { addColumnIfMissing(db, "recall_events", "query_id", "TEXT"); addColumnIfMissing(db, "recall_events", "user_memory_candidate_ids_json", "TEXT NOT NULL DEFAULT '[]'"); diff --git a/Memory/tests/repository/l3-field-repository.test.ts b/Memory/tests/repository/l3-field-repository.test.ts new file mode 100644 index 000000000..aece562ed --- /dev/null +++ b/Memory/tests/repository/l3-field-repository.test.ts @@ -0,0 +1,260 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import YAML from "yaml"; +import { MemoryDb, SCHEMA_VERSION } from "../../src/index.js"; +import { DEFAULT_MEMMY_CONFIG, loadMemmyConfig } from "../../src/config/index.js"; +import { isL3FieldCognitionActive } from "../../src/service/l3-field/flag.js"; +import type { ResolvedProject } from "../../src/service/l3-field/types.js"; +import { Repositories } from "../../src/storage/repositories.js"; + +const NEW_TABLES = ["l3_units", "l3_items", "l3_unit_batches", "l3_user_projects"]; +const USER = "l3-field-user"; +const MACHINE = "a".repeat(64); +const PROJECT: ResolvedProject = { + identity: "github.com/example/repo", + checkoutRoot: "/work/repo", + source: "git", + displayName: "repo" +}; + +const opened: MemoryDb[] = []; +const folders: string[] = []; + +function openRepos(): { db: MemoryDb; repos: Repositories } { + const db = new MemoryDb({ path: ":memory:" }); + opened.push(db); + return { db, repos: new Repositories(db.db) }; +} + +afterEach(() => { + for (const db of opened.splice(0)) db.close(); + for (const folder of folders.splice(0)) rmSync(folder, { recursive: true, force: true }); +}); + +function tableNames(db: MemoryDb): string[] { + return (db.db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table'`).pluck().all() as string[]); +} + +describe("L3 field cognition storage", () => { + it("creates the new tables while keeping schema version 9 and the old L3 tables", () => { + const { db } = openRepos(); + expect(SCHEMA_VERSION).toBe(9); + expect(db.schemaVersion().version).toBe(9); + const names = tableNames(db); + for (const table of NEW_TABLES) expect(names).toContain(table); + expect(names).toContain("l3_world_model_scopes"); + }); + + it("reopens an existing database without touching old L3 rows", () => { + const folder = mkdtempSync(join(tmpdir(), "l3-field-reopen-")); + folders.push(folder); + const path = join(folder, "memory.sqlite"); + const first = new MemoryDb({ path }); + first.db.exec(`DROP TABLE l3_user_projects; DROP TABLE l3_items; DROP TABLE l3_unit_batches; DROP TABLE l3_units;`); + first.db.prepare( + `INSERT INTO l3_world_model_scopes (scope_key, user_id, project_id, memory_id, next_scope_seq, updated_at) + VALUES ('legacy-scope', ?, NULL, NULL, 3, '2026-10-01T00:00:00.000Z')` + ).run(USER); + first.close(); + + const reopened = new MemoryDb({ path }); + opened.push(reopened); + expect(reopened.schemaVersion().version).toBe(9); + for (const table of NEW_TABLES) expect(tableNames(reopened)).toContain(table); + expect(reopened.db.prepare(`SELECT next_scope_seq FROM l3_world_model_scopes WHERE scope_key = 'legacy-scope'`).get()) + .toEqual({ next_scope_seq: 3 }); + }); + + it("keys machine and project units by user, machine and identity", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + expect(repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }).unitKey).toBe(machine.unitKey); + expect(repos.l3Field.ensureMachineUnit({ userId: "other-user", machineId: MACHINE }).unitKey).not.toBe(machine.unitKey); + const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + expect(project.kind).toBe("project"); + expect(project.projectIdentity).toBe(PROJECT.identity); + expect(project.unitKey).not.toBe(machine.unitKey); + expect(repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }).unitKey) + .toBe(project.unitKey); + }); + + it("refreshes a unit snapshot and bumps project snapshots when the machine unit changes", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + repos.l3Field.insertItem({ + unitKey: project.unitKey, + kind: "project_constraint", + content: "报告以 GitHub 上的 main 为准", + evidence: { fragmentIds: ["f1"], l1MemoryIds: ["l1"] }, + evidenceAt: "2026-10-20T00:00:00.000Z" + }); + const first = repos.l3Field.refreshSnapshot(project.unitKey); + expect(first.memoryLayer).toBe("L3"); + expect(first.properties.internal_info.schema_version).toBe(3); + expect(first.memoryValue).toContain("报告以 GitHub 上的 main 为准"); + const again = repos.l3Field.refreshSnapshot(project.unitKey); + expect(again.id).toBe(first.id); + expect(again.version).toBe(first.version + 1); + + repos.l3Field.insertItem({ + unitKey: machine.unitKey, + kind: "env_fact", + content: "这台电脑的 zsh 会展开 = 开头的参数", + evidence: { fragmentIds: ["f2"], l1MemoryIds: ["l2"] }, + evidenceAt: "2026-10-21T00:00:00.000Z" + }); + repos.l3Field.refreshSnapshot(machine.unitKey); + expect(repos.memories.get(first.id)!.version).toBe(again.version + 1); + }); + + it("replaces items by marking the old one inactive and pointing at its successor", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const old = repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content: "旧事实", + evidence: { fragmentIds: [], l1MemoryIds: ["l1"] }, evidenceAt: "2026-10-01T00:00:00.000Z" + }); + const next = repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content: "新事实", + evidence: { fragmentIds: [], l1MemoryIds: ["l2"] }, evidenceAt: "2026-10-02T00:00:00.000Z" + }); + repos.l3Field.inactivateItem(old.id, { replacedBy: next.id, at: "2026-10-02T00:00:00.000Z" }); + expect(repos.l3Field.activeItems(machine.unitKey).map((item) => item.id)).toEqual([next.id]); + expect(repos.l3Field.getItem(old.id)).toMatchObject({ status: "inactive", replacedBy: next.id }); + }); + + it("purges only items that have been inactive longer than the cutoff", () => { + const { repos } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const add = (content: string) => repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content, + evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" + }); + const keep = add("仍然有效"); + const oldInactive = add("失效 16 天"); + const recentInactive = add("失效 14 天"); + repos.l3Field.inactivateItem(oldInactive.id, { at: "2026-10-08T00:00:00.000Z" }); + repos.l3Field.inactivateItem(recentInactive.id, { at: "2026-10-10T00:00:00.000Z" }); + expect(repos.l3Field.purgeInactive("2026-10-09T00:00:00.000Z")).toBe(1); + expect(repos.l3Field.getItem(oldInactive.id)).toBeUndefined(); + expect(repos.l3Field.getItem(recentInactive.id)?.status).toBe("inactive"); + expect(repos.l3Field.getItem(keep.id)?.status).toBe("active"); + }); + + it("marks a deleted unit and inactivates its items", () => { + const { repos } = openRepos(); + const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + repos.l3Field.insertItem({ + unitKey: project.unitKey, kind: "project_constraint", content: "规矩", + evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-10-01T00:00:00.000Z" + }); + repos.l3Field.markUnitDeleted(project.unitKey, "2026-10-05T00:00:00.000Z"); + expect(repos.l3Field.getUnit(project.unitKey)?.deletedAt).toBe("2026-10-05T00:00:00.000Z"); + expect(repos.l3Field.activeItems(project.unitKey)).toEqual([]); + }); + + it("replaces the whole user project list", () => { + const { repos } = openRepos(); + repos.l3Field.replaceUserProjects(USER, [{ rootPath: "/a", name: "A" }, { rootPath: "/b", name: "B" }]); + repos.l3Field.replaceUserProjects(USER, [{ rootPath: "/b", name: "B2" }]); + expect(repos.l3Field.listUserProjects(USER)).toEqual([{ rootPath: "/b", name: "B2" }]); + repos.l3Field.replaceUserProjects(USER, []); + expect(repos.l3Field.listUserProjects(USER)).toEqual([]); + }); + + it("round-trips the new tables through export and import into an empty database", () => { + const source = openRepos().repos; + const project = source.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); + source.l3Field.insertItem({ + unitKey: project.unitKey, kind: "project_constraint", content: "规矩", + evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-10-01T00:00:00.000Z" + }); + source.l3Field.replaceUserProjects(USER, [{ rootPath: "/work/repo", name: "repo" }]); + const tables = source.runtime.exportBundleTables(true); + + const target = openRepos().repos; + target.runtime.importBundleTables(tables); + expect(target.l3Field.getUnit(project.unitKey)?.projectIdentity).toBe(PROJECT.identity); + expect(target.l3Field.activeItems(project.unitKey).map((item) => item.content)).toEqual(["规矩"]); + expect(target.l3Field.listUserProjects(USER)).toEqual([{ rootPath: "/work/repo", name: "repo" }]); + }); + + it("keeps a unit that already exists whole when importing, and fails only when asked to on conflict", () => { + const source = openRepos().repos; + const sourceMachine = source.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + source.l3Field.insertItem({ unitKey: sourceMachine.unitKey, kind: "env_fact", content: "旧备份里的事实", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-10-01T00:00:00.000Z" }); + source.l3Field.refreshSnapshot(sourceMachine.unitKey); + source.l3Field.replaceUserProjects(USER, [{ rootPath: "/work/repo", name: "repo" }]); + const tables = source.runtime.exportBundleTables(true); + + const target = openRepos().repos; + const machine = target.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + target.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机现在的事实", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-10-02T00:00:00.000Z" }); + const snapshot = target.l3Field.refreshSnapshot(machine.unitKey); + expect(() => target.runtime.importBundleTables(tables, { conflictStrategy: "error" })).toThrow(/l3 unit already exists/); + + const imported = target.runtime.importBundleTables(tables); + expect(imported.conflicts).toContainEqual(expect.objectContaining({ table: "l3_units", action: "skipped" })); + expect(target.l3Field.activeItems(machine.unitKey).map((item) => item.content)).toEqual(["本机现在的事实"]); + expect(target.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId).toBe(snapshot.id); + const snapshots = target.db.prepare( + `SELECT COUNT(*) AS n FROM memories WHERE json_extract(properties_json, '$.internal_info.l3_field.unit_key') = ?` + ).get(machine.unitKey) as { n: number }; + expect(snapshots.n).toBe(1); + expect(target.l3Field.listUserProjects(USER)).toEqual([{ rootPath: "/work/repo", name: "repo" }]); + }); + + it("drops unfinished unit batches and jobs from a redacted export", () => { + const { repos, db } = openRepos(); + const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + const insertBatch = (id: string, status: string) => db.db.prepare( + `INSERT INTO l3_unit_batches (id, machine_unit_key, scope_seq, user_id, session_id, trigger, + start_trace_seq, end_trace_seq, l1_memory_ids_json, raw_turn_ids_json, status, created_at, updated_at) + VALUES (?, ?, ?, ?, 's', 'session_close', 1, 1, '[]', '[]', ?, '2026-10-01T00:00:00.000Z', '2026-10-01T00:00:00.000Z')` + ).run(id, machine.unitKey, id === "done" ? 1 : 2, USER, status); + insertBatch("done", "applied"); + insertBatch("pending", "queued"); + const redacted = repos.runtime.exportBundleTables(false); + expect((redacted.l3_unit_batches ?? []).map((row) => row.id)).toEqual(["done"]); + }); + + it("empties the new tables when all memory data is cleared", () => { + const { repos, db } = openRepos(); + repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); + repos.l3Field.replaceUserProjects(USER, [{ rootPath: "/a", name: "A" }]); + repos.clearAllMemoryData(); + for (const table of NEW_TABLES) { + expect(db.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 }); + } + }); +}); + +describe("L3 field cognition flag", () => { + it("is off by default with a 15-day retention", () => { + expect(DEFAULT_MEMMY_CONFIG.algorithm.l3FieldCognition).toEqual({ enabled: false, inactiveRetentionDays: 15 }); + expect(isL3FieldCognitionActive(DEFAULT_MEMMY_CONFIG, { backendId: "sqlite-local" }, "local")).toBe(false); + }); + + it("is active only when enabled on local sqlite storage", () => { + const enabled = { + ...DEFAULT_MEMMY_CONFIG, + algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, l3FieldCognition: { enabled: true, inactiveRetentionDays: 15 } } + }; + expect(isL3FieldCognitionActive(enabled, { backendId: "sqlite-local" }, "local")).toBe(true); + expect(isL3FieldCognitionActive(enabled, { backendId: "sqlite-local" }, "cloud")).toBe(false); + expect(isL3FieldCognitionActive(enabled, { backendId: "openmem-cloud-rest" }, "local")).toBe(false); + }); + + it("reads the switch from memmyMemory.algorithm in the config file", () => { + const folder = mkdtempSync(join(tmpdir(), "l3-field-config-")); + folders.push(folder); + const file = join(folder, "config.yaml"); + writeFileSync(file, YAML.stringify({ memmyMemory: { algorithm: { l3FieldCognition: { enabled: true } } } })); + expect(loadMemmyConfig(file).config.algorithm.l3FieldCognition).toEqual({ enabled: true, inactiveRetentionDays: 15 }); + }); +}); diff --git a/Memory/tests/service/l3-field/text-units.test.ts b/Memory/tests/service/l3-field/text-units.test.ts new file mode 100644 index 000000000..0d2ae03fd --- /dev/null +++ b/Memory/tests/service/l3-field/text-units.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { countL3Units } from "../../../src/service/l3-field/text-units.js"; + +describe("countL3Units", () => { + it("counts CJK characters one by one", () => { + expect(countL3Units("这个仓库用 SSH 拉代码会失败")).toBe(12); + }); + + it("counts other text by whitespace-separated words", () => { + expect(countL3Units("pytest is the test runner here")).toBe(6); + }); + + it("counts mixed text by both rules and ignores punctuation-only tokens", () => { + expect(countL3Units("用 npm ci 安装依赖 , 不用 npm install")).toBe(11); + }); + + it("returns zero for blank text", () => { + expect(countL3Units(" \n ")).toBe(0); + }); +}); From f19b759467ed0e8d6fd56d313e111e351e3d37e6 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:29:19 +0200 Subject: [PATCH 2/9] feat(memory): resolve l3 projects from paths Decide which project a path belongs to for the new L3, by reading files only (no git process): - the nearer of a user-selected folder and a git repository wins; a shared root counts once with the git identity - git identity is the normalized origin (ssh, scp-like and https forms map to host/path), else the common git dir, so a clone's worktrees and subdirectories share one project - a session folder outside any repository is a plain-folder project; a tool path only joins the session project it lands in - temp dirs, Codex dated dirs, Claude scratch workspaces, ~/.codex/memories*, Memmy task folders and the home/Desktop/ Documents/Downloads folders themselves are never projects Co-Authored-By: Claude Opus 5.5 --- Memory/src/service/l3-field/git-identity.ts | 124 ++++++++++ .../src/service/l3-field/project-resolver.ts | 163 +++++++++++++ .../service/l3-field/project-resolver.test.ts | 219 ++++++++++++++++++ 3 files changed, 506 insertions(+) create mode 100644 Memory/src/service/l3-field/git-identity.ts create mode 100644 Memory/src/service/l3-field/project-resolver.ts create mode 100644 Memory/tests/service/l3-field/project-resolver.test.ts diff --git a/Memory/src/service/l3-field/git-identity.ts b/Memory/src/service/l3-field/git-identity.ts new file mode 100644 index 000000000..ed54b76f2 --- /dev/null +++ b/Memory/src/service/l3-field/git-identity.ts @@ -0,0 +1,124 @@ +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import path from "node:path"; + +export interface GitIdentity { + /** Working-tree root of this checkout (main clone or linked worktree). */ + repoRoot: string; + /** Git directory shared by every worktree of the clone. */ + commonDir: string; + /** Normalized origin URL, or `gitdir:` when there is no usable origin. */ + identity: string; +} + +/** + * Finds the repository containing `startDir` by reading files only (no git + * process). include/includeIf/insteadOf are not resolved; when origin cannot + * be read the clone is identified by its common git directory, so worktrees + * of one clone still match but separate clones do not. + */ +export function readGitIdentity(startDir: string): GitIdentity | null { + let current = startDir; + for (;;) { + const located = locateGitDir(current); + if (located) { + const commonDir = safeRealpath(readCommonDir(located.gitDir)); + const origin = readOriginUrl(commonDir); + const normalized = origin ? normalizeRemoteUrl(origin) : null; + return { + repoRoot: current, + commonDir, + identity: normalized ?? `gitdir:${commonDir}` + }; + } + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} + +/** Maps ssh, scp-like and http(s) remotes to `host[:port]/path`; local remotes return null. */ +export function normalizeRemoteUrl(url: string): string | null { + const trimmed = url.trim(); + if (!trimmed || trimmed.startsWith("/") || /^[A-Za-z]:[\\/]/.test(trimmed) || trimmed.startsWith("file:")) { + return null; + } + let host: string; + let port = ""; + let repoPath: string; + const scpLike = /^(?:[^@/]+@)?([^:/]+):(?!\/\/)(.+)$/.exec(trimmed); + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) { + let parsed: URL; + try { + parsed = new URL(trimmed); + } catch { + return null; + } + if (!parsed.hostname) return null; + host = parsed.hostname; + port = parsed.port; + repoPath = decodeURIComponent(parsed.pathname); + } else if (scpLike) { + host = scpLike[1]!; + repoPath = scpLike[2]!; + } else { + return null; + } + const cleanedPath = repoPath + .replace(/^\/+/, "") + .replace(/\/+$/, "") + .replace(/\.git$/i, "") + .toLowerCase(); + if (!cleanedPath) return null; + return `${host.toLowerCase()}${port ? `:${port}` : ""}/${cleanedPath}`; +} + +function locateGitDir(dir: string): { gitDir: string } | null { + const dotGit = path.join(dir, ".git"); + if (!existsSync(dotGit)) return null; + try { + if (statSync(dotGit).isDirectory()) return { gitDir: dotGit }; + const match = /^gitdir:\s*(.+)$/m.exec(readFileSync(dotGit, "utf8")); + if (!match) return null; + return { gitDir: path.resolve(dir, match[1]!.trim()) }; + } catch { + return null; + } +} + +function readCommonDir(gitDir: string): string { + try { + const relative = readFileSync(path.join(gitDir, "commondir"), "utf8").trim(); + return relative ? path.resolve(gitDir, relative) : gitDir; + } catch { + return gitDir; + } +} + +function readOriginUrl(commonDir: string): string | null { + let text: string; + try { + text = readFileSync(path.join(commonDir, "config"), "utf8"); + } catch { + return null; + } + let inOrigin = false; + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (line.startsWith("[")) { + inOrigin = /^\[remote\s+"origin"\]$/i.test(line); + continue; + } + if (!inOrigin) continue; + const match = /^url\s*=\s*(.+)$/i.exec(line); + if (match) return match[1]!.trim().replace(/^"(.*)"$/, "$1"); + } + return null; +} + +function safeRealpath(target: string): string { + try { + return realpathSync(target); + } catch { + return target; + } +} diff --git a/Memory/src/service/l3-field/project-resolver.ts b/Memory/src/service/l3-field/project-resolver.ts new file mode 100644 index 000000000..c06c2c7fd --- /dev/null +++ b/Memory/src/service/l3-field/project-resolver.ts @@ -0,0 +1,163 @@ +import { existsSync, realpathSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { readGitIdentity } from "./git-identity.js"; +import type { L3UserProject, ResolvedProject } from "./types.js"; + +export interface ResolverContext { + homeDir: string; + /** Folders the user picked in the desktop app. */ + userProjects: readonly L3UserProject[]; + /** Roots of throwaway task folders created by Memmy itself (default `~/Memmy`). */ + standaloneRoots: readonly string[]; + /** System temporary directories; defaults to the usual macOS/Linux locations. */ + tempRoots?: readonly string[]; + /** Remote workspaces are never resolved: the Memory service cannot see their files. */ + workspaceIsLocal: boolean; +} + +export interface ResolveProjectInput { + path: string; + /** A session folder may be a plain-folder project; a tool path only joins a project it lands in. */ + role: "session_dir" | "tool_path"; + sessionCwd: string | null; + sessionProjectRoot: string | null; +} + +type PathApi = Pick; + +export function defaultTempRoots(): string[] { + return [tmpdir(), "/tmp", "/private/tmp", "/var/folders", "/private/var/folders"]; +} + +/** True when `child` is `parent` or inside it, compared by path components (case-insensitive on Windows). */ +export function isPathWithin(parent: string, child: string, api: PathApi = path): boolean { + const caseInsensitive = api.sep === "\\"; + const from = api.resolve(caseInsensitive ? parent.toLowerCase() : parent); + const to = api.resolve(caseInsensitive ? child.toLowerCase() : child); + const relative = api.relative(from, to); + return relative === "" || (!relative.startsWith("..") && !api.isAbsolute(relative)); +} + +/** + * Decides which project a path belongs to: the nearer of a user-selected + * folder and a git repository, otherwise (for a session folder) the folder + * itself. Temporary and container folders never become projects. + */ +export function resolveProject(input: ResolveProjectInput, ctx: ResolverContext): ResolvedProject | null { + if (!ctx.workspaceIsLocal || !path.isAbsolute(input.path)) return null; + const target = lenientRealpath(input.path); + const home = lenientRealpath(ctx.homeDir); + if (isThrowawayLocation(target, home, ctx)) return null; + + const startDir = existingDirectory(target); + const selected = nearestUserProject(target, ctx.userProjects); + const gitRaw = startDir ? readGitIdentity(startDir) : null; + const git = gitRaw && isUsableRoot(gitRaw.repoRoot, home, ctx) ? gitRaw : null; + + if (selected && isUsableRoot(selected.root, home, ctx)) { + const gitIsNearer = git && git.repoRoot.length > selected.root.length; + if (!gitIsNearer) { + const sameRoot = git?.repoRoot === selected.root; + return { + identity: sameRoot ? git!.identity : `dir:${selected.root}`, + checkoutRoot: selected.root, + source: "user_selected", + displayName: selected.name + }; + } + } + if (git) { + return { + identity: git.identity, + checkoutRoot: git.repoRoot, + source: "git", + // Named after the remote repository so every checkout shows the same name. + displayName: git.identity.startsWith("gitdir:") + ? path.basename(git.repoRoot) + : git.identity.split("/").at(-1)! + }; + } + + if (input.role === "session_dir") { + const root = startDir ?? path.dirname(target); + if (!isUsableRoot(root, home, ctx)) return null; + return { identity: `dir:${root}`, checkoutRoot: root, source: "folder", displayName: path.basename(root) }; + } + const sessionFolder = input.sessionProjectRoot ?? input.sessionCwd; + if (sessionFolder && isPathWithin(lenientRealpath(sessionFolder), target)) { + return resolveProject({ ...input, path: sessionFolder, role: "session_dir" }, ctx); + } + return null; +} + +function nearestUserProject( + target: string, + projects: readonly L3UserProject[] +): { root: string; name: string } | null { + let best: { root: string; name: string } | null = null; + for (const project of projects) { + const root = lenientRealpath(project.rootPath); + if (isPathWithin(root, target) && (!best || root.length > best.root.length)) { + best = { root, name: project.name }; + } + } + return best; +} + +/** Inside a temp, Codex dated, Claude scratch, `~/.codex/memories*` or Memmy task folder. */ +function isThrowawayLocation(target: string, home: string, ctx: ResolverContext): boolean { + const roots = [ + ...(ctx.tempRoots ?? defaultTempRoots()), + ...ctx.standaloneRoots, + path.join(home, "Library", "Application Support", "Claude", "scratch-workspaces") + ].map(lenientRealpath); + if (roots.some((root) => isPathWithin(root, target))) return true; + const relativeToHome = isPathWithin(home, target) ? path.relative(home, target).split(path.sep) : []; + if (relativeToHome[0] === ".codex" && relativeToHome[1]?.startsWith("memories")) return true; + if (relativeToHome[0] === "Documents" && relativeToHome[1] === "Codex" && /^\d{4}-\d{2}-\d{2}$/.test(relativeToHome[2] ?? "")) { + return true; + } + return false; +} + +/** A project root may not be the home folder, a top-level container folder, a filesystem root or a throwaway location. */ +function isUsableRoot(root: string, home: string, ctx: ResolverContext): boolean { + if (path.dirname(root) === root) return false; + const containers = [home, ...["Desktop", "Documents", "Downloads"].map((name) => path.join(home, name))]; + if (containers.includes(root)) return false; + return !isThrowawayLocation(root, home, ctx); +} + +/** realpath that tolerates a missing tail: resolves the nearest existing ancestor and re-appends the rest. */ +function lenientRealpath(target: string): string { + const absolute = path.resolve(target); + const missing: string[] = []; + let current = absolute; + for (;;) { + try { + const real = realpathSync(current); + return missing.length ? path.join(real, ...missing.reverse()) : real; + } catch { + const parent = path.dirname(current); + if (parent === current) return absolute; + missing.push(path.basename(current)); + current = parent; + } + } +} + +/** The path itself when it is a directory, otherwise its nearest existing parent directory. */ +function existingDirectory(target: string): string | null { + let current = target; + for (;;) { + try { + if (existsSync(current) && statSync(current).isDirectory()) return current; + } catch { + // fall through to the parent + } + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} diff --git a/Memory/tests/service/l3-field/project-resolver.test.ts b/Memory/tests/service/l3-field/project-resolver.test.ts new file mode 100644 index 000000000..6b6844ff8 --- /dev/null +++ b/Memory/tests/service/l3-field/project-resolver.test.ts @@ -0,0 +1,219 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path, { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { normalizeRemoteUrl, readGitIdentity } from "../../../src/service/l3-field/git-identity.js"; +import { + isPathWithin, + resolveProject, + type ResolverContext +} from "../../../src/service/l3-field/project-resolver.js"; + +let home: string; +let ctx: ResolverContext; + +beforeEach(() => { + home = realpathSync(mkdtempSync(join(tmpdir(), "l3-resolver-home-"))); + ctx = { + homeDir: home, + userProjects: [], + standaloneRoots: [join(home, "Memmy")], + tempRoots: [join(home, "tmp")], + workspaceIsLocal: true + }; +}); + +afterEach(() => { + rmSync(home, { recursive: true, force: true }); +}); + +function dir(...parts: string[]): string { + const target = join(home, ...parts); + mkdirSync(target, { recursive: true }); + return target; +} + +function gitRepo(root: string, origin?: string): void { + mkdirSync(join(root, ".git"), { recursive: true }); + const remote = origin ? `[remote "origin"]\n\turl = ${origin}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n` : ""; + writeFileSync(join(root, ".git", "config"), `[core]\n\tbare = false\n${remote}`); +} + +function worktree(mainRoot: string, worktreeRoot: string, name: string): void { + const gitDir = join(mainRoot, ".git", "worktrees", name); + mkdirSync(gitDir, { recursive: true }); + writeFileSync(join(gitDir, "commondir"), "../..\n"); + mkdirSync(worktreeRoot, { recursive: true }); + writeFileSync(join(worktreeRoot, ".git"), `gitdir: ${path.relative(worktreeRoot, gitDir)}\n`); +} + +const asSession = (p: string) => ({ path: p, role: "session_dir" as const, sessionCwd: p, sessionProjectRoot: null }); + +describe("normalizeRemoteUrl", () => { + it("maps ssh, scp-like and https forms of the same repository to one identity", () => { + const expected = "github.com/memtensor/memmy-agent"; + expect(normalizeRemoteUrl("git@github.com:MemTensor/memmy-agent.git")).toBe(expected); + expect(normalizeRemoteUrl("ssh://git@GitHub.com/MemTensor/memmy-agent.git")).toBe(expected); + expect(normalizeRemoteUrl("https://github.com/MemTensor/memmy-agent")).toBe(expected); + expect(normalizeRemoteUrl("https://user@github.com/MemTensor/memmy-agent.git/")).toBe(expected); + }); + + it("keeps a non-default port and ignores local remotes", () => { + expect(normalizeRemoteUrl("ssh://git@git.example.test:2222/team/repo.git")).toBe("git.example.test:2222/team/repo"); + expect(normalizeRemoteUrl("/srv/git/repo.git")).toBeNull(); + expect(normalizeRemoteUrl("file:///srv/git/repo.git")).toBeNull(); + }); +}); + +describe("readGitIdentity", () => { + it("gives a worktree, its main clone and a subdirectory the same identity", () => { + const main = dir("code", "repo"); + gitRepo(main, "git@github.com:Org/Repo.git"); + const sub = dir("code", "repo", "src", "deep"); + const wt = join(home, "code", "repo-wt"); + worktree(main, wt, "repo-wt"); + const a = readGitIdentity(main)!; + expect(readGitIdentity(sub)!.identity).toBe(a.identity); + expect(readGitIdentity(sub)!.repoRoot).toBe(main); + expect(readGitIdentity(wt)!.identity).toBe(a.identity); + expect(readGitIdentity(wt)!.repoRoot).toBe(wt); + expect(a.identity).toBe("github.com/org/repo"); + }); + + it("falls back to the common git directory when there is no origin or the config is broken", () => { + const repo = dir("code", "local-only"); + gitRepo(repo); + expect(readGitIdentity(repo)!.identity).toBe(`gitdir:${join(repo, ".git")}`); + writeFileSync(join(repo, ".git", "config"), "\u0000not a config"); + expect(readGitIdentity(repo)!.identity).toBe(`gitdir:${join(repo, ".git")}`); + }); + + it("returns null outside any repository", () => { + expect(readGitIdentity(dir("plain"))).toBeNull(); + }); +}); + +describe("resolveProject", () => { + it("resolves a git repository by its origin, named after the remote repository", () => { + const repo = dir("code", "memmy-agent-feature-x"); + gitRepo(repo, "https://github.com/MemTensor/memmy-agent.git"); + expect(resolveProject(asSession(join(repo, "Memory")), ctx)).toEqual({ + identity: "github.com/memtensor/memmy-agent", + checkoutRoot: repo, + source: "git", + displayName: "memmy-agent" + }); + }); + + it("treats a plain session folder as a folder project", () => { + const folder = dir("Desktop", "reviews"); + expect(resolveProject(asSession(folder), ctx)).toEqual({ + identity: `dir:${folder}`, + checkoutRoot: folder, + source: "folder", + displayName: "reviews" + }); + }); + + it("does not treat temporary, Codex dated, Claude scratch or Memmy task folders as projects", () => { + expect(resolveProject(asSession(dir("tmp", "x")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir("Documents", "Codex", "2026-10-22", "na")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir("Library", "Application Support", "Claude", "scratch-workspaces", "a", "b")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir(".codex", "memories_v2")), ctx)).toBeNull(); + expect(resolveProject(asSession(dir("Memmy", "20261022-101010")), ctx)).toBeNull(); + }); + + it("does not treat the home directory, Desktop, Documents or Downloads themselves as projects", () => { + for (const container of [home, dir("Desktop"), dir("Documents"), dir("Downloads")]) { + expect(resolveProject(asSession(container), ctx)).toBeNull(); + } + }); + + it("ignores a git repository rooted at the home directory and falls back to the folder", () => { + gitRepo(home, "git@github.com:me/dotfiles.git"); + const notes = dir("notes"); + expect(resolveProject(asSession(notes), ctx)).toMatchObject({ source: "folder", checkoutRoot: notes }); + expect(resolveProject(asSession(home), ctx)).toBeNull(); + }); + + it("prefers the nearer root when a user-selected folder and a repository overlap", () => { + const repo = dir("code", "mono"); + gitRepo(repo, "git@github.com:org/mono.git"); + const selectedInside = dir("code", "mono", "packages", "app"); + const outer = dir("work"); + const repoInside = dir("work", "lib"); + gitRepo(repoInside, "git@github.com:org/lib.git"); + const withSelections: ResolverContext = { + ...ctx, + userProjects: [{ rootPath: selectedInside, name: "App" }, { rootPath: outer, name: "Work" }] + }; + expect(resolveProject(asSession(join(selectedInside, "src")), withSelections)).toMatchObject({ + source: "user_selected", checkoutRoot: selectedInside, displayName: "App" + }); + expect(resolveProject(asSession(repoInside), withSelections)).toMatchObject({ + source: "git", identity: "github.com/org/lib" + }); + }); + + it("counts a user-selected folder that is also the repository root once, with the git identity", () => { + const repo = dir("code", "same"); + gitRepo(repo, "git@github.com:org/same.git"); + const withSelection: ResolverContext = { ...ctx, userProjects: [{ rootPath: repo, name: "我的项目" }] }; + expect(resolveProject(asSession(repo), withSelection)).toEqual({ + identity: "github.com/org/same", + checkoutRoot: repo, + source: "user_selected", + displayName: "我的项目" + }); + }); + + it("compares by path components, so /repo does not contain /repo2", () => { + const repo = dir("code", "repo"); + gitRepo(repo, "git@github.com:org/repo.git"); + const sibling = dir("code", "repo2"); + expect(resolveProject(asSession(sibling), ctx)).toMatchObject({ source: "folder", checkoutRoot: sibling }); + }); + + it("resolves a tool path that does not exist yet through its nearest existing parent", () => { + const repo = dir("code", "repo"); + gitRepo(repo, "git@github.com:org/repo.git"); + expect(resolveProject({ + path: join(repo, "src", "new-dir", "new-file.ts"), role: "tool_path", sessionCwd: repo, sessionProjectRoot: repo + }, ctx)).toMatchObject({ identity: "github.com/org/repo" }); + }); + + it("falls back to the session project for a tool path under the session folder and ignores unrelated tool paths", () => { + const folder = dir("Desktop", "reviews"); + const file = join(folder, "paper-1", "notes.md"); + expect(resolveProject({ path: file, role: "tool_path", sessionCwd: folder, sessionProjectRoot: folder }, ctx)) + .toMatchObject({ source: "folder", checkoutRoot: folder }); + expect(resolveProject({ path: join(dir("elsewhere"), "x.txt"), role: "tool_path", sessionCwd: folder, sessionProjectRoot: folder }, ctx)) + .toBeNull(); + }); + + it("resolves a symlinked path to the same project as its target", () => { + const repo = dir("code", "real"); + gitRepo(repo, "git@github.com:org/real.git"); + const link = join(home, "link-to-real"); + symlinkSync(repo, link); + expect(resolveProject(asSession(link), ctx)).toMatchObject({ identity: "github.com/org/real", checkoutRoot: repo }); + }); + + it("returns null for a remote workspace", () => { + expect(resolveProject(asSession(dir("Desktop", "x")), { ...ctx, workspaceIsLocal: false })).toBeNull(); + }); +}); + +describe("isPathWithin", () => { + it("compares Windows paths by drive and components", () => { + expect(isPathWithin("C:\\work\\repo", "C:\\work\\repo\\src\\a.ts", path.win32)).toBe(true); + expect(isPathWithin("C:\\work\\repo", "c:\\work\\repo\\src", path.win32)).toBe(true); + expect(isPathWithin("C:\\work\\repo", "C:\\work\\repo2\\a.ts", path.win32)).toBe(false); + expect(isPathWithin("C:\\work\\repo", "D:\\work\\repo\\a.ts", path.win32)).toBe(false); + }); + + it("compares POSIX paths by components", () => { + expect(isPathWithin("/work/repo", "/work/repo")).toBe(true); + expect(isPathWithin("/work/repo", "/work/repo2")).toBe(false); + }); +}); From d2235b48c1cf2e1e13f1994c6d9f9cb73e64c283 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:43:51 +0200 Subject: [PATCH 3/9] feat(memory): accept user-selected project roots for l3 The desktop app now tells Memory which folders the user picked as projects, so L3 can treat them as project roots: - POST /api/v1/l3/user-projects replaces the user's whole list and the Memmy task-folder roots; relative paths are rejected; the list is stored regardless of the L3 field cognition flag - memmy-agent's long-running gateway reads the project registry every minute and pushes only when the list changed or the last push failed; a corrupt registry is never pushed - the contract is added identically to the Memory and App copies, with a test that keeps them in sync Co-Authored-By: Claude Opus 5.5 --- .../src/memory-l3-world-model.ts | 20 +++++ .../src/core/agent-runtime/loop.ts | 7 ++ .../src/entrypoints/cli/commands.ts | 1 + App/memmy-agent/src/memmy-memory/client.ts | 10 +++ .../src/memmy-memory/l3-user-project-sync.ts | 82 +++++++++++++++++++ App/memmy-agent/src/memmy-memory/register.ts | 29 ++++++- .../memmy-memory/l3-user-project-sync.test.ts | 78 ++++++++++++++++++ Memory/src/contracts/memory-l3-world-model.ts | 20 +++++ Memory/src/server/http.ts | 16 ++++ Memory/src/service/memory-service.ts | 15 +++- Memory/src/storage/l3-field-repository.ts | 19 +++++ .../contract/l3-field-user-projects.test.ts | 71 ++++++++++++++++ 12 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts create mode 100644 App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts create mode 100644 Memory/tests/contract/l3-field-user-projects.test.ts diff --git a/App/backend/local-api-contracts/src/memory-l3-world-model.ts b/App/backend/local-api-contracts/src/memory-l3-world-model.ts index 269d5f023..31b371365 100644 --- a/App/backend/local-api-contracts/src/memory-l3-world-model.ts +++ b/App/backend/local-api-contracts/src/memory-l3-world-model.ts @@ -83,6 +83,26 @@ export const L3WorldModelBoundaryResponseSchema = z.object({ }).strict(); export type L3WorldModelBoundaryResponse = z.infer; +const AbsolutePathSchema = z.string().min(1).max(4096) + .refine((value) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value), "path must be absolute"); + +/** Full list of folders the user picked in the desktop app; each push replaces the previous one. */ +export const L3UserProjectsReplaceRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + projects: z.array(z.object({ + rootPath: AbsolutePathSchema, + name: z.string().min(1).max(512) + }).strict()).max(500), + standaloneRoots: z.array(AbsolutePathSchema).max(20).default([]) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3UserProjectsReplaceRequest = z.input; + +export const L3UserProjectsReplaceResponseSchema = z.object({ + count: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3UserProjectsReplaceResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/App/memmy-agent/src/core/agent-runtime/loop.ts b/App/memmy-agent/src/core/agent-runtime/loop.ts index 44f5513cd..a4923b037 100644 --- a/App/memmy-agent/src/core/agent-runtime/loop.ts +++ b/App/memmy-agent/src/core/agent-runtime/loop.ts @@ -1288,6 +1288,13 @@ export class AgentLoop { return initialization; } + /** Long-running service only: keep Memory's copy of the desktop project list current. */ + startL3UserProjectSync(): void { + const projectStore = this.projectStore; + if (!projectStore) return; + this.memmyMemoryIntegration.startL3UserProjectSync?.(() => projectStore.snapshot()); + } + async closeRuntimeTools(): Promise { this.messageLoopReady = false; this.runtimeToolsClosed = true; diff --git a/App/memmy-agent/src/entrypoints/cli/commands.ts b/App/memmy-agent/src/entrypoints/cli/commands.ts index 513e5c4a5..0e57cbff4 100644 --- a/App/memmy-agent/src/entrypoints/cli/commands.ts +++ b/App/memmy-agent/src/entrypoints/cli/commands.ts @@ -1161,6 +1161,7 @@ export async function gateway({ cronService: cron, projectStore, }); + loop.startL3UserProjectSync?.(); reportStartupStage("agent-loop-created"); if (loop.sessions instanceof SessionManager) { loop.guiTranscriptMirror = new GuiTranscriptMirror(loop.sessions, canonicalWorkspace); diff --git a/App/memmy-agent/src/memmy-memory/client.ts b/App/memmy-agent/src/memmy-memory/client.ts index bb1a6ef95..14f35a5f9 100644 --- a/App/memmy-agent/src/memmy-memory/client.ts +++ b/App/memmy-agent/src/memmy-memory/client.ts @@ -1,11 +1,14 @@ import type { MemmyMemoryConnection, MemmyMemoryRequestEnvelope, JsonRecord } from "./types.js"; import { normalizeTimeZoneOffset } from "../utils/time-zone.js"; import { + L3UserProjectsReplaceResponseSchema, L3WorldModelBoundaryResponseSchema, L3WorldModelTraceHeadResponseSchema, MemoryHealthSnapshotSchema, SessionL3WorldModelContextResponseSchema, l3WorldModelGetTransport, + type L3UserProjectsReplaceRequest, + type L3UserProjectsReplaceResponse, type L3WorldModelBoundaryRequest, type L3WorldModelBoundaryResponse, type L3WorldModelRequestEnvelope, @@ -179,6 +182,13 @@ export class MemmyMemoryClient { return SessionL3WorldModelContextResponseSchema.parse(value); } + async replaceL3UserProjects( + request: L3UserProjectsReplaceRequest, + options: MemoryRequestOptions = {} + ): Promise { + return L3UserProjectsReplaceResponseSchema.parse(await this.post("/api/v1/l3/user-projects", request, options)); + } + } function safeJsonParse(text: string): any { diff --git a/App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts b/App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts new file mode 100644 index 000000000..fd97cb2c3 --- /dev/null +++ b/App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts @@ -0,0 +1,82 @@ +import { randomUUID } from "node:crypto"; +import type { L3UserProjectsReplaceRequest } from "@memmy/local-api-contracts"; +import type { WebuiProjectRegistrySnapshot } from "../entrypoints/frontend-bridge/projects.js"; +import type { MemmyMemoryClient } from "./client.js"; + +export const L3_USER_PROJECT_SYNC_INTERVAL_MS = 60_000; +/** An unchanged list is still pushed this often, so Memory gets it back after its data was cleared. */ +export const L3_USER_PROJECT_RESYNC_MS = 10 * 60_000; + +type Envelope = Omit; + +/** + * Keeps Memory's copy of the user's selected project folders current. It reads + * the registry on a timer and pushes the full list only when it differs from + * the last successful push, so startup, edits made by any process and a Memory + * service that was not reachable yet are all covered by the same loop. An + * unchanged list is pushed again every ten minutes in case Memory lost it. + */ +export class L3UserProjectSync { + private lastPushedKey: string | null = null; + private lastPushedAt = 0; + private inFlight: Promise | null = null; + private timer: ReturnType | null = null; + + constructor(private readonly options: { + client: Pick; + envelope: Envelope; + readProjects: () => WebuiProjectRegistrySnapshot; + standaloneRoots: () => string[]; + intervalMs?: number; + now?: () => number; + }) {} + + start(): void { + if (this.timer) return; + void this.syncNow(); + this.timer = setInterval(() => void this.syncNow(), this.options.intervalMs ?? L3_USER_PROJECT_SYNC_INTERVAL_MS); + this.timer.unref?.(); + } + + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + /** Pushes when the list changed or the last push failed; never throws. A corrupt registry is not pushed. */ + syncNow(): Promise { + if (this.inFlight) return this.inFlight; + this.inFlight = this.pushIfChanged().finally(() => { + this.inFlight = null; + }); + return this.inFlight; + } + + private async pushIfChanged(): Promise { + let snapshot: WebuiProjectRegistrySnapshot; + try { + snapshot = this.options.readProjects(); + } catch { + return false; + } + if (snapshot.state !== "ready") return false; + const projects = snapshot.projects.map((project) => ({ rootPath: project.rootPath, name: project.name })); + const standaloneRoots = this.options.standaloneRoots(); + const key = JSON.stringify({ projects, standaloneRoots }); + const now = (this.options.now ?? Date.now)(); + if (key === this.lastPushedKey && now - this.lastPushedAt < L3_USER_PROJECT_RESYNC_MS) return true; + try { + await this.options.client.replaceL3UserProjects({ + ...this.options.envelope, + requestId: randomUUID(), + projects, + standaloneRoots + }); + this.lastPushedKey = key; + this.lastPushedAt = now; + return true; + } catch { + return false; + } + } +} diff --git a/App/memmy-agent/src/memmy-memory/register.ts b/App/memmy-agent/src/memmy-memory/register.ts index 97913f2ce..d84b07af4 100644 --- a/App/memmy-agent/src/memmy-memory/register.ts +++ b/App/memmy-agent/src/memmy-memory/register.ts @@ -10,7 +10,10 @@ import { MemmyMemoryClient } from "./client.js"; import { resolveMemmyMemoryConfig } from "./config.js"; import { discoverMemmyMemoryConnection } from "./discovery.js"; import { MemmyMemoryHook } from "./hook.js"; +import { L3UserProjectSync } from "./l3-user-project-sync.js"; import type { MemmyMemoryInstallOptions } from "./types.js"; +import { getStandaloneTaskWorkspacesDir } from "../config/paths.js"; +import type { WebuiProjectRegistrySnapshot } from "../entrypoints/frontend-bridge/projects.js"; export type MemmyMemoryIntegration = { enabled: boolean; @@ -18,6 +21,8 @@ export type MemmyMemoryIntegration = { hook?: MemmyMemoryHook; dispose?: () => Promise; closeSession?: (sessionKey: string, reason?: string) => Promise; + /** Starts pushing the desktop project registry to Memory for L3 project recognition. */ + startL3UserProjectSync?: (readProjects: () => WebuiProjectRegistrySnapshot) => void; }; export { @@ -53,11 +58,33 @@ export function createMemmyMemoryIntegration( void hook.initialize().catch((error) => { hook.lastError = error instanceof Error ? error.message : String(error); }); + let projectSync: L3UserProjectSync | null = null; return { enabled: true, client, hook, - dispose: () => hook.dispose(), + dispose: async () => { + projectSync?.stop(); + await hook.dispose(); + }, + startL3UserProjectSync: (readProjects) => { + if (projectSync) return; + projectSync = new L3UserProjectSync({ + client, + envelope: { + adapterId: "memmy-agent", + source: "memmy-agent", + namespace: { + source: "memmy-agent", + profileId: "default", + ...(resolved.userId ? { userId: resolved.userId } : {}), + }, + }, + readProjects, + standaloneRoots: () => [getStandaloneTaskWorkspacesDir()], + }); + projectSync.start(); + }, closeSession: (sessionKey, reason = "deleted") => hook.sessionEnd(new AgentHookContext({ sessionKey, reason, diff --git a/App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts b/App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts new file mode 100644 index 000000000..571fd9d36 --- /dev/null +++ b/App/memmy-agent/tests/memmy-memory/l3-user-project-sync.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; +import type { WebuiProjectRegistrySnapshot } from "../../src/entrypoints/frontend-bridge/projects.js"; +import { L3UserProjectSync } from "../../src/memmy-memory/l3-user-project-sync.js"; + +const envelope = { + adapterId: "memmy-agent", + source: "memmy-agent", + namespace: { source: "memmy-agent", profileId: "default", userId: "u1" }, +}; + +function project(rootPath: string, name: string) { + return { id: name, name, rootPath, pinned: false, createdAt: "2026-10-24T00:00:00.000Z" }; +} + +function setup(initial: WebuiProjectRegistrySnapshot, now: () => number = () => 0) { + let snapshot = initial; + const replaceL3UserProjects = vi.fn().mockResolvedValue({ count: 0, serverTime: "2026-10-24T00:00:00.000Z" }); + const sync = new L3UserProjectSync({ + client: { replaceL3UserProjects }, + envelope, + readProjects: () => snapshot, + standaloneRoots: () => ["/Users/me/Memmy"], + now, + }); + return { sync, replaceL3UserProjects, set: (next: WebuiProjectRegistrySnapshot) => { snapshot = next; } }; +} + +describe("L3UserProjectSync", () => { + it("pushes the active project list once and again only after it changes", async () => { + const { sync, replaceL3UserProjects, set } = setup({ state: "ready", projects: [project("/work/a", "A")] }); + expect(await sync.syncNow()).toBe(true); + expect(await sync.syncNow()).toBe(true); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(1); + expect(replaceL3UserProjects.mock.calls[0]![0]).toMatchObject({ + ...envelope, + projects: [{ rootPath: "/work/a", name: "A" }], + standaloneRoots: ["/Users/me/Memmy"], + }); + set({ state: "ready", projects: [project("/work/a", "A2")] }); + await sync.syncNow(); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(2); + expect(replaceL3UserProjects.mock.calls[1]![0].projects).toEqual([{ rootPath: "/work/a", name: "A2" }]); + }); + + it("pushes an unchanged list again after ten minutes, in case Memory lost it", async () => { + let clock = 0; + const { sync, replaceL3UserProjects } = setup({ state: "ready", projects: [project("/work/a", "A")] }, () => clock); + await sync.syncNow(); + clock = 9 * 60_000; + await sync.syncNow(); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(1); + clock = 10 * 60_000; + await sync.syncNow(); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(2); + }); + + it("does not push a corrupt registry, so a failed load never wipes Memory's list", async () => { + const { sync, replaceL3UserProjects } = setup({ state: "corrupt", projects: [] }); + expect(await sync.syncNow()).toBe(false); + expect(replaceL3UserProjects).not.toHaveBeenCalled(); + }); + + it("retries on the next run after Memory was unreachable", async () => { + const { sync, replaceL3UserProjects } = setup({ state: "ready", projects: [project("/work/a", "A")] }); + replaceL3UserProjects.mockRejectedValueOnce(new Error("connect ECONNREFUSED")); + expect(await sync.syncNow()).toBe(false); + expect(await sync.syncNow()).toBe(true); + expect(replaceL3UserProjects).toHaveBeenCalledTimes(2); + }); + + it("runs one push at a time", async () => { + const { sync, replaceL3UserProjects } = setup({ state: "ready", projects: [project("/work/a", "A")] }); + const [first, second] = [sync.syncNow(), sync.syncNow()]; + expect(first).toBe(second); + await first; + expect(replaceL3UserProjects).toHaveBeenCalledTimes(1); + }); +}); diff --git a/Memory/src/contracts/memory-l3-world-model.ts b/Memory/src/contracts/memory-l3-world-model.ts index 269d5f023..31b371365 100644 --- a/Memory/src/contracts/memory-l3-world-model.ts +++ b/Memory/src/contracts/memory-l3-world-model.ts @@ -83,6 +83,26 @@ export const L3WorldModelBoundaryResponseSchema = z.object({ }).strict(); export type L3WorldModelBoundaryResponse = z.infer; +const AbsolutePathSchema = z.string().min(1).max(4096) + .refine((value) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value), "path must be absolute"); + +/** Full list of folders the user picked in the desktop app; each push replaces the previous one. */ +export const L3UserProjectsReplaceRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + projects: z.array(z.object({ + rootPath: AbsolutePathSchema, + name: z.string().min(1).max(512) + }).strict()).max(500), + standaloneRoots: z.array(AbsolutePathSchema).max(20).default([]) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3UserProjectsReplaceRequest = z.input; + +export const L3UserProjectsReplaceResponseSchema = z.object({ + count: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3UserProjectsReplaceResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index ec8d0447c..6ad90c733 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -7,6 +7,7 @@ import { } from "../agent-source/runtime.js"; import { EmbeddingInferenceInputSchema, + L3UserProjectsReplaceRequestSchema, L3WorldModelBoundaryRequestSchema, L3WorldModelRequestEnvelopeSchema, OpenSessionInputSchema @@ -75,6 +76,7 @@ export const API_ROUTES = [ "GET /api/v1/sessions/:sessionId/l3-world-model-trace-head", "POST /api/v1/sessions/:sessionId/l3-world-model-boundary", "GET /api/v1/l3-world-model/sessions/:sessionId/context", + "POST /api/v1/l3/user-projects", "POST /api/v1/turns/start", "POST /api/v1/turns/:turnId/complete", "POST /api/v1/source-turns/complete", @@ -651,6 +653,20 @@ async function routeRequest( return result; } + if (method === "POST" && path === "/api/v1/l3/user-projects") { + requireMemoryWrite(principal); + const parsed = L3UserProjectsReplaceRequestSchema.safeParse( + strictEnvelopeWithPrincipal(asObject(body, "l3.user-projects"), principal) + ); + if (!parsed.success) { + throw new MemoryServiceError( + "invalid_argument", + parsed.error.issues.map((issue) => `${issue.path.join(".") || "request"}: ${issue.message}`).join("; ") + ); + } + return service.replaceL3UserProjects(parsed.data); + } + const l3Context = match(path, /^\/api\/v1\/l3-world-model\/sessions\/([^/]+)\/context$/); if (method === "GET" && l3Context) { requireMemoryRead(principal); diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index 76d394b81..7bc876ed9 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -2,7 +2,9 @@ import { assertJsonValue, canonicalJson, isLocalWorkspaceUri, - sha256Hex + sha256Hex, + type L3UserProjectsReplaceRequest, + type L3UserProjectsReplaceResponse } from "../contracts/index.js"; import { languageSteeringLine, @@ -1283,6 +1285,17 @@ export class MemoryService { return this.repos.l3WorldModels.traceHead(sessionId); } + /** Stores the desktop app's selected project folders; kept regardless of the L3 field cognition flag. */ + replaceL3UserProjects(request: L3UserProjectsReplaceRequest): L3UserProjectsReplaceResponse { + const userId = normalizeNamespace(request.namespace).userId; + const at = nowIso(); + this.repos.transaction(() => { + this.repos.l3Field.replaceUserProjects(userId, request.projects, at); + this.repos.l3Field.setStandaloneRoots(userId, request.standaloneRoots ?? [], at); + }); + return { count: request.projects.length, serverTime: at }; + } + l3WorldModelBoundary( sessionId: string, request: L3WorldModelBoundaryRequest diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts index a235260b0..7b8b8d74e 100644 --- a/Memory/src/storage/l3-field-repository.ts +++ b/Memory/src/storage/l3-field-repository.ts @@ -269,6 +269,21 @@ export class L3FieldRepository { })(); } + /** Throwaway task roots Memmy itself creates (they are never projects); stored with the project list. */ + getStandaloneRoots(userId: string): string[] { + const row = this.db.prepare(`SELECT value_json FROM runtime_kv WHERE key = ?`) + .get(standaloneRootsKey(userId)) as { value_json: string } | undefined; + const value = row ? parseJson(row.value_json, []) : []; + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; + } + + setStandaloneRoots(userId: string, roots: readonly string[], at = nowIso()): void { + this.db.prepare( + `INSERT INTO runtime_kv (key, value_json, updated_at) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at` + ).run(standaloneRootsKey(userId), toJson([...roots]), at); + } + private writeSnapshot(unit: L3UnitRecord, at: string): MemoryRow { const memoryValue = renderL3UnitSnapshot(unit, this.activeItems(unit.unitKey)); const existing = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; @@ -338,6 +353,10 @@ export class L3FieldRepository { } } +function standaloneRootsKey(userId: string): string { + return `l3_field.standalone_roots:${userId}`; +} + function unitFromSql(row: SqlUnitRow): L3UnitRecord { return { unitKey: row.unit_key, diff --git a/Memory/tests/contract/l3-field-user-projects.test.ts b/Memory/tests/contract/l3-field-user-projects.test.ts new file mode 100644 index 000000000..5a687e764 --- /dev/null +++ b/Memory/tests/contract/l3-field-user-projects.test.ts @@ -0,0 +1,71 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { createMemoryHttpServer } from "../../src/index.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); + +async function withServer(run: (post: (body: unknown) => Promise, repos: Repositories) => Promise): Promise { + const { service, db } = createTestService(); + const server = createMemoryHttpServer({ + service, + startAgentSourceAutomation: false, + auth: { mode: "local", allowAnonymous: true } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const post = (body: unknown) => fetch(`http://127.0.0.1:${address.port}/api/v1/l3/user-projects`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }); + try { + return await run(post, new Repositories(db.db)); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +} + +const envelope = () => ({ + requestId: randomUUID(), + adapterId: "memmy-agent", + source: "memmy-agent", + namespace: { source: "memmy-agent", profileId: "default", userId: "desk-user" } +}); + +describe("L3 user projects endpoint", () => { + it("replaces the whole list and the standalone roots for the user", async () => { + await withServer(async (post, repos) => { + const first = await post({ ...envelope(), projects: [ + { rootPath: "/work/a", name: "A" }, { rootPath: "/work/b", name: "B" } + ], standaloneRoots: ["/Users/me/Memmy"] }); + expect(first.status).toBe(200); + expect(await first.json()).toMatchObject({ count: 2 }); + const second = await post({ ...envelope(), projects: [{ rootPath: "/work/b", name: "B2" }] }); + expect(second.status).toBe(200); + expect(repos.l3Field.listUserProjects("desk-user")).toEqual([{ rootPath: "/work/b", name: "B2" }]); + expect(repos.l3Field.getStandaloneRoots("desk-user")).toEqual([]); + }); + }); + + it("rejects relative project paths", async () => { + await withServer(async (post, repos) => { + const response = await post({ ...envelope(), projects: [{ rootPath: "work/a", name: "A" }] }); + expect(response.status).toBe(400); + expect(repos.l3Field.listUserProjects("desk-user")).toEqual([]); + }); + }); +}); + +describe("L3 user projects contract copies", () => { + it("keeps the Memory and App contract files identical", () => { + const memoryCopy = fileURLToPath(new URL("../../src/contracts/memory-l3-world-model.ts", import.meta.url)); + const appCopy = fileURLToPath(new URL("../../../App/backend/local-api-contracts/src/memory-l3-world-model.ts", import.meta.url)); + expect(readFileSync(appCopy, "utf8")).toBe(readFileSync(memoryCopy, "utf8")); + }); +}); From 608aeecfe476bee766c560cea93f2b64a92805f9 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:58:13 +0200 Subject: [PATCH 4/9] feat(memory): build l3 evidence fragments with unit candidates Cut a raw turn into evidence fragments for the new L3 so each fact can be attributed by the paths its own evidence touched: - one fragment for the user's words, one per tool call with its paired result (reusing the existing strict call/result pairing), and one per result that matches no call - paths come from file/dir arguments, cd and git -C targets and absolute command arguments; Codex code-mode snippets are read for their cmd/workdir literals; relative paths resolve against the call's workdir, else the session folder; URLs are ignored - each fragment gets the project units its paths resolve to, falling back to the session folder; long outputs keep head, tail and the error lines in between Co-Authored-By: Claude Opus 5.5 --- .../service/l3-field/evidence-fragments.ts | 311 ++++++++++++++++++ .../l3-field/evidence-fragments.test.ts | 141 ++++++++ 2 files changed, 452 insertions(+) create mode 100644 Memory/src/service/l3-field/evidence-fragments.ts create mode 100644 Memory/tests/service/l3-field/evidence-fragments.test.ts diff --git a/Memory/src/service/l3-field/evidence-fragments.ts b/Memory/src/service/l3-field/evidence-fragments.ts new file mode 100644 index 000000000..fb7df2944 --- /dev/null +++ b/Memory/src/service/l3-field/evidence-fragments.ts @@ -0,0 +1,311 @@ +import path from "node:path"; +import { sha256Hex } from "../../contracts/index.js"; +import type { RawTurnRecord } from "../../storage/repositories.js"; +import { matchToolResultIndices } from "../../utils/tool-call-pairing.js"; +import type { EvidenceFragment, ResolvedProject } from "./types.js"; + +const PATH_KEYS = ["file_path", "filePath", "path", "notebook_path", "workdir", "cwd", "directory"]; +const COMMAND_KEYS = ["command", "cmd"]; +const USER_TEXT_LIMIT = 4_000; +const CLIP_HEAD_CHARS = 600; +const CLIP_TAIL_CHARS = 600; +const ERROR_LINE = /error|fail|exception|denied|not found|traceback|refused|cannot|no such|fatal/i; +const COMMAND_SEPARATORS = new Set(["&&", "||", ";", "|"]); + +/** + * Cuts one raw turn into evidence fragments: the user's words, and each tool + * call together with its result. Every fragment carries the project units its + * own paths resolve to, so content about repository A never borrows evidence + * from a call that touched repository B. + */ +export function buildEvidenceFragments(input: { + rawTurn: RawTurnRecord; + l1MemoryId: string; + sessionCwd: string | null; + sessionProjectRoot: string | null; + resolve: (target: string, role: "session_dir" | "tool_path") => ResolvedProject | null; + unitKeyOf: (project: ResolvedProject) => string; +}): EvidenceFragment[] { + const { rawTurn } = input; + const sessionUnits = (): string[] => { + if (!input.sessionCwd) return []; + const project = input.resolve(input.sessionCwd, "session_dir"); + return project ? [input.unitKeyOf(project)] : []; + }; + const unitsForPaths = (paths: string[]): string[] => { + if (paths.length === 0) return sessionUnits(); + const keys = new Set(); + for (const target of paths) { + const project = input.resolve(target, "tool_path"); + if (project) keys.add(input.unitKeyOf(project)); + } + return [...keys]; + }; + const fragmentId = (kind: string, index: number) => + sha256Hex(`${rawTurn.id}:${kind}:${index}`).slice(0, 16); + + const fragments: EvidenceFragment[] = []; + const userText = rawTurn.userText?.trim(); + if (userText) { + fragments.push({ + id: fragmentId("user", 0), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: null, + input: null, + output: userText.slice(0, USER_TEXT_LIMIT), + isError: false, + paths: [], + candidateUnitKeys: sessionUnits(), + at: rawTurn.createdAt + }); + } + + rawTurn.toolCalls.forEach((call, index) => { + if (!isRecord(call)) return; + const paths = extractToolPaths(call, input.sessionCwd); + fragments.push({ + id: fragmentId("call", index), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: typeof call.name === "string" ? call.name : null, + input: call.input ?? null, + output: toolOutputText(call), + isError: isErrorRecord(call), + paths, + candidateUnitKeys: unitsForPaths(paths), + at: typeof call.endedAt === "string" ? call.endedAt : rawTurn.createdAt + }); + }); + + const matched = new Set( + matchToolResultIndices(rawTurn.toolCalls, rawTurn.toolResults).filter((index): index is number => index !== undefined) + ); + rawTurn.toolResults.forEach((result, index) => { + if (matched.has(index) || !isRecord(result)) return; + const name = [result.name, result.toolName, result.tool_name].find((value) => typeof value === "string"); + fragments.push({ + id: fragmentId("result", index), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: typeof name === "string" ? name : null, + input: null, + output: toolOutputText(result), + isError: isErrorRecord(result), + paths: [], + candidateUnitKeys: sessionUnits(), + at: rawTurn.createdAt + }); + }); + return fragments; +} + +/** + * Paths a tool call actually touched: file/dir arguments, `cd` and `git -C` + * targets, and absolute path arguments of commands. Relative paths resolve + * against the call's own workdir/cwd, else the session folder. + */ +export function extractToolPaths(call: Record, sessionCwd: string | null): string[] { + const rawArgs = call.input ?? call.arguments ?? call.args; + const args = parseArguments(rawArgs); + if (!args) { + return typeof rawArgs === "string" ? codeModePaths(rawArgs, sessionCwd) : []; + } + const workdir = stringValue(args.workdir) ?? stringValue(args.cwd) ?? stringValue(args.directory) ?? sessionCwd; + const found: string[] = []; + const add = (value: string, base: string | null) => { + const resolved = resolveAgainst(value, base); + if (resolved && !found.includes(resolved)) found.push(resolved); + }; + for (const key of PATH_KEYS) { + const value = stringValue(args[key]); + if (value) add(value, key === "workdir" || key === "cwd" || key === "directory" ? sessionCwd : workdir); + } + for (const key of COMMAND_KEYS) { + const raw = args[key]; + const command = typeof raw === "string" ? raw : Array.isArray(raw) ? raw.filter((part) => typeof part === "string").join(" ") : null; + if (command) for (const target of commandPaths(command, workdir)) add(target, null); + } + return found; +} + +/** + * Codex "code mode" passes a JavaScript snippet such as + * `tools.exec_command({cmd: "...", workdir: "..."})` instead of JSON; read the + * string literals of the known argument keys out of it. + */ +function codeModePaths(code: string, sessionCwd: string | null): string[] { + const literal = /\b(cmd|command|workdir|cwd|file_path|filePath|path)\s*:\s*("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)/g; + const values: Record = {}; + for (const match of code.matchAll(literal)) { + const value = unquoteJsString(match[2]!); + if (value !== null) (values[match[1]!] ??= []).push(value); + } + const found: string[] = []; + const add = (target: string | null) => { + if (target && !found.includes(target)) found.push(target); + }; + const workdirs = [...(values.workdir ?? []), ...(values.cwd ?? [])]; + for (const dir of workdirs) add(resolveAgainst(dir, sessionCwd)); + const base = workdirs[0] ? resolveAgainst(workdirs[0], sessionCwd) : sessionCwd; + for (const key of ["file_path", "filePath", "path"]) { + for (const value of values[key] ?? []) add(resolveAgainst(value, base)); + } + for (const command of [...(values.cmd ?? []), ...(values.command ?? [])]) { + for (const target of commandPaths(command, base)) add(target); + } + return found; +} + +function unquoteJsString(quoted: string): string | null { + const body = quoted.slice(1, -1); + if (quoted.startsWith("\"")) { + try { + return JSON.parse(quoted) as string; + } catch { + return null; + } + } + return body.replace(/\\(.)/g, (_, char: string) => (char === "n" ? "\n" : char === "t" ? "\t" : char)); +} + +/** Keeps head and tail plus any error-looking line from the middle, so a mid-output error is not lost. */ +export function clipToolOutput(text: string): string { + if (text.length <= CLIP_HEAD_CHARS + CLIP_TAIL_CHARS) return text; + const head = text.slice(0, CLIP_HEAD_CHARS); + const tail = text.slice(-CLIP_TAIL_CHARS); + const middle = text.slice(CLIP_HEAD_CHARS, text.length - CLIP_TAIL_CHARS); + const errorLines = middle.split(/\r?\n/).filter((line) => ERROR_LINE.test(line)).slice(0, 20); + return [head, "[已截断]", ...errorLines, ...(errorLines.length ? ["[已截断]"] : []), tail].join("\n"); +} + +function commandPaths(command: string, startDir: string | null): string[] { + const found: string[] = []; + let cwd = startDir; + let segment: string[] = []; + const flush = () => { + if (segment.length === 0) return; + const [executable, ...rest] = segment; + if (executable === "cd" && rest[0]) { + const target = resolveAgainst(rest[0], cwd); + if (target) { + found.push(target); + cwd = target; + } + } else { + for (let i = 0; i < rest.length; i += 1) { + const token = rest[i]!; + if (executable === "git" && token === "-C" && rest[i + 1]) { + const target = resolveAgainst(rest[i + 1]!, cwd); + if (target) found.push(target); + i += 1; + continue; + } + if (isAbsoluteAny(token) && !token.includes("://")) found.push(token); + } + } + segment = []; + }; + for (const token of shellTokens(command)) { + if (COMMAND_SEPARATORS.has(token)) flush(); + else segment.push(token); + } + flush(); + return found; +} + +/** Minimal shell word splitting: quotes, backslash escapes and the &&, ||, ;, | separators. */ +function shellTokens(command: string): string[] { + const tokens: string[] = []; + let current = ""; + let hasToken = false; + let quote: "'" | "\"" | null = null; + for (let i = 0; i < command.length; i += 1) { + const char = command[i]!; + if (quote) { + if (char === quote) quote = null; + else if (char === "\\" && quote === "\"" && i + 1 < command.length) current += command[++i]; + else current += char; + continue; + } + if (char === "'" || char === "\"") { + quote = char; + hasToken = true; + continue; + } + if (char === "\\" && i + 1 < command.length) { + current += command[++i]; + hasToken = true; + continue; + } + const pair = command.slice(i, i + 2); + if (pair === "&&" || pair === "||" || char === ";" || char === "|") { + if (hasToken) tokens.push(current); + tokens.push(pair === "&&" || pair === "||" ? pair : char); + current = ""; + hasToken = false; + if (pair === "&&" || pair === "||") i += 1; + continue; + } + if (/\s/.test(char)) { + if (hasToken) tokens.push(current); + current = ""; + hasToken = false; + continue; + } + current += char; + hasToken = true; + } + if (hasToken) tokens.push(current); + return tokens; +} + +function parseArguments(value: unknown): Record | null { + if (isRecord(value)) return value; + if (typeof value !== "string") return null; + try { + const parsed = JSON.parse(value) as unknown; + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } +} + +function resolveAgainst(value: string, base: string | null): string | null { + const trimmed = value.trim(); + if (!trimmed || trimmed.includes("://")) return null; + if (isAbsoluteAny(trimmed)) return trimmed; + if (!base) return null; + return isWindowsPath(base) ? path.win32.resolve(base, trimmed) : path.posix.resolve(base, trimmed); +} + +function isAbsoluteAny(value: string): boolean { + return value.startsWith("/") || isWindowsPath(value); +} + +function isWindowsPath(value: string): boolean { + return /^[A-Za-z]:[\\/]/.test(value); +} + +function toolOutputText(record: Record): string | null { + const error = typeof record.error === "string" ? record.error : null; + const output = record.output ?? record.result ?? record.content; + const text = typeof output === "string" ? output : output === undefined || output === null ? null : JSON.stringify(output); + const combined = [text, error && !text?.includes(error) ? error : null].filter(Boolean).join("\n"); + return combined ? clipToolOutput(combined) : null; +} + +function isErrorRecord(record: Record): boolean { + return record.success === false || + (typeof record.error === "string" && record.error.trim().length > 0) || + record.status === "failed" || + record.status === "error"; +} + +function stringValue(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} diff --git a/Memory/tests/service/l3-field/evidence-fragments.test.ts b/Memory/tests/service/l3-field/evidence-fragments.test.ts new file mode 100644 index 000000000..2d763f0b9 --- /dev/null +++ b/Memory/tests/service/l3-field/evidence-fragments.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import { + buildEvidenceFragments, + clipToolOutput, + extractToolPaths +} from "../../../src/service/l3-field/evidence-fragments.js"; +import type { ResolvedProject } from "../../../src/service/l3-field/types.js"; +import type { RawTurnRecord } from "../../../src/storage/repositories.js"; + +function rawTurn(toolCalls: unknown[], toolResults: unknown[] = [], userText = "帮我看一下"): RawTurnRecord { + return { + id: "raw-1", + sessionId: "s", + episodeId: "e", + turnId: "t", + userId: "u", + userText, + assistantText: "好的", + toolCalls, + toolResults, + sourceMemoryIds: [], + usage: {}, + status: "succeeded", + createdAt: "2026-10-24T01:00:00.000Z" + }; +} + +/** Projects are the first two path components under /work, e.g. /work/repo-a. */ +function resolve(target: string): ResolvedProject | null { + const match = /^\/work\/([^/]+)/.exec(target); + if (!match) return null; + return { identity: `id:${match[1]}`, checkoutRoot: `/work/${match[1]}`, source: "git", displayName: match[1]! }; +} +const unitKeyOf = (project: ResolvedProject) => `unit:${project.identity}`; + +function build(turn: RawTurnRecord, sessionCwd: string | null = "/work/repo-a") { + return buildEvidenceFragments({ rawTurn: turn, l1MemoryId: "l1-1", sessionCwd, sessionProjectRoot: sessionCwd, resolve, unitKeyOf }); +} + +describe("extractToolPaths", () => { + it("reads Claude Code file tools and resolves relative paths against the working directory", () => { + expect(extractToolPaths({ name: "Read", input: { file_path: "/work/repo-a/src/a.ts" } }, null)).toEqual(["/work/repo-a/src/a.ts"]); + expect(extractToolPaths({ name: "Edit", input: { file_path: "src/a.ts" } }, "/work/repo-a")).toEqual(["/work/repo-a/src/a.ts"]); + expect(extractToolPaths({ name: "NotebookEdit", input: { notebook_path: "/work/repo-a/n.ipynb" } }, null)).toEqual(["/work/repo-a/n.ipynb"]); + }); + + it("parses Codex arguments given as a JSON string, including workdir", () => { + const call = { name: "exec_command", input: JSON.stringify({ cmd: "npm test", workdir: "/work/repo-b" }) }; + expect(extractToolPaths(call, "/work/repo-a")).toEqual(["/work/repo-b"]); + }); + + it("reads cmd and workdir out of a Codex code-mode snippet", () => { + const code = [ + "const r = await Promise.allSettled([", + " tools.exec_command({cmd: \"sed -n 1,20p src/a.ts\", workdir: \"/work/repo-b\"}),", + " tools.exec_command({cmd: 'cat /work/repo-c/x.md'}),", + "]);" + ].join("\n"); + expect(extractToolPaths({ name: "exec", input: code }, "/work/repo-a")).toEqual([ + "/work/repo-b", "/work/repo-c/x.md" + ]); + }); + + it("skips arguments that are broken JSON", () => { + expect(extractToolPaths({ name: "exec_command", input: "{not json" }, "/work/repo-a")).toEqual([]); + }); + + it("takes cd and git -C targets and absolute path arguments from commands, but not URLs or the executable", () => { + const command = `cd /work/repo-a && git -C "/work/repo b" status && /usr/bin/env cat /work/repo-c/README.md && curl https://example.com/x`; + expect(extractToolPaths({ name: "Bash", input: { command } }, null)).toEqual([ + "/work/repo-a", "/work/repo b", "/work/repo-c/README.md" + ]); + }); + + it("follows a relative cd from the current directory", () => { + expect(extractToolPaths({ name: "Bash", input: { command: "cd packages/app && npm test" } }, "/work/mono")) + .toEqual(["/work/mono/packages/app"]); + }); + + it("recognizes Windows drive paths", () => { + expect(extractToolPaths({ name: "Read", input: { file_path: "C:\\work\\repo\\a.ts" } }, null)).toEqual(["C:\\work\\repo\\a.ts"]); + }); +}); + +describe("clipToolOutput", () => { + it("keeps head, tail and the error lines in between, and marks the cut", () => { + const middle = Array.from({ length: 400 }, (_, i) => `line ${i}`); + middle[200] = "Error: ENOENT: no such file or directory"; + const clipped = clipToolOutput(middle.join("\n")); + expect(clipped).toContain("line 0"); + expect(clipped).toContain("line 399"); + expect(clipped).toContain("Error: ENOENT: no such file or directory"); + expect(clipped).toContain("[已截断]"); + }); + + it("returns short output unchanged", () => { + expect(clipToolOutput("ok")).toBe("ok"); + }); +}); + +describe("buildEvidenceFragments", () => { + it("makes one fragment for the user text and one per tool call with its own candidates", () => { + const fragments = build(rawTurn([ + { id: "c1", name: "Edit", input: { file_path: "/work/repo-a/a.ts" }, output: "done", success: true }, + { id: "c2", name: "Bash", input: { command: "cd /work/repo-b && npm test" }, output: "1 failed", success: false } + ])); + expect(fragments.map((fragment) => [fragment.toolName, fragment.candidateUnitKeys, fragment.isError])).toEqual([ + [null, ["unit:id:repo-a"], false], + ["Edit", ["unit:id:repo-a"], false], + ["Bash", ["unit:id:repo-b"], true] + ]); + expect(fragments[0]!.output).toBe("帮我看一下"); + expect(new Set(fragments.map((fragment) => fragment.id)).size).toBe(3); + expect(fragments.every((fragment) => fragment.l1MemoryId === "l1-1" && fragment.rawTurnId === "raw-1")).toBe(true); + }); + + it("gives stable fragment ids for the same raw turn", () => { + const turn = rawTurn([{ id: "c1", name: "Read", input: { file_path: "/work/repo-a/a.ts" }, output: "x" }]); + expect(build(turn).map((fragment) => fragment.id)).toEqual(build(turn).map((fragment) => fragment.id)); + }); + + it("falls back to the session folder when a tool call has no path, and to nothing outside any project", () => { + expect(build(rawTurn([{ name: "WebSearch", input: { query: "x" }, output: "y" }]))[1]!.candidateUnitKeys) + .toEqual(["unit:id:repo-a"]); + expect(build(rawTurn([{ name: "WebSearch", input: { query: "x" }, output: "y" }]), "/Users/me")[1]!.candidateUnitKeys) + .toEqual([]); + }); + + it("keeps a tool result that matches no call as its own fragment", () => { + const fragments = build(rawTurn( + [{ id: "c1", name: "Read", input: { file_path: "/work/repo-a/a.ts" }, output: "x" }], + [{ id: "c1", output: "x" }, { id: "orphan", name: "Bash", output: "permission denied", success: false }] + )); + expect(fragments.at(-1)).toMatchObject({ toolName: "Bash", output: "permission denied", isError: true }); + }); + + it("treats an explicit error message as an error even without success=false", () => { + const fragments = build(rawTurn([{ id: "c1", name: "Bash", input: { command: "ls" }, error: "exit code 1" }])); + expect(fragments[1]).toMatchObject({ isError: true, output: "exit code 1" }); + }); +}); From b4e1950d62ee851dbc540432fc6a943f14225247 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:04:39 +0200 Subject: [PATCH 5/9] feat(memory): profile machine and project units for l3 - machine profile: probe OS (macOS product version, not the Darwin kernel), arch, shell and node/python3/git/npm/brew versions with 2-second timeouts, and render it in a fixed format by code only - project profile for the new L3: reuse the existing scan and the shared summarizeProjectProfile model call, but feed it only what the project declares (runtime versions probed on this machine belong to the machine profile); write it into the project unit and refresh its snapshot; skip the model when the scan is unchanged and drop the result when the unit was deleted meanwhile - at most one pending profile job per unit, and a unit scanned in the last ten minutes is not scanned again (each session open asks for one); each request carries its own scan id because the immutable dedupe index never frees a key - the old project-profile pipeline is unchanged apart from calling the extracted summarizeProjectProfile Co-Authored-By: Claude Opus 5.5 --- .../src/service/l3-field/machine-profile.ts | 164 ++++++++++++++++++ .../project-environment/profile-pipeline.ts | 55 +++--- .../project-environment-service.ts | 131 +++++++++++--- Memory/src/storage/l3-field-repository.ts | 7 + .../service/l3-field/machine-profile.test.ts | 85 +++++++++ .../service/l3-field/unit-profile.test.ts | 113 ++++++++++++ 6 files changed, 512 insertions(+), 43 deletions(-) create mode 100644 Memory/src/service/l3-field/machine-profile.ts create mode 100644 Memory/tests/service/l3-field/machine-profile.test.ts create mode 100644 Memory/tests/service/l3-field/unit-profile.test.ts diff --git a/Memory/src/service/l3-field/machine-profile.ts b/Memory/src/service/l3-field/machine-profile.ts new file mode 100644 index 000000000..0699ada69 --- /dev/null +++ b/Memory/src/service/l3-field/machine-profile.ts @@ -0,0 +1,164 @@ +import { execFile } from "node:child_process"; +import { arch, platform, release, tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { sha256Hex } from "../../contracts/index.js"; + +const execFileAsync = promisify(execFile); + +/** Always listed in the profile, as installed or not found. */ +export const MACHINE_TOOLS = ["node", "python3", "git", "npm", "brew"] as const; +/** Listed only when installed. */ +export const OPTIONAL_MACHINE_TOOLS = ["pnpm", "yarn", "uv", "conda", "docker", "java", "go", "rustc"] as const; +export type MachineTool = typeof MACHINE_TOOLS[number] | typeof OPTIONAL_MACHINE_TOOLS[number]; +const ALL_TOOLS: readonly MachineTool[] = [...MACHINE_TOOLS, ...OPTIONAL_MACHINE_TOOLS]; + +const TOOL_ARGS: Record = { + node: ["--version"], + python3: ["--version"], + git: ["--version"], + npm: ["--version"], + brew: ["--version"], + pnpm: ["--version"], + yarn: ["--version"], + uv: ["--version"], + conda: ["--version"], + docker: ["--version"], + java: ["-version"], + go: ["version"], + rustc: ["--version"] +}; +const PROBE_TIMEOUT_MS = 2_000; +// One login shell runs every probe; shell start-up files (nvm, conda, pyenv) can take a few seconds. +const LOGIN_SHELL_TIMEOUT_MS = 8_000; +const SECTION_MARKER = "__memmy_probe__"; + +export interface MachineProbe { + os: string; + osVersion: string; + arch: string; + shell: string | null; + tools: Record; +} + +export type VersionRunner = (executable: string, args: string[]) => Promise; +/** Runs a script in the user's login shell and returns its stdout, or null. */ +export type LoginShellRunner = (shell: string, script: string) => Promise; + +/** + * Probes this computer only by running `--version` style commands; never reads conversation content. + * Versions are read through the user's login shell first, because an app started from the Dock does not + * see the PATH set up in shell start-up files (nvm, conda, pyenv); tools it cannot see are run directly. + */ +export async function probeMachine(options: { + runVersion?: VersionRunner; + runLoginShell?: LoginShellRunner; + env?: NodeJS.ProcessEnv; + os?: NodeJS.Platform; +} = {}): Promise { + const runVersion = options.runVersion ?? defaultVersionRunner; + const runLoginShell = options.runLoginShell ?? defaultLoginShellRunner; + const env = options.env ?? process.env; + const os = options.os ?? platform(); + const fromShell = env.SHELL && os !== "win32" + ? parseShellProbe(await runLoginShell(env.SHELL, shellProbeScript()).catch(() => null)) + : new Map(); + const entries = await Promise.all(ALL_TOOLS.map(async (tool) => { + const viaShell = fromShell.get(tool); + if (viaShell) return [tool, viaShell] as const; + const output = await runVersion(tool, TOOL_ARGS[tool]).catch(() => null); + return [tool, output ? versionNumber(output) : null] as const; + })); + // os.release() is the Darwin kernel version on macOS; the product version is what users recognize. + const productVersion = os === "darwin" + ? await runVersion("sw_vers", ["-productVersion"]).catch(() => null) + : null; + return { + os, + osVersion: (productVersion ? versionNumber(productVersion) : null) ?? release(), + arch: arch(), + shell: env.SHELL ? path.basename(env.SHELL) : null, + tools: Object.fromEntries(entries) as Record + }; +} + +/** Fixed-format machine profile text; the same probe always renders the same text. */ +export function renderMachineProfile(probe: MachineProbe): string { + const lines = [ + `- 系统：${osLabel(probe.os)} ${probe.osVersion}（${probe.arch}）`, + ...(probe.shell ? [`- shell：${probe.shell}`] : []) + ]; + const installed = ALL_TOOLS + .filter((tool) => probe.tools[tool]) + .map((tool) => `${tool} ${probe.tools[tool]}`); + if (installed.length) lines.push(`- 已安装：${installed.join("，")}`); + const missing = MACHINE_TOOLS.filter((tool) => !probe.tools[tool]); + if (missing.length) lines.push(`- 未检测到：${missing.join("，")}`); + return lines.join("\n"); +} + +export function machineProfileFingerprint(probe: MachineProbe): string { + return sha256Hex(renderMachineProfile(probe)); +} + +function osLabel(os: string): string { + if (os === "darwin") return "macOS"; + if (os === "win32") return "Windows"; + if (os === "linux") return "Linux"; + return os; +} + +/** Keeps the version number only, e.g. `git version 2.39.5 (Apple Git-154)` -> `2.39.5`; output without one is not a version. */ +function versionNumber(output: string): string | null { + const version = /v?(\d+(?:\.\d+)+)/.exec(output); + return version ? version[1]! : null; +} + +/** Prints a marker line per tool, then its version; tools the shell cannot find print nothing. */ +function shellProbeScript(): string { + return ALL_TOOLS.map((tool) => + `printf '\\n${SECTION_MARKER}${tool}\\n'; command -v ${tool} >/dev/null 2>&1 && ${tool} ${TOOL_ARGS[tool].join(" ")} 2>&1 | head -n 3` + ).join("; "); +} + +function parseShellProbe(output: string | null): Map { + const found = new Map(); + if (!output) return found; + // Anything a start-up file prints before the first marker is ignored. + for (const section of output.split(SECTION_MARKER).slice(1)) { + const newline = section.indexOf("\n"); + const tool = (newline < 0 ? section : section.slice(0, newline)).trim() as MachineTool; + const version = newline < 0 ? null : versionNumber(section.slice(newline + 1)); + if (ALL_TOOLS.includes(tool) && version) found.set(tool, version); + } + return found; +} + +async function defaultLoginShellRunner(shell: string, script: string): Promise { + try { + const result = await execFileAsync(shell, ["-ilc", script], { + cwd: tmpdir(), + timeout: LOGIN_SHELL_TIMEOUT_MS, + maxBuffer: 64 * 1024, + windowsHide: true + }); + return result.stdout || null; + } catch { + return null; + } +} + +async function defaultVersionRunner(executable: string, args: string[]): Promise { + try { + const result = await execFileAsync(executable, args, { + cwd: tmpdir(), + timeout: PROBE_TIMEOUT_MS, + maxBuffer: 4096, + shell: false, + windowsHide: true + }); + return `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim() || null; + } catch { + return null; + } +} diff --git a/Memory/src/service/project-environment/profile-pipeline.ts b/Memory/src/service/project-environment/profile-pipeline.ts index 17941c4ea..70e490a0d 100644 --- a/Memory/src/service/project-environment/profile-pipeline.ts +++ b/Memory/src/service/project-environment/profile-pipeline.ts @@ -80,31 +80,9 @@ export class ProjectEnvironmentProfilePipeline { payload.userId, payload.projectId ).projectEnvironmentProfile; - const evidenceSupportsProfile = projectEnvironmentEvidenceSupportsProfile(derived); - const basePrompt = derived.projectKind === "code" ? CODE_PROFILE_PROMPT : FOLDER_PROFILE_PROMPT; - const language = currentProfile?.trim() - ? undefined - : steeredPromptLanguage(this.deps.language, [derived.compactFileTree]); - const systemPrompt = language ? `${basePrompt}\n\n${languageSteeringLine(language)}` : basePrompt; let output: ReturnType; try { - output = await completeStrictJson({ - llm: this.deps.llm, - operation: derived.projectKind === "code" - ? "project_environment_code_profile" - : "project_environment_folder_profile", - systemPrompt, - dynamicInput: profileDynamicInput(derived, currentProfile), - expectedSchema: { - op: "noop | create | update", - profile: "complete final profile; empty only for noop or update-clear" - }, - validate: (value) => validateProjectEnvironmentProfileOutput( - value, - currentProfile, - evidenceSupportsProfile - ) - }); + output = await summarizeProjectProfile(this.deps, derived, currentProfile); } catch (error) { const latest = this.deps.repos.projectEnvironments.getState(payload.userId, payload.projectId); const latestProfile = this.deps.repos.l3WorldModels.fields( @@ -131,6 +109,37 @@ export class ProjectEnvironmentProfilePipeline { } } +/** One model call that turns scan evidence into the project environment profile (shared by old and new L3). */ +export async function summarizeProjectProfile( + deps: { llm: LlmClient; language?: MemoryLanguage }, + derived: ProjectEnvironmentDerivedEvidence, + currentProfile: string | null +): Promise<{ op: "noop" | "create" | "update"; profile: string }> { + const evidenceSupportsProfile = projectEnvironmentEvidenceSupportsProfile(derived); + const basePrompt = derived.projectKind === "code" ? CODE_PROFILE_PROMPT : FOLDER_PROFILE_PROMPT; + const language = currentProfile?.trim() + ? undefined + : steeredPromptLanguage(deps.language, [derived.compactFileTree]); + const systemPrompt = language ? `${basePrompt}\n\n${languageSteeringLine(language)}` : basePrompt; + return completeStrictJson({ + llm: deps.llm, + operation: derived.projectKind === "code" + ? "project_environment_code_profile" + : "project_environment_folder_profile", + systemPrompt, + dynamicInput: profileDynamicInput(derived, currentProfile), + expectedSchema: { + op: "noop | create | update", + profile: "complete final profile; empty only for noop or update-clear" + }, + validate: (value) => validateProjectEnvironmentProfileOutput( + value, + currentProfile, + evidenceSupportsProfile + ) + }); +} + export function projectEnvironmentProfileJobPayload(value: Record): { userId: string; projectId: string; diff --git a/Memory/src/service/project-environment/project-environment-service.ts b/Memory/src/service/project-environment/project-environment-service.ts index b4ff673a5..bd23dac92 100644 --- a/Memory/src/service/project-environment/project-environment-service.ts +++ b/Memory/src/service/project-environment/project-environment-service.ts @@ -1,3 +1,4 @@ +import { pathToFileURL } from "node:url"; import type { MemoryLanguage } from "../../config/index.js"; import type { LlmClient } from "../../model/types.js"; import type { @@ -5,9 +6,15 @@ import type { Repositories, SessionRecord } from "../../storage/repositories.js"; +import { newId } from "../../utils/id.js"; +import { nowIso } from "../../utils/time.js"; import { scanLocalProject } from "./local-scanner.js"; import { parseDeterministicProjectFacts } from "./manifest-parsers.js"; -import { ProjectEnvironmentProfilePipeline, projectEnvironmentProfileJobPayload } from "./profile-pipeline.js"; +import { + ProjectEnvironmentProfilePipeline, + projectEnvironmentProfileJobPayload, + summarizeProjectProfile +} from "./profile-pipeline.js"; import { classifyProjectInventory } from "./project-classifier.js"; import { buildCompactFileTree, projectFingerprint } from "./scan-policy.js"; import type { ProjectEnvironmentDerivedEvidence } from "./types.js"; @@ -18,6 +25,10 @@ interface ProjectEnvironmentServiceDeps { readonly language?: MemoryLanguage; } +const L3_FIELD_MODE = "l3_field"; +/** A project is scanned for its new-L3 profile at most this often; each session open asks for one. */ +const L3_UNIT_PROFILE_INTERVAL_MS = 10 * 60_000; + export class ProjectEnvironmentService { private readonly profilePipeline: ProjectEnvironmentProfilePipeline; @@ -61,32 +72,58 @@ export class ProjectEnvironmentService { }); } + /** + * New L3: queue a profile scan of one checkout of a project unit. At most one + * such job is pending per unit, and a unit scanned in the last ten minutes is + * not scanned again; such a request is a no-op. + */ + requestL3UnitProfile(input: { userId: string; unitKey: string; checkoutRoot: string; at?: string }): boolean { + const at = input.at ?? nowIso(); + const recent = this.deps.repos.db.prepare( + `SELECT 1 FROM evolution_jobs + WHERE job_type = 'project_environment_profile' + AND json_extract(payload_json, '$.mode') = ? + AND json_extract(payload_json, '$.unitKey') = ? + AND (status IN ('queued', 'leased', 'failed') OR (status = 'succeeded' AND updated_at >= ?)) + LIMIT 1` + ).get(L3_FIELD_MODE, input.unitKey, new Date(Date.parse(at) - L3_UNIT_PROFILE_INTERVAL_MS).toISOString()); + if (recent) return false; + const scanId = newId("l3scan"); + // The immutable dedupe index never frees a key, so each request carries its own scan id. + const dedupeKey = ["project_environment_profile", L3_FIELD_MODE, input.unitKey, scanId].join(":"); + this.deps.repos.l3WorldModels.insertImmutableJob({ + id: newId("job"), + jobType: "project_environment_profile", + status: "queued", + dedupeKey, + userId: input.userId, + payload: { + mode: L3_FIELD_MODE, + userId: input.userId, + unitKey: input.unitKey, + checkoutRoot: input.checkoutRoot, + scanId + }, + attempts: 0, + maxAttempts: 3, + createdAt: at, + updatedAt: at + }); + return true; + } + async processProfileJob(job: EvolutionJobRecord): Promise { + if (job.payload.mode === L3_FIELD_MODE) { + await this.processL3UnitProfileJob(job); + return; + } const payload = projectEnvironmentProfileJobPayload(job.payload); if (job.userId !== payload.userId) throw new Error("project_environment_job_owner_mismatch"); if (!this.deps.repos.projectEnvironments.beginScan(payload.userId, payload.projectId, payload.scanId)) return; try { const scope = this.deps.repos.l3WorldModels.getScope(payload.userId, payload.projectId); if (!scope?.workspaceUri) throw new Error("project_environment_workspace_uri_missing"); - const scan = await scanLocalProject(scope.workspaceUri); - const classification = classifyProjectInventory(scan.entries); - const facts = parseDeterministicProjectFacts({ - entries: scan.entries, - textFiles: scan.textFiles, - runtimeProbes: scan.runtimeProbes - }); - const derived: ProjectEnvironmentDerivedEvidence = { - projectKind: classification.kind, - compactFileTree: buildCompactFileTree(scan.entries), - omittedCount: scan.omittedCount, - deterministicFacts: facts, - fingerprint: projectFingerprint({ - kind: classification.kind, - entries: scan.entries, - omittedCount: scan.omittedCount, - deterministicFacts: facts - }) - }; + const derived = await deriveScanEvidence(scope.workspaceUri); const state = this.deps.repos.projectEnvironments.getState(payload.userId, payload.projectId); if (!state || state.currentScanId !== payload.scanId) return; const currentProfile = this.deps.repos.l3WorldModels.fields( @@ -123,4 +160,58 @@ export class ProjectEnvironmentService { throw error; } } + + /** + * New L3 project profile. Only what the project itself declares goes in: + * runtime versions probed on this computer belong to the machine profile. + * The result is written only if the unit still exists and nobody changed + * its profile while the model was running. + */ + private async processL3UnitProfileJob(job: EvolutionJobRecord): Promise { + const userId = typeof job.payload.userId === "string" ? job.payload.userId : ""; + const unitKey = typeof job.payload.unitKey === "string" ? job.payload.unitKey : ""; + const checkoutRoot = typeof job.payload.checkoutRoot === "string" ? job.payload.checkoutRoot : ""; + if (!userId || !unitKey || !checkoutRoot || job.userId !== userId) { + throw new Error("invalid l3 field profile job payload"); + } + const unit = this.deps.repos.l3Field.getUnit(unitKey); + if (!unit || unit.deletedAt || unit.userId !== userId || unit.kind !== "project") return; + const scanned = await deriveScanEvidence(pathToFileURL(checkoutRoot).href); + const derived: ProjectEnvironmentDerivedEvidence = { + ...scanned, + deterministicFacts: { ...scanned.deterministicFacts, runtimeProbes: [] } + }; + if (unit.profileFingerprint === derived.fingerprint && unit.profileText) return; + const currentProfile = unit.profileText ?? null; + const output = await summarizeProjectProfile(this.deps, derived, currentProfile); + const latest = this.deps.repos.l3Field.getUnit(unitKey); + if (!latest || latest.deletedAt || (latest.profileText ?? null) !== currentProfile) return; + const nextProfile = output.op === "noop" ? currentProfile : (output.profile || null); + this.deps.repos.transaction(() => { + this.deps.repos.l3Field.setProfile(unitKey, { text: nextProfile, fingerprint: derived.fingerprint }); + if (nextProfile !== currentProfile) this.deps.repos.l3Field.refreshSnapshot(unitKey); + }); + } +} + +async function deriveScanEvidence(workspaceUri: string): Promise { + const scan = await scanLocalProject(workspaceUri); + const classification = classifyProjectInventory(scan.entries); + const facts = parseDeterministicProjectFacts({ + entries: scan.entries, + textFiles: scan.textFiles, + runtimeProbes: scan.runtimeProbes + }); + return { + projectKind: classification.kind, + compactFileTree: buildCompactFileTree(scan.entries), + omittedCount: scan.omittedCount, + deterministicFacts: facts, + fingerprint: projectFingerprint({ + kind: classification.kind, + entries: scan.entries, + omittedCount: scan.omittedCount, + deterministicFacts: facts + }) + }; } diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts index 7b8b8d74e..518be2dc5 100644 --- a/Memory/src/storage/l3-field-repository.ts +++ b/Memory/src/storage/l3-field-repository.ts @@ -269,6 +269,13 @@ export class L3FieldRepository { })(); } + /** Profile text comes only from code probes (machine) or scan summaries (project). */ + setProfile(unitKey: string, profile: { text: string | null; fingerprint: string }, at = nowIso()): void { + this.db.prepare( + `UPDATE l3_units SET profile_text = ?, profile_fingerprint = ?, updated_at = ? WHERE unit_key = ?` + ).run(profile.text, profile.fingerprint, at, unitKey); + } + /** Throwaway task roots Memmy itself creates (they are never projects); stored with the project list. */ getStandaloneRoots(userId: string): string[] { const row = this.db.prepare(`SELECT value_json FROM runtime_kv WHERE key = ?`) diff --git a/Memory/tests/service/l3-field/machine-profile.test.ts b/Memory/tests/service/l3-field/machine-profile.test.ts new file mode 100644 index 000000000..02e5276fd --- /dev/null +++ b/Memory/tests/service/l3-field/machine-profile.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { + machineProfileFingerprint, + probeMachine, + renderMachineProfile, + type LoginShellRunner, + type VersionRunner +} from "../../../src/service/l3-field/machine-profile.js"; + +const outputs: Record = { + node: "v25.1.0", + python3: "Python 3.11.5", + git: "git version 2.39.5 (Apple Git-154)", + npm: "11.6.2" +}; +const runner: VersionRunner = async (executable) => outputs[executable] ?? null; +const noShell: LoginShellRunner = async () => null; + +describe("machine profile", () => { + it("keeps only version numbers and marks tools that were not found", async () => { + const probe = await probeMachine({ runVersion: runner, runLoginShell: noShell, env: { SHELL: "/bin/zsh" } }); + expect(probe.shell).toBe("zsh"); + expect(probe.tools).toMatchObject({ node: "25.1.0", python3: "3.11.5", git: "2.39.5", npm: "11.6.2", brew: null, pnpm: null }); + const text = renderMachineProfile(probe); + expect(text).toContain("shell：zsh"); + expect(text).toContain("node 25.1.0，python3 3.11.5，git 2.39.5，npm 11.6.2"); + expect(text).toContain("未检测到：brew"); + }); + + it("reports the macOS product version rather than the Darwin kernel version", async () => { + const withSwVers: VersionRunner = async (executable) => (executable === "sw_vers" ? "26.1\n" : outputs[executable] ?? null); + const probe = await probeMachine({ runVersion: withSwVers, runLoginShell: noShell, env: {}, os: "darwin" }); + expect(probe.osVersion).toBe("26.1"); + expect(renderMachineProfile(probe)).toContain("系统：macOS 26.1"); + }); + + it("renders the same probe to the same text and fingerprint", async () => { + const a = await probeMachine({ runVersion: runner, runLoginShell: noShell, env: { SHELL: "/bin/zsh" } }); + const b = await probeMachine({ runVersion: runner, runLoginShell: noShell, env: { SHELL: "/bin/zsh" } }); + expect(renderMachineProfile(a)).toBe(renderMachineProfile(b)); + expect(machineProfileFingerprint(a)).toBe(machineProfileFingerprint(b)); + }); + + it("treats a failing or timed-out probe as not installed", async () => { + const failing: VersionRunner = async () => { + throw new Error("timeout"); + }; + const probe = await probeMachine({ runVersion: failing, runLoginShell: noShell, env: {} }); + expect(Object.values(probe.tools).every((value) => value === null)).toBe(true); + expect(probe.shell).toBeNull(); + }); + + it("prefers versions seen by the login shell and falls back to running tools directly", async () => { + const shellOutput = [ + "Welcome banner 1.2.3 from .zshrc", + "__memmy_probe__node", "v22.11.0", + "__memmy_probe__python3", "Python 3.12.4", + "__memmy_probe__conda", "conda 24.9.2", + "__memmy_probe__java", "The operation couldn't be completed. Unable to locate a Java Runtime.", + "__memmy_probe__go", "" + ].join("\n"); + let script = ""; + const shell: LoginShellRunner = async (_shell, body) => { + script = body; + return shellOutput; + }; + const probe = await probeMachine({ runVersion: runner, runLoginShell: shell, env: { SHELL: "/bin/zsh" }, os: "linux" }); + expect(script).toContain("command -v rustc"); + expect(probe.tools).toMatchObject({ node: "22.11.0", python3: "3.12.4", conda: "24.9.2", git: "2.39.5", java: null, go: null }); + const text = renderMachineProfile(probe); + expect(text).toContain("node 22.11.0，python3 3.12.4，git 2.39.5，npm 11.6.2，conda 24.9.2"); + expect(text).toContain("未检测到：brew"); + expect(text).not.toContain("java"); + }); + + it("does not use a login shell on Windows", async () => { + let called = false; + const shell: LoginShellRunner = async () => { + called = true; + return null; + }; + await probeMachine({ runVersion: runner, runLoginShell: shell, env: { SHELL: "bash" }, os: "win32" }); + expect(called).toBe(false); + }); +}); diff --git a/Memory/tests/service/l3-field/unit-profile.test.ts b/Memory/tests/service/l3-field/unit-profile.test.ts new file mode 100644 index 000000000..5290a65c0 --- /dev/null +++ b/Memory/tests/service/l3-field/unit-profile.test.ts @@ -0,0 +1,113 @@ +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/index.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { ProjectEnvironmentService } from "../../../src/service/project-environment/project-environment-service.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; + +const cleanups: Array<() => void> = []; +afterEach(() => { + for (const cleanup of cleanups.splice(0)) cleanup(); +}); + +function setup(reply = '{"op":"create","profile":"Node 项目，测试用 vitest"}') { + const db = new MemoryDb({ path: ":memory:" }); + const root = realpathSync(mkdtempSync(join(tmpdir(), "l3-unit-profile-"))); + cleanups.push(() => { + db.close(); + rmSync(root, { recursive: true, force: true }); + }); + const project = join(root, "repo"); + mkdirSync(project); + writeFileSync(join(project, "package.json"), JSON.stringify({ name: "demo", engines: { node: ">=20" }, scripts: { test: "vitest run" } })); + writeFileSync(join(project, "index.ts"), "export const a = 1;\n"); + const repos = new Repositories(db.db); + const complete = vi.fn().mockResolvedValue(reply); + const service = new ProjectEnvironmentService({ repos, llm: testLlm(complete) }); + const unit = repos.l3Field.ensureProjectUnit({ + userId: "u1", + machineId: "m".repeat(64), + project: { identity: "dir:repo", checkoutRoot: project, source: "folder", displayName: "repo" } + }); + const pendingJob = (): EvolutionJobRecord => { + const row = db.db.prepare( + `SELECT id, user_id, payload_json FROM evolution_jobs + WHERE job_type = 'project_environment_profile' AND status = 'queued' ORDER BY created_at DESC LIMIT 1` + ).get() as { id: string; user_id: string; payload_json: string }; + return { id: row.id, jobType: "project_environment_profile", userId: row.user_id, payload: JSON.parse(row.payload_json) } as EvolutionJobRecord; + }; + return { db, repos, service, complete, unit, project, pendingJob }; +} + +describe("L3 field cognition project profile", () => { + it("queues at most one pending profile job per unit, and scans a unit at most once in ten minutes", () => { + const { service, unit, project, db } = setup(); + const request = () => service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + expect(request()).toBe(true); + expect(request()).toBe(false); + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 9 * 60_000).toISOString()); + expect(request()).toBe(false); + db.db.prepare(`UPDATE evolution_jobs SET updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 11 * 60_000).toISOString()); + expect(request()).toBe(true); + }); + + it("writes the summary into the unit, refreshes its snapshot, and leaves machine runtime versions out", async () => { + const { service, repos, complete, unit, project, pendingJob } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + const updated = repos.l3Field.getUnit(unit.unitKey)!; + expect(updated.profileText).toBe("Node 项目，测试用 vitest"); + expect(updated.profileFingerprint).toBeTruthy(); + expect(repos.memories.get(updated.snapshotMemoryId!)!.memoryValue).toContain("Node 项目，测试用 vitest"); + const sent = JSON.stringify(complete.mock.calls); + expect(sent).toContain("runtime_probes\\\":[]"); + expect(sent).toContain(">=20"); + expect(repos.l3WorldModels.getScope("u1", "dir:repo")).toBeUndefined(); + }); + + it("skips the model when the scan is unchanged", async () => { + const { service, complete, unit, project, pendingJob, db } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + // Past the ten-minute gap, so the next session open scans again. + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 11 * 60_000).toISOString()); + expect(service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project })).toBe(true); + await service.processProfileJob(pendingJob()); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("drops the result when the unit was deleted", async () => { + const { service, repos, complete, unit, project, pendingJob } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + repos.l3Field.markUnitDeleted(unit.unitKey); + await service.processProfileJob(pendingJob()); + expect(complete).not.toHaveBeenCalled(); + expect(repos.l3Field.getUnit(unit.unitKey)!.profileText).toBeUndefined(); + }); +}); + +function testLlm(complete: LlmClient["complete"]): LlmClient { + return { + config: { + provider: "host", + endpoint: "http://localhost/unused", + model: "l3-unit-profile-test", + apiKey: "", + temperature: 0, + maxTokens: 4096, + timeoutMs: 30_000, + maxRetries: 0, + malformedRetries: 0, + enableThinking: false + }, + isConfigured: () => true, + complete, + completeJson: vi.fn(), + status: () => ({ provider: "host", model: "l3-unit-profile-test", configured: true, remote: false }) + }; +} From 9a86826585f8e3f019eaa71032b878b90b6e0b71 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:18:39 +0200 Subject: [PATCH 6/9] feat(memory): inject and display l3 field context The new-L3 project profile fingerprint leaves out the runtime versions probed on this machine (they belong to the machine profile), and an unchanged fingerprint skips the model even when the last summary found nothing worth a profile. The scan for it does not run the declared runtimes at all. A unit whose snapshot the user archived is not injected, and rewriting the snapshot keeps it archived. Deleting a snapshot and resetting its unit happen in one transaction. Co-Authored-By: Claude Opus 5.5 --- Memory/src/algorithm/plugin-algorithms.ts | 27 ++++ .../embedding/embedding-job-processor.ts | 2 + .../src/service/l3-field/l3-field-context.ts | 102 ++++++++++++ Memory/src/service/l3-field/session-units.ts | 71 +++++++++ .../src/service/l3-field/snapshot-memory.ts | 37 +++++ Memory/src/service/memory-service.ts | 42 ++++- .../project-environment/local-scanner.ts | 8 +- .../project-environment-service.ts | 28 ++-- Memory/src/service/worker/worker-runner.ts | 5 + Memory/src/storage/l3-field-repository.ts | 39 ++++- Memory/src/storage/repositories.ts | 3 +- .../service/l3-field/l3-field-context.test.ts | 145 ++++++++++++++++++ .../service/l3-field/unit-profile.test.ts | 36 ++++- .../project-environment/local-scanner.test.ts | 8 + 14 files changed, 526 insertions(+), 27 deletions(-) create mode 100644 Memory/src/service/l3-field/l3-field-context.ts create mode 100644 Memory/src/service/l3-field/session-units.ts create mode 100644 Memory/src/service/l3-field/snapshot-memory.ts create mode 100644 Memory/tests/service/l3-field/l3-field-context.test.ts diff --git a/Memory/src/algorithm/plugin-algorithms.ts b/Memory/src/algorithm/plugin-algorithms.ts index 5881dff18..d35d9982d 100644 --- a/Memory/src/algorithm/plugin-algorithms.ts +++ b/Memory/src/algorithm/plugin-algorithms.ts @@ -13,6 +13,7 @@ import { stableHash } from "../utils/id.js"; import { matchToolResultIndices } from "../utils/tool-call-pairing.js"; import { formatZonedTime } from "../utils/time.js"; import { isObservationMemory, isObservationMemoryProcessed } from "../service/observation-memory-state.js"; +import { isL3FieldSnapshotMemory, l3FieldSnapshotLegacyFields } from "../service/l3-field/snapshot-memory.js"; import { renderL3WorldModelFields, type L3WorldModelFields @@ -3441,6 +3442,31 @@ export function skillMetaFromMemory(memory: MemoryRow): SkillMemoryMeta | null { export function worldModelMetaFromMemory(memory: MemoryRow): WorldModelMemoryMeta | null { if (memory.memoryLayer !== "L3") return null; + const snapshotFields = l3FieldSnapshotLegacyFields(memory); + if (snapshotFields) { + // New L3 snapshots are shown on the world model page in the schema-2 shape; they are injected + // at session start only and never enter retrieval. + const internal = memory.properties.internal_info as Record; + const field = internal.l3_field as Record; + const project = field.kind === "project"; + return { + id: memory.id, + memory, + title: stringField(internal, "title") ?? (project ? "项目场域认知" : "本机场域认知"), + domainKey: `l3_field:${field.unit_key as string}`, + domainTags: [project ? "project" : "machine"], + policyIds: [], + confidence: 0, + cohesion: 1, + admission: "strict", + structure: { environment: [], inference: [], constraints: [] }, + summary: renderL3WorldModelFields(snapshotFields), + body: memory.memoryValue, + vec: memoryVector(memory, "vec"), + schemaVersion: 2, + fields: snapshotFields + }; + } const wm = getInternal>(memory, "world_model"); if (!wm) return null; const v2Fields = l3WorldModelV2Fields(memory, wm); @@ -5231,6 +5257,7 @@ function candidateFromMemory( } ): RankedMemoryCandidate | null { if (memory.status === "deleted" || memory.status === "archived") return null; + if (isL3FieldSnapshotMemory(memory)) return null; const trace = memory.memoryLayer === "L1" ? traceMetaFromMemory(memory) : null; const policy = memory.memoryLayer === "L2" ? policyMetaFromMemory(memory) : null; const skill = memory.memoryLayer === "Skill" ? skillMetaFromMemory(memory) : null; diff --git a/Memory/src/service/embedding/embedding-job-processor.ts b/Memory/src/service/embedding/embedding-job-processor.ts index ae965d5bd..2b3d2afb1 100644 --- a/Memory/src/service/embedding/embedding-job-processor.ts +++ b/Memory/src/service/embedding/embedding-job-processor.ts @@ -48,6 +48,7 @@ import { updateMemoryVectorField } from "./embedding-pipeline.js"; import { canonicalWorkMemoryText } from "../work-memory/work-memory-pipeline.js"; +import { isL3FieldSnapshotMemory } from "../l3-field/snapshot-memory.js"; import { isObservationMemory, isObservationMemoryProcessed, observationJobPayload } from "../observation-memory-state.js"; type TraceMeta = NonNullable>; @@ -209,6 +210,7 @@ export class EmbeddingJobProcessor { throw new Error(`embedding target not found: ${job.targetMemoryId ?? "unknown"}`); } if (!processingJobMatchesMemory(job, memory)) return null; + if (isL3FieldSnapshotMemory(memory)) return null; if (isObservationMemory(memory)) { if (!isObservationMemoryProcessed(memory)) return null; diff --git a/Memory/src/service/l3-field/l3-field-context.ts b/Memory/src/service/l3-field/l3-field-context.ts new file mode 100644 index 000000000..dbacd9d1b --- /dev/null +++ b/Memory/src/service/l3-field/l3-field-context.ts @@ -0,0 +1,102 @@ +import type { SessionL3WorldModelContextResponse } from "../../contracts/index.js"; +import type { Repositories, SessionRecord } from "../../storage/repositories.js"; +import { nowIso } from "../../utils/time.js"; +import { resolveSessionUnits, type L3FieldEnvironment } from "./session-units.js"; +import type { L3ItemRecord, L3UnitRecord, ProjectIdentitySource } from "./types.js"; + +const SOURCE_LABEL: Record = { + user_selected: "用户选定", + git: "git 仓库", + folder: "文件夹" +}; + +/** + * Session-start context for the new L3: this machine (profile and facts) and, + * when the session folder is a project, that project (profile, facts and + * constraints). The response keeps the schema-2 shape hooks already consume. + */ +export function loadL3FieldContext( + repos: Repositories, + env: L3FieldEnvironment, + session: SessionRecord +): SessionL3WorldModelContextResponse { + const units = resolveSessionUnits(repos, env, session); + // A unit whose snapshot the user archived is not injected, like an archived old-L3 world model. + const archived = (unit: L3UnitRecord) => + Boolean(unit.snapshotMemoryId && repos.memories.get(unit.snapshotMemoryId)?.status === "archived"); + const machine = archived(units.machine) ? null : units.machine; + const projectUnit = units.project && !archived(units.project.unit) ? units.project.unit : null; + const machineItems = machine ? repos.l3Field.activeItems(machine.unitKey) : []; + const projectItems = projectUnit ? repos.l3Field.activeItems(projectUnit.unitKey) : []; + + const machineFacts = bulletList(machineItems, "env_fact"); + const projectFacts = bulletList(projectItems, "env_fact"); + const constraints = bulletList(projectItems, "project_constraint"); + const machineProfile = trimmed(machine?.profileText); + const projectProfile = trimmed(projectUnit?.profileText); + const hasContent = Boolean(machineProfile || machineFacts || projectProfile || projectFacts || constraints); + // Clients treat projectId as the session's own project id, so it keeps that meaning. + const projectId = session.projectId ?? null; + if (!hasContent) return emptyResponse(projectId); + + const sections = [ + projectUnit + ? `当前项目：${projectUnit.displayName ?? projectUnit.projectIdentity}（${SOURCE_LABEL[projectUnit.identitySource ?? "folder"]}）` + : "当前项目：无", + section("本机环境画像", machineProfile), + section("本机环境事实", machineFacts), + section("项目环境画像", projectProfile), + section("项目环境事实", projectFacts), + section("项目约束", constraints) + ].filter(Boolean); + + // The version hooks cache on belongs to the project snapshot when there is a project; machine + // changes bump every project snapshot on the same machine, so this version still moves. + const snapshotUnit: L3UnitRecord = projectUnit ?? units.machine; + const snapshot = snapshotUnit.snapshotMemoryId + ? repos.memories.get(snapshotUnit.snapshotMemoryId) ?? repos.l3Field.refreshSnapshot(snapshotUnit.unitKey) + : repos.l3Field.refreshSnapshot(snapshotUnit.unitKey); + + return { + schemaVersion: 2, + projectId, + memoryId: snapshot.id, + memoryVersion: snapshot.version, + renderedContext: sections.join("\n\n"), + sourceMemoryIds: [...new Set([...machineItems, ...projectItems].flatMap((item) => item.evidence.l1MemoryIds))], + generalRulesAndSafetyConstraints: null, + projectEnvironmentProfile: [machineProfile, projectProfile].filter(Boolean).join("\n\n") || null, + projectContract: constraints, + domainKnowledge: [machineFacts, projectFacts].filter(Boolean).join("\n") || null, + serverTime: nowIso() + }; +} + +function bulletList(items: readonly L3ItemRecord[], kind: L3ItemRecord["kind"]): string | null { + const lines = items.filter((item) => item.kind === kind).map((item) => `- ${item.content}`); + return lines.length ? lines.join("\n") : null; +} + +function section(title: string, body: string | null): string | null { + return body ? `## ${title}\n${body}` : null; +} + +function trimmed(value: string | null | undefined): string | null { + return value?.trim() ? value.trim() : null; +} + +function emptyResponse(projectId: string | null): SessionL3WorldModelContextResponse { + return { + schemaVersion: 2, + projectId, + memoryId: null, + memoryVersion: null, + renderedContext: "", + sourceMemoryIds: [], + generalRulesAndSafetyConstraints: null, + projectEnvironmentProfile: null, + projectContract: null, + domainKnowledge: null, + serverTime: nowIso() + }; +} diff --git a/Memory/src/service/l3-field/session-units.ts b/Memory/src/service/l3-field/session-units.ts new file mode 100644 index 000000000..ea15aeff1 --- /dev/null +++ b/Memory/src/service/l3-field/session-units.ts @@ -0,0 +1,71 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { isLocalWorkspaceUri } from "../../contracts/index.js"; +import type { Repositories, SessionRecord } from "../../storage/repositories.js"; +import { resolveProject, type ResolverContext } from "./project-resolver.js"; +import type { L3UnitRecord, ResolvedProject } from "./types.js"; + +/** What the new L3 needs to know about the computer the Memory service runs on. */ +export interface L3FieldEnvironment { + /** Stable id of this installation (the workspace host id); machine units are keyed by it. */ + machineId: string; + homeDir: string; + /** Folders never treated as projects; defaults to the system temp folders. Tests override it. */ + tempRoots?: readonly string[]; +} + +export interface SessionUnits { + machine: L3UnitRecord; + project: { resolved: ResolvedProject; unit: L3UnitRecord } | null; + sessionCwd: string | null; + workspaceIsLocal: boolean; +} + +/** The session's working directory: the v2 workspace URI when present, else the legacy workspace path. */ +export function sessionWorkingDirectory(session: SessionRecord): { cwd: string | null; isLocal: boolean } { + const uri = typeof session.meta.workspace_uri === "string" ? session.meta.workspace_uri : null; + if (uri) { + if (!isLocalWorkspaceUri(uri)) return { cwd: null, isLocal: false }; + try { + return { cwd: fileURLToPath(uri), isLocal: true }; + } catch { + return { cwd: null, isLocal: false }; + } + } + const legacy = session.workspacePath; + return legacy && path.isAbsolute(legacy) ? { cwd: legacy, isLocal: true } : { cwd: null, isLocal: true }; +} + +export function resolverContextFor( + repos: Repositories, + env: L3FieldEnvironment, + userId: string, + workspaceIsLocal: boolean +): ResolverContext { + const standalone = repos.l3Field.getStandaloneRoots(userId); + return { + homeDir: env.homeDir, + userProjects: repos.l3Field.listUserProjects(userId), + standaloneRoots: standalone.length ? standalone : [path.join(env.homeDir, "Memmy")], + tempRoots: env.tempRoots, + workspaceIsLocal + }; +} + +/** Machine unit plus the project unit of the session folder (if it is a project); creates units on first use. */ +export function resolveSessionUnits( + repos: Repositories, + env: L3FieldEnvironment, + session: SessionRecord +): SessionUnits { + const { cwd, isLocal } = sessionWorkingDirectory(session); + const machine = repos.l3Field.ensureMachineUnit({ userId: session.userId, machineId: env.machineId }); + if (!cwd) return { machine, project: null, sessionCwd: null, workspaceIsLocal: isLocal }; + const resolved = resolveProject( + { path: cwd, role: "session_dir", sessionCwd: cwd, sessionProjectRoot: null }, + resolverContextFor(repos, env, session.userId, isLocal) + ); + if (!resolved) return { machine, project: null, sessionCwd: cwd, workspaceIsLocal: isLocal }; + const unit = repos.l3Field.ensureProjectUnit({ userId: session.userId, machineId: env.machineId, project: resolved }); + return { machine, project: { resolved, unit }, sessionCwd: cwd, workspaceIsLocal: isLocal }; +} diff --git a/Memory/src/service/l3-field/snapshot-memory.ts b/Memory/src/service/l3-field/snapshot-memory.ts new file mode 100644 index 000000000..d580b1e69 --- /dev/null +++ b/Memory/src/service/l3-field/snapshot-memory.ts @@ -0,0 +1,37 @@ +import type { L3WorldModelFields } from "../../contracts/index.js"; +import type { MemoryRow } from "../../types.js"; + +type SnapshotMemory = Pick; + +function l3FieldInfo(memory: SnapshotMemory): Record | null { + if (memory.memoryLayer !== "L3") return null; + const field = memory.properties.internal_info.l3_field; + return field && typeof field === "object" && !Array.isArray(field) ? field as Record : null; +} + +/** Unit key of an L3 field cognition snapshot memory, or null for any other memory. */ +export function l3FieldSnapshotUnitKey(memory: SnapshotMemory): string | null { + const unitKey = l3FieldInfo(memory)?.unit_key; + return typeof unitKey === "string" && unitKey ? unitKey : null; +} + +export function isL3FieldSnapshotMemory(memory: SnapshotMemory): boolean { + return l3FieldSnapshotUnitKey(memory) !== null; +} + +/** The schema-2 fields a snapshot carries for pages that only understand the old L3 shape. */ +export function l3FieldSnapshotLegacyFields(memory: SnapshotMemory): L3WorldModelFields | null { + if (!l3FieldSnapshotUnitKey(memory)) return null; + const legacy = l3FieldInfo(memory)?.legacy_fields; + if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) return null; + const value = (key: string): string | null => { + const field = (legacy as Record)[key]; + return typeof field === "string" ? field : null; + }; + return { + generalRulesAndSafetyConstraints: value("general_rules_and_safety_constraints"), + projectEnvironmentProfile: value("project_environment_profile"), + projectContract: value("project_contract"), + domainKnowledge: value("domain_knowledge") + }; +} diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index 7bc876ed9..393da3a53 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -1,11 +1,18 @@ +import { homedir } from "node:os"; import { assertJsonValue, canonicalJson, + deriveWorkspaceHostId, isLocalWorkspaceUri, sha256Hex, type L3UserProjectsReplaceRequest, type L3UserProjectsReplaceResponse } from "../contracts/index.js"; +import { getOrCreateInstallationId } from "../cli/analytics.js"; +import { isL3FieldCognitionActive } from "./l3-field/flag.js"; +import { loadL3FieldContext } from "./l3-field/l3-field-context.js"; +import type { L3FieldEnvironment } from "./l3-field/session-units.js"; +import { l3FieldSnapshotUnitKey } from "./l3-field/snapshot-memory.js"; import { languageSteeringLine, skillOutputLanguageFor, @@ -220,6 +227,8 @@ export interface MemoryServiceOptions { db?: MemoryDb; backend?: StorageBackend; mode?: "local" | "cloud" | "dev"; + /** Machine identity and home folder for the new L3; defaults to this installation. Tests override it. */ + l3FieldEnvironment?: () => L3FieldEnvironment; configPath?: string; configLoader?: (configPath?: string) => { config: MemmyConfig; @@ -566,6 +575,9 @@ export class MemoryService { runWorkerNoWrite: this.runWorkerNoWrite.bind(this), restartFailedProcessing: this.restartFailedProcessing.bind(this), enqueueObservationBackfill: (limit, at) => this.observationMemory.backfill(limit, at), + purgeL3InactiveItems: (at) => this.repos.l3Field.purgeInactive( + new Date(Date.parse(at) - this.config.algorithm.l3FieldCognition.inactiveRetentionDays * 86_400_000).toISOString() + ), previewPolicyEvidenceReconciliation: this.evolutionJobs.previewPolicyEvidenceReconciliation.bind(this.evolutionJobs), reconcileOrphanedPolicies: this.evolutionJobs.reconcileOrphanedPolicies.bind(this.evolutionJobs), enqueueJob: this.workerHandlers.enqueueJob, @@ -923,6 +935,20 @@ export class MemoryService { this.storageCapabilities().backendId === "sqlite-local"; } + private l3FieldActive(): boolean { + return isL3FieldCognitionActive(this.config, this.storageCapabilities(), this.mode); + } + + private l3FieldEnvironmentCache?: L3FieldEnvironment; + + private l3FieldEnvironment(): L3FieldEnvironment { + this.l3FieldEnvironmentCache ??= this.options.l3FieldEnvironment?.() ?? { + machineId: deriveWorkspaceHostId(getOrCreateInstallationId({ env: process.env })), + homeDir: homedir() + }; + return this.l3FieldEnvironmentCache; + } + private memorySearchEnabled(): boolean { return this.config.algorithm.enableMemorySearch; } @@ -1348,6 +1374,9 @@ export class MemoryService { if (session.status !== "open") { throw new MemoryServiceError("conflict", "l3_world_model_session_not_open"); } + if (this.l3FieldActive()) { + return loadL3FieldContext(this.repos, this.l3FieldEnvironment(), session); + } return this.l3WorldModelContextReadModel.load(session); } @@ -2572,9 +2601,16 @@ export class MemoryService { } const kind = kindFromMemory(memory); const at = nowIso(); - const deleted = strictV2WorldModel - ? this.repos.l3WorldModels.deleteScopeMemory(memory.id, at)?.deleted - : this.repos.memories.softDelete(memory.id, at); + const deletedL3Unit = l3FieldSnapshotUnitKey(memory); + const deleted = this.repos.transaction(() => { + const removed = strictV2WorldModel + ? this.repos.l3WorldModels.deleteScopeMemory(memory.id, at)?.deleted + : this.repos.memories.softDelete(memory.id, at); + // Deleting a new-L3 snapshot resets its unit: what it knew goes, later sessions start it afresh. + // Same transaction, so a failure never leaves the unit's items behind a deleted snapshot. + if (removed && deletedL3Unit) this.repos.l3Field.markUnitDeleted(deletedL3Unit, at); + return removed; + }); if (!deleted) { throw new MemoryServiceError("not_found", `memory not found: ${id}`); } diff --git a/Memory/src/service/project-environment/local-scanner.ts b/Memory/src/service/project-environment/local-scanner.ts index 4a21590b5..4e8d76fd1 100644 --- a/Memory/src/service/project-environment/local-scanner.ts +++ b/Memory/src/service/project-environment/local-scanner.ts @@ -57,7 +57,11 @@ interface InventorySnapshot { omittedCount: number; } -export async function scanLocalProject(workspaceUri: WorkspaceUri): Promise { +/** `withRuntimeProbes: false` skips running the declared runtimes (the new L3 keeps them in the machine profile). */ +export async function scanLocalProject( + workspaceUri: WorkspaceUri, + options: { withRuntimeProbes?: boolean } = {} +): Promise { const root = await resolveLocalWorkspaceRoot(workspaceUri); let stable = await scanInventory(root); let next = await scanInventory(root); @@ -73,7 +77,7 @@ export async function scanLocalProject(workspaceUri: WorkspaceUri): Promise { this.deps.repos.l3Field.setProfile(unitKey, { text: nextProfile, fingerprint: derived.fingerprint }); @@ -194,8 +199,11 @@ export class ProjectEnvironmentService { } } -async function deriveScanEvidence(workspaceUri: string): Promise { - const scan = await scanLocalProject(workspaceUri); +async function deriveScanEvidence( + workspaceUri: string, + options: { withRuntimeProbes?: boolean } = {} +): Promise { + const scan = await scanLocalProject(workspaceUri, options); const classification = classifyProjectInventory(scan.entries); const facts = parseDeterministicProjectFacts({ entries: scan.entries, diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index 3305648a2..9e92c1f55 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -47,6 +47,7 @@ import { import { ObservationDependenciesPendingError } from "../evidence/observation-memory-pipeline.js"; import { isObservationMemory, observationJobPayload, observationMemoryState, OBSERVATION_SUMMARY_MODEL_UNCONFIGURED } from "../observation-memory-state.js"; import { externalSkillGeneration, externalSkillJobPayload } from "../evidence/history-skill-policy.js"; +import { isL3FieldSnapshotMemory } from "../l3-field/snapshot-memory.js"; export const SUMMARY_WORKER_CONCURRENCY = 4; export const EMBEDDING_RETRY_LEASE_MS = 5 * 60_000; @@ -132,6 +133,8 @@ export interface WorkerRunnerDeps { runWorkerNoWrite: (request: RequestEnvelope) => Promise; restartFailedProcessing: (at: string, limit: number) => number; enqueueObservationBackfill?: (limit: number, at: string) => number; + /** Drops new-L3 items that have been inactive longer than the retention window. */ + purgeL3InactiveItems?: (at: string) => number; previewPolicyEvidenceReconciliation: (limit: number) => PolicyEvidencePreflightReport; reconcileOrphanedPolicies: (at: string, limit: number) => number; enqueueJob: (input: EnqueueJobInput) => EvolutionJobRecord; @@ -358,6 +361,7 @@ export class WorkerRunner { }, limit); for (const memory of retrievalMemories) { this.deps.repos.memories.reindexFts(memory); + if (isL3FieldSnapshotMemory(memory)) continue; if (!this.deps.capture.embedAfterCapture || retrievalDocumentIsCurrent(memory)) continue; if (this.deps.repos.runtime.hasPendingJob(memory.id, "embedding")) continue; const sourceHash = retrievalDocumentSourceHash(memory); @@ -377,6 +381,7 @@ export class WorkerRunner { enqueuedRetrievalReindexes += 1; } + this.deps.purgeL3InactiveItems?.(at); this.refreshObservationModelConfiguration(); return { requeuedJobs: interruptedJobs.length + failedJobs.length, diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts index 518be2dc5..9ed1bd017 100644 --- a/Memory/src/storage/l3-field-repository.ts +++ b/Memory/src/storage/l3-field-repository.ts @@ -71,6 +71,8 @@ export function l3UnitSnapshotMemoryKey(unitKey: string): string { return `world_model:l3_field:${unitKey}`; } +export { isL3FieldSnapshotMemory, l3FieldSnapshotUnitKey } from "../service/l3-field/snapshot-memory.js"; + /** Renders one unit (profile plus active items) as the text stored in its snapshot memory. */ export function renderL3UnitSnapshot(unit: L3UnitRecord, items: readonly L3ItemRecord[]): string { const facts = items.filter((item) => item.kind === "env_fact"); @@ -230,10 +232,18 @@ export class L3FieldRepository { return saved; } + /** + * Deleting a unit's L3 resets it rather than disabling it: items become + * inactive, the profile and snapshot link are cleared, and `deleted_at` + * records when, so work queued before the reset is discarded while later + * conversations build the unit up again. + */ markUnitDeleted(unitKey: string, at = nowIso()): void { this.db.transaction(() => { this.db.prepare( - `UPDATE l3_units SET deleted_at = COALESCE(deleted_at, ?), updated_at = ? WHERE unit_key = ?` + `UPDATE l3_units + SET deleted_at = ?, snapshot_memory_id = NULL, profile_text = NULL, profile_fingerprint = NULL, updated_at = ? + WHERE unit_key = ?` ).run(at, at, unitKey); this.db.prepare( `UPDATE l3_items @@ -292,12 +302,25 @@ export class L3FieldRepository { } private writeSnapshot(unit: L3UnitRecord, at: string): MemoryRow { - const memoryValue = renderL3UnitSnapshot(unit, this.activeItems(unit.unitKey)); + const items = this.activeItems(unit.unitKey); + const memoryValue = renderL3UnitSnapshot(unit, items); const existing = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; const title = unit.kind === "machine" ? "本机场域认知" : `项目场域认知：${unit.displayName ?? unit.projectIdentity}`; const tags = ["world_model", "l3_field", `scope:${unit.kind}`]; const info: Record = { l3_field_unit_key: unit.unitKey }; - const status = unit.deletedAt ? "deleted" as const : "activated" as const; + // Rewriting a snapshot keeps the user's archive; only the user unarchives it. + const status = existing?.status === "archived" ? "archived" as const : "activated" as const; + const bullets = (kind: L3ItemKind) => { + const lines = items.filter((item) => item.kind === kind).map((item) => `- ${item.content}`); + return lines.length ? lines.join("\n") : null; + }; + // Shape the desktop "world model" page already understands (it only reads schema-2 fields). + const legacyFields = { + general_rules_and_safety_constraints: null, + project_environment_profile: unit.profileText?.trim() ? unit.profileText.trim() : null, + project_contract: bullets("project_constraint"), + domain_knowledge: bullets("env_fact") + }; const memory: MemoryRow = { id: existing?.id ?? newId("memory"), timeline: existing?.timeline ?? at, @@ -322,7 +345,13 @@ export class L3FieldRepository { title, summary: [...memoryValue.replace(/\s+/gu, " ").trim()].slice(0, 240).join(""), body: memoryValue, - l3_field: { unit_key: unit.unitKey, kind: unit.kind } + l3_field: { + unit_key: unit.unitKey, + kind: unit.kind, + display_name: unit.displayName ?? null, + identity_source: unit.identitySource ?? null, + legacy_fields: legacyFields + } } }, memoryLayer: "L3", @@ -330,7 +359,7 @@ export class L3FieldRepository { version: existing?.version ?? 1, createdAt: existing?.createdAt ?? at, updatedAt: at, - deletedAt: unit.deletedAt ?? null + deletedAt: null }; const saved = existing ? this.memories.update(memory) : this.memories.insert(memory); if (!existing) { diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index 6794c77d7..fc205e458 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -56,6 +56,7 @@ import { } from "./sqlite-vec-store.js"; import { compactHistoryChangePayload, compactHistoryChangeRow } from "./history-change-snapshot.js"; import { L3FieldRepository } from "./l3-field-repository.js"; +import { isL3FieldSnapshotMemory } from "../service/l3-field/snapshot-memory.js"; type SqlValue = string | number | Buffer | null; const BUNDLE_TABLES = [ @@ -1322,7 +1323,7 @@ export class MemoryRepository { reindexFts(memory: MemoryRow): void { try { this.db.prepare(`DELETE FROM memories_fts WHERE id = ?`).run(memory.id); - if (!memory.deletedAt && memory.status !== "deleted") { + if (!memory.deletedAt && memory.status !== "deleted" && !isL3FieldSnapshotMemory(memory)) { this.db .prepare(`INSERT INTO memories_fts (id, identifier, memory_value, tags) VALUES (?, ?, ?, ?)`) .run(memory.id, memory.id, retrievalDocumentForMemory(memory), memory.tags.join(" ")); diff --git a/Memory/tests/service/l3-field/l3-field-context.test.ts b/Memory/tests/service/l3-field/l3-field-context.test.ts new file mode 100644 index 000000000..a307da57f --- /dev/null +++ b/Memory/tests/service/l3-field/l3-field-context.test.ts @@ -0,0 +1,145 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const MACHINE_ID = "c".repeat(64); + +function setup(enabled = true, l3MachineProbe: () => Promise | Promise = () => new Promise(() => undefined)) { + const root = fixture.createTestRoot("l3-field-context-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + const service = fixture.createTestMemoryService({ + db, + mode: "dev", + config, + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }) + }); + const repos = new Repositories(db.db); + return { db, service, repos, home, project }; +} + +function open(service: ReturnType["service"], sessionKey: string, workspace?: string) { + const namespace = { source: "codex", profileId: "default", sessionKey, userId: "u1" }; + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + ...(workspace ? { workspaceUri: pathToFileURL(workspace).href, workspaceHostId: "b".repeat(64) } : {}), + namespace + }); + const request = { + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", + adapterId: "codex-memory", + source: "codex", + namespace: opened.projectId ? { ...namespace, projectId: opened.projectId } : namespace + }; + return { opened, request, namespace: request.namespace }; +} + +function evidence() { + return { fragmentIds: ["f1"], l1MemoryIds: ["memory_l1"] }; +} + +describe("new L3 session context", () => { + it("is empty for a session without a folder until the machine knows something", () => { + const { service, repos } = setup(); + const { opened, request } = open(service, "machine-only"); + expect(service.l3WorldModelContext(opened.sessionId, request)).toMatchObject({ renderedContext: "", memoryId: null }); + + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const context = service.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toBe("当前项目：无\n\n## 本机环境事实\n- 本机用 pnpm 9"); + expect(context.memoryId).toBe(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId); + expect(context.projectContract).toBeNull(); + expect(context.domainKnowledge).toBe("- 本机用 pnpm 9"); + }); + + it("names the project and its source, and shows machine and project knowledge together", () => { + const { service, repos, project } = setup(); + const { opened, request } = open(service, "with-project", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + service.l3WorldModelContext(opened.sessionId, request); + const unit = repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project'`).get() as { unit_key: string }; + repos.l3Field.insertItem({ unitKey: unit.unit_key, kind: "project_constraint", content: "改完跑 npm test", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(unit.unit_key); + + const context = service.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toBe( + "当前项目：demo（文件夹）\n\n## 本机环境事实\n- 本机用 pnpm 9\n\n## 项目约束\n- 改完跑 npm test" + ); + expect(context.projectId).toBe(opened.projectId); + expect(context.memoryId).toBe(repos.l3Field.getUnit(unit.unit_key)!.snapshotMemoryId); + expect(context.sourceMemoryIds).toEqual(["memory_l1"]); + }); + + it("shows a snapshot on the world model page and keeps it out of full-text search", () => { + const { service, repos, db } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { request } = open(service, "panel"); + const detail = service.getMemory(snapshot.id, request); + expect(JSON.stringify(detail)).toContain("\"domainKnowledge\":\"- 本机用 pnpm 9\""); + const fts = db.db.prepare(`SELECT COUNT(*) AS n FROM memories_fts WHERE id = ?`).get(snapshot.id) as { n: number }; + expect(fts.n).toBe(0); + }); + + it("deleting a snapshot resets its unit, and the next session starts it afresh", () => { + const { service, repos } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const item = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { opened, request } = open(service, "delete"); + + service.deleteMemory(snapshot.id, { namespace: request.namespace, reason: "user" }); + expect(repos.l3Field.getItem(item.id)!.status).toBe("inactive"); + expect(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId).toBeUndefined(); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).toBe(""); + + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 bun", evidence: evidence(), evidenceAt: "2026-10-03T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const context = service.l3WorldModelContext(opened.sessionId, request); + expect(context.renderedContext).toContain("本机用 bun"); + expect(context.memoryId).not.toBe(snapshot.id); + }); + + it("on startup drops items inactive for over 15 days and never queues embeddings for snapshots", () => { + const { service, repos, db } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const old = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "旧", evidence: evidence(), evidenceAt: "2026-01-01T00:00:00.000Z" }); + const recent = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "新", evidence: evidence(), evidenceAt: "2026-01-01T00:00:00.000Z" }); + repos.l3Field.inactivateItem(old.id, { at: new Date(Date.now() - 16 * 86_400_000).toISOString() }); + repos.l3Field.inactivateItem(recent.id, { at: new Date(Date.now() - 14 * 86_400_000).toISOString() }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + + service.reconcileWorkerStartup(); + expect(repos.l3Field.getItem(old.id)).toBeUndefined(); + expect(repos.l3Field.getItem(recent.id)).toBeDefined(); + const jobs = db.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'embedding' AND target_memory_id = ?` + ).get(snapshot.id) as { n: number }; + expect(jobs.n).toBe(0); + }); + + it("keeps the old L3 when the switch is off", () => { + const { service, repos } = setup(false); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const { opened, request } = open(service, "flag-off"); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).not.toContain("pnpm"); + }); +}); diff --git a/Memory/tests/service/l3-field/unit-profile.test.ts b/Memory/tests/service/l3-field/unit-profile.test.ts index 5290a65c0..7067ce027 100644 --- a/Memory/tests/service/l3-field/unit-profile.test.ts +++ b/Memory/tests/service/l3-field/unit-profile.test.ts @@ -33,10 +33,16 @@ function setup(reply = '{"op":"create","profile":"Node 项目，测试用 vitest }); const pendingJob = (): EvolutionJobRecord => { const row = db.db.prepare( - `SELECT id, user_id, payload_json FROM evolution_jobs + `SELECT id, user_id, payload_json, created_at FROM evolution_jobs WHERE job_type = 'project_environment_profile' AND status = 'queued' ORDER BY created_at DESC LIMIT 1` - ).get() as { id: string; user_id: string; payload_json: string }; - return { id: row.id, jobType: "project_environment_profile", userId: row.user_id, payload: JSON.parse(row.payload_json) } as EvolutionJobRecord; + ).get() as { id: string; user_id: string; payload_json: string; created_at: string }; + return { + id: row.id, + jobType: "project_environment_profile", + userId: row.user_id, + payload: JSON.parse(row.payload_json), + createdAt: row.created_at + } as EvolutionJobRecord; }; return { db, repos, service, complete, unit, project, pendingJob }; } @@ -81,13 +87,31 @@ describe("L3 field cognition project profile", () => { expect(complete).toHaveBeenCalledTimes(1); }); - it("drops the result when the unit was deleted", async () => { - const { service, repos, complete, unit, project, pendingJob } = setup(); + it("does not ask the model again for an unchanged project that had nothing worth a profile (an empty folder)", async () => { + const { service, complete, unit, project, pendingJob, db } = setup('{"op":"noop","profile":""}'); + rmSync(join(project, "package.json")); + rmSync(join(project, "index.ts")); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); + await service.processProfileJob(pendingJob()); + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', updated_at = ? WHERE job_type = 'project_environment_profile'`) + .run(new Date(Date.now() - 11 * 60_000).toISOString()); service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project }); - repos.l3Field.markUnitDeleted(unit.unitKey); + await service.processProfileJob(pendingJob()); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("drops a job queued before the unit was deleted, but serves jobs queued after it", async () => { + const { service, repos, complete, unit, project, pendingJob, db } = setup(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project, at: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.markUnitDeleted(unit.unitKey, "2026-10-02T00:00:00.000Z"); await service.processProfileJob(pendingJob()); expect(complete).not.toHaveBeenCalled(); expect(repos.l3Field.getUnit(unit.unitKey)!.profileText).toBeUndefined(); + + db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE job_type = 'project_environment_profile'`).run(); + service.requestL3UnitProfile({ userId: "u1", unitKey: unit.unitKey, checkoutRoot: project, at: "2026-10-03T00:00:00.000Z" }); + await service.processProfileJob(pendingJob()); + expect(repos.l3Field.getUnit(unit.unitKey)!.profileText).toBe("Node 项目，测试用 vitest"); }); }); diff --git a/Memory/tests/service/project-environment/local-scanner.test.ts b/Memory/tests/service/project-environment/local-scanner.test.ts index 6736cce8f..74b2e0b48 100644 --- a/Memory/tests/service/project-environment/local-scanner.test.ts +++ b/Memory/tests/service/project-environment/local-scanner.test.ts @@ -39,6 +39,14 @@ describe("local project scanner", () => { ); }); + it("can skip running the declared runtimes", async () => { + const { root, uri } = await fixture(); + await writeFile(join(root, "package.json"), JSON.stringify({ engines: { node: ">=22" } })); + const result = await scanLocalProject(uri, { withRuntimeProbes: false }); + expect(result.runtimeProbes).toEqual([]); + expect(result.entries.map((entry) => entry.relativePath)).toEqual(["package.json"]); + }); + it("excludes gitignored, fixed, sensitive, binary, and symlink paths from the inventory", async () => { const { root, uri } = await fixture(); await mkdir(join(root, "node_modules")); From e2f88ffa5fb5c31a8eb44fb1850ac97cf6974892 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:47:36 +0200 Subject: [PATCH 7/9] feat(memory): update l3 field items from frozen batches Wherever a task closes (session close, new task, ended topic, idle timeout), its turns are frozen for the new L3, only through that task so a task still in progress is not cut; a conversation that simply stops therefore still reaches L3. With the switch off nothing changes. Older evidence never overturns newer state: a replace or delete whose cited evidence is older than the item's last confirmation is dropped, and what was said before a unit reset does not refill it, even when frozen or rerun later. The worker's next wake-up uses the same readiness rules as leasing (scope order, L1 summaries), so a job it cannot lease does not make it spin. A batch is shown the items removed on evidence newer than some of its own, with when (a batch rerun late, history scanned late, or one spanning the removal), so older evidence does not bring them back. Each lease of a batch job carries a token: a run that outlived its lease neither writes nor changes the job, and a run that finds its batch dead-lettered drops its result. Replace and delete skip items already gone, and ops citing a turn or L1 deleted during the model call are dropped. A machine probe that was running when the machine was reset is dropped. A snapshot is rewritten only when what it shows changed, and project snapshots move only when the machine's did, or when the machine snapshot is archived (so clients drop the cached machine part). Co-Authored-By: Claude Opus 5.5 --- AgentSourceCore/src/memory-token-budget.ts | 1 + .../local-api-contracts/src/memory-runtime.ts | 1 + Memory/src/contracts/memory-runtime.ts | 1 + .../service/evolution/evolution-logging.ts | 2 +- .../service/l3-field/evidence-fragments.ts | 157 ++++- .../src/service/l3-field/project-resolver.ts | 44 +- Memory/src/service/l3-field/types.ts | 16 + Memory/src/service/l3-field/unit-ops.ts | 149 ++++ .../service/l3-field/unit-update-pipeline.ts | 429 ++++++++++++ .../service/l3-field/unit-update-prompt.ts | 141 ++++ .../l3-world-model/strict-json-completion.ts | 10 +- Memory/src/service/memory-service.ts | 71 +- Memory/src/service/worker/job-handlers.ts | 24 +- Memory/src/service/worker/worker-runner.ts | 28 +- Memory/src/storage/l3-field-repository.ts | 147 +++- Memory/src/storage/repositories.ts | 198 +++++- Memory/src/storage/schema.ts | 3 + Memory/src/types.ts | 1 + .../repository/l3-field-repository.test.ts | 12 +- .../service/l3-field/ended-task-l3.test.ts | 203 ++++++ .../l3-field/evidence-fragments.test.ts | 96 ++- .../service/l3-field/l3-field-context.test.ts | 93 ++- .../service/l3-field/project-resolver.test.ts | 23 + .../service/l3-field/unit-update.test.ts | 661 ++++++++++++++++++ 24 files changed, 2414 insertions(+), 97 deletions(-) create mode 100644 Memory/src/service/l3-field/unit-ops.ts create mode 100644 Memory/src/service/l3-field/unit-update-pipeline.ts create mode 100644 Memory/src/service/l3-field/unit-update-prompt.ts create mode 100644 Memory/tests/service/l3-field/ended-task-l3.test.ts create mode 100644 Memory/tests/service/l3-field/unit-update.test.ts diff --git a/AgentSourceCore/src/memory-token-budget.ts b/AgentSourceCore/src/memory-token-budget.ts index 7524c4e8f..0ce567933 100644 --- a/AgentSourceCore/src/memory-token-budget.ts +++ b/AgentSourceCore/src/memory-token-budget.ts @@ -20,6 +20,7 @@ export const MEMORY_WORKER_JOB_TYPES = [ "l2_induction", "l3_abstraction", "l3_world_model_update", + "l3_unit_update", "project_environment_profile", "skill_crystallization", "skill_cluster_assign", diff --git a/App/backend/local-api-contracts/src/memory-runtime.ts b/App/backend/local-api-contracts/src/memory-runtime.ts index 3c909be58..cda827268 100644 --- a/App/backend/local-api-contracts/src/memory-runtime.ts +++ b/App/backend/local-api-contracts/src/memory-runtime.ts @@ -54,6 +54,7 @@ export const JobTypeSchema = z.enum([ "l2_induction", "l3_abstraction", "l3_world_model_update", + "l3_unit_update", "project_environment_profile", "skill_crystallization", "skill_trial_resolve", diff --git a/Memory/src/contracts/memory-runtime.ts b/Memory/src/contracts/memory-runtime.ts index 1ee3d88ca..e0aca4dd2 100644 --- a/Memory/src/contracts/memory-runtime.ts +++ b/Memory/src/contracts/memory-runtime.ts @@ -55,6 +55,7 @@ export const JobTypeSchema = z.enum([ "l2_induction", "l3_abstraction", "l3_world_model_update", + "l3_unit_update", "project_environment_profile", "skill_crystallization", "skill_cluster_assign", diff --git a/Memory/src/service/evolution/evolution-logging.ts b/Memory/src/service/evolution/evolution-logging.ts index 1292292e4..49bac5423 100644 --- a/Memory/src/service/evolution/evolution-logging.ts +++ b/Memory/src/service/evolution/evolution-logging.ts @@ -21,7 +21,7 @@ export function evolutionJobLogFields(job: EvolutionJobRecord): Record = {} ): void { diff --git a/Memory/src/service/l3-field/evidence-fragments.ts b/Memory/src/service/l3-field/evidence-fragments.ts index fb7df2944..355071e80 100644 --- a/Memory/src/service/l3-field/evidence-fragments.ts +++ b/Memory/src/service/l3-field/evidence-fragments.ts @@ -1,16 +1,24 @@ +import { existsSync } from "node:fs"; import path from "node:path"; import { sha256Hex } from "../../contracts/index.js"; import type { RawTurnRecord } from "../../storage/repositories.js"; import { matchToolResultIndices } from "../../utils/tool-call-pairing.js"; import type { EvidenceFragment, ResolvedProject } from "./types.js"; -const PATH_KEYS = ["file_path", "filePath", "path", "notebook_path", "workdir", "cwd", "directory"]; +const TARGET_KEYS = ["file_path", "filePath", "path", "notebook_path"]; +const WORKDIR_KEYS = ["workdir", "cwd", "directory"]; +const PATCH_KEYS = ["input", "patch"]; +const PATCH_FILE_LINE = /^\*\*\* (?:Update|Add|Delete) File: (.+)$|^\*\*\* Move to: (.+)$/gm; const COMMAND_KEYS = ["command", "cmd"]; const USER_TEXT_LIMIT = 4_000; -const CLIP_HEAD_CHARS = 600; -const CLIP_TAIL_CHARS = 600; +const CLIP_HEAD_CHARS = 300; +const CLIP_TAIL_CHARS = 300; +const ERROR_LINE_CHARS = 200; const ERROR_LINE = /error|fail|exception|denied|not found|traceback|refused|cannot|no such|fatal/i; const COMMAND_SEPARATORS = new Set(["&&", "||", ";", "|"]); +/** A redirection token, with its target attached (`2>/dev/null`) or as the next token (`>`). */ +const REDIRECT = /^\d*(?:>>?|<)|^&>/; +const REDIRECT_ONLY = /^(?:\d*(?:>>?|<)|&>)$/; /** * Cuts one raw turn into evidence fragments: the user's words, and each tool @@ -32,18 +40,43 @@ export function buildEvidenceFragments(input: { const project = input.resolve(input.sessionCwd, "session_dir"); return project ? [input.unitKeyOf(project)] : []; }; + const rootOf = new Map(); const unitsForPaths = (paths: string[]): string[] => { if (paths.length === 0) return sessionUnits(); const keys = new Set(); for (const target of paths) { const project = input.resolve(target, "tool_path"); - if (project) keys.add(input.unitKeyOf(project)); + if (!project) continue; + const key = input.unitKeyOf(project); + rootOf.set(key, project.checkoutRoot ?? null); + keys.add(key); } - return [...keys]; + return mostSpecificUnits([...keys], rootOf); }; const fragmentId = (kind: string, index: number) => sha256Hex(`${rawTurn.id}:${kind}:${index}`).slice(0, 16); + const calls: EvidenceFragment[] = []; + rawTurn.toolCalls.forEach((call, index) => { + if (!isRecord(call)) return; + const paths = extractToolPaths(call, input.sessionCwd); + calls.push({ + id: fragmentId("call", index), + rawTurnId: rawTurn.id, + l1MemoryId: input.l1MemoryId, + toolName: typeof call.name === "string" ? call.name : null, + input: clipToolInput(call.input), + output: toolOutputText(call), + isError: isErrorRecord(call), + paths, + candidateUnitKeys: unitsForPaths(paths), + at: typeof call.endedAt === "string" ? call.endedAt : rawTurn.createdAt + }); + }); + // The user's words belong to the projects this turn actually worked on; only a turn that touched + // no project falls back to the session folder (a session opened in a big parent folder). + const touched = mostSpecificUnits([...new Set(calls.flatMap((call) => call.candidateUnitKeys))], rootOf); + const fragments: EvidenceFragment[] = []; const userText = rawTurn.userText?.trim(); if (userText) { @@ -56,27 +89,11 @@ export function buildEvidenceFragments(input: { output: userText.slice(0, USER_TEXT_LIMIT), isError: false, paths: [], - candidateUnitKeys: sessionUnits(), + candidateUnitKeys: touched.length > 0 ? touched : sessionUnits(), at: rawTurn.createdAt }); } - - rawTurn.toolCalls.forEach((call, index) => { - if (!isRecord(call)) return; - const paths = extractToolPaths(call, input.sessionCwd); - fragments.push({ - id: fragmentId("call", index), - rawTurnId: rawTurn.id, - l1MemoryId: input.l1MemoryId, - toolName: typeof call.name === "string" ? call.name : null, - input: call.input ?? null, - output: toolOutputText(call), - isError: isErrorRecord(call), - paths, - candidateUnitKeys: unitsForPaths(paths), - at: typeof call.endedAt === "string" ? call.endedAt : rawTurn.createdAt - }); - }); + fragments.push(...calls); const matched = new Set( matchToolResultIndices(rawTurn.toolCalls, rawTurn.toolResults).filter((index): index is number => index !== undefined) @@ -101,31 +118,51 @@ export function buildEvidenceFragments(input: { } /** - * Paths a tool call actually touched: file/dir arguments, `cd` and `git -C` - * targets, and absolute path arguments of commands. Relative paths resolve - * against the call's own workdir/cwd, else the session folder. + * Paths a tool call actually touched: file arguments, files named in a patch, `cd` + * and `git -C` targets, and absolute path arguments of commands. Relative paths + * resolve against the call's own workdir/cwd, else the session folder. Each shell + * subcommand that names no target of its own counts as touching the directory it ran in. */ export function extractToolPaths(call: Record, sessionCwd: string | null): string[] { const rawArgs = call.input ?? call.arguments ?? call.args; const args = parseArguments(rawArgs); if (!args) { - return typeof rawArgs === "string" ? codeModePaths(rawArgs, sessionCwd) : []; + if (typeof rawArgs !== "string") return []; + // A code-mode snippet can hold a patch next to other calls; keep the paths of both. + return [...new Set([...patchPaths(rawArgs, sessionCwd), ...codeModePaths(rawArgs, sessionCwd)])]; } - const workdir = stringValue(args.workdir) ?? stringValue(args.cwd) ?? stringValue(args.directory) ?? sessionCwd; + const explicitWorkdir = WORKDIR_KEYS.map((key) => stringValue(args[key])).find(Boolean); + const workdir = explicitWorkdir ? resolveAgainst(explicitWorkdir, sessionCwd) : sessionCwd; const found: string[] = []; const add = (value: string, base: string | null) => { const resolved = resolveAgainst(value, base); if (resolved && !found.includes(resolved)) found.push(resolved); }; - for (const key of PATH_KEYS) { + for (const key of TARGET_KEYS) { const value = stringValue(args[key]); - if (value) add(value, key === "workdir" || key === "cwd" || key === "directory" ? sessionCwd : workdir); + if (value) add(value, workdir); + } + for (const key of PATCH_KEYS) { + const value = stringValue(args[key]); + if (value) for (const target of patchPaths(value, workdir)) add(target, null); } for (const key of COMMAND_KEYS) { const raw = args[key]; const command = typeof raw === "string" ? raw : Array.isArray(raw) ? raw.filter((part) => typeof part === "string").join(" ") : null; if (command) for (const target of commandPaths(command, workdir)) add(target, null); } + if (found.length === 0 && explicitWorkdir && workdir) found.push(workdir); + return found; +} + +/** Files an apply_patch style patch adds, updates, deletes or moves to. */ +function patchPaths(text: string, base: string | null): string[] { + if (!text.includes("*** Begin Patch")) return []; + const found: string[] = []; + for (const match of text.matchAll(PATCH_FILE_LINE)) { + const resolved = resolveAgainst((match[1] ?? match[2]!).trim(), base); + if (resolved && !found.includes(resolved)) found.push(resolved); + } return found; } @@ -145,6 +182,8 @@ function codeModePaths(code: string, sessionCwd: string | null): string[] { const add = (target: string | null) => { if (target && !found.includes(target)) found.push(target); }; + // A snippet can hold several calls and literals are not tied to the call they belong to, + // so every workdir counts here (unlike a single call, where a named target wins). const workdirs = [...(values.workdir ?? []), ...(values.cwd ?? [])]; for (const dir of workdirs) add(resolveAgainst(dir, sessionCwd)); const base = workdirs[0] ? resolveAgainst(workdirs[0], sessionCwd) : sessionCwd; @@ -175,10 +214,24 @@ export function clipToolOutput(text: string): string { const head = text.slice(0, CLIP_HEAD_CHARS); const tail = text.slice(-CLIP_TAIL_CHARS); const middle = text.slice(CLIP_HEAD_CHARS, text.length - CLIP_TAIL_CHARS); - const errorLines = middle.split(/\r?\n/).filter((line) => ERROR_LINE.test(line)).slice(0, 20); + const errorLines = middle.split(/\r?\n/).filter((line) => ERROR_LINE.test(line)).slice(0, 20) + .map((line) => (line.length > ERROR_LINE_CHARS ? `${line.slice(0, ERROR_LINE_CHARS)}…` : line)); return [head, "[已截断]", ...errorLines, ...(errorLines.length ? ["[已截断]"] : []), tail].join("\n"); } +/** A long tool input (a patch, a written file) is clipped like output; its paths were read before clipping. */ +function clipToolInput(value: unknown): unknown { + if (value === undefined || value === null) return null; + const text = typeof value === "string" ? value : JSON.stringify(value); + return text.length <= CLIP_HEAD_CHARS + CLIP_TAIL_CHARS ? value : clipToolOutput(text); +} + +/** + * Paths each subcommand works on: `cd` and `git -C` targets, absolute path arguments, and relative + * ones (`api/AGENTS.md`, or a bare name that exists in the current directory). + * A subcommand naming none works in the current directory, so that counts instead. Where + * output is redirected to (`> /tmp/log`) is not what the command works on. + */ function commandPaths(command: string, startDir: string | null): string[] { const found: string[] = []; let cwd = startDir; @@ -193,16 +246,27 @@ function commandPaths(command: string, startDir: string | null): string[] { cwd = target; } } else { + const targets: string[] = []; for (let i = 0; i < rest.length; i += 1) { const token = rest[i]!; + if (REDIRECT.test(token)) { + if (REDIRECT_ONLY.test(token)) i += 1; + continue; + } if (executable === "git" && token === "-C" && rest[i + 1]) { const target = resolveAgainst(rest[i + 1]!, cwd); - if (target) found.push(target); + if (target) targets.push(target); i += 1; continue; } - if (isAbsoluteAny(token) && !token.includes("://")) found.push(token); + if (isAbsoluteAny(token) && !token.includes("://")) targets.push(token); + else { + const relative = relativePathArgument(token, cwd); + if (relative) targets.push(relative); + } } + if (targets.length === 0 && cwd) targets.push(cwd); + found.push(...targets); } segment = []; }; @@ -271,6 +335,33 @@ function parseArguments(value: unknown): Record | null { } } +/** + * Work on a project inside a bigger folder belongs to that project: when a fragment touches both a + * folder and a project nested in it, only the nested one stays a candidate. + */ +function mostSpecificUnits(keys: string[], rootOf: ReadonlyMap): string[] { + const inside = (child: string, parent: string) => + child !== parent && [parent, parent + "/", parent + "\\"].some((prefix) => prefix !== parent && child.startsWith(prefix)); + return keys.filter((key) => { + const root = rootOf.get(key); + return !root || !keys.some((other) => { + const otherRoot = rootOf.get(other); + return Boolean(otherRoot) && inside(otherRoot!, root); + }); + }); +} + +/** Wildcards, patterns and expressions are not paths even when they contain a slash. */ +const NOT_A_PATH = /[*?[\]{}|()<>$`'"=!\s]/; + +/** A relative argument is a path when it has a slash, or when it names something in the current directory. */ +function relativePathArgument(token: string, cwd: string | null): string | null { + if (!cwd || token.startsWith("-") || token.includes("://") || NOT_A_PATH.test(token)) return null; + const target = resolveAgainst(token, cwd); + if (!target) return null; + return token.includes("/") || existsSync(target) ? target : null; +} + function resolveAgainst(value: string, base: string | null): string | null { const trimmed = value.trim(); if (!trimmed || trimmed.includes("://")) return null; diff --git a/Memory/src/service/l3-field/project-resolver.ts b/Memory/src/service/l3-field/project-resolver.ts index c06c2c7fd..8a3e9f671 100644 --- a/Memory/src/service/l3-field/project-resolver.ts +++ b/Memory/src/service/l3-field/project-resolver.ts @@ -52,8 +52,12 @@ export function resolveProject(input: ResolveProjectInput, ctx: ResolverContext) const startDir = existingDirectory(target); const selected = nearestUserProject(target, ctx.userProjects); - const gitRaw = startDir ? readGitIdentity(startDir) : null; - const git = gitRaw && isUsableRoot(gitRaw.repoRoot, home, ctx) ? gitRaw : null; + // A folder the user chose is a project wherever it is; otherwise tool install folders never are. + if (!selected && isToolLocation(target, home)) return null; + // A dependency that is its own git clone belongs to the project that contains it. + const gitStart = startDir ? outsidePackageDirectory(startDir) : null; + const gitRaw = gitStart ? readGitIdentity(gitStart) : null; + const git = gitRaw && isUsableRoot(gitRaw.repoRoot, home, ctx) && !isToolLocation(gitRaw.repoRoot, home) ? gitRaw : null; if (selected && isUsableRoot(selected.root, home, ctx)) { const gitIsNearer = git && git.repoRoot.length > selected.root.length; @@ -81,7 +85,7 @@ export function resolveProject(input: ResolveProjectInput, ctx: ResolverContext) if (input.role === "session_dir") { const root = startDir ?? path.dirname(target); - if (!isUsableRoot(root, home, ctx)) return null; + if (!isUsableRoot(root, home, ctx) || hasPackageDirectory(root)) return null; return { identity: `dir:${root}`, checkoutRoot: root, source: "folder", displayName: path.basename(root) }; } const sessionFolder = input.sessionProjectRoot ?? input.sessionCwd; @@ -121,6 +125,40 @@ function isThrowawayLocation(target: string, home: string, ctx: ResolverContext) return false; } +// Temp folders (/var/folders, /tmp) are handled as throwaway locations; /etc resolves to /private/etc on macOS. +const SYSTEM_ROOTS = [ + "/usr", "/bin", "/sbin", "/etc", "/private/etc", "/System", "/Library", "/Applications", + "/opt/homebrew", "/opt/local", "/nix" +]; +const WINDOWS_TOOL_DIRS = /\\(program files( \(x86\))?|programdata|appdata)(\\|$)/i; +const PACKAGE_DIRS = new Set(["node_modules", "site-packages", "dist-packages", "vendor", "bower_components"]); + +/** + * Where tools and the system live, not projects: system and package-manager folders, Windows + * program folders, ~/Library, and hidden folders right under home (~/.nvm, ~/.oh-my-zsh, ~/.pyenv, ...). + * Many of these are git clones, so the git check alone would turn them into projects. + */ +function isToolLocation(target: string, home: string): boolean { + if (WINDOWS_TOOL_DIRS.test(target)) return true; + if (isPathWithin(home, target)) { + const first = path.relative(home, target).split(path.sep)[0] ?? ""; + return first.startsWith(".") || first === "Library"; + } + return SYSTEM_ROOTS.some((root) => isPathWithin(root, target, path.posix)); +} + +/** A root inside a dependency folder is a downloaded package, not the user's project. */ +function hasPackageDirectory(root: string): boolean { + return root.split(/[\\/]/).some((part) => PACKAGE_DIRS.has(part)); +} + +/** The directory just above the first dependency folder on the path, or the path itself. */ +function outsidePackageDirectory(target: string): string { + const parts = target.split(path.sep); + const index = parts.findIndex((part) => PACKAGE_DIRS.has(part)); + return index > 0 ? parts.slice(0, index).join(path.sep) || path.sep : target; +} + /** A project root may not be the home folder, a top-level container folder, a filesystem root or a throwaway location. */ function isUsableRoot(root: string, home: string, ctx: ResolverContext): boolean { if (path.dirname(root) === root) return false; diff --git a/Memory/src/service/l3-field/types.ts b/Memory/src/service/l3-field/types.ts index 5a8937685..ac4a34f9f 100644 --- a/Memory/src/service/l3-field/types.ts +++ b/Memory/src/service/l3-field/types.ts @@ -73,3 +73,19 @@ export interface L3UserProject { rootPath: string; name: string; } + +export interface L3UnitBatchRecord { + id: string; + machineUnitKey: string; + scopeSeq: number; + userId: string; + sessionId: string; + trigger: string; + startTraceSeq: number; + endTraceSeq: number; + l1MemoryIds: string[]; + rawTurnIds: string[]; + status: L3UnitBatchStatus; + createdAt: string; + updatedAt: string; +} diff --git a/Memory/src/service/l3-field/unit-ops.ts b/Memory/src/service/l3-field/unit-ops.ts new file mode 100644 index 000000000..04f4c3be2 --- /dev/null +++ b/Memory/src/service/l3-field/unit-ops.ts @@ -0,0 +1,149 @@ +import type { EvidenceFragment, L3ItemKind, L3ItemRecord, L3UnitRecord } from "./types.js"; + +/** One model call: the machine plus at most one project, with the evidence that may support them. */ +export interface L3UpdateGroup { + machine: L3UnitRecord; + project: L3UnitRecord | null; + fragments: EvidenceFragment[]; + l1: Array<{ id: string; title: string; summary: string; at: string }>; +} + +/** A validated operation; every op names the unit it changes. */ +export type L3UnitOp = + | { op: "add"; unitKey: string; kind: L3ItemKind; content: string; evidence: string[] } + | { op: "replace"; unitKey: string; kind: L3ItemKind; ids: string[]; content: string; evidence: string[] } + | { op: "delete"; unitKey: string; id: string; evidence: string[] } + | { op: "confirm"; unitKey: string; id: string; evidence: string[] }; + +interface RawOp { + op: "add" | "replace" | "delete" | "confirm"; + unit?: string; + unit_key?: string; + kind?: string; + content?: string; + ids?: string[]; + id?: string; + evidence: string[]; +} + +const MIN_REPOSITORY_NAME_LENGTH = 4; + +/** JSON shape only; semantic checks run in validateL3UnitOps so the model can be told what was wrong. */ +export function parseL3UnitOutput(value: unknown): { reason: string; ops: RawOp[] } { + if (!isRecord(value) || !Array.isArray(value.ops)) throw new TypeError("expected {reason, ops[]}"); + const ops = value.ops.map((op, index): RawOp => { + if (!isRecord(op) || !["add", "replace", "delete", "confirm"].includes(op.op as string)) { + throw new TypeError(`ops[${index}]: unknown op`); + } + if (!isStringArray(op.evidence)) throw new TypeError(`ops[${index}]: evidence must be a string array`); + if (op.ids !== undefined && !isStringArray(op.ids)) throw new TypeError(`ops[${index}]: ids must be a string array`); + for (const key of ["unit", "unit_key", "kind", "content", "id"]) { + if (op[key] !== undefined && typeof op[key] !== "string") throw new TypeError(`ops[${index}]: ${key} must be a string`); + } + return op as unknown as RawOp; + }); + return { reason: typeof value.reason === "string" ? value.reason : "", ops }; +} + +/** Checks every op against the group; returns the ops that pass and one line per problem. */ +export function validateL3UnitOps( + rawOps: readonly RawOp[], + group: L3UpdateGroup, + activeItems: ReadonlyMap, + /** Items written earlier in this batch: shown to the model, but changes to them are dropped, not errors. */ + addedInBatch: ReadonlySet = new Set() +): { ops: L3UnitOp[]; problems: string[] } { + const fragments = new Map(group.fragments.map((fragment) => [fragment.id, fragment])); + const groupUnits = new Set([group.machine.unitKey, ...(group.project ? [group.project.unitKey] : [])]); + const usedIds = new Set(); + const ops: L3UnitOp[] = []; + const problems: string[] = []; + + rawOps.forEach((raw, index) => { + const fail = (message: string) => problems.push(`ops[${index}]: ${message}`); + const targets = raw.op === "replace" ? raw.ids ?? [] : raw.op === "add" ? [] : [raw.id ?? ""]; + if (targets.some((id) => addedInBatch.has(id))) return; + const evidence = [...new Set(raw.evidence)]; + if (evidence.length === 0) return fail("evidence is empty"); + const unknown = evidence.filter((id) => !fragments.has(id)); + if (unknown.length) return fail(`evidence ${unknown.join(", ")} is not a fragment of this input`); + + if (raw.op === "add" || raw.op === "replace") { + const content = raw.content?.trim() ?? ""; + if (!content) return fail("content is empty"); + let unitKey: string; + let kind: L3ItemKind; + let ids: string[] = []; + if (raw.op === "add") { + if (raw.unit === "machine") { + unitKey = group.machine.unitKey; + } else if (raw.unit === "project") { + if (!group.project) return fail("no project is listed, so unit must be machine"); + if (raw.unit_key && raw.unit_key !== group.project.unitKey) return fail(`unit_key ${raw.unit_key} is not a listed project`); + unitKey = group.project.unitKey; + } else { + return fail(`unit must be "machine" or "project"`); + } + if (raw.kind !== "env_fact" && raw.kind !== "project_constraint") return fail("kind must be env_fact or project_constraint"); + kind = raw.kind; + } else { + ids = [...new Set(raw.ids ?? [])]; + if (ids.length === 0) return fail("replace needs ids"); + const items = ids.map((id) => activeItems.get(id)); + const missing = ids.filter((_, i) => !items[i] || !groupUnits.has(items[i]!.unitKey)); + if (missing.length) return fail(`id ${missing.join(", ")} is not an active item of this group`); + const reused = ids.filter((id) => usedIds.has(id)); + if (reused.length) return fail(`id ${reused.join(", ")} is already changed by another op`); + unitKey = items[0]!.unitKey; + kind = items[0]!.kind; + if (items.some((item) => item!.unitKey !== unitKey || item!.kind !== kind)) { + return fail("replaced ids must share one unit and kind"); + } + } + const isMachine = unitKey === group.machine.unitKey; + if (isMachine && kind !== "env_fact") return fail("machine items must be env_fact"); + if (kind === "env_fact" && !evidence.some((id) => provesFact(fragments.get(id)!, unitKey, isMachine))) { + return fail("env_fact needs a cited tool output or error that touched this unit"); + } + if (isMachine && group.project && mentionsProject(content, group.project)) { + return fail("a machine item must not mention a project, repository or folder"); + } + ids.forEach((id) => usedIds.add(id)); + ops.push(raw.op === "add" + ? { op: "add", unitKey, kind, content, evidence } + : { op: "replace", unitKey, kind, ids, content, evidence }); + return; + } + + const id = raw.id ?? ""; + const item = activeItems.get(id); + if (!item || !groupUnits.has(item.unitKey)) return fail(`id ${id || "(missing)"} is not an active item of this group`); + if (usedIds.has(id)) return fail(`id ${id} is already changed by another op`); + usedIds.add(id); + ops.push({ op: raw.op, unitKey: item.unitKey, id, evidence }); + }); + return { ops, problems }; +} + +/** A tool call with output or an error, whose own paths belong to the unit (any call counts for the machine). */ +function provesFact(fragment: EvidenceFragment, unitKey: string, isMachine: boolean): boolean { + if (!fragment.toolName || !(fragment.isError || fragment.output?.trim())) return false; + return isMachine || fragment.candidateUnitKeys.includes(unitKey); +} + +function mentionsProject(content: string, project: L3UnitRecord): boolean { + const lower = content.toLowerCase(); + if (project.checkoutRoot && lower.includes(project.checkoutRoot.toLowerCase())) return true; + const name = project.displayName?.trim().toLowerCase(); + if (!name || name.length < MIN_REPOSITORY_NAME_LENGTH) return false; + const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`(^|[^\\p{L}\\p{N}_-])${escaped}($|[^\\p{L}\\p{N}_-])`, "u").test(lower); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} diff --git a/Memory/src/service/l3-field/unit-update-pipeline.ts b/Memory/src/service/l3-field/unit-update-pipeline.ts new file mode 100644 index 000000000..9b8635b6f --- /dev/null +++ b/Memory/src/service/l3-field/unit-update-pipeline.ts @@ -0,0 +1,429 @@ +import { canonicalJson, type JsonValue } from "../../contracts/index.js"; +import { createMemoryLogger } from "../../logging/logger.js"; +import { languageSteeringLine, steeredPromptLanguage, traceMetaFromMemory } from "../../algorithm/plugin-algorithms.js"; +import type { MemoryLanguage } from "../../config/index.js"; +import type { LlmClient } from "../../model/types.js"; +import { l3UnitKey } from "../../storage/l3-field-repository.js"; +import type { EvolutionJobRecord, Repositories } from "../../storage/repositories.js"; +import { nowIso } from "../../utils/time.js"; +import { evolutionJobLogFields, logEvolutionDecision } from "../evolution/evolution-logging.js"; +import { L3WorldModelTerminalEvidenceError } from "../evolution/l3-world-model-pipeline.js"; +import { storedGeneratedTitle } from "../read-model/display-fields.js"; +import { completeStrictJson } from "../l3-world-model/strict-json-completion.js"; +import { classifyProcessingError } from "../worker/job-handlers.js"; +import { buildEvidenceFragments } from "./evidence-fragments.js"; +import { resolveProject } from "./project-resolver.js"; +import { resolverContextFor, sessionWorkingDirectory, type L3FieldEnvironment } from "./session-units.js"; +import { countL3Units } from "./text-units.js"; +import type { EvidenceFragment, L3ItemKind, L3ItemRecord, L3UnitBatchRecord, L3UnitRecord, ResolvedProject } from "./types.js"; +import { L3_UNIT_UPDATE_PROMPT, L3_UNIT_UPDATE_SCHEMA, l3UnitRetryNote } from "./unit-update-prompt.js"; +import { parseL3UnitOutput, validateL3UnitOps, type L3UnitOp, type L3UpdateGroup } from "./unit-ops.js"; + +/** The evidence's turn and L1 still exist (neither deleted nor redacted since the input was built). */ +function evidenceStillThere(repos: Repositories, fragment: EvidenceFragment): boolean { + const rawTurn = repos.runtime.getRawTurn(fragment.rawTurnId); + const memory = repos.memories.get(fragment.l1MemoryId); + return Boolean(rawTurn && !rawTurn.deletedAt && !rawTurn.redactedAt && + memory && !memory.deletedAt && memory.status !== "deleted"); +} + +/** The run lost its job's lease to a newer run; it must leave the job and the batch alone. */ +export class L3LeaseLostError extends Error { + constructor(jobId: string) { + super(`l3 unit job ${jobId} was leased again; this run's result is dropped`); + this.name = "L3LeaseLostError"; + } +} + +interface PendingMachineItem { + id: string; + kind: L3ItemKind; + content: string; +} + +const l3FieldLogger = createMemoryLogger("l3-field"); +const MAX_INPUT_CHARS = 65_536; +/** After the model rejects an input as too long for its context, the group is asked once more with this much. */ +const FALLBACK_INPUT_CHARS = 16_384; +/** The output is a short list of ops; a smaller budget leaves room for the input in the context window. */ +const MAX_OUTPUT_TOKENS = 16_384; +const MAX_USER_PROJECTS = 50; +const MAX_ITEMS_PER_KIND = 10; +const OVER_LENGTH_UNITS = 150; +const L1_SUMMARY_CHARS = 600; + +/** + * Turns one frozen batch into item operations. Each group (the machine plus at most one project) + * gets one model call; nothing is written until every group has a valid answer, and then the + * whole batch is written in one transaction. + */ +export class L3UnitUpdatePipeline { + constructor(private readonly deps: { + repos: Repositories; + llm: LlmClient; + language?: MemoryLanguage; + environment: () => L3FieldEnvironment; + }) {} + + async process(job: EvolutionJobRecord): Promise { + const batchId = typeof job.payload.batchId === "string" ? job.payload.batchId : ""; + const batch = this.deps.repos.l3Field.getBatch(batchId); + if (!batch) throw new L3WorldModelTerminalEvidenceError(`missing l3 unit batch: ${batchId}`); + // A job recovered after a restart may run again for a batch that was already written. + if (batch.status === "applied") return; + if ( + batch.status !== "queued" || + job.userId !== batch.userId || + job.sessionId !== batch.sessionId || + job.scopeKey !== batch.machineUnitKey || + job.scopeSeq !== batch.scopeSeq + ) { + throw new L3WorldModelTerminalEvidenceError("l3 unit job does not match its batch"); + } + const machine = this.deps.repos.l3Field.getUnit(batch.machineUnitKey); + const session = this.deps.repos.runtime.getSession(batch.sessionId); + if (!machine || !session || session.userId !== batch.userId) { + throw new L3WorldModelTerminalEvidenceError("l3 unit batch scope mismatch"); + } + + const groups = this.buildGroups(batch, machine, session); + const decided: Array<{ group: L3UpdateGroup; ops: L3UnitOp[] }> = []; + // Machine items earlier groups of this batch decided to write; later groups see them so they + // do not write the same fact again in their own words. + const addedToMachine: PendingMachineItem[] = []; + for (const group of groups) { + const ops = await this.decide(job, group, addedToMachine); + decided.push({ group, ops }); + for (const op of ops) { + if ((op.op === "add" || op.op === "replace") && op.unitKey === group.machine.unitKey) { + addedToMachine.push({ id: `added-in-batch-${addedToMachine.length + 1}`, kind: op.kind, content: op.content }); + } + } + } + this.apply(job, batch, decided); + } + + private buildGroups( + batch: L3UnitBatchRecord, + machine: L3UnitRecord, + session: NonNullable> + ): L3UpdateGroup[] { + const repos = this.deps.repos; + const env = { ...this.deps.environment(), machineId: machine.machineId }; + const { cwd, isLocal } = sessionWorkingDirectory(session); + const ctx = resolverContextFor(repos, env, batch.userId, isLocal); + const sessionProject = cwd + ? resolveProject({ path: cwd, role: "session_dir", sessionCwd: cwd, sessionProjectRoot: null }, ctx) + : null; + const projects = new Map(); + const unitKeyOf = (project: ResolvedProject) => { + const key = l3UnitKey({ userId: batch.userId, machineId: machine.machineId, kind: "project", identity: project.identity }); + if (!projects.has(key)) projects.set(key, project); + return key; + }; + + const traces = repos.l3WorldModels.listInputTracesInRange(batch.sessionId, batch.startTraceSeq - 1, batch.endTraceSeq); + const l1ByRawTurn = new Map(); + for (const trace of traces) { + if (!batch.l1MemoryIds.includes(trace.l1MemoryId) || l1ByRawTurn.has(trace.rawTurnId)) continue; + const memory = repos.memories.get(trace.l1MemoryId); + if (memory && !memory.deletedAt && memory.status !== "deleted") l1ByRawTurn.set(trace.rawTurnId, memory.id); + } + + const fragments: EvidenceFragment[] = []; + for (const rawTurnId of batch.rawTurnIds) { + const rawTurn = repos.runtime.getRawTurn(rawTurnId); + const l1MemoryId = l1ByRawTurn.get(rawTurnId); + if (!rawTurn || !l1MemoryId || rawTurn.deletedAt || rawTurn.redactedAt) continue; + if (rawTurn.userId !== batch.userId || rawTurn.sessionId !== batch.sessionId) { + throw new L3WorldModelTerminalEvidenceError(`raw turn scope mismatch: ${rawTurnId}`); + } + fragments.push(...buildEvidenceFragments({ + rawTurn, + l1MemoryId, + sessionCwd: cwd, + sessionProjectRoot: sessionProject?.checkoutRoot ?? null, + resolve: (target, role) => resolveProject({ + path: target, + role, + sessionCwd: cwd, + sessionProjectRoot: sessionProject?.checkoutRoot ?? null + }, ctx), + unitKeyOf + })); + } + + const l1Entries = new Map([...new Set(l1ByRawTurn.values())].map((id) => { + const memory = repos.memories.get(id)!; + const summary = traceMetaFromMemory(memory)?.summary || memory.memoryValue; + return [id, { + id, + title: storedGeneratedTitle(memory) ?? "", + summary: [...summary].slice(0, L1_SUMMARY_CHARS).join(""), + at: memory.createdAt + }] as const; + })); + const groupOf = (picked: EvidenceFragment[], project: L3UnitRecord | null): L3UpdateGroup => ({ + machine, + project, + fragments: picked, + l1: [...new Set(picked.map((fragment) => fragment.l1MemoryId))].map((id) => l1Entries.get(id)!) + }); + + const groups: L3UpdateGroup[] = []; + for (const [unitKey, resolved] of projects) { + const picked = fragments.filter((fragment) => fragment.candidateUnitKeys.includes(unitKey)); + if (picked.length === 0) continue; + const unit = repos.l3Field.ensureProjectUnit({ userId: batch.userId, machineId: machine.machineId, project: resolved }); + groups.push(groupOf(picked, unit)); + } + const machineOnly = fragments.filter((fragment) => fragment.candidateUnitKeys.length === 0); + if (machineOnly.length > 0) groups.push(groupOf(machineOnly, null)); + return groups; + } + + private async decide( + job: EvolutionJobRecord, + group: L3UpdateGroup, + addedToMachine: readonly PendingMachineItem[] + ): Promise { + const repos = this.deps.repos; + const machineItems = repos.l3Field.activeItems(group.machine.unitKey); + // Removed on evidence newer than some of this group's: a batch processed late (rerun from dead + // letter, history scanned late) or spanning a removal must not bring them back on older evidence. + const oldestEvidenceAt = group.fragments.map((fragment) => fragment.at).sort()[0]; + const removedLater = oldestEvidenceAt + ? repos.l3Field.itemsRemovedAfter([group.machine.unitKey, ...(group.project ? [group.project.unitKey] : [])], oldestEvidenceAt) + : []; + const projectItems = group.project ? repos.l3Field.activeItems(group.project.unitKey) : []; + const activeItems = new Map([...machineItems, ...projectItems].map((item) => [item.id, item])); + let input = this.buildInput(job, group, machineItems, projectItems, addedToMachine, removedLater, MAX_INPUT_CHARS); + const addedIds = new Set(addedToMachine.map((item) => item.id)); + const language = activeItems.size === 0 + ? steeredPromptLanguage(this.deps.language, group.fragments.filter((f) => !f.toolName).map((f) => f.output)) + : undefined; + const systemPrompt = language ? `${L3_UNIT_UPDATE_PROMPT}\n\n${languageSteeringLine(language)}` : L3_UNIT_UPDATE_PROMPT; + const ask = (dynamicInput: JsonValue) => completeStrictJson({ + llm: this.deps.llm, + operation: "l3_field.unit_update", + systemPrompt, + dynamicInput, + expectedSchema: L3_UNIT_UPDATE_SCHEMA as unknown as JsonValue, + validate: parseL3UnitOutput, + maxTokens: MAX_OUTPUT_TOKENS + }); + + let first; + try { + first = await ask(input); + } catch (error) { + // Resending the same input would fail the same way; shrink it once instead. + if (classifyProcessingError(error).code !== "model_input_too_long") throw error; + logEvolutionDecision(job, "l3_unit_update", "input_too_long_shrunk", { maxChars: FALLBACK_INPUT_CHARS }); + input = this.buildInput(job, group, machineItems, projectItems, addedToMachine, removedLater, FALLBACK_INPUT_CHARS); + first = await ask(input); + } + const checked = validateL3UnitOps(first.ops, group, activeItems, addedIds); + if (checked.problems.length === 0) return checked.ops; + // Semantic problems get one regeneration with the reasons; a second failure fails the batch. + const second = await ask({ + ...input, + previous_output: first as unknown as JsonValue, + retry: l3UnitRetryNote(checked.problems) + }); + const rechecked = validateL3UnitOps(second.ops, group, activeItems, addedIds); + if (rechecked.problems.length > 0) { + throw new Error(`l3 unit output rejected twice: ${rechecked.problems.join("; ")}`); + } + return rechecked.ops; + } + + private buildInput( + job: EvolutionJobRecord, + group: L3UpdateGroup, + machineItems: readonly L3ItemRecord[], + projectItems: readonly L3ItemRecord[], + addedToMachine: readonly PendingMachineItem[], + removedLater: readonly L3ItemRecord[], + maxChars: number + ): Record { + const repos = this.deps.repos; + const session = repos.runtime.getSession(job.sessionId!)!; + const { cwd } = sessionWorkingDirectory(session); + const items = (list: readonly L3ItemRecord[]) => list.map((item) => ({ + id: item.id, + kind: item.kind, + content: item.content, + last_confirmed_at: item.lastConfirmedAt + })); + const evidence = group.fragments.map((fragment) => ({ + id: fragment.id, + type: fragment.toolName ? "tool" : "user_text", + ...(fragment.toolName + ? { tool: fragment.toolName, input: (fragment.input ?? null) as JsonValue, is_error: fragment.isError } + : {}), + ...(fragment.toolName ? { output: fragment.output } : { text: fragment.output }), + paths: fragment.paths, + candidate_units: ["machine", ...fragment.candidateUnitKeys], + at: fragment.at + })); + const input: Record = { + batch_l1: group.l1, + evidence, + machine: { + profile: group.machine.profileText ?? null, + items: [ + ...items(machineItems), + ...addedToMachine.map((item) => ({ id: item.id, kind: item.kind, content: item.content, added_in_this_batch: true })) + ] + }, + projects: group.project + ? [{ + unit_key: group.project.unitKey, + name: group.project.displayName ?? null, + source: group.project.identitySource ?? null, + root: group.project.checkoutRoot ?? null, + profile: group.project.profileText ?? null, + items: items(projectItems) + }] + : [], + // Only when some of this group's evidence is older than a removal: what was removed, and when. + ...(removedLater.length > 0 + ? { removed_by_newer_evidence: removedLater.map((item) => ({ + kind: item.kind, content: item.content, removed_at: item.removedEvidenceAt ?? null + })) } + : {}), + session: { cwd, current_project: group.project?.displayName ?? null }, + user_selected_projects: repos.l3Field.listUserProjects(job.userId) + .slice(0, MAX_USER_PROJECTS) + .map((project) => ({ name: project.name, root: project.rootPath })) + }; + // When the input is too long the oldest tool calls go first, the user's own words last; + // L1, items and profiles are kept. + const charsBefore = canonicalJson(input).length; + const evidenceBefore = evidence.length; + let droppedTool = 0; + let droppedUserText = 0; + while (canonicalJson(input).length > maxChars && evidence.length > 1) { + const oldestTool = evidence.findIndex((fragment) => fragment.type === "tool"); + const [removed] = evidence.splice(oldestTool >= 0 ? oldestTool : 0, 1); + if (removed?.type === "tool") droppedTool += 1; + else droppedUserText += 1; + } + if (droppedTool + droppedUserText > 0) { + l3FieldLogger.debug("input.truncated", { + ...evolutionJobLogFields(job), + unitKey: group.project?.unitKey ?? group.machine.unitKey, + maxChars, + charsBefore, + charsAfter: canonicalJson(input).length, + evidenceBefore, + evidenceAfter: evidence.length, + droppedToolFragments: droppedTool, + droppedUserTextFragments: droppedUserText + }); + } + return input; + } + + private apply(job: EvolutionJobRecord, batch: L3UnitBatchRecord, decided: Array<{ group: L3UpdateGroup; ops: L3UnitOp[] }>): void { + const repos = this.deps.repos; + // Two groups changing the same machine item: the first group's op wins. + const claimedMachineIds = new Set(); + const plan = decided.map(({ group, ops }) => ({ + group, + ops: ops.filter((op) => { + if (op.unitKey !== group.machine.unitKey || op.op === "add") return true; + const ids = op.op === "replace" ? op.ids : [op.id]; + if (ids.some((id) => claimedMachineIds.has(id))) { + logEvolutionDecision(job, "l3_unit_update", "machine_item_conflict_dropped", { ids }); + return false; + } + ids.forEach((id) => claimedMachineIds.add(id)); + return true; + }) + })); + + repos.transaction(() => { + // A run that outlived its lease (the job was leased again since) writes nothing. + if (!repos.runtime.holdsJobLease(job)) throw new L3LeaseLostError(job.id); + const status = repos.l3Field.getBatch(batch.id)?.status; + if (status === "applied") return; + // A run that outlived its lease can come back after a newer run of the same job dead-lettered + // the batch, and later batches may have changed the items since: its result is dropped (nothing + // is written) and the batch waits in dead letter for its rerun. + if (status !== "queued") throw new L3WorldModelTerminalEvidenceError(`l3 unit batch is ${status ?? "missing"}; result dropped`); + const at = nowIso(); + const touched = new Set(); + for (const { group, ops } of plan) { + const fragments = new Map(group.fragments.map((fragment) => [fragment.id, fragment])); + for (const op of ops) { + const unit = repos.l3Field.getUnit(op.unitKey); + if (!unit) continue; + const cited = op.evidence.map((id) => fragments.get(id)!); + // The user may have deleted a turn or its L1 while the model was answering: an op that + // cites it is dropped, as it would have been had the deletion come before. + if (cited.some((fragment) => !evidenceStillThere(repos, fragment))) { + logEvolutionDecision(job, "l3_unit_update", "deleted_evidence_op_dropped", { op: op.op }); + continue; + } + const evidenceAt = cited.map((fragment) => fragment.at).sort().at(-1) ?? at; + // A unit reset must not be refilled by what was said before the reset, even when that + // conversation is frozen later (a task closed late, or a batch rerun from dead letter). + if (unit.deletedAt && (batch.createdAt <= unit.deletedAt || evidenceAt <= unit.deletedAt)) continue; + // Older evidence must not overturn an item that newer evidence wrote or confirmed (a batch + // rerun from dead letter is applied after the batches that came after it), and an item + // already gone is not changed again. + if (op.op === "replace" || op.op === "delete") { + const targets = (op.op === "replace" ? op.ids : [op.id]).map((id) => repos.l3Field.getItem(id)); + if (targets.some((item) => !item || item.status !== "active" || item.lastConfirmedAt > evidenceAt)) { + logEvolutionDecision(job, "l3_unit_update", "stale_op_dropped", { op: op.op, evidenceAt }); + continue; + } + } + const evidence = { + fragmentIds: op.evidence, + l1MemoryIds: [...new Set(cited.map((fragment) => fragment.l1MemoryId))] + }; + if (op.op === "confirm") { + repos.l3Field.confirmItem(op.id, evidenceAt, at); + } else if (op.op === "delete") { + repos.l3Field.inactivateItem(op.id, { removedEvidenceAt: evidenceAt, at }); + } else { + const created = repos.l3Field.insertItem({ + unitKey: op.unitKey, kind: op.kind, content: op.content, evidence, evidenceAt, batchId: batch.id, at + }); + if (op.op === "replace") { + for (const id of op.ids) repos.l3Field.inactivateItem(id, { replacedBy: created.id, removedEvidenceAt: evidenceAt, at }); + } + if (countL3Units(op.content) > OVER_LENGTH_UNITS) { + logEvolutionDecision(job, "l3_unit_update", "item_over_length", { itemId: created.id }); + } + } + touched.add(op.unitKey); + } + } + for (const unitKey of touched) this.enforceLimit(job, unitKey, at); + repos.l3Field.setBatchStatus(batch.id, "applied", at); + // Refreshing the machine also refreshes every project snapshot on it. + if (touched.has(batch.machineUnitKey)) repos.l3Field.refreshSnapshot(batch.machineUnitKey, at); + for (const unitKey of touched) { + if (unitKey !== batch.machineUnitKey) repos.l3Field.refreshSnapshot(unitKey, at); + } + }); + } + + /** Over the limit, the items confirmed longest ago go first. */ + private enforceLimit(job: EvolutionJobRecord, unitKey: string, at: string): void { + const items = this.deps.repos.l3Field.activeItems(unitKey); + for (const kind of ["env_fact", "project_constraint"] as L3ItemKind[]) { + const ofKind = items + .filter((item) => item.kind === kind) + .sort((a, b) => a.lastConfirmedAt.localeCompare(b.lastConfirmedAt) || a.createdAt.localeCompare(b.createdAt)); + for (const item of ofKind.slice(0, Math.max(0, ofKind.length - MAX_ITEMS_PER_KIND))) { + this.deps.repos.l3Field.inactivateItem(item.id, { at }); + logEvolutionDecision(job, "l3_unit_update", "item_evicted_over_limit", { itemId: item.id, unitKey }); + } + } + } +} diff --git a/Memory/src/service/l3-field/unit-update-prompt.ts b/Memory/src/service/l3-field/unit-update-prompt.ts new file mode 100644 index 000000000..ebfdd4e51 --- /dev/null +++ b/Memory/src/service/l3-field/unit-update-prompt.ts @@ -0,0 +1,141 @@ +/** System prompt of the new L3 item update; approved draft: memmy-流程页面/L3提示词草稿.md (2026-10-24, polished 2026-10-25). */ +export const L3_UNIT_UPDATE_PROMPT = `You maintain "field cognition" for one computer and the projects on it: short, itemized notes about +what this computer and these projects are like, so that a future Agent session here starts out knowing them. + +The input contains: +- batch_l1: summaries of the turns that just happened, oldest first. Read them to understand what happened. +- evidence: raw fragments of those turns (the user's own words, and each tool call with its output). + Every item you write must cite them. Something that appears only in batch_l1 and not in evidence + cannot be written. +- machine: this computer's profile (read-only) and its current items. +- projects: each project touched by this batch, with its profile (read-only) and current items. +- removed_by_newer_evidence (only sometimes): items another conversation removed or replaced, with when that + evidence happened (removed_at). Do not write their content back on evidence older than removed_at; only + evidence after it can bring it back. +- session: the session's working folder and its current project, if any. +- user_selected_projects: folders the user has chosen as projects in the desktop app (read-only context). + +Field cognition has three parts: the environment profile, env_fact items and project_constraint items. +The profile is produced by code from scanning the computer and the project; it is given to you read-only. +You maintain only the two kinds of items. + +The most important test: write only what would still hold if a different person worked on this computer +or this project tomorrow. Field cognition records how the computer and the project themselves behave, not +what changes with the person working on them (their preferences, habits, current plans or mistakes), and +not what belongs to the Agent doing the work. + +1. env_fact — something an Agent will run into when working here, proven by a tool output or error. + Write it as "condition -> observable result". + Machine example: "the external drive /Volumes/data is mounted read-only -> write outputs elsewhere". + Project example: "git clone over SSH fails for this repo -> use the https remote". + Write a fact only if a future Agent here would act better for knowing it and could not easily find it + out on its own: it differs from what a competent Agent would expect. One occurrence is enough when a + tool output or error shows it. + Do NOT write: + - how a language, library, tool, model or online service behaves wherever it is used (general + knowledge), and results that only confirm normal, expected behavior. Ask: would the same thing + happen on any other computer with the same software and settings? If yes, it is not a fact about + this computer or project; + - limits of the Agent's own runtime: its sandbox, its permission or approval mode, the shell + environment it runs commands in, and how its own tools behave. Another Agent or a person working on + this computer would not meet them. When evidence from different Agents disagrees about the same + behavior, it is one of these: do not write it, and delete an existing item about it; + - what comes from whoever is working rather than from the environment: their own mistakes, choices + and plans; + - temporary states: what one ordinary action would change — something not installed yet, a file + that happened to be missing, the current branch — or what will be different next time; + - what is learned by reading content (the code, a document, a web page) rather than met when working here; + - guessed causes no output shows, advice without an observed result, or what the profile already + covers (versions, installed tools, OS, shell, stack, folder layout). + +2. project_constraint — a rule of this project that the user expects to hold for future tasks here. + Test: "next time someone does a different task in this project, must they still follow it?" + Examples: "Reports follow main on GitHub; pull before editing." "In the review folder, each paper + gets its own subfolder." + These are lasting rules: write them. + - How work in this project is done: how it is organized, what is authoritative, which conventions it + follows (what things are called, what is in or out of scope, which methods are used), and how + results are delivered. + - A requirement the user repeats across different tasks in this project. + Do NOT write: + - what only the current task or its deliverable needs, including the current plan of work; + - preferences of the person, or ways of working that are not specific to this project; + - anything the Agent decided on its own; + - what the work itself produces: its findings, conclusions, explanations or current state, even when + the user stated them. A rule changes only when the user changes it; what the work produces changes + as the work goes on. + If you cannot tell whether a requirement is meant to last, check the user's own words in evidence; + if it is still unclear, do not write it. + +Where an item goes: +- Machine items are only about this computer itself: its system, hardware, settings, and what is + installed on it or reachable from it. A machine item must not mention a specific project, repository + or folder. Machine items are env_fact only. +- A website, service, data source or tool that only some work uses is not part of this computer: its + behavior belongs to the project that uses it. +- project_constraint always goes to a project. An env_fact about a project goes to the project whose + files the evidence actually touched, which may differ from the session folder. +- Each evidence fragment lists its candidate_units; the target unit must be one of them. +- If something belongs neither to this computer nor to a listed project, leave it out; never put it on + the machine instead. + +Operations: +- add: a new item. +- replace: newer evidence changes or corrects existing items. The new content replaces them; list every + replaced id. All replaced ids must be the same unit and kind. Use this also to merge duplicates. +- delete: newer evidence shows an existing item is no longer true, and nothing replaces it. +- confirm: the batch shows an existing item is still true. It only refreshes when the item was last seen. +An empty list is the normal result: output only what changes. When new evidence conflicts with an +existing item, the newer evidence wins: replace or delete the old item. Replace an item only when its +meaning materially changes, never just to reword it; items this batch does not bear on stay as they are. +Two items that would lead an Agent to the same action are duplicates: keep the more precise one. + +Size: +- Each item is one or two sentences, about 60 characters (or 30 Chinese characters) at most. +- Each unit keeps at most 10 env_fact items, and a project at most 10 project_constraint items. + When an add would exceed that, first merge similar items with replace. If code still finds too many, + it drops the ones confirmed longest ago. + +Evidence: every operation carries "evidence", ids of fragments from this input that support it; use only +ids that appear in the input. An env_fact add or replace must cite at least one fragment containing a +tool output or error. + +Before you output, check every add and replace once more: +1. Would it still hold if a different person worked here tomorrow? +2. Is it about this computer or this project in particular — not true everywhere, and not about the + Agent itself? +3. Would it still hold after one ordinary action (installing, creating, switching)? +4. Is it in the right unit? +Drop any op that fails one of these. + +Return exactly one valid JSON object, no Markdown. Give a short "reason" (not stored) first, saying what +in this batch is worth keeping and what you left out: +{"reason":"...","ops":[ ... ]} +where each op is one of: +{"op":"add","unit":"machine","kind":"env_fact","content":"...","evidence":["f..."]} +{"op":"add","unit":"project","unit_key":"","kind":"env_fact"|"project_constraint","content":"...","evidence":["f..."]} +{"op":"replace","ids":["", ...],"content":"...","evidence":["f..."]} +{"op":"delete","id":"","evidence":["f..."]} +{"op":"confirm","id":"","evidence":["f..."]} +"unit_key" may be omitted only when exactly one project is listed. + +Write item content in the language of the existing items. When there are none, follow the interface language when one is pinned; otherwise use the dominant language of the user's own words in evidence. Do not translate content merely because this instruction is written in English.`; + +/** Added to the input when the previous output failed validation; the model regenerates once. */ +export function l3UnitRetryNote(problems: readonly string[]): string { + return [ + "Your previous output was rejected:", + ...problems.map((problem) => `- ${problem}`), + "Return the whole JSON again with these problems fixed. Drop an op if it cannot be fixed." + ].join("\n"); +} + +export const L3_UNIT_UPDATE_SCHEMA = { + reason: "string", + ops: [ + { op: "add", unit: "machine|project", unit_key: "string (optional)", kind: "env_fact|project_constraint", content: "string", evidence: ["fragment id"] }, + { op: "replace", ids: ["item id"], content: "string", evidence: ["fragment id"] }, + { op: "delete", id: "item id", evidence: ["fragment id"] }, + { op: "confirm", id: "item id", evidence: ["fragment id"] } + ] +} as const; diff --git a/Memory/src/service/l3-world-model/strict-json-completion.ts b/Memory/src/service/l3-world-model/strict-json-completion.ts index 3142b2302..bbbd854d5 100644 --- a/Memory/src/service/l3-world-model/strict-json-completion.ts +++ b/Memory/src/service/l3-world-model/strict-json-completion.ts @@ -18,6 +18,8 @@ export interface StrictJsonCompletionInput { dynamicInput: JsonValue; expectedSchema: JsonValue; validate(value: unknown): T; + /** Output budget; defaults to L3_WORLD_MODEL_MAX_OUTPUT_TOKENS. */ + maxTokens?: number; } /** Runs one strict JSON completion and at most one model-based schema repair. */ @@ -26,7 +28,7 @@ export async function completeStrictJson(input: StrictJsonCompletionInput) { role: "system", content: input.systemPrompt }, { role: "user", content: canonicalJson(input.dynamicInput) } ]; - const candidate = await input.llm.complete(messages, completionOptions(input.operation)); + const candidate = await input.llm.complete(messages, completionOptions(input.operation, input.maxTokens)); try { return parseAndValidate(candidate, input.validate); } catch (error) { @@ -41,16 +43,16 @@ export async function completeStrictJson(input: StrictJsonCompletionInput) validation_error: validationErrorMessage(error) }) } - ], completionOptions(`${input.operation}.repair`)); + ], completionOptions(`${input.operation}.repair`, input.maxTokens)); return parseAndValidate(repaired, input.validate); } } -function completionOptions(operation: string) { +function completionOptions(operation: string, maxTokens = L3_WORLD_MODEL_MAX_OUTPUT_TOKENS) { return { operation, temperature: 0, - maxTokens: L3_WORLD_MODEL_MAX_OUTPUT_TOKENS, + maxTokens, jsonMode: true } as const; } diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index 393da3a53..f4a374397 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -13,6 +13,9 @@ import { isL3FieldCognitionActive } from "./l3-field/flag.js"; import { loadL3FieldContext } from "./l3-field/l3-field-context.js"; import type { L3FieldEnvironment } from "./l3-field/session-units.js"; import { l3FieldSnapshotUnitKey } from "./l3-field/snapshot-memory.js"; +import { resolveSessionUnits } from "./l3-field/session-units.js"; +import { machineProfileFingerprint, probeMachine, renderMachineProfile, type MachineProbe } from "./l3-field/machine-profile.js"; +import { L3UnitUpdatePipeline } from "./l3-field/unit-update-pipeline.js"; import { languageSteeringLine, skillOutputLanguageFor, @@ -229,6 +232,8 @@ export interface MemoryServiceOptions { mode?: "local" | "cloud" | "dev"; /** Machine identity and home folder for the new L3; defaults to this installation. Tests override it. */ l3FieldEnvironment?: () => L3FieldEnvironment; + /** Probes this computer for the new L3 machine profile; tests override it. */ + l3MachineProbe?: () => Promise; configPath?: string; configLoader?: (configPath?: string) => { config: MemmyConfig; @@ -296,6 +301,8 @@ export class MemoryService { private readonly importJobs: ImportJobProcessor; private readonly l3WorldModelContextReadModel: L3WorldModelContextReadModel; private readonly projectEnvironment: ProjectEnvironmentService; + private readonly l3UnitUpdate: L3UnitUpdatePipeline; + private readonly l3MachineProfileRefreshedAt = new Map(); private readonly panelReadModel: PanelReadModel; private readonly retrieval: RetrievalService; private readonly sessionTurns: SessionTurnService; @@ -372,6 +379,15 @@ export class MemoryService { get llm() { return projectEnvironmentOwner.skillLlm; }, get language() { return projectEnvironmentOwner.config.language; } }); + this.l3UnitUpdate = new L3UnitUpdatePipeline({ + repos: this.repos, + get llm() { return projectEnvironmentOwner.skillLlm; }, + get language() { return projectEnvironmentOwner.config.language; }, + environment: () => projectEnvironmentOwner.l3FieldEnvironment() + }); + this.repos.l3WorldModels.routeBatchesToFieldCognition( + () => projectEnvironmentOwner.l3FieldActive() ? projectEnvironmentOwner.l3FieldEnvironment().machineId : null + ); const episodeTitleOwner = this; this.episodeTitle = new EpisodeTitleService({ repos: this.repos, @@ -402,6 +418,7 @@ export class MemoryService { abstractL3: (job) => this.evolutionJobs.abstractL3(job), updateL3WorldModel: (job) => this.evolutionJobs.updateL3WorldModel(job), updateProjectEnvironment: (job) => this.projectEnvironment.processProfileJob(job), + updateL3Unit: (job) => this.l3UnitUpdate.process(job), crystallizeSkill: (job) => this.evolutionJobs.crystallizeSkill(job), assignSkillCluster: (job) => this.evolutionJobs.assignSkillCluster(job), evolveSkillCluster: (job) => this.evolutionJobs.evolveSkillCluster(job), @@ -575,6 +592,7 @@ export class MemoryService { runWorkerNoWrite: this.runWorkerNoWrite.bind(this), restartFailedProcessing: this.restartFailedProcessing.bind(this), enqueueObservationBackfill: (limit, at) => this.observationMemory.backfill(limit, at), + l3FieldActive: () => this.l3FieldActive(), purgeL3InactiveItems: (at) => this.repos.l3Field.purgeInactive( new Date(Date.parse(at) - this.config.algorithm.l3FieldCognition.inactiveRetentionDays * 86_400_000).toISOString() ), @@ -941,6 +959,42 @@ export class MemoryService { private l3FieldEnvironmentCache?: L3FieldEnvironment; + /** New L3 profiles at session start and compaction: rescan the session's project, re-probe the machine. */ + private requestL3FieldProfiles(session: SessionRecord): void { + if (!this.memoryAddEnabled()) return; + const units = resolveSessionUnits(this.repos, this.l3FieldEnvironment(), session); + if (units.project?.unit.checkoutRoot) { + this.projectEnvironment.requestL3UnitProfile({ + userId: session.userId, + unitKey: units.project.unit.unitKey, + checkoutRoot: units.project.unit.checkoutRoot + }); + } + void this.refreshL3MachineProfile(units.machine.unitKey); + } + + /** At most once per ten minutes per machine; the profile is rewritten only when the probe changed. */ + private async refreshL3MachineProfile(unitKey: string): Promise { + const now = Date.now(); + if (now - (this.l3MachineProfileRefreshedAt.get(unitKey) ?? 0) < 10 * 60_000) return; + this.l3MachineProfileRefreshedAt.set(unitKey, now); + try { + // A reset (snapshot deleted) while probing makes this probe stale; it must not bring the profile back. + const resetBefore = this.repos.l3Field.getUnit(unitKey)?.deletedAt; + const probe = await (this.options.l3MachineProbe ?? probeMachine)(); + const fingerprint = machineProfileFingerprint(probe); + this.repos.transaction(() => { + const unit = this.repos.l3Field.getUnit(unitKey); + if (!unit || unit.deletedAt !== resetBefore || unit.profileFingerprint === fingerprint) return; + this.repos.l3Field.setProfile(unitKey, { text: renderMachineProfile(probe), fingerprint }); + this.repos.l3Field.refreshSnapshot(unitKey); + }); + } catch { + // A failed probe leaves the old profile; the next session start tries again. + this.l3MachineProfileRefreshedAt.delete(unitKey); + } + } + private l3FieldEnvironment(): L3FieldEnvironment { this.l3FieldEnvironmentCache ??= this.options.l3FieldEnvironment?.() ?? { machineId: deriveWorkspaceHostId(getOrCreateInstallationId({ env: process.env })), @@ -1268,7 +1322,9 @@ export class MemoryService { serverTime: string; } { const response = this.sessionTurns.openSession(this.withTimeZone(request)); - if (this.projectEnvironmentScanEnabled() && response.projectId) { + if (this.l3FieldActive()) { + this.requestL3FieldProfiles(this.requireSession(response.sessionId)); + } else if (this.projectEnvironmentScanEnabled() && response.projectId) { const session = this.requireSession(response.sessionId); const scope = this.repos.l3WorldModels.getScope(session.userId, response.projectId); if (scope?.workspaceUri && isLocalWorkspaceUri(scope.workspaceUri)) { @@ -1344,7 +1400,9 @@ export class MemoryService { if (!result.throughTraceSeq) { throw new MemoryServiceError("conflict", "through L1 memory was not registered"); } - if ( + if (request.trigger === "token_compaction" && this.l3FieldActive()) { + this.requestL3FieldProfiles(session); + } else if ( request.trigger === "token_compaction" && this.projectEnvironmentScanEnabled() && session.projectId @@ -2476,7 +2534,14 @@ export class MemoryService { this.assertMemoryInScope(memory, request.namespace); const kind = kindFromMemory(memory); const at = nowIso(); - const archived = this.repos.memories.archive(memory.id, at); + const archived = this.repos.transaction(() => { + const row = this.repos.memories.archive(memory.id, at); + // A project's session context includes the machine, so archiving the machine's new-L3 + // snapshot moves the project versions too, or clients keep using their cached context. + const l3Unit = row ? l3FieldSnapshotUnitKey(memory) : null; + if (l3Unit) this.repos.l3Field.refreshProjectContexts(l3Unit, at); + return row; + }); if (!archived) { throw new MemoryServiceError("not_found", `memory not found: ${id}`); } diff --git a/Memory/src/service/worker/job-handlers.ts b/Memory/src/service/worker/job-handlers.ts index 8f0642975..cd62bfb23 100644 --- a/Memory/src/service/worker/job-handlers.ts +++ b/Memory/src/service/worker/job-handlers.ts @@ -68,6 +68,7 @@ export interface WorkerJobProcessors { abstractL3(job: EvolutionJobRecord): MaybePromise; updateL3WorldModel(job: EvolutionJobRecord): MaybePromise; updateProjectEnvironment(job: EvolutionJobRecord): MaybePromise; + updateL3Unit(job: EvolutionJobRecord): MaybePromise; crystallizeSkill(job: EvolutionJobRecord): MaybePromise; assignSkillCluster(job: EvolutionJobRecord): MaybePromise; evolveSkillCluster(job: EvolutionJobRecord): MaybePromise; @@ -96,7 +97,7 @@ export interface WorkerJobProcessors { } export interface WorkerJobHandlerDeps { - repos: Pick; + repos: Pick; capture: { synthReflection: boolean }; reward: { feedbackWindowSec: number }; nowIso(): string; @@ -272,6 +273,16 @@ export async function processJob( case "project_environment_profile": await deps.processors.evolution.updateProjectEnvironment(job); return; + case "l3_unit_update": + try { + await deps.processors.evolution.updateL3Unit(job); + } catch (error) { + if (isTerminalL3WorldModelError(error) && deps.repos.runtime.holdsJobLease(job)) { + deps.repos.runtime.failJob(job.id, error instanceof Error ? error.message : String(error), deps.nowIso(), true); + } + throw error; + } + return; case "skill_crystallization": await deps.processors.evolution.crystallizeSkill(job); return; @@ -383,7 +394,15 @@ export function finalizeClosedEpisode( trigger: ClosedEpisodeTrigger ): EvolutionJobRecord[] { const current = deps.repos.runtime.getEpisode(episode.id) ?? episode; - if (current.status !== "closed" || current.l1MemoryIds.length === 0) return []; + if (current.status !== "closed") return []; + // Every way a task closes ends up here, so this is where its turns go to the new L3. + deps.repos.l3WorldModels.freezeClosedTask({ + sessionId: current.sessionId, + episodeId: current.id, + trigger: trigger === "idle_timeout" ? "episode_idle_close" : trigger === "session_closed" ? "session_close" : "new_task", + at + }); + if (current.l1MemoryIds.length === 0) return []; if (episodeHasPendingCaptureDecision(deps, current)) return []; // Titling is independent of reward and reflection, so it must be queued before // the mutually exclusive branches below can return. @@ -562,6 +581,7 @@ export function workerJobCanRunInParallel(job: EvolutionJobRecord): boolean { job.jobType === "observation_extract" || job.jobType === "embedding" || job.jobType === "l3_world_model_update" || + job.jobType === "l3_unit_update" || job.jobType === "project_environment_profile"; } diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index 9e92c1f55..7384c13d0 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -135,6 +135,8 @@ export interface WorkerRunnerDeps { enqueueObservationBackfill?: (limit: number, at: string) => number; /** Drops new-L3 items that have been inactive longer than the retention window. */ purgeL3InactiveItems?: (at: string) => number; + /** Whether the new L3 is on; while off its jobs stay queued and are not leased. */ + l3FieldActive?: () => boolean; previewPolicyEvidenceReconciliation: (limit: number) => PolicyEvidencePreflightReport; reconcileOrphanedPolicies: (at: string, limit: number) => number; enqueueJob: (input: EnqueueJobInput) => EvolutionJobRecord; @@ -174,12 +176,14 @@ export class WorkerRunner { return undefined; } const reconcileAt = this.deps.memoryBudgetNextReconcileAtMs?.(); - const heldSummaryJobs = this.summaryJobsHeld(); + // Wake only for jobs leasing would actually take, or a held job's runAfter spins the worker. + const heldJobs = this.excludedLeaseJobTypes(); if (this.deps.memoryBudgetPaused()) { const allowed = allowedMemoryBudgetJobTypes(this.deps.memoryBudgetModelSources()) - .filter((jobType) => !heldSummaryJobs?.includes(jobType)); + .filter((jobType) => !heldJobs?.includes(jobType)); const allowedAt = this.deps.repos.runtime.nextWorkerRunAt({ jobTypes: allowed, + ...(heldJobs ? { excludedJobTypes: heldJobs } : {}), includeEmbeddingRetries: !this.deps.memoryBudgetJobConsumes("embedding") }); const midnight = this.deps.memoryBudgetNextWakeAtMs(); @@ -187,7 +191,7 @@ export class WorkerRunner { return times.length > 0 ? Math.min(...times) : undefined; } const scheduled = this.deps.repos.runtime.nextWorkerRunAt( - heldSummaryJobs ? { excludedJobTypes: heldSummaryJobs } : undefined + heldJobs ? { excludedJobTypes: heldJobs } : undefined ); const times = [scheduled, reconcileAt].filter((time): time is number => Number.isFinite(time)); return times.length > 0 ? Math.min(...times) : undefined; @@ -440,7 +444,7 @@ export class WorkerRunner { targetMemoryIds, request.priorityCohortOnly, allowedWhenPaused, - this.summaryJobsHeld() + this.excludedLeaseJobTypes() ); const retryCapacity = Math.max(0, normalizedLimit - jobs.length); const results: WorkerJobRunResult[] = []; @@ -601,6 +605,7 @@ export class WorkerRunner { } completeLeasedWorkerJob(job: EvolutionJobRecord): WorkerJobRunResult { + if (!this.deps.repos.runtime.holdsJobLease(job)) return this.leaseLostResult(job); const completed = this.deps.repos.runtime.completeJob(job.id) ?? { ...job, status: "succeeded" as const, @@ -616,7 +621,15 @@ export class WorkerRunner { }; } + /** A run that outlived its lease leaves the job to the run that holds it now. */ + private leaseLostResult(job: EvolutionJobRecord): WorkerJobRunResult { + workerLogger.warn("job.lease_lost", workerJobLogFields(job)); + const current = this.deps.repos.runtime.getJob(job.id) ?? job; + return { succeeded: 0, failed: 0, ref: { ...jobToRef(current), status: current.status } }; + } + failLeasedWorkerJob(job: EvolutionJobRecord, error: unknown): WorkerJobRunResult { + if (!this.deps.repos.runtime.holdsJobLease(job)) return this.leaseLostResult(job); if (error instanceof ObservationDependenciesPendingError) { const at = this.deps.nowIso(); const deferred = this.deps.repos.runtime.deferProcessingJob(job.id, new Date(Date.parse(at) + 30_000).toISOString(), at) ?? job; @@ -891,6 +904,13 @@ export class WorkerRunner { return { succeeded: 0, failed: 1, item: null }; } + private excludedLeaseJobTypes(): readonly string[] | undefined { + const held = this.summaryJobsHeld() ?? []; + // Excluding l3_unit_update also keeps new-L3 profile jobs in the queue (see leaseQueuedJobs). + const excluded = this.deps.l3FieldActive?.() === false ? [...held, "l3_unit_update"] : held; + return excluded.length > 0 ? excluded : undefined; + } + private summaryJobsHeld(): readonly string[] | undefined { return this.deps.summaryModelConfigured() ? undefined : SUMMARY_JOBS_REQUIRING_MODEL; } diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts index 9ed1bd017..13690a1a7 100644 --- a/Memory/src/storage/l3-field-repository.ts +++ b/Memory/src/storage/l3-field-repository.ts @@ -4,6 +4,8 @@ import type { L3ItemEvidence, L3ItemKind, L3ItemRecord, + L3UnitBatchRecord, + L3UnitBatchStatus, L3UnitKind, L3UnitRecord, L3UserProject, @@ -154,6 +156,22 @@ export class L3FieldRepository { ).all(unitKey) as SqlItemRow[]).map(itemFromSql); } + /** + * Items of these units that a delete or replace removed on evidence newer than `evidenceAt`, + * newest last. Empty when all of a batch's evidence is newer (the usual case); a batch rerun late, + * history scanned late, or a batch spanning a removal sees what was removed. + */ + itemsRemovedAfter(unitKeys: readonly string[], evidenceAt: string, limit = 20): L3ItemRecord[] { + if (unitKeys.length === 0) return []; + return (this.db.prepare( + `SELECT * FROM l3_items + WHERE unit_key IN (${unitKeys.map(() => "?").join(", ")}) + AND status = 'inactive' AND removed_evidence_at > ? + ORDER BY removed_evidence_at DESC, id DESC + LIMIT ?` + ).all(...unitKeys, evidenceAt, limit) as SqlItemRow[]).map(itemFromSql).reverse(); + } + getItem(id: string): L3ItemRecord | undefined { const row = this.db.prepare(`SELECT * FROM l3_items WHERE id = ?`).get(id) as SqlItemRow | undefined; return row ? itemFromSql(row) : undefined; @@ -220,18 +238,31 @@ export class L3FieldRepository { refreshSnapshot(unitKey: string, at = nowIso()): MemoryRow { const unit = this.getUnit(unitKey); if (!unit) throw new Error(`L3 unit not found: ${unitKey}`); + const before = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; const saved = this.writeSnapshot(unit, at); - if (unit.kind === "machine") { - const projects = (this.db.prepare( - `SELECT * FROM l3_units - WHERE user_id = ? AND machine_id = ? AND kind = 'project' - AND snapshot_memory_id IS NOT NULL AND deleted_at IS NULL` - ).all(unit.userId, unit.machineId) as SqlUnitRow[]).map(unitFromSql); - for (const project of projects) this.writeSnapshot(project, at); - } + const changed = saved.id !== before?.id || saved.version !== before?.version; + if (unit.kind === "machine" && changed) this.bumpProjectSnapshots(unit, at); return saved; } + /** After a machine unit's snapshot changed outside a refresh (archived), its projects get new versions. */ + refreshProjectContexts(unitKey: string, at = nowIso()): void { + const unit = this.getUnit(unitKey); + if (unit?.kind === "machine") this.bumpProjectSnapshots(unit, at); + } + + /** + * Every project on the machine with a current snapshot gets a new version. A project reset + * earlier keeps its `deleted_at` (it guards old batches) but counts again once rebuilt. + */ + private bumpProjectSnapshots(machine: L3UnitRecord, at: string): void { + const projects = (this.db.prepare( + `SELECT * FROM l3_units + WHERE user_id = ? AND machine_id = ? AND kind = 'project' AND snapshot_memory_id IS NOT NULL` + ).all(machine.userId, machine.machineId) as SqlUnitRow[]).map(unitFromSql); + for (const project of projects) this.writeSnapshot(project, at, true); + } + /** * Deleting a unit's L3 resets it rather than disabling it: items become * inactive, the profile and snapshot link are cleared, and `deleted_at` @@ -250,6 +281,9 @@ export class L3FieldRepository { SET status = 'inactive', inactive_at = ?, updated_at = ? WHERE unit_key = ? AND status = 'active'` ).run(at, at, unitKey); + // Project context includes the machine, so its cached versions must move when the machine resets. + const unit = this.getUnit(unitKey); + if (unit?.kind === "machine") this.bumpProjectSnapshots(unit, at); })(); } @@ -286,6 +320,59 @@ export class L3FieldRepository { ).run(profile.text, profile.fingerprint, at, unitKey); } + /** + * Freezes one chunk of a session's input traces as a batch of the machine unit, claiming the + * unit's next sequence number. The caller queues the matching l3_unit_update job. + */ + freezeUnitBatch(input: { + userId: string; + machineId: string; + sessionId: string; + trigger: string; + startTraceSeq: number; + endTraceSeq: number; + l1MemoryIds: readonly string[]; + rawTurnIds: readonly string[]; + at?: string; + }): L3UnitBatchRecord { + const at = input.at ?? nowIso(); + const machine = this.ensureMachineUnit({ userId: input.userId, machineId: input.machineId, at }); + const claimed = this.db.prepare( + `UPDATE l3_units SET next_seq = next_seq + 1, updated_at = ? WHERE unit_key = ? + RETURNING next_seq - 1 AS scope_seq` + ).get(at, machine.unitKey) as { scope_seq: number }; + const id = newId("l3batch"); + this.db.prepare( + `INSERT INTO l3_unit_batches ( + id, machine_unit_key, scope_seq, user_id, session_id, trigger, start_trace_seq, end_trace_seq, + l1_memory_ids_json, raw_turn_ids_json, status, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', ?, ?)` + ).run( + id, + machine.unitKey, + claimed.scope_seq, + input.userId, + input.sessionId, + input.trigger, + input.startTraceSeq, + input.endTraceSeq, + toJson([...input.l1MemoryIds]), + toJson([...input.rawTurnIds]), + at, + at + ); + return this.getBatch(id)!; + } + + getBatch(id: string): L3UnitBatchRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_unit_batches WHERE id = ?`).get(id) as SqlBatchRow | undefined; + return row ? batchFromSql(row) : undefined; + } + + setBatchStatus(id: string, status: L3UnitBatchStatus, at = nowIso()): void { + this.db.prepare(`UPDATE l3_unit_batches SET status = ?, updated_at = ? WHERE id = ?`).run(status, at, id); + } + /** Throwaway task roots Memmy itself creates (they are never projects); stored with the project list. */ getStandaloneRoots(userId: string): string[] { const row = this.db.prepare(`SELECT value_json FROM runtime_kv WHERE key = ?`) @@ -301,7 +388,8 @@ export class L3FieldRepository { ).run(standaloneRootsKey(userId), toJson([...roots]), at); } - private writeSnapshot(unit: L3UnitRecord, at: string): MemoryRow { + /** `force` writes a new version even when nothing shown changed (a project carrying a machine change). */ + private writeSnapshot(unit: L3UnitRecord, at: string, force = false): MemoryRow { const items = this.activeItems(unit.unitKey); const memoryValue = renderL3UnitSnapshot(unit, items); const existing = unit.snapshotMemoryId ? this.memories.get(unit.snapshotMemoryId) : undefined; @@ -361,6 +449,13 @@ export class L3FieldRepository { updatedAt: at, deletedAt: null }; + // Nothing shown changed (a confirm only moves times): keep the version, so clients do not refetch + // and the change log does not fill with empty updates. + if ( + existing && !force && !existing.deletedAt && existing.status === status && + existing.contentHash === memory.contentHash && + toJson(existing.properties.internal_info) === toJson(memory.properties.internal_info) + ) return existing; const saved = existing ? this.memories.update(memory) : this.memories.insert(memory); if (!existing) { this.db.prepare( @@ -393,6 +488,40 @@ function standaloneRootsKey(userId: string): string { return `l3_field.standalone_roots:${userId}`; } +interface SqlBatchRow { + id: string; + machine_unit_key: string; + scope_seq: number; + user_id: string; + session_id: string; + trigger: string; + start_trace_seq: number; + end_trace_seq: number; + l1_memory_ids_json: string; + raw_turn_ids_json: string; + status: L3UnitBatchStatus; + created_at: string; + updated_at: string; +} + +function batchFromSql(row: SqlBatchRow): L3UnitBatchRecord { + return { + id: row.id, + machineUnitKey: row.machine_unit_key, + scopeSeq: Number(row.scope_seq), + userId: row.user_id, + sessionId: row.session_id, + trigger: row.trigger, + startTraceSeq: Number(row.start_trace_seq), + endTraceSeq: Number(row.end_trace_seq), + l1MemoryIds: parseJson(row.l1_memory_ids_json, []), + rawTurnIds: parseJson(row.raw_turn_ids_json, []), + status: row.status, + createdAt: row.created_at, + updatedAt: row.updated_at + }; +} + function unitFromSql(row: SqlUnitRow): L3UnitRecord { return { unitKey: row.unit_key, diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index fc205e458..e36b69c1f 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -3465,8 +3465,11 @@ export class RuntimeRepository { : `AND job_type IN (${jobTypes.map(() => "?").join(", ")})` : ""; const excluded = options?.excludedJobTypes?.length ? options.excludedJobTypes : undefined; + // Same rule as leasing: excluding l3_unit_update also keeps new-L3 profile jobs out. const excludeFilter = excluded - ? `AND job_type NOT IN (${excluded.map(() => "?").join(", ")})` + ? `AND job_type NOT IN (${excluded.map(() => "?").join(", ")})${excluded.includes("l3_unit_update") + ? ` AND NOT (job_type = 'project_environment_profile' AND json_extract(payload_json, '$.mode') IS 'l3_field')` + : ""}` : ""; const jobTypeParams = [ ...(jobTypes && jobTypes.length > 0 ? jobTypes : []), @@ -3479,6 +3482,7 @@ export class RuntimeRepository { WHERE status = 'queued' AND attempts < max_attempts AND json_type(payload_json, '$.runAfter') = 'text' + ${scopedJobReadySql()} ${jobTypeFilter} ${excludeFilter} ORDER BY run_after ASC @@ -3667,31 +3671,12 @@ export class RuntimeRepository { json_extract(payload_json, '$.runAfter') IS NULL OR CAST(json_extract(payload_json, '$.runAfter') AS TEXT) <= ? ) - AND ( - job_type NOT IN ('l3_world_model_update', 'work_memory_extract') - OR ( - scope_key IS NOT NULL - AND scope_seq IS NOT NULL - AND NOT EXISTS ( - SELECT 1 - FROM evolution_jobs AS leased_l3_job - WHERE leased_l3_job.job_type = evolution_jobs.job_type - AND leased_l3_job.scope_key = evolution_jobs.scope_key - AND leased_l3_job.status = 'leased' - AND leased_l3_job.id <> evolution_jobs.id - ) - AND NOT EXISTS ( - SELECT 1 - FROM evolution_jobs AS earlier_l3_job - WHERE earlier_l3_job.job_type = evolution_jobs.job_type - AND earlier_l3_job.scope_key = evolution_jobs.scope_key - AND earlier_l3_job.scope_seq < evolution_jobs.scope_seq - AND earlier_l3_job.status IN ('queued', 'leased', 'failed') - ) - ) - ) + ${scopedJobReadySql()} ${allowedJobTypes ? `AND job_type IN (${allowedJobTypes.map(() => "?").join(", ")})` : ""} ${excludedJobTypes?.length ? `AND job_type NOT IN (${excludedJobTypes.map(() => "?").join(", ")})` : ""} + ${excludedJobTypes?.includes("l3_unit_update") + ? `AND NOT (job_type = 'project_environment_profile' AND json_extract(payload_json, '$.mode') IS 'l3_field')` + : ""} ${targetFilter} ORDER BY ${evolutionJobOrderSql()} LIMIT ?` @@ -3705,16 +3690,24 @@ export class RuntimeRepository { : candidates; for (const row of rows) { + // A new-L3 batch job carries the token of its current lease: a run that outlived its lease + // (the job was leased again) may then neither write nor change the job (holdsJobLease). + // The idle flush carries one too: re-arming rewrites its payload, so the run it replaced + // cannot mark the re-armed job succeeded. + if (row.job_type === "l3_unit_update" || row.job_type === "work_memory_idle_flush") { + row.payload_json = toJson({ ...parseJson>(row.payload_json, {}), leaseToken: newId("lease") }); + } this.db .prepare( `UPDATE evolution_jobs SET status = 'leased', attempts = attempts + 1, leased_until = ?, - updated_at = ? + updated_at = ?, + payload_json = ? WHERE id = ?` ) - .run(leaseUntil, at, row.id); + .run(leaseUntil, at, row.payload_json, row.id); } return rows.map((row) => jobFromSql({ @@ -3729,6 +3722,14 @@ export class RuntimeRepository { return transaction(); } + /** Whether this run still holds the job's lease; only jobs leased with a token can lose it. */ + holdsJobLease(job: EvolutionJobRecord): boolean { + const token = job.payload.leaseToken; + if (typeof token !== "string") return true; + const current = this.getJob(job.id); + return current?.status === "leased" && current.payload.leaseToken === token; + } + completeJob(id: string, at = nowIso()): EvolutionJobRecord | undefined { this.db .prepare( @@ -3951,6 +3952,14 @@ export class RuntimeRepository { updateL3WorldModelBatchTerminalOutcome(this.db, batchId, at); } } + if (status === "dead_letter" && row.job_type === "l3_unit_update") { + const payload = parseJson>(row.payload_json, {}); + if (typeof payload.batchId === "string") { + this.db.prepare( + `UPDATE l3_unit_batches SET status = 'failed', updated_at = ? WHERE id = ? AND status = 'queued'` + ).run(at, payload.batchId); + } + } if (status === "dead_letter" && row.job_type === "project_environment_profile") { const payload = parseJson>(row.payload_json, {}); const userId = typeof payload.userId === "string" ? payload.userId : undefined; @@ -5125,11 +5134,31 @@ export class L3WorldModelScopeWorkspaceConflictError extends Error { } export class L3WorldModelRepository { + /** When it returns a machine id, frozen batches go to the new L3 (field cognition) instead. */ + private fieldCognitionMachineId?: () => string | null; + constructor( private readonly db: Database.Database, - private readonly memories: MemoryRepository + private readonly memories: MemoryRepository, + private readonly l3Field?: L3FieldRepository ) {} + routeBatchesToFieldCognition(machineId: () => string | null): void { + this.fieldCognitionMachineId = machineId; + } + + /** + * New L3 only: hand a closed task's turns to L3. Called wherever a task closes, so a conversation that + * simply stops (idle, or the user ended the topic) still reaches L3; what is already frozen is skipped. + * With the switch off nothing changes. + */ + freezeClosedTask(input: { sessionId: string; episodeId: string; trigger: L3WorldModelBatchTrigger; at?: string }): void { + if (!this.l3Field || !this.fieldCognitionMachineId?.()) return; + const row = this.db.prepare(`SELECT * FROM sessions WHERE id = ?`).get(input.sessionId) as SqlSessionRow | undefined; + if (!row || sessionFromSql(row).meta.l3_world_model_protocol_version !== 2) return; + this.freezeBatches({ sessionId: input.sessionId, trigger: input.trigger, episodeId: input.episodeId, at: input.at }); + } + getScope(userId: string, projectId?: string | null): L3WorldModelScopeRecord | undefined { const row = this.db.prepare( `SELECT * FROM l3_world_model_scopes @@ -5514,13 +5543,20 @@ export class L3WorldModelRepository { } insertImmutableJob(job: EvolutionJobRecord): EvolutionJobRecord { - if (job.jobType !== "l3_world_model_update" && job.jobType !== "project_environment_profile") { + if ( + job.jobType !== "l3_world_model_update" && + job.jobType !== "project_environment_profile" && + job.jobType !== "l3_unit_update" + ) { throw new TypeError("immutable L3 job must use an L3 World Model job type"); } if (!job.dedupeKey) { throw new TypeError("immutable L3 job requires dedupeKey"); } - if (job.jobType === "l3_world_model_update" && (!job.scopeKey || job.scopeSeq === undefined)) { + if ( + (job.jobType === "l3_world_model_update" || job.jobType === "l3_unit_update") && + (!job.scopeKey || job.scopeSeq === undefined) + ) { throw new TypeError("L3 field update job requires scopeKey and scopeSeq"); } if (job.jobType === "project_environment_profile" && (job.scopeKey || job.scopeSeq !== undefined)) { @@ -5553,6 +5589,43 @@ export class L3WorldModelRepository { return job; } + /** One new-L3 batch per chunk, run in order per machine through the l3_unit_update FIFO. */ + private freezeL3UnitBatch( + session: SessionRecord, + machineId: string, + trigger: L3WorldModelBatchTrigger, + chunk: readonly L3WorldModelInputTraceRecord[], + at: string + ): string { + const batch = this.l3Field!.freezeUnitBatch({ + userId: session.userId, + machineId, + sessionId: session.id, + trigger, + startTraceSeq: chunk[0]!.traceSeq, + endTraceSeq: chunk.at(-1)!.traceSeq, + l1MemoryIds: chunk.map((trace) => trace.l1MemoryId), + rawTurnIds: [...new Set(chunk.map((trace) => trace.rawTurnId))], + at + }); + this.insertImmutableJob({ + id: newId("job"), + jobType: "l3_unit_update", + status: "queued", + dedupeKey: `l3_unit_update:${batch.id}`, + userId: session.userId, + sessionId: session.id, + scopeKey: batch.machineUnitKey, + scopeSeq: batch.scopeSeq, + payload: { batchId: batch.id }, + attempts: 0, + maxAttempts: 3, + createdAt: at, + updatedAt: at + }); + return batch.id; + } + private freezeBatchesInTransaction(input: { sessionId: string; trigger: L3WorldModelBatchTrigger; @@ -5579,7 +5652,8 @@ export class L3WorldModelRepository { ).get(session.id, input.throughL1MemoryId) as { trace_seq: number } | undefined; if (!through) throw new Error("through L1 memory does not belong to the L3 World Model session trace"); endTraceSeq = through.trace_seq; - } else if (input.trigger === "episode_idle_close") { + } else if (input.episodeId || input.trigger === "episode_idle_close") { + // A given task freezes only through its own turns, never into a task still in progress. if (!input.episodeId) throw new TypeError("episode_idle_close requires episodeId"); const end = this.db.prepare( `SELECT COALESCE(MAX(trace_seq), 0) AS trace_seq @@ -5615,6 +5689,21 @@ export class L3WorldModelRepository { } const chunks = splitL3TracesByRawTurn(traces, 20); + const fieldMachineId = this.l3Field ? this.fieldCognitionMachineId?.() ?? null : null; + if (fieldMachineId) { + const batchIds = chunks.map((chunk) => this.freezeL3UnitBatch(session, fieldMachineId, input.trigger, chunk, at)); + this.db.prepare( + `UPDATE l3_world_model_session_cursors + SET last_scheduled_seq = ?, updated_at = ? WHERE session_id = ?` + ).run(endTraceSeq, at, session.id); + return { + scheduled: true, + throughL1MemoryId: traces.at(-1)?.l1MemoryId, + throughTraceSeq: endTraceSeq, + batchIds, + targetCount: batchIds.length + }; + } const scope = this.ensureScope(session.userId, session.projectId, at); const batchIds: string[] = []; let targetCount = 0; @@ -6098,8 +6187,8 @@ export class Repositories { this.userMemories = new UserMemoryRepository(db); this.processing = new MemoryProcessingRepository(db); this.runtime = new RuntimeRepository(db); - this.l3WorldModels = new L3WorldModelRepository(db, this.memories); this.l3Field = new L3FieldRepository(db, this.memories); + this.l3WorldModels = new L3WorldModelRepository(db, this.memories, this.l3Field); this.projectEnvironments = new ProjectEnvironmentRepository(db, this.l3WorldModels, this.runtime); } @@ -8259,6 +8348,51 @@ function evolutionJobOrderSql(): string { rowid ASC`; } +/** + * Whether a queued job is not held back by an earlier job of its scope (L3 and Work Memory run in + * order) or, for a new-L3 batch, by L1 summaries still being written. Leasing and the worker's + * next wake-up both use it; a wake-up time for a job that cannot be leased would spin the worker. + */ +function scopedJobReadySql(): string { + return ` + AND ( + job_type NOT IN ('l3_world_model_update', 'l3_unit_update', 'work_memory_extract') + OR ( + scope_key IS NOT NULL + AND scope_seq IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM evolution_jobs AS leased_l3_job + WHERE leased_l3_job.job_type = evolution_jobs.job_type + AND leased_l3_job.scope_key = evolution_jobs.scope_key + AND leased_l3_job.status = 'leased' + AND leased_l3_job.id <> evolution_jobs.id + ) + AND NOT EXISTS ( + SELECT 1 + FROM evolution_jobs AS earlier_l3_job + WHERE earlier_l3_job.job_type = evolution_jobs.job_type + AND earlier_l3_job.scope_key = evolution_jobs.scope_key + AND earlier_l3_job.scope_seq < evolution_jobs.scope_seq + AND earlier_l3_job.status IN ('queued', 'leased', 'failed') + ) + ) + ) + AND NOT ( + job_type = 'l3_unit_update' + AND EXISTS ( + SELECT 1 + FROM l3_unit_batches AS unit_batch, json_each(unit_batch.l1_memory_ids_json) AS batch_l1 + JOIN memory_processing_state AS l1_state ON l1_state.memory_id = batch_l1.value + JOIN memories AS batch_memory ON batch_memory.id = l1_state.memory_id + WHERE unit_batch.id = CAST(json_extract(evolution_jobs.payload_json, '$.batchId') AS TEXT) + AND l1_state.state IN ('summary_pending', 'summarizing') + -- A deleted L1 is skipped by the pipeline, so it must not hold the batch back. + AND batch_memory.deleted_at IS NULL AND batch_memory.status != 'deleted' + ) + )`; +} + function evolutionJobPrioritySql(): string { const onboardingFirstReportTarget = targetMemoryMatchesSql(onboardingFirstReportMemorySql("memories")); const importedTarget = targetMemoryMatchesSql(agentSourceMemorySql("memories")); @@ -8291,7 +8425,7 @@ function evolutionJobPrioritySql(): string { WHEN job_type = 'l2_association' THEN 40 WHEN job_type = 'l2_induction' THEN 50 WHEN job_type = 'project_environment_profile' THEN 55 - WHEN job_type IN ('l3_abstraction', 'l3_world_model_update') THEN 60 + WHEN job_type IN ('l3_abstraction', 'l3_world_model_update', 'l3_unit_update') THEN 60 WHEN job_type = 'skill_crystallization' THEN 70 WHEN job_type = 'skill_cluster_assign' THEN 72 WHEN job_type = 'skill_batch_evolve' THEN 73 diff --git a/Memory/src/storage/schema.ts b/Memory/src/storage/schema.ts index 5e07b3ae6..062c44428 100644 --- a/Memory/src/storage/schema.ts +++ b/Memory/src/storage/schema.ts @@ -595,6 +595,9 @@ const statements = [ ON evolution_jobs (dedupe_key) WHERE dedupe_key IS NOT NULL AND job_type IN ('l3_world_model_update', 'project_environment_profile')`, + `CREATE UNIQUE INDEX IF NOT EXISTS uq_evolution_jobs_l3_unit_dedupe + ON evolution_jobs (dedupe_key) + WHERE dedupe_key IS NOT NULL AND job_type = 'l3_unit_update'`, `CREATE UNIQUE INDEX IF NOT EXISTS uq_evolution_jobs_scope_seq ON evolution_jobs (scope_key, scope_seq) WHERE scope_key IS NOT NULL`, diff --git a/Memory/src/types.ts b/Memory/src/types.ts index 7ef5d0a1a..a2be3beaa 100644 --- a/Memory/src/types.ts +++ b/Memory/src/types.ts @@ -82,6 +82,7 @@ export type JobType = | "l2_induction" | "l3_abstraction" | "l3_world_model_update" + | "l3_unit_update" | "project_environment_profile" | "skill_crystallization" | "skill_cluster_assign" diff --git a/Memory/tests/repository/l3-field-repository.test.ts b/Memory/tests/repository/l3-field-repository.test.ts index aece562ed..81bc0a275 100644 --- a/Memory/tests/repository/l3-field-repository.test.ts +++ b/Memory/tests/repository/l3-field-repository.test.ts @@ -80,7 +80,7 @@ describe("L3 field cognition storage", () => { .toBe(project.unitKey); }); - it("refreshes a unit snapshot and bumps project snapshots when the machine unit changes", () => { + it("refreshes a unit snapshot only when it changed, and bumps project snapshots when the machine unit changes", () => { const { repos } = openRepos(); const machine = repos.l3Field.ensureMachineUnit({ userId: USER, machineId: MACHINE }); const project = repos.l3Field.ensureProjectUnit({ userId: USER, machineId: MACHINE, project: PROJECT }); @@ -97,7 +97,13 @@ describe("L3 field cognition storage", () => { expect(first.memoryValue).toContain("报告以 GitHub 上的 main 为准"); const again = repos.l3Field.refreshSnapshot(project.unitKey); expect(again.id).toBe(first.id); - expect(again.version).toBe(first.version + 1); + expect(again.version).toBe(first.version); + repos.l3Field.refreshSnapshot(machine.unitKey); + const machineVersion = repos.memories.get(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId!)!.version; + // A machine that did not change leaves the project snapshots alone. + repos.l3Field.refreshSnapshot(machine.unitKey); + expect(repos.memories.get(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId!)!.version).toBe(machineVersion); + expect(repos.memories.get(first.id)!.version).toBe(again.version + 1); repos.l3Field.insertItem({ unitKey: machine.unitKey, @@ -107,7 +113,7 @@ describe("L3 field cognition storage", () => { evidenceAt: "2026-10-21T00:00:00.000Z" }); repos.l3Field.refreshSnapshot(machine.unitKey); - expect(repos.memories.get(first.id)!.version).toBe(again.version + 1); + expect(repos.memories.get(first.id)!.version).toBe(again.version + 2); }); it("replaces items by marking the old one inactive and pointing at its successor", () => { diff --git a/Memory/tests/service/l3-field/ended-task-l3.test.ts b/Memory/tests/service/l3-field/ended-task-l3.test.ts new file mode 100644 index 000000000..d2025e525 --- /dev/null +++ b/Memory/tests/service/l3-field/ended-task-l3.test.ts @@ -0,0 +1,203 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { createMemoryServiceFixture, setRawTurnActivityAt } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +function setup(enabled = true) { + const root = fixture.createTestRoot("ended-task-l3-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + // What a later reloadConfig() reads, to turn the switch without a restart. + const reload = { config }; + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, configLoader: () => ({ config: reload.config }), + l3FieldEnvironment: () => ({ machineId: "7".repeat(64), homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const open = (key: string) => service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, workspaceHostId: "c".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: key, userId: "u1" } + }).sessionId; + const batches = (sessionId: string) => db.db.prepare( + `SELECT trigger FROM l3_unit_batches WHERE session_id = ? ORDER BY scope_seq` + ).all(sessionId).map((row) => (row as { trigger: string }).trigger); + const legacyBatches = (sessionId: string) => (db.db.prepare( + `SELECT COUNT(*) AS n FROM l3_world_model_evidence_batches WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; + const flush = (sessionId: string) => db.db.prepare( + `SELECT status, json_extract(payload_json, '$.runAfter') AS run_after FROM evolution_jobs + WHERE job_type = 'work_memory_idle_flush' AND session_id = ?` + ).get(sessionId) as { status: string; run_after: string } | undefined; + /** Turn the new L3 on or off as a config reload does, without restarting the service. */ + const switchTo = (on: boolean) => { + reload.config = structuredClone(config); + reload.config.algorithm.l3FieldCognition.enabled = on; + service.reloadConfig({ restartFailedProcessing: false }); + }; + return { db, service, open, batches, legacyBatches, flush, switchTo }; +} + +/** A conversation that goes quiet for over two hours, then a turn somewhere else runs the idle sweep. */ +async function goIdle(env: ReturnType) { + const quiet = env.open("quiet"); + const turn = env.service.completeTurn("quiet-turn", { + sessionId: quiet, query: "以后改完都跑 npm test", answer: "好", status: "succeeded" + }); + await env.service.runWorkerOnce(20); + setRawTurnActivityAt(env.db, turn.rawTurnId, new Date(Date.now() - 2 * 60 * 60 * 1000 - 5_000).toISOString()); + const other = env.open("other"); + await env.service.startTurn({ turnId: "other-turn", sessionId: other, query: "Start a separate task" }); + env.service.completeTurn("other-turn", { sessionId: other, query: "Start a separate task", answer: "Started." }); + await env.service.runWorkerOnce(20); + return quiet; +} + +describe("a task that ends without a session close still reaches the new L3", () => { + it("hands a conversation that went idle to L3 when the idle sweep closes its task", async () => { + const env = setup(); + const quiet = await goIdle(env); + expect(env.batches(quiet)).toEqual(["episode_idle_close"]); + }); + + it("hands a task to L3 when the user ends the topic", async () => { + const env = setup(); + const session = env.open("ended"); + env.service.completeTurn("work-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + env.service.completeTurn("end-turn", { sessionId: session, query: "结束会话", answer: "好的。" }); + expect(env.batches(session)).toEqual(["new_task"]); + }); + + it("does not cut into the task in progress when a closed task is finalized again later", async () => { + const env = setup(); + const session = env.open("ongoing"); + const ended = env.service.completeTurn("old-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + env.service.completeTurn("end-turn", { sessionId: session, query: "结束会话", answer: "好的。" }); + env.service.completeTurn("new-turn", { sessionId: session, query: "Start the next task", answer: "Started." }); + const handlers = (env.service as unknown as { workerHandlers: { finalizeClosedEpisode: (e: unknown, at: string, t: string) => unknown } }) + .workerHandlers; + const closed = (env.service as unknown as { repos: { runtime: { getEpisode(id: string): unknown } } }).repos.runtime.getEpisode(ended.episodeId); + handlers.finalizeClosedEpisode(closed, new Date().toISOString(), "episode_rewarded"); + expect(env.batches(session)).toEqual(["new_task"]); + }); + + it("hands a conversation that just stops, with no close and no later turn, to L3 once it has been quiet", async () => { + const env = setup(); + const session = env.open("stopped"); + env.service.completeTurn("stopped-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + expect(env.flush(session)).toMatchObject({ status: "queued" }); + expect(env.batches(session)).toEqual([]); + // Two hours later with nothing else happening anywhere, the session's own flush comes due. + const quietSince = new Date(Date.now() - 2 * 60 * 60 * 1000 - 5_000).toISOString(); + env.db.db.prepare(`UPDATE l3_world_model_input_traces SET created_at = ? WHERE session_id = ?`).run(quietSince, session); + env.db.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(quietSince); + await env.service.runWorkerOnce(50); + expect(env.batches(session)).toEqual(["session_idle"]); + }); + + it("keeps the flush queued when it comes due before the session has been quiet long enough", async () => { + const env = setup(); + const session = env.open("early"); + env.service.completeTurn("early-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + // Due now, but the last turn is recent (as when a turn lands after the flush was leased): it re-arms itself. + env.db.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(new Date(Date.now() - 1_000).toISOString()); + await env.service.runWorkerOnce(50); + const flush = env.flush(session)!; + expect(flush.status).toBe("queued"); + expect(Date.parse(flush.run_after)).toBeGreaterThan(Date.now()); + expect(env.batches(session)).toEqual([]); + }); + + it("re-freezes imported turns whose unfinished batch an import had to drop for a machine kept here", async () => { + const source = setup(); + const session = source.open("exported"); + source.service.completeTurn("exported-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + source.service.closeSession(session); + expect(source.batches(session)).toEqual(["session_close"]); + const bundle = source.service.exportBundle({ includeRawText: true }); + + const target = setup(); + target.open("local"); + target.service.completeTurn("local-turn", { sessionId: target.open("local"), query: "看看", answer: "好" }); + expect(target.service.importBundle({ bundle }).ok).toBe(true); + // The exported batch was dropped with the machine unit kept here; the closed session's turns are + // frozen afresh into this database's own queue. + expect(target.batches(session)).toEqual(["session_close"]); + const job = target.db.db.prepare( + `SELECT j.status FROM evolution_jobs j JOIN l3_unit_batches b ON b.id = json_extract(j.payload_json, '$.batchId') + WHERE b.session_id = ?` + ).get(session) as { status: string }; + expect(job.status).toBe("queued"); + // Importing the same bundle again finds those turns already frozen here and leaves them alone. + expect(target.service.importBundle({ bundle }).ok).toBe(true); + expect(target.batches(session)).toEqual(["session_close"]); + }); + + it("re-freezes the turns a replace import adds after what this database already froze", async () => { + const source = setup(); + const session = source.open("growing"); + source.service.completeTurn("first-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + const target = setup(); + target.service.completeTurn("local-turn", { sessionId: target.open("local"), query: "看看", answer: "好" }); + expect(target.service.importBundle({ bundle: source.service.exportBundle({ includeRawText: true }) }).ok).toBe(true); + const frozen = (sessionId: string) => target.db.db.prepare( + `SELECT start_trace_seq AS s, end_trace_seq AS e FROM l3_unit_batches WHERE session_id = ? ORDER BY scope_seq` + ).all(sessionId) as Array<{ s: number; e: number }>; + // The imported session goes quiet here and its idle flush freezes turn 1. + (target.service as unknown as { repos: { l3WorldModels: { freezeBatches(input: { sessionId: string; trigger: "session_idle" }): unknown } } }) + .repos.l3WorldModels.freezeBatches({ sessionId: session, trigger: "session_idle" }); + expect(frozen(session)).toEqual([{ s: 1, e: 1 }]); + // The source goes on: a second turn, then a close freezes [1,2] there; that bundle replaces this one. + source.service.completeTurn("second-turn", { sessionId: session, query: "提交信息用中文", answer: "好" }); + source.service.closeSession(session); + expect(target.service.importBundle({ + bundle: source.service.exportBundle({ includeRawText: true }), conflictStrategy: "replace" + }).ok).toBe(true); + expect(frozen(session).at(-1)!.e).toBe(2); + }); + + it("re-freezes a turn a replace import put at a position this database froze with a different turn", async () => { + const source = setup(); + const session = source.open("forked"); + source.service.completeTurn("shared-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + const target = setup(); + target.service.completeTurn("local-turn", { sessionId: target.open("local"), query: "看看", answer: "好" }); + expect(target.service.importBundle({ bundle: source.service.exportBundle({ includeRawText: true }) }).ok).toBe(true); + // Both databases go on in the same session with a different second turn; this one freezes [1,2] here. + target.service.completeTurn("target-turn", { sessionId: session, query: "提交信息用英文", answer: "好" }); + (target.service as unknown as { repos: { l3WorldModels: { freezeBatches(input: { sessionId: string; trigger: "session_idle" }): unknown } } }) + .repos.l3WorldModels.freezeBatches({ sessionId: session, trigger: "session_idle" }); + source.service.completeTurn("source-turn", { sessionId: session, query: "提交信息用中文", answer: "好" }); + source.service.closeSession(session); + expect(target.service.importBundle({ + bundle: source.service.exportBundle({ includeRawText: true }), conflictStrategy: "replace" + }).ok).toBe(true); + // Trace 2 is now the source's turn: some batch here must have frozen that turn, not just its position. + const trace = target.db.db.prepare( + `SELECT l1_memory_id AS id FROM l3_world_model_input_traces WHERE session_id = ? AND trace_seq = 2` + ).get(session) as { id: string }; + const owners = target.db.db.prepare( + `SELECT COUNT(*) AS n FROM l3_unit_batches b, json_each(b.l1_memory_ids_json) j + WHERE b.session_id = ? AND b.start_trace_seq <= 2 AND b.end_trace_seq >= 2 AND j.value = ?` + ).get(session, trace.id) as { n: number }; + expect(owners.n).toBe(1); + }); + + it("changes nothing with the switch off", async () => { + const env = setup(false); + const quiet = await goIdle(env); + expect(env.batches(quiet)).toEqual([]); + expect(env.legacyBatches(quiet)).toBe(0); + }); +}); diff --git a/Memory/tests/service/l3-field/evidence-fragments.test.ts b/Memory/tests/service/l3-field/evidence-fragments.test.ts index 2d763f0b9..f0407e2c5 100644 --- a/Memory/tests/service/l3-field/evidence-fragments.test.ts +++ b/Memory/tests/service/l3-field/evidence-fragments.test.ts @@ -1,3 +1,6 @@ +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { buildEvidenceFragments, @@ -57,7 +60,54 @@ describe("extractToolPaths", () => { "]);" ].join("\n"); expect(extractToolPaths({ name: "exec", input: code }, "/work/repo-a")).toEqual([ - "/work/repo-b", "/work/repo-c/x.md" + "/work/repo-b", "/work/repo-b/src/a.ts", "/work/repo-c/x.md" + ]); + }); + + it("reads relative paths, and bare names that exist, against the working directory", () => { + const big = mkdtempSync(join(tmpdir(), "l3-big-")); + mkdirSync(join(big, "api")); + const command = "cat api/AGENTS.md; rg -n 'listen|PORT' api --glob '!node_modules/**'; ls"; + expect(extractToolPaths({ name: "exec_command", input: { cmd: command, workdir: big } }, big)).toEqual([ + join(big, "api/AGENTS.md"), join(big, "api"), big + ]); + rmSync(big, { recursive: true, force: true }); + }); + + it("counts the workdir only when the call names no target of its own", () => { + expect(extractToolPaths({ name: "exec_command", input: { cmd: "git -C /work/repo-b status", workdir: "/work/repo-a" } }, "/work/repo-a")) + .toEqual(["/work/repo-b"]); + expect(extractToolPaths({ name: "exec_command", input: { cmd: "npm test", workdir: "/work/repo-b" } }, "/work/repo-a")) + .toEqual(["/work/repo-b"]); + }); + + it("judges each subcommand on its own and ignores where output is redirected", () => { + const run = (cmd: string) => extractToolPaths({ name: "exec_command", input: { cmd, workdir: "/work/repo-a" } }, "/work/repo-a"); + expect(run("git -C /work/repo-b status && npm test")).toEqual(["/work/repo-b", "/work/repo-a"]); + expect(run("npm test > /tmp/test.log 2>&1")).toEqual(["/work/repo-a"]); + expect(run("npm test 2>/dev/null | tail -5")).toEqual(["/work/repo-a"]); + }); + + it("keeps the other calls of a code-mode snippet that also holds a patch", () => { + const code = [ + "await tools.exec_command({cmd: \"npm test\", workdir: \"/work/repo-a\"});", + "await tools.apply_patch(`*** Begin Patch", + "*** Update File: /work/repo-b/file.ts", + "@@", + "-a", + "+b", + "*** End Patch`);" + ].join("\n"); + expect(extractToolPaths({ name: "exec", input: code }, "/work/repo-a").sort()).toEqual(["/work/repo-a", "/work/repo-b/file.ts"]); + }); + + it("reads the files a patch touches, relative ones against the workdir", () => { + const patch = "*** Begin Patch\n*** Update File: /work/repo-b/file.ts\n@@\n-a\n+b\n*** Add File: docs/new.md\n+x\n*** End Patch"; + expect(extractToolPaths({ name: "apply_patch", input: patch }, "/work/repo-a")).toEqual([ + "/work/repo-b/file.ts", "/work/repo-a/docs/new.md" + ]); + expect(extractToolPaths({ name: "apply_patch", input: { input: patch, workdir: "/work/repo-c" } }, "/work/repo-a")).toEqual([ + "/work/repo-b/file.ts", "/work/repo-c/docs/new.md" ]); }); @@ -96,6 +146,12 @@ describe("clipToolOutput", () => { it("returns short output unchanged", () => { expect(clipToolOutput("ok")).toBe("ok"); }); + + it("keeps only the start of a very long error line", () => { + const middle = Array.from({ length: 400 }, (_, i) => `line ${i}`); + middle[200] = `{"error":"${"x".repeat(50_000)}"}`; + expect(clipToolOutput(middle.join("\n")).length).toBeLessThan(1_000); + }); }); describe("buildEvidenceFragments", () => { @@ -105,7 +161,7 @@ describe("buildEvidenceFragments", () => { { id: "c2", name: "Bash", input: { command: "cd /work/repo-b && npm test" }, output: "1 failed", success: false } ])); expect(fragments.map((fragment) => [fragment.toolName, fragment.candidateUnitKeys, fragment.isError])).toEqual([ - [null, ["unit:id:repo-a"], false], + [null, ["unit:id:repo-a", "unit:id:repo-b"], false], ["Edit", ["unit:id:repo-a"], false], ["Bash", ["unit:id:repo-b"], true] ]); @@ -114,6 +170,42 @@ describe("buildEvidenceFragments", () => { expect(fragments.every((fragment) => fragment.l1MemoryId === "l1-1" && fragment.rawTurnId === "raw-1")).toBe(true); }); + it("gives the user's words to the repository the turn worked on, not the big folder the session opened in", () => { + const resolveBig = (target: string) => target.startsWith("/big/report") + ? { identity: "id:report", checkoutRoot: "/big/report", source: "git" as const, displayName: "report" } + : target.startsWith("/big") + ? { identity: "dir:/big", checkoutRoot: "/big", source: "folder" as const, displayName: "big" } + : null; + const turn = (calls: unknown[]) => buildEvidenceFragments({ + rawTurn: rawTurn(calls, [], "以后报告都以 main 为准"), l1MemoryId: "l1-1", + sessionCwd: "/big", sessionProjectRoot: "/big", resolve: resolveBig, unitKeyOf + }); + expect(turn([{ name: "Bash", input: { command: "git -C /big/report pull" }, output: "ok" }])[0]!.candidateUnitKeys) + .toEqual(["unit:id:report"]); + expect(turn([])[0]!.candidateUnitKeys).toEqual(["unit:dir:/big"]); + }); + + it("clips a long tool input but still reads its paths", () => { + const [, write] = build(rawTurn([ + { id: "c1", name: "Write", input: { file_path: "/work/repo-a/big.md", content: "y".repeat(20_000) }, output: "ok" } + ])); + expect(write!.candidateUnitKeys).toEqual(["unit:id:repo-a"]); + expect(JSON.stringify(write!.input).length).toBeLessThan(1_000); + }); + + it("keeps only the nested project when a call touches it and the big folder around it", () => { + const resolveBig = (target: string) => target.startsWith("/big/report") + ? { identity: "id:report", checkoutRoot: "/big/report", source: "git" as const, displayName: "report" } + : target.startsWith("/big") + ? { identity: "dir:/big", checkoutRoot: "/big", source: "folder" as const, displayName: "big" } + : null; + const fragments = buildEvidenceFragments({ + rawTurn: rawTurn([{ name: "exec_command", input: { cmd: "pwd; printf '\\n--- rules ---\\n'; cat /big/report/AGENTS.md", workdir: "/big" }, output: "ok" }], [], "看看 report 的规矩"), + l1MemoryId: "l1-1", sessionCwd: "/big", sessionProjectRoot: "/big", resolve: resolveBig, unitKeyOf + }); + expect(fragments.map((fragment) => fragment.candidateUnitKeys)).toEqual([["unit:id:report"], ["unit:id:report"]]); + }); + it("gives stable fragment ids for the same raw turn", () => { const turn = rawTurn([{ id: "c1", name: "Read", input: { file_path: "/work/repo-a/a.ts" }, output: "x" }]); expect(build(turn).map((fragment) => fragment.id)).toEqual(build(turn).map((fragment) => fragment.id)); diff --git a/Memory/tests/service/l3-field/l3-field-context.test.ts b/Memory/tests/service/l3-field/l3-field-context.test.ts index a307da57f..edf659d15 100644 --- a/Memory/tests/service/l3-field/l3-field-context.test.ts +++ b/Memory/tests/service/l3-field/l3-field-context.test.ts @@ -23,7 +23,8 @@ function setup(enabled = true, l3MachineProbe: () => Promise | Promise ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }) + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }), + l3MachineProbe: l3MachineProbe as never }); const repos = new Repositories(db.db); return { db, service, repos, home, project }; @@ -116,6 +117,96 @@ describe("new L3 session context", () => { expect(context.memoryId).not.toBe(snapshot.id); }); + it("does not inject a unit whose snapshot the user archived, and keeps it archived when it is rewritten", () => { + const { service, repos } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { opened, request } = open(service, "archive"); + service.archiveMemory(snapshot.id, { namespace: request.namespace, reason: "user" }); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).toBe(""); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 bun", evidence: evidence(), evidenceAt: "2026-10-02T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + expect(repos.memories.get(snapshot.id)!.status).toBe("archived"); + expect(service.l3WorldModelContext(opened.sessionId, request).renderedContext).toBe(""); + }); + + it("moves a project's context version when the machine snapshot is archived, so clients drop the cached machine part", () => { + const { service, repos, project } = setup(); + const { opened, request } = open(service, "archive-machine", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + service.l3WorldModelContext(opened.sessionId, request); + const unit = repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project'`).get() as { unit_key: string }; + repos.l3Field.insertItem({ unitKey: unit.unit_key, kind: "project_constraint", content: "改完跑 npm test", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const machineSnapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + repos.l3Field.refreshSnapshot(unit.unit_key); + const before = service.l3WorldModelContext(opened.sessionId, request); + expect(before.renderedContext).toContain("本机用 pnpm 9"); + + service.archiveMemory(machineSnapshot.id, { namespace: request.namespace, reason: "user" }); + const after = service.l3WorldModelContext(opened.sessionId, request); + expect(after.renderedContext).not.toContain("本机用 pnpm 9"); + expect(after.renderedContext).toContain("改完跑 npm test"); + expect(after.memoryVersion).not.toBe(before.memoryVersion); + }); + + it("deletes a snapshot and resets its unit together, or neither", () => { + const { service, repos } = setup(); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const item = repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { request } = open(service, "atomic-delete"); + const internal = (service as unknown as { repos: Repositories }).repos; + const reset = vi.spyOn(internal.l3Field, "markUnitDeleted").mockImplementation(() => { throw new Error("disk full"); }); + expect(() => service.deleteMemory(snapshot.id, { namespace: request.namespace, reason: "user" })).toThrow("disk full"); + reset.mockRestore(); + expect(repos.memories.get(snapshot.id)!.deletedAt).toBeFalsy(); + expect(repos.l3Field.getItem(item.id)!.status).toBe("active"); + }); + + it("drops a machine probe that was running when the user deleted the machine snapshot", async () => { + let finishProbe: (probe: unknown) => void = () => undefined; + const probe = () => new Promise((resolve) => { finishProbe = resolve; }); + const { service, repos } = setup(true, probe); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + const snapshot = repos.l3Field.refreshSnapshot(machine.unitKey); + const { request } = open(service, "probe-race"); + service.deleteMemory(snapshot.id, { namespace: request.namespace, reason: "user" }); + finishProbe({ os: { name: "macOS", version: "26.0" }, arch: "arm64", shell: "zsh", tools: [] }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(repos.l3Field.getUnit(machine.unitKey)!.profileText).toBeFalsy(); + expect(repos.l3Field.getUnit(machine.unitKey)!.snapshotMemoryId).toBeUndefined(); + }); + + it("moves the project version when the machine is reset, and keeps refreshing a project rebuilt after a reset", () => { + const { service, repos, project } = setup(); + const { opened, request } = open(service, "versions", project); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 pnpm 9", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + service.l3WorldModelContext(opened.sessionId, request); + const unitKey = (repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project'`).get() as { unit_key: string }).unit_key; + repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "改完跑测试", evidence: evidence(), evidenceAt: "2026-10-01T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const before = service.l3WorldModelContext(opened.sessionId, request); + + repos.l3Field.markUnitDeleted(machine.unitKey); + const afterMachineReset = service.l3WorldModelContext(opened.sessionId, request); + expect(afterMachineReset.memoryVersion).not.toBe(before.memoryVersion); + expect(afterMachineReset.renderedContext).not.toContain("pnpm"); + + repos.l3Field.markUnitDeleted(unitKey); + repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "新规则", evidence: evidence(), evidenceAt: "2026-10-02T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(unitKey); + const rebuilt = service.l3WorldModelContext(opened.sessionId, request); + repos.l3Field.insertItem({ unitKey: machine.unitKey, kind: "env_fact", content: "本机用 bun", evidence: evidence(), evidenceAt: "2026-10-03T00:00:00.000Z" }); + repos.l3Field.refreshSnapshot(machine.unitKey); + const afterMachineChange = service.l3WorldModelContext(opened.sessionId, request); + expect(afterMachineChange.memoryVersion).not.toBe(rebuilt.memoryVersion); + expect(afterMachineChange.renderedContext).toContain("本机用 bun"); + }); + it("on startup drops items inactive for over 15 days and never queues embeddings for snapshots", () => { const { service, repos, db } = setup(); const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); diff --git a/Memory/tests/service/l3-field/project-resolver.test.ts b/Memory/tests/service/l3-field/project-resolver.test.ts index 6b6844ff8..9d7fd30af 100644 --- a/Memory/tests/service/l3-field/project-resolver.test.ts +++ b/Memory/tests/service/l3-field/project-resolver.test.ts @@ -123,6 +123,29 @@ describe("resolveProject", () => { expect(resolveProject(asSession(dir("Memmy", "20261022-101010")), ctx)).toBeNull(); }); + it("does not treat tool installs as projects even when they are git clones", () => { + const nvm = dir(".nvm"); + gitRepo(nvm, "https://github.com/nvm-sh/nvm.git"); + const toolPath = (p: string) => ({ path: p, role: "tool_path" as const, sessionCwd: null, sessionProjectRoot: null }); + expect(resolveProject(toolPath(join(nvm, "nvm.sh")), ctx)).toBeNull(); + expect(resolveProject(asSession(nvm), ctx)).toBeNull(); + expect(resolveProject(toolPath("/opt/homebrew/bin/git"), ctx)).toBeNull(); + expect(resolveProject(toolPath("/usr/local/Homebrew/Library/Homebrew/brew.sh"), ctx)).toBeNull(); + expect(resolveProject(toolPath("C:\\Program Files\\Git\\bin\\git.exe"), ctx)).toBeNull(); + + const app = dir("code", "app"); + gitRepo(app, "git@github.com:me/app.git"); + const pkg = dir("code", "app", "node_modules", "left-pad"); + gitRepo(pkg, "https://github.com/x/left-pad.git"); + expect(resolveProject(toolPath(join(pkg, "index.js")), ctx)).toMatchObject({ identity: "github.com/me/app" }); + }); + + it("still takes a folder the user selected inside a hidden folder", () => { + const notes = dir(".config", "notes"); + ctx.userProjects = [{ rootPath: notes, name: "notes" }]; + expect(resolveProject(asSession(notes), ctx)).toMatchObject({ source: "user_selected", checkoutRoot: notes }); + }); + it("does not treat the home directory, Desktop, Documents or Downloads themselves as projects", () => { for (const container of [home, dir("Desktop"), dir("Documents"), dir("Downloads")]) { expect(resolveProject(asSession(container), ctx)).toBeNull(); diff --git a/Memory/tests/service/l3-field/unit-update.test.ts b/Memory/tests/service/l3-field/unit-update.test.ts new file mode 100644 index 000000000..3a419d6a2 --- /dev/null +++ b/Memory/tests/service/l3-field/unit-update.test.ts @@ -0,0 +1,661 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { ModelHttpError } from "../../../src/model/http.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { L3UnitUpdatePipeline } from "../../../src/service/l3-field/unit-update-pipeline.js"; +import { validateL3UnitOps, type L3UpdateGroup } from "../../../src/service/l3-field/unit-ops.js"; +import type { EvidenceFragment, L3ItemRecord, L3UnitRecord } from "../../../src/service/l3-field/types.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const MACHINE_ID = "d".repeat(64); +type Reply = (input: ModelInput, call: number) => unknown; + +interface ModelInput { + evidence: Array<{ id: string; type: string; candidate_units: string[]; paths: string[] }>; + machine: { items: Array<{ id: string; content: string }> }; + projects: Array<{ unit_key: string; name: string; items: Array<{ id: string }> }>; + retry?: string; +} + +function setup(options: { enabled?: boolean; reply?: Reply } = {}) { + const root = fixture.createTestRoot("l3-unit-update-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + const other = join(home, "code", "other-repo"); + mkdirSync(project, { recursive: true }); + mkdirSync(other, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = options.enabled ?? true; + const environment = () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }); + const service = fixture.createTestMemoryService({ + db, + mode: "dev", + config, + l3FieldEnvironment: environment, + l3MachineProbe: () => new Promise(() => undefined) + }); + const repos = new Repositories(db.db); + const inputs: ModelInput[] = []; + const reply = options.reply ?? defaultReply; + const complete = vi.fn(async (messages) => { + const input = JSON.parse(messages[1]!.content) as ModelInput; + inputs.push(input); + return JSON.stringify(reply(input, inputs.length)); + }); + const pipeline = new L3UnitUpdatePipeline({ repos, llm: testLlm(complete), environment }); + return { db, service, repos, pipeline, complete, inputs, home, project, other }; +} + +/** A tool error becomes a project fact; the user's words become a project rule. */ +const defaultReply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool"); + const user = input.evidence.find((fragment) => fragment.type === "user_text"); + const ops = []; + if (input.projects.length && tool) { + ops.push({ op: "add", unit: "project", kind: "env_fact", content: "npm test 需要先 npm ci -> 否则找不到 vitest", evidence: [tool.id] }); + } + if (input.projects.length && user) { + ops.push({ op: "add", unit: "project", kind: "project_constraint", content: "改完代码要跑 npm test", evidence: [user.id] }); + } + return { reason: "test", ops }; +}; + +function captureTurn( + service: ReturnType["service"], + sessionKey: string, + cwd: string, + turns: Array<{ query: string; workdir?: string }> = [{ query: "以后改完代码都要跑 npm test" }] +) { + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(cwd).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey, userId: "u1" } + }); + turns.forEach((turn, index) => { + service.completeTurn(`${sessionKey}-turn-${index}`, { + sessionId: opened.sessionId, + query: turn.query, + answer: "好的", + status: "succeeded", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: turn.workdir ?? cwd }, success: false }], + toolResults: [{ id: `call-${index}`, name: "exec_command", output: "sh: vitest: command not found", exitCode: 127 }] + }); + }); + service.closeSession(opened.sessionId); + return opened; +} + +function queuedUnitJob(repos: Repositories): EvolutionJobRecord { + const job = repos.runtime.listJobs("queued", 100).find((candidate) => candidate.jobType === "l3_unit_update"); + if (!job) throw new Error("expected a queued l3_unit_update job"); + return job; +} + +function projectUnitKey(repos: Repositories, name = "demo"): string { + return (repos.db.prepare(`SELECT unit_key FROM l3_units WHERE kind = 'project' AND display_name = ?`).get(name) as { unit_key: string }).unit_key; +} + +describe("new L3 update pipeline", () => { + it("freezes a session into a new-L3 batch whose job writes items the next session sees", async () => { + const { service, repos, pipeline, project } = setup(); + captureTurn(service, "e2e", project); + expect(repos.runtime.listJobs("queued", 100).some((job) => job.jobType === "l3_world_model_update")).toBe(false); + const job = queuedUnitJob(repos); + expect(job.scopeKey).toBe(repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }).unitKey); + + await pipeline.process(job); + const items = repos.l3Field.activeItems(projectUnitKey(repos)); + expect(items.map((item) => [item.kind, item.content]).sort()).toEqual([ + ["env_fact", "npm test 需要先 npm ci -> 否则找不到 vitest"], + ["project_constraint", "改完代码要跑 npm test"] + ]); + expect(repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + + const next = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: "e2e-next", userId: "u1" } + }); + const context = service.l3WorldModelContext(next.sessionId, { + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", + adapterId: "codex-memory", + source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "e2e-next", userId: "u1", projectId: next.projectId! } + }); + expect(context.renderedContext).toContain("当前项目：demo（文件夹）"); + expect(context.renderedContext).toContain("## 项目约束\n- 改完代码要跑 npm test"); + }); + + it("keeps the old batches and jobs when the switch is off", () => { + const { service, repos, project } = setup({ enabled: false }); + captureTurn(service, "flag-off", project); + expect(repos.runtime.listJobs("queued", 100).some((job) => job.jobType === "l3_world_model_update")).toBe(true); + expect(repos.runtime.listJobs("queued", 100).some((job) => job.jobType === "l3_unit_update")).toBe(false); + expect((repos.db.prepare(`SELECT COUNT(*) AS n FROM l3_unit_batches`).get() as { n: number }).n).toBe(0); + }); + + it("writes a batch only once even if its job runs again", async () => { + const { service, repos, pipeline, project, complete } = setup(); + captureTurn(service, "twice", project); + const job = queuedUnitJob(repos); + await pipeline.process(job); + await pipeline.process(job); + expect(repos.l3Field.activeItems(projectUnitKey(repos))).toHaveLength(2); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("regenerates once with the reasons, and fails without writing when rejected twice", async () => { + const userFact: Reply = (input) => ({ + reason: "bad", + ops: [{ op: "add", unit: "project", kind: "env_fact", content: "没有工具输出的事实", evidence: [input.evidence.find((f) => f.type === "user_text")!.id] }] + }); + const fixed = setup({ reply: (input, call) => (call === 1 ? userFact(input, call) : defaultReply(input, call)) }); + captureTurn(fixed.service, "retry-ok", fixed.project); + await fixed.pipeline.process(queuedUnitJob(fixed.repos)); + expect(fixed.inputs[1]!.retry).toContain("env_fact needs a cited tool output or error"); + expect(fixed.repos.l3Field.activeItems(projectUnitKey(fixed.repos))).toHaveLength(2); + + const broken = setup({ reply: userFact }); + captureTurn(broken.service, "retry-bad", broken.project); + const job = queuedUnitJob(broken.repos); + await expect(broken.pipeline.process(job)).rejects.toThrow("rejected twice"); + expect(broken.repos.l3Field.activeItems(projectUnitKey(broken.repos))).toHaveLength(0); + expect(broken.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("queued"); + }); + + it("drops results for a unit reset after the batch was frozen", async () => { + const { service, repos, pipeline, project } = setup(); + captureTurn(service, "reset", project); + const job = queuedUnitJob(repos); + repos.l3Field.markUnitDeleted(projectUnitKey(repos), new Date(Date.now() + 1000).toISOString()); + await pipeline.process(job); + expect(repos.l3Field.activeItems(projectUnitKey(repos))).toHaveLength(0); + expect(repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + }); + + it("does not let what was said before a unit reset refill it, even when frozen after the reset", async () => { + const { service, repos, pipeline, project } = setup(); + captureTurn(service, "late-freeze", project); + const job = queuedUnitJob(repos); + repos.l3Field.markUnitDeleted(projectUnitKey(repos), new Date(Date.now() + 1000).toISOString()); + repos.db.prepare(`UPDATE l3_unit_batches SET created_at = ?`).run(new Date(Date.now() + 2000).toISOString()); + await pipeline.process(job); + expect(repos.l3Field.activeItems(projectUnitKey(repos))).toHaveLength(0); + }); + + it("does not let older evidence replace or delete an item newer evidence wrote, as a late rerun would", async () => { + const reply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + const [first, second] = input.projects[0]!.items; + return { reason: "old", ops: [ + { op: "replace", ids: [first!.id], content: "Node 18", evidence: [tool.id] }, + { op: "delete", id: second!.id, evidence: [tool.id] } + ] }; + }; + const { service, repos, pipeline, project } = setup({ reply }); + captureTurn(service, "rerun", project, [{ query: "看看" }]); + const job = queuedUnitJob(repos); + const unitKey = projectUnitKey(repos); + const newer = (content: string) => repos.l3Field.insertItem({ + unitKey, kind: "env_fact", content, evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2099-01-01T00:00:00.000Z" + }); + const kept = [newer("Node 20"), newer("用 pnpm")]; + await pipeline.process(job); + expect(kept.map((item) => repos.l3Field.getItem(item.id)!.status)).toEqual(["active", "active"]); + expect(repos.l3Field.activeItems(unitKey).map((item) => item.content)).not.toContain("Node 18"); + }); + + it("drops the result of a run that outlived its lease once another run dead-lettered the batch", async () => { + let db: ReturnType["db"] | undefined; + const reply: Reply = (input, call) => { + // While this run waits on the model, a newer run of the same job fails and dead-letters the batch. + if (call === 1) db!.db.prepare(`UPDATE l3_unit_batches SET status = 'failed'`).run(); + return defaultReply(input, call); + }; + const env = setup({ reply }); + db = env.db; + captureTurn(env.service, "late-run", env.project); + const job = queuedUnitJob(env.repos); + await expect(env.pipeline.process(job)).rejects.toThrow(/result dropped/); + expect(env.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("failed"); + expect(env.repos.l3Field.activeItems(projectUnitKey(env.repos))).toEqual([]); + }); + + it("lets only the run that holds the lease write: a run that outlived it leaves job and batch alone", async () => { + let db: ReturnType["db"] | undefined; + const reply: Reply = (input, call) => { + // While this run waits on the model its lease runs out, the job is leased again and later put back + // in the queue by a rerun: the batch is "queued" again, but this run no longer holds the job. + if (call === 1) db!.db.prepare(`UPDATE evolution_jobs SET status = 'queued', + payload_json = json_set(payload_json, '$.leaseToken', 'newer-run') WHERE job_type = 'l3_unit_update'`).run(); + return defaultReply(input, call); + }; + const env = setup({ reply }); + db = env.db; + captureTurn(env.service, "lease-lost", env.project); + env.db.db.prepare(`UPDATE memory_processing_state SET state = 'ready'`).run(); + const [job] = env.repos.runtime.leaseQueuedJobs(10, 60, undefined, false, ["l3_unit_update"]); + expect(typeof job!.payload.leaseToken).toBe("string"); + await expect(env.pipeline.process(job!)).rejects.toThrow(/leased again/); + expect(env.repos.l3Field.getBatch(job!.payload.batchId as string)!.status).toBe("queued"); + expect(env.repos.l3Field.activeItems(projectUnitKey(env.repos))).toEqual([]); + expect(env.repos.runtime.getJob(job!.id)!.status).toBe("queued"); + }); + + it("drops ops citing a turn whose L1 the user deleted while the model was answering", async () => { + let db: ReturnType["db"] | undefined; + const reply: Reply = (input, call) => { + if (call === 1) db!.db.prepare(`UPDATE memories SET deleted_at = ?, status = 'deleted' WHERE memory_layer = 'L1'`).run(new Date().toISOString()); + return defaultReply(input, call); + }; + const env = setup({ reply }); + db = env.db; + captureTurn(env.service, "deleted-l1", env.project); + const job = queuedUnitJob(env.repos); + await env.pipeline.process(job); + expect(env.repos.l3Field.activeItems(projectUnitKey(env.repos))).toEqual([]); + expect(env.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + }); + + it("does not replace or delete an item that is already gone when the batch is written", async () => { + let gone: string | undefined; + let db: ReturnType["db"] | undefined; + const reply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + const [first, second] = input.projects[0]!.items; + // Another writer removes the first item while the model is answering. + db!.db.prepare(`UPDATE l3_items SET status = 'inactive', inactive_at = ? WHERE id = ?`).run(new Date().toISOString(), first!.id); + gone = first!.id; + return { reason: "x", ops: [ + { op: "replace", ids: [first!.id], content: "改写后的规则", evidence: [tool.id] }, + { op: "delete", id: second!.id, evidence: [tool.id] } + ] }; + }; + const env = setup({ reply }); + db = env.db; + captureTurn(env.service, "gone", env.project, [{ query: "看看" }]); + const job = queuedUnitJob(env.repos); + const unitKey = projectUnitKey(env.repos); + for (const content of ["规则一", "规则二"]) { + env.repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content, evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" }); + } + await env.pipeline.process(job); + expect(env.repos.l3Field.activeItems(unitKey).map((item) => item.content)).toEqual([]); + expect(env.repos.l3Field.getItem(gone!)!.replacedBy).toBeUndefined(); + }); + + it("shows a batch what was removed on evidence newer than some of its own: late or spanning yes, behind a backlog no", async () => { + const late = setup(); + captureTurn(late.service, "late", late.project); + const job = queuedUnitJob(late.repos); + const unitKey = projectUnitKey(late.repos); + const removed = late.repos.l3Field.insertItem({ + unitKey, kind: "project_constraint", content: "提交信息用英文", evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, + evidenceAt: "2099-01-01T00:00:00.000Z" + }); + // A newer conversation, applied while this batch sat in dead letter (or before this old history was + // scanned), deleted the rule. + late.repos.l3Field.inactivateItem(removed.id, { removedEvidenceAt: new Date(Date.now() + 60_000).toISOString() }); + await late.pipeline.process(job); + expect(late.inputs[0]).toMatchObject({ removed_by_newer_evidence: [{ kind: "project_constraint", content: "提交信息用英文" }] }); + + // Batches in order behind a backlog: the rule was removed on older evidence than this batch's, + // so this batch is free to bring it back. + const inOrder = setup(); + captureTurn(inOrder.service, "in-order", inOrder.project); + const olderRemoval = inOrder.repos.l3Field.insertItem({ + unitKey: projectUnitKey(inOrder.repos), kind: "project_constraint", content: "提交信息用中文", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" + }); + inOrder.repos.l3Field.inactivateItem(olderRemoval.id, { removedEvidenceAt: "2026-01-02T00:00:00.000Z", at: new Date(Date.now() + 60_000).toISOString() }); + await inOrder.pipeline.process(queuedUnitJob(inOrder.repos)); + expect(inOrder.inputs[0]).not.toHaveProperty("removed_by_newer_evidence"); + + // One batch spanning the removal: its older turn must not bring the rule back, so it is shown. + const spanning = setup(); + captureTurn(spanning.service, "spanning", spanning.project, [{ query: "提交信息用英文" }, { query: "再跑一次测试" }]); + const turns = spanning.db.db.prepare(`SELECT id FROM raw_turns ORDER BY created_at, id`).all() as Array<{ id: string }>; + spanning.db.db.prepare(`UPDATE raw_turns SET created_at = '2026-01-01T00:00:00.000Z' WHERE id = ?`).run(turns[0]!.id); + const removedBetween = spanning.repos.l3Field.insertItem({ + unitKey: projectUnitKey(spanning.repos), kind: "project_constraint", content: "提交信息用英文", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2025-12-01T00:00:00.000Z" + }); + spanning.repos.l3Field.inactivateItem(removedBetween.id, { removedEvidenceAt: "2026-06-01T00:00:00.000Z" }); + await spanning.pipeline.process(queuedUnitJob(spanning.repos)); + expect(spanning.inputs[0]).toMatchObject({ + removed_by_newer_evidence: [{ content: "提交信息用英文", removed_at: "2026-06-01T00:00:00.000Z" }] + }); + }); + + it("evicts the item confirmed longest ago when a kind goes over 10", async () => { + const { service, repos, pipeline, project } = setup(); + captureTurn(service, "evict-seed", project, [{ query: "看看" }]); + const seedJob = queuedUnitJob(repos); + const unitKey = projectUnitKey(repos); + for (let i = 0; i < 10; i += 1) { + repos.l3Field.insertItem({ + unitKey, kind: "project_constraint", content: `规则 ${i}`, + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: `2026-01-${String(10 + i)}T00:00:00.000Z` + }); + } + await pipeline.process(seedJob); + const constraints = repos.l3Field.activeItems(unitKey).filter((item) => item.kind === "project_constraint"); + expect(constraints).toHaveLength(10); + expect(constraints.map((item) => item.content)).not.toContain("规则 0"); + }); + + it("keeps only the first group's change when two groups change the same machine item", async () => { + const reply: Reply = (input) => { + const machineItem = input.machine.items[0]!; + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + return input.projects[0]?.name === "demo" + ? { reason: "a", ops: [{ op: "replace", ids: [machineItem.id], content: "本机 zsh 展开 = 参数 -> 要加引号", evidence: [tool.id] }] } + : { reason: "b", ops: [{ op: "delete", id: machineItem.id, evidence: [tool.id] }] }; + }; + const { service, repos, pipeline, project, other } = setup({ reply }); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + const old = repos.l3Field.insertItem({ + unitKey: machine.unitKey, kind: "env_fact", content: "旧的本机事实", + evidence: { fragmentIds: ["f"], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" + }); + captureTurn(service, "conflict", project, [{ query: "跑一下", workdir: project }, { query: "那边也跑", workdir: other }]); + await pipeline.process(queuedUnitJob(repos)); + const replaced = repos.l3Field.getItem(old.id)!; + expect(replaced.status).toBe("inactive"); + expect(replaced.replacedBy).toBeTruthy(); + expect(repos.l3Field.getItem(replaced.replacedBy!)!.content).toBe("本机 zsh 展开 = 参数 -> 要加引号"); + }); + + it("marks a batch done when the model changes nothing, and merges several items with one replace", async () => { + const empty = setup({ reply: () => ({ reason: "nothing worth keeping", ops: [] }) }); + captureTurn(empty.service, "empty", empty.project); + const job = queuedUnitJob(empty.repos); + await empty.pipeline.process(job); + expect(empty.repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + + const merge = setup({ + reply: (input) => ({ + reason: "merge", + ops: [{ op: "replace", ids: input.projects[0]!.items.map((entry) => entry.id), content: "合并后的规则", evidence: [input.evidence[0]!.id] }] + }) + }); + captureTurn(merge.service, "merge", merge.project); + const unitKey = projectUnitKey(merge.repos); + const a = merge.repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "规则 a", evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" }); + const b = merge.repos.l3Field.insertItem({ unitKey, kind: "project_constraint", content: "规则 b", evidence: { fragmentIds: [], l1MemoryIds: [] }, evidenceAt: "2026-01-01T00:00:00.000Z" }); + await merge.pipeline.process(queuedUnitJob(merge.repos)); + const active = merge.repos.l3Field.activeItems(unitKey); + expect(active.map((entry) => entry.content)).toEqual(["合并后的规则"]); + expect(merge.repos.l3Field.getItem(a.id)!.replacedBy).toBe(active[0]!.id); + expect(merge.repos.l3Field.getItem(b.id)!.replacedBy).toBe(active[0]!.id); + }); + + it("writes the probed machine profile at session start", async () => { + const root = fixture.createTestRoot("l3-machine-profile-"); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = true; + const probe = vi.fn(async () => ({ + os: "darwin", osVersion: "26.1", arch: "arm64", shell: "zsh", + tools: { node: "22.11.0", python3: null, git: "2.50.1", npm: null, brew: null, pnpm: null, yarn: null, uv: null, conda: null, docker: null, java: null, go: null, rustc: null } + })); + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: join(root, "home"), tempRoots: [] }), + l3MachineProbe: probe + }); + service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + namespace: { source: "codex", profileId: "default", sessionKey: "probe-1", userId: "u1" } + }); + service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + namespace: { source: "codex", profileId: "default", sessionKey: "probe-2", userId: "u1" } + }); + await vi.waitFor(() => { + const machine = new Repositories(db.db).l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + expect(machine.profileText).toContain("node 22.11.0，git 2.50.1"); + }); + expect(probe).toHaveBeenCalledTimes(1); + }); + + it("shows later groups the machine items earlier groups of the batch added, so they are not written twice", async () => { + const reply: Reply = (input) => { + const tool = input.evidence.find((fragment) => fragment.type === "tool")!; + const added = (input.machine.items as Array<{ id: string; added_in_this_batch?: boolean }>) + .find((entry) => entry.added_in_this_batch); + return added + ? { reason: "already there", ops: [{ op: "confirm", id: added.id, evidence: [tool.id] }] } + : { reason: "new", ops: [{ op: "add", unit: "machine", kind: "env_fact", content: "本机 git 会打印 xcrun_db 警告 -> 结果仍正常", evidence: [tool.id] }] }; + }; + const { service, repos, pipeline, project, other, complete, inputs } = setup({ reply }); + captureTurn(service, "dedupe", project, [{ query: "跑一下", workdir: project }, { query: "那边也跑", workdir: other }]); + await pipeline.process(queuedUnitJob(repos)); + const machine = repos.l3Field.ensureMachineUnit({ userId: "u1", machineId: MACHINE_ID }); + expect(repos.l3Field.activeItems(machine.unitKey).map((entry) => entry.content)).toEqual(["本机 git 会打印 xcrun_db 警告 -> 结果仍正常"]); + expect(complete).toHaveBeenCalledTimes(2); + expect(JSON.stringify(inputs[1]!.machine.items)).toContain("\"added_in_this_batch\":true"); + }); + + it("drops old tool calls before the user's own words when the input is too long", async () => { + const { service, repos, pipeline, project, inputs } = setup({ reply: () => ({ reason: "none", ops: [] }) }); + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: "long", userId: "u1" } + }); + for (let index = 0; index < 20; index += 1) { + service.completeTurn(`long-turn-${index}`, { + sessionId: opened.sessionId, + query: index === 0 ? "这个项目以后提交信息都用中文写" : `继续第 ${index} 步`, + answer: "好的", + status: "succeeded", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: project }, success: false }], + toolResults: [{ id: `call-${index}`, name: "exec_command", output: Array.from({ length: 50 }, () => `error: ${"y".repeat(400)}`).join("\n"), exitCode: 1 }] + }); + } + service.closeSession(opened.sessionId); + const job = repos.runtime.listJobs("queued", 500).find((candidate) => candidate.jobType === "l3_unit_update")!; + vi.stubEnv("MEMMY_LOG_LEVEL", "debug"); + const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + try { + await pipeline.process(job); + } finally { + vi.unstubAllEnvs(); + } + const truncated = stdout.mock.calls.map(([line]) => String(line)).filter((line) => line.includes("input.truncated")); + stdout.mockRestore(); + const evidence = inputs[0]!.evidence as Array<{ type: string; text?: string }>; + expect(evidence.filter((fragment) => fragment.type === "tool").length).toBeLessThan(20); + expect(evidence.some((fragment) => fragment.text === "这个项目以后提交信息都用中文写")).toBe(true); + expect(truncated).toHaveLength(1); + expect(JSON.parse(truncated[0]!).level).toBe("debug"); + expect(truncated[0]).toContain("maxChars=65536"); + expect(truncated[0]).toContain("droppedUserTextFragments=0"); + }); + + it("asks once more with a smaller input when the model rejects the input as too long", async () => { + const sizes: number[] = []; + const reply: Reply = (input) => { + const size = JSON.stringify(input).length; + sizes.push(size); + if (size > 20_000) { + throw new ModelHttpError("model request failed", "host", 400, undefined, "This model's maximum context length is 8192 tokens"); + } + return { reason: "none", ops: [] }; + }; + const { service, repos, pipeline, project, complete } = setup({ reply }); + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, + workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: "too-long", userId: "u1" } + }); + for (let index = 0; index < 20; index += 1) { + service.completeTurn(`too-long-turn-${index}`, { + sessionId: opened.sessionId, + query: `第 ${index} 步`, + answer: "好的", + status: "succeeded", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: project }, success: false }], + toolResults: [{ id: `call-${index}`, name: "exec_command", output: Array.from({ length: 50 }, () => `error: ${"y".repeat(400)}`).join("\n"), exitCode: 1 }] + }); + } + service.closeSession(opened.sessionId); + const job = repos.runtime.listJobs("queued", 500).find((candidate) => candidate.jobType === "l3_unit_update")!; + await pipeline.process(job); + expect(sizes).toHaveLength(2); + expect(sizes[1]!).toBeLessThan(20_000); + expect(repos.l3Field.getBatch(job.payload.batchId as string)!.status).toBe("applied"); + expect(complete.mock.calls.every(([, options]) => options?.maxTokens === 16_384)).toBe(true); + }); + + it("does not let a deleted L1 whose summary never finished hold its batch back", () => { + const { service, repos, project } = setup(); + captureTurn(service, "deleted-l1", project); + const job = queuedUnitJob(repos); + const l1 = repos.l3Field.getBatch(job.payload.batchId as string)!.l1MemoryIds[0]!; + repos.db.prepare( + `INSERT INTO memory_processing_state (memory_id, state, updated_at) VALUES (?, 'summary_pending', ?) + ON CONFLICT(memory_id) DO UPDATE SET state = 'summary_pending'` + ).run(l1, new Date().toISOString()); + repos.memories.softDelete(l1, new Date().toISOString()); + const leased = repos.runtime.leaseQueuedJobs(50, 60, undefined, false, ["l3_unit_update"]); + expect(leased.map((entry) => entry.id)).toContain(job.id); + }); + + it("does not wake the worker for new-L3 jobs it would not lease while the switch is off", () => { + const { service, repos, project } = setup({ enabled: false }); + captureTurn(service, "wake-off", project); + const now = new Date().toISOString(); + repos.db.prepare( + `INSERT INTO evolution_jobs (id, job_type, status, user_id, payload_json, attempts, max_attempts, created_at, updated_at) + VALUES ('held-unit', 'l3_unit_update', 'queued', 'u1', json_object('batchId', 'x', 'runAfter', '2000-01-01T00:00:00.000Z'), 0, 6, ?, ?), + ('held-profile', 'project_environment_profile', 'queued', 'u1', + json_object('mode', 'l3_field', 'runAfter', '2000-01-01T00:00:00.000Z'), 0, 3, ?, ?)` + ).run(now, now, now, now); + repos.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE id NOT IN ('held-unit', 'held-profile')`).run(); + expect(service.nextWorkerRunAt()).toBeUndefined(); + }); + + it("does not lease a batch while its L1 summary is pending, and leaves jobs queued while the switch is off", () => { + const { service, repos, project } = setup(); + captureTurn(service, "lease", project); + const job = queuedUnitJob(repos); + const l1 = (repos.l3Field.getBatch(job.payload.batchId as string)!.l1MemoryIds)[0]!; + const now = new Date().toISOString(); + repos.db.prepare( + `INSERT INTO memory_processing_state (memory_id, state, updated_at) VALUES (?, 'summary_pending', ?) + ON CONFLICT(memory_id) DO UPDATE SET state = 'summary_pending'` + ).run(l1, now); + const leasedIds = () => repos.runtime.leaseQueuedJobs(50, 60, undefined, false, ["l3_unit_update"]).map((leased) => leased.id); + expect(leasedIds()).not.toContain(job.id); + + repos.db.prepare(`UPDATE memory_processing_state SET state = 'ready' WHERE memory_id = ?`).run(l1); + const offLeased = repos.runtime.leaseQueuedJobs(50, 60, undefined, false, undefined, ["l3_unit_update"]); + expect(offLeased.some((leased) => leased.jobType === "l3_unit_update")).toBe(false); + expect(offLeased.some((leased) => leased.payload.mode === "l3_field")).toBe(false); + expect(leasedIds()).toContain(job.id); + }); +}); + +describe("new L3 op validation", () => { + const machine = unit("machine", "m"); + const demo = unit("project", "p", { displayName: "demo-app", checkoutRoot: "/Users/x/code/demo-app" }); + const fragment = (id: string, over: Partial = {}): EvidenceFragment => ({ + id, rawTurnId: "r", l1MemoryId: "l1", toolName: "exec_command", input: null, output: "error: boom", + isError: true, paths: [], candidateUnitKeys: ["p"], at: "2026-10-01T00:00:00.000Z", ...over + }); + const group: L3UpdateGroup = { + machine, + project: demo, + fragments: [ + fragment("tool-p"), + fragment("tool-other", { candidateUnitKeys: ["other"] }), + fragment("user", { toolName: null, output: "以后都这样", isError: false }) + ], + l1: [] + }; + const items = new Map([ + ["mi", item("mi", "m", "env_fact")], + ["pi", item("pi", "p", "project_constraint")], + ["pf", item("pf", "p", "env_fact")], + ["xi", item("xi", "elsewhere", "env_fact")] + ]); + const problems = (ops: unknown[]) => validateL3UnitOps(ops as never, group, items).problems; + + it("rejects ids of other units, inactive ids and an id changed twice", () => { + expect(problems([{ op: "delete", id: "xi", evidence: ["tool-p"] }])[0]).toContain("not an active item"); + expect(problems([{ op: "confirm", id: "gone", evidence: ["tool-p"] }])[0]).toContain("not an active item"); + expect(problems([ + { op: "confirm", id: "pi", evidence: ["user"] }, + { op: "delete", id: "pi", evidence: ["user"] } + ])[0]).toContain("already changed"); + }); + + it("rejects replace across units or kinds, and evidence from outside the input", () => { + expect(problems([{ op: "replace", ids: ["mi", "pf"], content: "x", evidence: ["tool-p"] }])[0]).toContain("one unit and kind"); + expect(problems([{ op: "replace", ids: ["pi", "pf"], content: "x", evidence: ["tool-p"] }])[0]).toContain("one unit and kind"); + expect(problems([{ op: "confirm", id: "pi", evidence: ["nope"] }])[0]).toContain("not a fragment"); + }); + + it("needs a tool output from this project for a project fact", () => { + expect(problems([{ op: "add", unit: "project", kind: "env_fact", content: "x", evidence: ["tool-other"] }])[0]).toContain("touched this unit"); + expect(problems([{ op: "add", unit: "project", kind: "env_fact", content: "x", evidence: ["user"] }])[0]).toContain("touched this unit"); + expect(problems([{ op: "add", unit: "project", kind: "env_fact", content: "x", evidence: ["tool-p"] }])).toEqual([]); + expect(problems([{ op: "add", unit: "machine", kind: "env_fact", content: "brew 无写权限", evidence: ["tool-other"] }])).toEqual([]); + }); + + it("keeps machine items about the machine only", () => { + expect(problems([{ op: "add", unit: "machine", kind: "project_constraint", content: "x", evidence: ["user"] }])[0]).toContain("env_fact"); + expect(problems([{ op: "add", unit: "machine", kind: "env_fact", content: "/Users/x/code/demo-app 里 npm 报错", evidence: ["tool-p"] }])[0]).toContain("must not mention"); + expect(problems([{ op: "add", unit: "machine", kind: "env_fact", content: "demo-app 的测试很慢", evidence: ["tool-p"] }])[0]).toContain("must not mention"); + const goGroup = { ...group, project: { ...demo, displayName: "go", checkoutRoot: "/Users/x/go" } }; + expect(validateL3UnitOps([{ op: "add", unit: "machine", kind: "env_fact", content: "go 1.22 编译报错", evidence: ["tool-p"] }] as never, goGroup, items).problems).toEqual([]); + }); + + it("asks for unit machine when no project is listed, and rejects an unknown unit_key", () => { + const noProject = { ...group, project: null }; + expect(validateL3UnitOps([{ op: "add", unit: "project", kind: "project_constraint", content: "x", evidence: ["user"] }] as never, noProject, items).problems[0]).toContain("unit must be machine"); + expect(problems([{ op: "add", unit: "project", unit_key: "zzz", kind: "project_constraint", content: "x", evidence: ["user"] }])[0]).toContain("not a listed project"); + }); +}); + +function unit(kind: "machine" | "project", unitKey: string, over: Partial = {}): L3UnitRecord { + return { unitKey, userId: "u1", machineId: MACHINE_ID, kind, nextSeq: 1, createdAt: "", updatedAt: "", ...over } as L3UnitRecord; +} + +function item(id: string, unitKey: string, kind: L3ItemRecord["kind"]): L3ItemRecord { + return { + id, unitKey, kind, content: id, evidence: { fragmentIds: [], l1MemoryIds: [] }, + evidenceAt: "", lastConfirmedAt: "", status: "active", createdAt: "", updatedAt: "" + }; +} + +function testLlm(complete: LlmClient["complete"]): LlmClient { + return { + config: { + provider: "host", endpoint: "http://localhost/unused", model: "l3-unit-update-test", apiKey: "", + temperature: 0, maxTokens: 4096, timeoutMs: 30_000, maxRetries: 0, malformedRetries: 0, enableThinking: false + }, + isConfigured: () => true, + complete, + completeJson: vi.fn(), + status: () => ({ provider: "host", model: "l3-unit-update-test", configured: true, remote: false }) + }; +} From 1eb2d9673d6fb118ab2f504216a54826fd73a1e5 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:19:15 +0200 Subject: [PATCH 8/9] fix(memory): retry and resume failed l3 jobs through the job queue Temporary model failures of L3 jobs wait 1, 5, 15, 30 and 60 minutes before retrying (runAfter) and the sixth goes to dead letter; any other failure (a rejected answer, a bug) would likely repeat, so it gets one retry. Later batches then continue: no failure leaves a job that holds back its scope until restart. Configuration problems (bad key, quota 40309) and an unreachable model endpoint (offline) wait 30 minutes without using an attempt, and L3 jobs are not leased at all while no evolution model is configured. Each dead-lettered new-L3 batch is rerun once automatically when the worker starts. Adds POST /api/v1/l3/failed-batches/retry, which resets the user's dead-letter l3_unit_update jobs and batches in place. Co-Authored-By: Claude Opus 5.5 --- .../src/memory-l3-world-model.ts | 12 + Memory/src/contracts/memory-l3-world-model.ts | 12 + Memory/src/server/http.ts | 18 ++ .../evolution/l3-world-model-pipeline.ts | 3 +- Memory/src/service/memory-service.ts | 30 ++ Memory/src/service/worker/job-handlers.ts | 51 +++ Memory/src/service/worker/worker-runner.ts | 48 ++- Memory/src/storage/l3-field-repository.ts | 38 +++ Memory/src/storage/repositories.ts | 22 ++ .../contract/memory-rest-service.test.ts | 6 +- .../service/evolution/l3-world-model.test.ts | 14 + .../tests/service/l3-field/l3-retry.test.ts | 301 ++++++++++++++++++ 12 files changed, 549 insertions(+), 6 deletions(-) create mode 100644 Memory/tests/service/l3-field/l3-retry.test.ts diff --git a/App/backend/local-api-contracts/src/memory-l3-world-model.ts b/App/backend/local-api-contracts/src/memory-l3-world-model.ts index 31b371365..338c318d6 100644 --- a/App/backend/local-api-contracts/src/memory-l3-world-model.ts +++ b/App/backend/local-api-contracts/src/memory-l3-world-model.ts @@ -103,6 +103,18 @@ export const L3UserProjectsReplaceResponseSchema = z.object({ }).strict(); export type L3UserProjectsReplaceResponse = z.infer; +/** Puts the user's new-L3 batches that ended in dead letter back in the queue. */ +export const L3FailedBatchesRetryRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3FailedBatchesRetryRequest = z.input; + +export const L3FailedBatchesRetryResponseSchema = z.object({ + requeued: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3FailedBatchesRetryResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/Memory/src/contracts/memory-l3-world-model.ts b/Memory/src/contracts/memory-l3-world-model.ts index 31b371365..338c318d6 100644 --- a/Memory/src/contracts/memory-l3-world-model.ts +++ b/Memory/src/contracts/memory-l3-world-model.ts @@ -103,6 +103,18 @@ export const L3UserProjectsReplaceResponseSchema = z.object({ }).strict(); export type L3UserProjectsReplaceResponse = z.infer; +/** Puts the user's new-L3 batches that ended in dead letter back in the queue. */ +export const L3FailedBatchesRetryRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3FailedBatchesRetryRequest = z.input; + +export const L3FailedBatchesRetryResponseSchema = z.object({ + requeued: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3FailedBatchesRetryResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index 6ad90c733..ac46fdfc3 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -8,6 +8,7 @@ import { import { EmbeddingInferenceInputSchema, L3UserProjectsReplaceRequestSchema, + L3FailedBatchesRetryRequestSchema, L3WorldModelBoundaryRequestSchema, L3WorldModelRequestEnvelopeSchema, OpenSessionInputSchema @@ -77,6 +78,7 @@ export const API_ROUTES = [ "POST /api/v1/sessions/:sessionId/l3-world-model-boundary", "GET /api/v1/l3-world-model/sessions/:sessionId/context", "POST /api/v1/l3/user-projects", + "POST /api/v1/l3/failed-batches/retry", "POST /api/v1/turns/start", "POST /api/v1/turns/:turnId/complete", "POST /api/v1/source-turns/complete", @@ -667,6 +669,22 @@ async function routeRequest( return service.replaceL3UserProjects(parsed.data); } + if (method === "POST" && path === "/api/v1/l3/failed-batches/retry") { + requireMemoryWrite(principal); + const parsed = L3FailedBatchesRetryRequestSchema.safeParse( + strictEnvelopeWithPrincipal(asObject(body, "l3.failed-batches.retry"), principal) + ); + if (!parsed.success) { + throw new MemoryServiceError( + "invalid_argument", + parsed.error.issues.map((issue) => `${issue.path.join(".") || "request"}: ${issue.message}`).join("; ") + ); + } + const retried = service.retryFailedL3Batches(parsed.data); + if (retried.requeued > 0) autoWorker.schedule(); + return retried; + } + const l3Context = match(path, /^\/api\/v1\/l3-world-model\/sessions\/([^/]+)\/context$/); if (method === "GET" && l3Context) { requireMemoryRead(principal); diff --git a/Memory/src/service/evolution/l3-world-model-pipeline.ts b/Memory/src/service/evolution/l3-world-model-pipeline.ts index 75fef7cb2..2c3ef175f 100644 --- a/Memory/src/service/evolution/l3-world-model-pipeline.ts +++ b/Memory/src/service/evolution/l3-world-model-pipeline.ts @@ -197,7 +197,8 @@ function strictJobPayload(job: EvolutionJobRecord): { batchId: string; targetField: L3WorldModelTargetField; } { - const keys = Object.keys(job.payload).sort(); + // runAfter is the queue's own retry time, not part of the job's evidence. + const keys = Object.keys(job.payload).filter((key) => key !== "runAfter").sort(); if (keys.join(",") !== "batchId,targetField") { throw new L3WorldModelTerminalEvidenceError("invalid L3 World Model job payload keys"); } diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index f4a374397..af8f7f65d 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -5,6 +5,8 @@ import { deriveWorkspaceHostId, isLocalWorkspaceUri, sha256Hex, + type L3FailedBatchesRetryRequest, + type L3FailedBatchesRetryResponse, type L3UserProjectsReplaceRequest, type L3UserProjectsReplaceResponse } from "../contracts/index.js"; @@ -593,6 +595,8 @@ export class MemoryService { restartFailedProcessing: this.restartFailedProcessing.bind(this), enqueueObservationBackfill: (limit, at) => this.observationMemory.backfill(limit, at), l3FieldActive: () => this.l3FieldActive(), + requeueL3DeadLettersOnce: (at) => this.requeueL3DeadLettersOnce(at), + evolutionModelConfigured: () => workerRunnerOwner.skillLlm.isConfigured(), purgeL3InactiveItems: (at) => this.repos.l3Field.purgeInactive( new Date(Date.parse(at) - this.config.algorithm.l3FieldCognition.inactiveRetentionDays * 86_400_000).toISOString() ), @@ -1378,6 +1382,32 @@ export class MemoryService { return { count: request.projects.length, serverTime: at }; } + /** Rerun entry for the new L3 only; dead letters of the old pipeline are not rerun. */ + retryFailedL3Batches(request: L3FailedBatchesRetryRequest): L3FailedBatchesRetryResponse { + this.assertMemoryAddEnabled(); + const userId = normalizeNamespace(request.namespace).userId; + const at = nowIso(); + const before = new Map(this.repos.runtime.listJobs("dead_letter", 10_000, userId).map((job) => [job.id, job])); + const requeued = this.repos.l3Field.requeueDeadLetterBatches(userId, at); + for (const id of requeued) { + const after = this.repos.runtime.getJob(id); + if (after) this.workerHandlers.appendJobChange(after, "queued", before.get(id)); + } + return { requeued: requeued.length, serverTime: at }; + } + + /** Worker startup: one automatic rerun for each dead-lettered new-L3 batch; nothing while the new L3 is off. */ + private requeueL3DeadLettersOnce(at: string): number { + if (!this.l3FieldActive()) return 0; + const before = new Map(this.repos.runtime.listJobs("dead_letter", 10_000).map((job) => [job.id, job])); + const requeued = this.repos.l3Field.requeueDeadLetterBatchesOnce(at); + for (const id of requeued) { + const after = this.repos.runtime.getJob(id); + if (after) this.workerHandlers.appendJobChange(after, "queued", before.get(id)); + } + return requeued.length; + } + l3WorldModelBoundary( sessionId: string, request: L3WorldModelBoundaryRequest diff --git a/Memory/src/service/worker/job-handlers.ts b/Memory/src/service/worker/job-handlers.ts index cd62bfb23..e8ea7969b 100644 --- a/Memory/src/service/worker/job-handlers.ts +++ b/Memory/src/service/worker/job-handlers.ts @@ -665,6 +665,57 @@ export function classifyProcessingError(error: unknown): { return { code: "processing_failed", retryAction: "retry" }; } +/** L3 jobs whose failures are retried later instead of right away. */ +export const L3_RETRY_JOB_TYPES: readonly string[] = ["l3_world_model_update", "l3_unit_update", "project_environment_profile"]; +/** Waits before retries 1-5 of a temporary failure; the sixth run that fails goes to dead letter. */ +export const L3_RETRY_DELAYS_MINUTES = [1, 5, 15, 30, 60] as const; +/** A configuration problem (bad key, quota) or no connection is tried again this often, without using up an attempt. */ +export const L3_SUSPEND_MINUTES = 30; + +export type L3FailureHandling = + | { kind: "retry_later"; delayMinutes: number; maxAttempts: number } + | { kind: "suspend" } + | { kind: "dead_letter" } + | { kind: "default" }; + +/** Connection errors that mean the model endpoint cannot be reached at all, as when the computer is offline. */ +const CANNOT_CONNECT_CODES = new Set([ + "ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED", "ENETUNREACH", "EHOSTUNREACH", "ENETDOWN", "UND_ERR_CONNECT_TIMEOUT" +]); + +/** + * How a failed L3 job goes back into the queue. A configuration problem or no connection waits + * for a fix without using an attempt, so going offline loses nothing. Temporary model trouble + * retries five times; any other failure (a rejected answer, a bug) would most likely fail the same + * way, so it gets one more try. Then the job goes to dead letter, never to "failed": a failed L3 + * job left in the queue would hold back every later batch of the same scope until a restart. + */ +export function l3FailureHandling(job: EvolutionJobRecord, error: unknown): L3FailureHandling { + if (!L3_RETRY_JOB_TYPES.includes(job.jobType) || isTerminalL3WorldModelError(error)) return { kind: "default" }; + if (cannotConnect(error)) return { kind: "suspend" }; + const message = error instanceof Error ? error.message : String(error); + // What the model clients really throw (model/http.ts, model/llm.ts): an empty body or missing + // output, and fetch's TypeError or the timeout's AbortError. Checked before configuration errors, + // because an empty reply also says "expected JSON". + const emptyReply = /received an empty response|response missing |unexpected end of json input/i + .test(message); + const network = error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError" || + (error instanceof TypeError && /fetch failed|network|socket/i.test(message))); + const classification = classifyProcessingError(error); + if (!emptyReply && !network && classification.retryAction === "open_settings") return { kind: "suspend" }; + const temporary = emptyReply || network || classification.code === "transient_provider_error"; + const delay = (temporary ? L3_RETRY_DELAYS_MINUTES : L3_RETRY_DELAYS_MINUTES.slice(0, 1))[job.attempts - 1]; + if (delay === undefined) return { kind: "dead_letter" }; + return { kind: "retry_later", delayMinutes: delay, maxAttempts: L3_RETRY_DELAYS_MINUTES.length + 1 }; +} + +/** fetch's "fetch failed" whose cause says the host was never reached (no cause: same, nothing was sent). */ +function cannotConnect(error: unknown): boolean { + if (!(error instanceof TypeError) || !/fetch failed/i.test(error.message)) return false; + const code = (error.cause as { code?: unknown } | undefined)?.code; + return typeof code !== "string" || CANNOT_CONNECT_CODES.has(code); +} + export function evolutionJobDedupeKey(input: Pick): string | undefined { const payload = input.payload ?? {}; const payloadString = (key: string): string | undefined => { diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index 7384c13d0..67d7c8d6b 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -38,6 +38,10 @@ import { memoryNeedsImportSummary, summaryProcessingJobType } from "../import/im import { classifyProcessingError, type EnqueueJobInput, + L3_RETRY_JOB_TYPES, + L3_SUSPEND_MINUTES, + l3FailureHandling, + type L3FailureHandling, processingJobMatchesMemory, processingStageForJob, sanitizeProcessingError, @@ -135,8 +139,12 @@ export interface WorkerRunnerDeps { enqueueObservationBackfill?: (limit: number, at: string) => number; /** Drops new-L3 items that have been inactive longer than the retention window. */ purgeL3InactiveItems?: (at: string) => number; + /** Gives each dead-lettered new-L3 batch one automatic rerun. */ + requeueL3DeadLettersOnce?: (at: string) => number; /** Whether the new L3 is on; while off its jobs stay queued and are not leased. */ l3FieldActive?: () => boolean; + /** Without an evolution model, L3 jobs stay queued (not leased, no attempt used) until one is configured. */ + evolutionModelConfigured?: () => boolean; previewPolicyEvidenceReconciliation: (limit: number) => PolicyEvidencePreflightReport; reconcileOrphanedPolicies: (at: string, limit: number) => number; enqueueJob: (input: EnqueueJobInput) => EvolutionJobRecord; @@ -386,6 +394,7 @@ export class WorkerRunner { } this.deps.purgeL3InactiveItems?.(at); + this.deps.requeueL3DeadLettersOnce?.(at); this.refreshObservationModelConfiguration(); return { requeuedJobs: interruptedJobs.length + failedJobs.length, @@ -641,12 +650,14 @@ export class WorkerRunner { this.deps.appendJobChange(deferred, "queued", job); return { succeeded: 0, failed: 0, ref: { ...jobToRef(deferred), status: "queued" } }; } + const l3Handling = l3FailureHandling(job, error); + if (l3Handling.kind === "retry_later" || l3Handling.kind === "suspend") return this.rescheduleFailedL3Job(job, error, l3Handling); const errorMessage = processingStageForJob(job.jobType) ? sanitizeProcessingError(error) : error instanceof Error ? error.message : String(error); const stage = processingStageForJob(job.jobType); const classification = classifyProcessingError(error); - const forceDeadLetter = classification.code === "40309" + const forceDeadLetter = l3Handling.kind === "dead_letter" || classification.code === "40309" || Boolean(stage && classification.retryAction !== "retry"); const failedJob = this.deps.repos.runtime.failJob( job.id, @@ -678,6 +689,34 @@ export class WorkerRunner { }; } + /** Temporary trouble waits and retries; a configuration problem waits for a fix without using an attempt. */ + private rescheduleFailedL3Job( + job: EvolutionJobRecord, + error: unknown, + handling: Extract + ): WorkerJobRunResult { + const at = this.deps.nowIso(); + const minutes = handling.kind === "retry_later" ? handling.delayMinutes : L3_SUSPEND_MINUTES; + const rescheduled = this.deps.repos.runtime.rescheduleFailedJob(job.id, { + runAfter: new Date(Date.parse(at) + minutes * 60_000).toISOString(), + error: sanitizeProcessingError(error), + refundAttempt: handling.kind === "suspend", + minMaxAttempts: handling.kind === "retry_later" ? handling.maxAttempts : undefined, + at + }) ?? job; + this.deps.appendJobChange(rescheduled, "queued", job); + workerLogger.warn(handling.kind === "suspend" ? "job.suspended" : "job.retry_scheduled", { + ...workerJobLogFields(rescheduled), + retryInMinutes: minutes, + ...memoryErrorFields(error) + }); + return { + succeeded: 0, + failed: handling.kind === "suspend" ? 0 : 1, + ref: { ...jobToRef(rescheduled), status: "queued" } + }; + } + markProcessingJobLeased(job: EvolutionJobRecord): void { if (!job.targetMemoryId) return; const stage = processingStageForJob(job.jobType); @@ -905,9 +944,12 @@ export class WorkerRunner { } private excludedLeaseJobTypes(): readonly string[] | undefined { - const held = this.summaryJobsHeld() ?? []; + const held = [ + ...(this.summaryJobsHeld() ?? []), + ...(this.deps.evolutionModelConfigured?.() === false ? L3_RETRY_JOB_TYPES : []) + ]; // Excluding l3_unit_update also keeps new-L3 profile jobs in the queue (see leaseQueuedJobs). - const excluded = this.deps.l3FieldActive?.() === false ? [...held, "l3_unit_update"] : held; + const excluded = this.deps.l3FieldActive?.() === false ? [...new Set([...held, "l3_unit_update"])] : held; return excluded.length > 0 ? excluded : undefined; } diff --git a/Memory/src/storage/l3-field-repository.ts b/Memory/src/storage/l3-field-repository.ts index 13690a1a7..ca846de82 100644 --- a/Memory/src/storage/l3-field-repository.ts +++ b/Memory/src/storage/l3-field-repository.ts @@ -369,6 +369,44 @@ export class L3FieldRepository { return row ? batchFromSql(row) : undefined; } + /** + * Rerun entry: the user's batches whose job ended in dead letter go back in the queue. The original + * job is reset in place (a copy would hit the dedupe and sequence indexes); applied batches are left alone. + * Returns the ids of the jobs put back. + */ + requeueDeadLetterBatches(userId: string, at = nowIso()): string[] { + return this.requeueDeadLetters(`AND j.user_id = ?`, [userId], at, false); + } + + /** + * Worker startup: every user's dead-lettered batches get one automatic rerun each, so a batch that + * failed through a long outage is not lost; a batch that dead-letters again stays for the manual rerun. + */ + requeueDeadLetterBatchesOnce(at = nowIso()): string[] { + return this.requeueDeadLetters(`AND json_extract(j.payload_json, '$.autoRequeued') IS NULL`, [], at, true); + } + + private requeueDeadLetters(filter: string, params: string[], at: string, markAuto: boolean): string[] { + return this.db.transaction(() => { + const jobs = this.db.prepare( + `SELECT j.id, b.id AS batch_id FROM evolution_jobs j + JOIN l3_unit_batches b ON b.id = CAST(json_extract(j.payload_json, '$.batchId') AS TEXT) + WHERE j.job_type = 'l3_unit_update' AND j.status = 'dead_letter' AND b.status != 'applied' ${filter} + ORDER BY j.scope_key, j.scope_seq` + ).all(...params) as Array<{ id: string; batch_id: string }>; + for (const job of jobs) { + this.db.prepare(`UPDATE l3_unit_batches SET status = 'queued', updated_at = ? WHERE id = ?`).run(at, job.batch_id); + this.db.prepare( + `UPDATE evolution_jobs SET status = 'queued', attempts = 0, leased_until = NULL, last_error = NULL, + payload_json = ${markAuto ? "json_set(json_remove(payload_json, '$.runAfter'), '$.autoRequeued', 1)" : "json_remove(payload_json, '$.runAfter')"}, + updated_at = ? + WHERE id = ?` + ).run(at, job.id); + } + return jobs.map((job) => job.id); + })(); + } + setBatchStatus(id: string, status: L3UnitBatchStatus, at = nowIso()): void { this.db.prepare(`UPDATE l3_unit_batches SET status = ?, updated_at = ? WHERE id = ?`).run(status, at, id); } diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index e36b69c1f..b2585a9b0 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -3743,6 +3743,28 @@ export class RuntimeRepository { return this.getJob(id); } + /** + * Puts a failed, leased job back in the queue to run after `runAfter`. A refunded attempt means the + * run did not count (waiting for a configuration fix); `minMaxAttempts` raises the attempt limit. + */ + rescheduleFailedJob(id: string, input: { + runAfter: string; + error: string; + refundAttempt?: boolean; + minMaxAttempts?: number; + at?: string; + }): EvolutionJobRecord | undefined { + const at = input.at ?? nowIso(); + this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, last_error = ?, + attempts = CASE WHEN ? THEN MAX(0, attempts - 1) ELSE attempts END, + max_attempts = MAX(max_attempts, ?), updated_at = ?, + payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE id = ? AND status = 'leased'`).run( + input.error, input.refundAttempt ? 1 : 0, input.minMaxAttempts ?? 0, at, input.runAfter, id + ); + return this.getJob(id); + } + /** Waiting for a dependency is not a model failure and does not consume an attempt. */ deferProcessingJob(id: string, runAfter: string, at = nowIso()): EvolutionJobRecord | undefined { this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, diff --git a/Memory/tests/contract/memory-rest-service.test.ts b/Memory/tests/contract/memory-rest-service.test.ts index c69edbce7..0a8d33363 100644 --- a/Memory/tests/contract/memory-rest-service.test.ts +++ b/Memory/tests/contract/memory-rest-service.test.ts @@ -738,7 +738,8 @@ describe("MemoryService / REST contract", () => { }); it("auto-drains L3 World Model jobs frozen by a REST session.close with no open episode", async () => { - const { db, service } = createTestService(); + // L3 jobs wait in the queue while no evolution model is configured, so configure one. + const { db, service } = createTestService({ skillLlm: createDeterministicSummaryLlm("unused") }); const opened = service.openSession({ l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", @@ -797,7 +798,8 @@ describe("MemoryService / REST contract", () => { }); expect(l3Jobs()).toHaveLength(1); - await waitFor(() => l3Jobs().every((job) => job.status !== "queued" && job.attempts > 0)); + // The job ran once; a failed L3 run goes back to the queue to retry later. + await waitFor(() => l3Jobs().every((job) => job.attempts > 0 && job.status !== "leased")); }); db.close(); diff --git a/Memory/tests/service/evolution/l3-world-model.test.ts b/Memory/tests/service/evolution/l3-world-model.test.ts index 2b7246113..2d9b4f2f4 100644 --- a/Memory/tests/service/evolution/l3-world-model.test.ts +++ b/Memory/tests/service/evolution/l3-world-model.test.ts @@ -165,6 +165,20 @@ describe("L3 World Model trace field pipeline", () => { db.close(); }); + it("accepts a job that the queue rescheduled with runAfter", async () => { + const llm = fieldLlm(); + const { db, service } = createTestService({ skillLlm: llm }); + const opened = openProject(service, "l3-run-after-user", "l3-run-after-session"); + service.completeTurn("l3-run-after-turn", { + sessionId: opened.sessionId, query: "这个项目必须先运行测试", answer: "好", status: "succeeded" + }); + service.closeSession(opened.sessionId); + const repos = new Repositories(db.db); + const job = repos.runtime.listJobs("queued", 100).find((candidate) => candidate.jobType === "l3_world_model_update")!; + await expect(new L3WorldModelTraceFieldPipeline({ repos, skillLlm: llm }) + .updateField({ ...job, payload: { ...job.payload, runAfter: "2026-10-24T00:00:00.000Z" } })).resolves.toBeUndefined(); + }); + it("removes reasoning from no-project generation and repair input", async () => { const complete = vi.fn() .mockResolvedValueOnce("{}") diff --git a/Memory/tests/service/l3-field/l3-retry.test.ts b/Memory/tests/service/l3-field/l3-retry.test.ts new file mode 100644 index 000000000..155982e49 --- /dev/null +++ b/Memory/tests/service/l3-field/l3-retry.test.ts @@ -0,0 +1,301 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMemoryHttpServer, DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { ModelHttpError } from "../../../src/model/http.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { l3FailureHandling } from "../../../src/service/worker/job-handlers.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const MACHINE_ID = "9".repeat(64); +const networkError = () => new Error("fetch failed: network connect ECONNRESET"); +/** Reached the host, then lost the connection: retried, unlike being offline. */ +const networkErrorAfterConnect = () => new TypeError("fetch failed", { cause: { code: "ECONNRESET" } }); + +function job(over: Partial = {}): EvolutionJobRecord { + return { id: "j", jobType: "l3_unit_update", status: "leased", userId: "u", payload: {}, attempts: 1, maxAttempts: 3, + createdAt: "", updatedAt: "", ...over } as EvolutionJobRecord; +} + +describe("L3 failure handling", () => { + it("waits 1, 5, 15, 30 and 60 minutes after temporary failures, then sends the sixth failure to dead letter", () => { + expect([1, 2, 3, 4, 5].map((attempts) => l3FailureHandling(job({ attempts }), networkError()))) + .toEqual([1, 5, 15, 30, 60].map((delayMinutes) => ({ kind: "retry_later", delayMinutes, maxAttempts: 6 }))); + expect(l3FailureHandling(job({ attempts: 6 }), networkError())).toEqual({ kind: "dead_letter" }); + }); + + it("recognizes the errors the model clients really throw", () => { + const aborted = Object.assign(new Error("This operation was aborted"), { name: "AbortError" }); + expect(l3FailureHandling(job(), aborted).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new TypeError("fetch failed", { cause: { code: "ECONNRESET" } })).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("openai_responses response missing output text")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("bedrock response missing output.message.content")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("host response missing text/content/output")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("deepseek HTTP 200: expected JSON but received an empty response")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("p HTTP 200: expected JSON but received text/html; check the configured model endpoint")).kind) + .toBe("suspend"); + expect(l3FailureHandling(job(), new TypeError("Cannot read properties of undefined (reading 'x')")).kind).toBe("retry_later"); + }); + + it("suspends without using an attempt when the model endpoint cannot be reached, as when offline", () => { + expect(l3FailureHandling(job(), new TypeError("fetch failed")).kind).toBe("suspend"); + expect(l3FailureHandling(job(), new TypeError("fetch failed", { cause: { code: "ENOTFOUND" } })).kind).toBe("suspend"); + expect(l3FailureHandling(job({ attempts: 6 }), new TypeError("fetch failed", { cause: { code: "ECONNREFUSED" } })).kind) + .toBe("suspend"); + }); + + it("retries empty replies, timeouts, rate limits and rejected answers later, and leaves other job types alone", () => { + const rejected = new Error("l3 unit output rejected twice: ops[0]: content is empty"); + expect(l3FailureHandling(job({ attempts: 1 }), rejected)).toEqual({ kind: "retry_later", delayMinutes: 1, maxAttempts: 6 }); + // Likely to fail the same way again: one more try only. + expect(l3FailureHandling(job({ attempts: 2 }), rejected)).toEqual({ kind: "dead_letter" }); + expect(l3FailureHandling(job(), new Error("openai_compatible response missing choices[0].message.content")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new TypeError("invalid JSON: Unexpected end of JSON input")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("request timed out after 180000ms")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new ModelHttpError("too many", "p", 429, undefined, "rate limit")).kind).toBe("retry_later"); + expect(l3FailureHandling(job(), new Error("l3 unit output rejected twice: ops[0]: content is empty")).kind).toBe("retry_later"); + expect(l3FailureHandling(job({ jobType: "reward" }), networkError()).kind).toBe("default"); + }); + + it("suspends on configuration problems such as a bad key or quota (40309)", () => { + expect(l3FailureHandling(job(), new ModelHttpError("quota", "p", 403, "40309", "")).kind).toBe("suspend"); + expect(l3FailureHandling(job(), new Error("401 unauthorized: invalid api key")).kind).toBe("suspend"); + }); +}); + +describe("L3 retries through the job queue", () => { + function setup(complete: LlmClient["complete"], configured = true) { + const root = fixture.createTestRoot("l3-retry-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = true; + const skillLlm = testLlm(complete, configured); + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, skillLlm, llm: testLlm(vi.fn(async () => { throw new Error("summary not used"); }), configured), + l3FieldEnvironment: () => ({ machineId: MACHINE_ID, homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const repos = new Repositories(db.db); + const sessions: string[] = []; + const freeze = (key: string) => { + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + workspaceUri: pathToFileURL(project).href, workspaceHostId: "b".repeat(64), + namespace: { source: "codex", profileId: "default", sessionKey: key, userId: "u1" } + }); + service.completeTurn(`${key}-turn`, { + sessionId: opened.sessionId, query: "以后改完都跑 npm test", answer: "好", status: "succeeded", + toolCalls: [{ id: "c", name: "exec_command", input: { cmd: "npm test", workdir: project }, success: false }], + toolResults: [{ id: "c", name: "exec_command", output: "vitest: command not found", exitCode: 127 }] + }); + service.closeSession(opened.sessionId); + sessions.push(opened.sessionId); + // This test is about the L3 job, not summaries: mark the L1 summaries done so the job can be leased. + repos.db.prepare(`UPDATE memory_processing_state SET state = 'ready'`).run(); + repos.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE job_type IN ('trace_summary', 'episode_title')`).run(); + }; + const unitJobs = () => repos.db.prepare( + `SELECT id, status, attempts, max_attempts, scope_seq, json_extract(payload_json, '$.runAfter') AS run_after, + json_extract(payload_json, '$.batchId') AS batch_id + FROM evolution_jobs WHERE job_type = 'l3_unit_update' ORDER BY scope_seq` + ).all() as Array<{ id: string; status: string; attempts: number; max_attempts: number; scope_seq: number; run_after: string | null; batch_id: string }>; + const runNow = async () => { + repos.db.prepare( + `UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', '2000-01-01T00:00:00.000Z') + WHERE job_type = 'l3_unit_update' AND status = 'queued' AND json_extract(payload_json, '$.runAfter') IS NOT NULL` + ).run(); + await service.runWorkerOnce(50); + }; + return { service, repos, freeze, unitJobs, runNow }; + } + + it("retries a temporary failure later, dead-letters the sixth failure, lets the next batch run, and reruns on request", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw networkError(); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, repos, freeze, unitJobs, runNow } = setup(complete); + freeze("first"); + await service.runWorkerOnce(50); + let [first] = unitJobs(); + expect(first).toMatchObject({ status: "queued", attempts: 1, max_attempts: 6 }); + expect(Date.parse(first!.run_after!) - Date.now()).toBeGreaterThan(50_000); + + for (let run = 2; run <= 6; run += 1) await runNow(); + [first] = unitJobs(); + expect(first).toMatchObject({ status: "dead_letter", attempts: 6 }); + expect(repos.l3Field.getBatch(first!.batch_id)!.status).toBe("failed"); + + failing = false; + freeze("second"); + await service.runWorkerOnce(50); + expect(unitJobs()[1]).toMatchObject({ status: "succeeded" }); + + expect(service.retryFailedL3Batches({ + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", + adapterId: "codex-memory", + source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "x", userId: "u1" } + })) + .toMatchObject({ requeued: 1 }); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0, run_after: null }); + await service.runWorkerOnce(50); + expect(unitJobs()[0]).toMatchObject({ status: "succeeded" }); + expect(repos.l3Field.getBatch(first!.batch_id)!.status).toBe("applied"); + }); + + it("gives a dead-lettered batch one automatic rerun when the worker starts, and only one", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw networkErrorAfterConnect(); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, repos, freeze, unitJobs, runNow } = setup(complete); + freeze("restart"); + await service.runWorkerOnce(50); + for (let run = 2; run <= 6; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + + service.reconcileWorkerStartup(); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0 }); + expect(repos.l3Field.getBatch(unitJobs()[0]!.batch_id)!.status).toBe("queued"); + await runNow(); + for (let run = 2; run <= 6; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + + service.reconcileWorkerStartup(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + failing = false; + }); + + it("wakes the worker when the rerun endpoint puts batches back", async () => { + let failing = true; + const complete = vi.fn(async () => { + if (failing) throw networkError(); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, freeze, unitJobs, runNow } = setup(complete); + freeze("http-rerun"); + await service.runWorkerOnce(50); + for (let run = 2; run <= 6; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter" }); + failing = false; + + // The startup fallback stays out of the window, so only the endpoint can wake the worker. + const server = createMemoryHttpServer({ + service, startAgentSourceAutomation: false, auth: { mode: "local", allowAnonymous: true }, workerStartupFallbackMs: 60_000 + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const response = await fetch(`http://127.0.0.1:${address.port}/api/v1/l3/failed-batches/retry`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", adapterId: "codex-memory", source: "codex", + namespace: { source: "codex", profileId: "default", sessionKey: "x", userId: "u1" } + }) + }); + expect(await response.json()).toMatchObject({ requeued: 1 }); + await vi.waitFor(() => expect(unitJobs()[0]).toMatchObject({ status: "succeeded" }), { timeout: 5000 }); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it("does not let a rejected answer hold back later batches until a restart", async () => { + let bad = true; + const complete = vi.fn(async () => + JSON.stringify(bad ? { reason: "bad", ops: [{ op: "confirm", id: "missing", evidence: [] }] } : { reason: "ok", ops: [] })); + const { service, freeze, unitJobs, runNow } = setup(complete); + freeze("rejected"); + freeze("after"); + await service.runWorkerOnce(50); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 1, max_attempts: 6 }); + await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "dead_letter", attempts: 2 }); + bad = false; + await runNow(); + expect(unitJobs()[1]).toMatchObject({ status: "succeeded" }); + }); + + it("waits out an offline spell without using attempts, then runs the batch", async () => { + let offline = true; + const complete = vi.fn(async () => { + if (offline) throw new TypeError("fetch failed", { cause: { code: "ENOTFOUND" } }); + return JSON.stringify({ reason: "ok", ops: [] }); + }); + const { service, freeze, unitJobs, runNow } = setup(complete); + freeze("offline"); + await service.runWorkerOnce(50); + const [suspended] = unitJobs(); + expect(suspended).toMatchObject({ status: "queued", attempts: 0 }); + expect(Date.parse(suspended!.run_after!) - Date.now()).toBeGreaterThan(29 * 60_000); + for (let run = 0; run < 10; run += 1) await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0 }); + offline = false; + await runNow(); + expect(unitJobs()[0]).toMatchObject({ status: "succeeded" }); + }); + + it("does not wake the worker for a batch whose wait is over but an earlier batch still holds it back", () => { + const db = new MemoryDb({ path: ":memory:" }); + const repos = new Repositories(db.db); + const at = new Date().toISOString(); + for (const [id, seq] of [["first", 1], ["second", 2]] as const) { + repos.l3WorldModels.insertImmutableJob({ + id, jobType: "l3_unit_update", status: "queued", dedupeKey: `k-${id}`, userId: "u", sessionId: "s", + scopeKey: "machine", scopeSeq: seq, payload: { batchId: `b-${id}` }, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at + }); + } + // The second batch failed and its wait is over; the first was put back from dead letter and now waits 30 minutes. + db.db.prepare(`UPDATE evolution_jobs SET status = 'leased', attempts = 1`).run(); + repos.runtime.rescheduleFailedJob("second", { runAfter: new Date(Date.now() - 1000).toISOString(), error: "x" }); + repos.runtime.rescheduleFailedJob("first", { runAfter: new Date(Date.now() + 30 * 60_000).toISOString(), error: "offline" }); + expect(repos.runtime.leaseQueuedJobs(10, 60)).toEqual([]); + expect(repos.runtime.nextWorkerRunAt()! - Date.now()).toBeGreaterThan(29 * 60_000); + db.close(); + }); + + it("suspends on a quota error without using an attempt", async () => { + const complete = vi.fn(async () => { throw new ModelHttpError("quota", "p", 403, "40309", ""); }); + const { service, freeze, unitJobs } = setup(complete); + freeze("quota"); + await service.runWorkerOnce(50); + const [suspended] = unitJobs(); + expect(suspended).toMatchObject({ status: "queued", attempts: 0 }); + expect(Date.parse(suspended!.run_after!) - Date.now()).toBeGreaterThan(29 * 60_000); + }); + + it("leaves L3 jobs in the queue while no evolution model is configured", async () => { + const complete = vi.fn(); + const { service, freeze, unitJobs } = setup(complete, false); + freeze("unconfigured"); + await service.runWorkerOnce(50); + expect(unitJobs()[0]).toMatchObject({ status: "queued", attempts: 0 }); + expect(complete).not.toHaveBeenCalled(); + }); +}); + +function testLlm(complete: LlmClient["complete"], configured: boolean): LlmClient { + return { + config: { + provider: "host", endpoint: "http://localhost/unused", model: "l3-retry-test", apiKey: "", + temperature: 0, maxTokens: 4096, timeoutMs: 30_000, maxRetries: 0, malformedRetries: 0, enableThinking: false + }, + isConfigured: () => configured, + complete, + completeJson: vi.fn(), + status: () => ({ provider: "host", model: "l3-retry-test", configured, remote: false }) + }; +} From 7cd5a20a9d6d98030de89f87f38a31f68d7c7464 Mon Sep 17 00:00:00 2001 From: Anson-Se <120631742+Anson-Se@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:44:23 +0200 Subject: [PATCH 9/9] feat(memory): feed scanned sessions into l3 With the new L3 on, conversations first opened by an agent-source scan use the L3 v2 session protocol with Work Memory disabled, so their turns register as L3 input like Hook sessions; with it off, scans stay out of L3 as before, and a session scanned while it was on never goes to the old L3 after it is switched off: its turns wait for the new L3. Every captured turn has a durable owner: with the new L3 on, the existing Work Memory idle flush (one re-armed job per session, runAfter two hours after the last turn) is armed for every v2 session, scanned ones included, and when it comes due it freezes the session's remaining turns into L3 (trigger session_idle). A conversation that just stops therefore still reaches L3 without a close or a later turn. On worker start and after an import, open sessions with unfrozen turns get it armed and closed ones are frozen right away. The scan end is the fast path: when a scan pass finishes a source, the sessions it touched are frozen (trigger scan_pass) and the worker is woken. The desktop scan run by the backend (persistent scan, including a cancelled pass) and hosted imports (importManaged) do the same through POST /api/v1/l3/scan-sessions/freeze, also when a pass or import stops early; it freezes the caller's own sessions and scanned sessions of its source. Freezing only writes rows, so neither scans nor imports wait on L3. Existing sessions and detached legacy history keep their protocol. Co-Authored-By: Claude Opus 5.5 --- .../src/memory-l3-world-model.ts | 13 ++ .../memory-client/http-memory-client.ts | 8 + .../memory-client/memory-layer-endpoints.ts | 1 + .../tests/http-memory-client.test.ts | 15 ++ .../tests/mock-memory-client.test.ts | 1 + .../adapters/outbound/memory-client/types.ts | 2 + .../src/services/agent-source-service.ts | 21 +- App/backend/src/services/ingestion-service.ts | 38 ++-- .../services/tests/ingestion-service.test.ts | 35 ++++ .../tests/native-persistent-scan.test.ts | 38 ++++ .../src/tests/support/mock-memory-client.ts | 5 + Memory/src/agent-source/runtime.ts | 33 +++ Memory/src/contracts/memory-l3-world-model.ts | 13 ++ Memory/src/server/http.ts | 18 ++ Memory/src/service/memory-service.ts | 64 +++++- .../service/session/session-turn-service.ts | 34 ++- .../work-memory/work-memory-pipeline.ts | 20 +- Memory/src/service/worker/worker-runner.ts | 3 + Memory/src/storage/repositories.ts | 25 ++- .../tests/contract/source-turn-http.test.ts | 40 +++- .../service/l3-field/ended-task-l3.test.ts | 18 ++ Memory/tests/service/l3-field/scan-l3.test.ts | 197 ++++++++++++++++++ 22 files changed, 617 insertions(+), 25 deletions(-) create mode 100644 Memory/tests/service/l3-field/scan-l3.test.ts diff --git a/App/backend/local-api-contracts/src/memory-l3-world-model.ts b/App/backend/local-api-contracts/src/memory-l3-world-model.ts index 338c318d6..bca1e15f9 100644 --- a/App/backend/local-api-contracts/src/memory-l3-world-model.ts +++ b/App/backend/local-api-contracts/src/memory-l3-world-model.ts @@ -115,6 +115,19 @@ export const L3FailedBatchesRetryResponseSchema = z.object({ }).strict(); export type L3FailedBatchesRetryResponse = z.infer; +/** End of a hosted import: freeze what it added to these scanned Sessions into L3 batches. */ +export const L3ScanSessionsFreezeRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + sessionIds: z.array(NonEmptyStringSchema).max(10_000) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3ScanSessionsFreezeRequest = z.input; + +export const L3ScanSessionsFreezeResponseSchema = z.object({ + batches: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3ScanSessionsFreezeResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts index 7c6c14ac8..ce8bb970c 100644 --- a/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts +++ b/App/backend/src/adapters/outbound/memory-client/http-memory-client.ts @@ -5,6 +5,7 @@ import { CloseSessionOutputSchema, CompleteTurnOutputSchema, SourceTurnCompleteOutputSchema, + L3ScanSessionsFreezeResponseSchema, DeleteMemoryOutputSchema, EmbeddingInferenceOutputSchema, DeletePanelTaskOutputSchema, @@ -26,6 +27,7 @@ import { RetryMemoryProcessingOutputSchema, WorkerRunOutputSchema } from "@memmy/local-api-contracts"; +import { randomUUID } from "node:crypto"; import { z, type ZodType } from "zod"; import { MemoryLayerError, MemoryLayerNetworkError } from "./errors.js"; import { buildMemoryLayerUrl, MEMORY_LAYER_PATHS } from "./memory-layer-endpoints.js"; @@ -186,6 +188,12 @@ export function createHttpMemoryClient( return request("POST", "completeSourceTurn", SourceTurnCompleteOutputSchema, { body: input, context }); }, + async freezeScanSessions(source, sessionIds, context) { + // The L3 envelope resolves the owner the same way the source turns that opened these Sessions did. + const body = { requestId: randomUUID(), adapterId: `agent-source:${source}`, source, sessionIds }; + return request("POST", "freezeScanSessions", L3ScanSessionsFreezeResponseSchema, { body, context }); + }, + async search(input, context) { return request("POST", "search", SearchOutputSchema, { body: input, context }); }, diff --git a/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts b/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts index abdf561c8..616b648ed 100644 --- a/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts +++ b/App/backend/src/adapters/outbound/memory-client/memory-layer-endpoints.ts @@ -11,6 +11,7 @@ export const MEMORY_LAYER_PATHS = Object.freeze({ startTurn: "/api/v1/turns/start", completeTurn: "/api/v1/turns/:turnId/complete", completeSourceTurn: "/api/v1/source-turns/complete", + freezeScanSessions: "/api/v1/l3/scan-sessions/freeze", search: "/api/v1/memory/search", embeddingInference: "/api/v1/models/embedding/infer", addMemory: "/api/v1/memory/add", diff --git a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts index bc0b36c8f..43ab190cb 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts @@ -30,6 +30,20 @@ describe("HttpMemoryClient", () => { expect(JSON.parse(String(calls[0]?.init?.body))).toEqual(input); expect(new Headers(calls[0]?.init?.headers).get("x-memmy-user-id")).toBe("fixture-user"); }); + it("hands scanned Sessions to L3 inside a full request envelope for their source", async () => { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const client = createHttpMemoryClient({ baseUrl: "http://memory.test", token: "fixture-token", timeoutMs: 500, maxRetries: 0 }, { + fetchImpl: (async (url, init) => { + calls.push({ url: String(url), init }); + return new Response(JSON.stringify({ batches: 2, serverTime: "2099-01-01T00:00:00.000Z" }), { status: 200, headers: { "content-type": "application/json" } }); + }) as typeof fetch + }); + expect(await client.freezeScanSessions("codex", ["s-1", "s-2"])).toEqual({ batches: 2, serverTime: "2099-01-01T00:00:00.000Z" }); + expect(calls[0]?.url).toBe("http://memory.test/api/v1/l3/scan-sessions/freeze"); + expect(JSON.parse(String(calls[0]?.init?.body))).toEqual({ + requestId: expect.stringMatching(/^[0-9a-f-]{36}$/), adapterId: "agent-source:codex", source: "codex", sessionIds: ["s-1", "s-2"] + }); + }); it("only defines path templates for the final memory HTTP APIs", () => { expect(Object.values(MEMORY_LAYER_PATHS)).toEqual([ "/api/v1/health", @@ -42,6 +56,7 @@ describe("HttpMemoryClient", () => { "/api/v1/turns/start", "/api/v1/turns/:turnId/complete", "/api/v1/source-turns/complete", + "/api/v1/l3/scan-sessions/freeze", "/api/v1/memory/search", "/api/v1/models/embedding/infer", "/api/v1/memory/add", diff --git a/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts index e68074f33..e1c729582 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/mock-memory-client.test.ts @@ -86,6 +86,7 @@ describe("createMockMemoryClient", () => { "completeSourceTurn", "deleteMemory", "enqueueImportSummaries", + "freezeScanSessions", "getMemory", "getMemoryProcessingStatus", "getMemoryTokenBudget", diff --git a/App/backend/src/adapters/outbound/memory-client/types.ts b/App/backend/src/adapters/outbound/memory-client/types.ts index 7a2d04fc6..bca20dca4 100644 --- a/App/backend/src/adapters/outbound/memory-client/types.ts +++ b/App/backend/src/adapters/outbound/memory-client/types.ts @@ -74,6 +74,8 @@ export interface MemoryClient { completeTurn(input: CompleteTurnInput & { turnId: string }, context?: MemoryRequestContext): Promise; completeSourceTurn(input: SourceTurnCompleteInput, context?: MemoryRequestContext): Promise; + /** Freeze what an import added to these scanned Sessions into L3 batches. */ + freezeScanSessions(source: string, sessionIds: string[], context?: MemoryRequestContext): Promise<{ batches: number }>; search(input: SearchInput, context?: MemoryRequestContext): Promise; embeddingInference?(input: EmbeddingInferenceInput, options?: { signal?: AbortSignal }): Promise; diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index 63dd1b137..ddc960d0a 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -132,7 +132,7 @@ export interface CreateAgentSourceServiceOptions { sourceRegistry: SourceRegistry; agentSourceRepository: AgentSourceRepository; ingestionService: IngestionService; - memoryClient: Pick; + memoryClient: Pick & Partial>; skillDistributionService: SkillDistributionService; agentSourceAnalytics?: AgentSourceLifecycleAnalytics; memoryAddAnalytics?: Pick< @@ -741,7 +741,20 @@ async function ingestPersistentStagedSource( store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "ingest", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); const preparedState = store.getSourceState(sourceId); store.saveSourceState({ ...(preparedState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "ingest", updatedAt: now() }); - const ingestion = await ingestPersistentSource(options, store, sourceId, scanOptions, []); + const sessionIds = new Set(); + let ingestion: Awaited>; + try { + ingestion = await ingestPersistentSource(options, store, sourceId, scanOptions, [], sessionIds); + } finally { + // Scanned history never ends its Sessions, so hand what this pass added to L3 now, + // including a pass that was cancelled midway. Freezing only writes rows in Memory; + // a failure here must not fail the scan. + if (sessionIds.size > 0 && options.memoryClient.freezeScanSessions) { + await options.memoryClient.freezeScanSessions(sourceId, [...sessionIds]).catch((error: unknown) => { + console.warn(`[agent-source] l3.freeze_failed ${error instanceof Error ? error.message : String(error)}`); + }); + } + } const scannedAt = now(); options.agentSourceRepository.setLastScannedAt(sourceId, scannedAt); const skillResult = await ingestSourceSkills(options, sourceId, scanOptions, store, mode); @@ -851,7 +864,8 @@ async function ingestPersistentSource( store: AppAgentSourceScanStore, sourceId: string, scanOptions: AgentSourceScanOptions, - initialErrors: readonly { conversationId: string; reason: string }[] + initialErrors: readonly { conversationId: string; reason: string }[], + sessionIds: Set ): Promise<{ memoryIds: string[]; memoryIdCount: number; deduped: number; errorCount: number; errors: Array<{ conversationId: string; reason: string }>; latestSeenAt: string | null; retryFrom: string | null; hasUncommittedSkips: boolean }> { const memoryIds: string[] = []; let memoryIdCount = 0; @@ -970,6 +984,7 @@ async function ingestPersistentSource( emitAddProgress("Capturing conversation turns"); continue; } + if (result.result?.sessionId) sessionIds.add(result.result.sessionId); const ids = result.result?.l1MemoryIds ?? []; if (result.status === "stored") { memoryIdCount += ids.length; diff --git a/App/backend/src/services/ingestion-service.ts b/App/backend/src/services/ingestion-service.ts index 169713030..4c7c645dc 100644 --- a/App/backend/src/services/ingestion-service.ts +++ b/App/backend/src/services/ingestion-service.ts @@ -56,6 +56,8 @@ export interface IngestionStats { dedupedMemories: number; failedMemories: number; memoryIds: string[]; + /** Memory Sessions this import added turns to; frozen into L3 when the import ends. */ + sessionIds?: string[]; /** * Subset of `memoryIds` written through the legacy add-memory path, which still needs a * summary job. A native turn is summarized inside the Memory capture transaction, so its @@ -77,7 +79,7 @@ export interface IngestionItemSkip { /** Contract for create ingestion service options. */ export interface CreateIngestionServiceOptions { - memoryClient: Pick; + memoryClient: Pick & Partial>; agentSourceRepository: Pick; memoryAddAnalytics?: Pick< MemoryDesktopAddAnalytics, @@ -117,20 +119,30 @@ export function createIngestionService(options: CreateIngestionServiceOptions): }; const conversations = groupByConversation(messages, stats)[Symbol.asyncIterator](); - while (true) { - if (ctx.signal?.aborted) { - break; + try { + while (true) { + if (ctx.signal?.aborted) { + break; + } + + const next = await conversations.next(); + if (next.done) { + break; + } + + stats.conversations += 1; + await processConversation(options, next.value, ctx, stats); } - - const next = await conversations.next(); - if (next.done) { - break; + } finally { + // Imported history never ends its Sessions, so hand what this import added to L3 now, also + // when it stopped early. Freezing only writes rows in Memory; a failure must not fail the import. + const sessionIds = [...new Set(stats.sessionIds ?? [])]; + if (sessionIds.length > 0 && options.memoryClient.freezeScanSessions) { + await options.memoryClient.freezeScanSessions(ctx.sourceId, sessionIds).catch((error: unknown) => { + console.warn(`[agent-source] l3.freeze_failed ${error instanceof Error ? error.message : String(error)}`); + }); } - - stats.conversations += 1; - await processConversation(options, next.value, ctx, stats); } - return stats; } }; @@ -333,6 +345,8 @@ async function processNativeConversation( ...(legacyImportTurnId ? { legacyImportTurnId } : {}), ...(ctx.scanMode === "initial_subset" || ctx.scanMode === "full" ? { captureLegacyHistory: true } : {}) }); + // Stored and already-stored turns both count: a re-import must freeze what a failed freeze left. + if (result.result?.sessionId) (stats.sessionIds ??= []).push(result.result.sessionId); if (result.status === "pending" || result.status === "conflict") { failed = true; stats.failed += turn.messages.length; diff --git a/App/backend/src/services/tests/ingestion-service.test.ts b/App/backend/src/services/tests/ingestion-service.test.ts index 3fc59ab1d..bc0d24547 100644 --- a/App/backend/src/services/tests/ingestion-service.test.ts +++ b/App/backend/src/services/tests/ingestion-service.test.ts @@ -47,6 +47,41 @@ describe("native Codex ingestion", () => { expect(markSeen).toHaveBeenCalledTimes(2); }); + it("freezes the sessions an import touched once it ends, and a failed freeze does not fail the import", async () => { + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } }); + const freezeScanSessions = vi.fn().mockResolvedValue({ batches: 1 }); + await createService({ completeSourceTurn, freezeScanSessions }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(freezeScanSessions).toHaveBeenCalledWith("codex", ["s-1"]); + + const failing = vi.fn().mockRejectedValue(new Error("memory unavailable")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const stats = await createService({ completeSourceTurn, freezeScanSessions: failing }) + .ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(stats.memoryIds).toEqual(["l1-native"]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("l3.freeze_failed")); + warn.mockRestore(); + + // Importing the same history again returns "existing"; the freeze is asked again so it can catch up. + const again = vi.fn().mockResolvedValue({ batches: 1 }); + const existing = vi.fn().mockResolvedValue({ status: "existing", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } }); + await createService({ completeSourceTurn: existing, freezeScanSessions: again }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" }); + expect(again).toHaveBeenCalledWith("codex", ["s-1"]); + }); + + it("still freezes what an import added when the import stops with an error", async () => { + const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } }); + const freezeScanSessions = vi.fn().mockResolvedValue({ batches: 1 }); + async function* brokenSource() { + yield* nativeMessages(); + // A later conversation fails to read after the first one was stored. + yield { ...nativeMessages()[0]!, conversationId: "next-conversation", messageId: "next-0" }; + throw new Error("source read failed"); + } + await expect(createService({ completeSourceTurn, freezeScanSessions }).ingest(brokenSource(), { sourceId: "codex" })) + .rejects.toThrow("source read failed"); + expect(freezeScanSessions).toHaveBeenCalledWith("codex", ["s-1"]); + }); + it("emits add analytics only when a native scan turn is newly stored", async () => { const events: Array<{ name: string; payload: Record }> = []; const analytics = { diff --git a/App/backend/src/services/tests/native-persistent-scan.test.ts b/App/backend/src/services/tests/native-persistent-scan.test.ts index 09450a08c..53eba5018 100644 --- a/App/backend/src/services/tests/native-persistent-scan.test.ts +++ b/App/backend/src/services/tests/native-persistent-scan.test.ts @@ -93,6 +93,44 @@ describe("persistent Codex scan", () => { expect(repository.getScanWatermark("codex")).toMatchObject({ latestSeenCreatedAt: at }); }); + it("hands the sessions a scan pass touched to L3 once, and a failed hand-off does not fail the scan", async () => { + const root = mkdtempSync(join(tmpdir(), "native-backend-scan-")); roots.push(root); + const store = createAppStateStore({ databasePath: join(root, "app.sqlite") }); stores.push(store); + const repository = store.repositories.agentSources; + const at = "2099-09-09T10:00:00.000Z"; + const event = (type: string, payload: Record) => ({ type, timestamp: at, payload }); + const turn = (turnId: string, text: string) => [ + event("event_msg", { type: "task_started", turn_id: turnId }), + event("response_item", { type: "message", role: "user", content: [{ text }] }), + event("response_item", { type: "message", role: "assistant", content: [{ text: "Done." }] }), + event("event_msg", { type: "task_complete", turn_id: turnId }) + ]; + writeFileSync(join(root, "rollout-a.jsonl"), [event("session_meta", { id: "session-a" }), ...turn("turn-1", "First."), ...turn("turn-2", "Second.")] + .map((value) => JSON.stringify(value)).join("\n") + "\n"); + const client = createMockMemoryClient(); + vi.spyOn(client, "completeSourceTurn").mockImplementation(async (input) => ({ status: "stored", result: { + turnId: input.sourceTurn.turnId, sessionId: "memory-session", episodeId: "episode", rawTurnId: "raw", l1MemoryId: input.sourceTurn.turnId, + l1MemoryIds: [input.sourceTurn.turnId], closedEpisodeIds: [], scheduledEvolution: false, jobs: [], serverTime: at + } })); + const freeze = vi.spyOn(client, "freezeScanSessions").mockRejectedValue(new Error("memory offline")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const service = createAgentSourceService({ + sourceRegistry: createSourceRegistry([createCodexSourceAdapter({ sessionsRoot: root })]), + memoryClient: client, + agentSourceRepository: repository, + ingestionService: createIngestionService({ memoryClient: client, agentSourceRepository: repository }), + skillDistributionService: { install: async () => undefined, uninstall: async () => undefined, installPlugin: async () => undefined, uninstallPlugin: async () => undefined }, + scanStoreDirectory: join(root, "scans") + }); + const result = await service.scanOne("codex", { scanJobId: "freeze-job", mode: "full" }); + expect(result.errors).toEqual([]); + expect(freeze).toHaveBeenCalledOnce(); + expect(freeze).toHaveBeenCalledWith("codex", ["memory-session"]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("l3.freeze_failed memory offline")); + expect(repository.getConversationCheckpoint("codex", "session-a")).not.toBeNull(); + warn.mockRestore(); + }); + it("starts the first watermark at a recent unfinished turn and captures it after it completes in place", async () => { const root = mkdtempSync(join(tmpdir(), "native-backend-scan-")); roots.push(root); const store = createAppStateStore({ databasePath: join(root, "app.sqlite") }); stores.push(store); diff --git a/App/backend/src/tests/support/mock-memory-client.ts b/App/backend/src/tests/support/mock-memory-client.ts index 2b24f9841..55935a44f 100644 --- a/App/backend/src/tests/support/mock-memory-client.ts +++ b/App/backend/src/tests/support/mock-memory-client.ts @@ -141,6 +141,11 @@ export function createMockMemoryClient(options: CreateMockMemoryClientOptions = }; }, + async freezeScanSessions() { + failIfNeeded(); + return { batches: 0 }; + }, + async completeSourceTurn(input) { failIfNeeded(); const l1MemoryId = randomUUID(); diff --git a/Memory/src/agent-source/runtime.ts b/Memory/src/agent-source/runtime.ts index 1e5e88e8c..aac225fbf 100644 --- a/Memory/src/agent-source/runtime.ts +++ b/Memory/src/agent-source/runtime.ts @@ -809,6 +809,7 @@ async function stageStandaloneSource( return { adapter, stored, sourceId, mode, staged, scanErrorCount, errors }; } +/** Ingests a source's staged turns; the Sessions it touched go to L3 even when the pass stops early. */ async function ingestStagedMessages( service: MemoryService, store: MemoryAgentSourceScanStore, @@ -822,6 +823,37 @@ async function ingestStagedMessages( "trackAddStarted" | "trackAddSucceeded" | "trackAddFailed" >, scanMode?: MemoryDesktopAddScanMode +): ReturnType { + const scannedSessionIds: string[] = []; + try { + return await ingestStagedTurns( + service, store, sourceId, signal, checkpoint, onProgress, scannedSessionIds, scheduleWorker, memoryAddAnalytics, scanMode + ); + } finally { + // Frozen batches are new queued jobs; wake the worker in case its drain already finished. + try { + if ((service.freezeScannedL3Sessions?.(scannedSessionIds) ?? 0) > 0) scheduleWorker?.(); + } catch { + // A failed freeze must not hide the scan's own result; the idle sweep hands these Sessions over later. + } + } +} + +async function ingestStagedTurns( + service: MemoryService, + store: MemoryAgentSourceScanStore, + sourceId: string, + signal: AbortSignal, + checkpoint: () => Promise, + onProgress: (progress: ScanProgress) => void, + /** Sessions this pass added turns to; the caller freezes them into L3 when the pass ends. */ + scannedSessionIds: string[], + scheduleWorker?: () => void, + memoryAddAnalytics?: Pick< + MemoryDesktopAddAnalytics, + "trackAddStarted" | "trackAddSucceeded" | "trackAddFailed" + >, + scanMode?: MemoryDesktopAddScanMode ): Promise<{ written: number; messageCount: number; errors: string[]; errorCount: number; latestSeenAt: string | null; retryFrom: string | null; hasUncommittedSkips: boolean }> { let written = 0; let messageCount = 0; @@ -964,6 +996,7 @@ async function ingestStagedMessages( }); reportedAdd = true; } + if (result.result?.sessionId) scannedSessionIds.push(result.result.sessionId); if (ids.length === 0) store.saveResult({ sourceId, conversationId: turn.conversationId }); for (const memoryId of ids) store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId }); messageCount += turn.messages.length; diff --git a/Memory/src/contracts/memory-l3-world-model.ts b/Memory/src/contracts/memory-l3-world-model.ts index 338c318d6..bca1e15f9 100644 --- a/Memory/src/contracts/memory-l3-world-model.ts +++ b/Memory/src/contracts/memory-l3-world-model.ts @@ -115,6 +115,19 @@ export const L3FailedBatchesRetryResponseSchema = z.object({ }).strict(); export type L3FailedBatchesRetryResponse = z.infer; +/** End of a hosted import: freeze what it added to these scanned Sessions into L3 batches. */ +export const L3ScanSessionsFreezeRequestSchema = z.object({ + ...L3WorldModelRequestEnvelopeShape, + sessionIds: z.array(NonEmptyStringSchema).max(10_000) +}).strict().superRefine(assertEnvelopeSourceConsistency); +export type L3ScanSessionsFreezeRequest = z.input; + +export const L3ScanSessionsFreezeResponseSchema = z.object({ + batches: z.number().int().nonnegative(), + serverTime: z.string().datetime() +}).strict(); +export type L3ScanSessionsFreezeResponse = z.infer; + export const SessionL3WorldModelContextResponseSchema = z.object({ schemaVersion: z.literal(2), projectId: NonEmptyStringSchema.nullable(), diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index ac46fdfc3..50c2d2526 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -9,6 +9,7 @@ import { EmbeddingInferenceInputSchema, L3UserProjectsReplaceRequestSchema, L3FailedBatchesRetryRequestSchema, + L3ScanSessionsFreezeRequestSchema, L3WorldModelBoundaryRequestSchema, L3WorldModelRequestEnvelopeSchema, OpenSessionInputSchema @@ -79,6 +80,7 @@ export const API_ROUTES = [ "GET /api/v1/l3-world-model/sessions/:sessionId/context", "POST /api/v1/l3/user-projects", "POST /api/v1/l3/failed-batches/retry", + "POST /api/v1/l3/scan-sessions/freeze", "POST /api/v1/turns/start", "POST /api/v1/turns/:turnId/complete", "POST /api/v1/source-turns/complete", @@ -685,6 +687,22 @@ async function routeRequest( return retried; } + if (method === "POST" && path === "/api/v1/l3/scan-sessions/freeze") { + requireMemoryWrite(principal); + const parsed = L3ScanSessionsFreezeRequestSchema.safeParse( + strictEnvelopeWithPrincipal(asObject(body, "l3.scan-sessions.freeze"), principal) + ); + if (!parsed.success) { + throw new MemoryServiceError( + "invalid_argument", + parsed.error.issues.map((issue) => `${issue.path.join(".") || "request"}: ${issue.message}`).join("; ") + ); + } + const frozen = service.freezeScanSessions(parsed.data); + if (frozen.batches > 0) autoWorker.schedule(); + return frozen; + } + const l3Context = match(path, /^\/api\/v1\/l3-world-model\/sessions\/([^/]+)\/context$/); if (method === "GET" && l3Context) { requireMemoryRead(principal); diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index af8f7f65d..37f35e66a 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -7,6 +7,8 @@ import { sha256Hex, type L3FailedBatchesRetryRequest, type L3FailedBatchesRetryResponse, + type L3ScanSessionsFreezeRequest, + type L3ScanSessionsFreezeResponse, type L3UserProjectsReplaceRequest, type L3UserProjectsReplaceResponse } from "../contracts/index.js"; @@ -373,7 +375,8 @@ export class MemoryService { get llm() { return serviceOwner.llm; }, get embedder() { return serviceOwner.embedder; }, get embedAfterCapture() { return serviceOwner.config.algorithm.capture.embedAfterCapture; }, - nowIso + nowIso, + l3FieldActive: () => serviceOwner.l3FieldActive() }); const projectEnvironmentOwner = this; this.projectEnvironment = new ProjectEnvironmentService({ @@ -596,6 +599,7 @@ export class MemoryService { enqueueObservationBackfill: (limit, at) => this.observationMemory.backfill(limit, at), l3FieldActive: () => this.l3FieldActive(), requeueL3DeadLettersOnce: (at) => this.requeueL3DeadLettersOnce(at), + armPendingL3Flushes: () => this.armPendingL3Flushes(), evolutionModelConfigured: () => workerRunnerOwner.skillLlm.isConfigured(), purgeL3InactiveItems: (at) => this.repos.l3Field.purgeInactive( new Date(Date.parse(at) - this.config.algorithm.l3FieldCognition.inactiveRetentionDays * 86_400_000).toISOString() @@ -726,6 +730,7 @@ export class MemoryService { maybeCreateDecisionRepair: this.feedbackExperience.maybeCreateDecisionRepair.bind(this.feedbackExperience), memoryAddEnabled: this.memoryAddEnabled.bind(this), memorySearchEnabled: this.memorySearchEnabled.bind(this), + l3FieldActive: () => this.l3FieldActive(), observeToolNoWrite: this.observeToolNoWrite.bind(this), openSessionNoWrite: this.openSessionNoWrite.bind(this), pendingTrialsForFeedback: this.skillTrials.pendingTrialsForFeedback.bind(this.skillTrials), @@ -1151,6 +1156,7 @@ export class MemoryService { const changed = stableStringify(previousConfig) !== stableStringify(nextConfig); const requiresRestart = stableStringify(previousConfig.storage) !== stableStringify(nextConfig.storage); const reloadedAt = nowIso(); + const l3FieldWasActive = this.l3FieldActive(); this.config = nextConfig; this.tokenBudgetLedger.setLimits(this.config.tokenBudget); @@ -1160,6 +1166,8 @@ export class MemoryService { if (!requiresRestart && request.restartFailedProcessing !== false) { this.restartFailedProcessing(reloadedAt); } + // Turned on without a restart: flushes that ran while it was off did not freeze, so hand those turns over now. + if (!l3FieldWasActive) this.armPendingL3Flushes(); serviceLogger.info("config.reloaded", { changed, requiresRestart, @@ -1382,6 +1390,39 @@ export class MemoryService { return { count: request.projects.length, serverTime: at }; } + /** + * End of an agent-source scan pass: freeze what the pass added to each touched Session into new-L3 + * batches, through the same entry Hook Session boundaries use. Only database rows are written (no + * model call), so the scan never waits on L3; a pass with nothing new freezes nothing. With the new + * L3 off, scans stay out of L3 as before. + */ + freezeScannedL3Sessions(sessionIds: readonly string[]): number { + if (!this.memoryAddEnabled() || !this.l3FieldActive()) return 0; + let batches = 0; + for (const sessionId of new Set(sessionIds)) { + const session = this.repos.runtime.getSession(sessionId); + if (!session || session.meta.l3_world_model_protocol_version !== 2) continue; + batches += this.repos.l3WorldModels.freezeBatches({ sessionId, trigger: "scan_pass" }).batchIds.length; + } + return batches; + } + + /** + * A scan or hosted import finished: freeze the scanned Sessions it touched. Capturing a scanned turn + * keeps the conversation's earlier owner even after an account switch, so the same rule applies + * here: the caller's own Sessions, and scanned Sessions of the caller's source. + */ + freezeScanSessions(request: L3ScanSessionsFreezeRequest): L3ScanSessionsFreezeResponse { + const namespace = normalizeNamespace(request.namespace); + const owned = request.sessionIds.filter((id) => { + const session = this.repos.runtime.getSession(id); + if (!session) return false; + return session.userId === namespace.userId || + (session.meta.work_memory_disabled === true && session.source === namespace.source); + }); + return { batches: this.freezeScannedL3Sessions(owned), serverTime: nowIso() }; + } + /** Rerun entry for the new L3 only; dead letters of the old pipeline are not rerun. */ retryFailedL3Batches(request: L3FailedBatchesRetryRequest): L3FailedBatchesRetryResponse { this.assertMemoryAddEnabled(); @@ -1396,6 +1437,24 @@ export class MemoryService { return { requeued: requeued.length, serverTime: at }; } + /** + * Worker startup (and after an import): turns not yet handed to the new L3 get someone responsible + * for them, also when the process stopped before a scan's hand-off or the data came from a backup. + * An open Session gets its idle flush armed; a closed one, which will not close again, is frozen + * now. Nothing while the new L3 is off. + */ + armPendingL3Flushes(): number { + if (!this.l3FieldActive()) return 0; + const pending = this.repos.l3WorldModels.sessionsWithUnfrozenTurns(); + this.repos.transaction(() => { + for (const { sessionId, open, lastActivityAt } of pending) { + if (open) this.workMemory.armIdleFlush(sessionId, lastActivityAt); + else this.repos.l3WorldModels.freezeBatches({ sessionId, trigger: "session_close" }); + } + }); + return pending.length; + } + /** Worker startup: one automatic rerun for each dead-lettered new-L3 batch; nothing while the new L3 is off. */ private requeueL3DeadLettersOnce(at: string): number { if (!this.l3FieldActive()) return 0; @@ -2508,6 +2567,9 @@ export class MemoryService { const result = this.repos.runtime.importBundleTables(request.bundle.tables, { conflictStrategy: request.conflictStrategy ?? "skip" }); + // Imported Sessions with turns not yet handed to the new L3 (including work the import had to + // drop for a unit kept here) get their idle flush armed. + this.armPendingL3Flushes(); const importedVectors = importedMemoryVectors(request.bundle.tables.memory_vectors); this.repos.vectors.importRows(importedVectors); result.inserted.memory_vectors = importedVectors.length; diff --git a/Memory/src/service/session/session-turn-service.ts b/Memory/src/service/session/session-turn-service.ts index 7474b72ed..8882c620e 100644 --- a/Memory/src/service/session/session-turn-service.ts +++ b/Memory/src/service/session/session-turn-service.ts @@ -1607,7 +1607,11 @@ export class SessionTurnService { const mapped = this.deps.repos.runtime.getSession(mappedSessionId); if (!mapped) return { status: "pending", reason: "source_session_missing" }; if (!inScope(mapped)) throw new MemoryServiceError("forbidden", "source_session_scope_conflict"); - if (mapped.status === "open") return mapped; + if (mapped.status === "open") { + const live = this.hookTakeoverOfScannedSession(request, mapped, inScope); + if (live) return live; + return mapped; + } if (!canStartAfter(mapped)) return { status: "pending", reason: "source_session_closed" }; } if (request.sessionId) { @@ -1634,9 +1638,37 @@ export class SessionTurnService { meta: { conversationId: identity.conversationId, source_namespace_key: scope.namespaceKey }, timeZone: request.timeZone }, { createNew: true, at: identity.startedAt }); + if (request.channel === "agent_source_scan" && this.deps.l3FieldActive()) { + // With the new L3 on, a newly scanned conversation feeds it like a Hook Session, but without + // Work Memory: a scan replays finished history, so there is no live task to extract working + // requirements from. With it off, scans stay out of L3 as before. Sessions opened earlier + // keep their protocol. + this.deps.repos.runtime.updateSessionMeta(opened.sessionId, { + l3_world_model_protocol_version: 2, + work_memory_disabled: true + }, identity.startedAt); + } return this.deps.repos.runtime.getSession(opened.sessionId)!; } + /** + * A conversation first seen by a scan, then continued live under a Hook: the Hook's own Session + * takes over, so its turns, compaction, close and Work Memory all act on one Session. The scan's + * Session is closed, which freezes what it still holds into L3. + */ + private hookTakeoverOfScannedSession( + request: SourceTurnCompleteRequest, + mapped: SessionRecord, + inScope: (candidate: SessionRecord) => boolean + ): SessionRecord | undefined { + if (request.channel !== "hook" || !request.sessionId || request.sessionId === mapped.id) return undefined; + if (mapped.meta.work_memory_disabled !== true) return undefined; + const live = this.deps.repos.runtime.getSession(request.sessionId); + if (!live || live.status !== "open" || !inScope(live)) return undefined; + this.closeSession(mapped.id); + return live; + } + completeTurn( turnId: string, request: TurnCompleteRequest & Record, diff --git a/Memory/src/service/work-memory/work-memory-pipeline.ts b/Memory/src/service/work-memory/work-memory-pipeline.ts index da59a64aa..5dd857d28 100644 --- a/Memory/src/service/work-memory/work-memory-pipeline.ts +++ b/Memory/src/service/work-memory/work-memory-pipeline.ts @@ -55,6 +55,8 @@ export interface WorkMemoryPipelineDependencies { embedder: Embedder; embedAfterCapture: boolean; nowIso: () => string; + /** With the new L3 on, the idle flush also hands a quiet Session's turns to L3. */ + l3FieldActive?: () => boolean; } const WORK_MEMORY_SCHEMA_VERSION = 1; @@ -111,7 +113,7 @@ export class WorkMemoryPipeline { at = this.deps.nowIso() ): WorkMemoryExtractionResult { const session = this.deps.repos.runtime.getSession(sessionId); - if (!session || session.meta.l3_world_model_protocol_version !== 2) { + if (!session || session.meta.l3_world_model_protocol_version !== 2 || session.meta.work_memory_disabled === true) { return { windows: 0, enqueued: 0, cursorAdvancedTo: 0 }; } // The first Work Memory pass of a Session starts from a clean slate: the L3 @@ -196,6 +198,8 @@ export class WorkMemoryPipeline { ): EvolutionJobRecord | undefined { const session = this.deps.repos.runtime.getSession(sessionId); if (!session || session.meta.l3_world_model_protocol_version !== 2) return undefined; + // A scanned Session has no Work Memory, but with the new L3 on the same flush hands its turns to L3. + if (session.meta.work_memory_disabled === true && this.deps.l3FieldActive?.() !== true) return undefined; return this.deps.repos.runtime.armWorkMemoryIdleFlush({ sessionId, userId: session.userId, @@ -217,9 +221,10 @@ export class WorkMemoryPipeline { if (!sessionId) throw new Error(`work memory idle flush job has no session: ${job.id}`); const empty: WorkMemoryExtractionResult = { windows: 0, enqueued: 0, cursorAdvancedTo: 0 }; const session = this.deps.repos.runtime.getSession(sessionId); - if (!session || session.status !== "open" || session.meta.l3_world_model_protocol_version !== 2) { - return empty; - } + if (!session || session.status !== "open" || session.meta.l3_world_model_protocol_version !== 2) return empty; + const workMemory = session.meta.work_memory_disabled !== true; + const l3Field = this.deps.l3FieldActive?.() === true; + if (!workMemory && !l3Field) return empty; const at = this.deps.nowIso(); const lastActivityAt = this.deps.repos.l3WorldModels.latestInputTraceCreatedAt(sessionId) ?? (typeof job.payload.lastActivityAt === "string" ? job.payload.lastActivityAt : at); @@ -230,7 +235,12 @@ export class WorkMemoryPipeline { return empty; } const throughTraceSeq = this.deps.repos.l3WorldModels.maxInputTraceSeq(sessionId); - return this.deps.repos.transaction(() => this.extractUnextracted(sessionId, throughTraceSeq, at)); + return this.deps.repos.transaction(() => { + // Every captured turn ends up here once its Session goes quiet, so a conversation that just + // stops (no close, no later turn) still reaches the new L3; what is already frozen is skipped. + if (l3Field) this.deps.repos.l3WorldModels.freezeBatches({ sessionId, trigger: "session_idle", at }); + return workMemory ? this.extractUnextracted(sessionId, throughTraceSeq, at) : empty; + }); } async extract(job: EvolutionJobRecord): Promise { diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index 67d7c8d6b..a0d91ae38 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -139,6 +139,8 @@ export interface WorkerRunnerDeps { enqueueObservationBackfill?: (limit: number, at: string) => number; /** Drops new-L3 items that have been inactive longer than the retention window. */ purgeL3InactiveItems?: (at: string) => number; + /** Arms the idle flush of every Session holding turns not yet handed to the new L3. */ + armPendingL3Flushes?: () => number; /** Gives each dead-lettered new-L3 batch one automatic rerun. */ requeueL3DeadLettersOnce?: (at: string) => number; /** Whether the new L3 is on; while off its jobs stay queued and are not leased. */ @@ -395,6 +397,7 @@ export class WorkerRunner { this.deps.purgeL3InactiveItems?.(at); this.deps.requeueL3DeadLettersOnce?.(at); + this.deps.armPendingL3Flushes?.(); this.refreshObservationModelConfiguration(); return { requeuedJobs: interruptedJobs.length + failedJobs.length, diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index b2585a9b0..ddcf8c62a 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -336,7 +336,11 @@ export type L3WorldModelBatchTrigger = | "token_compaction" | "token_compaction_attempt" | "session_close" - | "episode_idle_close"; + | "episode_idle_close" + /** A finished agent-source scan pass; new L3 only (scans stay out of the old pipeline). */ + | "scan_pass" + /** A Session that went quiet without a close (its idle flush); new L3 only. */ + | "session_idle"; export type L3WorldModelTargetField = Exclude; @@ -5169,6 +5173,20 @@ export class L3WorldModelRepository { this.fieldCognitionMachineId = machineId; } + /** v2 Sessions holding captured turns that were never frozen into a batch, with their last activity. */ + sessionsWithUnfrozenTurns(): Array<{ sessionId: string; open: boolean; lastActivityAt: string }> { + return (this.db.prepare( + `SELECT s.id AS sessionId, s.status = 'open' AS open, t.last_at AS lastActivityAt FROM sessions AS s + JOIN (SELECT session_id, MAX(trace_seq) AS last_seq, MAX(created_at) AS last_at + FROM l3_world_model_input_traces GROUP BY session_id) AS t + ON t.session_id = s.id + LEFT JOIN l3_world_model_session_cursors AS c ON c.session_id = s.id + WHERE json_extract(s.meta_json, '$.l3_world_model_protocol_version') = 2 + AND t.last_seq > COALESCE(c.last_scheduled_seq, 0)` + ).all() as Array<{ sessionId: string; open: number; lastActivityAt: string }>) + .map((row) => ({ ...row, open: row.open === 1 })); + } + /** * New L3 only: hand a closed task's turns to L3. Called wherever a task closes, so a conversation that * simply stops (idle, or the user ended the topic) still reaches L3; what is already frozen is skipped. @@ -5656,6 +5674,11 @@ export class L3WorldModelRepository { at?: string; }): FreezeL3WorldModelBatchesResult { const session = this.requireV2Session(input.sessionId); + // Scanned history only ever goes to the new L3. While it is off (switched off after the scan), the + // turns wait without moving the cursor, so they reach the new L3 once it is on again. + if (session.meta.work_memory_disabled === true && !(this.l3Field && this.fieldCognitionMachineId?.())) { + return { scheduled: false, batchIds: [], targetCount: 0 }; + } const at = input.at ?? nowIso(); this.db.prepare( `INSERT INTO l3_world_model_session_cursors (session_id, last_scheduled_seq, updated_at) diff --git a/Memory/tests/contract/source-turn-http.test.ts b/Memory/tests/contract/source-turn-http.test.ts index 1ab9d8088..c03626dc2 100644 --- a/Memory/tests/contract/source-turn-http.test.ts +++ b/Memory/tests/contract/source-turn-http.test.ts @@ -1,10 +1,11 @@ import { afterEach, describe, expect, it } from "vitest"; import { buildSourceTurnRequest } from "@memmy/agent-source-core"; -import { DEFAULT_MEMMY_CONFIG, createMemoryHttpServer } from "../../src/index.js"; +import { join } from "node:path"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb, createMemoryHttpServer } from "../../src/index.js"; import { Repositories } from "../../src/storage/repositories.js"; import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; -const { cleanup, createTestService } = createMemoryServiceFixture(); +const { cleanup, createTestService, createTestMemoryService, createTestRoot } = createMemoryServiceFixture(); afterEach(cleanup); describe("source turn HTTP contract", () => { @@ -40,6 +41,41 @@ describe("source turn HTTP contract", () => { } finally { await new Promise((resolve) => server.close(() => resolve())); } }); + it("freezes scanned Sessions through the request the desktop backend sends", async () => { + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = true; + const db = new MemoryDb({ path: join(createTestRoot(), "memory.sqlite") }); + const service = createTestMemoryService({ + db, + mode: "dev", + config, + l3FieldEnvironment: () => ({ machineId: "6".repeat(64), homeDir: "/nonexistent-home", tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + const server = createMemoryHttpServer({ service, startAgentSourceAutomation: false, auth: { mode: "local", allowAnonymous: true } }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const post = (path: string, body: unknown) => fetch(`http://127.0.0.1:${address.port}${path}`, { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) + }); + try { + const turn = await post("/api/v1/source-turns/complete", buildSourceTurnRequest({ source: "codex", conversationId: "scanned-conversation", + turnId: "scanned-turn", startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", sequence: 1, + completionEvidence: "final_answer:scanned-turn", query: "Rename the export helper.", answer: "Renamed.", status: "succeeded", + toolCalls: [], toolResults: [] }, "agent_source_scan")); + expect(turn.status).toBe(200); + const { result } = await turn.json() as { result: { sessionId: string } }; + const frozen = await post("/api/v1/l3/scan-sessions/freeze", { + requestId: crypto.randomUUID(), adapterId: "agent-source:codex", source: "codex", sessionIds: [result.sessionId] + }); + expect(frozen.status).toBe(200); + // The worker's startup pass may have handed the session over first; either way it is frozen once. + expect(await frozen.json()).toMatchObject({ batches: expect.any(Number) }); + expect(db.db.prepare(`SELECT COUNT(*) AS n FROM l3_unit_batches WHERE session_id = ?`).get(result.sessionId)).toEqual({ n: 1 }); + } finally { await new Promise((resolve) => server.close(() => resolve())); } + }); + it.each(["hook", "agent_source_scan", "concurrent"] as const)("reuses the installed Hook owner after account switch when %s arrives first", async (channel) => { const { service, db } = createTestService({ config: { ...DEFAULT_MEMMY_CONFIG, userId: "configured-owner" } }); const opened = service.openSession({ diff --git a/Memory/tests/service/l3-field/ended-task-l3.test.ts b/Memory/tests/service/l3-field/ended-task-l3.test.ts index d2025e525..f85a2deef 100644 --- a/Memory/tests/service/l3-field/ended-task-l3.test.ts +++ b/Memory/tests/service/l3-field/ended-task-l3.test.ts @@ -119,6 +119,24 @@ describe("a task that ends without a session close still reaches the new L3", () expect(env.batches(session)).toEqual([]); }); + it("hands over turns whose flush ran while the switch was off once it is turned on without a restart", async () => { + const env = setup(); + const session = env.open("toggled"); + env.service.completeTurn("toggled-turn", { sessionId: session, query: "以后改完都跑 npm test", answer: "好" }); + const quietSince = new Date(Date.now() - 2 * 60 * 60 * 1000 - 5_000).toISOString(); + env.db.db.prepare(`UPDATE l3_world_model_input_traces SET created_at = ? WHERE session_id = ?`).run(quietSince, session); + env.db.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(quietSince); + env.switchTo(false); + await env.service.runWorkerOnce(50); + expect(env.flush(session)!.status).toBe("succeeded"); + expect(env.batches(session)).toEqual([]); + env.switchTo(true); + expect(env.flush(session)!.status).toBe("queued"); + await env.service.runWorkerOnce(50); + expect(env.batches(session)).toEqual(["session_idle"]); + }); + it("re-freezes imported turns whose unfinished batch an import had to drop for a machine kept here", async () => { const source = setup(); const session = source.open("exported"); diff --git a/Memory/tests/service/l3-field/scan-l3.test.ts b/Memory/tests/service/l3-field/scan-l3.test.ts new file mode 100644 index 000000000..7ce6ce64c --- /dev/null +++ b/Memory/tests/service/l3-field/scan-l3.test.ts @@ -0,0 +1,197 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import type { SourceTurnCompleteRequest } from "../../../src/types.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +function setup(enabled = true) { + const root = fixture.createTestRoot("scan-l3-"); + const home = join(root, "home"); + const project = join(home, "code", "demo"); + mkdirSync(project, { recursive: true }); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + const service = fixture.createTestMemoryService({ + db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: "8".repeat(64), homeDir: home, tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + return { db, service, repos: new Repositories(db.db), project }; +} + +function sourceTurn( + channel: SourceTurnCompleteRequest["channel"], + conversationId: string, + index: number, + workspacePath?: string +): SourceTurnCompleteRequest { + const minute = String(index).padStart(2, "0"); + return { + namespace: { source: "codex", profileId: "default", userId: "scan-user" }, + source: "codex", + channel, + ...(workspacePath ? { workspacePath } : {}), + sourceTurn: { + source: "codex", profileId: "default", conversationId, turnId: `${conversationId}-turn-${index}`, + startedAt: `2099-01-01T10:${minute}:00.000Z`, completedAt: `2099-01-01T10:${minute}:30.000Z`, + sequence: index, completionEvidence: `task_complete:${index}` + }, + query: `Run the project tests, step ${index}.`, + answer: "The tests ran and one failed because vitest was missing.", + toolCalls: [{ id: `call-${index}`, name: "exec_command", input: { cmd: "npm test", workdir: workspacePath } }], + toolResults: [{ id: `call-${index}`, output: "sh: vitest: command not found" }] + }; +} + +function sessionOf(repos: Repositories, conversationId: string) { + const row = repos.db.prepare(`SELECT id FROM sessions WHERE conversation_id = ? OR host_session_key = ?`) + .get(conversationId, conversationId) as { id: string }; + return repos.runtime.getSession(row.id)!; +} + +describe("scanned sessions enter L3", () => { + it("opens a scanned conversation on the L3 v2 protocol with Work Memory off, but leaves Hook sessions alone", () => { + const { service, repos, project } = setup(); + expect(service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)).status).toBe("stored"); + expect(service.completeSourceTurn(sourceTurn("hook", "hooked", 1, project)).status).toBe("stored"); + expect(sessionOf(repos, "scanned").meta).toMatchObject({ l3_world_model_protocol_version: 2, work_memory_disabled: true }); + expect(sessionOf(repos, "hooked").meta.l3_world_model_protocol_version).toBeUndefined(); + }); + + it("never extracts Work Memory for a scanned session, on idle, new task or close", async () => { + const { service, repos, project } = setup(); + service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)); + service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 2, project)); + // Its idle flush (armed for the new L3) comes due and runs: it hands turns to L3, not to Work Memory. + const quietSince = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString(); + repos.db.prepare(`UPDATE l3_world_model_input_traces SET created_at = ?`).run(quietSince); + repos.db.prepare(`UPDATE evolution_jobs SET payload_json = json_set(payload_json, '$.runAfter', ?) + WHERE job_type = 'work_memory_idle_flush'`).run(quietSince); + await service.runWorkerOnce(50); + service.closeSession(sessionOf(repos, "scanned").id); + const workMemoryJobs = repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_extract'` + ).get() as { n: number }; + expect(workMemoryJobs.n).toBe(0); + expect(repos.db.prepare(`SELECT trigger FROM l3_unit_batches`).all()).toEqual([{ trigger: "session_idle" }]); + }); + + it("freezes what a scan pass added into new-L3 batches, and nothing when the pass added nothing", () => { + const { service, repos, project } = setup(); + service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)); + service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 2, project)); + const session = sessionOf(repos, "scanned"); + expect(service.freezeScannedL3Sessions([session.id, session.id])).toBe(1); + const batches = repos.db.prepare(`SELECT trigger, l1_memory_ids_json FROM l3_unit_batches`).all() as Array<{ trigger: string; l1_memory_ids_json: string }>; + expect(batches).toHaveLength(1); + expect(batches[0]!.trigger).toBe("scan_pass"); + expect(JSON.parse(batches[0]!.l1_memory_ids_json).length).toBeGreaterThanOrEqual(2); + expect(repos.runtime.listJobs("queued", 100).filter((job) => job.jobType === "l3_unit_update")).toHaveLength(1); + + expect(service.freezeScannedL3Sessions([session.id])).toBe(0); + service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 3, project)); + expect(service.freezeScannedL3Sessions([session.id])).toBe(1); + }); + + it("lets a live Hook take over a conversation a scan saw first", () => { + const { service, repos, project } = setup(); + service.completeSourceTurn(sourceTurn("agent_source_scan", "resumed", 1, project)); + const scanned = sessionOf(repos, "resumed"); + const hook = service.openSession({ + l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + namespace: { source: "codex", profileId: "default", userId: "scan-user", sessionKey: "codex-memory-resumed" } + }); + const result = service.completeSourceTurn({ ...sourceTurn("hook", "resumed", 2, project), sessionId: hook.sessionId }); + expect(result.status).toBe("stored"); + expect(result.result?.sessionId).toBe(hook.sessionId); + expect(repos.runtime.getSession(scanned.id)!.status).toBe("closed"); + expect(repos.runtime.getSession(hook.sessionId)!.meta.work_memory_disabled).toBeUndefined(); + const batches = repos.db.prepare(`SELECT session_id FROM l3_unit_batches`).all() as Array<{ session_id: string }>; + expect(batches.map((batch) => batch.session_id)).toEqual([scanned.id]); + const idleFlush = repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush' AND session_id = ?` + ).get(hook.sessionId) as { n: number }; + expect(idleFlush.n).toBe(1); + // Later turns follow the Hook Session. + expect(service.completeSourceTurn({ ...sourceTurn("hook", "resumed", 3, project), sessionId: hook.sessionId }).result?.sessionId) + .toBe(hook.sessionId); + }); + + it("lets a scan freeze the scanned sessions of its source, also after an account switch, but not another source's", () => { + const { service, repos, project } = setup(); + service.completeSourceTurn(sourceTurn("agent_source_scan", "imported", 1, project)); + const session = sessionOf(repos, "imported"); + const envelope = (userId: string, source: string) => ({ + requestId: "9353298b-4d3d-46f0-9178-27c27e81543e", adapterId: "memmy-desktop", source, + namespace: { source, profileId: "default", userId }, sessionIds: [session.id] + }); + expect(service.freezeScanSessions(envelope("someone-else", "claude_code")).batches).toBe(0); + // Capture kept the conversation's earlier owner after the switch; the hand-off follows it. + expect(service.freezeScanSessions(envelope("switched-account", "codex")).batches).toBe(1); + }); + + it("arms the idle flush of a scanned session only while the new L3 is on", () => { + const flushOf = (repos: Repositories, sessionId: string) => repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush' AND session_id = ? AND status = 'queued'` + ).get(sessionId) as { n: number }; + const on = setup(); + on.service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, on.project)); + expect(flushOf(on.repos, sessionOf(on.repos, "scanned").id)).toEqual({ n: 1 }); + const off = setup(false); + off.service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, off.project)); + expect(off.repos.db.prepare(`SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush'`).get()).toEqual({ n: 0 }); + }); + + it("re-arms the idle flush on worker start for sessions whose turns never reached L3, as after a crash mid-scan", () => { + const { service, repos, project } = setup(); + service.completeSourceTurn(sourceTurn("agent_source_scan", "crashed", 1, project)); + const sessionId = sessionOf(repos, "crashed").id; + // The process stopped before the scan's hand-off and the flush job was lost with an old build. + repos.db.prepare(`DELETE FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush'`).run(); + service.reconcileWorkerStartup(); + expect(repos.db.prepare( + `SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'work_memory_idle_flush' AND session_id = ? AND status = 'queued'` + ).get(sessionId)).toEqual({ n: 1 }); + expect(repos.db.prepare(`SELECT COUNT(*) AS n FROM l3_unit_batches`).get()).toEqual({ n: 0 }); + }); + + it("keeps a session scanned while the switch was on out of the old L3 after it is switched off", () => { + const on = setup(); + on.service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, on.project)); + const sessionId = sessionOf(on.repos, "scanned").id; + const reopen = (enabled: boolean) => { + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.l3FieldCognition.enabled = enabled; + return fixture.createTestMemoryService({ + db: on.db, mode: "dev", config, + l3FieldEnvironment: () => ({ machineId: "8".repeat(64), homeDir: join(on.project, "..", ".."), tempRoots: [] }), + l3MachineProbe: () => new Promise(() => undefined) + }); + }; + reopen(false).closeSession(sessionId); + const count = (table: string) => (on.repos.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; + expect([count("l3_world_model_evidence_batches"), count("l3_unit_batches")]).toEqual([0, 0]); + expect(on.repos.runtime.listJobs("queued", 100).filter((job) => job.jobType === "l3_world_model_update")).toEqual([]); + // Switched on again, the waiting turns still reach the new L3. + expect(reopen(true).freezeScannedL3Sessions([sessionId])).toBe(1); + }); + + it("keeps scans out of L3 when the switch is off, as before", () => { + const { service, repos, project } = setup(false); + service.completeSourceTurn(sourceTurn("agent_source_scan", "scanned", 1, project)); + const session = sessionOf(repos, "scanned"); + expect(session.meta.l3_world_model_protocol_version).toBeUndefined(); + expect(session.meta.work_memory_disabled).toBeUndefined(); + expect(service.freezeScannedL3Sessions([session.id])).toBe(0); + service.closeSession(session.id); + const count = (table: string) => (repos.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; + expect([count("l3_world_model_input_traces"), count("l3_world_model_evidence_batches"), count("l3_unit_batches")]).toEqual([0, 0, 0]); + expect(repos.runtime.listJobs("queued", 100).filter((job) => job.jobType.startsWith("l3_"))).toEqual([]); + }); +});