diff --git a/App/backend/local-api-contracts/src/index.ts b/App/backend/local-api-contracts/src/index.ts index 902be3f4d..77900b8ca 100644 --- a/App/backend/local-api-contracts/src/index.ts +++ b/App/backend/local-api-contracts/src/index.ts @@ -965,6 +965,7 @@ export const CatalogEndpointInputSchema = z.object({ endpointId: z.string().trim().min(1), apiBase: z.string().url(), protocol: ModelEndpointProtocolSchema, + inheritProviderDefaults: z.boolean().optional(), apiKey: z.string().optional(), extraHeaders: z.record(z.string(), z.string()).optional(), extraBody: z.record(z.string(), z.unknown()).optional(), @@ -1037,6 +1038,7 @@ export const ModelConfigTestInputSchema = z.object({ provider: ModelConfigTestProviderSchema, endpointId: z.string().trim().min(1), protocol: ModelEndpointProtocolSchema, + inheritProviderDefaults: z.boolean().optional(), apiBase: z.string().url(), modelId: z.string().min(1), apiKey: z.string().min(1).optional(), @@ -1146,6 +1148,7 @@ export const CatalogEndpointViewSchema = z.object({ endpointId: z.string().min(1), apiBase: z.string().url(), protocol: ModelEndpointProtocolSchema, + inheritProviderDefaults: z.boolean().optional(), hasApiKey: z.boolean(), apiKeyMasked: z.string(), apiKey: z.string().default(""), diff --git a/App/backend/local-api-contracts/src/model-catalog-resolver.ts b/App/backend/local-api-contracts/src/model-catalog-resolver.ts index af108ff8c..79c707d0c 100644 --- a/App/backend/local-api-contracts/src/model-catalog-resolver.ts +++ b/App/backend/local-api-contracts/src/model-catalog-resolver.ts @@ -9,6 +9,8 @@ export interface RuntimeCatalogEndpoint { apiBase: string; protocol: ModelEndpointProtocol; apiKey?: string; + /** False keeps credentials and request options local to this endpoint. */ + inheritProviderDefaults?: boolean; extraHeaders?: Record; extraBody?: Record; region?: string; @@ -155,10 +157,12 @@ export function resolveAssignedModel(input: ResolveAssignedModelInput): ModelSel } if (!providerOwnerMatches(preset, provider, input.activeAccountId)) return UNAVAILABLE; + const inherited = endpoint.inheritProviderDefaults === false ? {} : provider; + const apiKey = endpoint.apiKey ?? inherited.apiKey; let extraBody: Readonly>; try { extraBody = deepFreeze(structuredClone({ - ...(provider.extraBody ?? {}), + ...(inherited.extraBody ?? {}), ...(endpoint.extraBody ?? {}) })); } catch { @@ -183,12 +187,12 @@ export function resolveAssignedModel(input: ResolveAssignedModelInput): ModelSel endpointId: preset.endpoint, protocol: endpoint.protocol, apiBase: endpoint.apiBase, - ...(endpoint.apiKey ?? provider.apiKey - ? { apiKey: endpoint.apiKey ?? provider.apiKey } + ...(apiKey + ? { apiKey } : {}), ...(provider.ownerAccountId ? { ownerAccountId: provider.ownerAccountId } : {}), extraHeaders: Object.freeze({ - ...(provider.extraHeaders ?? {}), + ...(inherited.extraHeaders ?? {}), ...(endpoint.extraHeaders ?? {}) }), extraBody @@ -329,6 +333,7 @@ function isRuntimeEndpoint(value: unknown): value is RuntimeCatalogEndpoint { && typeof value.protocol === "string" && PROTOCOLS.has(value.protocol as ModelEndpointProtocol) && (value.apiKey === undefined || typeof value.apiKey === "string") + && (value.inheritProviderDefaults === undefined || typeof value.inheritProviderDefaults === "boolean") && validStringRecord(value.extraHeaders) && validUnknownRecord(value.extraBody) && (value.region === undefined || typeof value.region === "string"); diff --git a/App/backend/local-api-contracts/tests/endpoint-provider-inheritance.test.ts b/App/backend/local-api-contracts/tests/endpoint-provider-inheritance.test.ts new file mode 100644 index 000000000..674b63477 --- /dev/null +++ b/App/backend/local-api-contracts/tests/endpoint-provider-inheritance.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { CatalogEndpointInputSchema, CatalogEndpointViewSchema, ModelConfigTestInputSchema } from "../src/index.js"; +import { resolveAssignedModel, type RuntimeModelCatalog } from "../src/model-catalog-resolver.js"; +import { resolveAssignedModel as resolveMemoryModel } from "../../../../Memory/src/contracts/model-catalog-resolver.js"; + +function catalog(): RuntimeModelCatalog { + return { + providers: { openai: { + apiKey: "unrelated-cloud-key", extraHeaders: { Authorization: "unrelated-header", "X-Cloud": "cloud" }, + extraBody: { model: "unrelated-model", temperature: 0.9 }, + endpoints: { memory: { apiBase: "http://127.0.0.1:1234/v1", protocol: "openai-chat-completions" } } + } }, + modelPresets: { memory: { provider: "openai", endpoint: "memory", model: "own-model", source: "byok", capabilities: ["memory_summary"] } }, + modelAssignments: { byok: { memorySummary: "memory" } } + }; +} + +describe.each([["shared", resolveAssignedModel], ["standalone Memory", resolveMemoryModel]] as const)("%s endpoint inheritance", (_label, resolve) => { + it.each([undefined, "", "own-key"])("keeps an independent endpoint's own credential (%s) and request options", key => { + const c = catalog(), e = c.providers!.openai!.endpoints!.memory!; + Object.assign(e, { inheritProviderDefaults: false, extraHeaders: { "X-Own": "own" }, extraBody: { seed: 7 } }); + if (key !== undefined) e.apiKey = key; + const before = structuredClone(c); + const r = resolve({ catalog: c, mode: "byok", capability: "memory_summary" }); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.provider.apiKey).toBe(key || undefined); + expect(r.provider.extraHeaders).toEqual({ "X-Own": "own" }); + expect(r.provider.extraBody).toEqual({ seed: 7 }); + expect(c).toEqual(before); + c.providers!.openai!.apiKey = "rotated-cloud-key"; + c.providers!.openai!.extraHeaders = { Authorization: "rotated-header" }; + expect(resolve({ catalog: c, mode: "byok", capability: "memory_summary" })).toEqual(r); + }); + + it.each([undefined, true])("preserves inherited defaults and later credential rotation (%s)", flag => { + const c = catalog(), e = c.providers!.openai!.endpoints!.memory!; + if (flag !== undefined) Object.assign(e, { inheritProviderDefaults: flag }); + e.extraHeaders = { "X-Own": "own" }; e.extraBody = { seed: 7 }; + const r = resolve({ catalog: c, mode: "byok", capability: "memory_summary" }); + expect(r.ok && r.provider.apiKey).toBe("unrelated-cloud-key"); + expect(r.ok && r.provider.extraHeaders).toEqual({ Authorization: "unrelated-header", "X-Cloud": "cloud", "X-Own": "own" }); + expect(r.ok && r.provider.extraBody).toEqual({ model: "unrelated-model", temperature: 0.9, seed: 7 }); + c.providers!.openai!.apiKey = "rotated-key"; + const rotated = resolve({ catalog: c, mode: "byok", capability: "memory_summary" }); + expect(rotated.ok && rotated.provider.apiKey).toBe("rotated-key"); + }); + + it.each([null, "false", 0])("rejects malformed inheritance configuration (%s)", flag => { + const c = catalog(); Object.assign(c.providers!.openai!.endpoints!.memory!, { inheritProviderDefaults: flag }); + expect(resolve({ catalog: c, mode: "byok", capability: "memory_summary" })).toEqual({ ok: false, code: "model_selection_unavailable" }); + }); +}); + +it("retains the optional false flag in catalog and connection-test API round trips", () => { + const endpoint = { endpointId: "memory", apiBase: "http://127.0.0.1:1234/v1", protocol: "openai-chat-completions", inheritProviderDefaults: false }; + expect(CatalogEndpointInputSchema.parse(endpoint)).toHaveProperty("inheritProviderDefaults", false); + expect(CatalogEndpointViewSchema.parse({ ...endpoint, hasApiKey: false, apiKeyMasked: "" })).toHaveProperty("inheritProviderDefaults", false); + expect(ModelConfigTestInputSchema.parse({ ...endpoint, provider: "openai_compatible", modelId: "own-model" })).toHaveProperty("inheritProviderDefaults", false); + expect(CatalogEndpointInputSchema.safeParse({ ...endpoint, inheritProviderDefaults: "false" }).success).toBe(false); +}); diff --git a/App/backend/src/adapters/inbound/local-api/routes/composio-mcp.ts b/App/backend/src/adapters/inbound/local-api/routes/composio-mcp.ts index d6d0fbaca..9c37ad41f 100644 --- a/App/backend/src/adapters/inbound/local-api/routes/composio-mcp.ts +++ b/App/backend/src/adapters/inbound/local-api/routes/composio-mcp.ts @@ -134,6 +134,14 @@ async function dispatchComposioTool( * @param options Route dependencies. */ export function registerComposioMcpRoutes(app: FastifyInstance, options: RegisterComposioMcpRoutesOptions): void { + const notificationStreams = new Set(); + let closing = false; + app.addHook("preClose", async () => { + closing = true; + // GET holds an idle notification stream. Tool POSTs may still be writing + // external data, so leave those requests to finish normally. + await Promise.all([...notificationStreams].map(transport => transport.close())); + }); const handler = async (request: FastifyRequest, reply: FastifyReply): Promise => { const token = singleHeader(request.headers[MCP_TOKEN_HEADER]); if (!token || token !== options.mcpToken) { @@ -147,11 +155,20 @@ export function registerComposioMcpRoutes(app: FastifyInstance, options: Registe reply.hijack(); reply.raw.on("close", () => { + notificationStreams.delete(transport); void transport.close(); void server.close(); }); await server.connect(transport); + if (request.method === "GET") { + if (closing) { + await transport.close(); + reply.raw.end(); + return; + } + notificationStreams.add(transport); + } await transport.handleRequest(request.raw, reply.raw, request.body); }; diff --git a/App/backend/src/adapters/inbound/local-api/server.ts b/App/backend/src/adapters/inbound/local-api/server.ts index 5ecc34705..98decedec 100644 --- a/App/backend/src/adapters/inbound/local-api/server.ts +++ b/App/backend/src/adapters/inbound/local-api/server.ts @@ -52,6 +52,25 @@ export function createLocalApiServer(options: CreateLocalApiServerOptions): Fast const app = Fastify({ logger: false }); const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS; const authenticateRuntimeToken = createRuntimeTokenPreHandler(options.permissionManager); + const eventStreams = new Set<() => void>(); + let closing = false; + const registerEventStream = (stop: () => void) => { + // Authentication may finish after preClose has already drained existing streams. + if (closing) stop(); + else eventStreams.add(stop); + return () => { eventStreams.delete(stop); }; + }; + app.addHook("preClose", async () => { + closing = true; + // The renderer closes after this server; do not wait for its EventSource to + // disconnect. End only our notification streams, preserving ordinary writes. + for (const stop of eventStreams) stop(); + }); + app.addHook("onResponse", async () => { + // A request already running at preClose can become idle after Fastify's + // initial idle sweep. Preserve its response, then retire the idle socket. + if (closing) app.server.closeIdleConnections(); + }); if (options.knowledge) { registerKnowledgeRoutes(app, { ...options.knowledge, authenticate: authenticateRuntimeToken }); @@ -151,7 +170,7 @@ export function createLocalApiServer(options: CreateLocalApiServerOptions): Fast return reply.code(401).send({ error: "unauthorized" }); } - startSse(reply, heartbeatIntervalMs, getSingleHeaderValue(request.headers.origin), options.services.progressBus); + startSse(reply, heartbeatIntervalMs, getSingleHeaderValue(request.headers.origin), options.services.progressBus, registerEventStream); }); return app; @@ -216,7 +235,8 @@ function startSse( reply: FastifyReply, heartbeatIntervalMs: number, origin: string | undefined, - progressBus: BackendServices["progressBus"] + progressBus: BackendServices["progressBus"], + registerEventStream: (stop: () => void) => () => void ): void { // SSE is a long-lived response, so we take over the raw response to prevent Fastify from automatically ending the request. reply.hijack(); @@ -284,13 +304,22 @@ function startSse( }); }); + let cleanedUp = false; + let unregister = () => {}; const cleanup = () => { + if (cleanedUp) return; + cleanedUp = true; // Clean up the heartbeat timer promptly after the client disconnects, avoiding orphaned tasks lingering in the background. clearInterval(interval); unsubscribeScanProgress(); unsubscribeScanCompleted(); + unregister(); }; reply.raw.once("close", cleanup); reply.raw.once("error", cleanup); + unregister = registerEventStream(() => { + cleanup(); + reply.raw.end(); + }); } diff --git a/App/backend/src/infrastructure/memmy-config/index.ts b/App/backend/src/infrastructure/memmy-config/index.ts index 0da175c45..1115a3c74 100644 --- a/App/backend/src/infrastructure/memmy-config/index.ts +++ b/App/backend/src/infrastructure/memmy-config/index.ts @@ -108,6 +108,14 @@ export interface RuntimeProjectionResult { memoryConfigAffected: boolean; } +export interface CatalogEndpointRequestOptions { + apiBase: string; + protocol: string; + inheritProviderDefaults?: boolean; + apiKey?: string; + extraHeaders?: Record; +} + export interface MemmyConfigWriter { readModelConfig?(): Promise; @@ -119,6 +127,8 @@ export interface MemmyConfigWriter { readEndpointApiKey?(provider: string, endpointId: string): Promise; + readEndpointRequestOptions?(provider: string, endpointId: string): Promise; + /** Atomically persist the active account/BYOK namespace without rewriting the model catalog. */ writeUserMode?(mode: UserMode): Promise; @@ -206,6 +216,10 @@ export function createMemmyConfigWriter(options: CreateMemmyConfigWriterOptions return readCatalogEndpointApiKey(configPath, provider, endpointId); }, + async readEndpointRequestOptions(provider, endpointId) { + return readCatalogEndpointRequestOptions(configPath, provider, endpointId); + }, + async writeUserMode(mode) { await mutateRuntimeConfig(configPath, (config) => { const app = asRecord(config.app); @@ -276,7 +290,24 @@ export async function readCatalogEndpointApiKey( const provider = asRecord(asRecord(config.providers)?.[providerId]); const endpoint = asRecord(asRecord(provider?.endpoints)?.[endpointId]); if (!provider || !endpoint) return null; - return existingString(endpoint.apiKey) ?? existingString(provider.apiKey) ?? null; + return existingString(endpoint.apiKey) ?? (endpoint.inheritProviderDefaults === false ? null : existingString(provider.apiKey)) ?? null; +} + +export async function readCatalogEndpointRequestOptions( + configPath: string, providerId: string, endpointId: string +): Promise { + const config = await readCurrentRuntimeConfig(configPath); + const provider = asRecord(asRecord(config.providers)?.[providerId]); + const endpoint = asRecord(asRecord(provider?.endpoints)?.[endpointId]); + if (!provider || !endpoint || !existingString(endpoint.apiBase) || !existingString(endpoint.protocol)) return null; + const inherited = endpoint.inheritProviderDefaults === false ? {} : provider; + const apiKey = existingString(endpoint.apiKey) ?? existingString(inherited.apiKey); + const extraHeaders = Object.fromEntries(Object.entries({ + ...asRecord(inherited.extraHeaders), ...asRecord(endpoint.extraHeaders) + }).filter((entry): entry is [string, string] => typeof entry[1] === "string")); + return { apiBase: existingString(endpoint.apiBase)!, protocol: existingString(endpoint.protocol)!, + ...(typeof endpoint.inheritProviderDefaults === "boolean" ? { inheritProviderDefaults: endpoint.inheritProviderDefaults } : {}), + ...(apiKey ? { apiKey } : {}), ...(Object.keys(extraHeaders).length ? { extraHeaders } : {}) }; } async function readCurrentRuntimeConfig(configPath: string): Promise> { diff --git a/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts b/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts index 0e5f1657c..af0ba8a76 100644 --- a/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts +++ b/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts @@ -350,9 +350,7 @@ function memoryConnection(config: ConfigRecord, presetId: string): ConfigRecord const endpoint = record(record(provider.endpoints)[endpointId]); const apiBase = stringValue(endpoint.apiBase); if (!apiBase) return null; - const apiKey = stringValue(endpoint.apiKey) ?? stringValue(provider.apiKey); - const extraHeaders = { ...record(provider.extraHeaders), ...record(endpoint.extraHeaders) }; - const extraBody = { ...record(provider.extraBody), ...record(endpoint.extraBody) }; + const { apiKey, extraHeaders, extraBody } = effectiveEndpoint(provider, endpoint); return { provider: memoryProvider(providerId, stringValue(endpoint.protocol)), sourceProvider: providerId, @@ -364,6 +362,15 @@ function memoryConnection(config: ConfigRecord, presetId: string): ConfigRecord }; } +function effectiveEndpoint(provider: ConfigRecord, endpoint: ConfigRecord) { + const inherited = endpoint.inheritProviderDefaults === false ? {} : provider; + return { + apiKey: stringValue(endpoint.apiKey) ?? stringValue(inherited.apiKey), + extraHeaders: { ...record(inherited.extraHeaders), ...record(endpoint.extraHeaders) }, + extraBody: { ...record(inherited.extraBody), ...record(endpoint.extraBody) } + }; +} + function memoryProvider(providerId: string, protocol: string | undefined): string { if (protocol === "bedrock-converse") return "bedrock"; if (protocol === "openai-responses") return "openai_responses"; @@ -426,11 +433,9 @@ function relocateCustomPresets(config: ConfigRecord): boolean { function movedEndpoint(provider: ConfigRecord, endpoint: ConfigRecord): ConfigRecord { const next: ConfigRecord = { ...endpoint }; - const apiKey = stringValue(endpoint.apiKey) ?? stringValue(provider.apiKey); + const { apiKey, extraHeaders, extraBody } = effectiveEndpoint(provider, endpoint); if (apiKey) next.apiKey = apiKey; - const extraHeaders = { ...record(provider.extraHeaders), ...record(endpoint.extraHeaders) }; if (Object.keys(extraHeaders).length) next.extraHeaders = extraHeaders; - const extraBody = { ...record(provider.extraBody), ...record(endpoint.extraBody) }; if (Object.keys(extraBody).length) next.extraBody = extraBody; const region = stringValue(endpoint.region) ?? stringValue(provider.region); if (region) next.region = region; @@ -459,6 +464,7 @@ function placeCustomEndpoint(endpoints: ConfigRecord, preferredId: string, endpo function sameStoredEndpoint(left: ConfigRecord, right: ConfigRecord): boolean { return stableJson({ protocol: left.protocol ?? null, + inheritProviderDefaults: left.inheritProviderDefaults !== false, apiBase: normalizeApiBase(stringValue(left.apiBase) ?? ""), apiKey: stringValue(left.apiKey) ?? null, region: stringValue(left.region) ?? null, @@ -467,6 +473,7 @@ function sameStoredEndpoint(left: ConfigRecord, right: ConfigRecord): boolean { extraBody: left.extraBody ?? null }) === stableJson({ protocol: right.protocol ?? null, + inheritProviderDefaults: right.inheritProviderDefaults !== false, apiBase: normalizeApiBase(stringValue(right.apiBase) ?? ""), apiKey: stringValue(right.apiKey) ?? null, region: stringValue(right.region) ?? null, @@ -598,6 +605,7 @@ function mergeEndpoint(previous: ConfigRecord, input: CatalogEndpointInput): Con apiBase: input.apiBase, protocol: input.protocol }; + if (input.inheritProviderDefaults !== undefined) next.inheritProviderDefaults = input.inheritProviderDefaults; setOptionalSecret(next, "apiKey", input.apiKey, previous.apiKey); setOptionalRecord(next, "extraHeaders", input.extraHeaders, previous.extraHeaders); setOptionalRecord(next, "extraBody", input.extraBody, previous.extraBody); @@ -619,13 +627,15 @@ function canonicalizeEndpointDefinitions( Number(right.endpointId in previousEndpoints) - Number(left.endpointId in previousEndpoints)); for (const endpoint of ordered) { const merged = record(endpoints[endpoint.endpointId]); + const effective = effectiveEndpoint(provider, merged); const signature = stableJson({ provider: input.provider, protocol: merged.protocol, + inheritProviderDefaults: merged.inheritProviderDefaults !== false, apiBase: normalizeApiBase(stringValue(merged.apiBase) ?? ""), - apiKey: stringValue(merged.apiKey) ?? stringValue(provider.apiKey) ?? null, - extraHeaders: merged.extraHeaders ?? provider.extraHeaders ?? null, - extraBody: merged.extraBody ?? provider.extraBody ?? null + apiKey: effective.apiKey ?? null, + extraHeaders: effective.extraHeaders, + extraBody: effective.extraBody }); const canonicalId = canonicalIds.get(signature); if (canonicalId) { @@ -839,6 +849,7 @@ function providerView( endpointId, apiBase, protocol, + ...(typeof endpoint.inheritProviderDefaults === "boolean" ? { inheritProviderDefaults: endpoint.inheritProviderDefaults } : {}), hasApiKey: Boolean(endpointApiKey), apiKeyMasked: maskSecret(endpointApiKey), apiKey: "", @@ -869,7 +880,7 @@ function presetView( const protocol = endpoint.protocol as ModelEndpointProtocol; const source = preset.source as "account" | "byok"; const providerId = preset.provider as CatalogProviderId; - const hasCredential = Boolean(stringValue(endpoint.apiKey) ?? stringValue(provider.apiKey)); + const hasCredential = Boolean(effectiveEndpoint(provider, endpoint).apiKey); const ownerMatches = source === "byok" || ( stringValue(preset.ownerAccountId) && stringValue(preset.ownerAccountId) === stringValue(provider.ownerAccountId) @@ -883,7 +894,7 @@ function presetView( source, ...(stringValue(preset.ownerAccountId) ? { ownerAccountId: stringValue(preset.ownerAccountId)! } : {}), capabilities: arrayValue(preset.capabilities).filter(isModelCapability), - available: Boolean((hasCredential || API_KEY_OPTIONAL_PROVIDERS.has(providerId)) && ownerMatches), + available: Boolean((hasCredential || (source === "byok" && endpoint.inheritProviderDefaults === false) || API_KEY_OPTIONAL_PROVIDERS.has(providerId)) && ownerMatches), ...(preset.thinking && typeof preset.thinking === "object" ? { thinking: preset.thinking as TextModelItemView["thinking"] } : {}), ...(preset.custom === true ? { custom: true } : {}), ...(Array.isArray(preset.inputModalities) ? { inputModalities: preset.inputModalities as TextModelItemView["inputModalities"] } : {}) diff --git a/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts b/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts index aa6d155b6..1b22ed5b7 100644 --- a/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts +++ b/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts @@ -8,6 +8,7 @@ import { type ModelAssignments, type ModelConfigInput } from "@memmy/local-api-contracts"; +import { readCatalogEndpointApiKey } from "../index.js"; import { InvalidModelConfigError, ModelConfigChangedError, @@ -68,6 +69,124 @@ function openAiInput(revision: string, presetId?: string): ModelConfigInput { }; } +describe("independent endpoint inheritance", () => { + it("keeps no-auth custom endpoints isolated when relocating them from a named Provider", async () => { + const file = fixture({ + providers: { custom: { + apiKey: "custom-provider-secret", + endpoints: { memory: { + apiBase: "http://127.0.0.1:18789/v1", protocol: "openai-responses", + extraHeaders: { "x-memory": "local" }, extraBody: { temperature: 0 } + } } + }, openai: { + apiKey: "provider-secret", + extraHeaders: { Authorization: "Bearer provider-header", "x-provider": "global" }, + extraBody: { providerOption: true }, + endpoints: { memory: { + apiBase: "http://127.0.0.1:18789/v1", + protocol: "openai-responses", + inheritProviderDefaults: false, + extraHeaders: { "x-memory": "local" }, + extraBody: { temperature: 0 } + } } + } }, + modelPresets: { "legacy-custom": { + provider: "openai", endpoint: "memory", model: "memory-model", source: "byok", + capabilities: ["agent", "memory_summary"], custom: true + } }, + modelAssignments: emptyAssignments() + }); + + const view = await readModelConfigCatalog(file); + const raw = YAML.parse(readFileSync(file, "utf8")) as any; + expect(raw.modelPresets["legacy-custom"].endpoint).toBe("memory-custom"); + const endpoint = raw.providers.custom.endpoints["memory-custom"]; + expect(endpoint).toMatchObject({ inheritProviderDefaults: false, + extraHeaders: { "x-memory": "local" }, extraBody: { temperature: 0 } }); + expect(endpoint).not.toHaveProperty("apiKey"); + expect(endpoint.extraHeaders).not.toHaveProperty("Authorization"); + expect(endpoint.extraHeaders).not.toHaveProperty("x-provider"); + expect(endpoint.extraBody).not.toHaveProperty("providerOption"); + expect(view.providers.find(provider => provider.provider === "custom")?.models[0]) + .toMatchObject({ presetId: "legacy-custom", available: true, protocol: "openai-responses" }); + expect(await readCatalogEndpointApiKey(file, "custom", "memory-custom")).toBeNull(); + }); + + it("does not coalesce inherited and independent endpoints with currently identical options", async () => { + const file = fixture({ modelAssignments: emptyAssignments() }); + const input = openAiInput((await readModelConfigCatalog(file)).configRevision); + input.providers[0]!.endpoints.push({ + ...input.providers[0]!.endpoints[0]!, endpointId: "independent", + inheritProviderDefaults: false, apiKey: input.providers[0]!.apiKey + }); + input.providers[0]!.models.push({ endpointId: "independent", model: "memory-model", + source: "byok", capabilities: ["memory_summary"] }); + const first = await writeModelConfigCatalog(file, input); + expect(first.providers[0]!.endpoints.map(endpoint => endpoint.endpointId).sort()) + .toEqual(["chat", "independent"]); + expect(first.endpointAliases?.openai).toBeUndefined(); + + input.configRevision = first.configRevision; + input.providers[0]!.apiKey = "changed-provider-secret"; + input.providers[0]!.endpoints[1]!.apiKey = ""; + input.providers[0]!.models = first.providers[0]!.models.map(model => ({ ...model })); + await writeModelConfigCatalog(file, input); + expect(await readCatalogEndpointApiKey(file, "openai", "chat")).toBe("changed-provider-secret"); + expect(await readCatalogEndpointApiKey(file, "openai", "independent")).toBe("sk-new-secret"); + }); + + it("round-trips isolated no-auth memory endpoints without borrowing Provider defaults or changing cloud models", async () => { + const file = fixture({ app: { userMode: "byok" }, modelAssignments: emptyAssignments() }); + const input = openAiInput((await readModelConfigCatalog(file)).configRevision); + input.providers[0]!.apiKey = "provider-secret"; + input.providers[0]!.extraHeaders = { Authorization: "Bearer provider-header", "x-provider": "global" }; + input.providers[0]!.extraBody = { providerOption: true, temperature: 0.8 }; + input.providers[0]!.endpoints.push( + { endpointId: "memory", apiBase: "http://127.0.0.1:18789/v1", protocol: "openai-chat-completions", inheritProviderDefaults: false, extraHeaders: { "x-memory": "local" }, extraBody: { temperature: 0 } }, + { endpointId: "embed", apiBase: "http://127.0.0.1:18789/v1", protocol: "openai-embeddings", inheritProviderDefaults: false } + ); + input.providers[0]!.models.push( + { endpointId: "memory", model: "memory-no-auth", source: "byok", capabilities: ["memory_evolution"] }, + { endpointId: "memory", model: "summary-no-auth", source: "byok", capabilities: ["memory_summary"] }, + { endpointId: "embed", model: "embedding-no-auth", source: "byok", capabilities: ["embedding"] } + ); + const created = await writeModelConfigCatalog(file, input); + const models = created.providers[0]!.models; + const cloud = models.find(model => model.model === "gpt-5")!.presetId; + const memory = models.find(model => model.model === "memory-no-auth")!.presetId; + const summary = models.find(model => model.model === "summary-no-auth")!.presetId; + const embed = models.find(model => model.model === "embedding-no-auth")!.presetId; + expect(models.every(model => model.available)).toBe(true); + expect(created.providers[0]!.endpoints.find(endpoint => endpoint.endpointId === "memory")) + .toMatchObject({ inheritProviderDefaults: false, hasApiKey: false, apiKeyMasked: "" }); + const save = { ...input, configRevision: created.configRevision, + providers: [{ ...input.providers[0]!, apiKey: "", models: models.map(model => ({ ...model })) }], + modelAssignments: { ...created.modelAssignments, byok: { ...created.modelAssignments.byok, + agent: { candidates: [cloud], default: cloud }, memorySummary: summary, memoryEvolution: memory, embedding: embed } } }; + const saved = await writeModelConfigCatalog(file, save); + const raw = YAML.parse(readFileSync(file, "utf8")) as any; + for (const role of ["summary", "evolution"]) { + expect(raw.memmyMemory[role]).toMatchObject({ endpoint: "http://127.0.0.1:18789/v1", model: role === "summary" ? "summary-no-auth" : "memory-no-auth", extraHeaders: { "x-memory": "local" }, extraBody: { temperature: 0 } }); + expect(raw.memmyMemory[role]).not.toHaveProperty("apiKey"); + expect(raw.memmyMemory[role].extraHeaders).not.toHaveProperty("Authorization"); + expect(raw.memmyMemory[role].extraBody).not.toHaveProperty("providerOption"); + } + expect(raw.memmyMemory.embedding).toMatchObject({ mode: "custom", model: "embedding-no-auth" }); + expect(raw.memmyMemory.embedding).not.toHaveProperty("apiKey"); + expect(raw.memmyMemory.embedding).not.toHaveProperty("extraHeaders"); + expect(raw.providers.openai.apiKey).toBe("provider-secret"); + expect(await readCatalogEndpointApiKey(file, "openai", "memory")).toBeNull(); + expect(await readCatalogEndpointApiKey(file, "openai", "chat")).toBe("provider-secret"); + const clear = { ...save, configRevision: saved.configRevision, modelAssignments: structuredClone(saved.modelAssignments) }; + clear.modelAssignments.byok.embedding = null; + await writeModelConfigCatalog(file, clear); + const local = YAML.parse(readFileSync(file, "utf8")) as any; + expect(local.memmyMemory.embedding).toMatchObject({ provider: "local", mode: "local" }); + expect(local.memmyMemory.embedding).not.toHaveProperty("endpoint"); + expect(local.providers.openai).toEqual(raw.providers.openai); + }); +}); + describe("model config catalog", () => { it("stores Custom beside OpenAI and moves an existing custom preset onto provider custom", async () => { const file = fixture({ diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index 71edc95dd..9699ce644 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -3,7 +3,6 @@ import { randomUUID } from "node:crypto"; import { memoryScanUnavailable } from "../adapters/outbound/memory-client/errors.js"; import { MANAGED_AGENT_DISCOVERY_PENDING_DATA_PATH } from "@memmy/local-api-contracts"; import { - setImmediate as yieldToEventLoop, setTimeout as waitForWorkerProgress } from "node:timers/promises"; import type { @@ -43,12 +42,10 @@ import { export type { ScanProgress } from "@memmy/agent-source-readers/adapters/types"; const SCAN_MESSAGE_YIELD_INTERVAL = 100; -const IMPORT_WORKER_BATCH_SIZE = 1; const IMPORT_PROCESSING_COHORT_SIZE = 100; -// A targeted run leases one job so the timeout never depends on queue ordering. -// One summary can make three content attempts, each with four 180s HTTP attempts -// and backoff; keep a safety margin without replaying the worker request. -const IMPORT_WORKER_TIMEOUT_MS = 2_400_000; +// Allow a slow summary and its provider retries to finish without mistaking a +// quiet shared queue for completion. The producer never leases tasks itself. +const IMPORT_PROCESSING_TIMEOUT_MS = 2_400_000; const IMPORT_PROGRESS_POLL_INTERVAL_MS = 250; /** Contract for agent source service. */ export interface AgentSourceService { @@ -83,7 +80,7 @@ export interface CreateAgentSourceServiceOptions { sourceRegistry: SourceRegistry; agentSourceRepository: AgentSourceRepository; ingestionService: IngestionService; - memoryClient: Pick; + memoryClient: Pick; skillDistributionService: SkillDistributionService; agentSourceAnalytics?: AgentSourceLifecycleAnalytics; memoryAddAnalytics?: Pick< @@ -519,46 +516,24 @@ async function processPendingImportSummaries( while (pendingMemoryIds.size > 0) { scanOptions.signal?.throwIfAborted(); - const targets = [...pendingMemoryIds]; - const workerOutcome = options.memoryClient.runWorker({ - limit: IMPORT_WORKER_BATCH_SIZE, - targetMemoryIds: targets, - priorityCohortOnly: true, - signal: scanOptions.signal, - timeoutMs: IMPORT_WORKER_TIMEOUT_MS - }).then((result) => ({ kind: "worker" as const, result })); - let result: Awaited> | undefined; - - while (!result) { - const outcome = await Promise.race([ - workerOutcome, - waitForWorkerProgress(IMPORT_PROGRESS_POLL_INTERVAL_MS, undefined, { signal: scanOptions.signal }) - .then(() => ({ kind: "poll" as const })) - ]); - if (outcome.kind === "worker") result = outcome.result; - const previousPending = pendingMemoryIds.size; - await reconcileImportProcessing(options.memoryClient, pendingMemoryIds, failures); - if (pendingMemoryIds.size < previousPending) lastProgressAt = Date.now(); - emitProgress(scanOptions, { - sourceId: progressSourceId, - phase: "summarize", - current: completedMemoryCount + cohort.length - pendingMemoryIds.size, - total: ownedMemoryIds.length, - message: "Summarizing and indexing latest memories" - }); - if (pendingMemoryIds.size === 0 && !result) { - result = (await workerOutcome).result; - } - } - + const previousPending = pendingMemoryIds.size; + await reconcileImportProcessing(options.memoryClient, pendingMemoryIds, failures); + scanOptions.signal?.throwIfAborted(); + if (pendingMemoryIds.size < previousPending) lastProgressAt = Date.now(); + emitProgress(scanOptions, { + sourceId: progressSourceId, + phase: "summarize", + current: completedMemoryCount + cohort.length - pendingMemoryIds.size, + total: ownedMemoryIds.length, + message: "Summarizing and indexing latest memories" + }); if (pendingMemoryIds.size === 0) break; - if (Date.now() - lastProgressAt >= IMPORT_WORKER_TIMEOUT_MS) { + if (Date.now() - lastProgressAt >= IMPORT_PROCESSING_TIMEOUT_MS) { throw new Error(`Timed out waiting for ${pendingMemoryIds.size} imported memories to finish indexing`); } - if (result.leased === 0 && result.embeddingRetries.leased === 0) { - await waitForWorkerProgress(IMPORT_PROGRESS_POLL_INTERVAL_MS, undefined, { signal: scanOptions.signal }); - } - await yieldToEventLoop(); + // enqueueImportSummaries wakes the shared scheduler, including existing + // pending jobs. Observe durable target state rather than starting a drain. + await waitForWorkerProgress(IMPORT_PROGRESS_POLL_INTERVAL_MS, undefined, { signal: scanOptions.signal }); } completedMemoryCount += cohort.length; } diff --git a/App/backend/src/services/app-config-service.ts b/App/backend/src/services/app-config-service.ts index ec2330727..8ccc5ee3d 100644 --- a/App/backend/src/services/app-config-service.ts +++ b/App/backend/src/services/app-config-service.ts @@ -30,7 +30,7 @@ import type { BootstrapRepository } from "../infrastructure/app-state-store/repo import type { MemmyConfigWriter } from "../infrastructure/memmy-config/index.js"; import type { ScanPreferencesStore } from "../infrastructure/memmy-config/agent-access.js"; import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; -import { createHttpModelConfigTester, type ModelConfigTester } from "./model-config-tester.js"; +import { createHttpModelConfigTester, type ModelConfigTester, type ResolvedModelConfigTestInput } from "./model-config-tester.js"; export interface AppConfigService { updateSettings(input: PatchAppSettingsInput): Promise; @@ -89,10 +89,6 @@ const BUILT_IN_AVATARS = AvatarOptionSchema.array().parse([ kind: "video" } ]); -/** Type definition for normalized model config input. */ -/** Type definition for resolved model config test input. */ -type ResolvedModelConfigTestInput = ModelConfigTestInput & { apiKey: string }; - /** Creates create app config service. */ export function createAppConfigService(options: CreateAppConfigServiceOptions): AppConfigService { const modelConfigTester = options.modelConfigTester ?? createHttpModelConfigTester(); @@ -264,21 +260,23 @@ async function resolveModelConfigTestInput( configWriter: MemmyConfigWriter | undefined ): Promise { const directApiKey = input.apiKey?.trim(); - if (directApiKey) { - return { ...input, apiKey: directApiKey }; - } - const provider = canonicalCatalogProviderId(input.provider); if (!provider || !input.endpointId) { + if (directApiKey) return { ...input, apiKey: directApiKey }; throw Object.assign(new Error("Model config test requires a Provider endpoint"), { code: "invalid_argument" as const }); } - const storedApiKey = await configWriter?.readEndpointApiKey?.(provider, input.endpointId); - if (!storedApiKey) { + const endpoint = await configWriter?.readEndpointRequestOptions?.(provider, input.endpointId); + const independent = endpoint?.inheritProviderDefaults === false + && endpoint.protocol === input.protocol + && endpoint.apiBase.replace(/\/+$/u, "") === input.apiBase.replace(/\/+$/u, ""); + const storedApiKey = directApiKey ?? endpoint?.apiKey ?? await configWriter?.readEndpointApiKey?.(provider, input.endpointId); + if (!storedApiKey && !independent) { throw Object.assign(new Error("Model config API Key is not configured"), { code: "invalid_argument" as const }); } - return { ...input, apiKey: storedApiKey }; + return { ...input, apiKey: storedApiKey ?? "", + ...(independent && endpoint.extraHeaders ? { extraHeaders: endpoint.extraHeaders } : {}) }; } /** Reads get authenticated cloud account. */ diff --git a/App/backend/src/services/model-config-tester.ts b/App/backend/src/services/model-config-tester.ts index 064522b23..3da9821b3 100644 --- a/App/backend/src/services/model-config-tester.ts +++ b/App/backend/src/services/model-config-tester.ts @@ -12,7 +12,7 @@ export interface ModelConfigTester { test(input: ResolvedModelConfigTestInput): Promise; } -type ResolvedModelConfigTestInput = ModelConfigTestInput & { apiKey: string }; +export type ResolvedModelConfigTestInput = ModelConfigTestInput & { apiKey: string; extraHeaders?: Record }; export interface CreateHttpModelConfigTesterOptions { fetch?: FetchLike; @@ -51,7 +51,7 @@ export function createHttpModelConfigTester(options: CreateHttpModelConfigTester signal: AbortSignal.timeout(timeoutMs) }); if (!response.ok) { - const errorMessage = redactSecret(await readErrorMessage(response), input.apiKey); + const errorMessage = redactProbeSecrets(await readErrorMessage(response), input); return result( false, response.status === 404 @@ -64,7 +64,7 @@ export function createHttpModelConfigTester(options: CreateHttpModelConfigTester const body = await readJsonSafely(response); const providerError = extractErrorMessage(body); if (providerError) { - return result(false, redactSecret(providerError, input.apiKey), now); + return result(false, redactProbeSecrets(providerError, input), now); } if (!probe.isValidBody(body)) { return result(false, appendBaseUrlGuidance(INVALID_SUCCESS_BODY_MESSAGE, input.protocol), now); @@ -73,7 +73,7 @@ export function createHttpModelConfigTester(options: CreateHttpModelConfigTester const modelListed = probe.listedModels(body).some((model) => model === input.modelId); return result(true, SUCCESS_MESSAGE, now, modelListed); } catch (error) { - return result(false, redactSecret(normalizeThrownError(error), input.apiKey), now); + return result(false, redactProbeSecrets(normalizeThrownError(error), input), now); } } }; @@ -83,10 +83,10 @@ function listProbe(input: ResolvedModelConfigTestInput): ListProbe | null { if (input.protocol === "anthropic-messages") { return { url: versionedModelsUrl(input.apiBase, "v1"), - headers: { - "x-api-key": input.apiKey, + headers: probeHeaders({ + ...(input.apiKey ? { "x-api-key": input.apiKey } : {}), "anthropic-version": ANTHROPIC_VERSION - }, + }, input.extraHeaders), isValidBody: isAnthropicModelsBody, listedModels: anthropicModelIds }; @@ -94,7 +94,7 @@ function listProbe(input: ResolvedModelConfigTestInput): ListProbe | null { if (input.protocol === "gemini-generate-content") { return { url: versionedModelsUrl(input.apiBase, "v1beta"), - headers: { "x-goog-api-key": input.apiKey }, + headers: probeHeaders(input.apiKey ? { "x-goog-api-key": input.apiKey } : {}, input.extraHeaders), isValidBody: isGoogleModelsBody, listedModels: googleModelIds }; @@ -102,7 +102,7 @@ function listProbe(input: ResolvedModelConfigTestInput): ListProbe | null { if (supportsOpenAiModelList(input.protocol)) { return { url: resourceUrl(input.apiBase, "models"), - headers: { Authorization: `Bearer ${input.apiKey}` }, + headers: probeHeaders(input.apiKey ? { Authorization: `Bearer ${input.apiKey}` } : {}, input.extraHeaders), isValidBody: isOpenAiModelsBody, listedModels: openAiModelIds }; @@ -110,6 +110,20 @@ function listProbe(input: ResolvedModelConfigTestInput): ListProbe | null { return null; } +function probeHeaders(defaults: Record, extra: Record = {}): Record { + const headers = { ...defaults }; + for (const [key, value] of Object.entries(extra)) { + for (const existing of Object.keys(headers)) if (existing.toLowerCase() === key.toLowerCase()) delete headers[existing]; + headers[key] = value; + } + return headers; +} + +function redactProbeSecrets(message: string, input: ResolvedModelConfigTestInput): string { + for (const secret of [input.apiKey, ...Object.values(input.extraHeaders ?? {})]) message = redactSecret(message, secret); + return message; +} + function supportsOpenAiModelList(protocol: ModelEndpointProtocol): boolean { return protocol === "openai-chat-completions" || protocol === "openai-responses" diff --git a/App/backend/src/services/onboarding-first-report-memory-writer.ts b/App/backend/src/services/onboarding-first-report-memory-writer.ts index 62bc2caae..3b57daf97 100644 --- a/App/backend/src/services/onboarding-first-report-memory-writer.ts +++ b/App/backend/src/services/onboarding-first-report-memory-writer.ts @@ -34,7 +34,7 @@ export interface OnboardingFirstReportMemoryWriter { export function createOnboardingFirstReportMemoryWriter( memoryClient: Pick< MemoryClient, - "addMemory" | "enqueueImportSummaries" | "getMemoryProcessingStatus" | "runWorker" | "search" + "addMemory" | "enqueueImportSummaries" | "getMemoryProcessingStatus" | "search" >, now: () => number = Date.now ): OnboardingFirstReportMemoryWriter { @@ -150,7 +150,7 @@ function renderList(title: string, values: readonly string[], emptyLabel: string } async function processFirstReportMemory( - memoryClient: Pick, + memoryClient: Pick, memoryId: string, now: () => number ): Promise { @@ -167,15 +167,9 @@ async function processFirstReportMemory( throw new Error(`First-report memory was not indexed: ${processing.state}`); } - const run = await memoryClient.runWorker({ - limit: 4, - targetMemoryIds: [memoryId], - priorityCohortOnly: true, - timeoutMs: FIRST_REPORT_PROCESSING_TIMEOUT_MS - }); - if (run.leased === 0 && run.embeddingRetries.leased === 0) { - await delay(100); - } + // The enqueue request already wakes the shared scheduler. First reports + // retain their interactive policy without a separate targeted worker. + await delay(100); } throw new Error(`First-report memory indexing timed out: ${memoryId}`); } diff --git a/App/backend/src/services/tests/agent-source-service.test.ts b/App/backend/src/services/tests/agent-source-service.test.ts index 84a67e365..f0be2b125 100644 --- a/App/backend/src/services/tests/agent-source-service.test.ts +++ b/App/backend/src/services/tests/agent-source-service.test.ts @@ -31,6 +31,7 @@ let db: DatabaseSync | undefined; let tempDir: string | undefined; afterEach(() => { + vi.restoreAllMocks(); db?.close(); db = undefined; if (tempDir) { @@ -244,6 +245,70 @@ describe("agent source service", () => { ])); }); + it("waits for managed import processing without bypassing the shared scheduler", async () => { + const base = createReadyMemoryClient(); + const runWorker = vi.fn(base.runWorker); + const enqueueImportSummaries = vi.fn(base.enqueueImportSummaries); + let polls = 0; + const getMemoryProcessingStatus = vi.fn(async (ids: string[]) => { + const result = await base.getMemoryProcessingStatus(ids); + return ++polls === 1 + ? { ...result, items: result.items.map(item => ({ ...item, state: "summary_pending" as const })) } + : result; + }); + const memoryIds = ["managed-memory"]; + const service = createService({ + memoryClient: { ...base, runWorker, enqueueImportSummaries, getMemoryProcessingStatus }, + ingestionService: ingestionWithMemoryIds(memoryIds) + }); + const source = await service.addManual({ displayName: "Managed queue regression" }); + await expect(service.importManaged(source.sourceId, { mode: "full", messages: [] })) + .resolves.toMatchObject({ memoryIds }); + expect(enqueueImportSummaries).toHaveBeenCalledExactlyOnceWith(memoryIds); + expect(getMemoryProcessingStatus).toHaveBeenCalledTimes(2); + expect(runWorker).not.toHaveBeenCalled(); + }); + + it("keeps managed imports bounded and deduplicated without targeted worker drains", async () => { + const base = createReadyMemoryClient(); + const runWorker = vi.fn(base.runWorker); + const enqueueImportSummaries = vi.fn(base.enqueueImportSummaries); + const getMemoryProcessingStatus = vi.fn(base.getMemoryProcessingStatus); + const memoryIds = Array.from({ length: 205 }, (_, index) => `managed-${index}`); + const service = createService({ + memoryClient: { ...base, runWorker, enqueueImportSummaries, getMemoryProcessingStatus }, + ingestionService: ingestionWithMemoryIds([...memoryIds, memoryIds[0]!]) + }); + const source = await service.addManual({ displayName: "Managed cohort regression" }); + await service.importManaged(source.sourceId, { mode: "full", messages: [] }); + const batches = enqueueImportSummaries.mock.calls.map(([ids]) => ids!); + expect(batches.map(ids => ids.length)).toEqual([100, 100, 5]); + expect(batches.flat()).toEqual(memoryIds); + expect(getMemoryProcessingStatus.mock.calls.map(([ids]) => ids)).toEqual(batches); + expect(runWorker).not.toHaveBeenCalled(); + }); + + it("reports a stalled managed processing queue without issuing a worker call", async () => { + const base = createReadyMemoryClient(); + const runWorker = vi.fn(base.runWorker); + let clock = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => clock); + const service = createService({ + memoryClient: { ...base, runWorker, + async getMemoryProcessingStatus(ids) { + clock += 2_400_000; + const result = await base.getMemoryProcessingStatus(ids); + return { ...result, items: result.items.map(item => ({ ...item, state: "embedding_pending" as const })) }; + } + }, + ingestionService: ingestionWithMemoryIds(["managed-stalled"]) + }); + const source = await service.addManual({ displayName: "Managed timeout regression" }); + await expect(service.importManaged(source.sourceId, { mode: "full", messages: [] })) + .rejects.toThrow("Timed out waiting for 1 imported memories to finish indexing"); + expect(runWorker).not.toHaveBeenCalled(); + }); + it("persists the AI-discovered recipe and later syncs without another Agent session", async () => { tempDir = mkdtempSync(join(tmpdir(), "memmy-managed-sync-")); const historyPath = join(tempDir, "history.jsonl"); @@ -835,6 +900,13 @@ function createReadyMemoryClient(): MemoryClient { }; } +function ingestionWithMemoryIds(memoryIds: string[]): IngestionService { + const base = createFakeIngestionService(); + return { async ingest(messages, context) { + return { ...await base.ingest(messages, context), memoryIds }; + } }; +} + function createFakeIngestionService(): IngestionService { return { async ingest(messages) { diff --git a/App/backend/src/services/tests/independent-endpoint-probe.test.ts b/App/backend/src/services/tests/independent-endpoint-probe.test.ts new file mode 100644 index 000000000..6f20c8d40 --- /dev/null +++ b/App/backend/src/services/tests/independent-endpoint-probe.test.ts @@ -0,0 +1,73 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it } from "vitest"; +import { createMemmyConfigWriter } from "../../infrastructure/memmy-config/index.js"; +import { createAppConfigService } from "../app-config-service.js"; +import { createHttpModelConfigTester } from "../model-config-tester.js"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function fixture(endpoint: Record, provider: Record = {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-endpoint-probe-")); roots.push(root); + const configPath = join(root, "config.yaml"); + writeFileSync(configPath, YAML.stringify({ providers: { openai: { ...provider, endpoints: { local: { + apiBase: "http://127.0.0.1:18789/v1", protocol: "openai-chat-completions", ...endpoint + } } } } })); + const requests: Request[] = []; + const service = createAppConfigService({ bootstrapRepository: {} as never, + memmyConfigWriter: createMemmyConfigWriter({ configPath }), + modelConfigTester: createHttpModelConfigTester({ fetch: async (url, init) => { + requests.push(new Request(url, init)); + return new Response(JSON.stringify({ data: [{ id: "local-model" }] }), { status: 200 }); + } }) + }); + const input = { provider: "openai_compatible" as const, endpointId: "local", protocol: "openai-chat-completions" as const, + apiBase: "http://127.0.0.1:18789/v1", modelId: "local-model" }; + return { service, requests, input }; +} + +describe("saved independent endpoints through the real model test service and HTTP probe", () => { + it("sends no Authorization for a saved no-auth endpoint despite populated Provider defaults", async () => { + const h = fixture({ inheritProviderDefaults: false }, { apiKey: "provider-secret", extraHeaders: { Authorization: "Bearer provider-header" } }); + await expect(h.service.testModelConfig(h.input)).resolves.toMatchObject({ ok: true }); + expect(h.requests).toHaveLength(1); + expect(h.requests[0]!.url).toBe("http://127.0.0.1:18789/v1/models"); + expect(h.requests[0]!.headers.has("Authorization")).toBe(false); + expect(h.requests[0]!.headers.has("x-api-key")).toBe(false); + }); + it("keeps endpoint-owned Authorization and headers while excluding Provider headers", async () => { + const h = fixture({ inheritProviderDefaults: false, extraHeaders: { Authorization: "Bearer endpoint-owned", "x-local": "local" } }, + { apiKey: "provider-secret", extraHeaders: { Authorization: "Bearer provider-header", "x-provider": "must-not-send" } }); + await expect(h.service.testModelConfig(h.input)).resolves.toMatchObject({ ok: true }); + expect(h.requests[0]!.headers.get("Authorization")).toBe("Bearer endpoint-owned"); + expect(h.requests[0]!.headers.get("x-local")).toBe("local"); + expect(h.requests[0]!.headers.has("x-provider")).toBe(false); + }); + it("preserves endpoint-owned keys when independent, and legacy Provider keys by default", async () => { + for (const independent of [false, true]) { + const h = fixture(independent ? { inheritProviderDefaults: false, apiKey: "endpoint-secret" } : {}, { apiKey: "provider-secret" }); + await expect(h.service.testModelConfig(h.input)).resolves.toMatchObject({ ok: true }); + expect(h.requests[0]!.headers.get("Authorization")).toBe(independent ? "Bearer endpoint-secret" : "Bearer provider-secret"); + } + }); + it("keeps explicit endpoint Authorization even when a replacement key is supplied", async () => { + const h = fixture({ inheritProviderDefaults: false, apiKey: "old-key", extraHeaders: { authorization: "Bearer endpoint-owned" } }); + await expect(h.service.testModelConfig({ ...h.input, apiKey: "replacement-key" })).resolves.toMatchObject({ ok: true }); + expect(h.requests[0]!.headers.get("Authorization")).toBe("Bearer endpoint-owned"); + }); + + it("does not send saved endpoint headers to an edited URL before that endpoint is saved", async () => { + const h = fixture({ inheritProviderDefaults: false, extraHeaders: { Authorization: "Bearer endpoint-owned" } }); + await expect(h.service.testModelConfig({ ...h.input, apiBase: "http://127.0.0.1:18790/v1" })).rejects.toThrow(/API Key/); + expect(h.requests).toEqual([]); + }); + + it("does not let a self-reported flag bypass the default missing-key rule", async () => { + const h = fixture({}); + await expect(h.service.testModelConfig({ ...h.input, inheritProviderDefaults: false } as typeof h.input)) + .rejects.toThrow(/API Key/); + expect(h.requests).toEqual([]); + }); +}); diff --git a/App/backend/src/services/tests/onboarding-first-report-memory-writer.test.ts b/App/backend/src/services/tests/onboarding-first-report-memory-writer.test.ts index 036ff5ef2..0a0144524 100644 --- a/App/backend/src/services/tests/onboarding-first-report-memory-writer.test.ts +++ b/App/backend/src/services/tests/onboarding-first-report-memory-writer.test.ts @@ -108,13 +108,8 @@ describe("onboarding first-report memory writer", () => { expect(added?.content).toContain("## assistant\n\nMemmy 初见报告"); expect(added?.content).toContain("## 接下来可以做\n1. 运行测试。"); expect(enqueueImportSummaries).toHaveBeenCalledWith(["memory-first-report"]); - expect(runWorker).toHaveBeenCalledTimes(2); - expect(runWorker).toHaveBeenCalledWith({ - limit: 4, - targetMemoryIds: ["memory-first-report"], - priorityCohortOnly: true, - timeoutMs: 180_000 - }); + expect(runWorker).not.toHaveBeenCalled(); + expect(getMemoryProcessingStatus).toHaveBeenCalledTimes(3); expect(search).toHaveBeenCalledWith({ requestId: expect.stringMatching(/^first-report-search-log:/), adapterId: "agent-source:memmy-onboarding", @@ -123,7 +118,58 @@ describe("onboarding first-report memory writer", () => { layers: ["L1"] }); expect(addMemory.mock.invocationCallOrder[0]).toBeLessThan(enqueueImportSummaries.mock.invocationCallOrder[0] ?? 0); - expect(enqueueImportSummaries.mock.invocationCallOrder[0]).toBeLessThan(runWorker.mock.invocationCallOrder[0] ?? 0); - expect(runWorker.mock.invocationCallOrder.at(-1) ?? 0).toBeLessThan(search.mock.invocationCallOrder[0] ?? 0); + expect(enqueueImportSummaries.mock.invocationCallOrder[0]).toBeLessThan(getMemoryProcessingStatus.mock.invocationCallOrder[0] ?? 0); + expect(getMemoryProcessingStatus.mock.invocationCallOrder.at(-1) ?? 0).toBeLessThan(search.mock.invocationCallOrder[0] ?? 0); + }); + it.each(["failed", "ready_text_only", "missing"] as const)("does not search or start a worker when the target is %s", async (state) => { + const client = pollingClient(); + client.getMemoryProcessingStatus.mockResolvedValue({ + items: state === "missing" ? [] : [{ + memoryId: "memory-first-report", state, attemptCount: 1, manualRetryCount: 0, + retryAction: "none", updatedAt: "2026-08-05T10:00:00.000Z" + }], + serverTime: "2026-08-05T10:00:00.000Z" + }); + const writer = createOnboardingFirstReportMemoryWriter(client); + await expect(writer.write(firstReportInput())).rejects.toThrow(state === "missing" ? "processing state is missing" : `was not indexed: ${state}`); + expect(client.runWorker).not.toHaveBeenCalled(); + expect(client.search).not.toHaveBeenCalled(); + }); + + it("times out waiting for shared processing instead of launching a targeted worker", async () => { + const client = pollingClient(); + let clock = 0; + client.getMemoryProcessingStatus.mockImplementation(async () => { + clock = 180_000; + return { items: [{ memoryId: "memory-first-report", state: "summary_pending", + attemptCount: 0, manualRetryCount: 0, retryAction: "none", + updatedAt: "2026-08-05T10:00:00.000Z" }], serverTime: "2026-08-05T10:00:00.000Z" }; + }); + const writer = createOnboardingFirstReportMemoryWriter(client, () => clock); + await expect(writer.write(firstReportInput())).rejects.toThrow("First-report memory indexing timed out"); + expect(client.enqueueImportSummaries).toHaveBeenCalledWith(["memory-first-report"]); + expect(client.runWorker).not.toHaveBeenCalled(); + expect(client.search).not.toHaveBeenCalled(); }); + }); + + +function firstReportInput(): Parameters["write"]>[0] { + return { + locale: "zh-CN", reportMarkdown: "初见报告", projects: [], keywords: [], + taskContext: { topic: "", userGoal: "", latestRequest: "", status: "active", currentState: "", + agentActions: [], verifiedResults: [], unresolvedItems: [], continuationPoint: "", trajectorySummary: "" }, + latestConversation: { agentSource: "Codex", conversationId: "test-conversation", workspacePath: null } + }; +} + +function pollingClient() { + return { + addMemory: vi.fn(async () => ({ id: "memory-first-report" }) as Awaited>), + enqueueImportSummaries: vi.fn(async () => ({ enqueued: 0, memoryIds: ["memory-first-report"], serverTime: "2026-08-05T10:00:00.000Z" })), + getMemoryProcessingStatus: vi.fn(), + runWorker: vi.fn(), + search: vi.fn() + } satisfies FirstReportMemoryClient; +} diff --git a/App/backend/src/tests/backend-shutdown.test.ts b/App/backend/src/tests/backend-shutdown.test.ts new file mode 100644 index 000000000..26d6aa3f8 --- /dev/null +++ b/App/backend/src/tests/backend-shutdown.test.ts @@ -0,0 +1,164 @@ +import { Agent, request, type ClientRequest, type IncomingMessage } from "node:http"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import type { FastifyInstance } from "fastify"; +import { createLocalBackend } from "../index.js"; +import { createLocalApiServer } from "../adapters/inbound/local-api/server.js"; +import type { PermissionManager } from "../permission/index.js"; +import { createProgressBus } from "../services/progress-bus.js"; +import type { BackendServices } from "../services/index.js"; +import { createMockCloudClient } from "./support/mock-cloud-client.js"; +import { createMockMemoryClient } from "./support/mock-memory-client.js"; + +function deferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} +async function settlesWithin(promise: Promise, milliseconds = 500): Promise { + let timer: ReturnType | undefined; + try { return await Promise.race([promise.then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), milliseconds); })]); } + finally { clearTimeout(timer); } +} +function connections() { + const agent = new Agent({ keepAlive: true, maxSockets: 4 }); + const requests: ClientRequest[] = []; + const responses: IncomingMessage[] = []; + const bodies = new Map>(); + function open(url: string, headers?: Record, payload?: string) { + return new Promise((resolve, reject) => { + const clientRequest = request(url, { agent, headers, method: payload === undefined ? "GET" : "POST" }, response => { + responses.push(response); + bodies.set(response, new Promise(resolveBody => { + let body = ""; + response.on("data", chunk => { body += String(chunk); }); + response.once("end", () => resolveBody(body)); + response.on("error", () => {}); + })); + response.resume(); resolve(response); + }); + requests.push(clientRequest); clientRequest.on("error", reject); clientRequest.end(payload); + }); + } + return { open, body: (response: IncomingMessage) => bodies.get(response)!, destroy() { for (const response of responses) response.destroy(); for (const request of requests) request.destroy(); agent.destroy(); } }; +} +function api(verifyRuntimeToken: () => Promise = async () => true, executeRouterTool: BackendServices["integrations"]["executeRouterTool"] = async () => ({ saved: true })) { + const progressBus = createProgressBus(); + const unsubscribers: ReturnType[] = []; + const services = { + bootstrap: { async getBootstrap() { throw new Error("unused"); } }, + integrations: { executeRouterTool }, + progressBus: { ...progressBus, on: ((...args: Parameters) => { + const unsubscribe = vi.fn(progressBus.on(...args)); unsubscribers.push(unsubscribe); return unsubscribe; + }) } + } as unknown as BackendServices; + const app = createLocalApiServer({ + permissionManager: { verifyRuntimeToken } as PermissionManager, + composioMcpToken: "shutdown-test-mcp", services, heartbeatIntervalMs: 60_000, + }); + return { app, unsubscribers }; +} +async function listen(app: FastifyInstance) { return app.listen({ host: "127.0.0.1", port: 0 }); } + +describe("local backend graceful shutdown", () => { + it("closes with two connected renderer event streams and idle keepalive without client cancellation", async () => { + const root = mkdtempSync(join(tmpdir(), "memmy-backend-shutdown-")); + const clients = connections(); + const backend = await createLocalBackend({ + databasePath: join(root, "app.sqlite"), runtimeConfigPath: join(root, "runtime.json"), + memmyConfigPath: join(root, "config.yaml"), localToken: "test-token", heartbeatIntervalMs: 60_000, + memoryClient: createMockMemoryClient(), cloudClient: createMockCloudClient(), + }); + let shutdown: Promise | undefined; + try { + const base = backend.runtimeConfig.baseUrl; + await clients.body(await clients.open(`${base}/api/health`)); + const streams = await Promise.all([clients.open(`${base}/api/events?token=test-token`), clients.open(`${base}/api/events?token=test-token`)]); + for (const stream of streams) expect(stream.statusCode).toBe(200); + shutdown = backend.close(); + expect(await settlesWithin(shutdown)).toBe(true); + for (const stream of streams) expect(await clients.body(stream)).toContain("event: app.connected"); + } finally { clients.destroy(); await (shutdown ?? backend.close()); rmSync(root, { recursive: true, force: true }); } + }); + + it("unsubscribes each SSE listener once while preserving an in-flight ordinary request", async () => { + const { app, unsubscribers } = api(); + const clients = connections(); const started = deferred(); const release = deferred(); + let writeFinished = false; let shutdown: Promise | undefined; + app.get("/shutdown-write", async () => { started.resolve(); await release.promise; writeFinished = true; return { saved: true }; }); + const base = await listen(app); + try { + await clients.open(`${base}/api/events?token=test-token`); + const write = clients.open(`${base}/shutdown-write`); + await started.promise; + shutdown = app.close(); + expect(await settlesWithin(shutdown, 30)).toBe(false); + expect(writeFinished).toBe(false); + release.resolve(); + const response = await write; + expect(response.statusCode).toBe(200); + expect(JSON.parse(await clients.body(response))).toEqual({ saved: true }); + expect(await settlesWithin(shutdown)).toBe(true); + expect(writeFinished).toBe(true); + expect(unsubscribers).toHaveLength(2); + for (const unsubscribe of unsubscribers) expect(unsubscribe).toHaveBeenCalledOnce(); + } finally { release.resolve(); clients.destroy(); await (shutdown ?? app.close()); } + }); + + it("also ends an event stream whose authentication was already in flight when closing started", async () => { + const entered = deferred(); const authorize = deferred(); + const { app, unsubscribers } = api(async () => { entered.resolve(); await authorize.promise; return true; }); + const clients = connections(); const base = await listen(app); let shutdown: Promise | undefined; + try { + const stream = clients.open(`${base}/api/events?token=test-token`); + await entered.promise; + shutdown = app.close(); + expect(await settlesWithin(shutdown, 30)).toBe(false); + authorize.resolve(); + expect((await stream).statusCode).toBe(200); + expect(await settlesWithin(shutdown)).toBe(true); + expect(unsubscribers).toHaveLength(2); + for (const unsubscribe of unsubscribers) expect(unsubscribe).toHaveBeenCalledOnce(); + } finally { authorize.resolve(); clients.destroy(); await (shutdown ?? app.close()); } + }); + + it("also closes the idle Composio MCP notification stream without client cancellation", async () => { + const { app } = api(); const clients = connections(); const base = await listen(app); + let shutdown: Promise | undefined; + try { + const stream = await clients.open(`${base}/mcp/composio`, { + accept: "text/event-stream", "x-memmy-mcp-token": "shutdown-test-mcp", + }); + expect(stream.statusCode).toBe(200); + shutdown = app.close(); + expect(await settlesWithin(shutdown)).toBe(true); + } finally { clients.destroy(); await (shutdown ?? app.close()); } + }); + + it("lets an active MCP tool POST finish its write and full response while ending an idle MCP GET", async () => { + const entered = deferred(); const release = deferred(); let saved = false; + const { app } = api(undefined, async () => { entered.resolve(); await release.promise; saved = true; return { saved: true }; }); + const clients = connections(); const base = await listen(app); let shutdown: Promise | undefined; + try { + await clients.open(`${base}/mcp/composio`, { accept: "text/event-stream", "x-memmy-mcp-token": "shutdown-test-mcp" }); + const tool = clients.open(`${base}/mcp/composio`, { + accept: "application/json, text/event-stream", "content-type": "application/json", "x-memmy-mcp-token": "shutdown-test-mcp", + }, JSON.stringify({ jsonrpc: "2.0", id: 7, method: "tools/call", params: { name: "composio_execute_tool", arguments: { tool_slug: "FIXTURE_SAVE", arguments: {} } } })); + await entered.promise; + shutdown = app.close(); + expect(await settlesWithin(shutdown, 30)).toBe(false); + expect(saved).toBe(false); + release.resolve(); + const response = await tool; + expect(response.statusCode).toBe(200); + const body = await clients.body(response); + const result = JSON.parse(body.split("data: ")[1].trim()); + expect(result.id).toBe(7); + expect(JSON.parse(result.result.content[0].text)).toEqual({ saved: true }); + expect(await settlesWithin(shutdown)).toBe(true); + expect(saved).toBe(true); + } finally { release.resolve(); clients.destroy(); await (shutdown ?? app.close()); } + }); +}); diff --git a/App/frontend/desktop/src/api/config-client.ts b/App/frontend/desktop/src/api/config-client.ts index f1795014f..e30c90c1d 100644 --- a/App/frontend/desktop/src/api/config-client.ts +++ b/App/frontend/desktop/src/api/config-client.ts @@ -451,6 +451,7 @@ function rebaseEndpoints( const merged = structuredClone(latestEndpoint); merged.apiBase = mergeIntentValue(baseEndpoint.apiBase, desiredEndpoint.apiBase, latestEndpoint.apiBase, `Endpoint ${endpointId} URL`); merged.protocol = mergeIntentValue(baseEndpoint.protocol, desiredEndpoint.protocol, latestEndpoint.protocol, `Endpoint ${endpointId} protocol`); + mergeOptionalField(merged, baseEndpoint, desiredEndpoint, latestEndpoint, "inheritProviderDefaults", `Endpoint ${endpointId} defaults inheritance`); mergeOptionalField(merged, baseEndpoint, desiredEndpoint, latestEndpoint, "apiKey", `Endpoint ${endpointId} API key`); mergeOptionalField(merged, baseEndpoint, desiredEndpoint, latestEndpoint, "extraHeaders", `Endpoint ${endpointId} headers`); mergeOptionalField(merged, baseEndpoint, desiredEndpoint, latestEndpoint, "extraBody", `Endpoint ${endpointId} body`); diff --git a/App/frontend/desktop/src/pages/model-workspace-section.tsx b/App/frontend/desktop/src/pages/model-workspace-section.tsx index 5d24c83b4..86f267ee0 100644 --- a/App/frontend/desktop/src/pages/model-workspace-section.tsx +++ b/App/frontend/desktop/src/pages/model-workspace-section.tsx @@ -658,7 +658,7 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { return; } const credentials = editorCredentials; - if (!credentials || (!credentials.apiKey && !credentials.hasExistingApiKey)) { + if (!credentials || !editorHasUsableKey) { setEditorTest({ status: "error", message: t(credentials?.requiresApiKey ? "apiKey.credentialsChanged" : "settings.modelWorkspace.testKeyRequired") }); return; @@ -820,10 +820,18 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { editorExistingConnection.apiKeyMasked ) } : null) : null; - const editorHasUsableKey = Boolean(editorCredentials?.apiKey || editorCredentials?.hasExistingApiKey); + // A saved independent endpoint may deliberately use no API key. Its identity + // must still match before either testing it or reusing that authentication. + const editorHasIndependentCredentials = Boolean( + editorExistingConnection?.inheritProviderDefaults === false + && editorCredentials?.endpointId === editorExistingConnection.endpointId + ); + const editorHasUsableKey = Boolean( + editorCredentials?.apiKey || editorCredentials?.hasExistingApiKey || editorHasIndependentCredentials + ); const editorDetailsChanged = Boolean(editorExistingConnection && editorCredentials && ( editorCredentials.apiKey - || !editorCredentials.hasExistingApiKey + || (!editorCredentials.hasExistingApiKey && !editorHasIndependentCredentials) || editorRegion !== (editorExistingConnection.region ?? "").trim() || JSON.stringify(editorModels.map(editorDeclarationKey)) !== JSON.stringify(editorExistingConnection.modelEntries.map(savedDeclarationKey)) @@ -1277,7 +1285,7 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { showPassword={showEditorApiKey} onTogglePassword={() => setShowEditorApiKey((show) => !show)} /> - {editorCredentials?.requiresApiKey && ( + {editorCredentials?.requiresApiKey && !editorHasIndependentCredentials && (

{t("apiKey.credentialsChanged")}

)} {editorDetailsChanged && editorHasUsableKey && editorTest.status === "idle" && ( diff --git a/App/frontend/desktop/src/pages/tests/model-workspace-section.interaction.test.tsx b/App/frontend/desktop/src/pages/tests/model-workspace-section.interaction.test.tsx index 1895bd619..798caa0b2 100644 --- a/App/frontend/desktop/src/pages/tests/model-workspace-section.interaction.test.tsx +++ b/App/frontend/desktop/src/pages/tests/model-workspace-section.interaction.test.tsx @@ -178,6 +178,83 @@ describe("ModelWorkspaceSection BYOK connection deletion", () => { expect(alert.className).toContain("mb-5"); }); + it("tests and saves an existing independent no-auth connection without borrowing the Provider mask", async () => { + const seedConfig = createSeedConfig(1), endpoint = seedConfig.catalog!.providers[0]!.endpoints[0]!; + endpoint.inheritProviderDefaults = false; + endpoint.hasApiKey = false; endpoint.apiKeyMasked = ""; endpoint.apiKey = ""; + const configClient = { + getModelConfig: vi.fn(async () => seedConfig), + saveModelCatalog: vi.fn(async (_input: unknown) => seedConfig), + testModelConfig: vi.fn(async (_input: unknown) => ({ ok: true, message: "ok", checkedAt: "2026-08-13T00:00:00.000Z" })) + }; + await act(async () => { + root.render(); + await Promise.resolve(); + }); + act(() => container.querySelector('button[aria-label="编辑 openai 配置"]')!.click()); + const save = container.querySelector('button[aria-label="保存"]')!; + expect(save.disabled).toBe(false); + expect(container.querySelector('input[type="password"]')?.value).toBe(""); + const test = [...container.querySelectorAll("button")].find(button => button.textContent?.trim() === "测试")!; + await act(async () => { test.click(); await Promise.resolve(); }); + expect(configClient.testModelConfig).toHaveBeenCalledWith(expect.objectContaining({ apiKey: "", apiKeyMasked: "", endpointId: endpoint.endpointId }), "chat", expect.anything()); + await act(async () => { save.click(); await Promise.resolve(); }); + await vi.waitFor(() => expect(configClient.saveModelCatalog).toHaveBeenCalledTimes(1)); + const input = configClient.saveModelCatalog.mock.calls[0]![0] as { providers: Array<{ endpoints: Array<{ inheritProviderDefaults?: boolean; apiKey?: string }> }> }; + expect(input.providers[0]!.endpoints[0]).toMatchObject({ inheritProviderDefaults: false }); + expect(input.providers[0]!.endpoints[0]!.apiKey).toBeUndefined(); + }); + + it.each(["endpoint", "protocol"] as const)( + "does not reuse independent no-auth credentials after changing the %s", + async (changedField) => { + const seedConfig = createCustomSeedConfig(); + const endpoint = seedConfig.catalog!.providers[0]!.endpoints[0]!; + endpoint.inheritProviderDefaults = false; + endpoint.hasApiKey = false; + endpoint.apiKeyMasked = ""; + endpoint.apiKey = ""; + const configClient = { + getModelConfig: vi.fn(async () => seedConfig), + saveModelCatalog: vi.fn(async (_input: unknown) => seedConfig), + testModelConfig: vi.fn(async (_input: unknown) => ({ + ok: true, message: "ok", checkedAt: "2026-08-13T00:00:00.000Z" + })) + }; + await act(async () => { + root.render(); + await Promise.resolve(); + }); + act(() => container.querySelector('button[aria-label="编辑 custom 配置"]')!.click()); + const dialog = container.querySelector('[role="dialog"]')!; + const save = () => dialog.querySelector('button[aria-label="保存"]')!; + expect(save().disabled).toBe(false); + expect(dialog.querySelector('[role="alert"]')).toBeNull(); + if (changedField === "endpoint") { + const input = [...dialog.querySelectorAll("input")] + .find((element) => element.value === endpoint.apiBase)!; + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(input, "https://changed.example/v1"); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + } else { + const selector = [...dialog.querySelectorAll('[role="combobox"]')] + .find((button) => button.textContent?.includes("OpenAI Responses"))!; + act(() => selector.click()); + const option = [...document.querySelectorAll('[role="option"]')] + .find((button) => button.textContent?.includes("Anthropic Messages"))!; + act(() => option.click()); + } + expect(save().disabled).toBe(true); + const test = [...dialog.querySelectorAll("button")] + .find((button) => button.textContent?.trim() === "测试")!; + await act(async () => { test.click(); await Promise.resolve(); }); + expect(configClient.testModelConfig).not.toHaveBeenCalled(); + act(() => save().click()); + expect(configClient.saveModelCatalog).not.toHaveBeenCalled(); + } + ); + it("inherits the embedding connection type when adding bge-m3", async () => { const seedConfig = createEmbeddingSeedConfig(); const configClient = { diff --git a/App/frontend/desktop/src/state/model-workspace.ts b/App/frontend/desktop/src/state/model-workspace.ts index 41a3e7dd5..eb528c4a7 100644 --- a/App/frontend/desktop/src/state/model-workspace.ts +++ b/App/frontend/desktop/src/state/model-workspace.ts @@ -37,6 +37,7 @@ export interface ModelConnection { protocol: ModelEndpointProtocol; region: string; apiKeyMasked: string; + inheritProviderDefaults?: boolean; models: string[]; modelEntries: ModelConnectionModel[]; modelCapabilities: Record; @@ -86,6 +87,7 @@ export interface ModelConnectionInput { endpointId?: string; endpoint: string; protocol?: ModelEndpointProtocol; + inheritProviderDefaults?: boolean; region?: string; apiKey?: string; apiKeyMasked?: string; @@ -112,6 +114,7 @@ export interface ByokPresetInput { endpointId?: string; endpoint: string; protocol: ModelEndpointProtocol; + inheritProviderDefaults?: boolean; apiKey?: string; apiKeyMasked?: string; model: string; @@ -258,6 +261,7 @@ export function upsertByokPreset( endpointId: newId("endpoint"), apiBase, protocol: input.protocol, + ...(input.inheritProviderDefaults !== undefined ? { inheritProviderDefaults: input.inheritProviderDefaults } : {}), hasApiKey: Boolean(input.apiKey), apiKeyMasked: input.apiKey ? maskApiKey(input.apiKey) : input.apiKeyMasked ?? "", apiKey: input.apiKey ?? "" @@ -268,6 +272,7 @@ export function upsertByokPreset( endpoint.apiKeyMasked = maskApiKey(input.apiKey); endpoint.hasApiKey = true; } + if (input.inheritProviderDefaults !== undefined) endpoint.inheritProviderDefaults = input.inheritProviderDefaults; const existing = provider.models.find((item) => ( item.source === "byok" && item.endpointId === endpoint!.endpointId @@ -473,10 +478,13 @@ export function upsertModelConnection( next.providers.push(provider); } + const inheritProviderDefaults = input.inheritProviderDefaults + ?? (existingConnection?.provider === providerId ? existingConnection.inheritProviderDefaults : undefined); provider.endpoints.push({ endpointId, apiBase: endpoint, protocol, + ...(inheritProviderDefaults !== undefined ? { inheritProviderDefaults } : {}), hasApiKey: Boolean(input.apiKey || input.apiKeyMasked), apiKeyMasked: input.apiKey ? maskApiKey(input.apiKey) : input.apiKeyMasked ?? "", apiKey: input.apiKey ?? "", @@ -689,7 +697,8 @@ function createSpace(catalog: ModelConfigView, mode: ModelWorkspaceMode): ModelW endpoint: endpoint.apiBase, protocol: endpoint.protocol, region: endpoint.region ?? "", - apiKeyMasked: endpoint.apiKeyMasked || provider.apiKeyMasked, + apiKeyMasked: endpoint.apiKeyMasked || (endpoint.inheritProviderDefaults === false ? "" : provider.apiKeyMasked), + ...(endpoint.inheritProviderDefaults !== undefined ? { inheritProviderDefaults: endpoint.inheritProviderDefaults } : {}), models: models.map((model) => model.model), modelEntries: models.map((model) => ({ presetId: model.presetId, @@ -742,6 +751,7 @@ function endpointInput(endpoint: ModelConfigView["providers"][number]["endpoints endpointId: endpoint.endpointId, apiBase: endpoint.apiBase, protocol: endpoint.protocol, + ...(endpoint.inheritProviderDefaults !== undefined ? { inheritProviderDefaults: endpoint.inheritProviderDefaults } : {}), ...(endpoint.apiKey ? { apiKey: endpoint.apiKey } : {}), ...(endpoint.region ? { region: endpoint.region } : {}) }; diff --git a/App/frontend/desktop/src/state/tests/model-workspace.test.ts b/App/frontend/desktop/src/state/tests/model-workspace.test.ts index 2ca7ab61b..d7b14bc72 100644 --- a/App/frontend/desktop/src/state/tests/model-workspace.test.ts +++ b/App/frontend/desktop/src/state/tests/model-workspace.test.ts @@ -694,6 +694,79 @@ describe("canonical model workspace adapter", () => { expect((YAML.parse(readFileSync(file, "utf8")) as any).agents.defaults.concurrentMarker).toBe("keep-me"); }); + it("真实 Backend catalog：独立无 Key 连接编辑保存保留 false，切换本地与清空不影响云端模型", async () => { + const file = catalogFixture(), current = await readModelConfigCatalog(file); + const created = await writeModelConfigCatalog(file, { + configRevision: current.configRevision, + providers: [{ provider: "openai", apiKey: "sk-cloud", extraHeaders: { Authorization: "Bearer cloud", "x-cloud": "cloud" }, extraBody: { cloudDefault: true }, + endpoints: [ + { endpointId: "chat", apiBase: "https://api.openai.com/v1", protocol: "openai-chat-completions" }, + { endpointId: "memory-local", apiBase: "http://127.0.0.1:18789/v1", protocol: "openai-embeddings", inheritProviderDefaults: false, extraHeaders: { "x-local": "local" } } + ], models: [ + { endpointId: "chat", model: "gpt-cloud", source: "byok", capabilities: ["agent"] }, + { endpointId: "memory-local", model: "embed-local", source: "byok", capabilities: ["embedding"] } + ] }], modelAssignments: structuredClone(emptyAssignments) + }); + const cloud = created.providers[0]!.models.find(model => model.model === "gpt-cloud")!.presetId; + const independent = created.providers[0]!.models.find(model => model.model === "embed-local")!.presetId; + let workspace = createModelWorkspace(created); + workspace = assignCatalogPreset(workspace, "byok", "agent", cloud); + workspace = setModelAssignment(workspace, "byok", "embedding", independent); + const assigned = await writeModelConfigCatalog(file, modelConfigInput(workspace)); + workspace = createModelWorkspace(assigned); + const connection = workspace.spaces.byok.connections.find(connection => connection.endpointId === "memory-local")!; + expect(connection).toMatchObject({ inheritProviderDefaults: false, apiKeyMasked: "", available: true }); + expect(modelConfigInput(workspace).providers[0]!.endpoints.find(endpoint => endpoint.endpointId === "memory-local")) + .toMatchObject({ inheritProviderDefaults: false }); + const cloudBefore = (YAML.parse(readFileSync(file, "utf8")) as any).providers.openai; + const edited = upsertModelConnection(workspace, "byok", { + id: connection.id, provider: "openai", endpoint: connection.endpoint, protocol: connection.protocol, + apiKeyMasked: connection.apiKeyMasked, models: ["embed-local-edited"], + modelEntries: [{ presetId: independent, model: "embed-local-edited", capability: "embedding" }] + }); + expect(edited.error).toBeNull(); + const saved = await persistModelCatalogMutation(modelConfigInput(edited.workspace), { + read: () => readModelConfigCatalog(file), write: input => writeModelConfigCatalog(file, input) + }, assigned); + let raw = YAML.parse(readFileSync(file, "utf8")) as any; + expect(raw.providers.openai.endpoints["memory-local"]).toMatchObject({ inheritProviderDefaults: false, extraHeaders: { "x-local": "local" } }); + expect(raw.memmyMemory.embedding).toMatchObject({ mode: "custom", model: "embed-local-edited", extraHeaders: { "x-local": "local" } }); + expect(raw.memmyMemory.embedding).not.toHaveProperty("apiKey"); + expect(raw.memmyMemory.embedding.extraHeaders).not.toHaveProperty("Authorization"); + expect(raw.memmyMemory.embedding).not.toHaveProperty("extraBody"); + expect(raw.providers.openai.apiKey).toBe("sk-cloud"); + expect(raw.providers.openai.endpoints.chat).toEqual(cloudBefore.endpoints.chat); + for (const candidate of [BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID, null]) { + const view = await readModelConfigCatalog(file); + const local = setModelAssignment(createModelWorkspace(view), "byok", "embedding", candidate); + await writeModelConfigCatalog(file, modelConfigInput(local)); + raw = YAML.parse(readFileSync(file, "utf8")) as any; + expect(raw.memmyMemory.embedding).toMatchObject({ mode: "local", provider: "local" }); + expect(raw.memmyMemory.embedding).not.toHaveProperty("endpoint"); + expect(raw.providers.openai.endpoints.chat).toEqual(cloudBefore.endpoints.chat); + expect(raw.modelAssignments.byok.agent.default).toBe(cloud); + } + expect(saved.providers[0]!.endpoints.find(endpoint => endpoint.endpointId === "memory-local")) + .toMatchObject({ inheritProviderDefaults: false, hasApiKey: false, apiKeyMasked: "" }); + }); + + it("独立连接标记在有并发配置更新的保存重试中保留", async () => { + const file = catalogFixture(), base = await deletionCatalogFixture(file); + const desired = modelConfigInput(createModelWorkspace(base)); + desired.providers[0]!.endpoints.find(endpoint => endpoint.endpointId === "embedding")!.inheritProviderDefaults = false; + const concurrent = modelConfigInput(createModelWorkspace(base)); + concurrent.providers[0]!.apiKey = "provider-rotated"; + await writeModelConfigCatalog(file, concurrent); + const saved = await persistModelCatalogMutation(desired, { + read: () => readModelConfigCatalog(file), write: input => writeModelConfigCatalog(file, input) + }, base); + expect(saved.providers[0]!.endpoints.find(endpoint => endpoint.endpointId === "embedding")) + .toHaveProperty("inheritProviderDefaults", false); + const raw = YAML.parse(readFileSync(file, "utf8")) as any; + expect(raw.providers.openai.apiKey).toBe("provider-rotated"); + expect(raw.providers.openai.endpoints.chat.inheritProviderDefaults).toBeUndefined(); + }); + it("真实 Backend catalog：Provider 切换遇到目标同名 endpointId 时生成新 ID 并保留目标定义", async () => { const file = catalogFixture(); const empty = await readModelConfigCatalog(file); diff --git a/App/memmy-agent/src/config/schema.ts b/App/memmy-agent/src/config/schema.ts index 79cd7ccab..a86b20afe 100644 --- a/App/memmy-agent/src/config/schema.ts +++ b/App/memmy-agent/src/config/schema.ts @@ -617,6 +617,7 @@ export class ProviderConfig extends Base { export class ModelEndpointConfig extends Base { apiBase: string; protocol: ModelEndpointProtocol; + inheritProviderDefaults?: boolean; apiKey: string | null; extraHeaders: Dict | null; extraBody: Dict | null; @@ -632,6 +633,8 @@ export class ModelEndpointConfig extends Base { super(init); this.apiBase = assertRequiredString("provider endpoint apiBase", init.apiBase); this.protocol = assertOneOf("provider endpoint protocol", init.protocol, ENDPOINT_PROTOCOLS); + this.inheritProviderDefaults = init.inheritProviderDefaults === undefined ? undefined + : assertBoolean("provider endpoint inheritProviderDefaults", init.inheritProviderDefaults); this.apiKey = pick(init, ["apiKey"], null); this.extraHeaders = pick(init, ["extraHeaders"], null); this.extraBody = pick(init, ["extraBody"], null); @@ -642,6 +645,7 @@ export class ModelEndpointConfig extends Base { override toObject(): Dict { return omitUndefined({ ...this, + inheritProviderDefaults: this.inheritProviderDefaults, apiKey: this.apiKey ?? undefined, extraHeaders: this.extraHeaders ?? undefined, extraBody: this.extraBody ?? undefined, @@ -1452,7 +1456,10 @@ export class Config extends Base { model: string | null = null, opts: { preset?: ModelPresetConfig | null } = {}, ): string | null { - return this.getProvider(model, opts)?.apiKey ?? null; + const preset = opts.preset ?? this.resolvePreset(); + const provider = this.getProvider(model, { preset }); + const endpoint = provider?.endpoints[preset.endpoint]; + return endpoint?.inheritProviderDefaults === false ? endpoint.apiKey || null : provider?.apiKey ?? null; } getApiBase( diff --git a/App/memmy-agent/src/providers/anthropic-provider.ts b/App/memmy-agent/src/providers/anthropic-provider.ts index d8fec524d..210a18c59 100644 --- a/App/memmy-agent/src/providers/anthropic-provider.ts +++ b/App/memmy-agent/src/providers/anthropic-provider.ts @@ -31,6 +31,7 @@ export class AnthropicProvider extends LLMProvider { extraHeaders: Record | null = null; extraBody: Record | null = null; client: any = null; + private readonly inheritProviderDefaults: boolean; constructor( init: @@ -42,6 +43,7 @@ export class AnthropicProvider extends LLMProvider { spec?: any; extraHeaders?: Record | null; extraBody?: Record | null; + inheritProviderDefaults?: boolean; } | null = null, apiBase: string | null = null, @@ -53,9 +55,11 @@ export class AnthropicProvider extends LLMProvider { this.spec = init.spec ?? null; this.extraHeaders = init.extraHeaders ?? null; this.extraBody = init.extraBody ?? null; + this.inheritProviderDefaults = init.inheritProviderDefaults !== false; } else { super(init, apiBase); this.model = defaultModel; + this.inheritProviderDefaults = true; } this.defaultModel = this.model ?? "claude-sonnet-4-20250514"; this.client = this.buildClient(); @@ -63,11 +67,14 @@ export class AnthropicProvider extends LLMProvider { buildClient(): any { const clientOptions: Record = { - apiKey: this.apiKey ?? process.env.ANTHROPIC_API_KEY ?? null, + apiKey: this.inheritProviderDefaults ? this.apiKey ?? process.env.ANTHROPIC_API_KEY ?? null : this.apiKey, + ...(!this.inheritProviderDefaults ? { authToken: null } : {}), maxRetries: 0, }; if (this.apiBase) clientOptions.baseURL = this.apiBase; - if (this.extraHeaders) clientOptions.defaultHeaders = this.extraHeaders; + if (!this.inheritProviderDefaults && !this.apiKey) { + clientOptions.defaultHeaders = { "x-api-key": null, authorization: null, ...(this.extraHeaders ?? {}) }; + } else if (this.extraHeaders) clientOptions.defaultHeaders = this.extraHeaders; return new Anthropic(clientOptions); } @@ -465,7 +472,7 @@ export class AnthropicProvider extends LLMProvider { signal?: AbortSignal | null; }): Promise { const apiKey = this.apiKey ?? process.env.ANTHROPIC_API_KEY ?? null; - if (!apiKey) return new LLMResponse({ content: "Error calling LLM: missing Anthropic API key", finishReason: "error" }); + if (this.inheritProviderDefaults && !apiKey) return new LLMResponse({ content: "Error calling LLM: missing Anthropic API key", finishReason: "error" }); const kwargs = this.buildKwargs(args); try { if (args.signal?.aborted) throw createProviderAbortError(); @@ -487,7 +494,7 @@ export class AnthropicProvider extends LLMProvider { onToolCallDelta?: (delta: Record) => Promise | void; }): Promise { const apiKey = this.apiKey ?? process.env.ANTHROPIC_API_KEY ?? null; - if (!apiKey) return new LLMResponse({ content: "Error calling LLM: missing Anthropic API key", finishReason: "error" }); + if (this.inheritProviderDefaults && !apiKey) return new LLMResponse({ content: "Error calling LLM: missing Anthropic API key", finishReason: "error" }); const kwargs = { ...this.buildKwargs(args), stream: true }; const onContentDelta = args.onContentDelta; const onThinkingDelta = args.onThinkingDelta; diff --git a/App/memmy-agent/src/providers/factory.ts b/App/memmy-agent/src/providers/factory.ts index 57c89b7d6..3ad263f9d 100644 --- a/App/memmy-agent/src/providers/factory.ts +++ b/App/memmy-agent/src/providers/factory.ts @@ -52,12 +52,11 @@ function providerInit( ): any { const endpoint = exactEndpoint(config, providerConfig, preset); const protocol = endpoint?.protocol; + const effective = endpointOptions(providerConfig, endpoint); return { - apiKey: endpoint?.apiKey ?? providerConfig?.apiKey ?? null, + ...effective, apiBase: endpoint?.apiBase ?? config.getApiBase(model, { preset }), defaultModel: model, - extraHeaders: mergedRecord(providerConfig?.extraHeaders, endpoint?.extraHeaders), - extraBody: mergedRecord(providerConfig?.extraBody, endpoint?.extraBody), apiType: protocol === "openai-responses" ? "responses" : protocol === "openai-chat-completions" ? "chatCompletions" : "auto", @@ -72,6 +71,16 @@ function mergedRecord( return { ...(defaults ?? {}), ...(overrides ?? {}) }; } +function endpointOptions(provider: ProviderConfig | null, endpoint: ProviderConfig["endpoints"][string] | null) { + const independent = endpoint?.inheritProviderDefaults === false; + return { + apiKey: independent ? endpoint.apiKey || null : endpoint?.apiKey ?? provider?.apiKey ?? null, + extraHeaders: mergedRecord(independent ? null : provider?.extraHeaders, endpoint?.extraHeaders), + extraBody: mergedRecord(independent ? null : provider?.extraBody, endpoint?.extraBody), + inheritProviderDefaults: !independent, + }; +} + function backendForCustomProtocol(protocol: string | undefined): ProviderSpec["backend"] { if (protocol === "anthropic-messages") return "anthropic"; if (protocol === "bedrock-converse") return "bedrock"; @@ -116,7 +125,7 @@ function makeProviderCore( throw new ValueError("Azure OpenAI requires apiKey and apiBase in config."); } } else if (opts.validateCredentials !== false && backend === "openai_compat" && !model.startsWith("bedrock/")) { - const exempt = Boolean(spec?.isOauth || spec?.isLocal || spec?.isDirect); + const exempt = Boolean(spec?.isOauth || spec?.isLocal || spec?.isDirect || init.inheritProviderDefaults === false); if (!init.apiKey && !exempt) { throw new ValueError(`No API key configured for provider '${providerName}'.`); } @@ -200,14 +209,15 @@ export function providerSignature( const fallbackSignature = (fallback: ModelPresetConfig): any[] => { const fallbackProvider = config.getProvider(fallback.model, { preset: fallback }); const fallbackEndpoint = exactEndpoint(config, fallbackProvider, fallback); + const effective = endpointOptions(fallbackProvider, fallbackEndpoint); return [ fallback.model, fallback.provider, config.getProviderName(fallback.model, { preset: fallback }), - fallbackEndpoint?.apiKey ?? config.getApiKey(fallback.model, { preset: fallback }), + effective.apiKey, fallbackEndpoint?.apiBase ?? config.getApiBase(fallback.model, { preset: fallback }), - mergedRecord(fallbackProvider?.extraHeaders, fallbackEndpoint?.extraHeaders), - mergedRecord(fallbackProvider?.extraBody, fallbackEndpoint?.extraBody), + effective.extraHeaders, + effective.extraBody, fallbackEndpoint?.protocol ?? fallbackProvider?.apiType ?? "auto", (fallbackProvider as any)?.region ?? null, (fallbackProvider as any)?.profile ?? null, @@ -215,18 +225,20 @@ export function providerSignature( fallback.temperature, fallback.reasoningEffort, fallback.contextWindowTokens, + ...(!effective.inheritProviderDefaults ? [false] : []), ]; }; const endpoint = exactEndpoint(config, providerConfig, resolved); + const effective = endpointOptions(providerConfig, endpoint); return [ resolved.model, resolved.provider, config.getProviderName(resolved.model, { preset: resolved }), - endpoint?.apiKey ?? config.getApiKey(resolved.model, { preset: resolved }), + effective.apiKey, endpoint?.apiBase ?? config.getApiBase(resolved.model, { preset: resolved }), - mergedRecord(providerConfig?.extraHeaders, endpoint?.extraHeaders), - mergedRecord(providerConfig?.extraBody, endpoint?.extraBody), + effective.extraHeaders, + effective.extraBody, endpoint?.protocol ?? providerConfig?.apiType ?? "auto", (providerConfig as any)?.region ?? null, (providerConfig as any)?.profile ?? null, @@ -234,6 +246,7 @@ export function providerSignature( resolved.temperature, resolved.reasoningEffort, resolved.contextWindowTokens, + ...(!effective.inheritProviderDefaults ? [false] : []), fallbackPresets.map(fallbackSignature), ]; } diff --git a/App/memmy-agent/src/providers/model-catalog.ts b/App/memmy-agent/src/providers/model-catalog.ts index 61f52020e..ba4e53a30 100644 --- a/App/memmy-agent/src/providers/model-catalog.ts +++ b/App/memmy-agent/src/providers/model-catalog.ts @@ -246,6 +246,7 @@ function runtimeCatalog(config: Config): RuntimeModelCatalog { omitNullish({ apiBase: endpoint.apiBase, protocol: endpoint.protocol, + inheritProviderDefaults: endpoint.inheritProviderDefaults, apiKey: endpoint.apiKey, extraHeaders: endpoint.extraHeaders, extraBody: endpoint.extraBody, diff --git a/App/memmy-agent/src/providers/openai-compat-provider.ts b/App/memmy-agent/src/providers/openai-compat-provider.ts index 6ca9c49fe..7f1869e94 100644 --- a/App/memmy-agent/src/providers/openai-compat-provider.ts +++ b/App/memmy-agent/src/providers/openai-compat-provider.ts @@ -102,7 +102,9 @@ export class OpenAICompatProvider extends LLMProvider { apiType: OpenAICompatApiType = "auto"; effectiveBase: string | null = null; defaultHeaders: Record; - apiKeyForClient: string; + apiKeyForClient: string | null; + private readonly explicitNoAuth: boolean; + private readonly independent: boolean; isLocal: boolean; responsesFailures: Record = {}; responsesTrippedAt: Record = {}; @@ -118,6 +120,7 @@ export class OpenAICompatProvider extends LLMProvider { extraHeaders?: Record | null; extraBody?: Record | null; apiType?: OpenAICompatApiType; + inheritProviderDefaults?: boolean; } | null = null, apiBase: string | null = null, @@ -131,12 +134,16 @@ export class OpenAICompatProvider extends LLMProvider { this.extraHeaders = apiKeyOrInit.extraHeaders ?? {}; this.extraBody = apiKeyOrInit.extraBody ?? {}; this.apiType = normalizeApiType(apiKeyOrInit.apiType); + this.independent = apiKeyOrInit.inheritProviderDefaults === false; + this.explicitNoAuth = this.independent && !this.apiKey; } else { super(apiKeyOrInit, apiBase); this.model = defaultModel; this.spec = spec; this.extraHeaders = {}; this.extraBody = {}; + this.explicitNoAuth = false; + this.independent = false; } this.defaultModel = this.model ?? "gpt-4o"; @@ -144,7 +151,7 @@ export class OpenAICompatProvider extends LLMProvider { if (this.apiType !== "auto" && name !== "openai" && name !== "custom") { this.apiType = "auto"; } - if (this.apiKey && this.spec?.envKey) this.setupEnv(this.apiKey, this.apiBase); + if (this.apiKey && this.spec?.envKey && !this.independent) this.setupEnv(this.apiKey, this.apiBase); const effectiveBase = this.apiBase || resolveDefaultApiBase(this.spec); this.effectiveBase = effectiveBase; @@ -154,7 +161,7 @@ export class OpenAICompatProvider extends LLMProvider { Object.assign(this.defaultHeaders, OPENROUTER_ATTRIBUTION_HEADERS); } Object.assign(this.defaultHeaders, this.extraHeaders); - this.apiKeyForClient = + this.apiKeyForClient = this.explicitNoAuth ? null : this.apiKey ?? (specName(this.spec) === "openai" ? process.env.OPENAI_API_KEY : null) ?? "no-key"; @@ -178,15 +185,23 @@ export class OpenAICompatProvider extends LLMProvider { buildClient(): void { const timeoutS = openaiCompatTimeoutS(); const fetchOptions = this.isLocal ? ({ keepalive: false } as any) : undefined; + const defaultHeaders = this.explicitNoAuth + ? { authorization: null, "api-key": null, ...this.defaultHeaders } + : this.defaultHeaders; this.client = new OpenAI({ - apiKey: this.apiKeyForClient, + // The SDK requires a non-empty constructor credential. Keep its existing + // initialization placeholder internal; the public null headers below and + // cleared SDK key ensure it is never sent for an independent no-auth URL. + apiKey: this.apiKeyForClient ?? "no-key", + ...(this.independent ? { adminAPIKey: null, organization: null, project: null } : {}), baseURL: this.effectiveBase ?? undefined, - defaultHeaders: this.defaultHeaders, + defaultHeaders, maxRetries: 0, timeout: timeoutS * 1000, fetchOptions, }); - this.client.defaultHeaders = this.defaultHeaders; + if (this.explicitNoAuth) this.client.apiKey = null; + this.client.defaultHeaders = defaultHeaders; } static extractProviderErrorFacts( diff --git a/App/memmy-agent/tests/providers/endpoint-provider-defaults.test.ts b/App/memmy-agent/tests/providers/endpoint-provider-defaults.test.ts new file mode 100644 index 000000000..803805ae4 --- /dev/null +++ b/App/memmy-agent/tests/providers/endpoint-provider-defaults.test.ts @@ -0,0 +1,193 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Config, ModelEndpointConfig } from "../../src/config/schema.js"; +import { buildProviderSnapshot, makeProvider, providerSignature } from "../../src/providers/factory.js"; +import { resolveModelSelection } from "../../src/providers/model-catalog.js"; +import { OpenAICompatProvider } from "../../src/providers/openai-compat-provider.js"; +import { makeReloadingProviderSnapshotLoader } from "../../src/providers/snapshot-loader.js"; +const roots: string[] = []; +function data(inherit: boolean | undefined, ownKey?: string): any { + return { app: { userMode: "byok" }, agents: { defaults: { modelPreset: "independent" } }, + providers: { openai: { apiKey: "cloud-fixture-token", extraHeaders: { Authorization: "Bearer cloud-header", "X-Cloud": "unrelated" }, + extraBody: { model: "wrong-cloud-model", cloudOption: true }, endpoints: { + independent: { apiBase: "https://local-fixture.example/v1", protocol: "openai-chat-completions", + ...(inherit === undefined ? {} : { inheritProviderDefaults: inherit }), ...(ownKey === undefined ? {} : { apiKey: ownKey }), + extraHeaders: { "X-Own": "preserved" }, extraBody: { ownOption: true } }, + inherited: { apiBase: "https://cloud-fixture.example/v1", protocol: "openai-chat-completions" } } } }, + modelPresets: { + independent: { provider: "openai", endpoint: "independent", model: "local-fixture-model", source: "byok", capabilities: ["agent", "memory_summary"] }, + inherited: { provider: "openai", endpoint: "inherited", model: "cloud-fixture-model", source: "byok", capabilities: ["agent"] } }, + modelAssignments: { byok: { agent: { candidates: ["independent", "inherited"], default: "independent" }, memorySummary: "independent" } } }; +} +function writeConfig(value: unknown) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "memmy-endpoint-defaults-")); roots.push(root); + const file = path.join(root, "config.yaml"); fs.writeFileSync(file, YAML.stringify(value)); return file; +} +function anthropicData(key?: string) { + const input = data(false, key), provider = input.providers.openai; + provider.endpoints.independent.protocol = "anthropic-messages"; provider.endpoints.inherited.protocol = "anthropic-messages"; + input.providers = { anthropic: provider }; + input.modelPresets.independent.provider = "anthropic"; input.modelPresets.inherited.provider = "anthropic"; + return input; +} +function mockHttp() { + const requests: Array<{ url: string; headers: Headers; body: Record }> = []; + vi.stubGlobal("fetch", vi.fn(async (input: Request | URL | string, init?: RequestInit) => { + const url = String(input); requests.push({ url, headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) }); + if (JSON.parse(String(init?.body)).stream) { + if (url.includes("/messages")) { + const events = [ + { type: "message_start", message: { id: "msg-test", role: "assistant", model: "local-fixture-model", content: [], usage: { input_tokens: 1, output_tokens: 0 } } }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "OK" } }, + { type: "content_block_stop", index: 0 }, + { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }, { type: "message_stop" }, + ]; + return new Response(events.map(event => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), + { status: 200, headers: { "content-type": "text/event-stream" } }); + } + return new Response('data: {"choices":[{"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + { status: 200, headers: { "content-type": "text/event-stream" } }); + } + const body = url.includes("/messages") ? { id: "msg-test", type: "message", role: "assistant", model: "local-fixture-model", content: [{ type: "text", text: "OK" }], stop_reason: "end_turn", usage: { input_tokens: 1, output_tokens: 1 } } + : url.endsWith("/responses") ? { id: "resp-test", output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "OK" }] }], status: "completed", usage: { input_tokens: 1, output_tokens: 1 } } + : { id: "chat-test", object: "chat.completion", choices: [{ message: { role: "assistant", content: "OK" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }; + return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } }); + })); return requests; +} +afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); +describe("endpoint inheritance and actual Agent provider requests", () => { + it.each([false, true, undefined])("retains optional boolean %s through config serialization", inherit => { + const config = new Config(data(inherit)), serialized = config.toObject(), endpoint = (serialized.providers as any).openai.endpoints.independent; + expect(endpoint.inheritProviderDefaults).toBe(inherit); + expect(new Config(serialized).providers.openai.endpoints.independent.toObject()).toEqual(endpoint); + }); + it.each([null, "false", 0, {}])("rejects non-boolean endpoint inheritance %s", value => { + expect(() => new ModelEndpointConfig({ apiBase: "https://fixture.example/v1", protocol: "openai-chat-completions", inheritProviderDefaults: value })).toThrow(/boolean/); + }); + it.each(["own-fixture-token", undefined, ""])("sends only independent credentials/options (own key %s)", async key => { + vi.stubEnv("OPENAI_API_KEY", "environment-fixture-token"); vi.stubEnv("OPENAI_ADMIN_KEY", "environment-admin-token"); + vi.stubEnv("OPENAI_ORG_ID", "environment-org"); vi.stubEnv("OPENAI_PROJECT_ID", "environment-project"); + const requests = mockHttp(), config = new Config(data(false, key)), provider = makeProvider(config) as OpenAICompatProvider; + expect(provider.apiKey ?? null).toBe(key || null); + expect(config.getApiKey(null, { preset: config.modelPresets.independent })).toBe(key || null); + expect(provider.extraHeaders).toEqual({ "X-Own": "preserved" }); expect(provider.extraBody).toEqual({ ownOption: true }); + expect((await provider.chat({ messages: [{ role: "user", content: "synthetic" }] })).finishReason).toBe("stop"); + expect(requests).toHaveLength(1); const request = requests[0]!; + expect(request.headers.get("authorization")).toBe(key ? `Bearer ${key}` : null); + expect(request.headers.get("api-key")).toBeNull(); expect(request.headers.get("X-Cloud")).toBeNull(); + expect(request.headers.get("openai-organization")).toBeNull(); expect(request.headers.get("openai-project")).toBeNull(); + expect(request.headers.get("X-Own")).toBe("preserved"); expect(request.body).toMatchObject({ model: "local-fixture-model", ownOption: true }); + expect(request.body).not.toHaveProperty("cloudOption"); + if (!key) { expect(provider.apiKeyForClient).toBeNull(); expect(provider.client.apiKey).toBeNull(); } + }); + it("keeps endpoint Authorization when independent key authentication is absent", async () => { + vi.stubEnv("OPENAI_ORG_ID", "environment-org"); vi.stubEnv("OPENAI_PROJECT_ID", "environment-project"); + const input = data(false); input.providers.openai.endpoints.independent.extraHeaders.Authorization = "Bearer own-header"; + input.providers.openai.endpoints.independent.extraHeaders["OpenAI-Organization"] = "own-org"; + input.providers.openai.endpoints.independent.extraHeaders["OpenAI-Project"] = "own-project"; + const requests = mockHttp(); await makeProvider(new Config(input)).chat({ messages: [{ role: "user", content: "synthetic" }] }); + expect(requests[0]!.headers.get("authorization")).toBe("Bearer own-header"); expect(requests[0]!.headers.get("X-Cloud")).toBeNull(); + expect(requests[0]!.headers.get("openai-organization")).toBe("own-org"); expect(requests[0]!.headers.get("openai-project")).toBe("own-project"); + }); + it("does not export an independent endpoint key into other providers' environment fallback", () => { + vi.stubEnv("OPENAI_API_KEY", undefined); + expect(makeProvider(new Config(data(false, "own-fixture-token"))).apiKey).toBe("own-fixture-token"); + expect(process.env.OPENAI_API_KEY).toBeUndefined(); + }); + it("keeps both Responses and streaming Chat requests unauthenticated", async () => { + vi.stubEnv("OPENAI_API_KEY", "environment-fixture-token"); const requests = mockHttp(); + const responses = data(false); responses.providers.openai.endpoints.independent.protocol = "openai-responses"; + expect((await makeProvider(new Config(responses)).chat({ messages: [{ role: "user", content: "synthetic" }] })).content).toBe("OK"); + expect((await makeProvider(new Config(data(false))).chatStream({ messages: [{ role: "user", content: "synthetic" }] })).content).toBe("OK"); + expect(requests.map(request => request.url)).toEqual(["https://local-fixture.example/v1/responses", "https://local-fixture.example/v1/chat/completions"]); + for (const request of requests) expect(request.headers.get("authorization")).toBeNull(); + }); + it("forwards caller cancellation without retrying independent no-auth requests", async () => { + let entered!: () => void; const started = new Promise(resolve => { entered = resolve; }); + const request = vi.fn(async (_input: unknown, init: RequestInit) => { + expect(new Headers(init.headers).get("authorization")).toBeNull(); entered(); + return new Promise((_resolve, reject) => init.signal!.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")), { once: true })); + }); vi.stubGlobal("fetch", request); + const abort = new AbortController(), running = makeProvider(new Config(data(false))).chat({ messages: [{ role: "user", content: "synthetic" }], signal: abort.signal }); + await started; abort.abort(); const result = await running; + expect(result.finishReason).toBe("error"); expect(result.content).toMatch(/abort/i); expect(request).toHaveBeenCalledTimes(1); + }); + it.each(["own-anthropic-token", undefined])("keeps Anthropic requests independent of defaults and environment (key %s)", async key => { + vi.stubEnv("ANTHROPIC_API_KEY", "environment-anthropic-token"); vi.stubEnv("ANTHROPIC_AUTH_TOKEN", "environment-auth-token"); + const input = anthropicData(key); + const requests = mockHttp(), actual = makeProvider(new Config(input)); + const response = await actual.chat({ messages: [{ role: "user", content: "synthetic" }], maxTokens: 64 }); + expect(response.finishReason, response.content ?? "no response").toBe("stop"); + expect(requests).toHaveLength(1); + expect(requests[0]!.headers.get("x-api-key")).toBe(key ?? null); + expect(requests[0]!.headers.get("authorization")).toBeNull(); + expect(requests[0]!.headers.get("X-Cloud")).toBeNull(); + expect(requests[0]!.body.model).toBe("local-fixture-model"); + expect(requests[0]!.body).not.toHaveProperty("cloudOption"); + }); + it.each(["chat", "chatStream"] as const)("executes independent Anthropic %s without any environment key, including explicit header authentication", async method => { + vi.stubEnv("ANTHROPIC_API_KEY", undefined); vi.stubEnv("ANTHROPIC_AUTH_TOKEN", undefined); + const requests = mockHttp(); + for (const ownHeader of [false, true]) { + const input = anthropicData(); + if (ownHeader) input.providers.anthropic.endpoints.independent.extraHeaders.Authorization = "Bearer own-anthropic-header"; + const result = await makeProvider(new Config(input))[method]({ messages: [{ role: "user", content: "synthetic" }], maxTokens: 64 }); + expect(result.finishReason, result.content ?? "no response").toBe("stop"); expect(result.content).toBe("OK"); + const request = requests.at(-1)!; + expect(request.headers.get("x-api-key")).toBeNull(); + expect(request.headers.get("authorization")).toBe(ownHeader ? "Bearer own-anthropic-header" : null); + expect(request.headers.get("X-Cloud")).toBeNull(); + } + expect(requests).toHaveLength(2); + }); + it.each(["chat", "chatStream"] as const)("does not send pre-cancelled independent Anthropic %s", async method => { + vi.stubEnv("ANTHROPIC_API_KEY", undefined); vi.stubEnv("ANTHROPIC_AUTH_TOKEN", undefined); + const requests = mockHttp(), abort = new AbortController(); abort.abort(); + await expect(makeProvider(new Config(anthropicData()))[method]({ messages: [{ role: "user", content: "synthetic" }], maxTokens: 64, signal: abort.signal })) + .rejects.toMatchObject({ name: "AbortError" }); + expect(requests).toHaveLength(0); + }); + it("keeps the existing Anthropic missing-key rejection when inheritance is enabled", async () => { + vi.stubEnv("ANTHROPIC_API_KEY", undefined); vi.stubEnv("ANTHROPIC_AUTH_TOKEN", undefined); + const input = anthropicData(); delete input.providers.anthropic.apiKey; + delete input.providers.anthropic.endpoints.independent.inheritProviderDefaults; + const requests = mockHttp(), provider = makeProvider(new Config(input)); + expect((await provider.chat({ messages: [{ role: "user", content: "synthetic" }], maxTokens: 64 })).content).toMatch(/missing Anthropic API key/); + expect(requests).toHaveLength(0); + }); + it.each([undefined, true])("preserves default inheritance %s and provider key rotation", inherit => { + const input = data(inherit), first = buildProviderSnapshot(new Config(input)), rotated = structuredClone(input); + rotated.providers.openai.apiKey = "rotated-cloud-token"; rotated.providers.openai.extraHeaders["X-Cloud"] = "rotated"; + const second = buildProviderSnapshot(new Config(rotated)); + expect(first.provider.apiKey).toBe("cloud-fixture-token"); expect(second.provider.apiKey).toBe("rotated-cloud-token"); + expect((second.provider as OpenAICompatProvider).extraHeaders["X-Cloud"]).toBe("rotated"); expect(second.signature).not.toEqual(first.signature); + }); + it("keeps independent signatures stable across unrelated default rotation", () => { + const first = data(false, "own-fixture-token"), second = structuredClone(first); + second.providers.openai.apiKey = "rotated-cloud-token"; second.providers.openai.extraBody.model = "rotated-model"; + expect(providerSignature(new Config(second))).toEqual(providerSignature(new Config(first))); + first.agents.defaults.fallbackModels = ["inherited"]; second.agents.defaults.fallbackModels = ["inherited"]; + expect(providerSignature(new Config(second))).not.toEqual(providerSignature(new Config(first))); + }); + it("applies independent options when constructing a fallback", () => { + const input = data(false, "own-fixture-token"); input.agents.defaults.modelPreset = "inherited"; input.agents.defaults.fallbackModels = ["independent"]; + const snapshot = buildProviderSnapshot(new Config(input)); + const fallback = (snapshot.provider as any).providerFactory(new Config(input).modelPresets.independent); + expect(fallback.apiKey).toBe("own-fixture-token"); expect(fallback.extraHeaders).toEqual({ "X-Own": "preserved" }); + expect(fallback.extraBody).toEqual({ ownOption: true }); + }); + it("preserves independent options in catalog selection and reloaded snapshots", () => { + const configPath = writeConfig(data(false)), selection = resolveModelSelection({ configPath, capability: "memory_summary" }); + expect(selection).not.toBeNull(); expect(selection!.providerConfig.apiKey).toBeUndefined(); + expect(selection!.providerConfig.extraHeaders).toEqual({ "X-Own": "preserved" }); expect(selection!.providerConfig.extraBody).toEqual({ ownOption: true }); + expect(selection!.snapshot.provider.apiKey).toBeNull(); + const loader = makeReloadingProviderSnapshotLoader({ configPath }), first = loader(); + fs.writeFileSync(configPath, YAML.stringify(data(true, "own-fixture-token"))); const second = loader(); + expect((first.provider as OpenAICompatProvider).extraBody).toEqual({ ownOption: true }); + expect((second.provider as OpenAICompatProvider).extraBody).toMatchObject({ cloudOption: true }); expect(second.signature).not.toEqual(first.signature); + }); +}); diff --git a/App/shell/desktop/src/main/computer-use-agent-restart.ts b/App/shell/desktop/src/main/computer-use-agent-restart.ts index 76e38af8a..3cedd7b48 100644 --- a/App/shell/desktop/src/main/computer-use-agent-restart.ts +++ b/App/shell/desktop/src/main/computer-use-agent-restart.ts @@ -79,10 +79,13 @@ function stopVerifiedAgent(binary: string, agent: AgentProcess): Promise { const timer = setTimeout(() => finish(new Error('Native Computer Use agent shutdown timed out')), AGENT_STOP_TIMEOUT_MS); socket.setEncoding('utf8'); socket.on('connect', () => { + if (finished) return; connected = true; socket.write(JSON.stringify({ kind: 'agentInfo' }) + '\n'); }); socket.on('data', chunk => { + // Destruction is asynchronous: late events must never restart the handshake. + if (finished) return; buffered += chunk; if (buffered.length > 8192) { finish(new Error('Unexpected native agent lifecycle response')); return; } let end: number; diff --git a/App/shell/desktop/src/main/main.ts b/App/shell/desktop/src/main/main.ts index d1115931f..f98c706b3 100644 --- a/App/shell/desktop/src/main/main.ts +++ b/App/shell/desktop/src/main/main.ts @@ -6081,6 +6081,13 @@ function clearQuitCleanupForceExitTimer(): void { quitCleanupForceExitTimer = null; } +async function traceQuitCleanup(stage: string, operation: () => Promise): Promise { + const startedAt = Date.now(); + console.info(`[quit] ${stage}:start`); + try { return await operation(); } + finally { console.info(`[quit] ${stage}:finished elapsedMs=${Date.now() - startedAt}`); } +} + async function cleanupBeforeQuit(): Promise { if (requiredUpdateBackgroundFirstCheckTimer) { clearTimeout(requiredUpdateBackgroundFirstCheckTimer); @@ -6095,7 +6102,7 @@ async function cleanupBeforeQuit(): Promise { } clearPetWindowCloseActivateSuppression(); closePetWindow(); - await installPreparedRequiredUpdateOnQuit(); + await traceQuitCleanup("prepared-update", async () => { await installPreparedRequiredUpdateOnQuit(); }); ipcMain.removeHandler("memmy:get-runtime-config"); ipcMain.removeHandler("memmy:get-app-info"); ipcMain.removeHandler("memmy:get-installation-id"); @@ -6170,15 +6177,17 @@ async function cleanupBeforeQuit(): Promise { disposeBrowserWebviewPermissions?.(); disposeBrowserWebviewPermissions = null; // Native app agents belong to LaunchServices, outside the gateway process tree. - // Start their shutdown before awaiting services, which may fail or hit the quit deadline. + // Start their bounded shutdown before awaiting services, which may fail or hit the quit deadline. const nativeAgentShutdown = process.platform === 'darwin' - ? stopComputerUseAgentForPermissionRestart(app.isPackaged, process.resourcesPath, - process.env.MEMMY_DEV_COMPUTER_USE_BINARY).catch(error => { + ? traceQuitCleanup("native-history-agent", () => stopComputerUseAgentForPermissionRestart(app.isPackaged, process.resourcesPath, + process.env.MEMMY_DEV_COMPUTER_USE_BINARY)).catch(error => { console.warn("Failed to stop the native Computer Use agent during quit cleanup:", error); }) : Promise.resolve(); try { - await services?.close({ stopMemory: stopMemoryServiceForCurrentQuit }); + await traceQuitCleanup("runtime-services", async () => { + await services?.close({ stopMemory: stopMemoryServiceForCurrentQuit }); + }); } finally { await nativeAgentShutdown; } @@ -6189,9 +6198,9 @@ async function cleanupBeforeQuit(): Promise { pipScreenPermissionGuide = null; computerUseSurfaceWindows?.dispose(); computerUseSurfaceWindows = null; - await backend?.close(); - await stopPackagedRendererServer(); - await sendAppExitEventBeforeQuit(); + await traceQuitCleanup("backend", async () => backend?.close()); + await traceQuitCleanup("renderer", stopPackagedRendererServer); + await traceQuitCleanup("analytics", sendAppExitEventBeforeQuit); } function readStopMemoryServiceOnExitSetting(): boolean { diff --git a/App/shell/desktop/tests/computer-use-agent-restart.test.ts b/App/shell/desktop/tests/computer-use-agent-restart.test.ts index a998ef05e..955a33876 100644 --- a/App/shell/desktop/tests/computer-use-agent-restart.test.ts +++ b/App/shell/desktop/tests/computer-use-agent-restart.test.ts @@ -1,4 +1,4 @@ -import { once } from 'node:events'; +import { EventEmitter, once } from 'node:events'; import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import net from 'node:net'; import path from 'node:path'; @@ -284,6 +284,45 @@ describe.skipIf(hostPlatform === 'win32')('Computer Use cleanup for a complete d expect(native.requests).toEqual(['agentInfo']); }); + it.each(['timeout', 'close'] as const)('ignores late socket events after %s and clears the shutdown deadline', async completion => { + const f = fixture(); + processRows = [row(f, 900_001, f.socket('a'.repeat(16)))]; + const socket = Object.assign(new EventEmitter(), { + setEncoding: vi.fn(), write: vi.fn(), destroy: vi.fn(), + }); + vi.spyOn(net, 'createConnection').mockReturnValue(socket as unknown as net.Socket); + vi.useFakeTimers(); + try { + const stopping = stopComputerUseAgentForPermissionRestart(true, f.resources); + const outcome = completion === 'timeout' + ? expect(stopping).rejects.toThrow('Some native Computer Use agents did not complete shutdown') + : expect(stopping).resolves.toBeUndefined(); + socket.emit('connect'); + if (completion === 'timeout') await vi.advanceTimersByTimeAsync(1_000); + else { + socket.emit('data', `${JSON.stringify(f.info)}\n`); + socket.emit('data', '{"ok":true}\n'); + socket.emit('close'); + } + await outcome; + expect(vi.getTimerCount()).toBe(0); + const writes = socket.write.mock.calls.length; + expect(() => { + socket.emit('connect'); + socket.emit('data', `${JSON.stringify(f.info)}\n`); + socket.emit('data', '{not json}\n'); + socket.emit('error', new Error('late socket destruction error')); + socket.emit('error', new Error('another late socket error')); + socket.emit('close'); + }).not.toThrow(); + expect(socket.write).toHaveBeenCalledTimes(writes); + expect(socket.destroy).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + it('rechecks executable text before signalling and skips changed or unavailable processes', () => { const f = fixture(); processRows = [1, 2, 3, 4].map(index => row(f, 900_000 + index, f.socket(index.toString().repeat(16)))); diff --git a/App/shell/desktop/tests/mac-permission-restart-lifecycle.test.ts b/App/shell/desktop/tests/mac-permission-restart-lifecycle.test.ts index f8285663f..840b2807b 100644 --- a/App/shell/desktop/tests/mac-permission-restart-lifecycle.test.ts +++ b/App/shell/desktop/tests/mac-permission-restart-lifecycle.test.ts @@ -11,7 +11,7 @@ const source = ts.createSourceFile("main.ts", readFileSync(new URL("../src/main/ const functionNames = new Set([ "captureMacPermissionRestartRequest", "shouldIgnoreStaleReopenQuit", "relaunchAfterQuitCleanupIfRequested", "armQuitCleanupForceExitTimer", "clearQuitCleanupForceExitTimer", - "isMacSystemSessionEnding", "readMacSystemSessionProbe", "readMacQuitContext", + "isMacSystemSessionEnding", "readMacSystemSessionProbe", "readMacQuitContext", "traceQuitCleanup", ]); const variableNames = new Set([ "isQuitting", "isQuitCleanupInProgress", "isQuitCleanupComplete", "stopMemoryServiceForCurrentQuit", @@ -82,6 +82,7 @@ function setup(options: { requestSingleInstanceLock: ReturnType; }; const logs: string[] = []; + const quitTrace = vi.fn(); const events: string[] = []; let completeCleanup!: () => void; let failCleanup!: (error: Error) => void; @@ -134,7 +135,7 @@ function setup(options: { process: { platform: options.platform ?? "darwin", env: {}, resourcesPath: "/Applications/Memmy.app/Contents/Resources", argv: options.argv ?? ["Memmy", ...(options.relaunchMarker ? ["--memmy-permission-relaunch"] : [])] }, Date, setTimeout, clearTimeout, queueMicrotask, - console: { warn: vi.fn() }, + console: { warn: vi.fn(), info: quitTrace }, loadMenuBarAppearanceNative, isMacPermissionSystemQuit: policy, isMacSystemSessionQuit, MacPermissionRestartIntent, @@ -167,7 +168,7 @@ function setup(options: { const finishGuide = (token: number, approved: boolean) => runInContext(`macPermissionRestartIntent.finish(${token}, ${approved})`, context); if (options.permissionRequested) beginGuide(); return { - app, logs, events, readCurrentQuitContext, readSystemSessionEnding, readSystemSessionEndingState, + app, logs, events, quitTrace, readCurrentQuitContext, readSystemSessionEnding, readSystemSessionEndingState, loadMenuBarAppearanceNative, policy, cleanupBeforeQuit, closeServices, terminateSync, requestQuit, completeCleanup, failCleanup, stopComputerUseAgentsOnExitSync, stopComputerUseAgentForPermissionRestart, @@ -206,6 +207,30 @@ describe("native app-agent shutdown during desktop quit", () => { expect(test.events).toEqual(["native-async", "services-close", "native-sync", "relaunch", "quit"]); expect(test.stopComputerUseAgentsOnExitSync).toHaveBeenCalledExactlyOnceWith(true, "/Applications/Memmy.app/Contents/Resources", undefined); + expect(test.quitTrace).toHaveBeenCalledWith("[quit] runtime-services:start"); + expect(test.quitTrace).toHaveBeenCalledWith("[quit] native-history-agent:start"); + expect(test.quitTrace).toHaveBeenCalledWith(expect.stringMatching(/^\[quit\] runtime-services:finished elapsedMs=\d+$/)); + expect(test.quitTrace).toHaveBeenCalledWith(expect.stringMatching(/^\[quit\] native-history-agent:finished elapsedMs=\d+$/)); + }); + + it("closes services while native shutdown is pending, then waits for its bounded cleanup before quitting", async () => { + const test = setup({ actualCleanup: true, quitContext: userQuit }); + let completeNative!: () => void; + test.stopComputerUseAgentForPermissionRestart.mockImplementation(() => new Promise(resolve => { completeNative = resolve; })); + test.requestQuit(); + await vi.advanceTimersByTimeAsync(0); + expect(test.closeServices).toHaveBeenCalledOnce(); + test.completeCleanup(); + await vi.advanceTimersByTimeAsync(0); + expect(test.quitTrace).toHaveBeenCalledWith(expect.stringMatching(/^\[quit\] runtime-services:finished /)); + expect(test.quitTrace).not.toHaveBeenCalledWith(expect.stringMatching(/^\[quit\] native-history-agent:finished /)); + expect(test.stopComputerUseAgentsOnExitSync).not.toHaveBeenCalled(); + expect(test.events).not.toContain("quit"); + completeNative(); + await vi.advanceTimersByTimeAsync(0); + expect(test.quitTrace).toHaveBeenCalledWith(expect.stringMatching(/^\[quit\] native-history-agent:finished /)); + expect(test.stopComputerUseAgentsOnExitSync).toHaveBeenCalledOnce(); + expect(test.events).toContain("quit"); }); it("runs native exit cleanup after an earlier cleanup failure and before an ordinary quit", async () => { diff --git a/Memory/src/agent-source/runtime.ts b/Memory/src/agent-source/runtime.ts index 98819afeb..320b91518 100644 --- a/Memory/src/agent-source/runtime.ts +++ b/Memory/src/agent-source/runtime.ts @@ -311,7 +311,10 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti progressBeforePause = null; const controller = new AbortController(); scanAbortController = controller; - const activeScan = runScan(request, controller.signal, jobId).then(() => { + const sourceIds = request.sourceId === "all" + ? registry.list().map(adapter => adapter.descriptor.sourceId) : [request.sourceId]; + const activeScan = options.service.withAgentSourceScanPermit(sourceIds, controller.signal, + () => runScan(request, controller.signal, jobId)).then(() => { if (scan.jobId !== jobId || controller.signal.aborted) return; setScan({ ...scan, running: false, completedAt: scan.completedAt ?? new Date().toISOString() }); logger.info("scan.completed", { jobId, sourceId: request.sourceId }); diff --git a/Memory/src/config/model-catalog.ts b/Memory/src/config/model-catalog.ts index ba32a5e15..17f907ffd 100644 --- a/Memory/src/config/model-catalog.ts +++ b/Memory/src/config/model-catalog.ts @@ -114,7 +114,8 @@ function upsertMemoryCatalogPreset( const endpoint = record(value); return stringValue(endpoint.apiBase)?.replace(/\/+$/, "") === apiBase && endpoint.protocol === protocol - && (stringValue(endpoint.apiKey) ?? stringValue(provider.apiKey)) === apiKey + && endpoint.inheritProviderDefaults === false + && stringValue(endpoint.apiKey) === apiKey && stableJson(record(endpoint.extraHeaders)) === stableJson(extraHeaders) && stableJson(record(endpoint.extraBody)) === stableJson(extraBody); })?.[0]; @@ -124,6 +125,7 @@ function upsertMemoryCatalogPreset( endpoints ); endpoints[endpointId] = { + inheritProviderDefaults: false, apiBase, protocol, ...(apiKey ? { apiKey } : {}), diff --git a/Memory/src/contracts/model-catalog-resolver.ts b/Memory/src/contracts/model-catalog-resolver.ts index d32efb5fb..0c4cc2f13 100644 --- a/Memory/src/contracts/model-catalog-resolver.ts +++ b/Memory/src/contracts/model-catalog-resolver.ts @@ -9,6 +9,8 @@ export interface RuntimeCatalogEndpoint { apiBase: string; protocol: ModelEndpointProtocol; apiKey?: string; + /** False keeps credentials and request options local to this endpoint. */ + inheritProviderDefaults?: boolean; extraHeaders?: Record; extraBody?: Record; } @@ -152,10 +154,12 @@ export function resolveAssignedModel(input: ResolveAssignedModelInput): ModelSel } if (!providerOwnerMatches(preset, provider, input.activeAccountId)) return UNAVAILABLE; + const inherited = endpoint.inheritProviderDefaults === false ? {} : provider; + const apiKey = endpoint.apiKey ?? inherited.apiKey; let extraBody: Readonly>; try { extraBody = deepFreeze(structuredClone({ - ...(provider.extraBody ?? {}), + ...(inherited.extraBody ?? {}), ...(endpoint.extraBody ?? {}) })); } catch { @@ -179,12 +183,12 @@ export function resolveAssignedModel(input: ResolveAssignedModelInput): ModelSel endpointId: preset.endpoint, protocol: endpoint.protocol, apiBase: endpoint.apiBase, - ...(endpoint.apiKey ?? provider.apiKey - ? { apiKey: endpoint.apiKey ?? provider.apiKey } + ...(apiKey + ? { apiKey } : {}), ...(provider.ownerAccountId ? { ownerAccountId: provider.ownerAccountId } : {}), extraHeaders: Object.freeze({ - ...(provider.extraHeaders ?? {}), + ...(inherited.extraHeaders ?? {}), ...(endpoint.extraHeaders ?? {}) }), extraBody @@ -320,6 +324,7 @@ function isRuntimeEndpoint(value: unknown): value is RuntimeCatalogEndpoint { && typeof value.protocol === "string" && PROTOCOLS.has(value.protocol as ModelEndpointProtocol) && (value.apiKey === undefined || typeof value.apiKey === "string") + && (value.inheritProviderDefaults === undefined || typeof value.inheritProviderDefaults === "boolean") && validStringRecord(value.extraHeaders) && validUnknownRecord(value.extraBody); } diff --git a/Memory/src/logging/logger.ts b/Memory/src/logging/logger.ts index 3a4ef0728..9a16ed79c 100644 --- a/Memory/src/logging/logger.ts +++ b/Memory/src/logging/logger.ts @@ -200,19 +200,19 @@ function messageForEvent(component: string, event: string, fields: MemoryLogFiel case "json.failed": return `Model JSON parsing failed${details(fields, ["attempt", "maxTokens", "finishReason", "errorMessage"])}`; case "job.started": - return `Job started${details(fields, ["jobId", "attempt", "maxAttempts", "sessionId", "episodeId", "targetMemoryId"])}`; + return `Job started${details(fields, ["jobId", "queueLane", "enqueueSeq", "attempt", "maxAttempts", "sessionId", "episodeId", "targetMemoryId"])}`; case "job.succeeded": - return `Job succeeded${details(fields, ["jobId", "attempt", "maxAttempts", "targetMemoryId"])}`; + return `Job succeeded${details(fields, ["jobId", "queueLane", "enqueueSeq", "attempt", "maxAttempts", "targetMemoryId"])}`; case "job.failed": - return `Job failed${details(fields, ["jobId", "attempt", "maxAttempts", "terminal", "targetMemoryId", "errorMessage"])}`; + return `Job failed${details(fields, ["jobId", "queueLane", "enqueueSeq", "attempt", "maxAttempts", "terminal", "targetMemoryId", "errorMessage"])}`; case "embedding_retry.succeeded": - return `Embedding retry succeeded${details(fields, ["retryId", "targetMemoryId", "vectorField", "attempt", "maxAttempts"])}`; + return `Embedding retry succeeded${details(fields, ["retryId", "queueLane", "enqueueSeq", "targetMemoryId", "vectorField", "attempt", "maxAttempts"])}`; case "embedding_retry.retry_scheduled": - return `Embedding generation failed; retry scheduled${details(fields, ["retryId", "targetMemoryId", "vectorField", "attempt", "maxAttempts", "nextAttemptAt", "errorMessage"])}`; + return `Embedding generation failed; retry scheduled${details(fields, ["retryId", "queueLane", "enqueueSeq", "targetMemoryId", "vectorField", "attempt", "maxAttempts", "nextAttemptAt", "errorMessage"])}`; case "embedding_retry.failed": - return `Embedding retry failed after the final attempt${details(fields, ["retryId", "targetMemoryId", "vectorField", "attempt", "maxAttempts", "errorMessage"])}`; + return `Embedding retry failed after the final attempt${details(fields, ["retryId", "queueLane", "enqueueSeq", "targetMemoryId", "vectorField", "attempt", "maxAttempts", "errorMessage"])}`; case "drain.completed": - return `Worker drain completed${details(fields, ["leased", "succeeded", "failed", "embeddingRetriesLeased", "embeddingRetriesSucceeded", "embeddingRetriesFailed"])}`; + return `Worker drain completed${details(fields, ["algorithm", "laneWeights", "leased", "succeeded", "failed", "embeddingRetriesLeased", "embeddingRetriesSucceeded", "embeddingRetriesFailed"])}`; case "drain.failed": return `Worker drain failed${details(fields, ["errorMessage"])}`; case "startup.reconciliation_failed": diff --git a/Memory/src/model/task-routing.ts b/Memory/src/model/task-routing.ts index b7736e648..dd8f72590 100644 --- a/Memory/src/model/task-routing.ts +++ b/Memory/src/model/task-routing.ts @@ -1,6 +1,8 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { embeddingIdentity, rememberEmbeddingIdentity } from "./embedding-identity.js"; import type { MemmyConfig } from "../config/index.js"; +import { providerDescriptor, type WorkerConcurrencyController } from "../service/worker/worker-concurrency.js"; +import { observeWorker, type WorkerObserver } from "../service/worker/worker-observability.js"; import type { Embedder, LlmClient, @@ -20,9 +22,16 @@ export interface MemoryModelTaskContext { export class MemoryModelTaskRouter { private readonly storage = new AsyncLocalStorage(); + private readonly lifecycle = new AsyncLocalStorage(); + private readonly activeTasks = new Set(); + private closing = false; + private stopped?: Promise; + private resolveStopped?: () => void; constructor( - private readonly resolveContext: () => MemoryModelTaskContext + private readonly resolveContext: () => MemoryModelTaskContext, + private readonly concurrency?: WorkerConcurrencyController, + private readonly observer?: WorkerObserver ) {} client(role: MemoryModelRole): LlmClient { @@ -37,9 +46,90 @@ export class MemoryModelTaskRouter { return this.storage.getStore() ?? this.resolveContext(); } + /** Resolve once before admission; execution must use precisely these delegates. */ + snapshot(): MemoryModelTaskContext { + return Object.freeze({ ...this.resolveContext() }); + } + + runWith(context: MemoryModelTaskContext, operation: () => T): T { + // A worker may already own a claim while its execution microtask is queued. + if (this.closing && !this.concurrency?.hasActiveWorkerPermit()) throw new Error("Model task router is closing"); + return this.track(() => this.storage.run(context, operation)); + } + + request(config: LlmClient["config"] | Embedder["config"], operation: () => Promise, embedding = false, + kind: "summary" | "evolution" | "embedding" = embedding ? "embedding" : "summary", batchSize?: number): Promise { + if (!this.concurrency && !this.observer) return operation(); + const descriptor = providerDescriptor(config, this.concurrency?.limits); + const measured = this.observer ? async () => { + // Called only after actual request admission, never while waiting for a + // provider permit. Delegate retries remain part of this logical request. + const started = performance.now(); + let failure: "429" | "timeout" | "other" | undefined; + try { return await operation(); } + catch (error) { + const status = error && typeof error === "object" && "httpStatus" in error ? error.httpStatus : undefined; + const name = error instanceof Error ? error.name : ""; + failure = status === 429 ? "429" : /^(?:AbortError|TimeoutError)$/.test(name) ? "timeout" : "other"; + throw error; + } finally { + observeWorker(this.observer, { type: "model_request", kind, + providerKey: descriptor.providers[0]!.key, modelKey: descriptor.providers.at(-1)!.key, + durationMs: performance.now() - started, outcome: failure ? "failed" : "succeeded", failure, batchSize }); + } + } : operation; + return this.concurrency + ? this.concurrency.runRequest(descriptor, measured, embedding) + : measured(); + } + + embeddingBatchSize(): number { + return this.concurrency?.limits.embeddingBatchSize ?? 16; + } + run(operation: () => T): T { - if (this.storage.getStore()) return operation(); - return this.storage.run(this.resolveContext(), operation); + if (this.storage.getStore() && this.isActiveTask()) return operation(); + if (this.closing) throw new Error("Model task router is closing"); + const start = () => this.storage.run(this.resolveContext(), operation); + return this.track(() => this.concurrency ? this.concurrency.runNewForegroundTask(start) : start()); + } + + assertCanStop(): void { + // An admitted worker's reentry is checked by WorkerRunner first. A foreground + // task must not await its own completion through service.stop either. + if (this.isActiveTask() && !this.concurrency?.hasActiveWorkerPermit()) { + throw new Error("Cannot stop Memory service from an active foreground model task"); + } + } + + stop(): Promise { + this.assertCanStop(); + if (this.stopped) return this.stopped; + this.closing = true; + this.stopped = new Promise(resolve => { this.resolveStopped = resolve; }); + if (this.activeTasks.size === 0) this.resolveStopped?.(); + return this.stopped; + } + + private track(operation: () => T): T { + const token = {}; + this.activeTasks.add(token); + const finish = () => { + this.activeTasks.delete(token); + if (this.closing && this.activeTasks.size === 0) this.resolveStopped?.(); + }; + try { + const result = this.lifecycle.run(token, operation); + if (result && typeof (result as { then?: unknown }).then === "function") { + void Promise.resolve(result).then(finish, finish); + } else finish(); + return result; + } catch (error) { finish(); throw error; } + } + + private isActiveTask(): boolean { + const token = this.lifecycle.getStore(); + return token !== undefined && this.activeTasks.has(token); } } @@ -56,10 +146,18 @@ class TaskRoutedEmbedder implements Embedder { embed(texts: string[], role?: "query" | "document"): Promise { return this.router.run(async () => { + if (texts.length === 0) return []; const delegate = this.delegate(); const identity = embeddingIdentity(delegate.config); - const vectors = await delegate.embed(texts, role); - return vectors.map((vector) => rememberEmbeddingIdentity(vector, identity)); + const batchSize = this.router.embeddingBatchSize(); + const vectors: number[][] = []; + // A business task may generate many texts internally (for example Work + // Memory retrieval). Limit actual requests without charging extra jobs. + for (let offset = 0; offset < texts.length; offset += batchSize) { + const batch = texts.slice(offset, offset + batchSize); + vectors.push(...await this.router.request(delegate.config, () => delegate.embed(batch, role), true, "embedding", batch.length)); + } + return vectors.map(vector => rememberEmbeddingIdentity(vector, identity)); }); } @@ -67,7 +165,8 @@ class TaskRoutedEmbedder implements Embedder { return this.router.run(async () => { const delegate = this.delegate(); const identity = embeddingIdentity(delegate.config); - return rememberEmbeddingIdentity(await delegate.embedOne(text, role), identity); + const vector = await this.router.request(delegate.config, () => delegate.embedOne(text, role), true, "embedding", 1); + return rememberEmbeddingIdentity(vector, identity); }); } @@ -98,14 +197,20 @@ class TaskRoutedLlmClient implements LlmClient { messages: LlmMessage[], options: LlmCompletionOptions ): Promise { - return this.router.run(() => this.delegate().complete(messages, options)); + return this.router.run(() => { + const delegate = this.delegate(); + return this.router.request(delegate.config, () => delegate.complete(messages, options), false, this.role); + }); } completeJson>( messages: LlmMessage[], options: LlmCompletionOptions ): Promise { - return this.router.run(() => this.delegate().completeJson(messages, options)); + return this.router.run(() => { + const delegate = this.delegate(); + return this.router.request(delegate.config, () => delegate.completeJson(messages, options), false, this.role); + }); } status(): ModelStatus { diff --git a/Memory/src/model/token-usage.ts b/Memory/src/model/token-usage.ts index 988955be7..1a4878234 100644 --- a/Memory/src/model/token-usage.ts +++ b/Memory/src/model/token-usage.ts @@ -5,6 +5,7 @@ import { randomUUID } from "node:crypto"; import { isBudgetedMemoryUsage } from "@memmy/agent-source-core"; import type { ActualModelContext } from "../contracts/index.js"; import { createMemoryLogger, memoryErrorFields } from "../logging/logger.js"; +import { withoutJobLeaseGuard } from "../storage/job-lease-context.js"; export type MemoryLlmModelRole = "memory_summary" | "memory_evolution"; export type MemoryTokenUsageKind = MemoryLlmModelRole | "embedding"; @@ -200,6 +201,12 @@ export class HttpByokTokenUsageRecorder implements MemoryTokenUsageSink { } record(event: MemoryModelUsageEvent): MemoryTokenUsageRecordStatus { + // A model request can consume tokens even when its task lease is lost. + // Accounting and the delivery/probe timers it starts outlive that task. + return withoutJobLeaseGuard(() => this.recordUsage(event)); + } + + private recordUsage(event: MemoryModelUsageEvent): MemoryTokenUsageRecordStatus { const context = event.actualModelContext; if (!context || context.source !== "byok" || context.capability !== event.kind || isEmptyUsage(event.usage)) { return "skipped"; diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index 9c84e0148..b3b85e0ab 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -31,6 +31,7 @@ import type { } from "../types.js"; import { DEFAULT_NAMESPACE_SOURCE } from "../types.js"; import { MemoryService } from "../service/memory-service.js"; +import { withoutWorkerSchedulerContext } from "../service/worker/worker-scheduler.js"; import { MemoryServiceError, statusForCode } from "../utils/error.js"; import { stableHash } from "../utils/id.js"; import { resolveTimeZone } from "../utils/time.js"; @@ -374,8 +375,15 @@ function createAutoWorkerDrain( let activeDrain: Promise | undefined; const maxCycles = 40; const workerBatchSize = 4; + const admissionAbort = new AbortController(); - async function drain(): Promise { + function drain(): Promise { + // A wake-up can originate from a running job's async descendants. It queues + // independent work rather than awaiting that job's own execution boundary. + return withoutWorkerSchedulerContext(drainUnscoped); + } + + async function drainUnscoped(): Promise { if (disposed) { return; } @@ -401,7 +409,12 @@ function createAutoWorkerDrain( for (let cycle = 0; cycle < maxCycles && !disposed; cycle += 1) { const result = await service.runWorkerOnce(workerBatchSize, { priorityCohortOnly: true - }); + }, { continuous: true, signal: admissionAbort.signal, onYieldToPendingRequests: () => { + // Keep an automatic continuation behind the accepted manual work. + // Awaiting the shared scheduler provides the wake-up; polling zero + // results would either lose the tail or spin while manual work runs. + requested = true; + } }); if (result.leased === 0 && result.embeddingRetries.leased === 0) { break; } @@ -453,6 +466,7 @@ function createAutoWorkerDrain( if (disposed) { return; } + service.notifyWorkerWorkAvailable(); startupReleased = true; requested = true; if (startupTimer) { @@ -482,7 +496,8 @@ function createAutoWorkerDrain( clearTimeout(delayedTimer); delayedTimer = undefined; } - scheduleNextDueJob(); + service.notifyWorkerWorkAvailable(); + if (!running) scheduleNextDueJob(); }); // Capture failures still leave durable work, even when the HTTP response fails. @@ -521,6 +536,7 @@ function createAutoWorkerDrain( schedule, async dispose(): Promise { disposed = true; + admissionAbort.abort(); service.setAppBudgetReconcileListener(undefined); service.setPersistRecoveredListener(undefined); service.setCapturePendingListener(undefined); @@ -936,9 +952,9 @@ async function routeRequest( "worker.import-summaries.enqueue.memoryIds" ); const result = service.enqueuePendingImportSummaries(10_000, memoryIds); - if (result.enqueued > 0) { - autoWorker.schedule(); - } + // An existing queued task also needs a wake-up. Producers enqueue and + // observe status instead of starting their own targeted worker execution. + autoWorker.schedule(); return result; } diff --git a/Memory/src/service/embedding/embedding-job-processor.ts b/Memory/src/service/embedding/embedding-job-processor.ts index 16ffa2e76..7f750584a 100644 --- a/Memory/src/service/embedding/embedding-job-processor.ts +++ b/Memory/src/service/embedding/embedding-job-processor.ts @@ -318,10 +318,7 @@ export class EmbeddingJobProcessor { attemptCount: item.job.attempts, source: "worker.embedding", sourceHash: item.sourceHash, - allowedProcessingStates: ["embedding_pending", "embedding"], - finalize: (_saved, hadProcessing, at) => { - if (hadProcessing) this.deps.repos.runtime.completeJob(item.job.id, at); - } + allowedProcessingStates: ["embedding_pending", "embedding"] }); } diff --git a/Memory/src/service/embedding/embedding-pipeline.ts b/Memory/src/service/embedding/embedding-pipeline.ts index dd6250316..a75aa4073 100644 --- a/Memory/src/service/embedding/embedding-pipeline.ts +++ b/Memory/src/service/embedding/embedding-pipeline.ts @@ -123,10 +123,11 @@ export function embeddingRetryVectorFieldForMemory(memory: MemoryRow): Embedding } export function embeddingRetryBackoffMs(attemptNo: number): number { - return Math.min( + const base = Math.min( EMBEDDING_RETRY_MAX_BACKOFF_MS, EMBEDDING_RETRY_BASE_BACKOFF_MS * 2 ** Math.max(0, attemptNo - 1) ); + return Math.min(EMBEDDING_RETRY_MAX_BACKOFF_MS, Math.round(base * (0.8 + Math.random() * 0.4))); } export function embeddingRetryToRunItem(retry: EmbeddingRetryRecord): EmbeddingRetryRunItem { diff --git a/Memory/src/service/import/import-job-processor.ts b/Memory/src/service/import/import-job-processor.ts index ef000809c..0d69a86c1 100644 --- a/Memory/src/service/import/import-job-processor.ts +++ b/Memory/src/service/import/import-job-processor.ts @@ -134,6 +134,7 @@ export interface ImportJobProcessorDeps { }; runtime: { appendChange(change: Record): number; + getJob(id: string): EvolutionJobRecord | undefined; latestJobForTarget(targetMemoryId: string, jobType: JobType): { payload: Record } | undefined; }; } @@ -431,7 +432,10 @@ export class ImportJobProcessor { if (!d.config.algorithm.enableMemoryAdd) return 0; let restarted = 0; for (const failed of d.processing.listByStates(["failed"], limit)) { + // Configuration blocks resume via the worker's configuration fingerprint + // change or an explicit manual retry, never on every service restart. if (!failed.stage || failed.retryAction === "none") continue; + if (failed.activeJobId && d.runtime.getJob(failed.activeJobId)?.blockedReason === "configuration") continue; const memory = d.memories.get(failed.memoryId); if (!memory) continue; if (failed.stage !== "summary" && d.memories.hasVector(memory.id, "vec_summary")) { diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index fc4d85c79..2255cef17 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -43,7 +43,8 @@ import { HttpByokTokenUsageRecorder, type HttpByokTokenUsageRecorderOptions, type MemoryLlmModelRole, - type MemoryModelUsageEvent + type MemoryModelUsageEvent, + type MemoryTokenUsageSink } from "../model/token-usage.js"; import { TokenUsageOutbox } from "../storage/token-usage-outbox.js"; import { MemoryTokenBudgetLedger } from "./memory-token-budget-ledger.js"; @@ -202,7 +203,11 @@ import { type ClosedEpisodeTrigger, type EnqueueJobInput } from "./worker/job-handlers.js"; -import { WorkerRunner } from "./worker/worker-runner.js"; +import { WorkerRunner, type WorkerRunExecution } from "./worker/worker-runner.js"; +import { WorkerConcurrencyController, providerDescriptor, workerModelBatchKey, type PreparedWorkerExecution } from "./worker/worker-concurrency.js"; +import { observeWorker, readWorkerQueueMetrics, WorkerObservability } from "./worker/worker-observability.js"; +import { jobPolicyRegistry, UnknownJobPolicyError } from "./worker/job-policy-registry.js"; +import type { FairClaimedWork } from "./worker/fair-worker-queue.js"; const serviceLogger = createMemoryLogger("memory-service"); @@ -220,6 +225,10 @@ export type { FeedbackResponse } from "./feedback/feedback-experience.js"; export interface MemoryServiceOptions { + /** One controller is shared by worker admission and actual model requests. */ + concurrency?: WorkerConcurrencyController; + /** Optional process-local diagnostics sink, never a public status API. */ + observability?: WorkerObservability; db?: MemoryDb; backend?: StorageBackend; mode?: "local" | "cloud" | "dev"; @@ -303,6 +312,13 @@ export class MemoryService { private readonly mode: "local" | "cloud" | "dev"; private config: MemmyConfig; private readonly modelTasks: MemoryModelTaskRouter; + private readonly concurrency: WorkerConcurrencyController; + private readonly workerMetrics: WorkerObservability; + private readonly stopConcurrencyMetrics: () => void; + private readonly workerExecutionSnapshots = new WeakMap; + }>(); private llm: LlmClient; private skillLlm: LlmClient; private embedder: Embedder; @@ -311,6 +327,7 @@ export class MemoryService { private readonly embeddingRetryWorkerId = `embedding-retry-${newId("worker")}`; private readonly tokenBudgetLedger: MemoryTokenBudgetLedger; private readonly tokenUsageRecorder: HttpByokTokenUsageRecorder; + private readonly modelUsageSink: MemoryTokenUsageSink; private readonly fetchAppMemoryBudgetFn: (signal?: AbortSignal) => Promise<{ dailyUsed: number; lifetimeUsed: number } | null>; private readonly budgetAbort = new AbortController(); private appBudgetReconcile: { @@ -327,11 +344,17 @@ export class MemoryService { private persistRecoveredListener?: () => void; private capturePendingListener?: () => void; private closing = false; + private stopping?: Promise; private viewerEndpoint?: string; constructor(private readonly options: MemoryServiceOptions) { + this.concurrency = options.concurrency ?? new WorkerConcurrencyController(); this.viewerEndpoint = options.viewerEndpoint; this.repos = options.backend?.repositories() ?? new Repositories(requireMemoryDb(options).db); + this.workerMetrics = options.observability ?? new WorkerObservability({ + queueSnapshot: (at) => readWorkerQueueMetrics(this.repos.db, at) + }); + this.stopConcurrencyMetrics = this.concurrency.observe(this.workerMetrics); this.l3WorldModelContextReadModel = new L3WorldModelContextReadModel(this.repos); this.mode = options.mode ?? "local"; this.config = cloneMemmyConfig(options.config ?? DEFAULT_MEMMY_CONFIG); @@ -348,10 +371,17 @@ export class MemoryService { touchBudget: () => this.tokenBudgetLedger.touch(), onPersistRecovered: () => this.persistRecoveredListener?.() }); + this.modelUsageSink = { + record: (event) => { + // Preserve the original sink's return/throw and budget/outbox semantics. + try { return this.tokenUsageRecorder.record(event); } + finally { this.observeModelTokens(event); } + } + }; this.fetchAppMemoryBudgetFn = options.fetchAppMemoryBudget ?? ((signal) => fetchAppMemoryBudget({ signal })); this.startAppBudgetReconcile(); - this.modelTasks = new MemoryModelTaskRouter(() => this.resolveModelTaskContext()); + this.modelTasks = new MemoryModelTaskRouter(() => this.resolveModelTaskContext(), this.concurrency, this.workerMetrics); this.llm = this.modelTasks.client("summary"); this.skillLlm = this.modelTasks.client("evolution"); this.embedder = this.modelTasks.embedder(); @@ -560,6 +590,10 @@ export class MemoryService { }); const workerRunnerOwner = this; this.workerRunner = new WorkerRunner({ + concurrency: this.concurrency, + observer: this.workerMetrics, + prepareExecution: this.prepareWorkerExecution.bind(this), + prepareExecutionBatch: this.prepareWorkerExecutionBatch.bind(this), onTurnJudgeDeadLetter: (job) => this.embeddingJobs.markTurnJudgeFailed(job), repos: this.repos, get embedder() { return workerRunnerOwner.embedder; }, @@ -572,6 +606,27 @@ export class MemoryService { memoryBudgetNextWakeAtMs: () => this.tokenBudgetLedger.nextWakeAtMs(), memoryBudgetNextReconcileAtMs: () => this.nextAppBudgetReconcileAtMs(), summaryModelConfigured: () => workerRunnerOwner.llm.isConfigured(), + configurationFingerprint: () => { + const config = workerRunnerOwner.effectiveModelConfig(); + return stableHash({ summary: config.summary, evolution: config.evolution, + embedding: config.embedding, roleRouting: config.roleRouting }); + }, + beforeRun: async request => { + await this.ensureAppBudgetReconciled(); + // Run upgrade planning inside the shared worker entry so stop and scoped + // requests retain the same boundary as ordinary durable work. + if (request && this.config.algorithm.enableMemoryAdd && !this.shouldDeferBudgetedEmbedding() && !request.targetMemoryIds) { + this.embeddingUpgradeInFlight ??= this.withModelTaskContext(() => this.embeddingMaintenance().resumeUpgrade()) + .finally(() => { this.embeddingUpgradeInFlight = undefined; }); + await this.embeddingUpgradeInFlight; + } + }, + maintenanceNextRunAtMs: () => this.config.algorithm.enableMemoryAdd && !this.shouldDeferBudgetedEmbedding() + ? this.embeddingMaintenance().nextUpgradeAttemptAt() : undefined, + afterRun: () => { + if (this.config.algorithm.enableMemoryAdd) this.embeddingMaintenance().upgradeProgress(); + this.reconcileAutomaticHistorySkills(nowIso()); + }, nowIso, encodeChangeCursor: this.encodeChangeCursor.bind(this), namespaceIdFromMemory, @@ -760,6 +815,7 @@ export class MemoryService { this.sessionTurns.migrateLegacyCaptures(); serviceLogger.info("initialized", memoryConfigLogFields(this.config)); this.tokenUsageRecorder.start(); + this.workerMetrics.start(); } stopTokenUsageDelivery(): void { @@ -767,12 +823,21 @@ export class MemoryService { } async stop(): Promise { - if (this.closing) { - return; - } + this.modelTasks.assertCanStop(); + const stopped = this.workerRunner.stop(); + if (this.stopping) return this.stopping; this.closing = true; this.tokenUsageRecorder.stop(); this.budgetAbort.abort(); + // Close request admission synchronously before the worker releases provider + // slots, then keep storage alive through complete foreground business work. + const modelsStopped = this.modelTasks.stop(); + const requestsStopped = this.concurrency.close(); + this.stopping = Promise.all([stopped, modelsStopped, requestsStopped]).then(() => { + this.stopConcurrencyMetrics(); + this.workerMetrics.stop(); + }); + return this.stopping; } private createConfiguredMemoryLlm(config: MemmyConfig, modelRole: MemoryLlmModelRole): LlmClient { @@ -780,7 +845,7 @@ export class MemoryService { modelRole === "memory_summary" ? config.summary : resolveEvolutionConfig(config), { modelRole, - usageRecorder: this.tokenUsageRecorder + usageRecorder: this.modelUsageSink } ); } @@ -793,6 +858,17 @@ export class MemoryService { }); } + private observeModelTokens(event: MemoryModelUsageEvent): void { + try { + const descriptor = providerDescriptor({ provider: event.provider, endpoint: event.endpoint, model: event.model, + actualModelContext: event.actualModelContext }, this.concurrency.limits); + observeWorker(this.workerMetrics, { type: "tokens", kind: event.kind === "memory_summary" ? "summary" + : event.kind === "memory_evolution" ? "evolution" : "embedding", + providerKey: descriptor.providers[0]!.key, modelKey: descriptor.providers.at(-1)!.key, + inputTokens: event.usage.inputTokens, outputTokens: event.usage.outputTokens, totalTokens: event.usage.totalTokens }); + } catch { /* Diagnostics must not change token persistence or budget accounting. */ } + } + private budgetModelSources(): MemoryBudgetModelSources { return { memory_summary: { @@ -925,18 +1001,20 @@ export class MemoryService { ]); } + private effectiveModelConfig(): MemmyConfig { + return this.options.configPath || this.options.configLoader + ? (this.options.configLoader ?? loadMemmyConfig)(this.options.configPath).config + : this.config; + } + private resolveModelTaskContext(): MemoryModelTaskContext { - const taskConfig = cloneMemmyConfig( - this.options.configPath || this.options.configLoader - ? (this.options.configLoader ?? loadMemmyConfig)(this.options.configPath).config - : this.config - ); + const taskConfig = cloneMemmyConfig(this.effectiveModelConfig()); const summary = this.options.llm ?? this.createConfiguredMemoryLlm(taskConfig, "memory_summary"); const evolution = this.options.skillLlm ?? this.createConfiguredMemoryLlm(taskConfig, "memory_evolution"); const embedding = this.options.embedder ?? createEmbedder(taskConfig.embedding, { - usageRecorder: this.tokenUsageRecorder + usageRecorder: this.modelUsageSink }); freezeModelSelectionConfig(taskConfig); return { @@ -947,6 +1025,70 @@ export class MemoryService { }; } + private prepareWorkerExecutionBatch(): (work: FairClaimedWork) => PreparedWorkerExecution { + // One synchronous fill observes one configuration snapshot. Provider work is + // still resolved per candidate; injected runner factories may vary per work. + const context = this.modelTasks.snapshot(); + return work => this.prepareWorkerExecution(work, context); + } + + private prepareWorkerExecution(work: FairClaimedWork, snapshot?: MemoryModelTaskContext): PreparedWorkerExecution { + let roles: readonly ("summary" | "evolution" | "embedding")[]; + try { + roles = jobPolicyRegistry.resolve({ jobType: work.queue === "main" ? work.job.jobType : "embedding_retry" }).concurrency.providerRoles; + } catch (error) { + if (!(error instanceof UnknownJobPolicyError)) throw error; + // Unregistered jobs are admitted only for controlled dead-letter cleanup. + return { providers: [], run: operation => operation() }; + } + const context = snapshot ?? this.modelTasks.snapshot(); + let cached = this.workerExecutionSnapshots.get(context); + if (!cached) { + cached = { batchKey: workerModelBatchKey([context.summary.config, context.evolution.config, context.embedding.config]), providers: new Map() }; + this.workerExecutionSnapshots.set(context, cached); + } + const roleKey = roles.join("/"); + let providers = cached.providers.get(roleKey); + if (!providers) { + providers = roles.flatMap(role => providerDescriptor(context[role].config, this.concurrency.limits).providers); + cached.providers.set(roleKey, providers); + } + return { + providers, + embedding: roles.includes("embedding"), + batchKey: cached.batchKey, + run: operation => this.modelTasks.runWith(context, operation) + }; + } + + notifyWorkerWorkAvailable(): void { + this.workerRunner.notifyWorkAvailable(); + } + + /** Source producers share the global background budget, without inheriting + * a model permit: any foreground request still admits its own providers. */ + async withAgentSourceScanPermit(sourceIds: string | readonly string[], signal: AbortSignal, operation: () => Promise): Promise { + const scopeKeys = [...new Set(typeof sourceIds === "string" ? [sourceIds] : sourceIds)] + .map(sourceId => `agent-source-scan:${sourceId}`); + for (;;) { + signal.throwIfAborted(); + if (this.stopping) throw new Error("Memory service is stopping"); + const permit = this.concurrency.tryAcquire({ resourceClass: "scan", providers: [], + scopeKeys }); + if (permit) { + try { signal.throwIfAborted(); return await operation(); } + finally { permit.release(); } + } + await new Promise((resolve, reject) => { + const finish = () => { unsubscribe(); signal.removeEventListener("abort", canceled); resolve(); }; + const canceled = () => { unsubscribe(); signal.removeEventListener("abort", canceled); reject(signal.reason); }; + const unsubscribe = this.concurrency.subscribe(finish); + signal.addEventListener("abort", canceled, { once: true }); + if (signal.aborted) canceled(); + }); + } + } + private withModelTaskContext(operation: () => T): T { return this.modelTasks.run(operation); } @@ -1089,10 +1231,12 @@ export class MemoryService { this.appBudgetReconcile.nextAttemptAtMs = 0; this.startAppBudgetReconcile(); if (!requiresRestart && request.restartFailedProcessing !== false) { + this.workerRunner.resumeJobsAfterConfigurationChange(); this.restartFailedProcessing(reloadedAt); for (const { before, after } of this.repos.runtime.requeueFailedEpisodeAssignments(reloadedAt)) this.workerHandlers.appendJobChange(after, "queued", before); this.capturePendingListener?.(); } + this.workerRunner.notifyWorkAvailable(); serviceLogger.info("config.reloaded", { changed, requiresRestart, @@ -3264,20 +3408,10 @@ export class MemoryService { request: RequestEnvelope & { targetMemoryIds?: string[]; priorityCohortOnly?: boolean; - } = {} + } = {}, + execution: WorkerRunExecution = {} ): ReturnType { - await this.ensureAppBudgetReconciled(); - // Run the bounded one-time provenance repair through the normal durable worker. - // Respect explicit write/budget gates and scoped worker requests. - if (this.config.algorithm.enableMemoryAdd && !this.shouldDeferBudgetedEmbedding() && !request.targetMemoryIds) { - this.embeddingUpgradeInFlight ??= this.withModelTaskContext(() => this.embeddingMaintenance().resumeUpgrade()) - .finally(() => { this.embeddingUpgradeInFlight = undefined; }); - await this.embeddingUpgradeInFlight; - } - const result = await this.workerRunner.runWorkerOnce(limit, request); - if (this.config.algorithm.enableMemoryAdd) this.embeddingMaintenance().upgradeProgress(); - this.reconcileAutomaticHistorySkills(nowIso()); - return result; + return this.workerRunner.runWorkerOnce(limit, request, execution); } private async queryVector(query: string): Promise { diff --git a/Memory/src/service/session/session-turn-service.ts b/Memory/src/service/session/session-turn-service.ts index b3a3ca929..bc96e978a 100644 --- a/Memory/src/service/session/session-turn-service.ts +++ b/Memory/src/service/session/session-turn-service.ts @@ -537,7 +537,7 @@ export class SessionTurnService { return this.deps.llm.isConfigured() ? undefined : this.ensureEpisode(session); } - queueAssignment(raw: RawTurnRecord, turnJudge = false, requestedEpisodeId = raw.episodeId): EvolutionJobRecord[] { + queueAssignment(raw: RawTurnRecord, turnJudge = false, requestedEpisodeId = raw.episodeId, captureOrigin?: "agent_source_scan"): EvolutionJobRecord[] { if (!this.deps.llm.isConfigured()) return []; const at = nowIso(); const generation = stableHash({ user: raw.userText, assistant: raw.assistantText, tools: raw.toolResults, turnJudge }); @@ -547,7 +547,7 @@ export class SessionTurnService { } }); return (["episode_assignment", "episode_finalize"] as const).map(jobType => this.deps.enqueueJob({ jobType, userId: raw.userId, sessionId: raw.sessionId, - dedupeKey: `${jobType}:${raw.id}:${generation}`, payload: { rawTurnId: raw.id, turnAt: raw.createdAt, generation }, createdAt: at + dedupeKey: `${jobType}:${raw.id}:${generation}`, payload: { rawTurnId: raw.id, turnAt: raw.createdAt, generation, ...(captureOrigin ? { captureOrigin } : {}) }, createdAt: at })); } @@ -1709,7 +1709,8 @@ export class SessionTurnService { throw new MemoryServiceError("forbidden", "source_raw_turn_scope_conflict"); } const result = this.completeTurn(identity.turnId, { ...normalized, sessionId: session.id, - ...(gapEpisode ? { episodeId: gapEpisode.id } : {}) }, sourceTurn); + ...(gapEpisode ? { episodeId: gapEpisode.id } : {}) }, sourceTurn, undefined, + allowsLegacyHistoryCapture(request) ? "agent_source_scan" : undefined); if (gapEpisode) this.deps.repos.runtime.orderEpisodeTurnsBySourceTime(gapEpisode.id); const response: SourceTurnCompleteResponse = result.l1MemoryIds.length > 0 ? { status: "stored", result } @@ -1791,7 +1792,7 @@ export class SessionTurnService { const result = this.completeTurn(sourceTurn.turnId, { ...withoutEpisode, sessionId: opened.sessionId - }, sourceTurn); + }, sourceTurn, undefined, allowsLegacyHistoryCapture(request) ? "agent_source_scan" : undefined); const response: SourceTurnCompleteResponse = result.l1MemoryIds.length > 0 ? { status: "stored", result } : { status: "rejected", reason: "capture_policy", result }; @@ -1952,7 +1953,8 @@ export class SessionTurnService { turnId: string, request: TurnCompleteRequest & Record, sourceTurn?: SourceTurnIdentity, - acceptedAt?: string + acceptedAt?: string, + captureOrigin?: "agent_source_scan" ): CompleteTurnResponse { request = sanitizeTurnCompleteRequest(request); if (request.status === "cancelled") { @@ -2303,7 +2305,7 @@ export class SessionTurnService { const l1MemoryIds: string[] = []; const captureClaimByRawTurnId = new Map(); let changeSeq = 0; - const jobs: EvolutionJobRecord[] = [...route.jobs, ...userMemoryCapture.jobs, ...this.queueAssignment(rawTurn, this.deps.config.algorithm.session.turnJudge && this.deps.llm.isConfigured(), request.episodeId)]; + const jobs: EvolutionJobRecord[] = [...route.jobs, ...userMemoryCapture.jobs, ...this.queueAssignment(rawTurn, this.deps.config.algorithm.session.turnJudge && this.deps.llm.isConfigured(), request.episodeId, captureOrigin)]; for (const step of capturedSteps) { const stepRawTurnId = step.rawTurnId ?? rawTurn.id; @@ -2364,6 +2366,8 @@ export class SessionTurnService { }, error_signatures: step.errorSignatures, ...observation.internal, + // Scheduling provenance is separate from the native capture pipeline. + ...(captureOrigin ? { capture_origin: captureOrigin } : {}), trace: { key: step.key, ts: step.ts, diff --git a/Memory/src/service/worker/admission-lane-queue-selector.ts b/Memory/src/service/worker/admission-lane-queue-selector.ts new file mode 100644 index 000000000..77fb04d5a --- /dev/null +++ b/Memory/src/service/worker/admission-lane-queue-selector.ts @@ -0,0 +1,163 @@ +import { QUEUE_LANES, type QueueLane } from "./job-policy-registry.js"; +import { DEFAULT_LANE_WEIGHTS, LaneQueueSelector, validateLaneWeights, type LaneQueueCandidate } from "./lane-queue-selector.js"; + +export interface LaneQueueTurns { + version: 1; + aging?: { servedMask: number; used: boolean }; + interactive?: { spent: boolean; upgradeAdvance?: boolean }; + lanes: ReadonlyArray<{ lane: QueueLane; remaining: number }>; +} + +/** + * WRR turns survive admission refusals. Only an exhausted successful lane moves + * to the tail: unrelated fast work cannot reset an older blocked lane's turn. + * There is one bounded turn per lane, never a credit per rejected job. + */ +export class AdmissionLaneQueueSelector { + private readonly weights: Readonly>; + private turns: Array<{ lane: QueueLane; remaining: number }>; + private cursor: number; + private empty = new Set(); + private aging = { servedMask: 0, used: false }; + private interactiveSpent = false; + private upgradeAdvance = false; + + constructor(weights = DEFAULT_LANE_WEIGHTS, initialCursor = 0, savedTurns?: unknown) { + this.weights = validateLaneWeights(weights); + this.cursor = new LaneQueueSelector(this.weights, initialCursor).getCursor(); + let offset = this.cursor, first = 0; + while (offset >= this.weights[QUEUE_LANES[first]!]) { + offset -= this.weights[QUEUE_LANES[first]!]!; + first++; + } + // Old cursor-only state past the interactive segment cannot mint a fresh + // quota. It may advance one existing ticket once during upgrade. New state + // distinguishes an empty skip from four actually consumed tickets. + this.interactiveSpent = first !== 0; + this.upgradeAdvance = this.interactiveSpent && savedTurns === undefined; + this.turns = QUEUE_LANES.map((_, i) => { + const lane = QUEUE_LANES[(first + i) % QUEUE_LANES.length]!; + return { lane, remaining: this.weights[lane] - (i === 0 ? offset : 0) }; + }); + // The extension is optional under the existing KV key. Old releases and + // malformed/unknown versions safely resume from the compatible cursor. + if (savedTurns === null || typeof savedTurns !== "object" || Array.isArray(savedTurns)) return; + const saved = savedTurns as { version?: unknown; lanes?: unknown; aging?: unknown; interactive?: unknown }; + if (saved.version !== 1 || !Array.isArray(saved.lanes) || saved.lanes.length !== QUEUE_LANES.length) return; + const restored: typeof this.turns = [], seen = new Set(); + for (const value of saved.lanes) { + if (value === null || typeof value !== "object" || Array.isArray(value)) return; + const { lane, remaining } = value as { lane?: unknown; remaining?: unknown }; + if (typeof lane !== "string" || !(QUEUE_LANES as readonly string[]).includes(lane) || seen.has(lane) || + typeof remaining !== "number" || !Number.isSafeInteger(remaining) || remaining < 1 || remaining > this.weights[lane as QueueLane]) return; + seen.add(lane); restored.push({ lane: lane as QueueLane, remaining }); + } + this.turns = restored; + this.upgradeAdvance = this.interactiveSpent && saved.interactive === undefined; + if (saved.interactive && typeof saved.interactive === "object" && !Array.isArray(saved.interactive) && + typeof (saved.interactive as { spent?: unknown }).spent === "boolean") { + this.interactiveSpent = (saved.interactive as { spent: boolean }).spent; + this.upgradeAdvance = (saved.interactive as { upgradeAdvance?: unknown }).upgradeAdvance === true; + } + if (saved.aging && typeof saved.aging === "object" && !Array.isArray(saved.aging)) { + const state = saved.aging as { servedMask?: unknown; used?: unknown }; + if (typeof state.servedMask === "number" && Number.isSafeInteger(state.servedMask) && state.servedMask >= 0 && + state.servedMask < (1 << QUEUE_LANES.length) && typeof state.used === "boolean") { + this.aging = { servedMask: state.servedMask, used: state.used }; + } + } + } + + getCursor(): number { return this.cursor; } + + snapshot(): LaneQueueTurns { + return { version: 1, lanes: this.turns.map(turn => ({ ...turn })), aging: { ...this.aging }, + interactive: { spent: this.interactiveSpent, upgradeAdvance: this.upgradeAdvance } }; + } + + /** Only lanes proven to have no eligible row, not resource-denied lanes. */ + dropEmpty(empty: ReadonlySet): void { + // Skipping an empty lane is tentative until some other lane really claims + // a job. A later slice may receive new interactive work before any CAS wins. + this.empty = new Set(empty); + } + + choose(candidates: readonly T[]): T | undefined { + const heads = this.heads(candidates); + for (const turn of this.turns) { + const head = heads.get(turn.lane); + if (head) return head; + } + return undefined; + } + + /** Prefer already assigned interactive tickets. Old ambiguous state may + * advance one ticket from the next turn once, never a newly minted quota. */ + chooseInteractive(candidates: readonly T[]): T | undefined { + return this.interactiveSpent && !this.upgradeAdvance ? undefined : this.heads(candidates).get("interactive"); + } + + private heads(candidates: readonly T[]): Map { + const heads = new Map(), seen = new Set(); + for (const candidate of candidates) { + if (!(QUEUE_LANES as readonly string[]).includes(candidate.lane)) throw new Error(`Unknown queue candidate lane: ${candidate.lane}`); + if (!Number.isSafeInteger(candidate.enqueueSeq) || candidate.enqueueSeq < 1) throw new RangeError(`Invalid enqueue sequence for queue candidate: ${candidate.id}`); + const key = `${candidate.queue}:${candidate.id}`; + if (seen.has(key)) throw new Error(`Duplicate queue candidate: ${key}`); + seen.add(key); + const head = heads.get(candidate.lane); + if (!head || candidate.enqueueSeq < head.enqueueSeq || (candidate.enqueueSeq === head.enqueueSeq && + (candidate.queue.localeCompare(head.queue) || candidate.id.localeCompare(head.id)) < 0)) heads.set(candidate.lane, candidate); + } + return heads; + } + + /** One FIFO head may spend an already assigned ticket earlier in this round. */ + chooseAged( + candidates: readonly T[], nowMs: number, thresholdMs: number, excluded: ReadonlySet = new Set() + ): T | undefined { + if (this.aging.used || !Number.isFinite(nowMs)) return undefined; + const heads = this.heads(candidates); + return [...heads.values()].filter(candidate => !excluded.has(candidate.lane) && + !(this.aging.servedMask & (1 << QUEUE_LANES.indexOf(candidate.lane))) && + candidate.firstQueuedAt && Date.parse(candidate.firstQueuedAt) <= nowMs - thresholdMs) + .sort((a, b) => Date.parse(a.firstQueuedAt!) - Date.parse(b.firstQueuedAt!) || a.enqueueSeq - b.enqueueSeq)[0]; + } + + /** Call only after a successful lease CAS; a rejection/rollback consumes nothing. */ + commit(lane: QueueLane, promoted = false): void { + const index = this.turns.findIndex(turn => turn.lane === lane); + if (index < 0) throw new Error(`Unknown queue lane: ${lane}`); + if (promoted && (this.aging.used || this.aging.servedMask & (1 << QUEUE_LANES.indexOf(lane)))) { + throw new Error("Aging may only advance one unspent lane ticket per round"); + } + if (promoted) this.aging.used = true; + if (lane === "interactive" && this.upgradeAdvance) { + // Old releases did not distinguish an empty skip from a spent quota. + // Advance one ticket from the existing turn, not a fresh four-ticket + // quota. Persist the consumed flag with the successful lease CAS. + this.upgradeAdvance = false; + if (this.interactiveSpent) this.aging.servedMask |= 1; + } + for (const empty of this.empty) this.aging.servedMask |= 1 << QUEUE_LANES.indexOf(empty); + const skipped = this.turns.slice(0, index).filter(turn => this.empty.has(turn.lane)); + this.turns = [...this.turns.filter(turn => !skipped.includes(turn)), + ...skipped.map(turn => ({ lane: turn.lane, remaining: this.weights[turn.lane] }))]; + const turn = this.turns.find(item => item.lane === lane)!; + const start = QUEUE_LANES.slice(0, QUEUE_LANES.indexOf(lane)).reduce((sum, item) => sum + this.weights[item], 0); + const size = QUEUE_LANES.reduce((sum, item) => sum + this.weights[item], 0); + turn.remaining--; + this.cursor = (start + this.weights[lane] - turn.remaining) % size; + if (turn.remaining === 0) { + if (lane === "interactive") this.interactiveSpent = true; + this.aging.servedMask |= 1 << QUEUE_LANES.indexOf(lane); + this.turns.splice(this.turns.indexOf(turn), 1); + this.turns.push({ lane, remaining: this.weights[lane] }); + } + if (this.aging.servedMask === (1 << QUEUE_LANES.length) - 1) { + this.aging = { servedMask: 0, used: false }; + this.interactiveSpent = false; + this.upgradeAdvance = false; + } + } +} diff --git a/Memory/src/service/worker/fair-worker-queue.ts b/Memory/src/service/worker/fair-worker-queue.ts new file mode 100644 index 000000000..7a9fbd616 --- /dev/null +++ b/Memory/src/service/worker/fair-worker-queue.ts @@ -0,0 +1,394 @@ +import { createHash } from "node:crypto"; +import type { Repositories, EvolutionJobRecord, EmbeddingRetryRecord } from "../../storage/repositories.js"; +import type { QueueEligibilityOptions } from "../../storage/queue-eligibility.js"; +import type { SchedulingCandidate } from "../../storage/queue-scheduling-store.js"; +import { LaneQueueSelector } from "./lane-queue-selector.js"; +import { AdmissionLaneQueueSelector, type LaneQueueTurns } from "./admission-lane-queue-selector.js"; +import { QUEUE_LANES, type QueueLane } from "./job-policy-registry.js"; +import type { ResolvedWorkerSchedulingOptions } from "./worker-scheduling-options.js"; +import { WorkerAdmissionWaitSet } from "./worker-admission-wait-set.js"; + +export interface WorkAdmission { release(): void } +export interface WorkAdmissionBlocked { blockedOn: string } +export type FairClaimedWork = ({ queue: "main"; job: EvolutionJobRecord } | + { queue: "embedding_retry"; retry: EmbeddingRetryRecord }) & { admission?: WorkAdmission; + selectionReason?: "wrr" | "aging" | "interactive" | "legacy" }; +export interface FairWorkerClaimInput extends QueueEligibilityOptions { + at: string; limit: number; includeRetries: boolean; owner: string; + retryOwner: string; leaseSeconds: number; retryLeaseMs: number; + leaseFor?: (work: FairClaimedWork) => { durationSec: number }; + options: ResolvedWorkerSchedulingOptions; + priorityCohortOnly?: boolean; + /** Optional resumable admission scan. Without it the synchronous API is unchanged. */ + scan?: FairWorkerScan; + /** Acquire all permits synchronously, or report the capacity to wait for. */ + admit?: (work: FairClaimedWork) => WorkAdmission | WorkAdmissionBlocked | undefined; +} +const CURSOR_KEY = "worker.scheduler.wrr_cursor.v1"; +export const FAIR_CLAIM_BATCH_LIMIT = 200; + +export const FAIR_SCAN_CANDIDATE_LIMIT = 64; +type ScanBucket = { lane: SchedulingCandidate["lane"]; queue: SchedulingCandidate["queue"]; after: number; exhausted: boolean }; +type ScanContinuation = { + fingerprint: string; + buckets: ScanBucket[]; + candidates: SchedulingCandidate[]; + mainSeen: string[]; + retrySeen: string[]; + mainDone: boolean; + retryDone: boolean; + cohort?: number; + turns?: LaneQueueTurns; + agingSkippedLanes: QueueLane[]; + interactiveDenied?: boolean; +}; +/** Owned by one drain. Capacity releases only wake the matching parked rows; + * configuration, classification and due-time changes still invalidate the scan. */ +export interface FairWorkerScan { + inspected: number; + /** Waiters survive a completed pass or a filled execution slot. */ + blocked: boolean; + /** Whether to yield and immediately continue; independent of cursor lifetime. */ + more: boolean; + continuation?: ScanContinuation; + fingerprint?: string; + waiting: WorkerAdmissionWaitSet; +} +export function createFairWorkerScan(): FairWorkerScan { + return { inspected: 0, blocked: false, more: false, waiting: new WorkerAdmissionWaitSet() }; +} +export function releaseFairWorkerResources(scan: FairWorkerScan, resources: readonly string[]): void { + const lanes = scan.waiting.release(resources); + if (!scan.continuation) return; + scan.continuation.agingSkippedLanes = scan.continuation.agingSkippedLanes.filter(lane => !lanes.has(lane)); + if (lanes.has("interactive")) scan.continuation.interactiveDenied = false; +} + +function dependencyKey(job: EvolutionJobRecord): string | undefined { + if (job.sessionId && (["episode_assignment", "episode_finalize", "feedback_experience"].includes(job.jobType) || + (job.jobType === "trace_summary" && job.payload.turnJudge === true))) { + return JSON.stringify(["dependency", "session", job.sessionId]); + } + if (["l3_world_model_update", "work_memory_extract"].includes(job.jobType) && job.scopeKey) { + return JSON.stringify(["dependency", "scope", job.jobType, job.scopeKey]); + } + if (job.jobType === "reward" && typeof job.payload.l1MemoryId === "string") { + return JSON.stringify(["dependency", "memory", job.payload.l1MemoryId]); + } + return undefined; +} + +/** Completion only wakes related SQL dependencies, not every historical task. */ +export function releaseFairWorkerDependencies(scan: FairWorkerScan, work: FairClaimedWork): void { + const keys: string[] = []; + if (work.queue === "main") { + if (["l3_world_model_update", "work_memory_extract"].includes(work.job.jobType)) { + const key = dependencyKey(work.job); if (key) keys.push(key); + } + if (work.job.sessionId) keys.push(JSON.stringify(["dependency", "session", work.job.sessionId])); + if (work.job.targetMemoryId) keys.push(JSON.stringify(["dependency", "memory", work.job.targetMemoryId])); + } else keys.push(JSON.stringify(["dependency", "memory", work.retry.targetId])); + releaseFairWorkerResources(scan, keys); +} + +function normalizeFilter(values: readonly string[] | undefined): string[] | undefined { + return values === undefined ? undefined : [...new Set(values)].sort(); +} + +function scopeCursorKey(filters: QueueEligibilityOptions & { includeRetries: boolean }): string { + if (filters.targetMemoryIds === undefined && filters.allowedJobTypes === undefined && + !filters.excludedJobTypes?.length && filters.includeRetries) return CURSOR_KEY; + // A manual/targeted drain must not skip other lanes in the automatic round. + // Keep a persistent round for each equivalent scope, including limit=1 calls. + // Hash the full sets: truncating large target lists would merge distinct scopes. + const scope = JSON.stringify({ targets: filters.targetMemoryIds ?? null, + allowed: filters.allowedJobTypes ?? null, excluded: filters.excludedJobTypes ?? [], + includeRetries: filters.includeRetries }); + return `${CURSOR_KEY}.scope.${createHash("sha256").update(scope).digest("hex")}`; +} + +/** Both backing tables share one selection, one budget and one atomic cursor. */ +export function claimFairWorkerBatch( + repos: Pick, + input: FairWorkerClaimInput +): FairClaimedWork[] { + if (input.admit) return claimAdmittedWorkerBatch(repos, input); + const request = { ...input, + targetMemoryIds: normalizeFilter(input.targetMemoryIds), + allowedJobTypes: normalizeFilter(input.allowedJobTypes), + excludedJobTypes: normalizeFilter(input.excludedJobTypes), + includeDueFailed: true + }; + const cursorKey = scopeCursorKey(request); + return repos.transaction(() => { + repos.runtime.scheduling.refreshActivePolicies({ includeRetries: input.includeRetries }); + const weightsKey = QUEUE_LANES.map((lane) => input.options.laneWeights[lane]).join("/"); + const saved = repos.runtime.getKv(cursorKey)?.value as { weightsKey?: unknown; cursor?: unknown } | undefined; + const cursor = saved?.weightsKey === weightsKey && Number.isSafeInteger(saved.cursor) && Number(saved.cursor) >= 0 + ? Number(saved.cursor) : 0; + const selector = new LaneQueueSelector(input.options.laneWeights, cursor); + const limit = Math.min(FAIR_CLAIM_BATCH_LIMIT, input.limit); + const candidates = repos.runtime.scheduling.candidates({ ...request, limit, includeUnregistered: true }); + const claimed: FairClaimedWork[] = []; + while (claimed.length < limit && candidates.length > 0) { + const before = selector.getCursor(); + const chosen = selector.select(candidates, 1).selected[0]; + if (!chosen) break; + candidates.splice(candidates.findIndex((item) => item.queue === chosen.queue && item.id === chosen.id), 1); + if (chosen.queue === "main") { + const job = repos.runtime.claimQueuedJob(chosen.id, request); + if (job) claimed.push({ queue: "main", job, selectionReason: "wrr" }); + else selector.setCursor(before); + } else { + const retry = repos.runtime.claimEmbeddingRetryById(chosen.id, { + now: Date.parse(input.at), workerId: input.retryOwner, + leaseUntil: Date.parse(input.at) + input.retryLeaseMs, + targetMemoryIds: request.targetMemoryIds + }); + if (retry) claimed.push({ queue: "embedding_retry", retry, selectionReason: "wrr" }); + else selector.setCursor(before); + } + } + // Commit the cursor with the claims. Rollback cannot consume opportunities, + // and a process restart or a series of limit=1 calls cannot reset the round. + if (claimed.length > 0) repos.runtime.setKv(cursorKey, { weightsKey, cursor: selector.getCursor() }, input.at); + return claimed; + }); +} + +/** Admission and ID-CAS share the transaction; waiting tasks retain their FIFO position. */ +function claimAdmittedWorkerBatch( + repos: Pick, input: FairWorkerClaimInput +): FairClaimedWork[] { + const request = { ...input, + targetMemoryIds: normalizeFilter(input.targetMemoryIds), + allowedJobTypes: normalizeFilter(input.allowedJobTypes), + excludedJobTypes: normalizeFilter(input.excludedJobTypes), + includeDueFailed: input.options.algorithm === "wrr" + }; + const admissions: WorkAdmission[] = []; + let admissionBlocked = false; + const limit = Math.max(0, Math.min(FAIR_CLAIM_BATCH_LIMIT, Math.floor(input.limit))); + function tryClaim(work: FairClaimedWork, candidate?: SchedulingCandidate): FairClaimedWork | undefined { + const acquired = input.admit!(work); + if (!acquired) return undefined; + if ("blockedOn" in acquired) { + admissionBlocked = true; + if (candidate) input.scan?.waiting.park(candidate, acquired.blockedOn); + return undefined; + } + let released = false; + const admission = { release() { + if (released) return; + released = true; + acquired.release(); + } }; + admissions.push(admission); + if (work.queue === "main") { + const job = repos.runtime.claimQueuedJob(work.job.id, { ...request, + leaseSeconds: input.leaseFor?.(work).durationSec ?? input.leaseSeconds }); + if (job) return { queue: "main", job, admission }; + } else { + const retry = repos.runtime.claimEmbeddingRetryById(work.retry.id, { + now: Date.parse(input.at), workerId: input.retryOwner, + leaseUntil: Date.parse(input.at) + input.retryLeaseMs, + targetMemoryIds: request.targetMemoryIds + }); + if (retry) return { queue: "embedding_retry", retry, admission }; + } + admission.release(); + return undefined; + } + try { + return repos.transaction(() => { + if (limit === 0) return []; + const claimed: FairClaimedWork[] = []; + const fingerprint = JSON.stringify([scopeCursorKey(request), input.options, input.priorityCohortOnly ?? false, + repos.runtime.scheduling.getSelectionRevision()]); + const scan = input.scan; + if (scan) { + scan.inspected = 0; scan.blocked = false; scan.more = false; + if (scan.fingerprint !== fingerprint) scan.waiting.clear(); + scan.fingerprint = fingerprint; + } + const continuing = scan?.continuation?.fingerprint === fingerprint ? scan.continuation : undefined; + // Buffers are copied so transaction rollback cannot leave a half-consumed + // continuation. On rollback all drain-local wait references are discarded. + const state: ScanContinuation = continuing ? { ...continuing, mainDone: false, + buckets: continuing.buckets.map(bucket => ({ ...bucket, exhausted: false })), + candidates: [...continuing.candidates], agingSkippedLanes: [...continuing.agingSkippedLanes], mainSeen: [...continuing.mainSeen], retrySeen: [...continuing.retrySeen] + } : { fingerprint, buckets: QUEUE_LANES.flatMap(lane => + (input.includeRetries ? ["main", "embedding_retry"] as const : ["main"] as const) + .map(queue => ({ lane, queue, after: 0, exhausted: false }))), candidates: [], + mainSeen: [], retrySeen: [], mainDone: false, retryDone: false, agingSkippedLanes: [] }; + let inspected = 0; + const inspectLimit = scan ? FAIR_SCAN_CANDIDATE_LIMIT : Number.POSITIVE_INFINITY; + const saveScan = (more: boolean) => { + if (scan) { + scan.inspected = inspected; + scan.more = more || scan.waiting.hasPendingPrune; + scan.continuation = input.options.algorithm === "wrr" || more ? state : undefined; + scan.blocked = scan.waiting.size > 0; + } + }; + if (input.options.algorithm === "legacy") { + // Offsets become invalid when another macro task removes or leases an + // examined row. Exclude examined IDs instead, keeping the original SQL + // priority/recency ordering for every remaining eligible candidate. + mainPages: while (!state.mainDone && claimed.length < limit && inspected < inspectLimit) { + const pageLimit = Math.min(200, inspectLimit - inspected); + const page = repos.runtime.previewLegacyJobPage({ ...request, limit: pageLimit, excludedIds: state.mainSeen }); + for (const { job, priority } of page) { + if (claimed.length >= limit) break mainPages; + if (input.priorityCohortOnly && state.cohort !== undefined && priority !== state.cohort) { + state.mainDone = true; break mainPages; + } + inspected++; + state.mainSeen.push(job.id); + const result = tryClaim({ queue: "main", job }); + if (result) { claimed.push({ ...result, selectionReason: "legacy" }); state.cohort ??= priority; } + } + if (page.length < pageLimit) state.mainDone = true; + } + retryPages: while (state.mainDone && input.includeRetries && !state.retryDone && claimed.length < limit && inspected < inspectLimit) { + const pageLimit = Math.min(200, inspectLimit - inspected); + const page = repos.runtime.previewLegacyRetryPage({ now: Date.parse(input.at), + targetMemoryIds: request.targetMemoryIds, limit: pageLimit, excludedIds: state.retrySeen }); + for (const retry of page) { + if (claimed.length >= limit) break retryPages; + inspected++; + state.retrySeen.push(retry.id); + const result = tryClaim({ queue: "embedding_retry", retry }); + if (result) claimed.push({ ...result, selectionReason: "legacy" }); + } + if (page.length < pageLimit) state.retryDone = true; + } + saveScan(claimed.length < limit && (!state.mainDone || (input.includeRetries && !state.retryDone))); + return claimed; + } + + // An upgrade may leave thousands of older rows with stale policy. Finish + // their active-task projection before FIFO selection, but never do all writes in one + // event-loop turn. Recheck each slice for metadata dirtied between turns. + const beforeProjectionRevision = repos.runtime.scheduling.getSelectionRevision(); + const projection = repos.runtime.scheduling.refreshActivePolicies({ includeRetries: input.includeRetries, + ...(scan ? { maxRows: inspectLimit } : {}) }); + if (scan) { + inspected += projection.refreshed; + if (repos.runtime.scheduling.getSelectionRevision() !== beforeProjectionRevision) { + scan.waiting.clear(); + state.candidates = []; + for (const bucket of state.buckets) { bucket.after = 0; bucket.exhausted = false; } + } + if (projection.hasMore || inspected >= inspectLimit) { saveScan(true); return []; } + } + if (scan) { + inspected += scan.waiting.pruneHeads(candidate => repos.runtime.scheduling.isPendingCandidate(candidate), inspectLimit - inspected, + (candidate, resource) => resource.startsWith('["dependency",') && + repos.runtime.scheduling.isEligibleCandidate(candidate, input.at, request)); + if (inspected >= inspectLimit) { saveScan(true); return []; } + } + const cursorKey = scopeCursorKey(request); + const weightsKey = QUEUE_LANES.map(lane => input.options.laneWeights[lane]).join("/"); + const saved = repos.runtime.getKv(cursorKey)?.value as { weightsKey?: unknown; cursor?: unknown; turns?: unknown } | undefined; + const cursor = saved?.weightsKey === weightsKey && Number.isSafeInteger(saved.cursor) && Number(saved.cursor) >= 0 ? Number(saved.cursor) : 0; + const selector = new AdmissionLaneQueueSelector(input.options.laneWeights, cursor, + state.turns ?? (saved?.weightsKey === weightsKey ? saved.turns : undefined)); + const { buckets, candidates } = state; + function refill() { + for (const bucket of buckets) { + if (bucket.exhausted || candidates.some(candidate => candidate.queue === bucket.queue && candidate.lane === bucket.lane)) continue; + const page = repos.runtime.scheduling.candidatePage({ ...request, ...bucket, limit: 200, + afterEnqueueSeq: bucket.after, includeUnregistered: true, includeDependencyBlocked: true }); + if (page.length) bucket.after = page[page.length - 1]!.enqueueSeq; + bucket.exhausted = page.length < 200; + candidates.push(...page); + } + } + let exhausted = false; + const deniedLanes = new Set(); + while (claimed.length < limit && inspected < inspectLimit) { + refill(); + // A deleted ID may already have been reinserted with a new sequence. + // Discard stale ready heads before combining them with discovery pages. + const heads = scan?.waiting.heads() ?? []; + const ready = heads.filter(candidate => { + if (repos.runtime.scheduling.isPendingCandidate(candidate)) return true; + scan?.waiting.take(candidate); + return false; + }); + if (ready.length !== heads.length) { inspected++; continue; } + const readyKeys = new Set(ready.map(candidate => `${candidate.queue}:${candidate.id}`)); + const available = [...candidates.filter(candidate => !readyKeys.has(`${candidate.queue}:${candidate.id}`)), ...ready]; + // A denied lane consumes no ticket, but must let other lanes try their + // already assigned tickets before probing its next blocked row. + let selectable = available.filter(candidate => !deniedLanes.has(candidate.lane)); + if (!selectable.length) { deniedLanes.clear(); selectable = available; } + // A page that has never contained a row proves an empty lane. Exhausted + // pages after rejected candidates do not: their unspent turn survives. + selector.dropEmpty(new Set(QUEUE_LANES.filter(lane => + buckets.filter(bucket => bucket.lane === lane).every(bucket => bucket.after === 0 && bucket.exhausted) && + !selectable.some(candidate => candidate.lane === lane)))); + const interactive = state.interactiveDenied ? undefined : selector.chooseInteractive(selectable); + const aged = !interactive && input.options.aging?.enabled ? selector.chooseAged(selectable, Date.parse(input.at), + input.options.aging.thresholdMs, new Set(state.agingSkippedLanes)) : undefined; + const chosen = interactive ?? aged ?? selector.choose(selectable); + if (!chosen) { exhausted = true; break; } + const index = candidates.findIndex(candidate => candidate.queue === chosen.queue && candidate.id === chosen.id); + if (index >= 0) candidates.splice(index, 1); + inspected++; + let result: FairClaimedWork | undefined; + const acquiredBefore = admissions.length; + admissionBlocked = false; + const blocked = scan?.waiting.isBlocked(chosen); + if (blocked && scan) scan.blocked = true; + if (!blocked) scan?.waiting.take(chosen); + const eligible = !blocked && repos.runtime.scheduling.isEligibleCandidate(chosen, input.at, request); + if (!blocked && !eligible && scan && chosen.queue === "main") { + const job = repos.runtime.getJob(chosen.id); + const key = job && dependencyKey(job); + if (key && repos.runtime.scheduling.isPendingCandidate(chosen)) { + scan.waiting.park(chosen, key); admissionBlocked = true; + } + } + if (eligible && chosen.queue === "main") { + const job = repos.runtime.getJob(chosen.id); + if (job) result = tryClaim({ queue: "main", job }, chosen); + } else if (eligible && chosen.queue === "embedding_retry") { + const retry = repos.runtime.getEmbeddingRetry(chosen.id); + if (retry) result = tryClaim({ queue: "embedding_retry", retry }, chosen); + } + if (!result && (blocked || admissionBlocked)) deniedLanes.add(chosen.lane); + if (!result && admissions.length > acquiredBefore && scan && repos.runtime.scheduling.isEligibleCandidate(chosen, input.at, request)) { + // A transient CAS miss must not permanently hide a still-runnable row. + // Retry on the next fill, preserving its original sequence. + scan.waiting.park(chosen, "claim_retry"); + } + if (result) { + claimed.push({ ...result, selectionReason: interactive ? "interactive" : aged ? "aging" : "wrr" }); + selector.commit(chosen.lane, aged !== undefined); + } + else if (aged && admissions.length === acquiredBefore) { + // A busy aged lane must not make every later candidate in that lane + // jump ahead of other runnable work. Retry aging after admission state + // changes; ordinary FIFO scanning remains available in this fill. + state.agingSkippedLanes.push(chosen.lane); + } + if (interactive && !result && admissions.length === acquiredBefore) state.interactiveDenied = true; + } + scan?.waiting.release(["claim_retry"]); + state.turns = selector.snapshot(); + saveScan(!exhausted && claimed.length < limit); + if (claimed.length) repos.runtime.setKv(cursorKey, { weightsKey, cursor: selector.getCursor(), turns: selector.snapshot() }, input.at); + return claimed; + }); + } catch (error) { + if (input.scan) { input.scan.more = false; input.scan.continuation = undefined; input.scan.waiting.clear(); } + // Rollback also covers successful earlier CAS operations. Returning all + // permits here prevents a failed transaction from leaking execution slots. + for (const admission of admissions) { + try { admission.release(); } catch { /* Keep releasing the remaining permits and preserve the transaction error. */ } + } + throw error; + } +} diff --git a/Memory/src/service/worker/job-handlers.ts b/Memory/src/service/worker/job-handlers.ts index 88a569554..5f82a5b0a 100644 --- a/Memory/src/service/worker/job-handlers.ts +++ b/Memory/src/service/worker/job-handlers.ts @@ -25,7 +25,6 @@ import { import { episodeTitleIsCurrent } from "../episode-title/episode-title-service.js"; import { memoryHasImportPipeline } from "../import/import-job-processor.js"; import { observationMemoryState } from "../observation-memory-state.js"; -import { isTerminalL3WorldModelError } from "../evolution/l3-world-model-pipeline.js"; import { isNeutralFeedback, turnJudgeJobFlags } from "../evolution/turn-judge.js"; import { namespaceForMemory, @@ -208,6 +207,16 @@ export function enqueueJob( input: EnqueueJobInput ): EvolutionJobRecord { const at = input.createdAt ?? deps.nowIso(); + const payload = { ...input.payload }; + if (input.jobType === "episode_title") { + // Title jobs have no target memory. Only a wholly historical episode can + // inherit the historical lane; mixed or incomplete episodes stay interactive. + delete payload.captureOrigin; + const episode = input.episodeId ? deps.repos.runtime.getEpisode(input.episodeId) : undefined; + if (episode && deps.repos.memories.areAllHistoricalCaptureMemories(episode.l1MemoryIds)) { + payload.captureOrigin = "agent_source_scan"; + } + } const job = deps.repos.runtime.enqueueJob({ id: newId("job"), jobType: input.jobType, @@ -217,7 +226,7 @@ export function enqueueJob( sessionId: input.sessionId, episodeId: input.episodeId, targetMemoryId: input.targetMemoryId, - payload: input.payload ?? {}, + payload, attempts: 0, maxAttempts: input.maxAttempts ?? 3, createdAt: at, @@ -256,19 +265,7 @@ export async function processJob( await deps.processors.evolution.abstractL3(job); return; case "l3_world_model_update": - try { - await deps.processors.evolution.updateL3WorldModel(job); - } catch (error) { - if (isTerminalL3WorldModelError(error)) { - deps.repos.runtime.failJob( - job.id, - error instanceof Error ? error.message : String(error), - deps.nowIso(), - true - ); - } - throw error; - } + await deps.processors.evolution.updateL3WorldModel(job); return; case "project_environment_profile": await deps.processors.evolution.updateProjectEnvironment(job); @@ -619,8 +616,13 @@ export function classifyProcessingError(error: unknown): { if (!hasStructuredCode && /\b40309\b/.test(normalized)) { return { code: "40309", retryAction: "open_settings" }; } - // Providers can report shared model/account failures as HTTP 400. Classify - // their reason before generic 4xx handling, which permits item isolation. + // Temporary HTTP failures can mention API keys; keep their retry semantics. + if (error instanceof ModelHttpError && (error.httpStatus === 408 || error.httpStatus === 429 || + (error.httpStatus >= 500 && error.httpStatus < 600))) { + return { code: "transient_provider_error", retryAction: "retry" }; + } + // Shared HTTP 400 configuration/quota failures must not fan out into one + // request per item when an embedding batch is rejected. if (error instanceof ModelHttpError) { const providerCode = error.errorCode?.toLowerCase().replace(/[\s_-]/g, ""); if (providerCode === "operationnotsupported" || providerCode === "insufficientquota" || @@ -630,6 +632,7 @@ export function classifyProcessingError(error: unknown): { if (providerCode === "ratelimitexceeded" || /rate[\s_-]*limit/.test(normalized)) { return { code: "transient_provider_error", retryAction: "retry" }; } + } if (/api.?key|unauthorized|forbidden|\b401\b|\b403\b|\b404\b|model.+not configured|missing.+model|expected json|html instead of json|configured model endpoint/.test(normalized)) { return { code: "model_configuration", retryAction: "open_settings" }; diff --git a/Memory/src/service/worker/job-policy-registry.ts b/Memory/src/service/worker/job-policy-registry.ts new file mode 100644 index 000000000..7788b84ac --- /dev/null +++ b/Memory/src/service/worker/job-policy-registry.ts @@ -0,0 +1,172 @@ +import type { JobType } from "../../types.js"; +import { resolveWorkerLeaseOptions, type ResolvedWorkerLeaseOptions } from "./worker-lease-options.js"; + +export const JOB_POLICY_VERSION = 4; +export const QUEUE_LANES = ["interactive", "repair_retry", "import", "evolution", "maintenance"] as const; +export type QueueLane = typeof QUEUE_LANES[number]; +export const RESOURCE_CLASSES = ["summary", "embedding", "evolution_llm", "database", "scan"] as const; +export type ResourceClass = typeof RESOURCE_CLASSES[number]; +export type PolicyJobType = JobType | "embedding_retry"; +export type JobOperationMode = "default" | "onboarding_first_report" | "manual_retry" | + "startup_repair" | "historical_import" | "maintenance"; + +/** The scheduler needs metadata, never a memory's conversation text or vectors. */ +export interface JobPolicyTarget { + agentId?: string; + memoryLayer?: string; + memoryKind?: string; + tags?: readonly string[]; + pluginAlgorithm?: string; + /** Scheduling provenance only; never identifies an import processing pipeline. */ + captureOrigin?: string; +} + +export interface JobPolicyInput { + jobType: string; + payload?: Record; + target?: JobPolicyTarget; + operationMode?: JobOperationMode; +} + +export interface JobPolicy { + queueLane: QueueLane; + resourceClass: ResourceClass; + /** Audit/urgency information; this is not a replacement for lane fairness or FIFO. */ + basePriority: number; + policyVersion: number; + lease: Readonly<{ durationSec: number; heartbeatMs: number }>; + /** Describes future admission requirements; phase 3 does not acquire permits. */ + concurrency: Readonly<{ + providerRoles: readonly ("summary" | "evolution" | "embedding")[]; + scope: "target" | "user_project" | "episode" | "none"; + batchable: boolean; + }>; +} + +type PolicyDefinition = Omit; +type ProviderRole = JobPolicy["concurrency"]["providerRoles"][number]; +type Scope = JobPolicy["concurrency"]["scope"]; + +function definition( + queueLane: QueueLane, + resourceClass: ResourceClass, + basePriority: number, + providerRoles: readonly ProviderRole[], + scope: Scope = "target", + batchable = false +): PolicyDefinition { + return Object.freeze({ + queueLane, resourceClass, basePriority, + concurrency: Object.freeze({ providerRoles: Object.freeze([...providerRoles]), scope, batchable }) + }); +} + +// Record makes a newly added JobType a compilation error until its policy is reviewed. +// Resource classes describe the principal work; providerRoles also records secondary +// model calls (for example Work Memory's retrieval query embedding). +const DEFINITIONS: Readonly> = Object.freeze({ + episode_assignment: definition("interactive", "summary", 1, ["summary"], "none"), + episode_finalize: definition("interactive", "database", 1, [], "none"), + episode_idle_close: definition("maintenance", "database", 10, [], "episode"), + episode_title: definition("interactive", "summary", 15, ["summary"], "episode"), + trace_summary: definition("interactive", "summary", 1, ["summary", "evolution"]), + import_summary: definition("import", "summary", 2, ["summary", "evolution"]), + observation_extract: definition("interactive", "summary", 1, ["summary"]), + embedding: definition("evolution", "embedding", 3, ["embedding"], "target", true), + user_memory_embedding: definition("interactive", "embedding", 3, ["embedding"], "target", true), + embedding_retry: definition("repair_retry", "embedding", 0, ["embedding"], "target", true), + reflection: definition("evolution", "evolution_llm", 20, ["summary", "evolution"], "episode"), + reward: definition("evolution", "evolution_llm", 30, ["summary", "evolution"], "episode"), + span_big_turn: definition("evolution", "summary", 35, ["summary"]), + negative_experience: definition("evolution", "evolution_llm", 100, ["evolution"], "episode"), + decision_repair: definition("repair_retry", "evolution_llm", 25, ["evolution"]), + l2_association: definition("evolution", "database", 40, []), + l2_induction: definition("evolution", "evolution_llm", 50, ["evolution"]), + l3_abstraction: definition("evolution", "database", 60, [], "user_project"), + l3_world_model_update: definition("evolution", "evolution_llm", 60, ["evolution"], "user_project"), + project_environment_profile: definition("maintenance", "evolution_llm", 55, ["evolution"], "user_project"), + skill_crystallization: definition("evolution", "evolution_llm", 70, ["evolution"]), + skill_cluster_assign: definition("evolution", "evolution_llm", 72, ["embedding", "evolution"], "user_project"), + skill_batch_evolve: definition("evolution", "evolution_llm", 73, ["evolution"], "user_project"), + skill_trial_resolve: definition("evolution", "database", 80, [], "episode"), + work_memory_extract: definition("evolution", "summary", 100, ["summary", "embedding"], "user_project"), + work_memory_idle_flush: definition("evolution", "database", 100, [], "user_project"), + feedback_experience: definition("evolution", "evolution_llm", 100, ["evolution", "embedding"]) +}); + +export const REGISTERED_JOB_TYPES = Object.freeze(Object.keys(DEFINITIONS) as PolicyJobType[]); +const FIRST_REPORT_JOB_TYPES = new Set(["trace_summary", "import_summary", "observation_extract", "embedding", "embedding_retry"]); +const STARTUP_REPAIR_SOURCES = new Set([ + "startup.processing_repair", "startup.placeholder_summary_repair", + "startup.retrieval_document_v2", "startup.observation_backfill" +]); +const SUMMARY_JOB_TYPES = new Set(["trace_summary", "import_summary", "observation_extract"]); + +export class UnknownJobPolicyError extends Error { + constructor(readonly jobType: string) { + super(`No scheduling policy registered for job type: ${jobType}`); + this.name = "UnknownJobPolicyError"; + } +} + +export function isOnboardingFirstReportTarget(target: JobPolicyTarget | undefined): boolean { + return target?.agentId?.toLowerCase() === "memmy-onboarding" || + target?.tags?.some((tag) => ["first-encounter-report", "onboarding-report"].includes(tag.toLowerCase())) === true; +} + +export function isImportedPolicyTarget(target: JobPolicyTarget | undefined): boolean { + return target?.pluginAlgorithm?.startsWith("memory.add.import_async.") === true || + target?.tags?.some((tag) => tag.toLowerCase() === "agent-source") === true; +} + +export class JobPolicyRegistry { + constructor(private readonly leases: Readonly = resolveWorkerLeaseOptions()) {} + + resolve(input: JobPolicyInput): JobPolicy { + if (!Object.hasOwn(DEFINITIONS, input.jobType)) throw new UnknownJobPolicyError(input.jobType); + const base = DEFINITIONS[input.jobType as PolicyJobType]; + const source = typeof input.payload?.source === "string" ? input.payload.source : ""; + // Embedding producers use `reason`, while summary/observation producers use + // `source`. Both identify the operation; neither contains the model provider. + const reason = typeof input.payload?.reason === "string" ? input.payload.reason : ""; + let queueLane = base.queueLane; + let basePriority = base.basePriority; + + // First reports are user-visible results, even when the original data came + // from a historical scan or is being repaired on startup. + if (FIRST_REPORT_JOB_TYPES.has(input.jobType) && + (input.operationMode === "onboarding_first_report" || isOnboardingFirstReportTarget(input.target))) { + queueLane = "interactive"; + basePriority = -100; + } else if (input.operationMode === "manual_retry" || source === "memory.processing.manual_retry") { + queueLane = "repair_retry"; + basePriority = 0; + } else if (input.operationMode === "startup_repair" || input.operationMode === "maintenance" || + STARTUP_REPAIR_SOURCES.has(source) || STARTUP_REPAIR_SOURCES.has(reason)) { + queueLane = "maintenance"; + basePriority = 90; + } else if (input.operationMode === "historical_import" || + (["episode_title", "episode_assignment", "episode_finalize"].includes(input.jobType) && input.payload?.captureOrigin === "agent_source_scan") || + ((SUMMARY_JOB_TYPES.has(input.jobType) || input.jobType === "embedding") && + (source === "agent_source.scan.summary_stage" || reason === "import.summary.updated" || + input.target?.captureOrigin === "agent_source_scan" || isImportedPolicyTarget(input.target)))) { + queueLane = "import"; + basePriority = 2; + } else if (input.jobType === "embedding") { + if (isImportedPolicyTarget(input.target)) { + queueLane = "import"; + basePriority = 2; + } else if (input.target?.memoryLayer === "L1" || input.target?.memoryKind === "user_memory") { + queueLane = "interactive"; + basePriority = 1; + } + } + + return Object.freeze({ + ...base, queueLane, basePriority, policyVersion: JOB_POLICY_VERSION, + lease: input.jobType === "embedding_retry" ? this.leases.embeddingRetry : this.leases.resources[base.resourceClass] + }); + } +} + +export const jobPolicyRegistry = new JobPolicyRegistry(); diff --git a/Memory/src/service/worker/lane-queue-selector.ts b/Memory/src/service/worker/lane-queue-selector.ts new file mode 100644 index 000000000..12008a10d --- /dev/null +++ b/Memory/src/service/worker/lane-queue-selector.ts @@ -0,0 +1,130 @@ +import { QUEUE_LANES, type QueueLane } from "./job-policy-registry.js"; + +/** User-confirmed starting weights, shared by production and shadow selection. */ +export const DEFAULT_LANE_WEIGHTS: Readonly> = Object.freeze({ + interactive: 4, + repair_retry: 3, + import: 2, + evolution: 2, + maintenance: 1 +}); + +export interface LaneQueueCandidate { + queue: "main" | "embedding_retry"; + id: string; + lane: QueueLane; + enqueueSeq: number; + basePriority: number; +} + +export interface LaneQueueSelection { + selected: readonly T[]; + nextCursor: number; + algorithm: "wrr_runnable_fifo"; + weights: Readonly>; +} + +/** Validate all five lanes, returning a detached immutable configuration. */ +export function validateLaneWeights(weights: unknown): Readonly> { + if (weights === null || typeof weights !== "object" || Array.isArray(weights)) { + throw new TypeError("Lane weights must be an object containing every registered lane"); + } + for (const key of Object.keys(weights)) { + if (!(QUEUE_LANES as readonly string[]).includes(key)) throw new RangeError(`Unknown lane weight: ${key}`); + } + const normalized = {} as Record; + for (const lane of QUEUE_LANES) { + const weight = (weights as Record)[lane]; + if (!Object.hasOwn(weights, lane) || typeof weight !== "number" || + !Number.isSafeInteger(weight) || weight < 1 || weight > 1_024) { + throw new RangeError(`Invalid lane weight for ${lane}`); + } + normalized[lane] = weight; + } + return Object.freeze(normalized); +} + +/** + * Pure selection shared by production and shadow execution. Candidates must + * already pass readiness checks; claims and durable cursor commits belong to + * the caller's transaction. No input records are modified. + */ +export class LaneQueueSelector { + private readonly schedule: readonly QueueLane[]; + private readonly weights: Readonly>; + private cursor = 0; + + constructor(weights: Readonly> = DEFAULT_LANE_WEIGHTS, initialCursor = 0) { + this.weights = validateLaneWeights(weights); + const schedule: QueueLane[] = []; + for (const lane of QUEUE_LANES) { + const weight = this.weights[lane]; + schedule.push(...Array(weight).fill(lane)); + } + this.schedule = Object.freeze(schedule); + this.setCursor(initialCursor); + } + + getCursor(): number { + return this.cursor; + } + + /** Accept persisted cursors after a weight change by normalizing the offset. */ + setCursor(cursor: number): void { + if (!Number.isSafeInteger(cursor) || cursor < 0) throw new RangeError(`Invalid queue cursor: ${cursor}`); + this.cursor = cursor % this.schedule.length; + } + + preview(candidates: readonly T[], limit: number): LaneQueueSelection { + return this.select(candidates, limit, { advanceCursor: false }); + } + + select( + candidates: readonly T[], limit: number, options: { advanceCursor?: boolean } = {} + ): LaneQueueSelection { + if (!Number.isSafeInteger(limit) || limit < 0) throw new RangeError(`Invalid queue selection limit: ${limit}`); + const buckets = new Map(QUEUE_LANES.map((lane) => [lane, []])); + const identities = new Set(); + for (const candidate of candidates) { + const bucket = buckets.get(candidate.lane); + if (!bucket) throw new Error(`Unknown queue candidate lane: ${candidate.lane}`); + if (!Number.isSafeInteger(candidate.enqueueSeq) || candidate.enqueueSeq < 1) { + throw new RangeError(`Invalid enqueue sequence for queue candidate: ${candidate.id}`); + } + const identity = `${candidate.queue}:${candidate.id}`; + if (identities.has(identity)) throw new Error(`Duplicate queue candidate: ${identity}`); + identities.add(identity); + bucket.push(candidate); + } + for (const bucket of buckets.values()) { + bucket.sort((a, b) => a.enqueueSeq - b.enqueueSeq || + a.queue.localeCompare(b.queue) || a.id.localeCompare(b.id)); + } + + const offsets = new Map(); + const selected: T[] = []; + let nextCursor = this.cursor; + while (selected.length < Math.min(limit, candidates.length)) { + let found = false; + // Empty/blocked lanes never consume execution capacity. A full schedule + // traversal without a candidate means the available input is exhausted. + for (let scanned = 0; scanned < this.schedule.length; scanned += 1) { + const lane = this.schedule[nextCursor]!; + nextCursor = (nextCursor + 1) % this.schedule.length; + const offset = offsets.get(lane) ?? 0; + const candidate = buckets.get(lane)![offset]; + if (!candidate) continue; + selected.push(candidate); + offsets.set(lane, offset + 1); + found = true; + break; + } + if (!found) break; + } + if (options.advanceCursor !== false) this.cursor = nextCursor; + return Object.freeze({ + selected: Object.freeze(selected), nextCursor, + algorithm: "wrr_runnable_fifo", weights: this.weights + }); + } +} diff --git a/Memory/src/service/worker/shadow-queue-selector.ts b/Memory/src/service/worker/shadow-queue-selector.ts new file mode 100644 index 000000000..e7f112ced --- /dev/null +++ b/Memory/src/service/worker/shadow-queue-selector.ts @@ -0,0 +1,7 @@ +/** Compatibility names for the stage-3 shadow observer. */ +export { + DEFAULT_LANE_WEIGHTS, + LaneQueueSelector as ShadowQueueSelector, + type LaneQueueCandidate as ShadowQueueCandidate, + type LaneQueueSelection as ShadowQueueSelection +} from "./lane-queue-selector.js"; diff --git a/Memory/src/service/worker/worker-admission-wait-set.ts b/Memory/src/service/worker/worker-admission-wait-set.ts new file mode 100644 index 000000000..512be26f8 --- /dev/null +++ b/Memory/src/service/worker/worker-admission-wait-set.ts @@ -0,0 +1,123 @@ +import type { SchedulingCandidate } from "../../storage/queue-scheduling-store.js"; +import { QUEUE_LANES, type QueueLane } from "./job-policy-registry.js"; + +type Entry = { candidate: SchedulingCandidate; group: { ready: boolean } }; +type BlockedGroup = { ready: boolean; root?: Node }; +type Node = { entry: Entry; child?: Node; sibling?: Node }; +const keyOf = (candidate: SchedulingCandidate) => `${candidate.queue}:${candidate.id}:${candidate.enqueueSeq}`; +function before(a: Node, b: Node): boolean { + const x = a.entry.candidate, y = b.entry.candidate; + return x.enqueueSeq < y.enqueueSeq || (x.enqueueSeq === y.enqueueSeq && + (x.queue.localeCompare(y.queue) || x.id.localeCompare(y.id)) < 0); +} +/** A meldable min-heap: releasing a resource moves its waiters in O(1), + * without walking a large backlog in the resource-release callback. */ +function meld(a: Node | undefined, b: Node | undefined): Node | undefined { + if (!a) return b; + if (!b) return a; + if (before(b, a)) [a, b] = [b, a]; + b.sibling = a.child; a.child = b; + return a; +} +function pop(root: Node): Node | undefined { + const pairs: Node[] = []; + let child = root.child; + while (child) { + const a = child, b = a.sibling; + child = b?.sibling; + a.sibling = undefined; + if (b) b.sibling = undefined; + pairs.push(meld(a, b)!); + } + let next: Node | undefined; + for (let i = pairs.length - 1; i >= 0; i--) next = meld(pairs[i], next); + return next; +} + +/** Drain-local references only; SQLite remains authoritative for eligibility and + * lease CAS. Waiting never changes the persisted enqueue sequence or owns permits. */ +export class WorkerAdmissionWaitSet { + private readonly entries = new Map(); + private readonly blocked = new Map>(); + private readonly ready = new Map(); + + private pruning?: Generator<{ resource: string; lane: QueueLane; lanes: Map; group: BlockedGroup }>; + private *pruneGroups() { + for (const [resource, lanes] of this.blocked) for (const [lane, group] of lanes) yield { resource, lanes, lane, group }; + } + + get size(): number { return this.entries.size; } + /** Finish this bounded sweep before sleeping; a completed sweep stays idle + * until the next refill/wake initiates another one. */ + get hasPendingPrune(): boolean { return this.pruning !== undefined; } + clear(): void { this.entries.clear(); this.blocked.clear(); this.ready.clear(); this.pruning = undefined; } + isBlocked(candidate: SchedulingCandidate): boolean { + const entry = this.entries.get(keyOf(candidate)); + return entry !== undefined && !entry.group.ready; + } + park(candidate: SchedulingCandidate, resource: string): void { + if (this.entries.has(keyOf(candidate))) return; + let lanes = this.blocked.get(resource); + if (!lanes) { lanes = new Map(); this.blocked.set(resource, lanes); } + let group = lanes.get(candidate.lane); + if (!group) { group = { ready: false }; lanes.set(candidate.lane, group); } + const entry = { candidate, group }; + this.entries.set(keyOf(candidate), entry); + group.root = meld(group.root, { entry }); + } + release(resources: readonly string[]): Set { + const lanes = new Set(); + for (const resource of resources) { + const groups = this.blocked.get(resource); + if (!groups) continue; + this.blocked.delete(resource); + for (const [lane, group] of groups) { + group.ready = true; + this.ready.set(lane, meld(this.ready.get(lane), group.root)); + group.root = undefined; + lanes.add(lane); + } + } + return lanes; + } + /** Probe only heap heads, never walk the stable blocked backlog on a refill. + * Terminal/deleted entries are removed in bounded slices. */ + pruneHeads(isPending: (candidate: SchedulingCandidate) => boolean, limit: number, + isReady?: (candidate: SchedulingCandidate, resource: string) => boolean): number { + let removed = 0, checked = 0; + this.pruning ??= this.pruneGroups(); + while (checked < limit) { + const next = this.pruning.next(); + if (next.done) { this.pruning = undefined; break; } + const { resource, lane, lanes, group } = next.value; + if (group.ready || !group.root) continue; + while (group.root && checked < limit) { + checked++; + if (isPending(group.root.entry.candidate)) { + if (isReady?.(group.root.entry.candidate, resource)) this.release([resource]); + break; + } + const root = group.root; + this.entries.delete(keyOf(root.entry.candidate)); + group.root = pop(root); removed++; + } + if (!group.root) lanes.delete(lane); + if (!lanes.size) this.blocked.delete(resource); + // Restart at this resource next slice after a full batch of removals. + if (removed === limit) { this.pruning = undefined; break; } + } + return removed; + } + heads(): SchedulingCandidate[] { + return QUEUE_LANES.flatMap(lane => { + const root = this.ready.get(lane); + return root ? [root.entry.candidate] : []; + }); + } + take(candidate: SchedulingCandidate): void { + const root = this.ready.get(candidate.lane); + if (!root || keyOf(root.entry.candidate) !== keyOf(candidate)) return; + this.ready.set(candidate.lane, pop(root)); + this.entries.delete(keyOf(candidate)); + } +} diff --git a/Memory/src/service/worker/worker-concurrency.ts b/Memory/src/service/worker/worker-concurrency.ts new file mode 100644 index 000000000..f84cb590d --- /dev/null +++ b/Memory/src/service/worker/worker-concurrency.ts @@ -0,0 +1,387 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { createHash } from "node:crypto"; +import { RESOURCE_CLASSES, type ResourceClass } from "./job-policy-registry.js"; +import { observeWorker, type AdmissionRejectReason, type WorkerObserver } from "./worker-observability.js"; + +export interface WorkerConcurrencyOptions { + maxActiveJobs?: number; + resourceLimits?: Partial>>; + localModelConcurrency?: number; + cloudModelConcurrency?: number; + embeddingRequestConcurrency?: number; + embeddingBatchSize?: number; +} +export interface WorkerConcurrencyLimits { + readonly maxActiveJobs: number; + readonly maxBackgroundJobs: number; + readonly interactiveReservedSlots: number; + readonly resourceLimits: Readonly>; + readonly localModelConcurrency: number; + readonly cloudModelConcurrency: number; + readonly embeddingRequestConcurrency: number; + readonly embeddingBatchSize: number; +} +export interface ProviderRequirement { key: string; limit: number } +/** Opaque, process-local keys used only to wake the waiters of a changed capacity. */ +export function workerCapacityKey(kind: "global" | "background" | "resource" | "scope" | "embedding_request" | "provider", key?: string): string { + return JSON.stringify([kind, key ?? null]); +} +export interface WorkerRequirements { + resourceClass: ResourceClass; + providers: readonly ProviderRequirement[]; + scopeKeys: readonly string[]; + embedding?: boolean; + /** Persisted scheduling lane; only interactive may use the reserved slot. */ + queueLane?: string; +} +export interface WorkerPermit { + release(): void; + run(operation: () => T): T; +} +export interface PreparedWorkerExecution { + providers: readonly ProviderRequirement[]; + embedding?: boolean; + /** Opaque fingerprint of the entire model snapshot; never log credentials. */ + batchKey?: string; + run(operation: () => T): T; +} +export interface WorkerProviderDescriptor { + readonly providers: readonly ProviderRequirement[]; + readonly local: boolean; +} +interface ReservationGroup { + providers: readonly ProviderRequirement[]; + embedding: boolean; + resourceClass: ResourceClass; + members: number; + logicalJobs: number; + activeRequests: number; + serial: Map>; +} +interface PermitState { + requirements: WorkerRequirements; + logicalJobs: number; + released: boolean; + group: ReservationGroup; +} +const defaults: WorkerConcurrencyLimits = Object.freeze({ + maxActiveJobs: 4, maxBackgroundJobs: 3, interactiveReservedSlots: 1, + resourceLimits: Object.freeze({ summary: 4, embedding: 4, evolution_llm: 4, database: 1, scan: 1 }), + localModelConcurrency: 1, cloudModelConcurrency: 2, embeddingRequestConcurrency: 1, embeddingBatchSize: 16 +}); +function positive(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1 || Number(value) > 1024) { + throw new RangeError(`${name} must be an integer between 1 and 1024`); + } + return Number(value); +} +export function resolveWorkerConcurrencyOptions( + options: WorkerConcurrencyOptions = {}, env: Readonly> = process.env +): WorkerConcurrencyLimits { + const number = (key: Exclude, name: string) => { + const raw = options[key] ?? (env[name] === undefined ? defaults[key] : Number(env[name])); + return positive(raw, key); + }; + let resources: unknown = options.resourceLimits; + if (resources === undefined && env.MEMMY_WORKER_RESOURCE_LIMITS !== undefined) { + try { resources = JSON.parse(env.MEMMY_WORKER_RESOURCE_LIMITS); } + catch { throw new TypeError("MEMMY_WORKER_RESOURCE_LIMITS must be a JSON object"); } + } + if (resources !== undefined && (!resources || typeof resources !== "object" || Array.isArray(resources))) { + throw new TypeError("resourceLimits must be an object"); + } + const resourceLimits = { ...defaults.resourceLimits }; + for (const [key, value] of Object.entries(resources ?? {})) { + if (!(RESOURCE_CLASSES as readonly string[]).includes(key)) throw new TypeError(`Unknown resource class: ${key}`); + resourceLimits[key as ResourceClass] = positive(value, `resourceLimits.${key}`); + } + const maxActiveJobs = number("maxActiveJobs", "MEMMY_WORKER_MAX_ACTIVE_JOBS"); + // A single-slot configuration must still permit background progress. + const interactiveReservedSlots = maxActiveJobs > 1 ? 1 : 0; + return Object.freeze({ + maxActiveJobs, interactiveReservedSlots, maxBackgroundJobs: maxActiveJobs - interactiveReservedSlots, + resourceLimits: Object.freeze(resourceLimits), + localModelConcurrency: number("localModelConcurrency", "MEMMY_WORKER_LOCAL_MODEL_CONCURRENCY"), + cloudModelConcurrency: number("cloudModelConcurrency", "MEMMY_WORKER_CLOUD_MODEL_CONCURRENCY"), + embeddingRequestConcurrency: number("embeddingRequestConcurrency", "MEMMY_WORKER_EMBEDDING_REQUEST_CONCURRENCY"), + embeddingBatchSize: number("embeddingBatchSize", "MEMMY_WORKER_EMBEDDING_BATCH_SIZE") + }); +} + +/** Provider identity omits credentials, query parameters, headers and user text. */ +export function providerDescriptor(config: { + provider?: string; sourceProvider?: string; endpoint?: string; model?: string; mode?: string; + actualModelContext?: { source?: string }; +}, limits: Pick = defaults): WorkerProviderDescriptor { + const source = (config.sourceProvider || config.provider || "unknown").trim().toLowerCase(); + const defaultEndpoints: Readonly> = { + openai_compatible: "https://api.openai.com/v1", openai_responses: "https://api.openai.com/v1", gemini: "https://generativelanguage.googleapis.com/v1beta", + anthropic: "https://api.anthropic.com/v1/messages", cohere: "https://api.cohere.com/v2/embed", + voyage: "https://api.voyageai.com/v1/embeddings", mistral: "https://api.mistral.ai/v1/embeddings" + }; + let endpoint = (config.endpoint || defaultEndpoints[config.provider ?? ""] || "").trim().replace(/\/+$/, ""); + // Mirror HttpLlmClient's actual URL construction before deriving the root: + // a bare Anthropic base and its default /v1/messages endpoint send to the + // same place, while an explicit /v1 base actually sends /v1/v1/messages. + if (config.provider === "anthropic" && !endpoint.endsWith("/v1/messages")) endpoint += "/v1/messages"; + let hostname = ""; + if (endpoint) { + try { + const url = new URL(endpoint); + hostname = url.hostname.toLowerCase(); + if (hostname === "localhost" || hostname === "[::1]" || /^127\./.test(hostname)) url.hostname = "localhost"; + url.username = ""; url.password = ""; url.search = ""; url.hash = ""; + if (config.provider === "openai_compatible") url.pathname = url.pathname.replace(/\/(?:chat\/completions|embeddings)\/?$/, ""); + if (config.provider === "openai_responses") url.pathname = url.pathname.replace(/\/responses\/?$/, ""); + // Native Mistral embeddings share the API service with compatible chat. + // Keep the host and preceding base path so independent proxies stay separate. + if (config.provider === "mistral") url.pathname = url.pathname.replace(/\/embeddings\/?$/, ""); + if (config.provider === "anthropic") url.pathname = url.pathname.replace(/\/messages\/?$/, ""); + endpoint = url.toString().replace(/\/+$/, ""); + } catch { throw new TypeError("Invalid model provider endpoint"); } + } + const local = config.actualModelContext?.source !== "account" && (config.mode === "local" + || ["local", "local_only", "lmstudio", "lm_studio", "ollama"].includes(source) + || hostname === "localhost" || hostname === "[::1]" || hostname === "::1" || /^127\./.test(hostname)); + const limit = local ? limits.localModelConcurrency : limits.cloudModelConcurrency; + const hash = (value: string) => createHash("sha256").update(value).digest("hex"); + const root = `provider:${hash(endpoint || source)}`; + return Object.freeze({ local, providers: Object.freeze([ + Object.freeze({ key: root, limit }), + Object.freeze({ key: `${root}:model:${hash(config.model ?? "default")}`, limit }) + ]) }); +} + +/** Unlike bounded audit hashes, batching identity must include every config byte. */ +export function workerModelBatchKey(configs: readonly unknown[]): string { + const serialized = JSON.stringify(configs, (_key, value: unknown) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + return Object.fromEntries(Object.keys(value).sort().map(key => [key, (value as Record)[key]])); + }); + return createHash("sha256").update(serialized).digest("hex"); +} + +/** In-process admission only: failed acquisition never holds any partial permit. */ +export class WorkerConcurrencyController { + readonly limits: WorkerConcurrencyLimits; + private activeJobs = 0; + private backgroundActiveJobs = 0; + private readonly resources = new Map(); + private readonly providers = new Map>(); + private readonly scopes = new Set(); + private embeddingRequests = 0; + private readonly states = new WeakMap(); + private readonly context = new AsyncLocalStorage(); + private readonly waiters = new Set<() => void>(); + private readonly listeners = new Set<(resources: readonly string[]) => void>(); + private readonly groups = new Set(); + private readonly observers = new Set(); + private closing = false; + private closed?: Promise; + private resolveClosed?: () => void; + + constructor(options: WorkerConcurrencyOptions = {}, env: Readonly> = process.env) { + this.limits = resolveWorkerConcurrencyOptions(options, env); + } + availableJobs(): number { return Math.max(0, this.limits.maxActiveJobs - this.activeJobs); } + observe(observer: WorkerObserver): () => void { + this.observers.add(observer); + this.observeConcurrency(); + return () => { this.observers.delete(observer); }; + } + hasActivePermit(): boolean { + const state = this.context.getStore(); + return state !== undefined && !state.released; + } + hasActiveWorkerPermit(): boolean { + const state = this.context.getStore(); + return state !== undefined && !state.released && state.logicalJobs > 0; + } + /** Only a new router business boundary may discard a completed foreground + * request's inherited ALS. Released worker permits remain fenced, including + * their late callbacks; an active foreground reservation is never discarded. */ + runNewForegroundTask(operation: () => T): T { + const state = this.context.getStore(); + return state?.released && state.logicalJobs === 0 + ? this.context.run(undefined, operation) + : operation(); + } + close(): Promise { + if (this.closed) return this.closed; + this.closing = true; + this.closed = new Promise(resolve => { this.resolveClosed = resolve; }); + this.notifyReleased(); + return this.closed; + } + subscribe(listener: (resources: readonly string[]) => void): () => void { + this.listeners.add(listener); + return () => { this.listeners.delete(listener); }; + } + snapshot() { + return { + activeJobs: this.activeJobs, availableJobs: this.availableJobs(), + backgroundActiveJobs: this.backgroundActiveJobs, maxBackgroundJobs: this.limits.maxBackgroundJobs, + interactiveReservedSlots: this.limits.interactiveReservedSlots, + resources: Object.fromEntries(this.resources), + providers: Object.fromEntries([...this.providers].map(([key, groups]) => [key, groups.size])), + scopeKeys: [...this.scopes], embeddingRequests: this.embeddingRequests + }; + } + tryAcquire(requirements: WorkerRequirements, logicalJobs = 1, shareWith?: WorkerPermit, + onBlocked?: (resource: string) => void): WorkerPermit | undefined { + positive(logicalJobs, "logicalJobs"); + return this.acquire(requirements, logicalJobs, shareWith, onBlocked); + } + private acquire(input: WorkerRequirements, logicalJobs: number, shareWith?: WorkerPermit, + onBlocked?: (resource: string) => void): WorkerPermit | undefined { + const reject = (reason: AdmissionRejectReason, resource?: string): undefined => { + if (resource) onBlocked?.(resource); + for (const observer of this.observers) observeWorker(observer, { type: "admission_rejected", reason }); + return undefined; + }; + if (this.closing) return reject("closing"); + if (!(RESOURCE_CLASSES as readonly string[]).includes(input.resourceClass)) throw new TypeError("Unknown resource class"); + const providerMap = new Map(); + for (const provider of input.providers) { + if (!provider.key?.trim()) throw new TypeError("Provider key is required"); + const limit = positive(provider.limit, "provider limit"); + providerMap.set(provider.key, Math.min(providerMap.get(provider.key) ?? limit, limit)); + } + const providers = [...providerMap].sort(([a], [b]) => a.localeCompare(b)).map(([key, limit]) => ({ key, limit })); + const scopeKeys = [...new Set(input.scopeKeys)]; + if (scopeKeys.some(key => typeof key !== "string" || !key.trim())) throw new TypeError("Scope keys must be nonempty strings"); + const requirements: WorkerRequirements = { resourceClass: input.resourceClass, providers, scopeKeys, embedding: input.embedding === true, queueLane: input.queueLane }; + const parent = shareWith ? this.states.get(shareWith) : undefined; + if (shareWith && (!parent || parent.released || parent.group.resourceClass !== "embedding" || input.resourceClass !== "embedding" + || !parent.group.embedding || !input.embedding || JSON.stringify(parent.group.providers) !== JSON.stringify(providers))) return reject("share_invalid"); + if (input.resourceClass === "embedding" && input.embedding && + logicalJobs + (parent?.group.logicalJobs ?? 0) > this.limits.embeddingBatchSize) return reject("batch"); + if (logicalJobs > this.availableJobs()) return reject("global", workerCapacityKey("global")); + // Charge every logical member independently, including shared embedding batches. + // Missing/unknown lanes do not gain interactive privilege; foreground calls charge zero. + const backgroundJobs = input.queueLane === "interactive" ? 0 : logicalJobs; + if (this.backgroundActiveJobs + backgroundJobs > this.limits.maxBackgroundJobs) return reject("background", workerCapacityKey("background")); + if ((this.resources.get(input.resourceClass) ?? 0) + logicalJobs > this.limits.resourceLimits[input.resourceClass]) return reject("resource", workerCapacityKey("resource", input.resourceClass)); + const heldScope = scopeKeys.find(key => this.scopes.has(key)); + if (heldScope) return reject("scope", workerCapacityKey("scope", heldScope)); + if (!parent) { + if (requirements.embedding && this.embeddingRequests >= this.limits.embeddingRequestConcurrency) return reject("embedding_request", workerCapacityKey("embedding_request")); + for (const provider of providers) { + const active = this.providers.get(provider.key); + const cap = Math.min(provider.limit, ...active?.values() ?? []); + if ((active?.size ?? 0) >= cap) return reject("provider", workerCapacityKey("provider", provider.key)); + } + } + const group: ReservationGroup = parent?.group ?? { providers, embedding: requirements.embedding === true, + resourceClass: input.resourceClass, members: 0, logicalJobs: 0, activeRequests: 0, serial: new Map() }; + if (!parent) { + this.groups.add(group); + for (const provider of providers) { + const active = this.providers.get(provider.key) ?? new Map(); + active.set(group, provider.limit); this.providers.set(provider.key, active); + } + if (group.embedding) this.embeddingRequests++; + } + group.members++; + group.logicalJobs += logicalJobs; + this.activeJobs += logicalJobs; + this.backgroundActiveJobs += backgroundJobs; + this.resources.set(input.resourceClass, (this.resources.get(input.resourceClass) ?? 0) + logicalJobs); + for (const key of scopeKeys) this.scopes.add(key); + const state: PermitState = { requirements, logicalJobs, released: false, group }; + const permit: WorkerPermit = { + run: operation => { if (state.released) throw new Error("Worker permit already released"); return this.context.run(state, operation); }, + release: () => { + if (state.released) return; + state.released = true; + group.logicalJobs -= logicalJobs; + this.activeJobs -= logicalJobs; + this.backgroundActiveJobs -= backgroundJobs; + const count = (this.resources.get(input.resourceClass) ?? 0) - logicalJobs; + if (count) this.resources.set(input.resourceClass, count); else this.resources.delete(input.resourceClass); + for (const key of scopeKeys) this.scopes.delete(key); + group.members--; + const freed = scopeKeys.map(key => workerCapacityKey("scope", key)); + if (logicalJobs > 0) freed.push(workerCapacityKey("global"), workerCapacityKey("resource", input.resourceClass)); + if (backgroundJobs > 0) freed.push(workerCapacityKey("background")); + this.notifyReleased([...freed, ...this.releaseGroupIfIdle(group)]); + } + }; + this.states.set(permit, state); + this.observeConcurrency(); + return permit; + } + + private observeConcurrency(): void { + if (this.observers.size === 0) return; + const state = { + activeJobs: this.activeJobs, maxActiveJobs: this.limits.maxActiveJobs, + backgroundActiveJobs: this.backgroundActiveJobs, maxBackgroundJobs: this.limits.maxBackgroundJobs, + interactiveReservedSlots: this.limits.interactiveReservedSlots, + resources: Object.fromEntries(this.resources), resourceLimits: this.limits.resourceLimits, + providers: [...this.providers].map(([key, groups]) => ({ key, active: groups.size, limit: Math.min(...groups.values()) })), + embeddingRequests: this.embeddingRequests, embeddingRequestLimit: this.limits.embeddingRequestConcurrency + }; + for (const observer of this.observers) observeWorker(observer, { type: "concurrency", state }); + } + + private releaseGroupIfIdle(group: ReservationGroup): string[] { + if (group.members || group.activeRequests || !this.groups.delete(group)) return []; + for (const provider of group.providers) { + const active = this.providers.get(provider.key)!; + active.delete(group); if (active.size === 0) this.providers.delete(provider.key); + } + if (group.embedding) this.embeddingRequests--; + return [...group.providers.map(provider => workerCapacityKey("provider", provider.key)), + ...(group.embedding ? [workerCapacityKey("embedding_request")] : [])]; + } + + private notifyReleased(resources: readonly string[] = []): void { + this.observeConcurrency(); + for (const wake of this.waiters) wake(); + this.waiters.clear(); + for (const listener of this.listeners) { + try { listener(resources); } catch { /* Observers cannot prevent resource release. */ } + } + if (this.closing && this.activeJobs === 0 && this.groups.size === 0) this.resolveClosed?.(); + } + + /** Reserve once per task; serialize internal calls using the shared group, never acquire our own slot twice. */ + async runRequest(descriptor: WorkerProviderDescriptor, operation: () => Promise, embedding = false): Promise { + const state = this.context.getStore(); + if (state) { + const owned = new Set(state.group.providers.map(provider => provider.key)); + if (state.released || descriptor.providers.some(provider => !owned.has(provider.key)) || (embedding && !state.group.embedding)) { + throw new Error("Model provider request was not admitted by the worker permit"); + } + const keys = descriptor.providers.map(provider => provider.key); + if (embedding) keys.push("embedding-request"); + const previous = [...new Set(keys.map(key => state.group.serial.get(key)).filter(Boolean))]; + const request = Promise.all(previous.map(promise => promise!.catch(() => {}))).then(async () => { + if (state.released) throw new Error("Worker permit released before model request started"); + state.group.activeRequests++; + try { return await operation(); } + finally { + state.group.activeRequests--; + if (state.group.members === 0) { + this.notifyReleased(this.releaseGroupIfIdle(state.group)); + } + } + }); + const settled = request.then(() => {}, () => {}); + for (const key of keys) state.group.serial.set(key, settled); + try { return await request; } + finally { for (const key of keys) if (state.group.serial.get(key) === settled) state.group.serial.delete(key); } + } + // Foreground operations share provider/request caps but consume no background job slots. + const requirements: WorkerRequirements = { resourceClass: embedding ? "embedding" : "summary", providers: descriptor.providers, scopeKeys: [], embedding }; + for (;;) { + if (this.closing) throw new Error("Model request admission is closed"); + const permit = this.acquire(requirements, 0); + if (permit) { + try { return await permit.run(operation); } finally { permit.release(); } + } + await new Promise(resolve => this.waiters.add(resolve)); + } + } +} diff --git a/Memory/src/service/worker/worker-lease-options.ts b/Memory/src/service/worker/worker-lease-options.ts new file mode 100644 index 000000000..cd281c59a --- /dev/null +++ b/Memory/src/service/worker/worker-lease-options.ts @@ -0,0 +1,53 @@ +import type { ResourceClass } from "./job-policy-registry.js"; + +export interface WorkerLeasePolicy { durationSec: number; heartbeatMs: number } +export interface WorkerLeaseOptions { + resources?: Partial>>; + embeddingRetry?: Partial; +} +export interface ResolvedWorkerLeaseOptions { + resources: Readonly>>; + embeddingRetry: Readonly; +} +const classes: readonly ResourceClass[] = ["summary", "embedding", "evolution_llm", "database", "scan"]; + +/** Resolve once at worker startup; invalid timing must fail before any claim. */ +export function resolveWorkerLeaseOptions( + options: WorkerLeaseOptions = {}, env: Readonly> = process.env +): Readonly { + let configured: WorkerLeaseOptions = {}; + if (env.MEMMY_WORKER_LEASE_OPTIONS !== undefined) { + try { configured = JSON.parse(env.MEMMY_WORKER_LEASE_OPTIONS) as WorkerLeaseOptions; } + catch { throw new TypeError("MEMMY_WORKER_LEASE_OPTIONS must be a JSON object"); } + } + const object = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + for (const value of [configured, options]) { + if (!object(value) || Object.keys(value).some(key => !["resources", "embeddingRetry"].includes(key))) { + throw new TypeError("Worker lease options must contain resources or embeddingRetry"); + } + if (value.resources !== undefined && (!object(value.resources) || Object.keys(value.resources).some(key => !classes.includes(key as ResourceClass)))) { + throw new TypeError("Unknown worker lease resource class"); + } + } + const policy = (defaults: WorkerLeasePolicy, ...values: unknown[]): Readonly => { + let result = { ...defaults }; + for (const value of values) { + if (value === undefined) continue; + if (!object(value) || Object.keys(value).some(key => !["durationSec", "heartbeatMs"].includes(key))) { + throw new TypeError("Invalid worker lease policy"); + } + result = { ...result, ...value }; + } + if (!Number.isSafeInteger(result.durationSec) || result.durationSec < 1 || result.durationSec > 86_400 || + !Number.isSafeInteger(result.heartbeatMs) || result.heartbeatMs < 1 || result.heartbeatMs > result.durationSec * 500) { + throw new RangeError("Worker lease duration must be 1..86400 seconds and heartbeat at most half its duration"); + } + return Object.freeze(result); + }; + const resources = Object.fromEntries(classes.map(resource => [resource, + policy({ durationSec: 60, heartbeatMs: 20_000 }, configured.resources?.[resource], options.resources?.[resource]) + ])) as Record>; + return Object.freeze({ resources: Object.freeze(resources), + embeddingRetry: policy({ durationSec: 300, heartbeatMs: 20_000 }, configured.embeddingRetry, options.embeddingRetry) }); +} diff --git a/Memory/src/service/worker/worker-observability.ts b/Memory/src/service/worker/worker-observability.ts new file mode 100644 index 000000000..f3be52e66 --- /dev/null +++ b/Memory/src/service/worker/worker-observability.ts @@ -0,0 +1,299 @@ +import { createHash } from "node:crypto"; +import type Database from "better-sqlite3"; +import { QUEUE_LANES, RESOURCE_CLASSES, type ResourceClass } from "./job-policy-registry.js"; + +export type AdmissionRejectReason = "global" | "background" | "resource" | "scope" | "provider" | "embedding_request" | "batch" | "closing" | "share_invalid"; +export type WorkerSkipReason = AdmissionRejectReason | "budget" | "configuration" | "dependency" | "unknown"; +export type WorkerMetricOutcome = "succeeded" | "failed" | "released" | "fenced"; +export interface ObservedWorkerWork { + lane?: string | null; + queue: "main" | "embedding_retry"; + firstQueuedAt?: string | null; + leaseCount?: number; + leaseRecoveryCount?: number; + selectionReason?: "wrr" | "aging" | "interactive" | "legacy"; +} +export interface ObservedConcurrency { + activeJobs: number; maxActiveJobs: number; + backgroundActiveJobs?: number; maxBackgroundJobs?: number; interactiveReservedSlots?: number; + resources: Readonly>; + resourceLimits: Readonly>; + providers: ReadonlyArray<{ key: string; active: number; limit: number }>; + embeddingRequests: number; embeddingRequestLimit: number; +} +/** Only metadata is accepted. Do not add IDs, scope values, endpoint URLs or model inputs. */ +export type WorkerObservation = + | { type: "claimed" | "started"; work: ObservedWorkerWork } + | { type: "finished"; work: ObservedWorkerWork; outcome: WorkerMetricOutcome } + | { type: "skipped"; lane?: string | null; reason: WorkerSkipReason } + | { type: "lease"; queue: "main" | "embedding_retry"; action: "renewed" | "renew_failed" | "expired" | "recovered" | "fenced"; count?: number } + | { type: "admission_rejected"; reason: AdmissionRejectReason } + | { type: "concurrency"; state: ObservedConcurrency } + | { type: "model_request"; providerKey: string; modelKey: string; kind: "summary" | "evolution" | "embedding"; + durationMs: number; outcome: "succeeded" | "failed"; failure?: "429" | "timeout" | "other"; batchSize?: number } + | { type: "tokens"; providerKey: string; modelKey: string; kind: "summary" | "evolution" | "embedding"; + inputTokens: number; outputTokens: number; totalTokens: number }; +export interface WorkerObserver { record(event: WorkerObservation): void } +export function observeWorker(observer: WorkerObserver | undefined, event: WorkerObservation): void { + try { observer?.record(event); } catch { /* Metrics never affect transactions, admission or execution. */ } +} + +type Sample = { at: number; value: number }; +class Samples { + private readonly values: Sample[] = []; + private next = 0; + constructor(private readonly capacity: number) {} + add(at: number, value: number): void { + const sample = { at, value }; + if (this.values.length < this.capacity) this.values.push(sample); + else { this.values[this.next] = sample; this.next = (this.next + 1) % this.capacity; } + } + snapshot(since: number, until: number) { + const values = this.values.filter(sample => sample.at >= since && sample.at <= until).map(sample => sample.value).sort((a, b) => a - b); + const p = (fraction: number) => values.length ? values[Math.ceil(values.length * fraction) - 1] : null; + return { sampleCount: values.length, p50: p(0.5), p95: p(0.95), max: values.at(-1) ?? null }; + } +} +interface Bucket { at: number; counts: Map; peaks: Map } +export interface WorkerQueueMetric { + lane: string; queued: number; failed: number; leased: number; dueTimeOnly: number; + oldestWaitingAt: string | null; +} +export interface WorkerObservabilityOptions { + now?: () => number; + /** Fixed production defaults: five minute window and thirty second log interval. */ + windowMs?: number; + logIntervalMs?: number; + queueSnapshot?: (atMs: number) => WorkerQueueMetric[]; + write?: (snapshot: Record) => void; +} +const LANE_KEYS = [...QUEUE_LANES, "unknown"] as const; +const MODEL_LIMIT = 32; +const PROVIDER_LIMIT = 32; +const BUCKET_MS = 10_000; +const laneKey = (lane: unknown): string => (QUEUE_LANES as readonly unknown[]).includes(lane) ? String(lane) : "unknown"; +const key = (value: string): string => createHash("sha256").update(value).digest("hex").slice(0, 16); +const count = (value: number | undefined): number => Number.isFinite(value) && value! > 0 ? Math.trunc(value!) : 0; + +/** Bounded process-local diagnostics. All counters describe this process and rolling window, not durable history. */ +export class WorkerObservability implements WorkerObserver { + private readonly now: () => number; + private readonly windowMs: number; + private readonly logIntervalMs: number; + private readonly buckets: Bucket[] = []; + private readonly waits = new Map(LANE_KEYS.map(lane => [lane as string, new Samples(512)])); + private readonly models = new Map(); + private readonly providerKeys = new Set(); + private current?: ObservedConcurrency; + private lastLoggedAt: number; + private readonly startedAt: number; + private timer?: ReturnType; + private stopped = false; + + constructor(private readonly options: WorkerObservabilityOptions = {}) { + this.now = options.now ?? Date.now; + this.windowMs = options.windowMs ?? 300_000; + this.logIntervalMs = options.logIntervalMs ?? 30_000; + if (!Number.isSafeInteger(this.windowMs) || this.windowMs < BUCKET_MS || this.windowMs > 3_600_000 || + !Number.isSafeInteger(this.logIntervalMs) || this.logIntervalMs < 1 || this.logIntervalMs > 3_600_000) throw new RangeError("Invalid worker metrics window"); + this.lastLoggedAt = this.startedAt = this.now(); + } + start(): void { + if (this.stopped || this.timer) return; + this.timer = setInterval(() => this.flush(), this.logIntervalMs); + this.timer.unref?.(); + } + stop(): void { + if (this.stopped) return; + this.stopped = true; + if (this.timer) clearInterval(this.timer); + this.timer = undefined; + this.flush(true); + } + record(event: WorkerObservation): void { + if (this.stopped) return; + const at = this.now(); + const bucket = this.bucket(at); + const add = (name: string, value = 1) => bucket.counts.set(name, (bucket.counts.get(name) ?? 0) + value); + if (event.type === "claimed" || event.type === "started" || event.type === "finished") { + const lane = laneKey(event.work.lane); + add(`lane.${lane}.${event.type}`); + if (event.type === "claimed") { + add(`queue.${event.work.queue}.claimed`); + const reason = ["wrr", "aging", "interactive", "legacy"].includes(event.work.selectionReason ?? "") + ? event.work.selectionReason : "unspecified"; + add(`selection.${lane}.${reason}`); + // Record recoveries on the separately reported transition, not repeatedly + // on every future claim of an item which recovered once in the past. + if (count(event.work.leaseRecoveryCount)) add(`queue.${event.work.queue}.claims_with_recovery_history`); + } + if (event.type === "started") { + const queuedAt = Date.parse(event.work.firstQueuedAt ?? ""); + if (Number.isFinite(queuedAt)) this.waits.get(lane)!.add(at, Math.max(0, at - queuedAt)); + else add(`lane.${lane}.unknown_wait`); + } + if (event.type === "finished") add(`lane.${lane}.${event.outcome}`); + } else if (event.type === "skipped") add(`skip.${laneKey(event.lane)}.${event.reason}`); + else if (event.type === "admission_rejected") add(`admission_rejected.${event.reason}`); + else if (event.type === "lease") add(`lease.${event.queue}.${event.action}`, event.count === undefined ? 1 : count(event.count)); + else if (event.type === "concurrency") { + // Bound labels across configuration churn, not just within one snapshot. + const providerValues = new Map(); + for (const provider of event.state.providers) { + let id = key(provider.key); + if (!this.providerKeys.has(id)) { + if (this.providerKeys.size < PROVIDER_LIMIT) this.providerKeys.add(id); else id = "overflow"; + } + const value = providerValues.get(id) ?? { key: id, active: 0, limit: 0 }; + value.active += count(provider.active); value.limit += count(provider.limit); providerValues.set(id, value); + } + this.current = { activeJobs: count(event.state.activeJobs), maxActiveJobs: count(event.state.maxActiveJobs), + backgroundActiveJobs: count(event.state.backgroundActiveJobs), maxBackgroundJobs: count(event.state.maxBackgroundJobs), + interactiveReservedSlots: count(event.state.interactiveReservedSlots), + resources: Object.fromEntries(RESOURCE_CLASSES.map(resource => [resource, count(event.state.resources[resource])])), + resourceLimits: Object.fromEntries(RESOURCE_CLASSES.map(resource => [resource, count(event.state.resourceLimits[resource])])) as Record, + embeddingRequests: count(event.state.embeddingRequests), embeddingRequestLimit: count(event.state.embeddingRequestLimit), + providers: [...providerValues.values()] }; + const peak = (name: string, value: number) => bucket.peaks.set(name, Math.max(bucket.peaks.get(name) ?? 0, value)); + peak("background", count(event.state.backgroundActiveJobs)); peak("global", event.state.activeJobs); peak("embedding_request", event.state.embeddingRequests); + for (const resource of RESOURCE_CLASSES) peak(`resource.${resource}`, event.state.resources[resource] ?? 0); + for (const provider of this.current.providers) peak(`provider.${provider.key}`, provider.active); + } else if (event.type === "tokens" || event.type === "model_request") { + const series = this.modelSeries(event.providerKey, event.modelKey, event.kind); + if (event.type === "tokens") { + add(`model.${series.id}.usage_reports`); + add(`model.${series.id}.input_tokens`, count(event.inputTokens)); + add(`model.${series.id}.output_tokens`, count(event.outputTokens)); + add(`model.${series.id}.total_tokens`, count(event.totalTokens)); + } else { + add(`model.${series.id}.requests`); add(`model.${series.id}.${event.outcome}`); + if (event.failure) add(`model.${series.id}.${event.failure}`); + series.value.durations.add(at, Math.max(0, event.durationMs)); + if (event.batchSize !== undefined) series.value.batches.add(at, count(event.batchSize)); + } + } + } + snapshot(): Record { + const at = this.now(), since = at - this.windowMs; + this.prune(at); + const counts = new Map(), peaks = new Map(); + for (const bucket of this.buckets) { + if (bucket.at > at) continue; + for (const [name, value] of bucket.counts) counts.set(name, (counts.get(name) ?? 0) + value); + for (const [name, value] of bucket.peaks) peaks.set(name, Math.max(peaks.get(name) ?? 0, value)); + } + // A long-running reservation may span an entire window without a change + // event. Its current occupancy still belongs to that window's peak. + if (this.current) { + const peak = (name: string, value: number) => peaks.set(name, Math.max(peaks.get(name) ?? 0, value)); + peak("background", count(this.current.backgroundActiveJobs)); peak("global", this.current.activeJobs); peak("embedding_request", this.current.embeddingRequests); + for (const resource of RESOURCE_CLASSES) peak(`resource.${resource}`, this.current.resources[resource] ?? 0); + for (const provider of this.current.providers) peak(`provider.${provider.key}`, provider.active); + } + const elapsedMinutes = Math.max(1 / 60, Math.min(this.windowMs, at - this.startedAt) / 60_000); + let queue: WorkerQueueMetric[] = [], queueSnapshotError = false; + try { queue = this.options.queueSnapshot?.(at) ?? []; } catch { queueSnapshotError = true; } + const queueSnapshotAvailable = Boolean(this.options.queueSnapshot) && !queueSnapshotError; + return { + windowMs: this.windowMs, bucketMs: BUCKET_MS, processLocal: true, + rateDenominatorMs: elapsedMinutes * 60_000, + counterBasis: "rolling_10s_buckets_including_boundary_bucket_future_buckets_excluded_on_clock_rollback_oldest_evicted_at_hard_cap", + waitBasis: "started_in_window_total_since_first_queued_at_last_512_per_lane", + requestBasis: "logical_request_operation_only_excludes_permit_wait_includes_delegate_internal_retries", + embeddingBatchBasis: "texts_submitted_to_delegate_which_may_further_split_or_cache", + dueBasis: "time_metadata_only_not_dependency_configuration_or_permit_admissibility", + skipBasis: "candidate_scan_attempts_not_distinct_waiting_jobs", + tokenBasis: "reported_usage_only_missing_usage_is_unknown", + leaseBasis: "process_events_in_window_not_complete_historical_claim_or_recovery_totals", + completionBasis: "execution_attempts_succeeded_or_failed_not_distinct_terminal_jobs", + utilizationBasis: "instantaneous_active_divided_by_limit_not_time_weighted", + runnableBasis: "successful_admissions_only_not_a_full_queue_runnable_census", + queueSnapshotError, + queueSnapshotAvailable, + lanes: Object.fromEntries(LANE_KEYS.map(lane => { + const rows = queue.filter(row => laneKey(row.lane) === lane); + const oldest = rows.flatMap(row => row.oldestWaitingAt ? [Date.parse(row.oldestWaitingAt)] : []).filter(Number.isFinite); + const sum = (field: "queued" | "failed" | "leased" | "dueTimeOnly") => queueSnapshotAvailable + ? rows.reduce((total, row) => total + row[field], 0) : null; + return [lane, { queued: sum("queued"), failed: sum("failed"), leased: sum("leased"), dueTimeOnly: sum("dueTimeOnly"), + oldestWaitingAgeMs: oldest.length ? Math.max(0, at - Math.min(...oldest)) : null, + waitMs: this.waits.get(lane)!.snapshot(since, at), + admitted: counts.get(`lane.${lane}.claimed`) ?? 0, started: counts.get(`lane.${lane}.started`) ?? 0, + succeeded: counts.get(`lane.${lane}.succeeded`) ?? 0, failedInWindow: counts.get(`lane.${lane}.failed`) ?? 0, + completedPerMinute: ((counts.get(`lane.${lane}.succeeded`) ?? 0) + (counts.get(`lane.${lane}.failed`) ?? 0)) / elapsedMinutes }]; + })), + concurrency: this.current ? { ...this.current, peaks: Object.fromEntries(peaks), providerSeriesLimit: PROVIDER_LIMIT, + overflowBasis: "additional_provider_reservations_aggregated_not_a_single_provider_limit", + utilization: { global: this.current.activeJobs / this.current.maxActiveJobs, + embeddingRequest: this.current.embeddingRequests / this.current.embeddingRequestLimit, + resources: Object.fromEntries(RESOURCE_CLASSES.map(resource => [resource, + (this.current!.resources[resource] ?? 0) / this.current!.resourceLimits[resource]])) } } : null, + models: [...this.models].map(([id, model]) => ({ id, provider: model.provider, model: model.model, kind: model.kind, + latencyMs: model.durations.snapshot(since, at), embeddingBatchSize: model.batches.snapshot(since, at), + requests: counts.get(`model.${id}.requests`) ?? 0, failed: counts.get(`model.${id}.failed`) ?? 0, + failureRate: counts.get(`model.${id}.requests`) ? (counts.get(`model.${id}.failed`) ?? 0) / counts.get(`model.${id}.requests`)! : null, + rateLimited: counts.get(`model.${id}.429`) ?? 0, timedOut: counts.get(`model.${id}.timeout`) ?? 0, + usageReports: counts.get(`model.${id}.usage_reports`) ?? 0, + reportedTokens: counts.get(`model.${id}.usage_reports`) ? { input: counts.get(`model.${id}.input_tokens`) ?? 0, + output: counts.get(`model.${id}.output_tokens`) ?? 0, total: counts.get(`model.${id}.total_tokens`) ?? 0 } : null })), + counters: Object.fromEntries(counts) + }; + } + flush(final = false): void { + try { + if (!final && (this.stopped || this.now() - this.lastLoggedAt < this.logIntervalMs)) return; + this.lastLoggedAt = this.now(); + const snapshot = { ...this.snapshot(), final }; + if (this.options.write) this.options.write(snapshot); + // The ordinary human-readable logger intentionally clips long fields; + // emit this bounded, metadata-only snapshot as one complete JSON line. + else if (!["silent", "warn", "error"].includes(process.env.MEMMY_LOG_LEVEL?.trim().toLowerCase() ?? "info")) process.stdout.write(`${JSON.stringify({ + timestamp: new Date(this.now()).toISOString(), level: "info", component: "worker-metrics", event: "worker.metrics", + message: "[worker-metrics] worker.metrics", metrics: snapshot + })}\n`); + } catch { /* Failed metrics collection or logging must not interrupt shutdown. */ } + } + private bucket(at: number): Bucket { + this.prune(at); + const start = Math.floor(at / BUCKET_MS) * BUCKET_MS; + let bucket = this.buckets.find(value => value.at === start); + if (!bucket) { + bucket = { at: start, counts: new Map(), peaks: new Map() }; this.buckets.push(bucket); + this.buckets.sort((a, b) => a.at - b.at); + } + // Wall clocks can move backwards repeatedly. Retaining duplicate or future + // buckets must never turn a rolling diagnostic into unbounded storage. + const maximum = Math.ceil(this.windowMs / BUCKET_MS) + 2; + while (this.buckets.length > maximum) this.buckets.shift(); + return bucket; + } + private prune(at: number): void { + while (this.buckets.length && this.buckets[0]!.at < Math.floor((at - this.windowMs) / BUCKET_MS) * BUCKET_MS) this.buckets.shift(); + } + private modelSeries(providerKey: string, modelKey: string, kind: string) { + let id = key(`${providerKey}:${modelKey}:${kind}`); + if (!this.models.has(id)) { + if (this.models.size >= MODEL_LIMIT) id = "overflow"; + if (!this.models.has(id)) this.models.set(id, { provider: id === "overflow" ? "overflow" : key(providerKey), + model: id === "overflow" ? "overflow" : key(modelKey), kind: id === "overflow" ? "mixed" : kind, + durations: new Samples(256), batches: new Samples(256) }); + } + return { id, value: this.models.get(id)! }; + } +} + +/** Read only counts and timestamps; never hydrate memory bodies, payloads or vectors. */ +export function readWorkerQueueMetrics(db: Database.Database, atMs: number): WorkerQueueMetric[] { + const at = new Date(atMs).toISOString(); + const main = db.prepare(`SELECT queue_lane AS lane, + SUM(status = 'queued') AS queued, SUM(status = 'failed') AS failed, SUM(status = 'leased') AS leased, + SUM(status IN ('queued','failed') AND (available_at IS NULL OR available_at <= ?)) AS dueTimeOnly, + MIN(CASE WHEN status IN ('queued','failed') THEN first_queued_at END) AS oldestWaitingAt + FROM evolution_jobs WHERE status IN ('queued','failed','leased') GROUP BY queue_lane`).all(at); + const retry = db.prepare(`SELECT queue_lane AS lane, + SUM(status = 'pending') AS queued, SUM(status = 'failed') AS failed, SUM(status = 'in_progress') AS leased, + SUM(status = 'pending' AND next_attempt_at <= ?) AS dueTimeOnly, + MIN(CASE WHEN status = 'pending' THEN first_queued_at END) AS oldestWaitingAt + FROM embedding_retry_queue WHERE status IN ('pending','failed','in_progress') GROUP BY queue_lane`).all(atMs); + return [...main, ...retry] as WorkerQueueMetric[]; +} diff --git a/Memory/src/service/worker/worker-runner.ts b/Memory/src/service/worker/worker-runner.ts index e53ffbf1a..e8e490375 100644 --- a/Memory/src/service/worker/worker-runner.ts +++ b/Memory/src/service/worker/worker-runner.ts @@ -8,12 +8,16 @@ import { ModelHttpError } from "../../model/http.js"; import { allowedMemoryBudgetJobTypes } from "@memmy/agent-source-core"; +import { setImmediate as yieldWorkerTurn } from "node:timers/promises"; import type { Embedder } from "../../model/types.js"; import { retrievalDocumentIsCurrent, retrievalDocumentSourceHash } from "../../algorithm/plugin-algorithms.js"; import { createMemoryLogger, memoryErrorFields } from "../../logging/logger.js"; +import { JobLeaseLostError, withJobLeaseGuard, withoutJobLeaseGuard } from "../../storage/job-lease-context.js"; +import { newId } from "../../utils/id.js"; +import { isTerminalL3WorldModelError } from "../evolution/l3-world-model-pipeline.js"; import { jobToRef, type EmbeddingRetryRecord, @@ -49,10 +53,22 @@ import { import { ObservationDependenciesPendingError } from "../evidence/observation-memory-pipeline.js"; import { isObservationMemory, observationJobPayload, observationMemoryState, OBSERVATION_SUMMARY_MODEL_UNCONFIGURED } from "../observation-memory-state.js"; import { externalSkillGeneration, externalSkillJobPayload } from "../evidence/history-skill-policy.js"; +import { WorkerSchedulingShadow, type WorkerSchedulingComparison } from "./worker-scheduling-shadow.js"; +import { WorkerScheduler } from "./worker-scheduler.js"; +import { claimFairWorkerBatch, createFairWorkerScan, releaseFairWorkerResources, releaseFairWorkerDependencies, FAIR_CLAIM_BATCH_LIMIT, type FairClaimedWork } from "./fair-worker-queue.js"; +import { resolveWorkerSchedulingOptions, type WorkerSchedulingOptions, type ResolvedWorkerSchedulingOptions } from "./worker-scheduling-options.js"; +import { JobPolicyRegistry, UnknownJobPolicyError } from "./job-policy-registry.js"; +import { resolveWorkerLeaseOptions, type WorkerLeaseOptions } from "./worker-lease-options.js"; +import { WorkerConcurrencyController, workerCapacityKey, type PreparedWorkerExecution, type WorkerPermit } from "./worker-concurrency.js"; +import { workerWorkRequirements, workerEmbeddingGroupKey } from "./worker-work-requirements.js"; +import { observeWorker, type WorkerObserver, type WorkerMetricOutcome } from "./worker-observability.js"; export const SUMMARY_WORKER_CONCURRENCY = 4; export const EMBEDDING_RETRY_LEASE_MS = 5 * 60_000; +export const WORKER_LEASE_SECONDS = 60; +export const WORKER_HEARTBEAT_MS = 20_000; const SUMMARY_JOBS_REQUIRING_MODEL: readonly string[] = ["episode_assignment", "trace_summary", "import_summary", "observation_extract", "episode_title"]; +const WORKER_TURN_JOB_LIMIT = 32; const workerLogger = createMemoryLogger("worker"); @@ -88,6 +104,14 @@ export interface WorkerRunSummary { serverTime: string; } +/** Internal execution controls; the HTTP manual request does not expose these. */ +export interface WorkerRunExecution { + continuous?: boolean; + signal?: AbortSignal; + /** Internal handoff signal; a zero-claim yield does not mean the queue is idle. */ + onYieldToPendingRequests?: () => void; +} + export interface WorkerStartupReconciliation { requeuedJobs: number; requeuedEmbeddingRetries: number; @@ -102,6 +126,7 @@ export interface WorkerStartupReconciliation { export interface EmbeddingRetryClaim { workerId: string; leaseUntil: number; + leaseToken: string; } type EmbeddingRetryResult = { @@ -116,8 +141,8 @@ type EmbeddingRetryResult = { * that the scheduler invokes. */ export interface WorkerRunnerDeps { - repos: Pick; - // Mark exhausted judgment attempts; the durable assignment waits for an explicit retry. + repos: Pick & Partial>; + // Mark exhausted judgment attempts; durable assignment waits for an explicit retry. onTurnJudgeDeadLetter?: (job: EvolutionJobRecord) => void; embedder: Embedder; capture: { embedAfterCapture: boolean }; @@ -129,6 +154,20 @@ export interface WorkerRunnerDeps { memoryBudgetNextWakeAtMs: () => number; memoryBudgetNextReconcileAtMs?: () => number | undefined; summaryModelConfigured: () => boolean; + configurationFingerprint?: () => string; + beforeRun?: (request?: { targetMemoryIds?: readonly string[] }) => Promise; + maintenanceNextRunAtMs?: () => number | undefined; + afterRun?: () => void; + scheduling?: WorkerSchedulingOptions; + leases?: WorkerLeaseOptions; + concurrency?: WorkerConcurrencyController; + observer?: WorkerObserver; + prepareExecution?: (work: FairClaimedWork) => PreparedWorkerExecution; + prepareExecutionBatch?: () => (work: FairClaimedWork) => PreparedWorkerExecution; + schedulingShadow?: { + enabled?: boolean; + onComparison?: (comparison: WorkerSchedulingComparison) => void; + }; nowIso: () => string; nowMs?: () => number; encodeChangeCursor: (changeSeq: number) => string; @@ -168,12 +207,34 @@ export interface WorkerRunnerDeps { } export class WorkerRunner { - constructor(private readonly deps: WorkerRunnerDeps) {} + private readonly jobPolicies: JobPolicyRegistry; + private readonly scheduler = new WorkerScheduler(); + private readonly leaseOwner = newId("worker"); + private readonly executionConfiguration = new WeakMap(); + private readonly observedFinished = new WeakSet(); + private readonly observedStarted = new WeakSet(); + private readonly schedulingShadow: WorkerSchedulingShadow; + private readonly concurrency: WorkerConcurrencyController; + private readonly wakeListeners = new Set<() => void>(); + private wakeVersion = 0; + private stopping = false; + private readonly unsubscribeConcurrency: () => void; + + constructor(private readonly deps: WorkerRunnerDeps) { + this.jobPolicies = new JobPolicyRegistry(resolveWorkerLeaseOptions(deps.leases)); + // Environment settings are fixed at process launch. Reject invalid startup + // configuration instead of repeatedly waking an un-runnable worker. + resolveWorkerSchedulingOptions(deps.scheduling); + this.schedulingShadow = new WorkerSchedulingShadow(deps.repos.runtime); + this.concurrency = deps.concurrency ?? new WorkerConcurrencyController(); + this.unsubscribeConcurrency = this.concurrency.subscribe(() => this.notifyWorkAvailable()); + } nextWorkerRunAt(): number | undefined { if (!this.deps.memoryAddEnabled()) { return undefined; } + this.refreshConfigurationBlockedJobs(); const reconcileAt = this.deps.memoryBudgetNextReconcileAtMs?.(); const heldSummaryJobs = this.summaryJobsHeld(); if (this.deps.memoryBudgetPaused()) { @@ -209,14 +270,17 @@ export class WorkerRunner { } const at = this.deps.nowIso(); + this.refreshConfigurationBlockedJobs(); this.deps.enqueueObservationBackfill?.(Math.min(limit, 100), at); const interruptedJobs = this.deps.repos.runtime.requeueLeasedJobsAfterRestart(at); + observeWorker(this.deps.observer, { type: "lease", queue: "main", action: "recovered", count: interruptedJobs.length }); const failedJobs = [...this.deps.repos.runtime.requeueFailedEpisodeAssignments(at), ...this.deps.repos.runtime.requeueFailedJobs(limit, at)]; for (const { before, after } of [...interruptedJobs, ...failedJobs]) { this.deps.appendJobChange(after, "queued", before); } const embeddingRetries = this.deps.repos.runtime.requeueEmbeddingRetriesAfterRestart(Date.parse(at)); + observeWorker(this.deps.observer, { type: "lease", queue: "embedding_retry", action: "recovered", count: embeddingRetries.length }); for (const { before, after } of embeddingRetries) { this.deps.appendEmbeddingRetryChange(after, "queued", before); } @@ -326,6 +390,9 @@ export class WorkerRunner { for (const memory of this.deps.repos.memories.listImportMemoriesNeedingSummary(limit)) { const processing = this.deps.repos.processing.get(memory.id); if (processing?.state === "summary_pending" || processing?.state === "summarizing") continue; + // Startup reconciliation must not erase an explicit failure or recreate a + // terminal task. Manual retry/configuration recovery owns those states. + if (processing?.state === "failed") continue; const jobType = summaryProcessingJobType(memory); let job = this.deps.repos.runtime.getPendingJob(memory.id, jobType, memory.contentHash ?? undefined, String(observationMemoryState(memory)?.generation ?? "") || undefined); if (!job) { @@ -397,28 +464,71 @@ export class WorkerRunner { }; } - async runWorkerOnce( + runWorkerOnce( limit = 100, request: RequestEnvelope & { targetMemoryIds?: string[]; priorityCohortOnly?: boolean; - } = {} + } = {}, + execution: WorkerRunExecution = {} + ): Promise { + // Wake callbacks may originate from an earlier job's async context. A new + // drain establishes fresh per-job guards instead of inheriting that claim. + return this.runScheduled(async () => { + const wake = () => this.notifyWorkAvailable(); + execution.signal?.addEventListener("abort", wake, { once: true }); + try { return await this.runWorkerOnceUnscoped(limit, request, execution); } + finally { execution.signal?.removeEventListener("abort", wake); } + }, request); + } + + private runScheduled(execute: () => Promise, request?: { targetMemoryIds?: readonly string[] }): Promise { + // Every entry, including retry-only requests, must wake an automatic pump + // so it can yield the shared entry boundary to the waiting request. + this.notifyWorkAvailable(); + return withoutJobLeaseGuard(() => this.scheduler.run(async () => { + if (this.deps.beforeRun) await this.deps.beforeRun(request); + const result = await execute(); + this.deps.afterRun?.(); + return result; + })); + } + + notifyWorkAvailable(): void { + this.wakeVersion++; + for (const wake of [...this.wakeListeners]) wake(); + } + + stop(): Promise { + const stopped = this.scheduler.close(); + this.stopping = true; + this.notifyWorkAvailable(); + void stopped.then(this.unsubscribeConcurrency, this.unsubscribeConcurrency); + return stopped; + } + + private async runWorkerOnceUnscoped( + limit: number, + request: RequestEnvelope & { targetMemoryIds?: string[]; priorityCohortOnly?: boolean }, + execution: WorkerRunExecution = {} ): Promise { if (!this.deps.memoryAddEnabled()) { return this.deps.runWorkerNoWrite(request); } + const options = resolveWorkerSchedulingOptions(this.deps.scheduling); const normalizedLimit = Math.max(1, Math.floor(limit)); const targetMemoryIds = request.targetMemoryIds; + this.refreshConfigurationBlockedJobs(); // A large upgrade drains in bounded batches over normal worker rounds. // Do not create a wake-up loop while configuration or budget blocks models. if (!targetMemoryIds && this.deps.summaryModelConfigured() && !this.deps.memoryBudgetPaused()) { this.deps.enqueueObservationBackfill?.(Math.min(normalizedLimit, 100), this.deps.nowIso()); } - const requeuedJobs = this.deps.repos.runtime.requeueFailedJobs( + const requeuedJobs = options.algorithm === "legacy" ? this.deps.repos.runtime.requeueFailedJobs( normalizedLimit, this.deps.nowIso(), targetMemoryIds - ); + ) : []; for (const { before, after } of requeuedJobs) { this.deps.appendJobChange(after, "queued", before); } @@ -435,17 +545,340 @@ export class WorkerRunner { for (const { before, after } of expiredBudgeted) { this.deps.appendJobChange(after, "queued", before); } + observeWorker(this.deps.observer, { type: "lease", queue: "main", action: "recovered", count: expiredBudgeted.length }); } - const jobs = this.deps.repos.runtime.leaseQueuedJobs( - normalizedLimit, - 60, - targetMemoryIds, - request.priorityCohortOnly, - allowedWhenPaused, - this.summaryJobsHeld() - ); - const retryCapacity = Math.max(0, normalizedLimit - jobs.length); - const results: WorkerJobRunResult[] = []; + // Diagnostics compare both algorithms; they never control real claims. A preview failure cannot prevent + // the existing worker from claiming or executing its normal selection. + if ((this.deps.schedulingShadow?.enabled ?? process.env.MEMMY_WORKER_SCHEDULER_SHADOW !== "0")) { + try { + const comparison = this.schedulingShadow.sample({ + at: this.deps.nowIso(), limit: normalizedLimit, targetMemoryIds, laneWeights: options.laneWeights, + priorityCohortOnly: request.priorityCohortOnly, + allowedJobTypes: allowedWhenPaused, excludedJobTypes: this.summaryJobsHeld(), + includeRetries: !(this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding")) + }); + if (comparison) { + workerLogger.info("scheduler.shadow", { ...comparison }); + this.deps.schedulingShadow?.onComparison?.(comparison); + } + } catch (error) { + workerLogger.warn("scheduler.shadow_failed", memoryErrorFields(error)); + } + } + return this.runFairWorkerOnce(normalizedLimit, request, options, false, execution); + } + + private async runFairWorkerOnce( + limit: number, + request: RequestEnvelope & { targetMemoryIds?: string[]; priorityCohortOnly?: boolean }, + options: ResolvedWorkerSchedulingOptions, + retryOnly = false, + execution: WorkerRunExecution = {} + ): Promise { + type Admission = { permit: WorkerPermit; execution: PreparedWorkerExecution; group: symbol; priorRecoveries: number; release(): void }; + // Automatic execution stays open across refill waves. Finite manual requests + // retain their exact budget and target filters through the same scheduler. + let remaining = execution.continuous ? Number.POSITIVE_INFINITY : limit; + let leased = 0; + let fatal: unknown; + let failedDrain = false; + let attemptedReconcileDeadline: number | undefined; + let filled = false; + let jobsThisTurn = 0; + let scan = createFairWorkerScan(); + let claiming = false; + const unsubscribeScanAdmission = this.concurrency.subscribe(resources => { + // A CAS-lost reservation is returned synchronously inside the claim. It + // does not free capacity that was unavailable before that reservation. + if (claiming) return; + if (options.algorithm === "legacy") scan = createFairWorkerScan(); + else releaseFairWorkerResources(scan, resources); + }); + let scanFilters = ""; + let scanDenied = false; + let scanDeadline: number | undefined; + let deadlineRevision: { revision: number; rescanRevision: number } | undefined; + let prepareExecution: WorkerRunnerDeps["prepareExecution"]; + const active = new Set>(); + const selectedMain: string[] = []; + const claimKey = (job: EvolutionJobRecord) => `${job.id}:${job.leaseToken}`; + const results = new Map(); + const retries: EmbeddingRetryRunSummary = { leased: 0, succeeded: 0, failed: 0, items: [] }; + const waitForWake = (version: number) => { + let wake!: () => void; + const promise = new Promise(resolve => { + wake = resolve; + this.wakeListeners.add(wake); + if (this.wakeVersion !== version || this.stopping) resolve(); + }); + return { promise, dispose: () => this.wakeListeners.delete(wake) }; + }; + try { + while (remaining > 0 && this.deps.memoryAddEnabled() && !this.stopping && !failedDrain && !execution.signal?.aborted && + !(execution.continuous && this.scheduler.hasPendingRequests())) { + if (jobsThisTurn >= WORKER_TURN_JOB_LIMIT) { + // Resolved promises alone do not give HTTP, cancellation or timers a + // turn. Yield before the next admission wave, not before starting + // already claimed work. Re-enter the guards after the macro task. + await yieldWorkerTurn(); + jobsThisTurn = 0; + continue; + } + const maintenanceTimes = [this.deps.memoryBudgetNextReconcileAtMs?.(), this.deps.maintenanceNextRunAtMs?.()] + .filter((time): time is number => time !== undefined && Number.isFinite(time)); + const reconcileAt = maintenanceTimes.length ? Math.min(...maintenanceTimes) : undefined; + if (execution.continuous && this.deps.beforeRun && reconcileAt !== undefined && + Number.isFinite(reconcileAt) && reconcileAt <= this.nowMs() && reconcileAt !== attemptedReconcileDeadline) { + // A long automatic run no longer re-enters beforeRun every four jobs. + // Preserve budget reconciliation at its due time, without retrying an + // unchanged in-flight deadline on every completion/refill. + attemptedReconcileDeadline = reconcileAt; + await this.deps.beforeRun(retryOnly ? undefined : request); + if (!this.deps.memoryAddEnabled() || this.stopping || failedDrain || execution.signal?.aborted || this.scheduler.hasPendingRequests()) break; + } + let denied = false; + const admissions = new Map(); + let previous: { key: string; permit: WorkerPermit; group: symbol; count: number } | undefined; + const keyOf = (work: FairClaimedWork) => `${work.queue}:${work.queue === "main" ? work.job.id : work.retry.id}`; + const paused = this.deps.memoryBudgetPaused(); + const filters = JSON.stringify([retryOnly, paused, this.deps.memoryBudgetModelSources(), this.summaryJobsHeld(), + this.deps.configurationFingerprint?.()]); + if (!scan.more) scanDenied = scan.waiting.size > 0; + const revision = this.deps.repos.runtime.scheduling.getDeadlineRevision(); + const filtersChanged = scanFilters !== filters; + // Use one cutoff for both the expiry check and the future-only query. + // A deadline crossed during the snapshot must survive until discovery + // revisits older sequences, rather than being filtered out and forgotten. + const deadlineNow = this.nowMs(); + const deadlineExpired = scanDeadline !== undefined && scanDeadline <= deadlineNow; + const changed = revision.revision !== deadlineRevision?.revision; + const previousDeadlineRevision = deadlineRevision; + if (filtersChanged || deadlineExpired || changed) { + // Read the token and deadline in one snapshot. Notifications alone do + // not invalidate timing: ordinary completions only request a refill. + this.deps.repos.runtime.scheduling.readSnapshot(() => { + deadlineRevision = this.deps.repos.runtime.scheduling.getDeadlineRevision(); + scanDeadline = this.nextAdmissionDeadline(deadlineNow, request.targetMemoryIds, retryOnly); + }); + } + // Use the snapshot's token: another connection may edit an old row + // between the cheap change probe and opening that snapshot. + if (filtersChanged || deadlineExpired || (previousDeadlineRevision && + deadlineRevision?.rescanRevision !== previousDeadlineRevision.rescanRevision)) { + scan = createFairWorkerScan(); + scanFilters = filters; + scanDenied = false; + prepareExecution = this.deps.prepareExecutionBatch?.() ?? this.deps.prepareExecution; + } + const capacity = Math.min(remaining, this.concurrency.availableJobs(), FAIR_CLAIM_BATCH_LIMIT); + if (filled && capacity > 0 && !retryOnly && options.algorithm === "legacy") { + // Legacy selection requires an explicit failed -> queued transition. + // A deadline wake within a long run must perform that same step. + for (const { before, after } of this.deps.repos.runtime.requeueFailedJobs( + capacity, this.deps.nowIso(), request.targetMemoryIds + )) this.deps.appendJobChange(after, "queued", before); + } + filled = true; + claiming = true; + const work = capacity > 0 ? this.deps.repos.transaction(() => { + const before = this.deps.repos.runtime.scheduling.getDeadlineRevision(); + const claimed = claimFairWorkerBatch(this.deps.repos, { + at: this.deps.nowIso(), limit: capacity, targetMemoryIds: request.targetMemoryIds, + priorityCohortOnly: request.priorityCohortOnly, + allowedJobTypes: retryOnly ? [] : paused ? allowedMemoryBudgetJobTypes(this.deps.memoryBudgetModelSources()) : undefined, + excludedJobTypes: this.summaryJobsHeld(), + includeRetries: !(paused && this.deps.memoryBudgetJobConsumes("embedding")), + owner: this.leaseOwner, retryOwner: this.deps.embeddingRetryWorkerId, + leaseSeconds: WORKER_LEASE_SECONDS, + retryLeaseMs: this.leasePolicyFor("embedding_retry").durationSec * 1000, + leaseFor: work => this.leasePolicyFor(work.queue === "main" ? work.job.jobType : "embedding_retry"), + options, scan, + admit: candidate => { + const execution = prepareExecution?.(candidate) ?? { providers: [], run: (operation: () => T) => operation() }; + const requirements = workerWorkRequirements(candidate, this.deps.repos, execution); + // Reject held business state before loading any embedding body. + const heldScopes = this.concurrency.snapshot().scopeKeys; + const heldScope = requirements.scopeKeys.find(scope => heldScopes.includes(scope)); + if (heldScope) { + observeWorker(this.deps.observer, { type: "skipped", lane: candidate.queue === "main" ? candidate.job.queueLane : candidate.retry.queueLane, reason: "scope" }); + denied = true; return { blockedOn: workerCapacityKey("scope", heldScope) }; + } + const canShare = previous && previous.count < this.concurrency.limits.embeddingBatchSize; + let batchKey: string | undefined; + if (canShare) { + const key = workerEmbeddingGroupKey(candidate, this.deps.repos); + batchKey = key ? JSON.stringify([key, execution.batchKey ?? null]) : undefined; + } + const share = batchKey && previous?.key === batchKey ? previous : undefined; + let blockedOn: string | undefined; + const onBlocked = (resource: string) => { blockedOn = resource; }; + let permit = this.concurrency.tryAcquire(requirements, 1, share?.permit, onBlocked); + const shared = permit !== undefined && share !== undefined; + if (!permit && share) { + blockedOn = undefined; + permit = this.concurrency.tryAcquire(requirements, 1, undefined, onBlocked); + } + if (!permit) { denied = true; return blockedOn ? { blockedOn } : undefined; } + if (!canShare) { + const key = workerEmbeddingGroupKey(candidate, this.deps.repos); + batchKey = key ? JSON.stringify([key, execution.batchKey ?? null]) : undefined; + } + const group = shared ? share!.group : Symbol("worker-execution"); + const beforeAdmission = previous; + const acquiredPermit = permit; + const admission: Admission = { permit, execution, group, + priorRecoveries: (candidate.queue === "main" ? candidate.job : candidate.retry).leaseRecoveryCount ?? 0, + release: () => { + acquiredPermit.release(); + // A lost ID-CAS must not strand the surviving embedding group. + if (previous?.permit === acquiredPermit) previous = beforeAdmission; + } }; + admissions.set(keyOf(candidate), admission); + previous = batchKey ? { key: batchKey, permit, group, count: shared ? share!.count + 1 : 1 } : undefined; + return admission; + } + }); + // This transaction only classifies/claims rows. Account for its own + // leases without requerying the queue, but never acknowledge a change + // that was already present before it acquired the database snapshot. + const after = this.deps.repos.runtime.scheduling.getDeadlineRevision(); + if (deadlineRevision?.revision === before.revision && + after.revision - before.revision === claimed.length && + after.rescanRevision - before.rescanRevision === claimed.length) { + deadlineRevision = after; + for (const item of claimed) { + const expiry = item.queue === "main" ? Date.parse(item.job.leasedUntil!) : item.retry.leaseUntil!; + if (Number.isFinite(expiry)) scanDeadline = scanDeadline === undefined ? expiry : Math.min(scanDeadline, expiry); + } + } + return claimed; + }) : []; + claiming = false; + scanDenied = denied || scan.blocked; + denied = scanDenied; + remaining -= work.length; + jobsThisTurn += work.length; + const groups: Array<{ work: FairClaimedWork[]; admission: Admission }> = []; + for (const item of work) { + this.observeWork(item, "claimed"); + const admission = admissions.get(keyOf(item))!; + const recoveries = (item.queue === "main" ? item.job : item.retry).leaseRecoveryCount ?? 0; + if (recoveries > admission.priorRecoveries) { + const count = recoveries - admission.priorRecoveries; + observeWorker(this.deps.observer, { type: "lease", queue: item.queue, action: "expired", count }); + observeWorker(this.deps.observer, { type: "lease", queue: item.queue, action: "recovered", count }); + } + if (item.queue === "main") { leased++; selectedMain.push(claimKey(item.job)); } + const last = groups.at(-1); + if (last?.admission.group === admission.group) last.work.push(item); + else groups.push({ work: [item], admission }); + } + for (const group of groups) { + let task!: Promise; + task = Promise.resolve().then(() => group.admission.permit.run(() => group.admission.execution.run(async () => { + const jobs = group.work.flatMap(item => item.queue === "main" ? [item.job] : []); + const retryRows = group.work.flatMap(item => item.queue === "embedding_retry" ? [item.retry] : []); + const claims = new Map(retryRows.map(retry => [retry.id, { + workerId: this.deps.embeddingRetryWorkerId, leaseUntil: retry.leaseUntil!, leaseToken: retry.leaseToken! + }])); + const stopMainHeartbeat = this.startLeaseHeartbeat(jobs); + const stopRetryHeartbeat = this.startEmbeddingRetryHeartbeat(retryRows.map(retry => ({ retry, claim: claims.get(retry.id)! }))); + try { + if (jobs.length) { + const completed: WorkerJobRunResult[] = []; + await this.runClaimedMainJobs(jobs, completed, true); + for (const result of completed) { + const claim = jobs.find(job => job.id === result.ref.jobId)!; + results.set(claimKey(claim), result); + } + } else { + const result = await this.runLeasedEmbeddingRetries(retryRows, claims); + retries.leased += result.leased; + retries.succeeded += result.succeeded; + retries.failed += result.failed; + retries.items.push(...result.items); + } + } finally { stopMainHeartbeat(); stopRetryHeartbeat(); } + }))).catch(error => { if (!failedDrain) fatal = error; failedDrain = true; }).finally(() => { + for (const item of group.work) { + releaseFairWorkerDependencies(scan, item); + item.admission?.release(); + } + active.delete(task); + this.notifyWorkAvailable(); + }); + active.add(task); + } + if (remaining <= 0 || !this.deps.memoryAddEnabled() || this.stopping || failedDrain || execution.signal?.aborted || + (execution.continuous && this.scheduler.hasPendingRequests())) break; + if (capacity > 0 && scan.more) { + // No await occurs inside the claim transaction. Start this slice's + // claimed work above, then let HTTP, timers and cancellation run even + // when thousands of inspected rows acquired no execution slot. + await yieldWorkerTurn(); + jobsThisTurn = 0; + continue; + } + // SQLite discovery can cross a saved deadline within this synchronous + // turn. Handle it before either exiting or replacing it with a future + // deadline, otherwise a skipped older sequence can remain invisible. + const now = this.nowMs(); + if ((scanDeadline !== undefined && scanDeadline <= now) || + this.deps.repos.runtime.scheduling.getDeadlineRevision().revision !== deadlineRevision?.revision) continue; + if (active.size === 0) { + const snapshot = this.concurrency.snapshot(); + if (!denied || (snapshot.activeJobs === 0 && snapshot.embeddingRequests === 0 && Object.keys(snapshot.providers).length === 0)) break; + } + const wake = waitForWake(this.wakeVersion); + // Budget deadlines are cheap in-memory state and may change independently + // of durable queue timestamps. Do not issue another SQL query to combine them. + const deadlines = [scanDeadline, this.deps.memoryBudgetNextReconcileAtMs?.(), this.deps.maintenanceNextRunAtMs?.(), + ...(paused ? [this.deps.memoryBudgetNextWakeAtMs()] : [])] + .filter((time): time is number => time !== undefined && Number.isFinite(time) && time > now); + const deadline = deadlines.length ? Math.min(...deadlines) : undefined; + // Already due rows may be blocked on a permit or dependency. They must + // not hide a future deadline or create a repeating zero-delay timer. + const timer = deadline === undefined ? undefined : setTimeout( + () => this.notifyWorkAvailable(), Math.min(2_147_483_647, Math.max(1, deadline - now)) + ); + try { await Promise.race([...active, wake.promise]); } + finally { if (timer !== undefined) clearTimeout(timer); wake.dispose(); } + } + } catch (error) { failedDrain = true; fatal = error; } + finally { unsubscribeScanAdmission(); } + // Stop admission first, then wait for every guarded execution before releasing + // the shared entry boundary or allowing service shutdown to close storage. + await Promise.all(active); + if (failedDrain) throw fatal; + if (execution.continuous && !this.stopping && !execution.signal?.aborted && + this.deps.memoryAddEnabled() && this.scheduler.hasPendingRequests()) { + execution.onYieldToPendingRequests?.(); + } + const ordered = selectedMain.flatMap(id => results.has(id) ? [results.get(id)!] : []); + const succeeded = ordered.reduce((sum, item) => sum + item.succeeded, 0); + const failed = ordered.reduce((sum, item) => sum + item.failed, 0); + if (leased > 0 || retries.leased > 0) workerLogger.info("drain.completed", { + algorithm: options.algorithm, laneWeights: options.laneWeights, leased, succeeded, failed, + embeddingRetriesLeased: retries.leased, embeddingRetriesSucceeded: retries.succeeded, + embeddingRetriesFailed: retries.failed + }); + const changeSeq = this.deps.repos.runtime.latestChangeSeq(); + return { leased, succeeded, failed, jobs: ordered.map(item => item.ref), embeddingRetries: retries, + changeSeq, syncCursor: this.deps.encodeChangeCursor(changeSeq), serverTime: this.deps.nowIso() }; + } + + private nextAdmissionDeadline(now: number, targetMemoryIds: string[] | undefined, retryOnly: boolean): number | undefined { + const paused = this.deps.memoryBudgetPaused(); + const scheduled = this.deps.repos.runtime.nextWorkerRunAt({ + futureAfterMs: now, targetMemoryIds, + jobTypes: retryOnly ? [] : paused ? allowedMemoryBudgetJobTypes(this.deps.memoryBudgetModelSources()) : undefined, + excludedJobTypes: this.summaryJobsHeld(), + includeEmbeddingRetries: !(paused && this.deps.memoryBudgetJobConsumes("embedding")) + }); + return scheduled; + } + + private async runClaimedMainJobs(jobs: EvolutionJobRecord[], results: WorkerJobRunResult[], preserveRoleOrder = false): Promise { const queue = [...jobs]; while (queue.length > 0) { if (this.deps.memoryBudgetPaused()) { @@ -471,7 +904,7 @@ export class WorkerRunner { continue; } if (batchType === "embedding") { - results.push(...await this.runLeasedEmbeddingJobs(batch)); + results.push(...await this.runLeasedEmbeddingJobs(batch, preserveRoleOrder)); } else { for (let offset = 0; offset < batch.length; offset += SUMMARY_WORKER_CONCURRENCY) { if (offset > 0 && this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes(batchType)) { @@ -489,52 +922,21 @@ export class WorkerRunner { queue.shift(); results.push(await this.runLeasedWorkerJob(job)); } - // Joining already-persisted results is local work. Finish ready bindings in - // this drain, without waiting for the next scheduler tick or another model. - const finalizeCapacity = Math.max(0, normalizedLimit - jobs.length); - if (finalizeCapacity > 0) { - const ready = this.deps.repos.runtime.leaseQueuedJobs(finalizeCapacity, 60, undefined, false, ["episode_finalize"]); - jobs.push(...ready); - for (const job of ready) results.push(await this.runLeasedWorkerJob(job)); - } - const embeddingHeld = this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding"); - const embeddingRetries = !embeddingHeld && retryCapacity > 0 - ? await this.runEmbeddingRetryOnce(retryCapacity, targetMemoryIds) - : { leased: 0, succeeded: 0, failed: 0, items: [] }; - - const succeeded = results.reduce((sum, result) => sum + result.succeeded, 0); - const failed = results.reduce((sum, result) => sum + result.failed, 0); - if (jobs.length > 0 || embeddingRetries.leased > 0) { - workerLogger.info("drain.completed", { - leased: jobs.length, - succeeded, - failed, - embeddingRetriesLeased: embeddingRetries.leased, - embeddingRetriesSucceeded: embeddingRetries.succeeded, - embeddingRetriesFailed: embeddingRetries.failed - }); - } - const changeSeq = this.deps.repos.runtime.latestChangeSeq(); - return { - leased: jobs.length, - succeeded, - failed, - jobs: results.map((result) => result.ref), - embeddingRetries, - changeSeq, - syncCursor: this.deps.encodeChangeCursor(changeSeq), - serverTime: this.deps.nowIso() - }; } async runLeasedWorkerJob(job: EvolutionJobRecord): Promise { + if (!this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso())) return this.lostLeaseResult(job); this.deps.appendJobChange(job, "leased"); this.markProcessingJobLeased(job); workerLogger.info("job.started", workerJobLogFields(job)); + this.observeWork({ queue: "main", job }, "started"); try { - await this.deps.jobHandlers.processJob(job); + if (job.queueLane === "unregistered") throw new UnknownJobPolicyError(job.jobType); + this.captureExecutionConfiguration(job); + await this.withJobLease(job, () => this.deps.jobHandlers.processJob(job)); return this.completeLeasedWorkerJob(job); } catch (error) { + if (error instanceof JobLeaseLostError) return this.lostLeaseResult(job); if (error instanceof SummaryModelUnconfiguredError) { return this.holdUnconfiguredSummaryJob(job); } @@ -542,16 +944,21 @@ export class WorkerRunner { } } - async runLeasedEmbeddingJobs(jobs: EvolutionJobRecord[]): Promise { + async runLeasedEmbeddingJobs(jobs: EvolutionJobRecord[], preserveRoleOrder = false): Promise { const results: WorkerJobRunResult[] = []; const prepared: PreparedEmbeddingJob[] = []; for (const job of jobs) { + if (!this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso())) { + results.push(this.lostLeaseResult(job)); + continue; + } this.deps.appendJobChange(job, "leased"); this.markProcessingJobLeased(job); workerLogger.info("job.started", workerJobLogFields(job)); + this.observeWork({ queue: "main", job }, "started"); try { - const item = this.deps.embeddingJobs.prepareEmbeddingJob(job); + const item = this.withJobLease(job, () => this.deps.embeddingJobs.prepareEmbeddingJob(job)); if (item) { prepared.push(item); } else { @@ -562,15 +969,31 @@ export class WorkerRunner { } } - for (const role of ["document", "query"] as const) { - const batch = prepared.filter((item) => item.role === role); + const batches: PreparedEmbeddingJob[][] = []; + if (preserveRoleOrder) { + for (const item of prepared) { + const last = batches.at(-1); + if (last?.[0]?.role === item.role) last.push(item); + else batches.push([item]); + } + } else { + for (const role of ["document", "query"] as const) batches.push(prepared.filter((item) => item.role === role)); + } + for (let offset = 0; offset < batches.length; offset++) { + const batch = batches[offset]!; if (batch.length === 0) continue; + if (this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding")) { + results.push(...this.holdPreparedEmbeddingJobs(batches.slice(offset).flat())); + break; + } + const role = batch[0]!.role; try { + for (const item of batch) this.captureExecutionConfiguration(item.job); const vectors = await this.deps.embedder.embed(batch.map((item) => item.text || "(empty)"), role); for (const [index, item] of batch.entries()) { try { - this.deps.embeddingJobs.applyEmbeddingVector(item, vectors[index] ?? []); + this.withJobLease(item.job, () => this.deps.embeddingJobs.applyEmbeddingVector(item, vectors[index] ?? [])); results.push(this.completeLeasedWorkerJob(item.job)); } catch (error) { results.push(this.failLeasedWorkerJob(item.job, error)); @@ -591,9 +1014,13 @@ export class WorkerRunner { } async runLeasedEmbeddingItem(item: PreparedEmbeddingJob): Promise { + if (this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding")) { + return this.holdPreparedEmbeddingJobs([item])[0]!; + } try { + this.captureExecutionConfiguration(item.job); const vector = await this.deps.embedder.embedOne(item.text || "(empty)", item.role); - this.deps.embeddingJobs.applyEmbeddingVector(item, vector); + this.withJobLease(item.job, () => this.deps.embeddingJobs.applyEmbeddingVector(item, vector)); return this.completeLeasedWorkerJob(item.job); } catch (error) { return this.finishFailedEmbeddingItem(item, error); @@ -601,24 +1028,24 @@ export class WorkerRunner { } finishFailedEmbeddingItem(item: PreparedEmbeddingJob, error: unknown): WorkerJobRunResult { + if (error instanceof JobLeaseLostError || !this.deps.repos.runtime.isJobLeaseHeld(item.job, this.deps.nowIso())) { + return this.lostLeaseResult(item.job); + } if (classifyProcessingError(error).retryAction !== "retry") { return this.failLeasedWorkerJob(item.job, error); } - if (!this.deps.embeddingJobs.enqueueEmbeddingRetryAfterFailure(item, error)) { + if (!this.withJobLease(item.job, () => this.deps.embeddingJobs.enqueueEmbeddingRetryAfterFailure(item, error))) { return this.failLeasedWorkerJob(item.job, error); } return this.completeLeasedWorkerJob(item.job); } completeLeasedWorkerJob(job: EvolutionJobRecord): WorkerJobRunResult { - const completed = this.deps.repos.runtime.completeJob(job.id) ?? { - ...job, - status: "succeeded" as const, - leasedUntil: null, - updatedAt: this.deps.nowIso() - }; + const completed = this.deps.repos.runtime.completeJob(job.id, this.deps.nowIso(), job); + if (!completed) return this.lostLeaseResult(job); this.deps.appendJobChange(completed, "succeeded", job); workerLogger.info("job.succeeded", workerJobLogFields(completed)); + this.observeWork({ queue: "main", job }, "finished", "succeeded"); return { succeeded: 1, failed: 0, @@ -627,58 +1054,75 @@ export class WorkerRunner { } failLeasedWorkerJob(job: EvolutionJobRecord, error: unknown): WorkerJobRunResult { + if (error instanceof JobLeaseLostError || !this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso())) { + return this.lostLeaseResult(job); + } if (error instanceof ObservationDependenciesPendingError) { const at = this.deps.nowIso(); - const deferred = this.deps.repos.runtime.deferProcessingJob(job.id, new Date(Date.parse(at) + 30_000).toISOString(), at) ?? job; + const deferred = this.deps.repos.runtime.deferProcessingJob(job.id, new Date(Date.parse(at) + 30_000).toISOString(), at, job); + if (!deferred) return this.lostLeaseResult(job); const memory = job.targetMemoryId ? this.deps.repos.memories.get(job.targetMemoryId) : undefined; if (memory && processingJobMatchesMemory(job, memory)) { this.deps.repos.processing.update(memory.id, { state: "summary_pending", stage: "summary", activeJobId: job.id, attemptCount: deferred.attempts, updatedAt: at }, ["summary_pending", "summarizing"]); } this.deps.appendJobChange(deferred, "queued", job); + observeWorker(this.deps.observer, { type: "skipped", lane: job.queueLane, reason: "dependency" }); + this.observeWork({ queue: "main", job }, "finished", "released"); return { succeeded: 0, failed: 0, ref: { ...jobToRef(deferred), status: "queued" } }; } const errorMessage = processingStageForJob(job.jobType) ? sanitizeProcessingError(error) : error instanceof Error ? error.message : String(error); - const stage = processingStageForJob(job.jobType); const classification = classifyProcessingError(error); + const configurationBlocked = classification.retryAction === "open_settings" && classification.code !== "40309"; const forceDeadLetter = classification.code === "40309" - || Boolean(stage && classification.retryAction !== "retry"); + || classification.retryAction === "none" + || error instanceof UnknownJobPolicyError + || isTerminalL3WorldModelError(error) + || /unsupported job type/i.test(errorMessage); + const failureNo = (job.failureCount ?? 0) + 1; + const availableAt = configurationBlocked || forceDeadLetter ? null + : new Date(this.nowMs() + workerFailureBackoffMs(failureNo)).toISOString(); const failedJob = this.deps.repos.runtime.failJob( job.id, errorMessage, this.deps.nowIso(), - forceDeadLetter - ) ?? { - ...job, - status: "failed" as const, - leasedUntil: null, - lastError: errorMessage, - updatedAt: this.deps.nowIso() - }; + forceDeadLetter, + job, + { availableAt, blockedReason: configurationBlocked ? "configuration" : null, countFailure: !configurationBlocked } + ); + if (!failedJob) return this.lostLeaseResult(job); const failOp = failedJob.status === "dead_letter" ? "dead_letter" : "failed"; this.deps.appendJobChange(failedJob, failOp, job); this.updateProcessingAfterJobFailure(failedJob, error); + const resumed = configurationBlocked && this.executionConfigurationChanged(job) + ? this.resumeConfigurationBlockedMainJobs([job.id])[0]?.after + : undefined; if (failedJob.status === "dead_letter" && failedJob.jobType === "trace_summary" && failedJob.payload.turnJudge === true) { this.deps.onTurnJudgeDeadLetter?.(failedJob); } workerLogger.error("job.failed", { ...workerJobLogFields(failedJob), terminal: failedJob.status === "dead_letter", - ...memoryErrorFields(error) + ...memoryErrorFields(error), + availableAt: failedJob.availableAt, + blockedReason: failedJob.blockedReason }); + if (configurationBlocked) observeWorker(this.deps.observer, { type: "skipped", lane: job.queueLane, reason: "configuration" }); + this.observeWork({ queue: "main", job }, "finished", "failed"); return { succeeded: 0, failed: 1, ref: { ...jobToRef(job), - status: failedJob.status === "dead_letter" ? "dead_letter" : "failed" + status: resumed?.status ?? (failedJob.status === "dead_letter" ? "dead_letter" : "failed") } }; } markProcessingJobLeased(job: EvolutionJobRecord): void { + if (!this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso())) throw new JobLeaseLostError(job.id); if (!job.targetMemoryId) return; const stage = processingStageForJob(job.jobType); if (!stage) return; @@ -724,11 +1168,11 @@ export class WorkerRunner { return; } this.deps.repos.processing.update(job.targetMemoryId, { - state: terminal ? "failed" : stage === "summary" ? "summary_pending" : "embedding_pending", + state: terminal || job.blockedReason === "configuration" ? "failed" : stage === "summary" ? "summary_pending" : "embedding_pending", stage, activeJobId: terminal ? null : job.id, attemptCount: job.attempts, - retryAction: terminal ? classification.retryAction : "retry", + retryAction: job.blockedReason === "configuration" ? "open_settings" : terminal ? classification.retryAction : "retry", errorCode: classification.code, errorMessage: message, failedAt: this.deps.nowIso(), @@ -738,68 +1182,85 @@ export class WorkerRunner { : ["embedding_pending", "embedding", "failed"]); } - async runEmbeddingRetryOnce( + runEmbeddingRetryOnce( limit: number, targetMemoryIds?: readonly string[] ): Promise { - const now = this.nowMs(); - const retries = this.deps.repos.runtime.claimDueEmbeddingRetries({ - now, - workerId: this.deps.embeddingRetryWorkerId, - leaseUntil: now + EMBEDDING_RETRY_LEASE_MS, - limit, - targetMemoryIds + return this.runScheduled(async () => { + if (!this.deps.memoryAddEnabled()) return { leased: 0, succeeded: 0, failed: 0, items: [] }; + const options = resolveWorkerSchedulingOptions(this.deps.scheduling); + this.refreshConfigurationBlockedJobs(); + return (await this.runFairWorkerOnce(Math.max(1, Math.floor(limit)), + { targetMemoryIds: targetMemoryIds ? [...targetMemoryIds] : undefined }, options, true)).embeddingRetries; }); + } + + private async runLeasedEmbeddingRetries( + retries: EmbeddingRetryRecord[], prefetchedClaims?: ReadonlyMap + ): Promise { + const now = this.nowMs(); const results: EmbeddingRetryResult[] = []; const claimed: Array<{ retry: EmbeddingRetryRecord; claim: EmbeddingRetryClaim; attemptNo: number }> = []; for (const retry of retries) { - const claim = { + const claim = prefetchedClaims?.get(retry.id) ?? { workerId: this.deps.embeddingRetryWorkerId, - leaseUntil: retry.leaseUntil ?? now + EMBEDDING_RETRY_LEASE_MS + leaseUntil: retry.leaseUntil ?? now + this.leasePolicyFor("embedding_retry").durationSec * 1000, + leaseToken: retry.leaseToken! }; if (!this.deps.repos.runtime.isEmbeddingRetryClaimHeld(retry.id, claim)) { + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); results.push({ succeeded: 0, failed: 0, item: null }); continue; } claimed.push({ retry, claim, attemptNo: retry.attempts + 1 }); } + const stopHeartbeat = prefetchedClaims ? () => {} : this.startEmbeddingRetryHeartbeat(claimed); + try { + for (const role of ["document", "query"] as const) { + const batch = claimed.filter((item) => item.retry.embedRole === role); + if (batch.length === 0) continue; + if (this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding")) { + for (const item of batch) results.push(this.holdUnstartedEmbeddingRetry(item.retry)); + continue; + } - for (const role of ["document", "query"] as const) { - const batch = claimed.filter((item) => item.retry.embedRole === role); - if (batch.length === 0) continue; - - try { - const vectors = await this.deps.embedder.embed( - batch.map((item) => item.retry.sourceText || "(empty)"), - role - ); - for (const [index, item] of batch.entries()) { - try { - results.push(this.applyEmbeddingRetryVector(item.retry, item.claim, vectors[index] ?? [])); - } catch (error) { - results.push(this.failClaimedEmbeddingRetry(item.retry, item.claim, item.attemptNo, error)); + try { + for (const item of batch) this.observeWork({ queue: "embedding_retry", retry: item.retry }, "started"); + for (const item of batch) this.captureExecutionConfiguration(item.claim); + const vectors = await this.deps.embedder.embed( + batch.map((item) => item.retry.sourceText || "(empty)"), + role + ); + for (const [index, item] of batch.entries()) { + try { + results.push(this.applyEmbeddingRetryVector(item.retry, item.claim, vectors[index] ?? [])); + } catch (error) { + results.push(this.failClaimedEmbeddingRetry(item.retry, item.claim, item.attemptNo, error)); + } + } + } catch (error) { + if (batch.length > 1 && shouldIsolateEmbeddingBatch(error)) { + for (const item of batch) { + results.push(await this.runClaimedEmbeddingRetryItem(item.retry, item.claim, item.attemptNo)); + } + continue; } - } - } catch (error) { - if (batch.length > 1 && shouldIsolateEmbeddingBatch(error)) { for (const item of batch) { - results.push(await this.runClaimedEmbeddingRetryItem(item.retry, item.claim, item.attemptNo)); + results.push(this.failClaimedEmbeddingRetry(item.retry, item.claim, item.attemptNo, error)); } - continue; - } - for (const item of batch) { - results.push(this.failClaimedEmbeddingRetry(item.retry, item.claim, item.attemptNo, error)); } } - } - return { - leased: retries.length, - succeeded: results.reduce((sum, result) => sum + result.succeeded, 0), - failed: results.reduce((sum, result) => sum + result.failed, 0), - items: results.map((result) => result.item).filter((item): item is EmbeddingRetryRunSummary["items"][number] => Boolean(item)) - }; + return { + leased: retries.length, + succeeded: results.reduce((sum, result) => sum + result.succeeded, 0), + failed: results.reduce((sum, result) => sum + result.failed, 0), + items: results.map((result) => result.item).filter((item): item is EmbeddingRetryRunSummary["items"][number] => Boolean(item)) + }; + } finally { + stopHeartbeat(); + } } async runClaimedEmbeddingRetryItem( @@ -807,7 +1268,12 @@ export class WorkerRunner { claim: EmbeddingRetryClaim, attemptNo: number ): Promise { + if (this.deps.memoryBudgetPaused() && this.deps.memoryBudgetJobConsumes("embedding")) { + return this.holdUnstartedEmbeddingRetry(retry); + } try { + this.observeWork({ queue: "embedding_retry", retry }, "started"); + this.captureExecutionConfiguration(claim); const vector = await this.deps.embedder.embedOne(retry.sourceText || "(empty)", retry.embedRole); return this.applyEmbeddingRetryVector(retry, claim, vector); } catch (error) { @@ -820,6 +1286,10 @@ export class WorkerRunner { claim: EmbeddingRetryClaim, vector: number[] ): EmbeddingRetryResult { + if (!this.isEmbeddingRetryLeaseHeld(retry, claim)) { + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); + return { succeeded: 0, failed: 0, item: null }; + } const memory = this.deps.repos.memories.get(retry.targetId); if (!memory) { throw new Error(`embedding retry target not found: ${retry.targetKind}:${retry.targetId}`); @@ -832,6 +1302,11 @@ export class WorkerRunner { ...claim, now: this.nowMs() }); + if (!completed) { + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); + return { succeeded: 0, failed: 0, item: null }; + } + this.observeWork({ queue: "embedding_retry", retry }, "finished", "released"); if (completed) this.deps.appendEmbeddingRetryChange(completed, "succeeded", retry); const replacement = this.deps.enqueueEmbeddingRetry( memory, @@ -843,7 +1318,7 @@ export class WorkerRunner { return { succeeded: 0, failed: 0, item: completed ? embeddingRetryToRunItem(completed) : null }; } let completed: EmbeddingRetryRecord | undefined; - this.deps.embeddingJobs.persistEmbeddingVector({ + withJobLeaseGuard(retry, () => this.isEmbeddingRetryLeaseHeld(retry, claim), () => this.deps.embeddingJobs.persistEmbeddingVector({ memoryId: memory.id, vectorField: retry.vectorField, vector, @@ -859,12 +1334,14 @@ export class WorkerRunner { now: this.nowMs() }); } - }); + })); if (completed) { this.deps.appendEmbeddingRetryChange(completed, "succeeded", retry); workerLogger.info("embedding_retry.succeeded", embeddingRetryLogFields(completed)); + this.observeWork({ queue: "embedding_retry", retry }, "finished", "succeeded"); return { succeeded: 1, failed: 0, item: embeddingRetryToRunItem(completed) }; } + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); return { succeeded: 0, failed: 0, item: null }; } @@ -874,9 +1351,19 @@ export class WorkerRunner { attemptNo: number, error: unknown ): EmbeddingRetryResult { + if (error instanceof JobLeaseLostError || !this.isEmbeddingRetryLeaseHeld(retry, claim)) { + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); + return { succeeded: 0, failed: 0, item: null }; + } const message = sanitizeProcessingError(error); - const terminal = classifyProcessingError(error).retryAction !== "retry" || attemptNo >= retry.maxAttempts; - const updated = terminal + const classification = classifyProcessingError(error); + const configurationBlocked = classification.retryAction === "open_settings" && classification.code !== "40309"; + const terminal = classification.retryAction !== "retry" || attemptNo >= retry.maxAttempts; + const updated = configurationBlocked + ? this.deps.repos.runtime.markEmbeddingRetryConfigurationBlockedClaimed(retry.id, { + ...claim, error: message, now: this.nowMs() + }) + : terminal ? this.deps.repos.runtime.markEmbeddingRetryFailedClaimed(retry.id, { ...claim, attempts: attemptNo, @@ -891,10 +1378,13 @@ export class WorkerRunner { now: this.nowMs() }); if (updated) { + if (configurationBlocked) observeWorker(this.deps.observer, { type: "skipped", lane: retry.queueLane, reason: "configuration" }); + this.observeWork({ queue: "embedding_retry", retry }, "finished", "failed"); this.deps.appendEmbeddingRetryChange(updated, terminal ? "failed" : "retry", retry); const fields = { ...embeddingRetryLogFields(updated), terminal, + blockedReason: updated.blockedReason, ...memoryErrorFields(error) }; if (terminal) { @@ -902,9 +1392,14 @@ export class WorkerRunner { } else { workerLogger.warn("embedding_retry.retry_scheduled", fields); } - return { succeeded: 0, failed: 1, item: embeddingRetryToRunItem(updated) }; + const resumed = configurationBlocked && this.executionConfigurationChanged(claim) + ? this.deps.repos.runtime.resumeConfigurationBlockedEmbeddingRetries(this.nowMs(), [retry.id])[0]?.after + : undefined; + if (resumed) this.deps.appendEmbeddingRetryChange(resumed, "queued", updated); + return { succeeded: 0, failed: 1, item: embeddingRetryToRunItem(resumed ?? updated) }; } - return { succeeded: 0, failed: 1, item: null }; + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); + return { succeeded: 0, failed: 0, item: null }; } private summaryJobsHeld(): readonly string[] | undefined { @@ -932,7 +1427,8 @@ export class WorkerRunner { } private holdUnconfiguredSummaryJob(job: EvolutionJobRecord): WorkerJobRunResult { - this.requeueUnstartedBudgetedJobs([job]); + if (!this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso())) return this.lostLeaseResult(job); + this.requeueUnstartedBudgetedJobs([job], "configuration"); if (job.targetMemoryId) { const memory = this.deps.repos.memories.get(job.targetMemoryId); this.deps.repos.processing.update(job.targetMemoryId, { @@ -953,22 +1449,218 @@ export class WorkerRunner { }; } - private requeueUnstartedBudgetedJobs(jobs: readonly EvolutionJobRecord[]): void { + private requeueUnstartedBudgetedJobs(jobs: readonly EvolutionJobRecord[], reason: "budget" | "configuration" = "budget"): void { if (jobs.length === 0) { return; } - const requeued = this.deps.repos.runtime.requeueUnstartedLeasedJobs( - jobs.map((job) => job.id), - this.deps.nowIso() - ); + const requeued = this.deps.repos.runtime.releaseClaimedJobs(jobs, this.deps.nowIso()); for (const { before, after } of requeued) { this.deps.appendJobChange(after, "queued", before); + observeWorker(this.deps.observer, { type: "skipped", lane: before.queueLane, reason }); + this.observeWork({ queue: "main", job: before }, "finished", "released"); + } + } + + /** Prepared items are marked embedding already, but no new request may start + * after a budget pause. Return their claims and processing state atomically; + * a replacement lease or another active job must retain its own state. */ + private holdPreparedEmbeddingJobs(items: readonly PreparedEmbeddingJob[]): WorkerJobRunResult[] { + const transitions = this.deps.repos.transaction(() => { + const released = this.deps.repos.runtime.releaseClaimedJobs(items.map(item => item.job), this.deps.nowIso()); + for (const { before, after } of released) { + if (!after.targetMemoryId) continue; + const processing = this.deps.repos.processing.get(after.targetMemoryId); + const memory = this.deps.repos.memories.get(after.targetMemoryId); + if (processing?.activeJobId !== before.id || !memory || !processingJobMatchesMemory(before, memory)) continue; + this.deps.repos.processing.update(after.targetMemoryId, { + state: "embedding_pending", stage: "embedding", activeJobId: after.id, + attemptCount: after.attempts, updatedAt: this.deps.nowIso() + }, ["embedding"]); + } + return released; + }); + const releasedById = new Map(transitions.map(transition => [transition.after.id, transition])); + return items.map(({ job }) => { + const transition = releasedById.get(job.id); + if (!transition) return this.lostLeaseResult(job); + this.deps.appendJobChange(transition.after, "queued", transition.before); + observeWorker(this.deps.observer, { type: "skipped", lane: job.queueLane, reason: "budget" }); + this.observeWork({ queue: "main", job }, "finished", "released"); + return { succeeded: 0, failed: 0, ref: jobToRef(transition.after) }; + }); + } + + private holdUnstartedEmbeddingRetry(retry: EmbeddingRetryRecord): EmbeddingRetryResult { + const released = this.deps.repos.runtime.releaseClaimedEmbeddingRetries([retry], this.nowMs())[0]; + if (!released) { + this.observeWork({ queue: "embedding_retry", retry }, "finished", "fenced"); + return { succeeded: 0, failed: 0, item: null }; } + this.deps.appendEmbeddingRetryChange(released.after, "queued", released.before); + observeWorker(this.deps.observer, { type: "skipped", lane: retry.queueLane, reason: "budget" }); + this.observeWork({ queue: "embedding_retry", retry }, "finished", "released"); + return { succeeded: 0, failed: 0, item: embeddingRetryToRunItem(released.after) }; } private nowMs(): number { return this.deps.nowMs?.() ?? Date.now(); } + + private withJobLease(job: EvolutionJobRecord, work: () => T): T { + return withJobLeaseGuard(job, () => this.deps.repos.runtime.isJobLeaseHeld(job, this.deps.nowIso()), work); + } + + private lostLeaseResult(job: EvolutionJobRecord): WorkerJobRunResult { + workerLogger.warn("lease.fenced", { jobId: job.id, jobType: job.jobType }); + this.observeWork({ queue: "main", job }, "finished", "fenced"); + // A rejected result is neither a business failure nor a success. Report the + // durable current state without inventing a terminal outcome for the caller. + const current = this.deps.repos.runtime.getJob(job.id) ?? job; + return { succeeded: 0, failed: 0, ref: jobToRef(current) }; + } + + private observeWork(item: FairClaimedWork, type: "claimed" | "started" | "finished", outcome?: WorkerMetricOutcome): void { + if (!this.deps.observer) return; + const row = item.queue === "main" ? item.job : item.retry; + if (type === "started") { + if (this.observedStarted.has(row)) return; + this.observedStarted.add(row); + } + if (type === "finished") { + if (this.observedFinished.has(row)) return; + this.observedFinished.add(row); + if (outcome === "fenced") observeWorker(this.deps.observer, { type: "lease", queue: item.queue, action: "fenced" }); + } + const work = { queue: item.queue, lane: row.queueLane, firstQueuedAt: row.firstQueuedAt, + leaseCount: row.leaseCount, leaseRecoveryCount: row.leaseRecoveryCount, + ...(item.selectionReason ? { selectionReason: item.selectionReason } : {}) }; + if (type === "finished") observeWorker(this.deps.observer, { type, work, outcome: outcome! }); + else observeWorker(this.deps.observer, { type, work }); + } + + private leasePolicyFor(jobType: string) { + try { return this.jobPolicies.resolve({ jobType }).lease; } + catch (error) { + if (!(error instanceof UnknownJobPolicyError)) throw error; + // Lease an unregistered persisted row only so its existing terminal + // error path can quarantine it; do not let it block the entire drain. + return { durationSec: WORKER_LEASE_SECONDS, heartbeatMs: WORKER_HEARTBEAT_MS }; + } + } + + private startLeaseHeartbeat(jobs: readonly EvolutionJobRecord[]): () => void { + if (jobs.length === 0) return () => {}; + const tracked = new Map(jobs.map(job => { + const policy = this.leasePolicyFor(job.jobType); + return [job.id, { job, policy, nextRenewAt: this.nowMs() + policy.heartbeatMs }]; + })); + const timer = setInterval(() => { + for (const [id, entry] of tracked) { + const { job, policy } = entry; + if (this.nowMs() < entry.nextRenewAt) continue; + try { + const renewed = this.deps.repos.runtime.renewJobLease(job, policy.durationSec, this.deps.nowIso()); + if (!renewed) { + observeWorker(this.deps.observer, { type: "lease", queue: "main", action: "renew_failed" }); + tracked.delete(id); + continue; + } + job.leasedUntil = renewed.leasedUntil; + entry.nextRenewAt = this.nowMs() + policy.heartbeatMs; + workerLogger.debug("lease.renewed", { jobId: id, leasedUntil: renewed.leasedUntil }); + observeWorker(this.deps.observer, { type: "lease", queue: "main", action: "renewed" }); + } catch (error) { + // A busy database may recover on a later heartbeat. If the deadline + // passes first, all subsequent writes are rejected by the lease guard. + workerLogger.warn("lease.renew_failed", { jobId: id, ...memoryErrorFields(error) }); + observeWorker(this.deps.observer, { type: "lease", queue: "main", action: "renew_failed" }); + } + } + }, Math.min(...[...tracked.values()].map(entry => entry.policy.heartbeatMs))); + timer.unref?.(); + return () => clearInterval(timer); + } + + private captureExecutionConfiguration(execution: object): void { + const fingerprint = this.deps.configurationFingerprint?.(); + if (fingerprint) this.executionConfiguration.set(execution, fingerprint); + } + + private executionConfigurationChanged(execution: object): boolean { + const startedWith = this.executionConfiguration.get(execution); + const current = this.deps.configurationFingerprint?.(); + // A config reload can happen while a request from the previous config is + // still in flight. Its late failure must not block work under the new config. + // Resume only this execution, so a genuinely invalid current config stays held. + return Boolean(startedWith && current && startedWith !== current); + } + + private resumeConfigurationBlockedMainJobs(ids?: readonly string[]) { + const transitions = this.deps.repos.runtime.resumeConfigurationBlockedJobs(this.deps.nowIso(), ids); + for (const { before, after } of transitions) { + const stage = processingStageForJob(after.jobType); + if (after.targetMemoryId && stage) { + this.deps.repos.processing.update(after.targetMemoryId, { + state: stage === "summary" ? "summary_pending" : "embedding_pending", stage, + activeJobId: after.id, attemptCount: after.attempts, retryAction: "retry", + errorCode: null, errorMessage: null, failedAt: null, updatedAt: this.deps.nowIso() + }, ["failed"]); + } + this.deps.appendJobChange(after, "queued", before); + } + return transitions; + } + + private refreshConfigurationBlockedJobs(): void { + const fingerprint = this.deps.configurationFingerprint?.(); + if (!fingerprint) return; + const key = "worker.model_configuration_fingerprint"; + if (this.deps.repos.runtime.getKv(key)?.value === fingerprint) return; + this.resumeConfigurationBlockedMainJobs(); + for (const { before, after } of this.deps.repos.runtime.resumeConfigurationBlockedEmbeddingRetries(this.nowMs())) { + this.deps.appendEmbeddingRetryChange(after, "queued", before); + } + this.deps.repos.runtime.setKv(key, fingerprint, this.deps.nowIso()); + } + + resumeJobsAfterConfigurationChange(): void { + this.refreshConfigurationBlockedJobs(); + } + + private isEmbeddingRetryLeaseHeld(retry: EmbeddingRetryRecord, claim: EmbeddingRetryClaim): boolean { + return claim.leaseUntil > this.nowMs() && this.deps.repos.runtime.isEmbeddingRetryClaimHeld(retry.id, claim); + } + + private startEmbeddingRetryHeartbeat(items: Array<{ retry: EmbeddingRetryRecord; claim: EmbeddingRetryClaim }>): () => void { + if (items.length === 0) return () => {}; + const policy = this.leasePolicyFor("embedding_retry"); + const tracked = new Set(items); + const timer = setInterval(() => { + for (const item of tracked) { + try { + const now = this.nowMs(); + const renewed = this.deps.repos.runtime.renewEmbeddingRetryClaim(item.retry.id, item.claim, now + policy.durationSec * 1000, now); + if (!renewed?.leaseUntil) { + observeWorker(this.deps.observer, { type: "lease", queue: "embedding_retry", action: "renew_failed" }); + tracked.delete(item); continue; + } + item.claim.leaseUntil = renewed.leaseUntil; + item.retry.leaseUntil = renewed.leaseUntil; + observeWorker(this.deps.observer, { type: "lease", queue: "embedding_retry", action: "renewed" }); + } catch (error) { + workerLogger.warn("embedding_retry.renew_failed", { retryId: item.retry.id, ...memoryErrorFields(error) }); + observeWorker(this.deps.observer, { type: "lease", queue: "embedding_retry", action: "renew_failed" }); + } + } + }, policy.heartbeatMs); + timer.unref?.(); + return () => clearInterval(timer); + } +} + +export function workerFailureBackoffMs(failureNo: number): number { + const base = Math.min(5 * 60_000, 5_000 * 2 ** Math.min(10, Math.max(0, failureNo - 1))); + return Math.min(5 * 60_000, Math.round(base * (0.8 + Math.random() * 0.4))); } function emptyPolicyEvidencePreflight(): PolicyEvidencePreflightReport { @@ -984,6 +1676,8 @@ function workerJobLogFields(job: EvolutionJobRecord): Record { return { jobId: job.id, jobType: job.jobType, + queueLane: job.queueLane, + enqueueSeq: job.enqueueSeq, status: job.status, attempt: job.attempts, maxAttempts: job.maxAttempts, @@ -996,6 +1690,8 @@ function workerJobLogFields(job: EvolutionJobRecord): Record { function embeddingRetryLogFields(retry: EmbeddingRetryRecord): Record { return { retryId: retry.id, + queueLane: retry.queueLane, + enqueueSeq: retry.enqueueSeq, targetKind: retry.targetKind, targetMemoryId: retry.targetId, vectorField: retry.vectorField, diff --git a/Memory/src/service/worker/worker-scheduler.ts b/Memory/src/service/worker/worker-scheduler.ts new file mode 100644 index 000000000..de3ace55d --- /dev/null +++ b/Memory/src/service/worker/worker-scheduler.ts @@ -0,0 +1,96 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +export class WorkerSchedulerReentryError extends Error { + constructor() { + super("A running worker cannot await another drain through the same scheduler"); + this.name = "WorkerSchedulerReentryError"; + } +} + +export class WorkerSchedulerStoppedError extends Error { + constructor() { + super("Worker scheduler has stopped accepting requests"); + this.name = "WorkerSchedulerStoppedError"; + } +} + +interface Execution { active: boolean; scheduler: WorkerScheduler } +const executionContext = new AsyncLocalStorage(); + +/** Fire-and-forget wake callbacks create new requests, not nested awaited work. */ +export function withoutWorkerSchedulerContext(work: () => T): T { + return executionContext.exit(work); +} +interface PendingRequest { + start: (execution: Execution) => void; + reject: (error: Error) => void; +} + +/** + * The single execution boundary shared by automatic, manual and retry drains. + * Requests retain their own budgets and filters. The admitted runner uses the + * shared resource controller to execute and continuously refill within a drain. + */ +export class WorkerScheduler { + private readonly pending: PendingRequest[] = []; + private execution: Execution | undefined; + private stopped = false; + private readonly idleWaiters: Array<() => void> = []; + + hasPendingRequests(): boolean { return this.pending.length > 0; } + + run(execute: () => T | Promise): Promise { + if (this.isReentry()) return Promise.reject(new WorkerSchedulerReentryError()); + if (this.stopped) return Promise.reject(new WorkerSchedulerStoppedError()); + return new Promise((resolve, reject) => { + this.pending.push({ + reject, + start: (execution) => { + const finish = () => { + execution.active = false; + this.execution = undefined; + this.startNext(); + }; + try { + const outcome = executionContext.run(execution, execute); + void Promise.resolve(outcome).then( + (value) => { finish(); resolve(value); }, + (error: unknown) => { finish(); reject(error); } + ); + } catch (error) { + finish(); + reject(error); + } + } + }); + this.startNext(); + }); + } + + /** Reject requests that have not started; let the current guarded drain settle. */ + close(): Promise { + // Fail before lifecycle callers mutate their shutdown state. Awaiting our + // own completion from the running drain would otherwise deadlock. + if (this.isReentry()) throw new WorkerSchedulerReentryError(); + this.stopped = true; + for (const request of this.pending.splice(0)) request.reject(new WorkerSchedulerStoppedError()); + return this.execution ? new Promise((resolve) => this.idleWaiters.push(resolve)) : Promise.resolve(); + } + + private startNext(): void { + if (this.execution) return; + const next = this.pending.shift(); + if (next && !this.stopped) { + const execution = { active: true, scheduler: this }; + this.execution = execution; + next.start(execution); + return; + } + for (const resolve of this.idleWaiters.splice(0)) resolve(); + } + + private isReentry(): boolean { + const context = executionContext.getStore(); + return context?.scheduler === this && context.active; + } +} diff --git a/Memory/src/service/worker/worker-scheduling-options.ts b/Memory/src/service/worker/worker-scheduling-options.ts new file mode 100644 index 000000000..4f669df6f --- /dev/null +++ b/Memory/src/service/worker/worker-scheduling-options.ts @@ -0,0 +1,54 @@ +import type { QueueLane } from "./job-policy-registry.js"; +import { DEFAULT_LANE_WEIGHTS, validateLaneWeights } from "./lane-queue-selector.js"; + +export type WorkerSchedulingAlgorithm = "wrr" | "legacy"; +export interface WorkerAgingOptions { enabled?: boolean; thresholdMs?: number } +export interface WorkerSchedulingOptions { + algorithm?: WorkerSchedulingAlgorithm; + aging?: WorkerAgingOptions; + laneWeights?: Readonly>; +} +export interface ResolvedWorkerSchedulingOptions { + algorithm: WorkerSchedulingAlgorithm; + aging: Readonly<{ enabled: boolean; thresholdMs: number }>; + laneWeights: Readonly>; +} +type SchedulingEnvironment = Partial>; + +/** Explicit dependency settings win over environment settings, field by field. */ +export function resolveWorkerSchedulingOptions( + options: WorkerSchedulingOptions = {}, env: SchedulingEnvironment = process.env +): Readonly { + const algorithmValue: unknown = options.algorithm !== undefined ? options.algorithm + : env.MEMMY_WORKER_SCHEDULER_ALGORITHM?.trim() || "wrr"; + const algorithm = typeof algorithmValue === "string" ? algorithmValue.trim().toLowerCase() : undefined; + if (algorithm !== "wrr" && algorithm !== "legacy") { + throw new RangeError("Worker scheduler algorithm must be wrr or legacy"); + } + let weights: unknown = options.laneWeights; + if (weights === undefined) { + const encoded = env.MEMMY_WORKER_LANE_WEIGHTS?.trim(); + if (encoded) { + try { weights = JSON.parse(encoded) as unknown; } + catch { throw new TypeError("MEMMY_WORKER_LANE_WEIGHTS must contain a JSON object with all five lane weights"); } + } else weights = DEFAULT_LANE_WEIGHTS; + } + if (options.aging !== undefined && (options.aging === null || typeof options.aging !== "object" || Array.isArray(options.aging))) { + throw new TypeError("Worker aging options must be an object"); + } + if (options.aging?.enabled !== undefined && typeof options.aging.enabled !== "boolean") { + throw new TypeError("Worker aging enabled option must be a boolean"); + } + const flag: unknown = options.aging?.enabled !== undefined ? options.aging.enabled : env.MEMMY_WORKER_AGING_ENABLED?.trim() ?? true; + const enabled = typeof flag === "boolean" ? flag : flag === "true" || flag === "1" ? true : flag === "false" || flag === "0" ? false : undefined; + if (enabled === undefined) throw new TypeError("Worker aging enabled must be a boolean or true/false/1/0"); + const threshold: unknown = options.aging?.thresholdMs !== undefined ? options.aging.thresholdMs : + (env.MEMMY_WORKER_AGING_THRESHOLD_MS === undefined ? 300_000 : Number(env.MEMMY_WORKER_AGING_THRESHOLD_MS)); + if (typeof threshold !== "number" || !Number.isSafeInteger(threshold) || threshold < 1) { + throw new RangeError("Worker aging thresholdMs must be a positive safe integer"); + } + return Object.freeze({ algorithm, laneWeights: validateLaneWeights(weights), + aging: Object.freeze({ enabled: algorithm === "wrr" && enabled, thresholdMs: threshold }) }); +} diff --git a/Memory/src/service/worker/worker-scheduling-shadow.ts b/Memory/src/service/worker/worker-scheduling-shadow.ts new file mode 100644 index 000000000..0e8f1b86a --- /dev/null +++ b/Memory/src/service/worker/worker-scheduling-shadow.ts @@ -0,0 +1,73 @@ +import type { RuntimeRepository } from "../../storage/repositories.js"; +import type { QueueEligibilityOptions } from "../../storage/queue-eligibility.js"; +import { ShadowQueueSelector, type ShadowQueueSelection } from "./shadow-queue-selector.js"; +import type { QueueLane } from "./job-policy-registry.js"; + +export interface WorkerSchedulingShadowInput extends QueueEligibilityOptions { + at: string; + limit: number; + priorityCohortOnly?: boolean; + includeRetries?: boolean; + laneWeights?: Readonly>; +} +export interface WorkerSchedulingComparison { + mode: "shadow"; + sampledAt: string; + sampleLimit: number; + algorithm: ShadowQueueSelection["algorithm"]; + weights: ShadowQueueSelection["weights"]; + nextCursor: number; + coverageComplete: boolean; + pendingPolicyCount: number; + unregisteredJobCount: number; + candidatePoolSize: number; + legacySelection: string[]; + shadowSelection: string[]; + shadowLanes: string[]; + sameSelection: boolean; +} + +/** Observe at most once per five seconds, capped at 32 choices per sample. */ +export class WorkerSchedulingShadow { + private lastSampleAt: number | undefined; + private selector = new ShadowQueueSelector(); + private weightsKey: string | undefined; + + constructor(private readonly runtime: RuntimeRepository) {} + + sample(input: WorkerSchedulingShadowInput): WorkerSchedulingComparison | undefined { + const now = Date.parse(input.at); + if (this.lastSampleAt !== undefined && now >= this.lastSampleAt && now - this.lastSampleAt < 5_000) return; + this.lastSampleAt = now; + if (input.laneWeights && JSON.stringify(input.laneWeights) !== this.weightsKey) { + this.selector = new ShadowQueueSelector(input.laneWeights); + this.weightsKey = JSON.stringify(input.laneWeights); + } + const sampleLimit = Math.max(1, Math.min(32, Math.floor(input.limit))); + // Only the new projection fields are refreshed. Claims, task state, attempt + // counts, deadlines and change cursors remain entirely on the legacy path. + this.runtime.scheduling.refreshPolicies(500); + return this.runtime.scheduling.readSnapshot(() => { + const pendingPolicyCount = this.runtime.scheduling.pendingPolicyCount(); + const unregisteredJobCount = this.runtime.scheduling.unregisteredPolicyCount(); + const candidates = this.runtime.scheduling.candidates({ ...input, limit: sampleLimit }); + const legacyMain = this.runtime.previewQueuedJobs({ ...input, limit: sampleLimit }); + const legacyRetries = input.includeRetries === false ? [] : this.runtime.previewDueEmbeddingRetries({ + now, limit: sampleLimit - legacyMain.length, targetMemoryIds: input.targetMemoryIds + }); + const legacySelection = [...legacyMain.map((job) => `main:${job.id}`), + ...legacyRetries.map((retry) => `embedding_retry:${retry.id}`)]; + const selection = this.selector.select(candidates, sampleLimit); + const shadowSelection = selection.selected.map((item) => `${item.queue}:${item.id}`); + if (legacySelection.length === 0 && shadowSelection.length === 0 && pendingPolicyCount === 0 && unregisteredJobCount === 0) return; + return { + mode: "shadow", sampledAt: input.at, sampleLimit, algorithm: selection.algorithm, + weights: selection.weights, nextCursor: selection.nextCursor, + coverageComplete: pendingPolicyCount === 0 && unregisteredJobCount === 0, + pendingPolicyCount, unregisteredJobCount, candidatePoolSize: candidates.length, + legacySelection, shadowSelection, shadowLanes: selection.selected.map((item) => item.lane), + sameSelection: JSON.stringify(legacySelection) === JSON.stringify(shadowSelection) + }; + }); + } +} diff --git a/Memory/src/service/worker/worker-work-requirements.ts b/Memory/src/service/worker/worker-work-requirements.ts new file mode 100644 index 000000000..8197c39d4 --- /dev/null +++ b/Memory/src/service/worker/worker-work-requirements.ts @@ -0,0 +1,64 @@ +import type { Repositories } from "../../storage/repositories.js"; +import type { FairClaimedWork } from "./fair-worker-queue.js"; +import { jobPolicyRegistry, UnknownJobPolicyError } from "./job-policy-registry.js"; +import type { PreparedWorkerExecution, WorkerRequirements } from "./worker-concurrency.js"; + +function text(value: unknown): string | undefined { return typeof value === "string" && value ? value : undefined; } + +/** Scope keys describe shared business state, rather than an individual job type. */ +export function workerWorkRequirements( + work: FairClaimedWork, + repos: Pick & Partial>, + execution?: PreparedWorkerExecution +): WorkerRequirements { + const job = work.queue === "main" ? work.job : undefined; + const targetId = job?.targetMemoryId ?? (work.queue === "embedding_retry" ? work.retry.targetId : undefined); + const payload = job?.payload ?? {}; + let policy; + try { + // Target metadata affects lane/urgency, not these execution requirements. + // Avoid reading long conversation bodies while skipping resource-blocked rows. + policy = jobPolicyRegistry.resolve({ jobType: job?.jobType ?? "embedding_retry", payload }); + } catch (error) { + if (!(error instanceof UnknownJobPolicyError)) throw error; + return { resourceClass: "database", providers: [], scopeKeys: targetId ? [JSON.stringify(["target", targetId])] : [] }; + } + const scopeKeys = targetId ? [JSON.stringify(["target", targetId])] : []; + if (policy.concurrency.scope === "episode") { + const episode = job?.episodeId ?? text(payload.episodeId) ?? text(payload.episode_id); + if (episode) scopeKeys.push(JSON.stringify(["episode", job?.userId, episode])); + } + if (policy.concurrency.scope === "user_project") { + // Production L3 payloads contain only batchId/targetField; Work Memory + // payloads contain only trajectory data. Their opaque scope hashes differ. + // Resolve the authoritative project so both share the same business permit. + const session = job?.sessionId ? repos.runtime.getSession(job.sessionId) : undefined; + const batch = job?.jobType === "l3_world_model_update" && text(payload.batchId) + ? repos.l3WorldModels?.getBatch(String(payload.batchId)) : undefined; + // Assignment discovers its destination cluster only after model work. Both + // Skill operations therefore reserve the project, using their persisted + // episode/cluster rather than a missing target or a possibly absent session. + const skillOperation = job?.jobType === "skill_cluster_assign" || job?.jobType === "skill_batch_evolve"; + const skillEpisode = skillOperation && job?.episodeId ? repos.runtime.getEpisode(job.episodeId) : undefined; + const skillCluster = job?.jobType === "skill_batch_evolve" && text(payload.clusterId) + ? repos.runtime.getSkillCluster(String(payload.clusterId)) : undefined; + const memory = !batch && !session && !skillEpisode && !skillCluster && targetId ? repos.memories.get(targetId) : undefined; + const internal = memory?.properties.internal_info; + const project = batch ? batch.projectId ?? null : skillCluster ? skillCluster.projectId ?? null + : skillEpisode ? skillEpisode.projectId ?? null : session ? session.projectId ?? null + : text(payload.projectId) ?? text(payload.project_id) ?? text(internal?.project_id) + ?? text(internal?.projectId) ?? job?.scopeKey ?? null; + scopeKeys.push(JSON.stringify(["user_project", job?.userId ?? memory?.userId, project])); + } + return { queueLane: work.queue === "main" ? work.job.queueLane ?? undefined : work.retry.queueLane ?? undefined, + resourceClass: policy.resourceClass, providers: execution?.providers ?? [], scopeKeys, + embedding: execution?.embedding === true || policy.concurrency.providerRoles.includes("embedding") }; +} + +/** Only neighbouring, compatible embedding items may share one request permit. */ +export function workerEmbeddingGroupKey(work: FairClaimedWork, repos: Pick): string | undefined { + if (work.queue === "embedding_retry") return `retry:${work.retry.embedRole}`; + if (work.job.jobType !== "embedding") return undefined; + const memory = work.job.targetMemoryId ? repos.memories.get(work.job.targetMemoryId) : undefined; + return `main:${memory?.memoryLayer === "L1" ? "document" : "query"}`; +} diff --git a/Memory/src/storage/db.ts b/Memory/src/storage/db.ts index 2fbba53fd..954319330 100644 --- a/Memory/src/storage/db.ts +++ b/Memory/src/storage/db.ts @@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url"; import { getLoadablePath as getSqliteVecLoadablePath } from "sqlite-vec"; import { getSchemaVersion, migrate, SCHEMA_VERSION } from "./schema.js"; import { SQLITE_VEC_VERSION } from "./sqlite-vec-store.js"; +import { protectJobLeaseWrites } from "./job-lease-context.js"; export interface MemoryDbOptions { path?: string; @@ -38,6 +39,7 @@ export class MemoryDb { this.createPreMigrationBackup(); migrate(this.db); } + protectJobLeaseWrites(this.db); } catch (error) { this.db.close(); throw error; diff --git a/Memory/src/storage/job-lease-context.ts b/Memory/src/storage/job-lease-context.ts new file mode 100644 index 000000000..d902f5db4 --- /dev/null +++ b/Memory/src/storage/job-lease-context.ts @@ -0,0 +1,103 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type Database from "better-sqlite3"; + +interface JobLeaseContext { + jobId: string; + isHeld: () => boolean; +} + +const jobLeaseContext = new AsyncLocalStorage(); +const protectedConnections = new WeakSet(); + +export class JobLeaseLostError extends Error { + constructor(readonly jobId: string) { + super(`worker lease is no longer held: ${jobId}`); + this.name = "JobLeaseLostError"; + } +} + +/** Carry a claim across model awaits without holding a SQLite transaction open. */ +export function withJobLeaseGuard(job: { id: string }, isHeld: () => boolean, work: () => T): T { + return jobLeaseContext.run({ jobId: job.id, isHeld }, () => { + assertCurrentJobLease(); + return work(); + }); +} + +/** + * Infrastructure with an independent lifecycle (usage accounting and worker + * wake-ups) must not inherit the business task's expiring ownership. Callers + * must keep actual task-result persistence inside withJobLeaseGuard. + */ +export function withoutJobLeaseGuard(work: () => T): T { + return jobLeaseContext.exit(work); +} + +/** No active worker context means an ordinary foreground or administrative write. */ +export function assertCurrentJobLease(): void { + const claim = jobLeaseContext.getStore(); + if (claim && !claim.isHeld()) throw new JobLeaseLostError(claim.jobId); +} + +/** + * Guard the actual write boundary rather than relying on every business handler + * to remember a check after every model await. Better-sqlite3 exposes readonly + * for prepared statements, including WITH and RETURNING statements. + * + * The service has one synchronous SQLite writer. No asynchronous work occurs + * between a claim check and its statement; multi-statement changes continue to + * use the repository's synchronous transactions. This is not a cross-process + * distributed transaction mechanism. + */ +export function protectJobLeaseWrites(db: Database.Database): void { + if (protectedConnections.has(db)) return; + protectedConnections.add(db); + const prepare = db.prepare.bind(db); + db.prepare = ((sql: string) => { + const statement = prepare(sql); + if (statement.readonly) return statement; + + for (const name of ["run", "get", "all"] as const) { + const execute = statement[name]; + Object.defineProperty(statement, name, { + configurable: true, + writable: true, + value: (...args: unknown[]) => { + assertCurrentJobLease(); + return Reflect.apply(execute, statement, args); + } + }); + } + + const iterate = statement.iterate; + Object.defineProperty(statement, "iterate", { + configurable: true, + writable: true, + value: (...args: unknown[]) => { + assertCurrentJobLease(); + const iterator = Reflect.apply(iterate, statement, args) as IterableIterator; + return { + [Symbol.iterator]() { return this; }, + next(...nextArgs: [] | [unknown]) { + assertCurrentJobLease(); + return iterator.next(...nextArgs); + }, + return(value?: unknown) { + return iterator.return ? iterator.return(value) : { done: true, value }; + }, + throw(error?: unknown) { + if (iterator.throw) return iterator.throw(error); + throw error; + } + }; + } + }); + return statement; + }) as typeof db.prepare; + + const exec = db.exec.bind(db); + db.exec = (sql: string) => { + assertCurrentJobLease(); + return exec(sql); + }; +} diff --git a/Memory/src/storage/queue-deadline-schema.ts b/Memory/src/storage/queue-deadline-schema.ts new file mode 100644 index 000000000..5ed8a609f --- /dev/null +++ b/Memory/src/storage/queue-deadline-schema.ts @@ -0,0 +1,44 @@ +import type Database from "better-sqlite3"; + +export const MAIN_DEADLINE = "MAX(COALESCE(available_at, ''), COALESCE(CAST(json_extract(payload_json, '$.runAfter') AS TEXT), ''))"; +export const MAIN_TIMED = `status IN ('queued', 'failed') AND failure_count < max_attempts + AND (blocked_reason IS NULL OR blocked_reason != 'configuration') + AND (available_at IS NOT NULL OR json_type(payload_json, '$.runAfter') = 'text')`; +export const MAIN_LEASED = "status = 'leased' AND failure_count < max_attempts AND leased_until IS NOT NULL"; +export const RETRY_LEASED = "status = 'in_progress'"; +export const RETRY_DEADLINE = "CASE WHEN claimed_by IS NULL OR lease_until IS NULL THEN next_attempt_at ELSE MAX(next_attempt_at, lease_until) END"; + +/** Transactional change tokens, including raw/other-connection producers. New + * sequences need discovery; changes to existing rows also invalidate old cursors. + * Terminal removals cannot introduce an earlier deadline: a cached, earlier wake + * is harmless and will recompute the remaining deadlines when it arrives. */ +export function installQueueDeadlineSchema(db: Database.Database): void { + db.exec(`CREATE TABLE IF NOT EXISTS worker_deadline_revision ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), revision INTEGER NOT NULL, rescan_revision INTEGER NOT NULL + ); INSERT OR IGNORE INTO worker_deadline_revision VALUES (1, 0, 0)`); + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + const main = table === "evolution_jobs"; + const active = main ? "NEW.status IN ('queued','failed','leased')" : "NEW.status IN ('pending','in_progress')"; + const columns = main + ? ["status", "available_at", "leased_until", "lease_owner", "job_type", "target_memory_id", "failure_count", "max_attempts", "blocked_reason", "payload_json"] + : ["status", "next_attempt_at", "lease_until", "claimed_by", "target_id"]; + const changed = columns.map(column => column === "payload_json" + ? "json_extract(OLD.payload_json, '$.runAfter') IS NOT json_extract(NEW.payload_json, '$.runAfter')" + : `OLD.${column} IS NOT NEW.${column}`).join(" OR "); + db.exec(`CREATE TRIGGER IF NOT EXISTS ${table}_deadline_insert AFTER INSERT ON ${table} WHEN ${active} BEGIN + UPDATE worker_deadline_revision SET revision = revision + 1 WHERE singleton = 1; + END; + CREATE TRIGGER IF NOT EXISTS ${table}_deadline_update AFTER UPDATE OF ${columns.join(",")} ON ${table} + WHEN (${active}) AND (${changed}) BEGIN + UPDATE worker_deadline_revision SET revision = revision + 1, rescan_revision = rescan_revision + 1 WHERE singleton = 1; + END`); + } + // Upgrade the earlier lease-only expression once; keep the ordered index on + // normal reopens while allowing unowned legacy rows to wake at their backoff. + const retryIndex = db.prepare("SELECT sql FROM sqlite_master WHERE type='index' AND name='worker_retry_lease_deadline'") + .get() as { sql: string } | undefined; + if (retryIndex && !retryIndex.sql.includes(RETRY_DEADLINE)) db.exec("DROP INDEX worker_retry_lease_deadline"); + db.exec(`CREATE INDEX IF NOT EXISTS worker_main_deadline ON evolution_jobs(${MAIN_DEADLINE}) WHERE ${MAIN_TIMED}; + CREATE INDEX IF NOT EXISTS worker_main_lease_deadline ON evolution_jobs(leased_until) WHERE ${MAIN_LEASED}; + CREATE INDEX IF NOT EXISTS worker_retry_lease_deadline ON embedding_retry_queue(${RETRY_DEADLINE}) WHERE ${RETRY_LEASED}`); +} diff --git a/Memory/src/storage/queue-eligibility.ts b/Memory/src/storage/queue-eligibility.ts new file mode 100644 index 000000000..bb9f1a1c3 --- /dev/null +++ b/Memory/src/storage/queue-eligibility.ts @@ -0,0 +1,149 @@ +/** Shared eligibility for legacy leasing and read-only scheduling previews. */ +export interface QueueEligibilityOptions { + targetMemoryIds?: readonly string[]; + allowedJobTypes?: readonly string[]; + excludedJobTypes?: readonly string[]; + /** WRR admits due failures directly; legacy leasing still requires requeue. */ + includeDueFailed?: boolean; +} + +export function mainQueueEligibility(at: string, options: QueueEligibilityOptions = {}, includeDependencyBlocked = false): { + sql: string; params: Array; +} { + const { targetMemoryIds, allowedJobTypes, excludedJobTypes } = options; + if (targetMemoryIds?.length === 0 || allowedJobTypes?.length === 0) return { sql: "0", params: [] }; + const targetFilter = targetMemoryIds + ? `AND target_memory_id IN (${targetMemoryIds.map(() => "?").join(", ")})` + : ""; + return { + sql: `(status = 'queued' + ${options.includeDueFailed ? "OR status = 'failed'" : ""} + OR (status = 'leased' AND (leased_until IS NULL OR leased_until <= ?))) + AND failure_count < max_attempts + AND (available_at IS NULL OR available_at <= ?) + AND (blocked_reason IS NULL OR blocked_reason != 'configuration') + AND ${includeDependencyBlocked ? "1=1" : `NOT ( + job_type = 'reward' + AND json_extract(payload_json, '$.l1MemoryId') IS NOT NULL + AND ( + json_extract(payload_json, '$.polarity') IS NULL + OR json_extract(payload_json, '$.polarity') = 'positive' + ) + AND EXISTS ( + SELECT 1 + FROM memory_processing_state + JOIN memories ON memories.id = memory_processing_state.memory_id + WHERE memory_processing_state.memory_id = + CAST(json_extract(evolution_jobs.payload_json, '$.l1MemoryId') AS TEXT) + AND NOT EXISTS ( + SELECT 1 + FROM memory_vector_entries + WHERE memory_vector_entries.memory_id = memory_processing_state.memory_id + AND memory_vector_entries.vector_field = 'vec_summary' + ) + AND ( + memory_processing_state.state IN ('embedding_pending', 'embedding') + OR memory_processing_state.state IN ('summary_pending', 'summarizing') + ) + ) + )`} + AND ( + json_extract(payload_json, '$.runAfter') IS NULL + OR CAST(json_extract(payload_json, '$.runAfter') AS TEXT) <= ? + ) + AND ${includeDependencyBlocked ? "1=1" : `( + job_type NOT IN ('l3_world_model_update', 'work_memory_extract') + OR ( + scope_key IS NOT NULL + AND scope_seq IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM evolution_jobs AS leased_l3_job + WHERE leased_l3_job.job_type = evolution_jobs.job_type + AND leased_l3_job.scope_key = evolution_jobs.scope_key + AND leased_l3_job.status = 'leased' + AND leased_l3_job.id <> evolution_jobs.id + ) + AND NOT EXISTS ( + SELECT 1 + FROM evolution_jobs AS earlier_l3_job + WHERE earlier_l3_job.job_type = evolution_jobs.job_type + AND earlier_l3_job.scope_key = evolution_jobs.scope_key + AND earlier_l3_job.scope_seq < evolution_jobs.scope_seq + AND earlier_l3_job.status IN ('queued', 'leased', 'failed') + ) + ) + )`} + AND ${includeDependencyBlocked ? "1=1" : `NOT ( + job_type = 'feedback_experience' + AND EXISTS ( + SELECT 1 FROM raw_turns pending_feedback_turn + WHERE pending_feedback_turn.id = json_extract(evolution_jobs.payload_json, '$.rawTurnId') + AND pending_feedback_turn.deleted_at IS NULL AND pending_feedback_turn.redacted_at IS NULL + AND (pending_feedback_turn.episode_id IS NULL OR + json_extract(pending_feedback_turn.message_payload_json, '$.episode_assignment.status') IN ('pending', 'selected', 'applying')) + ) + ) + AND NOT ( + job_type = 'trace_summary' + AND COALESCE(json_extract(payload_json, '$.turnJudge'), 0) = 1 + AND EXISTS ( + SELECT 1 + FROM evolution_jobs AS earlier_turn_job + WHERE earlier_turn_job.job_type = 'trace_summary' + AND earlier_turn_job.session_id = evolution_jobs.session_id + AND COALESCE(json_extract(earlier_turn_job.payload_json, '$.turnJudge'), 0) = 1 + AND earlier_turn_job.status IN ('queued', 'leased', 'failed') + AND earlier_turn_job.id <> evolution_jobs.id + AND ( + CAST(json_extract(earlier_turn_job.payload_json, '$.turnAt') AS TEXT) + < CAST(json_extract(evolution_jobs.payload_json, '$.turnAt') AS TEXT) + OR (json_extract(earlier_turn_job.payload_json, '$.turnAt') = json_extract(evolution_jobs.payload_json, '$.turnAt') + AND COALESCE((SELECT rowid FROM raw_turns WHERE id = COALESCE( + json_extract(earlier_turn_job.payload_json, '$.rawTurnId'), + (SELECT json_extract(properties_json, '$.internal_info.trace.raw_turn_id') FROM memories WHERE id = earlier_turn_job.target_memory_id) + )), earlier_turn_job.rowid) + < COALESCE((SELECT rowid FROM raw_turns WHERE id = COALESCE( + json_extract(evolution_jobs.payload_json, '$.rawTurnId'), + (SELECT json_extract(properties_json, '$.internal_info.trace.raw_turn_id') FROM memories WHERE id = evolution_jobs.target_memory_id) + )), evolution_jobs.rowid)) + ) + ) + ) + AND NOT ( + (job_type IN ('episode_assignment', 'episode_finalize') OR (job_type = 'trace_summary' AND COALESCE(json_extract(payload_json, '$.turnJudge'), 0) = 1)) + AND json_extract(payload_json, '$.rawTurnId') IS NOT NULL + AND EXISTS ( + SELECT 1 FROM raw_turns prior JOIN raw_turns current ON current.id = json_extract(evolution_jobs.payload_json, '$.rawTurnId') + WHERE prior.session_id = current.session_id AND prior.deleted_at IS NULL AND prior.redacted_at IS NULL + AND json_extract(prior.message_payload_json, '$.episode_assignment.status') IN ('pending', 'selected', 'applying') + AND (prior.created_at < current.created_at OR (prior.created_at = current.created_at AND prior.rowid < current.rowid)) + ) + ) + AND (job_type <> 'episode_finalize' OR EXISTS ( + SELECT 1 FROM raw_turns r WHERE r.id = json_extract(evolution_jobs.payload_json, '$.rawTurnId') + AND (r.deleted_at IS NOT NULL OR r.redacted_at IS NOT NULL OR ( + json_extract(r.message_payload_json, '$.episode_assignment.selection') IS NOT NULL + AND (COALESCE(json_extract(r.message_payload_json, '$.episode_assignment.turnJudge'), 0) = 0 + OR json_extract(r.message_payload_json, '$.episode_assignment.judgment') IS NOT NULL + OR json_extract(r.message_payload_json, '$.turn_judge.status') = 'skipped') + )) + ))`} + ${allowedJobTypes ? `AND job_type IN (${allowedJobTypes.map(() => "?").join(", ")})` : ""} + ${excludedJobTypes?.length ? `AND job_type NOT IN (${excludedJobTypes.map(() => "?").join(", ")})` : ""} + ${targetFilter}`, + params: [at, at, at, ...(allowedJobTypes ?? []), ...(excludedJobTypes ?? []), ...(targetMemoryIds ?? [])] + }; +} + +export function retryQueueEligibility(now: number, targetMemoryIds?: readonly string[]): { + sql: string; params: Array; +} { + if (targetMemoryIds?.length === 0) return { sql: "0", params: [] }; + return { + sql: `(q.status = 'pending' OR (q.status = 'in_progress' AND (q.claimed_by IS NULL OR q.lease_until IS NULL OR q.lease_until <= ?))) + AND q.next_attempt_at <= ? + ${targetMemoryIds ? `AND q.target_id IN (${targetMemoryIds.map(() => "?").join(", ")})` : ""}`, + params: [now, now, ...(targetMemoryIds ?? [])] + }; +} diff --git a/Memory/src/storage/queue-scheduling-schema.ts b/Memory/src/storage/queue-scheduling-schema.ts new file mode 100644 index 000000000..0c91d1f57 --- /dev/null +++ b/Memory/src/storage/queue-scheduling-schema.ts @@ -0,0 +1,110 @@ +import { installQueueDeadlineSchema } from "./queue-deadline-schema.js"; +import type Database from "better-sqlite3"; +import { JOB_POLICY_VERSION } from "../service/worker/job-policy-registry.js"; + +// A persisted, event-maintained worklist: valid rows occupy no entries. Literal +// version predicates let SQLite use this partial index without scanning valid jobs. +export const pendingPolicyPredicate = `(policy_version IS NULL OR policy_version != ${JOB_POLICY_VERSION})`; +export const pendingPolicyIndex = (table: string, active = false): string => `${table}_policy_pending_v${JOB_POLICY_VERSION}${active ? "_active" : ""}`; +export const activePolicyPredicate = (table: string, prefix = ""): string => table === "evolution_jobs" + ? `${prefix}status IN ('queued', 'leased', 'failed')` : `${prefix}status IN ('pending', 'in_progress')`; + +/** Metadata only: none of these triggers changes job state, claims or eligibility. */ +export function installQueueSchedulingSchema(db: Database.Database): void { + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + const names = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map((c) => c.name)); + for (const [name, type] of Object.entries({ + queue_lane: "TEXT", resource_class: "TEXT", enqueue_seq: "INTEGER", first_queued_at: "TEXT", + base_priority: "INTEGER", policy_version: "INTEGER", policy_promotion: "TEXT" + })) { + if (!names.has(name)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); + } + } + // An auxiliary counter holds no task facts. Keeping it after deletion prevents + // reuse of a retired sequence, including when the newest row was deleted. + db.exec(`CREATE TABLE IF NOT EXISTS worker_enqueue_counter ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), value INTEGER NOT NULL + )`); + db.prepare(`INSERT INTO worker_enqueue_counter(singleton, value) + VALUES (1, MAX( + COALESCE((SELECT MAX(enqueue_seq) FROM evolution_jobs), 0), + COALESCE((SELECT MAX(enqueue_seq) FROM embedding_retry_queue), 0))) + ON CONFLICT(singleton) DO UPDATE SET value = MAX(value, excluded.value)`).run(); + const missing = db.prepare(`SELECT 'evolution_jobs' AS table_name, id, created_at AS queued_at, + (julianday(created_at) - 2440587.5) * 86400000 AS queued_ms, rowid AS tie + FROM evolution_jobs WHERE enqueue_seq IS NULL + UNION ALL + SELECT 'embedding_retry_queue', id, strftime('%Y-%m-%dT%H:%M:%fZ', created_at / 1000.0, 'unixepoch'), + created_at, rowid FROM embedding_retry_queue WHERE enqueue_seq IS NULL + ORDER BY queued_ms, table_name, tie`).all() as Array<{ table_name: string; id: string; queued_at: string }>; + const next = db.prepare("UPDATE worker_enqueue_counter SET value = value + 1 WHERE singleton = 1 RETURNING value"); + for (const row of missing) { + const seq = (next.get() as { value: number }).value; + db.prepare(`UPDATE ${row.table_name} SET enqueue_seq = ?, first_queued_at = COALESCE(first_queued_at, ?) WHERE id = ?`) + .run(seq, row.queued_at, row.id); + } + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + // Some import producers deliberately backdate created_at to the source + // conversation. New waiting time must start at insertion, not that date. + const firstAt = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + db.exec(`CREATE TRIGGER IF NOT EXISTS ${table}_enqueue_sequence AFTER INSERT ON ${table} BEGIN + UPDATE worker_enqueue_counter SET value = value + 1 WHERE singleton = 1; + UPDATE ${table} SET enqueue_seq = (SELECT value FROM worker_enqueue_counter WHERE singleton = 1), + first_queued_at = ${firstAt}, policy_version = NULL WHERE id = NEW.id; + END`); + const main = table === "evolution_jobs"; + const columns = main ? ["job_type", "payload_json", "target_memory_id", "user_id", "session_id", "episode_id", "scope_key", "scope_seq"] + : ["target_kind", "target_id", "vector_field", "source_text", "embed_role"]; + const newCycle = main + ? "OLD.status IN ('succeeded', 'dead_letter') AND NEW.status IN ('queued', 'leased', 'failed')" + : "OLD.status IN ('succeeded', 'failed') AND NEW.status IN ('pending', 'in_progress')"; + // Replace older broad triggers on existing installations as well. A reused + // unfinished task retains its promotion; a new work cycle must earn it again. + db.exec(`DROP TRIGGER IF EXISTS ${table}_policy_dirty`); + db.exec(`CREATE TRIGGER ${table}_policy_dirty AFTER UPDATE OF ${columns.join(", ")}, status ON ${table} + WHEN (${columns.map(column => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")}) OR (${newCycle}) BEGIN + UPDATE ${table} SET policy_version = NULL, + policy_promotion = CASE WHEN ${newCycle} THEN NULL ELSE NEW.policy_promotion END WHERE id = NEW.id; + END`); + // Rule upgrades build a new worklist once; processing removes entries in FIFO + // order. Reopening resumes remaining entries rather than restarting a scan. + const obsolete = db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name=? AND name GLOB ?") + .all(table, `${table}_policy_pending_v*`) as Array<{ name: string }>; + for (const index of obsolete) { + if (![pendingPolicyIndex(table), pendingPolicyIndex(table, true)].includes(index.name)) db.exec(`DROP INDEX "${index.name.replaceAll('"', '""')}"`); + } + db.exec(`CREATE INDEX IF NOT EXISTS ${pendingPolicyIndex(table)} ON ${table}(enqueue_seq) WHERE ${pendingPolicyPredicate}`); + db.exec(`CREATE INDEX IF NOT EXISTS ${pendingPolicyIndex(table, true)} ON ${table}(enqueue_seq) + WHERE ${pendingPolicyPredicate} AND ${activePolicyPredicate(table)}`); + db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS ${table}_enqueue_seq ON ${table}(enqueue_seq)`); + const due = table === "evolution_jobs" ? "available_at" : "next_attempt_at"; + db.exec(`CREATE INDEX IF NOT EXISTS ${table}_lane_due ON ${table}(status, queue_lane, ${due}, enqueue_seq)`); + db.exec(`CREATE INDEX IF NOT EXISTS ${table}_lane_fifo ON ${table}(queue_lane, enqueue_seq)`); + db.exec(`CREATE INDEX IF NOT EXISTS ${table}_policy_dirty ON ${table}(policy_version)`); + } + // Target-only invalidations cannot use the existing (target_kind, target_id) + // index's left prefix; avoid a retry-table scan for every imported memory. + db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_retry_policy_target ON embedding_retry_queue(target_id)`); + db.exec("DROP TRIGGER IF EXISTS memories_worker_policy_dirty"); + db.exec(`CREATE TRIGGER memories_worker_policy_dirty + AFTER UPDATE OF agent_id, tags_json, properties_json, memory_layer ON memories + WHEN OLD.agent_id IS NOT NEW.agent_id OR OLD.tags_json IS NOT NEW.tags_json + OR OLD.memory_layer IS NOT NEW.memory_layer + OR json_extract(OLD.properties_json, '$.internal_info.plugin_algorithm') IS NOT json_extract(NEW.properties_json, '$.internal_info.plugin_algorithm') + OR json_extract(OLD.properties_json, '$.internal_info.capture_origin') IS NOT json_extract(NEW.properties_json, '$.internal_info.capture_origin') + OR json_extract(OLD.properties_json, '$.internal_info.project_id') IS NOT json_extract(NEW.properties_json, '$.internal_info.project_id') + OR json_extract(OLD.properties_json, '$.internal_info.projectId') IS NOT json_extract(NEW.properties_json, '$.internal_info.projectId') + BEGIN + UPDATE evolution_jobs SET policy_version = NULL WHERE target_memory_id = NEW.id; + UPDATE embedding_retry_queue SET policy_version = NULL WHERE target_id = NEW.id; + END`); + db.exec(`CREATE TRIGGER IF NOT EXISTS memories_worker_policy_insert AFTER INSERT ON memories BEGIN + UPDATE evolution_jobs SET policy_version = NULL WHERE target_memory_id = NEW.id; + UPDATE embedding_retry_queue SET policy_version = NULL WHERE target_id = NEW.id; + END`); + db.exec(`CREATE TRIGGER IF NOT EXISTS memories_worker_policy_delete AFTER DELETE ON memories BEGIN + UPDATE evolution_jobs SET policy_version = NULL WHERE target_memory_id = OLD.id; + UPDATE embedding_retry_queue SET policy_version = NULL WHERE target_id = OLD.id; + END`); + installQueueDeadlineSchema(db); +} diff --git a/Memory/src/storage/queue-scheduling-store.ts b/Memory/src/storage/queue-scheduling-store.ts new file mode 100644 index 000000000..a38b24a1f --- /dev/null +++ b/Memory/src/storage/queue-scheduling-store.ts @@ -0,0 +1,312 @@ +import type Database from "better-sqlite3"; +import { + jobPolicyRegistry, JOB_POLICY_VERSION, QUEUE_LANES, + type QueueLane, UnknownJobPolicyError +} from "../service/worker/job-policy-registry.js"; +import { mainQueueEligibility, retryQueueEligibility, type QueueEligibilityOptions } from "./queue-eligibility.js"; +import { createMemoryLogger, memoryErrorFields } from "../logging/logger.js"; +import { activePolicyPredicate, pendingPolicyIndex, pendingPolicyPredicate } from "./queue-scheduling-schema.js"; +import { JobLeaseLostError } from "./job-lease-context.js"; + +const logger = createMemoryLogger("worker"); +// Repositories may be recreated around one connection. All its selectors must +// observe reclassification, including refreshOne completing before the next slice. +const selectionRevisions = new WeakMap(); + +export interface SchedulingCandidate { + queue: "main" | "embedding_retry"; + id: string; + jobType: string; + lane: QueueLane; + enqueueSeq: number; + basePriority: number; + firstQueuedAt?: string | null; + unregistered?: boolean; +} +export interface SchedulingMetadata { + queueLane?: string | null; + resourceClass?: string | null; + enqueueSeq?: number | null; + firstQueuedAt?: string | null; + basePriority?: number | null; + policyVersion?: number | null; +} +interface PolicyRow { + id: string; job_type: string; payload_json: string; agent_id: string | null; + memory_layer: string | null; tags_json: string | null; plugin_algorithm: string | null; capture_origin: unknown; + previous_queue_lane: string | null; + policy_promotion: string | null; + status: string; +} +export interface PolicyRefreshResult { + refreshed: number; + unknown: Array<{ id: string; jobType: string }>; + /** Present for active/runnable maintenance; runnable probes may conservatively report an exhausted budget. */ + hasMore?: boolean; +} + +/** A metadata projection and read-only candidate adapter; never claims or completes jobs. */ +export class QueueSchedulingStore { + constructor(private readonly db: Database.Database) {} + + getDeadlineRevision(): { revision: number; rescanRevision: number } { + return this.db.prepare("SELECT revision, rescan_revision AS rescanRevision FROM worker_deadline_revision WHERE singleton = 1") + .get() as { revision: number; rescanRevision: number }; + } + + getSelectionRevision(): number { + return selectionRevisions.get(this.db) ?? 0; + } + + isPendingCandidate(candidate: SchedulingCandidate): boolean { + const table = candidate.queue === "main" ? "evolution_jobs" : "embedding_retry_queue"; + return !!this.db.prepare(`SELECT 1 FROM ${table} WHERE id = ? AND enqueue_seq = ? + AND ${activePolicyPredicate(table)}`).get(candidate.id, candidate.enqueueSeq); + } + + isEligibleCandidate(candidate: SchedulingCandidate, at: string, options: QueueEligibilityOptions): boolean { + const condition = candidate.queue === "main" ? mainQueueEligibility(at, options) + : retryQueueEligibility(Date.parse(at), options.targetMemoryIds); + const table = candidate.queue === "main" ? "evolution_jobs" : "embedding_retry_queue q"; + return !!this.db.prepare(`SELECT 1 FROM ${table} WHERE ${condition.sql} AND id = ? AND enqueue_seq = ?`) + .get(...condition.params, candidate.id, candidate.enqueueSeq); + } + + readSnapshot(read: () => T): T { + return this.db.transaction(read)(); + } + + refreshPolicies(limit = 500): PolicyRefreshResult { + return this.refreshPendingPolicies(Math.max(0, limit), false, true); + } + + /** Admission maintenance is independent of eligibility. Even delayed/blocked + * dirty tasks leave the worklist after one bounded visit; they cannot be scanned + * again on every refill. Terminal history is absent from the active index. */ + refreshActivePolicies(input: { maxRows?: number; includeRetries?: boolean } = {}): PolicyRefreshResult { + if (input.maxRows !== undefined && (!Number.isSafeInteger(input.maxRows) || input.maxRows < 0)) { + throw new RangeError("maxRows must be a non-negative safe integer"); + } + return this.refreshPendingPolicies(input.maxRows, true, input.includeRetries !== false); + } + + private refreshPendingPolicies(limit: number | undefined, active: boolean, includeRetries: boolean): PolicyRefreshResult { + const result: PolicyRefreshResult = { refreshed: 0, unknown: [], ...(active ? { hasMore: false } : {}) }; + if (this.db.readonly) return result; + return this.db.transaction(() => { + for (const queue of ["main", "embedding_retry"] as const) { + if (queue === "embedding_retry" && !includeRetries) continue; + const table = queue === "main" ? "evolution_jobs" : "embedding_retry_queue"; + const target = queue === "main" ? "q.target_memory_id" : "q.target_id"; + const predicate = `${pendingPolicyPredicate}${active ? ` AND ${activePolicyPredicate(table, "q.")}` : ""}`; + for (;;) { + const remaining = limit === undefined ? 200 : Math.min(200, limit - result.refreshed); + if (remaining === 0) { + if (active && this.db.prepare(`SELECT 1 FROM ${table} q INDEXED BY ${pendingPolicyIndex(table, active)} + WHERE ${predicate} LIMIT 1`).get()) result.hasMore = true; + break; + } + const rows = this.db.prepare(`SELECT q.id, q.queue_lane AS previous_queue_lane, q.policy_promotion, q.status, + ${queue === "main" ? "q.job_type, q.payload_json" : "'embedding_retry' AS job_type, '{}' AS payload_json"}, + m.agent_id, m.memory_layer, m.tags_json, + json_extract(m.properties_json, '$.internal_info.plugin_algorithm') AS plugin_algorithm, + json_extract(m.properties_json, '$.internal_info.capture_origin') AS capture_origin + FROM ${table} q INDEXED BY ${pendingPolicyIndex(table, active)} LEFT JOIN memories m ON m.id = ${target} + WHERE ${predicate} ORDER BY q.enqueue_seq LIMIT ?`).all(remaining) as PolicyRow[]; + for (const row of rows) this.writePolicy(table, row, result); + if (rows.length < remaining) break; + } + } + return result; + })(); + } + + /** + * Complete the projection for the currently runnable set before real FIFO + * selection. Callers using maxRows must finish all slices before selecting: + * an older dirty row may precede an already-classified row in FIFO order. + * Terminal history, future deadlines and blocked dependencies are skipped. + */ + refreshRunnablePolicies(input: QueueEligibilityOptions & { + at: string; includeRetries?: boolean; maxRows?: number; + }): PolicyRefreshResult { + if (input.maxRows !== undefined && (!Number.isSafeInteger(input.maxRows) || input.maxRows < 0)) { + throw new RangeError("maxRows must be a non-negative safe integer"); + } + const result: PolicyRefreshResult = { + refreshed: 0, unknown: [], ...(input.maxRows !== undefined ? { hasMore: false } : {}) + }; + if (this.db.readonly) return result; + const main = mainQueueEligibility(input.at, input); + const retry = retryQueueEligibility(Date.parse(input.at), input.targetMemoryIds); + // Start from the event-maintained pending index, then check each dirty ID. + // Never materialize the whole eligible queue to discover missing metadata. + return this.db.transaction(() => { + for (const queue of ["main", "embedding_retry"] as const) { + if (queue === "embedding_retry" && input.includeRetries === false) continue; + if ((queue === "main" ? main.sql : retry.sql) === "0") continue; + const table = queue === "main" ? "evolution_jobs" : "embedding_retry_queue"; + const target = queue === "main" ? "q.target_memory_id" : "q.target_id"; + const ready = queue === "main" + ? `EXISTS (SELECT 1 FROM evolution_jobs WHERE id = q.id AND (${main.sql}))` + : retry.sql; + const parameters = queue === "main" ? main.params : retry.params; + if (input.maxRows === 0) { + const found = this.db.prepare(`SELECT 1 FROM ${table} q INDEXED BY ${pendingPolicyIndex(table)} + WHERE ${pendingPolicyPredicate} AND (${ready}) LIMIT 1`) + .get(...parameters); + if (found) { result.hasMore = true; return result; } + continue; + } + for (;;) { + const remaining = input.maxRows === undefined ? 200 : Math.min(200, input.maxRows - result.refreshed); + if (remaining === 0) { result.hasMore = true; return result; } + const rows = this.db.prepare(`SELECT q.id, q.queue_lane AS previous_queue_lane, q.policy_promotion, q.status, + ${queue === "main" ? "q.job_type, q.payload_json" : "'embedding_retry' AS job_type, '{}' AS payload_json"}, + m.agent_id, m.memory_layer, m.tags_json, + json_extract(m.properties_json, '$.internal_info.plugin_algorithm') AS plugin_algorithm, + json_extract(m.properties_json, '$.internal_info.capture_origin') AS capture_origin + FROM ${table} q INDEXED BY ${pendingPolicyIndex(table)} LEFT JOIN memories m ON m.id = ${target} + WHERE ${pendingPolicyPredicate} AND (${ready}) + ORDER BY q.enqueue_seq LIMIT ?`).all(...parameters, remaining) as PolicyRow[]; + if (rows.length === 0) break; + for (const row of rows) this.writePolicy(table, row, result); + } + } + return result; + })(); + } + + refreshOne(queue: SchedulingCandidate["queue"], id: string): PolicyRefreshResult { + const result: PolicyRefreshResult = { refreshed: 0, unknown: [] }; + try { + const table = queue === "main" ? "evolution_jobs" : "embedding_retry_queue"; + const target = queue === "main" ? "q.target_memory_id" : "q.target_id"; + const row = this.db.prepare(`SELECT q.id, q.queue_lane AS previous_queue_lane, q.policy_promotion, q.status, + ${queue === "main" ? "q.job_type, q.payload_json" : "'embedding_retry' AS job_type, '{}' AS payload_json"}, + m.agent_id, m.memory_layer, m.tags_json, + json_extract(m.properties_json, '$.internal_info.plugin_algorithm') AS plugin_algorithm, + json_extract(m.properties_json, '$.internal_info.capture_origin') AS capture_origin + FROM ${table} q LEFT JOIN memories m ON m.id = ${target} WHERE q.id = ? AND ${pendingPolicyPredicate}`).get(id) as PolicyRow | undefined; + if (row) this.writePolicy(table, row, result); + } catch (error) { + if (error instanceof JobLeaseLostError) throw error; + // A projection error must not turn a successful real enqueue into a + // failed one. Insert/update triggers leave it dirty for a later retry. + logger.warn("scheduler.policy_refresh_failed", { queue, jobId: id, ...memoryErrorFields(error) }); + } + return result; + } + + private writePolicy(table: string, row: PolicyRow, result: PolicyRefreshResult): void { + let policy: { queueLane: string; resourceClass: string; basePriority: number; policyVersion: number }; + let promotion: "onboarding_first_report" | "manual_retry" | null = null; + try { + const payload: unknown = JSON.parse(row.payload_json); + const tags: unknown = row.tags_json ? JSON.parse(row.tags_json) : []; + const input = { jobType: row.job_type, + payload: payload && typeof payload === "object" && !Array.isArray(payload) + ? payload as Record : {}, target: { + agentId: typeof row.agent_id === "string" ? row.agent_id : undefined, + memoryLayer: typeof row.memory_layer === "string" ? row.memory_layer : undefined, + tags: Array.isArray(tags) ? tags.filter((tag): tag is string => typeof tag === "string") : [], + pluginAlgorithm: typeof row.plugin_algorithm === "string" ? row.plugin_algorithm : undefined, + captureOrigin: typeof row.capture_origin === "string" ? row.capture_origin : undefined + } + }; + policy = jobPolicyRegistry.resolve(input); + // Business urgency, never lane weights: first report > explicit retry > + // ordinary classification. Keep earned urgency only for this unfinished task. + const unfinished = table === "evolution_jobs" + ? ["queued", "leased", "failed"].includes(row.status) + : ["pending", "in_progress"].includes(row.status); + if (unfinished) { + if (row.policy_promotion === "onboarding_first_report" || + (policy.queueLane === "interactive" && policy.basePriority === -100)) promotion = "onboarding_first_report"; + else if (row.policy_promotion === "manual_retry" || input.payload.source === "memory.processing.manual_retry") promotion = "manual_retry"; + } + if (promotion) policy = jobPolicyRegistry.resolve({ ...input, operationMode: promotion }); + } catch (error) { + if (!(error instanceof UnknownJobPolicyError)) throw error; + // Keep quarantine explicit. WRR admits these only for terminal cleanup; + // legacy retains unsupported-type handling. Never silently assign a policy. + result.unknown.push({ id: row.id, jobType: row.job_type }); + logger.warn("scheduler.unregistered_job", { jobId: row.id, jobType: row.job_type }); + policy = { queueLane: "unregistered", resourceClass: "unregistered", + basePriority: 0, policyVersion: JOB_POLICY_VERSION }; + } + const updated = this.db.prepare(`UPDATE ${table} SET queue_lane = ?, resource_class = ?, base_priority = ?, policy_version = ?, policy_promotion = ? WHERE id = ?`) + .run(policy.queueLane, policy.resourceClass, policy.basePriority, policy.policyVersion, promotion, row.id); + if (updated.changes > 0 && row.previous_queue_lane !== null) { + // Do not invalidate deep scans for ordinary new imports. An existing row + // can change lane/resource/scope even while ending at the same policy version. + // Rollback may over-invalidate; never roll the revision back to a cached value. + selectionRevisions.set(this.db, this.getSelectionRevision() + 1); + } + result.refreshed++; + } + + pendingPolicyCount(): number { + return Number((this.db.prepare(`SELECT + (SELECT COUNT(*) FROM evolution_jobs INDEXED BY ${pendingPolicyIndex("evolution_jobs")} WHERE ${pendingPolicyPredicate}) + + (SELECT COUNT(*) FROM embedding_retry_queue INDEXED BY ${pendingPolicyIndex("embedding_retry_queue")} WHERE ${pendingPolicyPredicate}) AS n`) + .get() as { n: number }).n); + } + + unregisteredPolicyCount(): number { + return Number((this.db.prepare(`SELECT + (SELECT COUNT(*) FROM evolution_jobs WHERE queue_lane = 'unregistered') + + (SELECT COUNT(*) FROM embedding_retry_queue WHERE queue_lane = 'unregistered') AS n`) + .get() as { n: number }).n); + } + + /** Fetch bounded candidates per lane/table. Type filters apply to main jobs, + * matching legacy semantics; includeRetries separately gates the retry table. */ + candidates(input: QueueEligibilityOptions & { + at: string; limit: number; includeRetries?: boolean; includeUnregistered?: boolean; + }): SchedulingCandidate[] { + const result: SchedulingCandidate[] = []; + if (input.limit <= 0) return result; + for (const lane of QUEUE_LANES) { + result.push(...this.candidatePage({ ...input, lane, queue: "main" })); + if (input.includeRetries !== false) result.push(...this.candidatePage({ ...input, lane, queue: "embedding_retry" })); + } + return result; + } + + /** Keyset pages let admission skip busy resources without hiding later FIFO rows. */ + candidatePage(input: QueueEligibilityOptions & { + at: string; limit: number; queue: SchedulingCandidate["queue"]; lane: QueueLane; + afterEnqueueSeq?: number; includeUnregistered?: boolean; + /** Discovery only; ID eligibility and lease CAS still enforce dependencies. */ + includeDependencyBlocked?: boolean; + }): SchedulingCandidate[] { + if (input.limit <= 0) return []; + const main = mainQueueEligibility(input.at, input, input.includeDependencyBlocked); + const retry = retryQueueEligibility(Date.parse(input.at), input.targetMemoryIds); + const limit = Math.max(1, Math.min(200, Math.floor(input.limit))); + const includeUnknown = input.includeUnregistered === true && input.lane === "maintenance"; + const laneFilter = includeUnknown ? "queue_lane IN (?, 'unregistered')" : "queue_lane = ?"; + const after = input.afterEnqueueSeq ?? 0; + if (input.queue === "main") { + const rows = this.db.prepare(`SELECT id, job_type, queue_lane, enqueue_seq, base_priority, first_queued_at FROM evolution_jobs + WHERE ${main.sql} AND ${laneFilter} AND policy_version = ? AND enqueue_seq > ? + ORDER BY enqueue_seq LIMIT ?`).all(...main.params, input.lane, JOB_POLICY_VERSION, after, limit) as Array<{ + id: string; job_type: string; queue_lane: QueueLane | "unregistered"; enqueue_seq: number; base_priority: number; first_queued_at: string | null + }>; + return rows.map((row) => ({ queue: "main" as const, id: row.id, jobType: row.job_type, + lane: row.queue_lane === "unregistered" ? "maintenance" : row.queue_lane, + enqueueSeq: row.enqueue_seq, basePriority: row.base_priority, firstQueuedAt: row.first_queued_at, + ...(row.queue_lane === "unregistered" ? { unregistered: true } : {}) })); + } + const retries = this.db.prepare(`SELECT q.id, q.queue_lane, q.enqueue_seq, q.base_priority, q.first_queued_at FROM embedding_retry_queue q + WHERE ${retry.sql} AND q.${laneFilter} AND q.policy_version = ? AND q.enqueue_seq > ? + ORDER BY q.enqueue_seq LIMIT ?`).all(...retry.params, input.lane, JOB_POLICY_VERSION, after, limit) as Array<{ + id: string; queue_lane: QueueLane | "unregistered"; enqueue_seq: number; base_priority: number; first_queued_at: string | null + }>; + return retries.map((row) => ({ queue: "embedding_retry" as const, id: row.id, jobType: "embedding_retry", + lane: row.queue_lane === "unregistered" ? "maintenance" : row.queue_lane, + enqueueSeq: row.enqueue_seq, basePriority: row.base_priority, firstQueuedAt: row.first_queued_at, + ...(row.queue_lane === "unregistered" ? { unregistered: true } : {}) })); + } +} diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index 897936263..d62a20140 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -1,3 +1,4 @@ +import { MAIN_DEADLINE, MAIN_TIMED, MAIN_LEASED, RETRY_DEADLINE, RETRY_LEASED } from "./queue-deadline-schema.js"; import type { EmbeddingMaintenanceEntry } from "./sqlite-vec-store.js"; import { sameEmbeddingIdentity, vectorEmbeddingIdentity, type EmbeddingIdentity } from "../model/embedding-identity.js"; import type Database from "better-sqlite3"; @@ -43,6 +44,7 @@ import { newId, stableHash } from "../utils/id.js"; import { asStringArray, isRecord, parseJson, toJson } from "../utils/json.js"; import { firstSemanticUserLine, semanticUserText } from "../utils/text.js"; import { nowIso } from "../utils/time.js"; +import { assertCurrentJobLease, withoutJobLeaseGuard } from "./job-lease-context.js"; import { attachMemoryVectors, dirtyMemoryVectorEntries, @@ -56,6 +58,8 @@ import { VECTOR_SEARCH_WINDOW, type VectorSearchCandidate } from "./sqlite-vec-store.js"; +import { QueueSchedulingStore, type SchedulingMetadata } from "./queue-scheduling-store.js"; +import { mainQueueEligibility, retryQueueEligibility, type QueueEligibilityOptions } from "./queue-eligibility.js"; import { compactHistoryChangePayload, compactHistoryChangeRow } from "./history-change-snapshot.js"; type SqlValue = string | number | Buffer | null; @@ -307,7 +311,7 @@ export interface ApiLogRecord { calledAt: string; } -export interface EvolutionJobRecord { +export interface EvolutionJobRecord extends SchedulingMetadata { id: string; jobType: JobType; status: JobStatus; @@ -322,6 +326,13 @@ export interface EvolutionJobRecord { attempts: number; maxAttempts: number; leasedUntil?: string | null; + leaseOwner?: string | null; + leaseToken?: string | null; + availableAt?: string | null; + blockedReason?: string | null; + failureCount?: number; + leaseCount?: number; + leaseRecoveryCount?: number; lastError?: string | null; createdAt: string; updatedAt: string; @@ -441,7 +452,7 @@ export interface MemoryVectorSearchOptions { anyOfTags?: string[]; } -export interface EmbeddingRetryRecord { +export interface EmbeddingRetryRecord extends SchedulingMetadata { id: string; targetKind: EmbeddingRetryTargetKind; targetId: string; @@ -450,10 +461,16 @@ export interface EmbeddingRetryRecord { embedRole: "document" | "query"; status: EmbeddingRetryStatus; attempts: number; + /** Cumulative claims; migrated history is a lower bound. */ + leaseCount?: number; + /** Expired-lease recoveries observed since counter migration. */ + leaseRecoveryCount?: number; maxAttempts: number; nextAttemptAt: number; claimedBy?: string | null; leaseUntil?: number | null; + leaseToken?: string | null; + blockedReason?: string | null; lastError?: string | null; createdAt: number; updatedAt: number; @@ -840,6 +857,30 @@ export class MemoryRepository { return ids.map((id) => byId.get(id)).filter((row): row is MemoryRow => Boolean(row)); } + /** Title scheduling needs provenance, never conversation bodies or vectors. */ + areAllHistoricalCaptureMemories(ids: readonly string[]): boolean { + const distinctIds = [...new Set(ids)]; + if (distinctIds.length === 0) return false; + const row = this.db.prepare(`SELECT COUNT(*) AS found, + COALESCE(SUM(CASE WHEN + json_extract(properties_json, '$.internal_info.capture_origin') = 'agent_source_scan' + AND lower(COALESCE(agent_id, '')) != 'memmy-onboarding' + AND NOT EXISTS (SELECT 1 FROM ( + SELECT value FROM json_each(CASE WHEN json_type(tags_json) = 'array' THEN tags_json ELSE '[]' END) + UNION ALL + SELECT value FROM json_each(CASE WHEN json_type(info_json, '$.tags') = 'array' + THEN json_extract(info_json, '$.tags') ELSE '[]' END) + UNION ALL + SELECT value FROM json_each(CASE WHEN json_type(properties_json, '$.tags') = 'array' + THEN json_extract(properties_json, '$.tags') ELSE '[]' END) + ) WHERE lower(value) IN ('first-encounter-report', 'onboarding-report')) + THEN 1 ELSE 0 END), 0) AS historical + FROM memories + WHERE id IN (SELECT value FROM json_each(?)) AND deleted_at IS NULL`) + .get(toJson(distinctIds)) as { found: number; historical: number }; + return row.found === distinctIds.length && row.historical === row.found; + } + list(filter: MemoryFilter = {}, limit = 50, offset = 0): MemoryRow[] { const built = buildMemoryWhere(filter); const rows = this.db @@ -1856,11 +1897,14 @@ function sourceTurnCaptureFromSql(row: Record): SourceTurnCaptu } export class RuntimeRepository { + readonly scheduling: QueueSchedulingStore; + private readonly leaseOwner = newId("worker"); private readonly scheduledLogPrunes = new Set(); private historyLogCompactionScheduled = false; private historyLogCompactionCursor = 0; constructor(private readonly db: Database.Database) { + this.scheduling = new QueueSchedulingStore(db); this.scheduleHistoryLogCompaction(); } @@ -1869,7 +1913,7 @@ export class RuntimeRepository { if (reset) this.historyLogCompactionCursor = 0; if (this.historyLogCompactionScheduled || !this.db.open || this.db.readonly) return; this.historyLogCompactionScheduled = true; - setImmediate(() => { + withoutJobLeaseGuard(() => setImmediate(() => { this.historyLogCompactionScheduled = false; if (!this.db.open || this.db.readonly) return; try { @@ -1890,7 +1934,7 @@ export class RuntimeRepository { // Best-effort maintenance must not interfere with capture or worker writes. // An interrupted pass is retried on the next service start. } - }); + })); } getKv(key: string): { value: unknown; updatedAt: string } | undefined { @@ -3456,24 +3500,35 @@ export class RuntimeRepository { enqueueJobInTransaction(job: EvolutionJobRecord): EvolutionJobRecord { const existing = job.dedupeKey ? this.getActiveJobByDedupeKey(job.dedupeKey) : undefined; if (existing) { + // Capture an older/raw producer's explicit promotion before merging a new + // delivery's payload. Current projections are a cheap indexed no-op. + this.scheduling.refreshOne("main", existing.id); const updatedAt = job.updatedAt ?? nowIso(); const payload = mergeJobPayload(existing.payload, job.payload); + const manualRetry = existing.status === "failed" && job.payload.source === "memory.processing.manual_retry"; this.db .prepare( `UPDATE evolution_jobs - SET status = CASE WHEN status = 'failed' THEN 'queued' ELSE status END, + SET status = CASE WHEN @manualRetry = 1 THEN 'queued' ELSE status END, + attempts = CASE WHEN @manualRetry = 1 THEN 0 ELSE attempts END, + failure_count = CASE WHEN @manualRetry = 1 THEN 0 ELSE failure_count END, + available_at = CASE WHEN @manualRetry = 1 THEN NULL ELSE available_at END, + blocked_reason = CASE WHEN @manualRetry = 1 THEN NULL ELSE blocked_reason END, + lease_owner = CASE WHEN @manualRetry = 1 THEN NULL ELSE lease_owner END, + lease_token = CASE WHEN @manualRetry = 1 THEN NULL ELSE lease_token END, + leased_until = CASE WHEN @manualRetry = 1 THEN NULL ELSE leased_until END, + last_error = CASE WHEN @manualRetry = 1 THEN NULL ELSE last_error END, session_id = COALESCE(@sessionId, session_id), episode_id = COALESCE(@episodeId, episode_id), target_memory_id = COALESCE(@targetMemoryId, target_memory_id), payload_json = @payloadJson, max_attempts = MAX(max_attempts, @maxAttempts), - leased_until = CASE WHEN status = 'failed' THEN NULL ELSE leased_until END, - last_error = CASE WHEN status = 'failed' THEN NULL ELSE last_error END, updated_at = @updatedAt WHERE id = @id` ) .run({ id: existing.id, + manualRetry: manualRetry ? 1 : 0, sessionId: job.sessionId ?? null, episodeId: job.episodeId ?? null, targetMemoryId: job.targetMemoryId ?? null, @@ -3481,13 +3536,12 @@ export class RuntimeRepository { maxAttempts: job.maxAttempts, updatedAt }); + this.scheduling.refreshOne("main", existing.id); return this.getJob(existing.id) ?? { ...existing, payload, updatedAt, - status: existing.status === "failed" ? "queued" : existing.status, - leasedUntil: existing.status === "failed" ? null : existing.leasedUntil, - lastError: existing.status === "failed" ? null : existing.lastError + status: existing.status }; } this.db @@ -3514,7 +3568,8 @@ export class RuntimeRepository { leasedUntil: job.leasedUntil ?? null, lastError: job.lastError ?? null }); - return job; + this.scheduling.refreshOne("main", job.id); + return this.getJob(job.id) ?? job; } /** Allocate the next FIFO sequence for Work Memory jobs in a merge scope. */ @@ -3600,6 +3655,8 @@ export class RuntimeRepository { payload_json = ?, attempts = 0, leased_until = NULL, + lease_owner = NULL, lease_token = NULL, + available_at = NULL, blocked_reason = NULL, failure_count = 0, last_error = NULL, updated_at = ? WHERE id = ?` @@ -3658,6 +3715,9 @@ export class RuntimeRepository { jobTypes?: readonly string[]; excludedJobTypes?: readonly string[]; includeEmbeddingRetries?: boolean; + targetMemoryIds?: readonly string[]; + /** Ignore already due rows when scheduling a running pump's next wake. */ + futureAfterMs?: number; }): number | undefined { const jobTypes = options?.jobTypes ?? (options?.jobType ? [options.jobType] : undefined); const jobTypeFilter = jobTypes @@ -3673,32 +3733,26 @@ export class RuntimeRepository { ...(jobTypes && jobTypes.length > 0 ? jobTypes : []), ...(excluded ?? []) ]; + const targets = options?.targetMemoryIds; + if (targets?.length === 0) return undefined; + const targetFilter = targets ? `AND target_memory_id IN (${targets.map(() => "?").join(", ")})` : ""; + const retryTargetFilter = targets ? `AND target_id IN (${targets.map(() => "?").join(", ")})` : ""; + const future = options?.futureAfterMs; + const futureIso = future === undefined ? undefined : new Date(future).toISOString(); + const mainParams = [...jobTypeParams, ...(targets ?? []), ...(futureIso ? [futureIso] : [])]; const queuedJob = this.db - .prepare( - `SELECT CAST(json_extract(payload_json, '$.runAfter') AS TEXT) AS run_after - FROM evolution_jobs - WHERE status = 'queued' - AND attempts < max_attempts - AND json_type(payload_json, '$.runAfter') = 'text' - ${jobTypeFilter} - ${excludeFilter} - ORDER BY run_after ASC - LIMIT 1` - ) - .get(...jobTypeParams) as { run_after: string } | undefined; + .prepare(`SELECT ${MAIN_DEADLINE} AS run_after + FROM evolution_jobs INDEXED BY worker_main_deadline + WHERE ${MAIN_TIMED} ${jobTypeFilter} ${excludeFilter} ${targetFilter} + ${futureIso ? `AND ${MAIN_DEADLINE} > ?` : ""} + ORDER BY ${MAIN_DEADLINE} ASC LIMIT 1`) + .get(...mainParams) as { run_after: string } | undefined; const leasedJob = this.db - .prepare( - `SELECT leased_until - FROM evolution_jobs - WHERE status = 'leased' - AND attempts < max_attempts - AND leased_until IS NOT NULL - ${jobTypeFilter} - ${excludeFilter} - ORDER BY leased_until ASC - LIMIT 1` - ) - .get(...jobTypeParams) as { leased_until: string } | undefined; + .prepare(`SELECT leased_until FROM evolution_jobs INDEXED BY worker_main_lease_deadline + WHERE ${MAIN_LEASED} ${jobTypeFilter} ${excludeFilter} ${targetFilter} + ${futureIso ? "AND leased_until > ?" : ""} + ORDER BY leased_until ASC LIMIT 1`) + .get(...mainParams) as { leased_until: string } | undefined; const includeEmbeddingRetries = options?.includeEmbeddingRetries !== false; const pendingEmbedding = includeEmbeddingRetries ? this.db @@ -3706,24 +3760,25 @@ export class RuntimeRepository { `SELECT next_attempt_at FROM embedding_retry_queue WHERE status = 'pending' + ${retryTargetFilter} + ${future !== undefined ? "AND next_attempt_at > ?" : ""} ORDER BY next_attempt_at ASC LIMIT 1` ) - .get() as { next_attempt_at: number } | undefined + .get(...(targets ?? []), ...(future === undefined ? [] : [future])) as { next_attempt_at: number } | undefined : undefined; const inProgressEmbedding = includeEmbeddingRetries ? this.db .prepare( - `SELECT CASE - WHEN claimed_by IS NULL OR lease_until IS NULL THEN next_attempt_at - ELSE MAX(next_attempt_at, lease_until) - END AS run_at - FROM embedding_retry_queue - WHERE status = 'in_progress' + `SELECT ${RETRY_DEADLINE} AS run_at + FROM embedding_retry_queue INDEXED BY worker_retry_lease_deadline + WHERE ${RETRY_LEASED} + ${retryTargetFilter} + ${future !== undefined ? `AND ${RETRY_DEADLINE} > ?` : ""} ORDER BY run_at ASC LIMIT 1` ) - .get() as { run_at: number } | undefined + .get(...(targets ?? []), ...(future === undefined ? [] : [future])) as { run_at: number } | undefined : undefined; const times = [ queuedJob ? Date.parse(queuedJob.run_after) : Number.NaN, @@ -3829,196 +3884,180 @@ export class RuntimeRepository { return Boolean(row); } + /** Legacy order without side effects, for stage-3 shadow comparison only. */ + previewQueuedJobs(input: { + at: string; limit: number; targetMemoryIds?: readonly string[]; + priorityCohortOnly?: boolean; allowedJobTypes?: readonly string[]; excludedJobTypes?: readonly string[]; + }): EvolutionJobRecord[] { + const eligibility = mainQueueEligibility(input.at, input); + const rows = this.db.prepare(`SELECT *, ${evolutionJobPrioritySql()} AS queue_priority + FROM evolution_jobs WHERE ${eligibility.sql} ORDER BY ${evolutionJobOrderSql()} LIMIT ?`) + .all(...eligibility.params, input.limit) as Array; + return (input.priorityCohortOnly && rows[0] + ? rows.filter((row) => row.queue_priority === rows[0]!.queue_priority) : rows).map(jobFromSql); + } + + previewDueEmbeddingRetries(input: { now: number; limit: number; targetMemoryIds?: readonly string[] }): EmbeddingRetryRecord[] { + if (input.limit <= 0) return []; + const eligibility = retryQueueEligibility(input.now, input.targetMemoryIds); + return (this.db.prepare(`SELECT q.* FROM embedding_retry_queue q LEFT JOIN memories m ON m.id = q.target_id + WHERE ${eligibility.sql} ORDER BY ${embeddingRetryOrderSql()} LIMIT ?`) + .all(...eligibility.params, input.limit) as SqlEmbeddingRetryRow[]).map(embeddingRetryFromSql); + } + + /** Legacy ordering for resource admission, paged without acquiring a lease. */ + previewLegacyJobPage(input: QueueEligibilityOptions & { + at: string; limit: number; offset?: number; excludedIds?: readonly string[]; + }): Array<{ job: EvolutionJobRecord; priority: number }> { + if (input.limit <= 0) return []; + const eligibility = mainQueueEligibility(input.at, input); + const rows = this.db.prepare(`SELECT *, ${evolutionJobPrioritySql()} AS queue_priority + FROM evolution_jobs WHERE ${eligibility.sql} + ${input.excludedIds?.length ? "AND id NOT IN (SELECT value FROM json_each(?))" : ""} + ORDER BY ${evolutionJobOrderSql()} LIMIT ? OFFSET ?`) + .all(...eligibility.params, ...(input.excludedIds?.length ? [JSON.stringify(input.excludedIds)] : []), + Math.min(200, Math.floor(input.limit)), Math.max(0, Math.floor(input.offset ?? 0))) as Array; + return rows.map(row => ({ job: jobFromSql(row), priority: row.queue_priority })); + } + + previewLegacyRetryPage(input: { now: number; limit: number; offset?: number; targetMemoryIds?: readonly string[]; excludedIds?: readonly string[] }): EmbeddingRetryRecord[] { + if (input.limit <= 0) return []; + const eligibility = retryQueueEligibility(input.now, input.targetMemoryIds); + return (this.db.prepare(`SELECT q.* FROM embedding_retry_queue q LEFT JOIN memories m ON m.id = q.target_id + WHERE ${eligibility.sql} + ${input.excludedIds?.length ? "AND q.id NOT IN (SELECT value FROM json_each(?))" : ""} + ORDER BY ${embeddingRetryOrderSql()} LIMIT ? OFFSET ?`) + .all(...eligibility.params, ...(input.excludedIds?.length ? [JSON.stringify(input.excludedIds)] : []), + Math.min(200, Math.floor(input.limit)), Math.max(0, Math.floor(input.offset ?? 0))) as SqlEmbeddingRetryRow[]).map(embeddingRetryFromSql); + } + leaseQueuedJobs( limit = 10, leaseSeconds = 60, targetMemoryIds?: readonly string[], priorityCohortOnly = false, allowedJobTypes?: readonly string[], - excludedJobTypes?: readonly string[] + excludedJobTypes?: readonly string[], + owner = this.leaseOwner ): EvolutionJobRecord[] { if (targetMemoryIds?.length === 0 || allowedJobTypes?.length === 0) { return []; } const at = nowIso(); const leaseUntil = new Date(Date.now() + leaseSeconds * 1000).toISOString(); - const targetFilter = targetMemoryIds - ? `AND target_memory_id IN (${targetMemoryIds.map(() => "?").join(", ")})` - : ""; + const eligibility = mainQueueEligibility(at, { targetMemoryIds, allowedJobTypes, excludedJobTypes }); const transaction = this.db.transaction(() => { - const candidates = this.db - .prepare( - `SELECT *, ${evolutionJobPrioritySql()} AS queue_priority - FROM evolution_jobs - WHERE (status = 'queued' - OR (status = 'leased' AND leased_until IS NOT NULL AND leased_until <= ?)) - AND attempts < max_attempts - AND NOT ( - job_type = 'feedback_experience' - AND EXISTS ( - SELECT 1 FROM raw_turns pending_feedback_turn - WHERE pending_feedback_turn.id = json_extract(evolution_jobs.payload_json, '$.rawTurnId') - AND pending_feedback_turn.deleted_at IS NULL AND pending_feedback_turn.redacted_at IS NULL - AND (pending_feedback_turn.episode_id IS NULL OR - json_extract(pending_feedback_turn.message_payload_json, '$.episode_assignment.status') IN ('pending', 'selected', 'applying')) - ) - ) - AND NOT ( - job_type = 'reward' - AND json_extract(payload_json, '$.l1MemoryId') IS NOT NULL - AND ( - json_extract(payload_json, '$.polarity') IS NULL - OR json_extract(payload_json, '$.polarity') = 'positive' - ) - AND EXISTS ( - SELECT 1 - FROM memory_processing_state - JOIN memories ON memories.id = memory_processing_state.memory_id - WHERE memory_processing_state.memory_id = - CAST(json_extract(evolution_jobs.payload_json, '$.l1MemoryId') AS TEXT) - AND NOT EXISTS ( - SELECT 1 - FROM memory_vector_entries - WHERE memory_vector_entries.memory_id = memory_processing_state.memory_id - AND memory_vector_entries.vector_field = 'vec_summary' - ) - AND ( - memory_processing_state.state IN ('embedding_pending', 'embedding') - OR memory_processing_state.state IN ('summary_pending', 'summarizing') - ) - ) - ) - AND ( - json_extract(payload_json, '$.runAfter') IS NULL - OR CAST(json_extract(payload_json, '$.runAfter') AS TEXT) <= ? - ) - AND ( - job_type NOT IN ('l3_world_model_update', 'work_memory_extract') - OR ( - scope_key IS NOT NULL - AND scope_seq IS NOT NULL - AND NOT EXISTS ( - SELECT 1 - FROM evolution_jobs AS leased_l3_job - WHERE leased_l3_job.job_type = evolution_jobs.job_type - AND leased_l3_job.scope_key = evolution_jobs.scope_key - AND leased_l3_job.status = 'leased' - AND leased_l3_job.id <> evolution_jobs.id - ) - AND NOT EXISTS ( - SELECT 1 - FROM evolution_jobs AS earlier_l3_job - WHERE earlier_l3_job.job_type = evolution_jobs.job_type - AND earlier_l3_job.scope_key = evolution_jobs.scope_key - AND earlier_l3_job.scope_seq < evolution_jobs.scope_seq - AND earlier_l3_job.status IN ('queued', 'leased', 'failed') - ) - ) - ) - AND NOT ( - job_type = 'trace_summary' - AND json_extract(payload_json, '$.turnJudge') = 1 - AND EXISTS ( - SELECT 1 - FROM evolution_jobs AS earlier_turn_job - WHERE earlier_turn_job.job_type = 'trace_summary' - AND earlier_turn_job.session_id = evolution_jobs.session_id - AND json_extract(earlier_turn_job.payload_json, '$.turnJudge') = 1 - AND earlier_turn_job.status IN ('queued', 'leased', 'failed') - AND earlier_turn_job.id <> evolution_jobs.id - AND ( - CAST(json_extract(earlier_turn_job.payload_json, '$.turnAt') AS TEXT) - < CAST(json_extract(evolution_jobs.payload_json, '$.turnAt') AS TEXT) - OR (json_extract(earlier_turn_job.payload_json, '$.turnAt') = json_extract(evolution_jobs.payload_json, '$.turnAt') - AND COALESCE((SELECT rowid FROM raw_turns WHERE id = COALESCE( - json_extract(earlier_turn_job.payload_json, '$.rawTurnId'), - (SELECT json_extract(properties_json, '$.internal_info.trace.raw_turn_id') FROM memories WHERE id = earlier_turn_job.target_memory_id) - )), earlier_turn_job.rowid) - < COALESCE((SELECT rowid FROM raw_turns WHERE id = COALESCE( - json_extract(evolution_jobs.payload_json, '$.rawTurnId'), - (SELECT json_extract(properties_json, '$.internal_info.trace.raw_turn_id') FROM memories WHERE id = evolution_jobs.target_memory_id) - )), evolution_jobs.rowid)) - ) - ) - ) - AND NOT ( - (job_type IN ('episode_assignment', 'episode_finalize') OR (job_type = 'trace_summary' AND COALESCE(json_extract(payload_json, '$.turnJudge'), 0) = 1)) - AND json_extract(payload_json, '$.rawTurnId') IS NOT NULL - AND EXISTS ( - SELECT 1 FROM raw_turns prior JOIN raw_turns current ON current.id = json_extract(evolution_jobs.payload_json, '$.rawTurnId') - WHERE prior.session_id = current.session_id AND prior.deleted_at IS NULL AND prior.redacted_at IS NULL - AND json_extract(prior.message_payload_json, '$.episode_assignment.status') IN ('pending', 'selected', 'applying') - AND (prior.created_at < current.created_at OR (prior.created_at = current.created_at AND prior.rowid < current.rowid)) - ) - ) - AND (job_type <> 'episode_finalize' OR EXISTS ( - SELECT 1 FROM raw_turns r WHERE r.id = json_extract(evolution_jobs.payload_json, '$.rawTurnId') - AND (r.deleted_at IS NOT NULL OR r.redacted_at IS NOT NULL OR ( - json_extract(r.message_payload_json, '$.episode_assignment.selection') IS NOT NULL - AND (COALESCE(json_extract(r.message_payload_json, '$.episode_assignment.turnJudge'), 0) = 0 - OR json_extract(r.message_payload_json, '$.episode_assignment.judgment') IS NOT NULL - OR json_extract(r.message_payload_json, '$.turn_judge.status') = 'skipped') - )) - )) - ${allowedJobTypes ? `AND job_type IN (${allowedJobTypes.map(() => "?").join(", ")})` : ""} - ${excludedJobTypes?.length ? `AND job_type NOT IN (${excludedJobTypes.map(() => "?").join(", ")})` : ""} - ${targetFilter} - ORDER BY ${evolutionJobOrderSql()} - LIMIT ?` - ) - .all(at, at, ...(allowedJobTypes ?? []), ...(excludedJobTypes ?? []), ...(targetMemoryIds ?? []), limit) as Array; + const candidates = this.db.prepare(`SELECT *, ${evolutionJobPrioritySql()} AS queue_priority + FROM evolution_jobs WHERE ${eligibility.sql} ORDER BY ${evolutionJobOrderSql()} LIMIT ?`) + .all(...eligibility.params, limit) as Array; const queuePriority = candidates[0]?.queue_priority; const rows = priorityCohortOnly && queuePriority !== undefined ? candidates.filter((row) => row.queue_priority === queuePriority) : candidates; + const claimed: EvolutionJobRecord[] = []; for (const row of rows) { + const token = newId("lease"); + const recovered = row.status === "leased" ? 1 : 0; this.db .prepare( `UPDATE evolution_jobs SET status = 'leased', - attempts = attempts + 1, + attempts = attempts + 1 - ?, leased_until = ?, + lease_owner = ?, lease_token = ?, + lease_count = lease_count + 1, + lease_recovery_count = lease_recovery_count + ?, + available_at = NULL, updated_at = ? WHERE id = ?` ) - .run(leaseUntil, at, row.id); + .run(recovered, leaseUntil, owner, token, recovered, at, row.id); + claimed.push(this.getJob(row.id)!); } - return rows.map((row) => - jobFromSql({ - ...row, - status: "leased", - attempts: row.attempts + 1, - leased_until: leaseUntil, - updated_at: at - }) - ); + return claimed; }); return transaction(); } - completeJob(id: string, at = nowIso()): EvolutionJobRecord | undefined { - this.db - .prepare( - `UPDATE evolution_jobs - SET status = 'succeeded', - leased_until = NULL, - updated_at = ? - WHERE id = ?` - ) - .run(at, id); - return this.getJob(id); + /** Claim the specific WRR-selected candidate; never substitute a legacy head. */ + claimQueuedJob(id: string, input: QueueEligibilityOptions & { + at?: string; leaseSeconds?: number; owner?: string; + } = {}): EvolutionJobRecord | undefined { + const at = input.at ?? nowIso(); + const leaseUntil = new Date(Date.parse(at) + (input.leaseSeconds ?? 60) * 1000).toISOString(); + const eligibility = mainQueueEligibility(at, input); + return this.db.transaction(() => { + const row = this.db.prepare(`UPDATE evolution_jobs SET + attempts = attempts + CASE WHEN status = 'leased' THEN 0 ELSE 1 END, + lease_recovery_count = lease_recovery_count + CASE WHEN status = 'leased' THEN 1 ELSE 0 END, + status = 'leased', leased_until = ?, lease_owner = ?, lease_token = ?, + lease_count = lease_count + 1, available_at = NULL, updated_at = ? + WHERE id = ? AND (${eligibility.sql}) RETURNING *`) + .get(leaseUntil, input.owner ?? this.leaseOwner, newId("lease"), at, id, ...eligibility.params) as SqlJobRow | undefined; + return row ? jobFromSql(row) : undefined; + })(); + } + + /** A missing claim is reserved for unowned legacy/admin rows only. */ + private jobClaimWhere(claim: EvolutionJobRecord | undefined, at: string): { sql: string; params: SqlValue[] } { + return claim ? { + sql: "status = 'leased' AND lease_owner = ? AND lease_token = ? AND leased_until > ?", + params: [claim.leaseOwner ?? "", claim.leaseToken ?? "", at] + } : { sql: "status IN ('queued', 'leased') AND lease_owner IS NULL AND lease_token IS NULL", params: [] }; + } + + isJobLeaseHeld(job: EvolutionJobRecord, at = nowIso()): boolean { + if (!job.leaseOwner || !job.leaseToken) return false; + const claim = this.jobClaimWhere(job, at); + return Boolean(this.db.prepare(`SELECT 1 FROM evolution_jobs WHERE id = ? AND ${claim.sql}`) + .get(job.id, ...claim.params)); + } + + renewJobLease(job: EvolutionJobRecord, leaseSeconds = 60, at = nowIso()): EvolutionJobRecord | undefined { + if (!job.leaseOwner || !job.leaseToken) return undefined; + const claim = this.jobClaimWhere(job, at); + const until = new Date(Date.parse(at) + Math.max(1, leaseSeconds) * 1000).toISOString(); + const result = this.db.prepare(`UPDATE evolution_jobs SET leased_until = MAX(leased_until, ?), updated_at = ? + WHERE id = ? AND ${claim.sql}`).run(until, at, job.id, ...claim.params); + return result.changes > 0 ? this.getJob(job.id) : undefined; + } + + completeJob(id: string, at = nowIso(), claim?: EvolutionJobRecord): EvolutionJobRecord | undefined { + const where = this.jobClaimWhere(claim, at); + const result = this.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded', + leased_until = NULL, lease_owner = NULL, lease_token = NULL, + available_at = NULL, blocked_reason = NULL, updated_at = ? + WHERE id = ? AND ${where.sql}`).run(at, id, ...where.params); + return result.changes > 0 ? this.getJob(id) : undefined; } /** Waiting for a dependency is not a model failure and does not consume an attempt. */ - deferProcessingJob(id: string, runAfter: string, at = nowIso(), restoreAttempt = true): EvolutionJobRecord | undefined { - this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, - attempts = MAX(0, attempts - ?), updated_at = ?, + deferProcessingJob(id: string, runAfter: string, at = nowIso(), claim?: EvolutionJobRecord): EvolutionJobRecord | undefined { + const where = this.jobClaimWhere(claim, at); + const result = this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, + lease_owner = NULL, lease_token = NULL, available_at = ?, blocked_reason = NULL, + attempts = MAX(0, attempts - 1), updated_at = ?, payload_json = json_set(payload_json, '$.runAfter', ?) - WHERE id = ? AND status = 'leased'`).run(restoreAttempt ? 1 : 0, at, runAfter, id); - return this.getJob(id); + WHERE id = ? AND ${where.sql}`).run(runAfter, at, runAfter, id, ...where.params); + return result.changes > 0 ? this.getJob(id) : undefined; + } + + releaseClaimedJobs(jobs: readonly EvolutionJobRecord[], at = nowIso()): Array<{ before: EvolutionJobRecord; after: EvolutionJobRecord }> { + return this.db.transaction(() => { + const result: Array<{ before: EvolutionJobRecord; after: EvolutionJobRecord }> = []; + for (const job of jobs) { + if (!this.isJobLeaseHeld(job, at)) continue; + const before = this.getJob(job.id)!; + const where = this.jobClaimWhere(job, at); + const changed = this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', + leased_until = NULL, lease_owner = NULL, lease_token = NULL, + attempts = MAX(0, attempts - 1), updated_at = ? + WHERE id = ? AND ${where.sql}`).run(at, job.id, ...where.params); + if (changed.changes > 0) result.push({ before, after: this.getJob(job.id)! }); + } + return result; + })(); } requeueFailedEpisodeAssignments(at = nowIso(), rawTurnId?: string): Array<{ before: EvolutionJobRecord; after: EvolutionJobRecord }> { @@ -4031,7 +4070,7 @@ export class RuntimeRepository { AND json_extract(r.message_payload_json,'$.episode_assignment.generation') = json_extract(j.payload_json,'$.generation'))`) .all(...(rawTurnId ? [rawTurnId] : [])) as SqlJobRow[]; return rows.map(row => { - this.db.prepare("UPDATE evolution_jobs SET status='queued',attempts=0,leased_until=NULL,last_error=NULL,updated_at=?,payload_json=json_remove(payload_json,'$.runAfter') WHERE id=?").run(at,row.id); + this.db.prepare("UPDATE evolution_jobs SET status='queued',attempts=0,failure_count=0,leased_until=NULL,lease_owner=NULL,lease_token=NULL,available_at=NULL,blocked_reason=NULL,last_error=NULL,updated_at=?,payload_json=json_remove(payload_json,'$.runAfter') WHERE id=?").run(at,row.id); return { before: jobFromSql(row), after: this.getJob(row.id)! }; }); })(); @@ -4054,12 +4093,14 @@ export class RuntimeRepository { `SELECT * FROM evolution_jobs WHERE status = 'failed' - AND attempts < max_attempts + AND failure_count < max_attempts + AND (blocked_reason IS NULL OR blocked_reason != 'configuration') + AND (available_at IS NULL OR available_at <= ?) ${targetFilter} ORDER BY ${evolutionJobOrderSql()} LIMIT ?` ) - .all(...(targetMemoryIds ?? []), limit) as SqlJobRow[]; + .all(at, ...(targetMemoryIds ?? []), limit) as SqlJobRow[]; for (const row of rows) { this.db @@ -4067,6 +4108,7 @@ export class RuntimeRepository { `UPDATE evolution_jobs SET status = 'queued', leased_until = NULL, + lease_owner = NULL, lease_token = NULL, updated_at = ? WHERE id = ?` ) @@ -4090,42 +4132,24 @@ export class RuntimeRepository { at = nowIso(), activeCaptureIds: readonly string[] = [] ): Array<{ before: EvolutionJobRecord; after: EvolutionJobRecord }> { - const transaction = this.db.transaction(() => { - const rows = this.db - .prepare( - `SELECT * - FROM evolution_jobs - WHERE status = 'leased' - ${activeCaptureIds.length ? `AND id NOT IN (${activeCaptureIds.map(() => '?').join(', ')})` : ''} - ORDER BY ${evolutionJobOrderSql()}` - ) - .all(...activeCaptureIds) as SqlJobRow[]; - - for (const row of rows) { - this.db - .prepare( - `UPDATE evolution_jobs - SET status = 'queued', - attempts = MAX(0, attempts - 1), - leased_until = NULL, - updated_at = ? - WHERE id = ?` - ) - .run(at, row.id); - } + return this.requeueLeasedJobsWhere( + `status = 'leased' AND (leased_until IS NULL OR leased_until <= ?) + ${activeCaptureIds.length ? `AND id NOT IN (${activeCaptureIds.map(() => "?").join(", ")})` : ""}`, [at, ...activeCaptureIds], at + ); + } - return rows.map((row) => ({ - before: jobFromSql(row), - after: jobFromSql({ - ...row, - status: "queued", - attempts: Math.max(0, row.attempts - 1), - leased_until: null, - updated_at: at - }) - })); - }); - return transaction(); + resumeConfigurationBlockedJobs(at = nowIso(), ids?: readonly string[]): Array<{ before: EvolutionJobRecord; after: EvolutionJobRecord }> { + if (ids?.length === 0) return []; + return this.db.transaction(() => { + const filter = ids ? ` AND id IN (${ids.map(() => "?").join(", ")})` : ""; + const rows = this.db.prepare(`SELECT * FROM evolution_jobs + WHERE status = 'failed' AND blocked_reason = 'configuration'${filter}`).all(...(ids ?? [])) as SqlJobRow[]; + return rows.map((row) => { + this.db.prepare(`UPDATE evolution_jobs SET status = 'queued', available_at = ?, + blocked_reason = NULL, updated_at = ? WHERE id = ?`).run(at, at, row.id); + return { before: jobFromSql(row), after: this.getJob(row.id)! }; + }); + })(); } requeueUnstartedLeasedJobs( @@ -4136,9 +4160,10 @@ export class RuntimeRepository { return []; } return this.requeueLeasedJobsWhere( - `status = 'leased' AND id IN (${ids.map(() => "?").join(", ")})`, + `status = 'leased' AND lease_owner IS NULL AND lease_token IS NULL AND id IN (${ids.map(() => "?").join(", ")})`, [...ids], - at + at, + false ); } @@ -4162,7 +4187,8 @@ export class RuntimeRepository { private requeueLeasedJobsWhere( whereSql: string, params: readonly unknown[], - at: string + at: string, + recovery = true ): Array<{ before: EvolutionJobRecord; after: EvolutionJobRecord }> { const transaction = this.db.transaction(() => { const rows = this.db @@ -4181,21 +4207,17 @@ export class RuntimeRepository { SET status = 'queued', attempts = MAX(0, attempts - 1), leased_until = NULL, + lease_owner = NULL, lease_token = NULL, + lease_recovery_count = lease_recovery_count + ?, updated_at = ? WHERE id = ?` ) - .run(at, row.id); + .run(recovery ? 1 : 0, at, row.id); } return rows.map((row) => ({ before: jobFromSql(row), - after: jobFromSql({ - ...row, - status: "queued", - attempts: Math.max(0, row.attempts - 1), - leased_until: null, - updated_at: at - }) + after: this.getJob(row.id)! })); }); return transaction(); @@ -4205,27 +4227,34 @@ export class RuntimeRepository { id: string, error: string, at = nowIso(), - forceDeadLetter = false + forceDeadLetter = false, + claim?: EvolutionJobRecord, + options: { availableAt?: string | null; blockedReason?: string | null; countFailure?: boolean } = {} ): EvolutionJobRecord | undefined { + const where = this.jobClaimWhere(claim, at); return this.db.transaction(() => { const row = this.db - .prepare(`SELECT * FROM evolution_jobs WHERE id = ?`) - .get(id) as SqlJobRow | undefined; + .prepare(`SELECT * FROM evolution_jobs WHERE id = ? AND ${where.sql}`) + .get(id, ...where.params) as SqlJobRow | undefined; if (!row) return undefined; if (row.status === "dead_letter") return jobFromSql(row); - const status: JobStatus = forceDeadLetter || row.attempts >= row.max_attempts + const failures = row.failure_count + (options.countFailure === false ? 0 : 1); + const status: JobStatus = forceDeadLetter || failures >= row.max_attempts ? "dead_letter" : "failed"; this.db .prepare( `UPDATE evolution_jobs SET status = ?, - leased_until = NULL, + leased_until = NULL, lease_owner = NULL, lease_token = NULL, + failure_count = ?, attempts = MAX(0, attempts - ?), + available_at = ?, blocked_reason = ?, last_error = ?, updated_at = ? - WHERE id = ?` + WHERE id = ? AND ${where.sql}` ) - .run(status, error, at, id); + .run(status, failures, options.countFailure === false ? 1 : 0, + options.availableAt ?? null, options.blockedReason ?? null, error, at, id, ...where.params); if (status === "dead_letter" && row.job_type === "l3_world_model_update") { const payload = parseJson>(row.payload_json, {}); const batchId = typeof payload.batchId === "string" ? payload.batchId : undefined; @@ -4267,6 +4296,10 @@ export class RuntimeRepository { now?: number; }): EmbeddingRetryRecord { const now = input.now ?? Date.now(); + const unchangedActive = `(embedding_retry_queue.source_text = excluded.source_text + AND embedding_retry_queue.embed_role = excluded.embed_role + AND embedding_retry_queue.status IN ('pending', 'in_progress'))`; + const unchangedPending = `(${unchangedActive} AND embedding_retry_queue.status = 'pending')`; this.db .prepare( `INSERT INTO embedding_retry_queue ( @@ -4281,29 +4314,21 @@ export class RuntimeRepository { ON CONFLICT(target_kind, target_id, vector_field) DO UPDATE SET source_text = excluded.source_text, embed_role = excluded.embed_role, - status = CASE - WHEN embedding_retry_queue.status IN ('failed', 'succeeded', 'in_progress') THEN 'pending' - ELSE embedding_retry_queue.status - END, - attempts = CASE - WHEN embedding_retry_queue.status IN ('failed', 'succeeded') THEN 0 - ELSE embedding_retry_queue.attempts - END, + status = CASE WHEN ${unchangedPending} THEN embedding_retry_queue.status ELSE 'pending' END, + attempts = CASE WHEN ${unchangedActive} THEN embedding_retry_queue.attempts ELSE 0 END, max_attempts = excluded.max_attempts, - next_attempt_at = MIN(embedding_retry_queue.next_attempt_at, excluded.next_attempt_at), + next_attempt_at = CASE WHEN ${unchangedActive} THEN embedding_retry_queue.next_attempt_at ELSE excluded.next_attempt_at END, claimed_by = NULL, lease_until = NULL, - last_error = CASE - WHEN embedding_retry_queue.status IN ('failed', 'succeeded') THEN NULL - ELSE embedding_retry_queue.last_error - END, + lease_token = NULL, + blocked_reason = CASE WHEN ${unchangedActive} THEN embedding_retry_queue.blocked_reason ELSE NULL END, + last_error = CASE WHEN ${unchangedActive} THEN embedding_retry_queue.last_error ELSE NULL END, updated_at = excluded.updated_at - WHERE NOT ( - embedding_retry_queue.status = 'in_progress' + WHERE NOT (${unchangedActive} + AND embedding_retry_queue.status = 'in_progress' AND embedding_retry_queue.claimed_by IS NOT NULL AND embedding_retry_queue.lease_until IS NOT NULL - AND embedding_retry_queue.lease_until > excluded.next_attempt_at - )` + AND embedding_retry_queue.lease_until > excluded.next_attempt_at)` ) .run({ id: input.id ?? newId("embed_retry"), @@ -4319,7 +4344,8 @@ export class RuntimeRepository { if (!record) { throw new Error(`embedding retry was not persisted: ${input.targetKind}:${input.targetId}`); } - return record; + this.scheduling.refreshOne("embedding_retry", record.id); + return this.getEmbeddingRetry(record.id)!; } getEmbeddingRetry(id: string): EmbeddingRetryRecord | undefined { @@ -4488,20 +4514,21 @@ export class RuntimeRepository { .prepare( `SELECT * FROM embedding_retry_queue - WHERE status IN ('in_progress', 'failed') + WHERE status = 'in_progress' AND (lease_until IS NULL OR lease_until <= ?) ORDER BY updated_at ASC, id ASC` ) - .all() as SqlEmbeddingRetryRow[]; + .all(now) as SqlEmbeddingRetryRow[]; for (const row of rows) { this.db .prepare( `UPDATE embedding_retry_queue SET status = 'pending', - attempts = CASE WHEN status = 'failed' THEN 0 ELSE attempts END, next_attempt_at = ?, + lease_recovery_count = lease_recovery_count + 1, claimed_by = NULL, lease_until = NULL, + lease_token = NULL, updated_at = ? WHERE id = ?` ) @@ -4513,10 +4540,12 @@ export class RuntimeRepository { after: embeddingRetryFromSql({ ...row, status: "pending", - attempts: row.status === "failed" ? 0 : row.attempts, + attempts: row.attempts, + lease_recovery_count: row.lease_recovery_count + 1, next_attempt_at: now, claimed_by: null, lease_until: null, + lease_token: null, updated_at: now }) })); @@ -4535,35 +4564,24 @@ export class RuntimeRepository { return []; } const limit = Math.max(1, Math.min(200, Math.floor(input.limit ?? 25))); - const targetFilter = input.targetMemoryIds - ? `AND q.target_id IN (${input.targetMemoryIds.map(() => "?").join(", ")})` - : ""; + const eligibility = retryQueueEligibility(input.now, input.targetMemoryIds); const transaction = this.db.transaction(() => { - const rows = this.db - .prepare( - `SELECT q.* - FROM embedding_retry_queue q - LEFT JOIN memories m ON m.id = q.target_id - WHERE ( - q.status = 'pending' - OR (q.status = 'in_progress' AND ( - q.claimed_by IS NULL OR q.lease_until IS NULL OR q.lease_until <= ? - )) - ) - AND q.next_attempt_at <= ? - ${targetFilter} - ORDER BY ${embeddingRetryOrderSql()} - LIMIT ?` - ) - .all(input.now, input.now, ...(input.targetMemoryIds ?? []), limit) as SqlEmbeddingRetryRow[]; + const rows = this.db.prepare(`SELECT q.* FROM embedding_retry_queue q + LEFT JOIN memories m ON m.id = q.target_id WHERE ${eligibility.sql} + ORDER BY ${embeddingRetryOrderSql()} LIMIT ?`) + .all(...eligibility.params, limit) as SqlEmbeddingRetryRow[]; const claimed: EmbeddingRetryRecord[] = []; for (const row of rows) { + const leaseToken = newId("retry_lease"); const result = this.db .prepare( `UPDATE embedding_retry_queue - SET status = 'in_progress', + SET lease_count = lease_count + 1, + lease_recovery_count = lease_recovery_count + CASE WHEN status = 'in_progress' THEN 1 ELSE 0 END, + status = 'in_progress', claimed_by = ?, lease_until = ?, + lease_token = ?, updated_at = ? WHERE id = ? AND ( @@ -4574,13 +4592,16 @@ export class RuntimeRepository { ) AND next_attempt_at <= ?` ) - .run(input.workerId, input.leaseUntil, input.now, row.id, input.now, input.now); + .run(input.workerId, input.leaseUntil, leaseToken, input.now, row.id, input.now, input.now); if (result.changes > 0) { claimed.push(embeddingRetryFromSql({ ...row, status: "in_progress", + lease_count: row.lease_count + 1, + lease_recovery_count: row.lease_recovery_count + (row.status === "in_progress" ? 1 : 0), claimed_by: input.workerId, lease_until: input.leaseUntil, + lease_token: leaseToken, updated_at: input.now })); } @@ -4590,10 +4611,84 @@ export class RuntimeRepository { return transaction(); } + /** Claim only the chosen retry id, rechecking readiness under the write lock. */ + claimEmbeddingRetryById(id: string, input: { + now: number; workerId: string; leaseUntil: number; targetMemoryIds?: readonly string[]; + }): EmbeddingRetryRecord | undefined { + const eligibility = retryQueueEligibility(input.now, input.targetMemoryIds); + return this.db.transaction(() => { + const row = this.db.prepare(`UPDATE embedding_retry_queue AS q SET + lease_count = lease_count + 1, + lease_recovery_count = lease_recovery_count + CASE WHEN status = 'in_progress' THEN 1 ELSE 0 END, + status = 'in_progress', + claimed_by = ?, lease_until = ?, lease_token = ?, updated_at = ? + WHERE q.id = ? AND (${eligibility.sql}) RETURNING *`) + .get(input.workerId, input.leaseUntil, newId("retry_lease"), input.now, id, ...eligibility.params) as SqlEmbeddingRetryRow | undefined; + return row ? embeddingRetryFromSql(row) : undefined; + })(); + } + + /** Return unstarted retry claims without changing retry/attempt history. */ + releaseClaimedEmbeddingRetries(retries: readonly EmbeddingRetryRecord[], now = Date.now()): Array<{ before: EmbeddingRetryRecord; after: EmbeddingRetryRecord }> { + return this.db.transaction(() => { + const transitions: Array<{ before: EmbeddingRetryRecord; after: EmbeddingRetryRecord }> = []; + for (const retry of retries) { + if (!retry.claimedBy || !retry.leaseToken || retry.leaseUntil == null) continue; + const before = this.getEmbeddingRetry(retry.id); + const row = this.db.prepare(`UPDATE embedding_retry_queue SET status = 'pending', + claimed_by = NULL, lease_until = NULL, lease_token = NULL, updated_at = ? + WHERE id = ? AND status = 'in_progress' AND claimed_by = ? AND lease_token = ? + AND lease_until = ? AND lease_until > ? RETURNING *`) + .get(now, retry.id, retry.claimedBy, retry.leaseToken, retry.leaseUntil, now) as SqlEmbeddingRetryRow | undefined; + if (row && before) transitions.push({ before, after: embeddingRetryFromSql(row) }); + } + return transitions; + })(); + } + + markEmbeddingRetryConfigurationBlockedClaimed(id: string, input: { + workerId: string; leaseUntil: number; leaseToken: string; error: string; now: number; + }): EmbeddingRetryRecord | undefined { + const result = this.db.prepare(`UPDATE embedding_retry_queue SET status = 'failed', + blocked_reason = 'configuration', claimed_by = NULL, lease_until = NULL, lease_token = NULL, + last_error = ?, updated_at = ? + WHERE id = ? AND status = 'in_progress' AND claimed_by = ? AND lease_until = ? AND lease_token = ? AND lease_until > ?`) + .run(input.error, input.now, id, input.workerId, input.leaseUntil, input.leaseToken, input.now); + return result.changes > 0 ? this.getEmbeddingRetry(id) : undefined; + } + + resumeConfigurationBlockedEmbeddingRetries(now = Date.now(), ids?: readonly string[]): Array<{ before: EmbeddingRetryRecord; after: EmbeddingRetryRecord }> { + if (ids?.length === 0) return []; + const targetFilter = ids ? ` AND id IN (${ids.map(() => "?").join(", ")})` : ""; + return this.db.transaction(() => { + const rows = this.db.prepare(`SELECT * FROM embedding_retry_queue + WHERE status = 'failed' AND blocked_reason = 'configuration'${targetFilter}`).all(...(ids ?? [])) as SqlEmbeddingRetryRow[]; + return rows.map((row) => { + this.db.prepare(`UPDATE embedding_retry_queue SET status = 'pending', blocked_reason = NULL, + next_attempt_at = ?, updated_at = ? WHERE id = ?`).run(now, now, row.id); + return { before: embeddingRetryFromSql(row), after: this.getEmbeddingRetry(row.id)! }; + }); + })(); + } + + renewEmbeddingRetryClaim( + id: string, + claim: { workerId: string; leaseUntil: number; leaseToken: string }, + newUntil: number, + now = Date.now() + ): EmbeddingRetryRecord | undefined { + if (newUntil <= now) return undefined; + const result = this.db.prepare(`UPDATE embedding_retry_queue SET lease_until = ?, updated_at = ? + WHERE id = ? AND status = 'in_progress' AND claimed_by = ? AND lease_until = ? AND lease_token = ? AND lease_until > ?`) + .run(Math.max(claim.leaseUntil, newUntil), now, id, claim.workerId, claim.leaseUntil, claim.leaseToken, now); + return result.changes > 0 ? this.getEmbeddingRetry(id) : undefined; + } + isEmbeddingRetryClaimHeld(id: string, input: { workerId: string; leaseUntil: number; - }): boolean { + leaseToken: string; + }, now = Date.now()): boolean { const row = this.db .prepare( `SELECT COUNT(*) AS count @@ -4601,15 +4696,18 @@ export class RuntimeRepository { WHERE id = ? AND status = 'in_progress' AND claimed_by = ? - AND lease_until = ?` + AND lease_until = ? + AND lease_token = ? + AND lease_until > ?` ) - .get(id, input.workerId, input.leaseUntil) as { count: number } | undefined; + .get(id, input.workerId, input.leaseUntil, input.leaseToken, now) as { count: number } | undefined; return (row?.count ?? 0) > 0; } markEmbeddingRetryRetryClaimed(id: string, input: { workerId: string; leaseUntil: number; + leaseToken: string; attempts: number; nextAttemptAt: number; error: string; @@ -4618,17 +4716,20 @@ export class RuntimeRepository { const result = this.db .prepare( `UPDATE embedding_retry_queue - SET status = 'pending', + SET status = 'pending', blocked_reason = NULL, attempts = ?, next_attempt_at = ?, claimed_by = NULL, lease_until = NULL, + lease_token = NULL, last_error = ?, updated_at = ? WHERE id = ? AND status = 'in_progress' AND claimed_by = ? - AND lease_until = ?` + AND lease_until = ? + AND lease_token = ? + AND lease_until > ?` ) .run( input.attempts, @@ -4637,7 +4738,9 @@ export class RuntimeRepository { input.now, id, input.workerId, - input.leaseUntil + input.leaseUntil, + input.leaseToken, + input.now ); return result.changes > 0 ? this.getEmbeddingRetry(id) : undefined; } @@ -4645,6 +4748,7 @@ export class RuntimeRepository { markEmbeddingRetryFailedClaimed(id: string, input: { workerId: string; leaseUntil: number; + leaseToken: string; attempts: number; error: string; now: number; @@ -4652,40 +4756,47 @@ export class RuntimeRepository { const result = this.db .prepare( `UPDATE embedding_retry_queue - SET status = 'failed', + SET status = 'failed', blocked_reason = NULL, attempts = ?, claimed_by = NULL, lease_until = NULL, + lease_token = NULL, last_error = ?, updated_at = ? WHERE id = ? AND status = 'in_progress' AND claimed_by = ? - AND lease_until = ?` + AND lease_until = ? + AND lease_token = ? + AND lease_until > ?` ) - .run(input.attempts, input.error, input.now, id, input.workerId, input.leaseUntil); + .run(input.attempts, input.error, input.now, id, input.workerId, input.leaseUntil, input.leaseToken, input.now); return result.changes > 0 ? this.getEmbeddingRetry(id) : undefined; } markEmbeddingRetrySucceededClaimed(id: string, input: { workerId: string; leaseUntil: number; + leaseToken: string; now: number; }): EmbeddingRetryRecord | undefined { const result = this.db .prepare( `UPDATE embedding_retry_queue - SET status = 'succeeded', + SET status = 'succeeded', blocked_reason = NULL, next_attempt_at = ?, claimed_by = NULL, lease_until = NULL, + lease_token = NULL, updated_at = ? WHERE id = ? AND status = 'in_progress' AND claimed_by = ? - AND lease_until = ?` + AND lease_until = ? + AND lease_token = ? + AND lease_until > ?` ) - .run(input.now, input.now, id, input.workerId, input.leaseUntil); + .run(input.now, input.now, id, input.workerId, input.leaseUntil, input.leaseToken, input.now); return result.changes > 0 ? this.getEmbeddingRetry(id) : undefined; } @@ -5389,14 +5500,14 @@ export class RuntimeRepository { return; } this.scheduledLogPrunes.add(table); - setImmediate(() => { + withoutJobLeaseGuard(() => setImmediate(() => { this.scheduledLogPrunes.delete(table); try { this.pruneLogTable(table); } catch { // Log retention is best-effort and must not affect memory write paths. } - }); + })); } private pruneLogTable(table: LogTableName): number { @@ -6435,7 +6546,10 @@ export class Repositories { } transaction(fn: () => T): T { - return this.db.transaction(fn)(); + return this.db.transaction(() => { + assertCurrentJobLease(); + return fn(); + })(); } clearAllMemoryData(): Record { @@ -7892,7 +8006,16 @@ function recallEventFromSql(row: SqlRecallEventRow): RecallEventRecord { }; } -interface SqlJobRow { +interface SqlSchedulingMetadata { + queue_lane: string | null; resource_class: string | null; enqueue_seq: number | null; + first_queued_at: string | null; base_priority: number | null; policy_version: number | null; +} +function schedulingMetadataFromSql(row: SqlSchedulingMetadata): SchedulingMetadata { + return { queueLane: row.queue_lane, resourceClass: row.resource_class, enqueueSeq: row.enqueue_seq, + firstQueuedAt: row.first_queued_at, basePriority: row.base_priority, policyVersion: row.policy_version }; +} + +interface SqlJobRow extends SqlSchedulingMetadata { id: string; job_type: JobType; status: JobStatus; @@ -7907,6 +8030,13 @@ interface SqlJobRow { attempts: number; max_attempts: number; leased_until: string | null; + lease_owner: string | null; + lease_token: string | null; + available_at: string | null; + blocked_reason: string | null; + failure_count: number; + lease_count: number; + lease_recovery_count: number; last_error: string | null; created_at: string; updated_at: string; @@ -7927,7 +8057,7 @@ interface SqlMemoryProcessingRow { updated_at: string; } -interface SqlEmbeddingRetryRow { +interface SqlEmbeddingRetryRow extends SqlSchedulingMetadata { id: string; target_kind: EmbeddingRetryTargetKind; target_id: string; @@ -7936,10 +8066,14 @@ interface SqlEmbeddingRetryRow { embed_role: "document" | "query"; status: EmbeddingRetryStatus; attempts: number; + lease_count: number; + lease_recovery_count: number; max_attempts: number; next_attempt_at: number; claimed_by: string | null; lease_until: number | null; + lease_token: string | null; + blocked_reason: string | null; last_error: string | null; created_at: number; updated_at: number; @@ -8026,6 +8160,7 @@ interface SqlAuditLogRow { function jobFromSql(row: SqlJobRow): EvolutionJobRecord { return { + ...schedulingMetadataFromSql(row), id: row.id, jobType: row.job_type, status: row.status, @@ -8040,6 +8175,13 @@ function jobFromSql(row: SqlJobRow): EvolutionJobRecord { attempts: row.attempts, maxAttempts: row.max_attempts, leasedUntil: row.leased_until, + leaseOwner: row.lease_owner, + leaseToken: row.lease_token, + availableAt: row.available_at, + blockedReason: row.blocked_reason, + failureCount: row.failure_count, + leaseCount: row.lease_count, + leaseRecoveryCount: row.lease_recovery_count, lastError: row.last_error, createdAt: row.created_at, updatedAt: row.updated_at @@ -8078,6 +8220,7 @@ function mergeJobPayload(existing: Record, next: Record= 0), + lease_recovery_count INTEGER NOT NULL DEFAULT 0 CHECK (lease_recovery_count >= 0), + blocked_reason TEXT, last_error TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -695,7 +711,7 @@ export function migrate(db: Database.Database): void { const hasMemories = tableExists(db, "memories"); const version = currentSchemaVersion(db); - if (hasMemories && version !== SCHEMA_VERSION && version !== 2 && version !== 3 && version !== 4 && version !== 5 && version !== 6 && version !== 7 && version !== 8 && version !== 9) { + if (hasMemories && version !== SCHEMA_VERSION && version !== 2 && version !== 3 && version !== 4 && version !== 5 && version !== 6 && version !== 7 && version !== 8 && version !== 9 && version !== 10 && version !== 11 && version !== 12) { throw new Error( `Unsupported memory database schema version ${version}; the database was left unchanged` ); @@ -717,6 +733,37 @@ export function migrate(db: Database.Database): void { addColumnIfMissing(db, "evolution_jobs", "scope_key", "TEXT"); addColumnIfMissing(db, "evolution_jobs", "scope_seq", "INTEGER"); } + if (tableExists(db, "evolution_jobs")) { + const needsFailureBackfill = !columnExists(db, "evolution_jobs", "failure_count"); + addColumnIfMissing(db, "evolution_jobs", "lease_owner", "TEXT"); + addColumnIfMissing(db, "evolution_jobs", "lease_token", "TEXT"); + addColumnIfMissing(db, "evolution_jobs", "available_at", "TEXT"); + addColumnIfMissing(db, "evolution_jobs", "blocked_reason", "TEXT"); + addColumnIfMissing(db, "evolution_jobs", "failure_count", "INTEGER NOT NULL DEFAULT 0"); + addColumnIfMissing(db, "evolution_jobs", "lease_count", "INTEGER NOT NULL DEFAULT 0"); + addColumnIfMissing(db, "evolution_jobs", "lease_recovery_count", "INTEGER NOT NULL DEFAULT 0"); + if (version > 0 && needsFailureBackfill) { + // Historic attempts counted claims. A leased/succeeded row includes + // the current or successful claim, which was not a business failure. + db.prepare(`UPDATE evolution_jobs SET + failure_count = MAX(0, attempts - CASE WHEN status IN ('leased', 'succeeded') THEN 1 ELSE 0 END), + lease_count = MAX(0, attempts)`).run(); + } + } + if (tableExists(db, "embedding_retry_queue")) { + const needsLeaseCountBackfill = !columnExists(db, "embedding_retry_queue", "lease_count"); + addColumnIfMissing(db, "embedding_retry_queue", "blocked_reason", "TEXT"); + addColumnIfMissing(db, "embedding_retry_queue", "lease_token", "TEXT"); + addColumnIfMissing(db, "embedding_retry_queue", "lease_count", "INTEGER NOT NULL DEFAULT 0 CHECK (lease_count >= 0)"); + addColumnIfMissing(db, "embedding_retry_queue", "lease_recovery_count", "INTEGER NOT NULL DEFAULT 0 CHECK (lease_recovery_count >= 0)"); + if (version > 0 && version < 13 && needsLeaseCountBackfill) { + // Historic attempts count failures, so they prove only a lower bound + // on claims. An active/succeeded row proves one more claim. Previous + // recoveries cannot be reconstructed and start at zero after upgrade. + db.prepare(`UPDATE embedding_retry_queue SET lease_count = MAX(0, attempts) + + CASE WHEN status IN ('in_progress', 'succeeded') THEN 1 ELSE 0 END`).run(); + } + } // Rebuild only the parent table, with foreign keys disabled outside this // transaction. Preserve rowid: it is the tie-breaker for turn ordering. if (tableExists(db, "raw_turns") && (db.prepare("PRAGMA table_info(raw_turns)").all() as Array<{ name: string; notnull: number }>).some(c => c.name === "episode_id" && c.notnull)) { @@ -750,6 +797,8 @@ export function migrate(db: Database.Database): void { WHERE dedupe_key IS NOT NULL AND status IN ('queued', 'leased', 'failed')` ).run(); + installQueueSchedulingSchema(db); + // This boundary belongs to the Memory database and is shared by Hook and scanners. db.prepare(`INSERT INTO runtime_kv (key, value_json, updated_at) VALUES ('source_turn_capture_activated_at', ?, ?) diff --git a/Memory/tests/agent-source-runtime.test.ts b/Memory/tests/agent-source-runtime.test.ts index 0a874c77c..0db721b34 100644 --- a/Memory/tests/agent-source-runtime.test.ts +++ b/Memory/tests/agent-source-runtime.test.ts @@ -1,3 +1,4 @@ +import { withUnrestrictedScanPermit } from "./fixtures/scan-admission-fixture.js"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -139,7 +140,7 @@ describe("standalone Agent source executor", () => { .mockImplementationOnce(async () => { await first; return true; }) .mockImplementationOnce(async () => { await second; return true; }); const executor = createAgentSourceExecutor({ - service: {} as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit,} as MemoryService, configPath: join(root, "config.yaml"), sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "fixture-agent", displayName: "Fixture Agent", builtin: true, dataPath: root }, @@ -180,7 +181,7 @@ describe("standalone Agent source executor", () => { }; const addMemory = vi.fn(); const executor = createAgentSourceExecutor({ - service: { addMemory } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory } as unknown as MemoryService, configPath: join(root, "config.yaml"), sourceRegistry: createSourceRegistry([adapter]), }); @@ -211,7 +212,7 @@ describe("standalone Agent source executor", () => { const detection = new Promise((done) => { finishDetection = done; }); const detect = vi.fn(async () => { await detection; return true; }); const executor = createAgentSourceExecutor({ - service: {} as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit,} as MemoryService, configPath, initialScanDelayMs: 0, sourceRegistry: createSourceRegistry([{ @@ -258,7 +259,7 @@ describe("standalone Agent source executor", () => { .mockReturnValue({ id: "memory-1", duplicate: true }); const enqueuePendingImportSummaries = vi.fn(); const scheduleWorker = vi.fn(); - const service = { addMemory, enqueuePendingImportSummaries } as unknown as MemoryService; + const service = { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries } as unknown as MemoryService; const adapter: SourceAdapter = { descriptor: { sourceId: "fixture-agent", @@ -367,7 +368,7 @@ describe("standalone Agent source executor", () => { } }); const executor = createAgentSourceExecutor({ - service: { addMemory, clearAllData: clearMemory, enqueuePendingImportSummaries } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, clearAllData: clearMemory, enqueuePendingImportSummaries } as unknown as MemoryService, configPath: join(root, "config.yaml"), statePath, scanStoreDirectory, @@ -434,7 +435,7 @@ describe("standalone Agent source executor", () => { const clearingMemory = new Promise((resolve) => { finishClear = resolve; }); const clearMemory = vi.fn(() => clearingMemory); const executor = createAgentSourceExecutor({ - service: { addMemory, clearAllData: clearMemory } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, clearAllData: clearMemory } as unknown as MemoryService, configPath: join(root, "config.yaml"), sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "fixture-agent", displayName: "Fixture Agent", builtin: true, dataPath: root }, @@ -505,7 +506,7 @@ describe("standalone Agent source executor", () => { } }; const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, @@ -551,7 +552,7 @@ describe("standalone Agent source executor", () => { } }; const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, @@ -603,7 +604,7 @@ describe("standalone Agent source executor", () => { } }; const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory: vi.fn(), enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, @@ -678,7 +679,7 @@ describe("standalone Agent source executor", () => { const enqueuePendingImportSummaries = vi.fn(); const scheduleWorker = vi.fn(); const executor = createAgentSourceExecutor({ - service: { addMemory, enqueuePendingImportSummaries } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries } as unknown as MemoryService, configPath, statePath: join(root, "agent-sources.json"), sourceRegistry: createSourceRegistry([adapter]), @@ -715,7 +716,7 @@ describe("standalone Agent source executor", () => { const root = tempRoot(); const addMemory = vi.fn(() => ({ id: "memory-1", duplicate: false })); const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, @@ -751,7 +752,7 @@ describe("standalone Agent source executor", () => { const events: Array<{ name: string; payload: Record }> = []; const root = tempRoot(); const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory: vi.fn(() => { throw new Error("write failed"); }), @@ -781,7 +782,7 @@ describe("standalone Agent source executor", () => { const root = tempRoot(); const addMemory = vi.fn(() => ({ id: "memory-1", duplicate: true })); const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, @@ -808,7 +809,7 @@ describe("standalone Agent source executor", () => { completeTurn("conversation-new", "2026-08-28T02:00:00.000Z") ]; const executor = createAgentSourceExecutor({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, @@ -846,7 +847,7 @@ describe("standalone Agent source executor", () => { writeFileSync(skillPath, "---\nname: sample-skill\nversion: 1\n---\n\nUse the sample procedure.\n"); vi.stubEnv("CODEX_HOME", codexRoot); const addMemory = vi.fn(() => ({ id: "skill-memory-1" })); - const service = { + const service = { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService; @@ -885,7 +886,7 @@ describe("standalone Agent source executor", () => { } vi.stubEnv("CODEX_HOME", codexRoot); const events: Array<{ name: string; payload: Record }> = []; - const service = { + const service = { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory: vi.fn((input: { sourceSkillId?: string }) => { if (input.sourceSkillId === "duplicate-skill") return { id: "dup", duplicate: true, status: "activated" }; if (input.sourceSkillId === "deleted-skill") return { id: "gone", status: "deleted" }; @@ -943,7 +944,7 @@ describe("standalone Agent source executor", () => { detect: async () => true, async *scan() {} }; - const service = { + const service = { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory: vi.fn(), enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService; diff --git a/Memory/tests/agent-source-scan-responsiveness.test.ts b/Memory/tests/agent-source-scan-responsiveness.test.ts index 93b0ca98b..12414d20b 100644 --- a/Memory/tests/agent-source-scan-responsiveness.test.ts +++ b/Memory/tests/agent-source-scan-responsiveness.test.ts @@ -1,3 +1,4 @@ +import { withUnrestrictedScanPermit } from "./fixtures/scan-admission-fixture.js"; import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -59,7 +60,7 @@ describe("standalone scan HTTP responsiveness", () => { const completeSourceTurn = vi.fn(() => ({ status: "existing" })); const addMemory = vi.fn(() => ({ id: "synthetic-memory" })); const executor = createAgentSourceExecutor({ - service: { completeSourceTurn, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, completeSourceTurn, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, configPath: join(root, "config.yaml"), scanStoreDirectory: join(root, "scans"), sourceRegistry: createSourceRegistry([{ @@ -72,7 +73,7 @@ describe("standalone scan HTTP responsiveness", () => { resolveAgentSkillRoot: () => null, }); const { server, url } = await listenMemoryHttpServer({ - service: { + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, health: () => ({ ok: true, scanRunning: executor.scanStatus().running }), setAppBudgetReconcileListener: () => undefined, setCapturePendingListener: () => undefined, @@ -124,7 +125,7 @@ describe("standalone scan HTTP responsiveness", () => { let firstJob = true; const addMemory = vi.fn(() => ({ id: "synthetic-skill" })); const executor = createAgentSourceExecutor({ - service: { addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory, enqueuePendingImportSummaries: vi.fn() } as unknown as MemoryService, configPath: join(root, "config.yaml"), scanStoreDirectory: join(root, "scans"), sourceRegistry: createSourceRegistry([{ @@ -181,7 +182,7 @@ describe("standalone scan HTTP responsiveness", () => { }); const completeSourceTurn = vi.fn(() => ({ status: "existing" })); const create = () => createAgentSourceExecutor({ - service: { completeSourceTurn } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, completeSourceTurn } as unknown as MemoryService, configPath: join(root, "config.yaml"), scanStoreDirectory: join(root, "scans"), sourceRegistry: createSourceRegistry([syntheticAdapter(root)]), @@ -227,7 +228,7 @@ describe("standalone scan HTTP responsiveness", () => { }); const completeSourceTurn = vi.fn((_request: Parameters[0]) => ({ status: "existing" })); const executor = createAgentSourceExecutor({ - service: { completeSourceTurn } as unknown as MemoryService, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit, completeSourceTurn } as unknown as MemoryService, configPath: join(root, "config.yaml"), sourceRegistry: createSourceRegistry([syntheticAdapter(root)]), resolveAgentSkillRoot: () => null, diff --git a/Memory/tests/contract/memory-rest-service.test.ts b/Memory/tests/contract/memory-rest-service.test.ts index e359f5dba..bbdd56b5e 100644 --- a/Memory/tests/contract/memory-rest-service.test.ts +++ b/Memory/tests/contract/memory-rest-service.test.ts @@ -21,6 +21,7 @@ import { createFailingLlm, createMemoryServiceFixture, runWorkerRounds, + runWorkerRoundsWithDueRetries, setRawTurnActivityAt } from "../fixtures/memory-service-fixture.js"; @@ -402,7 +403,7 @@ describe("MemoryService / REST contract", () => { }); const namespace = { source: "hermes", profileId: "default", userId: "http-retry-user" }; const added = addAgentSourceImport(service, namespace, "retry through HTTP", "http-retry"); - await runWorkerRounds(service, 3, 1); + await runWorkerRoundsWithDueRetries(service, db, [added.id], 3, 1); expect(service.memoryProcessingStatus([added.id], { namespace }).items[0]?.state).toBe("failed"); const server = createMemoryHttpServer({ service }); @@ -1044,6 +1045,9 @@ describe("MemoryService / REST contract", () => { expect(repos.runtime.getPendingJob(interruptedMemory.id, "embedding")?.jobType).toBe("embedding"); const [interruptedJob] = repos.runtime.leaseQueuedJobs(1, 600, [interruptedMemory.id]); expect(interruptedJob?.jobType).toBe("embedding"); + // Startup recovery must wait for expiry rather than revoke a valid owner. + db.db.prepare("UPDATE evolution_jobs SET leased_until = ? WHERE id = ?") + .run(new Date(Date.now() - 1_000).toISOString(), interruptedJob!.id); const failedMemory = addAgentSourceImport( service, diff --git a/Memory/tests/embedding-provenance-upgrade.test.ts b/Memory/tests/embedding-provenance-upgrade.test.ts index bad0bcd35..2b75f9c98 100644 --- a/Memory/tests/embedding-provenance-upgrade.test.ts +++ b/Memory/tests/embedding-provenance-upgrade.test.ts @@ -252,7 +252,9 @@ describe("one-time upgrade repair", () => { } return texts.map(() => vector()); }; - await service.runWorkerOnce(); + // Limit the first drain to the stale input so continuous refill cannot also + // finish its replacement before the intermediate-state assertion. + await service.runWorkerOnce(1); expect(service.embeddingMaintenanceProgress()).toMatchObject({ done: false, updated: 0 }); await service.runWorkerOnce(); expect(service.embeddingMaintenanceProgress()).toMatchObject({ done: true, updated: 1 }); diff --git a/Memory/tests/fixtures/memory-service-fixture.ts b/Memory/tests/fixtures/memory-service-fixture.ts index d960e33d8..0d14f42f6 100644 --- a/Memory/tests/fixtures/memory-service-fixture.ts +++ b/Memory/tests/fixtures/memory-service-fixture.ts @@ -16,12 +16,13 @@ export const LEGACY_TEST_CONFIG = { ...DEFAULT_MEMMY_CONFIG, algorithm: { ...DEF session: { ...DEFAULT_MEMMY_CONFIG.algorithm.session, turnJudge: false } } }; export function createMemoryServiceFixture(): { - cleanup: () => void; + cleanup: () => Promise; createTestMemoryService: ( options: ConstructorParameters[0] ) => MemoryService; createTestRoot: (prefix?: string) => string; createTestService: (options?: { + concurrency?: ConstructorParameters[0]["concurrency"]; mode?: "local" | "cloud" | "dev"; config?: typeof DEFAULT_MEMMY_CONFIG; llm?: LlmClient; @@ -65,6 +66,7 @@ export function createMemoryServiceFixture(): { } function createTestService(options: { + concurrency?: ConstructorParameters[0]["concurrency"]; mode?: "local" | "cloud" | "dev"; config?: typeof DEFAULT_MEMMY_CONFIG; llm?: LlmClient; @@ -85,6 +87,7 @@ export function createMemoryServiceFixture(): { root, db, service: createTestMemoryService({ + concurrency: options.concurrency, db, mode: options.mode ?? "dev", config: options.config, @@ -97,18 +100,23 @@ export function createMemoryServiceFixture(): { }; } - function cleanup(): void { - for (const service of services.splice(0)) { - void service.stop(); - } - for (const database of databases.splice(0)) { + async function cleanup(): Promise { + // Snapshot before awaiting: even callers which forget to await cleanup + // must never close the databases belonging to the next test's services. + const stoppingServices = services.splice(0); + const closingDatabases = databases.splice(0); + const removingRoots = roots.splice(0); + const stopped = await Promise.allSettled(stoppingServices.map(async service => { await service.stop(); })); + for (const database of closingDatabases) { if (database.db.open) { database.close(); } } - for (const root of roots.splice(0)) { + for (const root of removingRoots) { rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); } + const failed = stopped.find(result => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; } return { @@ -129,6 +137,25 @@ export async function runWorkerRounds( } } +/** Lifecycle tests can explicitly make selected retries due without real sleeps. + * Backoff timing itself is covered by worker-lease-safety using a fake clock. + * Configuration blocks are never bypassed by this helper. + */ +export async function runWorkerRoundsWithDueRetries( + service: MemoryService, db: MemoryDb, memoryIds: readonly string[], rounds: number, limit = 100 +) { + if (memoryIds.length === 0) throw new Error("retry test requires explicit memory ids"); + const placeholders = memoryIds.map(() => "?").join(", "); + const results = []; + for (let index = 0; index < rounds; index += 1) { + db.db.prepare(`UPDATE evolution_jobs SET available_at = ? + WHERE status = 'failed' AND blocked_reason IS NULL AND target_memory_id IN (${placeholders})`) + .run(new Date().toISOString(), ...memoryIds); + results.push(await service.runWorkerOnce(limit)); + } + return results; +} + export function addAgentSourceImport( service: MemoryService, namespace: { source: string; profileId: string; userId: string }, diff --git a/Memory/tests/fixtures/scan-admission-fixture.ts b/Memory/tests/fixtures/scan-admission-fixture.ts new file mode 100644 index 000000000..e6df197ba --- /dev/null +++ b/Memory/tests/fixtures/scan-admission-fixture.ts @@ -0,0 +1,8 @@ +/** Unit-only service doubles do not own a concurrency controller. Integration + * admission tests use the real MemoryService instead of this fixture. */ +export async function withUnrestrictedScanPermit( + _sourceId: string | readonly string[], signal: AbortSignal, operation: () => Promise +): Promise { + signal.throwIfAborted(); + return operation(); +} diff --git a/Memory/tests/http-startup.test.ts b/Memory/tests/http-startup.test.ts index 63489680e..5c73b3fc5 100644 --- a/Memory/tests/http-startup.test.ts +++ b/Memory/tests/http-startup.test.ts @@ -290,6 +290,7 @@ function stubService(reconcile: () => void): StartupServiceStub { return { ok: true }; }, reconcileWorkerStartup: reconcile, + notifyWorkerWorkAvailable() {}, async runWorkerOnce() { return workerResult(0); }, diff --git a/Memory/tests/http-worker-continuous-admission.test.ts b/Memory/tests/http-worker-continuous-admission.test.ts new file mode 100644 index 000000000..80a1e7079 --- /dev/null +++ b/Memory/tests/http-worker-continuous-admission.test.ts @@ -0,0 +1,123 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type { Server } from "node:http"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMemoryHttpServer, type LlmClient, type LlmCompletionOptions, type LlmMessage } from "../src/index.js"; +import { closeMemoryHttpServer } from "../src/server/http.js"; +import { addAgentSourceImport, createBatchReflectionLlm, createMemoryServiceFixture } from "./fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +const servers: Server[] = []; +const namespace = { source: "codex", profileId: "continuous-http", userId: "continuous-http-user" }; +afterEach(async () => { + await Promise.all(servers.splice(0).map(closeMemoryHttpServer)); + fixture.cleanup(); vi.restoreAllMocks(); +}); +function deferred() { let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } +async function listen(server: Server) { + servers.push(server); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected local TCP address"); + return `http://127.0.0.1:${address.port}`; +} +async function post(base: string, path: string, body: unknown) { + return fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); +} +async function waitFor(check: () => boolean, timeoutMs = 1_500) { + const deadline = Date.now() + timeoutMs; + while (!check()) { + if (Date.now() >= deadline) throw new Error("timed out waiting for continuous worker progress"); + await new Promise(resolve => setTimeout(resolve, 5)); + } +} +function controlledCloudLlm(): LlmClient { + const base = createBatchReflectionLlm([], "Controlled cloud summary"); + return { + ...base, + config: { ...base.config, provider: "openai_compatible", endpoint: "https://controlled-cloud.invalid/v1", model: "controlled-cloud-model" }, + status: () => ({ provider: "openai_compatible", model: "controlled-cloud-model", configured: true, remote: true }) + }; +} + +// Actual HTTP auto scheduling, MemoryService, SQLite and default provider caps. +// The model is an in-process substitute: no network model or desktop data. +describe("HTTP automatic continuous admission", () => { + it("continues after fast imports and embedding finish while the first cloud summary is still slow", async () => { + const gate = deferred(), context = new AsyncLocalStorage(); + const llm = controlledCloudLlm(), complete = llm.completeJson.bind(llm); + let slowId = "", active = 0, peak = 0, maximumLeased = 0; + const summaryStarted: string[] = []; + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + active++; peak = Math.max(peak, active); + try { if (context.getStore() === slowId) await gate.promise; return await complete(messages, options); } + finally { active--; } + }; + const { service, db } = fixture.createTestService({ llm, skillLlm: llm }); + const targets = Array.from({ length: 8 }, (_, i) => addAgentSourceImport(service, namespace, `cloud import ${i}`, `continuous-cloud-${i}`).id); + slowId = targets[0]!; + const internal = service as unknown as { workerRunner: { deps: { jobHandlers: { processJob: (job: { jobType: string; targetMemoryId?: string }) => Promise } } } }; + const handlers = internal.workerRunner.deps.jobHandlers, original = handlers.processJob.bind(handlers); + handlers.processJob = job => { + if (job.jobType === "import_summary") summaryStarted.push(job.targetMemoryId!); + const leased = (db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get() as { n: number }).n; + maximumLeased = Math.max(maximumLeased, leased); + return context.run(job.targetMemoryId ?? "", () => original(job)); + }; + const base = await listen(createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 })); + try { + expect((await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: targets })).status).toBe(200); + await waitFor(() => summaryStarted.includes(targets[4]!)); + expect(service.memoryProcessingStatus([slowId], { namespace }).items[0]?.state).toBe("summarizing"); + expect(active).toBeGreaterThanOrEqual(1); + expect(peak).toBe(2); + expect(maximumLeased).toBeLessThanOrEqual(4); + } finally { gate.resolve(); } + await waitFor(() => targets.every(id => service.memoryProcessingStatus([id], { namespace }).items[0]?.state === "ready")); + }); + + it("preserves the manual HTTP budget and target filter without turning it into an automatic drain", async () => { + const { service, db } = fixture.createTestService({ llm: controlledCloudLlm() }); + const targets = Array.from({ length: 3 }, (_, i) => addAgentSourceImport(service, namespace, `manual target ${i}`, `manual-bound-${i}`).id); + const base = await listen(createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 })); + const response = await post(base, "/api/v1/worker/run", { limit: 1, targetMemoryIds: [targets[1]] }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ leased: 1, succeeded: 1, embeddingRetries: { leased: 0 } }); + const rows = db.db.prepare("SELECT target_memory_id,status,attempts FROM evolution_jobs WHERE job_type='import_summary' ORDER BY enqueue_seq").all(); + expect(rows).toEqual([ + { target_memory_id: targets[0], status: "queued", attempts: 0 }, + { target_memory_id: targets[1], status: "succeeded", attempts: 1 }, + { target_memory_id: targets[2], status: "queued", attempts: 0 } + ]); + expect((db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get() as { n: number }).n).toBe(0); + }); + + it("stops automatic admission on HTTP shutdown while allowing admitted cloud requests to finish", async () => { + const gate = deferred(), llm = controlledCloudLlm(), complete = llm.completeJson.bind(llm); + let active = 0; + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + active++; + try { await gate.promise; return await complete(messages, options); } finally { active--; } + }; + const { service, db } = fixture.createTestService({ llm, skillLlm: llm }); + const targets = Array.from({ length: 3 }, (_, i) => addAgentSourceImport(service, namespace, `shutdown import ${i}`, `continuous-stop-${i}`).id); + const server = createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 }); + const base = await listen(server); + let closed = false, closing: Promise | undefined; + try { + await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: targets }); + await waitFor(() => active === 2); + closing = closeMemoryHttpServer(server).then(() => { closed = true; }); + await new Promise(resolve => setTimeout(resolve, 10)); + expect(closed).toBe(false); + } finally { gate.resolve(); await closing; } + expect(closed).toBe(true); + expect(active).toBe(0); + const rows = db.db.prepare("SELECT status,attempts,lease_token FROM evolution_jobs WHERE job_type='import_summary' ORDER BY enqueue_seq").all(); + expect(rows).toEqual([ + { status: "succeeded", attempts: 1, lease_token: null }, + { status: "succeeded", attempts: 1, lease_token: null }, + { status: "queued", attempts: 0, lease_token: null } + ]); + expect((db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get() as { n: number }).n).toBe(0); + }); +}); diff --git a/Memory/tests/http-worker-entry.test.ts b/Memory/tests/http-worker-entry.test.ts new file mode 100644 index 000000000..7b3e18ab1 --- /dev/null +++ b/Memory/tests/http-worker-entry.test.ts @@ -0,0 +1,177 @@ +import type { Server } from "node:http"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMemoryHttpServer, type LlmCompletionOptions, type LlmMessage } from "../src/index.js"; +import { closeMemoryHttpServer } from "../src/server/http.js"; +import { + addAgentSourceImport, createBatchReflectionLlm, createMemoryServiceFixture +} from "./fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +const servers: Server[] = []; +const namespace = { source: "codex", profileId: "entry-test", userId: "entry-user" }; +afterEach(async () => { + await Promise.all(servers.splice(0).map(closeMemoryHttpServer)); + fixture.cleanup(); + vi.restoreAllMocks(); +}); +async function listen(server: Server): Promise { + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + return `http://127.0.0.1:${address.port}`; +} +async function post(base: string, path: string, body: unknown, token?: string) { + return fetch(`${base}${path}`, { method: "POST", headers: { + "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}) + }, body: JSON.stringify(body) }); +} +async function waitFor(predicate: () => boolean, timeoutMs = 3_000) { + const started = Date.now(); + while (!predicate()) { + if (Date.now() - started >= timeoutMs) throw new Error("timed out waiting for background worker"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +// Real HTTP, MemoryService, SQLite claims and state transitions. Only model +// requests are local substitutes; no current desktop/user data is involved. +describe("Memory HTTP unified worker entry", () => { + it("wakes existing queued imports even when enqueue creates no new jobs", async () => { + const { service } = fixture.createTestService({ llm: createBatchReflectionLlm([]) }); + const added = addAgentSourceImport(service, namespace, "already queued before wake", "already-queued"); + expect(service.enqueuePendingImportSummaries(10_000, [added.id]).enqueued).toBe(0); + const run = vi.spyOn(service, "runWorkerOnce"); + const base = await listen(createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 })); + expect(run).not.toHaveBeenCalled(); + const response = await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: [added.id] }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ enqueued: 0 }); + await waitFor(() => service.memoryProcessingStatus([added.id], { namespace }).items[0]?.state === "ready", 1_000); + expect(run).toHaveBeenCalledWith(expect.any(Number), { priorityCohortOnly: true }, + expect.objectContaining({ continuous: true, signal: expect.any(AbortSignal) })); + }); + + it("shares one execution boundary between automatic drain and a manual HTTP worker call", async () => { + let active = 0, peak = 0; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + active++; peak = Math.max(peak, active); + try { await gate; return await complete(messages, options); } + finally { active--; } + }; + const { service, db } = fixture.createTestService({ llm }); + const targets = Array.from({ length: 8 }, (_, i) => addAgentSourceImport(service, namespace, + `parallel entry ${i}`, `parallel-entry-${i}`).id); + const run = vi.spyOn(service, "runWorkerOnce"); + const base = await listen(createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 })); + let manual: Promise | undefined; + try { + expect((await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: targets })).status).toBe(200); + await waitFor(() => active === 1); + manual = post(base, "/api/v1/worker/run", { limit: 4, priorityCohortOnly: true }); + await waitFor(() => run.mock.calls.length >= 2); + // Both routes entered the real service, but only the active scheduler + // request can claim. The local provider permits one request; the second entry cannot add another. + expect(active).toBe(1); + expect((db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get() as { n: number }).n).toBe(1); + } finally { release(); } + expect((await manual!).status).toBe(200); + await waitFor(() => targets.every(id => service.memoryProcessingStatus([id], { namespace }).items[0]?.state === "ready")); + expect(peak).toBe(1); + }); + + it("waits for an active automatic drain and its queued manual request when closing", async () => { + let entered = 0; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + entered++; + await gate; + return complete(messages, options); + }; + const { service, db } = fixture.createTestService({ llm }); + const targets = Array.from({ length: 8 }, (_, i) => addAgentSourceImport(service, namespace, + `shutdown entry ${i}`, `shutdown-entry-${i}`).id); + const run = vi.spyOn(service, "runWorkerOnce"); + const server = createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 }); + const base = await listen(server); + let manual: Promise | undefined; + let closing: Promise | undefined; + try { + await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: targets }); + await waitFor(() => entered === 1); + // Closing sockets may reject the client, but accepted server work must + // still settle before cleanup completes. + const pending = db.db.prepare("SELECT target_memory_id FROM evolution_jobs WHERE job_type='import_summary' AND status='queued'").all() as Array<{ target_memory_id: string }>; + manual = post(base, "/api/v1/worker/run", { limit: 4, targetMemoryIds: pending.map(row => row.target_memory_id) }).catch(error => error as Error); + await waitFor(() => run.mock.calls.length >= 2); + let closed = false; + closing = closeMemoryHttpServer(server).then(() => { closed = true; }); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(closed).toBe(false); + expect(entered).toBe(1); + } finally { release(); } + await closing; + await manual; + const completedRuns = await Promise.all(run.mock.results.map(result => result.value)); + const automaticIndex = run.mock.calls.findIndex(([, , execution]) => execution?.continuous === true); + const manualIndex = run.mock.calls.findIndex(([, , execution]) => execution?.continuous !== true); + expect(automaticIndex).toBeGreaterThanOrEqual(0); + expect(manualIndex).toBeGreaterThanOrEqual(0); + // Shutdown cancels further automatic admission. Its sole active local-model + // task settles, and the already accepted manual request retains budget four. + const automatic = completedRuns[automaticIndex]!; + const manualResult = completedRuns[manualIndex]!; + expect(automatic.leased + automatic.embeddingRetries.leased).toBe(1); + expect(manualResult.leased + manualResult.embeddingRetries.leased).toBe(4); + expect(entered).toBeGreaterThan(1); + const completedSummaries = db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type='import_summary' AND status='succeeded'").get() as { n: number }; + expect(completedSummaries.n).toBe(entered); + // Every admitted execution settles; work not admitted before cancellation + // or beyond the finite manual budget stays queued without lease history. + expect((db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get() as { n: number }).n).toBe(0); + const waiting = db.db.prepare("SELECT attempts,lease_token FROM evolution_jobs WHERE status='queued'").all(); + expect(waiting.every(row => (row as { attempts: number; lease_token: string | null }).attempts === 0 + && (row as { lease_token: string | null }).lease_token === null)).toBe(true); + }); + + it("retains manual retry wake-up through the same background scheduler", async () => { + let rejectRequest = true; + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + if (rejectRequest) throw new Error("unauthorized API key"); + return complete(messages, options); + }; + const { service } = fixture.createTestService({ llm }); + const added = addAgentSourceImport(service, namespace, "retry from HTTP", "http-manual-retry"); + await service.runWorkerOnce(1); + expect(service.memoryProcessingStatus([added.id], { namespace }).items[0]?.state).toBe("failed"); + rejectRequest = false; + const base = await listen(createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 })); + const response = await post(base, `/api/v1/memory/${added.id}/processing/retry`, { namespace }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ accepted: true }); + await waitFor(() => service.memoryProcessingStatus([added.id], { namespace }).items[0]?.state === "ready"); + }); + + it("does not wake the worker on an unauthenticated enqueue request", async () => { + const calls: Parameters[0] = []; + const { service } = fixture.createTestService({ llm: createBatchReflectionLlm(calls) }); + const added = addAgentSourceImport(service, namespace, "authentication before wake", "authenticated-wake"); + const run = vi.spyOn(service, "runWorkerOnce"); + const base = await listen(createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000, + auth: { localServiceToken: "entry-test-token" } })); + expect((await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: [added.id] })).status).toBe(401); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(run).not.toHaveBeenCalled(); expect(calls).toHaveLength(0); + expect((await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: [added.id] }, "entry-test-token")).status).toBe(200); + await waitFor(() => service.memoryProcessingStatus([added.id], { namespace }).items[0]?.state === "ready"); + }); +}); diff --git a/Memory/tests/http-worker-handoff.test.ts b/Memory/tests/http-worker-handoff.test.ts new file mode 100644 index 000000000..f881d3ee5 --- /dev/null +++ b/Memory/tests/http-worker-handoff.test.ts @@ -0,0 +1,101 @@ +import type { Server } from "node:http"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMemoryHttpServer } from "../src/index.js"; +import { closeMemoryHttpServer } from "../src/server/http.js"; +import { Repositories } from "../src/storage/repositories.js"; +import type { WorkerConcurrencyController } from "../src/service/worker/worker-concurrency.js"; +import type { EvolutionJobRecord } from "../src/storage/repositories.js"; +import { createBatchReflectionLlm, createMemoryServiceFixture } from "./fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(), servers: Server[] = []; +afterEach(async () => { + await Promise.all(servers.splice(0).map(closeMemoryHttpServer)); + fixture.cleanup(); vi.restoreAllMocks(); +}); +function deferred() { let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } +async function waitFor(check: () => boolean) { + const deadline = Date.now() + 3000; + while (!check()) { + if (Date.now() >= deadline) throw new Error("timed out waiting for worker handoff"); + await new Promise(resolve => setTimeout(resolve, 5)); + } +} +async function listen(server: Server) { + servers.push(server); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + return `http://127.0.0.1:${address.port}`; +} +function post(base: string, path: string, body: unknown) { + return fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); +} + +describe("HTTP automatic worker handoff", () => { + it("resumes queued work after an automatic request is interleaved between several manual requests", async () => { + const { service, db, root } = fixture.createTestService({ llm: createBatchReflectionLlm([]), + fetchAppMemoryBudget: async () => ({ dailyUsed: 0, lifetimeUsed: 0 }) }); + const repos = new Repositories(db.db); + const gates = new Map(["first", "manual-1", "manual-2", "manual-3"].map(id => [id, deferred()])); + const starts: string[] = [], automaticResults: number[] = []; + let active = 0, peak = 0, peakJobs = 0, peakLeased = 0; + const internal = service as unknown as { workerRunner: { concurrency: WorkerConcurrencyController; + deps: { jobHandlers: { processJob(job: EvolutionJobRecord): Promise } } } }; + internal.workerRunner.deps.jobHandlers.processJob = async job => { + starts.push(job.id); peak = Math.max(peak, ++active); + peakJobs = Math.max(peakJobs, internal.workerRunner.concurrency.snapshot().activeJobs); + peakLeased = Math.max(peakLeased, Number((db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get() as { n: number }).n)); + try { await gates.get(job.id)?.promise; } finally { active--; } + }; + for (const id of ["first", "tail", ...[...gates.keys()].filter(id => id !== "first")]) { + const at = new Date().toISOString(); + repos.runtime.enqueueJob({ id, jobType: "trace_summary", targetMemoryId: id, userId: "handoff-user", + payload: {}, status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + if (id.startsWith("manual")) db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id=?").run("2100-01-01T00:00:00.000Z", id); + } + const original = service.runWorkerOnce.bind(service); + const run = vi.spyOn(service, "runWorkerOnce").mockImplementation(async (...args) => { + const result = await original(...args); + if (args[2]?.continuous) automaticResults.push(result.leased); + return result; + }); + const base = await listen(createMemoryHttpServer({ service, configPath: join(root, "config.yaml"), workerStartupFallbackMs: 60_000 })); + const automaticCalls = () => run.mock.calls.filter(([, , execution]) => execution?.continuous).length; + const pending: Array> = []; + const manual = async (id: string) => { + db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE id=?").run(id); + pending.push(post(base, "/api/v1/worker/run", { limit: 1, targetMemoryIds: [id] })); + await waitFor(() => run.mock.calls.some(([, request, execution]) => !execution?.continuous && request?.targetMemoryIds?.[0] === id)); + }; + try { + expect((await post(base, "/api/v1/worker/import-summaries/enqueue", { memoryIds: [] })).status).toBe(200); + await waitFor(() => starts.includes("first")); + await manual("manual-1"); await manual("manual-2"); + gates.get("first")!.resolve(); + await waitFor(() => starts.includes("manual-1") && automaticCalls() >= 2); + // auto#2 is now behind manual#2. Queue manual#3 behind auto#2 so its + // legitimate yield must not be mistaken for an empty durable queue. + await manual("manual-3"); + gates.get("manual-1")!.resolve(); await waitFor(() => starts.includes("manual-2")); + gates.get("manual-2")!.resolve(); await waitFor(() => starts.includes("manual-3") && automaticResults.length >= 2); + expect(repos.runtime.getJob("tail")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + gates.get("manual-3")!.resolve(); + const responses = await Promise.all(pending); + expect(responses.every(response => response.status === 200)).toBe(true); + for (const response of responses) expect(await response.json()).toMatchObject({ leased: 1, succeeded: 1 }); + await waitFor(() => repos.runtime.getJob("tail")?.status === "succeeded"); + expect(starts).toEqual(["first", "manual-1", "manual-2", "manual-3", "tail"]); + expect(automaticResults.slice(0, 2)).toEqual([1, 0]); + expect(peak).toBe(1); expect(peakJobs).toBeLessThanOrEqual(4); expect(peakLeased).toBe(1); + await waitFor(() => internal.workerRunner.concurrency.snapshot().activeJobs === 0); + await waitFor(() => automaticResults.length === automaticCalls() && automaticResults.at(-1) === 0); + const settledCalls = automaticCalls(); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(automaticCalls()).toBe(settledCalls); + expect(settledCalls).toBeLessThanOrEqual(5); + expect(internal.workerRunner.concurrency.snapshot().providers).toEqual({}); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get()).toEqual({ n: 0 }); + } finally { for (const gate of gates.values()) gate.resolve(); await Promise.allSettled(pending); } + }); +}); diff --git a/Memory/tests/logger.test.ts b/Memory/tests/logger.test.ts index b5af7c23f..d3e2b68c3 100644 --- a/Memory/tests/logger.test.ts +++ b/Memory/tests/logger.test.ts @@ -47,6 +47,88 @@ describe("Memory structured logger", () => { expect(line).toContain("[redacted]"); }); + it.each([ + ["info", "job.started"], + ["info", "job.succeeded"], + ["error", "job.failed"], + ["info", "embedding_retry.succeeded"], + ["warn", "embedding_retry.retry_scheduled"], + ["error", "embedding_retry.failed"] + ] as const)("preserves scheduling identity in %s %s output", (level, event) => { + process.env.MEMMY_LOG_LEVEL = "info"; + const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + createMemoryLogger("worker")[level](event, { + jobId: "job-1", + retryId: "retry-1", + queueLane: "repair_retry", + enqueueSeq: 17, + attempt: 1, + maxAttempts: 3, + errorMessage: "Authorization: Bearer abc.def", + apiKey: "secret-value", + content: "private memory content" + }); + + const stream = level === "info" ? stdout : stderr; + const otherStream = level === "info" ? stderr : stdout; + expect(stream).toHaveBeenCalledTimes(1); + expect(otherStream).not.toHaveBeenCalled(); + const record = JSON.parse(String(stream.mock.calls[0]?.[0])) as Record; + expect(Object.keys(record)).toEqual(["timestamp", "level", "message"]); + expect(record.level).toBe(level); + expect(record.message).toContain("queueLane=repair_retry"); + expect(record.message).toContain("enqueueSeq=17"); + expect(record.message).not.toContain("abc.def"); + expect(record.message).not.toContain("secret-value"); + expect(record.message).not.toContain("private memory content"); + }); + + it("includes the actual scheduler and WRR weights in drain output", () => { + process.env.MEMMY_LOG_LEVEL = "info"; + const write = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const laneWeights = { interactive: 4, repair_retry: 3, import: 2, evolution: 2, maintenance: 1 }; + + createMemoryLogger("worker").info("drain.completed", { + algorithm: "wrr", + laneWeights, + leased: 4, + succeeded: 4, + failed: 0, + embeddingRetriesLeased: 2, + embeddingRetriesSucceeded: 2, + embeddingRetriesFailed: 0 + }); + + const record = JSON.parse(String(write.mock.calls[0]?.[0])) as Record; + expect(Object.keys(record)).toEqual(["timestamp", "level", "message"]); + expect(record.message).toContain("algorithm=wrr"); + expect(record.message).toContain("laneWeights=interactive:4|repair_retry:3|import:2|evolution:2|maintenance:1"); + expect(record.message).toContain("leased=4, succeeded=4, failed=0"); + expect(record.message).toContain("embeddingRetriesLeased=2"); + }); + + it("keeps legacy drain output concise and sanitizes scheduler fields", () => { + process.env.MEMMY_LOG_LEVEL = "info"; + const write = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const logger = createMemoryLogger("worker"); + logger.info("drain.completed", { algorithm: "legacy", leased: 1 }); + logger.info("drain.completed", { + algorithm: "wrr", + laneWeights: { interactive: 4, apiKey: "nested-secret" }, + content: "private memory content" + }); + + const legacy = JSON.parse(String(write.mock.calls[0]?.[0])) as Record; + expect(legacy.message).toContain("algorithm=legacy"); + expect(legacy.message).not.toContain("laneWeights="); + const sanitized = String(write.mock.calls[1]?.[0]); + expect(sanitized).toContain("[redacted]"); + expect(sanitized).not.toContain("nested-secret"); + expect(sanitized).not.toContain("private memory content"); + }); + it("honors MEMMY_LOG_LEVEL", () => { process.env.MEMMY_LOG_LEVEL = "warn"; const write = vi.spyOn(process.stdout, "write").mockImplementation(() => true); diff --git a/Memory/tests/model-catalog-inheritance.test.ts b/Memory/tests/model-catalog-inheritance.test.ts new file mode 100644 index 000000000..2fbee1fd1 --- /dev/null +++ b/Memory/tests/model-catalog-inheritance.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { syncMemoryModelCatalog } from "../src/config/model-catalog.js"; +import { resolveAssignedModel, type RuntimeModelCatalog } from "../src/contracts/model-catalog-resolver.js"; + +describe("Memory settings preserve independent endpoint options", () => { + it.each(["summary", "evolution", "embedding"] as const)("isolates %s without changing existing cloud defaults", role => { + const root: Record = { + app: { userMode: "byok" }, providers: { openai: { apiKey: "cloud-key", extraHeaders: { Authorization: "cloud-header" }, extraBody: { model: "cloud-model" }, + endpoints: { cloud: { apiBase: "https://cloud.example.test/v1", protocol: role === "embedding" ? "openai-embeddings" : "openai-chat-completions" } } } }, + modelPresets: {}, modelAssignments: { byok: {} } + }; + const original = structuredClone(root.providers.openai); + const connection = { provider: "openai_compatible", endpoint: "http://127.0.0.1:1234/v1", model: "own-model", extraHeaders: { "X-Own": "own" }, extraBody: { seed: 7 }, ...(role === "embedding" ? { mode: "custom" } : {}) }; + const memory: Record = { roleRouting: { summary: "fixed", evolution: "fixed" }, [role]: connection }; + syncMemoryModelCatalog(root, memory, { [role]: connection }); + const field = role === "summary" ? "memorySummary" : role === "evolution" ? "memoryEvolution" : "embedding"; + const id = root.modelAssignments.byok[field], preset = root.modelPresets[id]; + const endpoint = root.providers.openai.endpoints[preset.endpoint]; + expect(endpoint.inheritProviderDefaults).toBe(false); + const r = resolveAssignedModel({ catalog: root as RuntimeModelCatalog, mode: "byok", capability: role === "summary" ? "memory_summary" : role === "evolution" ? "memory_evolution" : "embedding" }); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.provider.apiKey).toBeUndefined(); + expect(r.provider.extraHeaders).toEqual(connection.extraHeaders); + expect(r.provider.extraBody).toEqual(connection.extraBody); + } + expect({ ...root.providers.openai, endpoints: original.endpoints }).toEqual(original); + const once = structuredClone(root); + syncMemoryModelCatalog(root, memory, { [role]: connection }); + expect(root).toEqual(once); + }); + it("does not reuse a currently equal inherited endpoint that may rotate later", () => { + const connection = { provider: "openai_compatible", endpoint: "https://custom.example.test/v1", model: "custom-model", apiKey: "initial-key" }; + const root: Record = { + providers: { openai: { apiKey: "initial-key", endpoints: { + inherited: { apiBase: connection.endpoint, protocol: "openai-chat-completions" }, + } } }, + modelPresets: { existing: { provider: "openai", endpoint: "inherited", model: connection.model, source: "byok", capabilities: ["memory_summary"] } }, + modelAssignments: { byok: { memorySummary: "existing" } }, + }; + const memory = { roleRouting: { summary: "fixed" }, summary: connection }; + syncMemoryModelCatalog(root, memory, { summary: connection }); + const id = root.modelAssignments.byok.memorySummary; + expect(id).not.toBe("existing"); + root.providers.openai.apiKey = "rotated-key"; + const saved = resolveAssignedModel({ catalog: root as RuntimeModelCatalog, mode: "byok", capability: "memory_summary" }); + expect(saved.ok && saved.provider.apiKey).toBe("initial-key"); + const inherited = resolveAssignedModel({ catalog: { ...root, modelAssignments: { byok: { memorySummary: "existing" } } } as RuntimeModelCatalog, mode: "byok", capability: "memory_summary" }); + expect(inherited.ok && inherited.provider.apiKey).toBe("rotated-key"); + const once = structuredClone(root); + syncMemoryModelCatalog(root, memory, { summary: connection }); + expect(root).toEqual(once); + }); + +}); diff --git a/Memory/tests/model/token-usage-outbox-delivery.test.ts b/Memory/tests/model/token-usage-outbox-delivery.test.ts index d9a5a16a6..ba9a7160b 100644 --- a/Memory/tests/model/token-usage-outbox-delivery.test.ts +++ b/Memory/tests/model/token-usage-outbox-delivery.test.ts @@ -11,6 +11,7 @@ import { import { MemoryTokenBudgetLedger } from "../../src/service/memory-token-budget-ledger.js"; import { Repositories } from "../../src/storage/repositories.js"; import { TokenUsageOutbox } from "../../src/storage/token-usage-outbox.js"; +import { JobLeaseLostError, withJobLeaseGuard } from "../../src/storage/job-lease-context.js"; const roots: string[] = []; const recorders: HttpByokTokenUsageRecorder[] = []; @@ -26,6 +27,62 @@ afterEach(() => { }); describe("durable BYOK token usage delivery", () => { + it("accounts for consumed tokens after a task loses its lease while keeping business writes fenced", async () => { + const harness = createSqliteHarness(); + const fetchMock = vi.fn(async () => jsonResponse({ ok: true })); + const recorder = sqliteRecorder(harness, fetchMock); + let held = true; + withJobLeaseGuard({ id: "late-model-result" }, () => held, () => { + held = false; + expect(recorder.record(usageEvent({ totalTokens: 37 }))).toBe("recorded"); + expect(harness.ledger.snapshot().lifetimeUsed).toBe(37); + expect(() => harness.db.db.prepare("INSERT INTO runtime_kv(key, value_json, updated_at) VALUES ('obsolete', '{}', 'now')").run()) + .toThrow(JobLeaseLostError); + }); + await waitFor(() => fetchMock.mock.calls.length === 1 && !harness.outbox.hasPending()); + expect(recorder.isPersistUnreliable()).toBe(false); + }); + + it("deletes an acknowledged outbox event after the producing task has finished", async () => { + let release!: (response: Response) => void; + const fetchMock = vi.fn(() => new Promise((done) => { release = done; })); + const harness = createSqliteHarness(); + const recorder = sqliteRecorder(harness, fetchMock); + let held = true; + withJobLeaseGuard({ id: "finished-job" }, () => held, () => { + expect(recorder.record(usageEvent({ totalTokens: 23 }))).toBe("recorded"); + }); + await waitFor(() => fetchMock.mock.calls.length === 1); + held = false; + release(jsonResponse({ ok: true })); + await waitFor(() => !harness.outbox.hasPending()); + expect(harness.ledger.snapshot().lifetimeUsed).toBe(23); + expect(recorder.isPersistUnreliable()).toBe(false); + }); + + it("lets delivery retries outlive the producing task lease", async () => { + const harness = createSqliteHarness(); + let firstResponse!: (response: Response) => void; + let requests = 0; + const fetchMock = vi.fn((): Promise => { + requests++; + return requests === 1 + ? new Promise((done) => { firstResponse = done; }) + : Promise.resolve(jsonResponse({ ok: true })); + }); + const recorder = sqliteRecorder(harness, fetchMock); + let held = true; + withJobLeaseGuard({ id: "retry-producing-job" }, () => held, () => { + recorder.record(usageEvent({ totalTokens: 11 })); + }); + await waitFor(() => fetchMock.mock.calls.length === 1); + held = false; + firstResponse(new Response("temporarily unavailable", { status: 503 })); + await waitFor(() => fetchMock.mock.calls.length === 2 && !harness.outbox.hasPending()); + expect(harness.ledger.snapshot().lifetimeUsed).toBe(11); + expect(recorder.isPersistUnreliable()).toBe(false); + }); + it("persists then sends the original payload and deletes after { ok: true }", async () => { const fetchMock = vi.fn(async () => jsonResponse({ ok: true })); const { recorder, outbox, ledger } = createRecorder(fetchMock); diff --git a/Memory/tests/repository/embedding-retry-lease-counters.test.ts b/Memory/tests/repository/embedding-retry-lease-counters.test.ts new file mode 100644 index 000000000..bba4466b5 --- /dev/null +++ b/Memory/tests/repository/embedding-retry-lease-counters.test.ts @@ -0,0 +1,204 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Database from "better-sqlite3"; +import { describe, expect, it } from "vitest"; +import { pendingPolicyIndex } from "../../src/storage/queue-scheduling-schema.js"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { SCHEMA_VERSION } from "../../src/storage/schema.js"; + +const now = Date.parse("2026-10-23T00:00:00.000Z"); +const counters = ["lease_count", "lease_recovery_count"] as const; + +function usingDatabase(run: (repos: Repositories, db: MemoryDb) => void) { + const root = mkdtempSync(join(tmpdir(), "memmy-retry-counters-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + try { run(new Repositories(db.db), db); } + finally { db.close(); rmSync(root, { recursive: true, force: true }); } +} + +function enqueue(repos: Repositories, id: string) { + return repos.runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, + vectorField: "vec_summary", sourceText: `source-${id}`, now }); +} + +describe("Embedding retry lease counters", () => { + it.each(["id", "batch"] as const)("counts %s claims and expired takeovers without spending business attempts or weakening fencing", api => { + usingDatabase(repos => { + enqueue(repos, "takeover"); + const claim = (at: number, workerId: string, leaseUntil: number) => api === "id" + ? repos.runtime.claimEmbeddingRetryById("takeover", { now: at, workerId, leaseUntil }) + : repos.runtime.claimDueEmbeddingRetries({ now: at, workerId, leaseUntil, targetMemoryIds: ["takeover"] })[0]; + const first = claim(now, "A", now + 1000)!; + expect(first).toMatchObject({ attempts: 0, leaseCount: 1, leaseRecoveryCount: 0 }); + expect(claim(now + 500, "B", now + 5000)).toBeUndefined(); + const second = claim(now + 1001, "B", now + 5000)!; + expect(second).toMatchObject({ attempts: 0, leaseCount: 2, leaseRecoveryCount: 1 }); + const stale = { workerId: "A", leaseUntil: first.leaseUntil!, leaseToken: first.leaseToken!, now: now + 1002 }; + expect(repos.runtime.markEmbeddingRetrySucceededClaimed("takeover", stale)).toBeUndefined(); + expect(repos.runtime.markEmbeddingRetryFailedClaimed("takeover", { ...stale, attempts: 1, error: "late failure" })).toBeUndefined(); + expect(repos.runtime.renewEmbeddingRetryClaim("takeover", stale, now + 9000, now + 1002)).toBeUndefined(); + expect(repos.runtime.getEmbeddingRetry("takeover")).toMatchObject({ leaseToken: second.leaseToken, + attempts: 0, leaseCount: 2, leaseRecoveryCount: 1 }); + }); + }); + + it("counts each restart recovery once, retains valid leases, and does not count the following claim as another recovery", () => { + usingDatabase(repos => { + enqueue(repos, "expired"); enqueue(repos, "valid"); + repos.runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 1000, targetMemoryIds: ["expired"] }); + repos.runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 9000, targetMemoryIds: ["valid"] }); + expect(repos.runtime.requeueEmbeddingRetriesAfterRestart(now + 1001).map(item => item.after.id)).toEqual(["expired"]); + expect(repos.runtime.requeueEmbeddingRetriesAfterRestart(now + 1001)).toEqual([]); + expect(repos.runtime.getEmbeddingRetry("expired")).toMatchObject({ status: "pending", attempts: 0, leaseCount: 1, leaseRecoveryCount: 1 }); + expect(repos.runtime.getEmbeddingRetry("valid")).toMatchObject({ status: "in_progress", attempts: 0, leaseCount: 1, leaseRecoveryCount: 0 }); + const next = repos.runtime.claimDueEmbeddingRetries({ now: now + 1001, workerId: "B", leaseUntil: now + 9000, targetMemoryIds: ["expired"] })[0]!; + expect(next).toMatchObject({ attempts: 0, leaseCount: 2, leaseRecoveryCount: 1 }); + expect(repos.runtime.releaseClaimedEmbeddingRetries([next], now + 1002)[0]!.after) + .toMatchObject({ attempts: 0, leaseCount: 2, leaseRecoveryCount: 1 }); + }); + }); + + it("preserves cumulative lease counters across business failures, duplicate deliveries and changed input", () => { + usingDatabase(repos => { + enqueue(repos, "cumulative"); + const claimed = repos.runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 5000 })[0]!; + repos.runtime.markEmbeddingRetryRetryClaimed(claimed.id, { workerId: "A", leaseUntil: claimed.leaseUntil!, + leaseToken: claimed.leaseToken!, attempts: 1, nextAttemptAt: now + 3000, error: "temporary", now: now + 1 }); + expect(enqueue(repos, "cumulative")).toMatchObject({ attempts: 1, leaseCount: 1, leaseRecoveryCount: 0, nextAttemptAt: now + 3000 }); + const changed = repos.runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "cumulative", + vectorField: "vec_summary", sourceText: "changed source", now: now + 2 }); + expect(changed).toMatchObject({ attempts: 0, leaseCount: 1, leaseRecoveryCount: 0 }); + const configured = repos.runtime.claimEmbeddingRetryById(changed.id, { now: now + 3, workerId: "B", leaseUntil: now + 5000 })!; + repos.runtime.markEmbeddingRetryConfigurationBlockedClaimed(configured.id, { workerId: "B", leaseUntil: configured.leaseUntil!, + leaseToken: configured.leaseToken!, error: "missing credentials", now: now + 4 }); + expect(repos.runtime.resumeConfigurationBlockedEmbeddingRetries(now + 5)[0]!.after) + .toMatchObject({ attempts: 0, leaseCount: 2, leaseRecoveryCount: 0 }); + }); + }); + + it("rolls back a failed claim transaction without consuming lease or recovery counters", () => { + usingDatabase(repos => { + enqueue(repos, "rollback"); + expect(() => repos.transaction(() => { + repos.runtime.claimEmbeddingRetryById("rollback", { now, workerId: "A", leaseUntil: now + 5000 }); + throw new Error("abort claim"); + })).toThrow("abort claim"); + expect(repos.runtime.getEmbeddingRetry("rollback")).toMatchObject({ status: "pending", attempts: 0, + leaseCount: 0, leaseRecoveryCount: 0, leaseToken: null }); + }); + }); +}); + +function downgradeTo(db: MemoryDb, version: 9 | 12) { + // These auxiliary objects did not exist in either historical schema. + for (const table of ["evolution_jobs", "embedding_retry_queue"]) + for (const suffix of ["deadline_insert", "deadline_update"]) db.db.exec(`DROP TRIGGER ${table}_${suffix}`); + for (const index of ["worker_main_deadline", "worker_main_lease_deadline", "worker_retry_lease_deadline"]) + db.db.exec(`DROP INDEX ${index}`); + db.db.exec("DROP TABLE worker_deadline_revision"); + const columns = () => (db.db.prepare("PRAGMA table_info(embedding_retry_queue)").all() as Array<{ name: string }>).map(row => row.name); + for (const column of counters) if (columns().includes(column)) db.db.exec(`ALTER TABLE embedding_retry_queue DROP COLUMN ${column}`); + if (version === 9) { + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + for (const suffix of ["enqueue_sequence", "policy_dirty"]) db.db.exec(`DROP TRIGGER ${table}_${suffix}`); + db.db.exec(`DROP INDEX ${pendingPolicyIndex(table)}`); + db.db.exec(`DROP INDEX ${pendingPolicyIndex(table, true)}`); + for (const suffix of ["enqueue_seq", "lane_due", "lane_fifo", "policy_dirty"]) db.db.exec(`DROP INDEX ${table}_${suffix}`); + } + for (const suffix of ["dirty", "insert", "delete"]) db.db.exec(`DROP TRIGGER memories_worker_policy_${suffix}`); + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + for (const column of ["queue_lane", "resource_class", "enqueue_seq", "first_queued_at", "base_priority", "policy_version", "policy_promotion"]) + db.db.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); + } + db.db.exec("DROP TABLE worker_enqueue_counter"); + db.db.exec("DROP INDEX idx_evolution_jobs_available"); + for (const column of ["lease_owner", "lease_token", "available_at", "blocked_reason", "failure_count", "lease_count", "lease_recovery_count"]) + db.db.exec(`ALTER TABLE evolution_jobs DROP COLUMN ${column}`); + for (const column of ["blocked_reason", "lease_token"]) db.db.exec(`ALTER TABLE embedding_retry_queue DROP COLUMN ${column}`); + } + db.db.exec("DELETE FROM schema_migrations"); + db.db.prepare("INSERT INTO schema_migrations VALUES (?,?,?,'legacy')") + .run(version === 9 ? "009_token_usage_outbox" : "012_queue_scheduling_metadata", version, new Date(now).toISOString()); +} + +describe("Embedding retry counter schema upgrade", () => { + it.each([9, 12] as const)("upgrades v%s with a backup, a documented historical lower bound and stable subsequent reopens", version => { + const root = mkdtempSync(join(tmpdir(), "memmy-retry-counter-upgrade-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined, backup: Database.Database | undefined; + try { + db = new MemoryDb({ path }); downgradeTo(db, version); + const insert = db.db.prepare(`INSERT INTO embedding_retry_queue + (id,target_kind,target_id,vector_field,source_text,status,attempts,max_attempts,next_attempt_at,claimed_by,lease_until,created_at,updated_at) + VALUES (?,'trace',?,'vec_summary','retained',?,?,6,?,?,?, ?,?)`); + for (const [status, attempts] of [["pending", 2], ["in_progress", 3], ["succeeded", 1], ["failed", 4]] as const) { + insert.run(status, status, status, attempts, now, status === "in_progress" ? "old-worker" : null, + status === "in_progress" ? now + 1000 : null, now, now); + } + if (version === 12) db.db.prepare("UPDATE embedding_retry_queue SET lease_token='old-token' WHERE id='in_progress'").run(); + db.close(); db = new MemoryDb({ path }); + expect(db.schemaVersion().version).toBe(13); + expect(SCHEMA_VERSION).toBe(13); + const repos = new Repositories(db.db); + // Failures prove prior claims; an active/succeeded row proves one more. + // Historical lease losses are unknowable, so recovery starts at zero. + for (const [id, attempts, lowerBound] of [["pending", 2, 2], ["in_progress", 3, 4], ["succeeded", 1, 2], ["failed", 4, 4]] as const) + expect(repos.runtime.getEmbeddingRetry(id)).toMatchObject({ attempts, leaseCount: lowerBound, leaseRecoveryCount: 0 }); + expect(repos.runtime.requeueEmbeddingRetriesAfterRestart(now)).toEqual([]); + if (version === 12) expect(repos.runtime.isEmbeddingRetryClaimHeld("in_progress", { + workerId: "old-worker", leaseUntil: now + 1000, leaseToken: "old-token" + }, now)).toBe(true); + expect(repos.runtime.claimEmbeddingRetryById("in_progress", { now, workerId: "B", leaseUntil: now + 9000 })).toBeUndefined(); + const recovered = repos.runtime.claimEmbeddingRetryById("in_progress", { now: now + 1001, workerId: "B", leaseUntil: now + 9000 })!; + expect(recovered).toMatchObject({ attempts: 3, leaseCount: 5, leaseRecoveryCount: 1, leaseToken: expect.any(String) }); + expect(repos.runtime.markEmbeddingRetrySucceededClaimed("in_progress", { workerId: "old-worker", + leaseUntil: now + 1000, leaseToken: "old-token", now: now + 1002 })).toBeUndefined(); + const backupPath = `${path}.pre-v13.bak`; + expect(existsSync(backupPath)).toBe(true); + const originalBackup = readFileSync(backupPath); + backup = new Database(backupPath, { readonly: true }); + const backupColumns = (backup.prepare("PRAGMA table_info(embedding_retry_queue)").all() as Array<{ name: string }>).map(row => row.name); + for (const column of counters) expect(backupColumns).not.toContain(column); + expect(backup.prepare("SELECT MAX(version) AS version FROM schema_migrations").get()).toEqual({ version }); + backup.close(); backup = undefined; + db.close(); db = new MemoryDb({ path }); + expect(new Repositories(db.db).runtime.getEmbeddingRetry("in_progress")).toMatchObject({ attempts: 3, + leaseCount: 5, leaseRecoveryCount: 1, leaseToken: recovered.leaseToken }); + expect(readFileSync(backupPath)).toEqual(originalBackup); + } finally { + backup?.close(); if (db?.db.open) db.close(); rmSync(root, { recursive: true, force: true }); + } + }); + + it("rolls a failed v12 migration back completely and keeps its original backup for the successful retry", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-retry-counter-rollback-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined, raw: Database.Database | undefined; + try { + db = new MemoryDb({ path }); downgradeTo(db, 12); + db.db.prepare(`INSERT INTO embedding_retry_queue + (id,target_kind,target_id,vector_field,source_text,status,attempts,max_attempts,next_attempt_at, + claimed_by,lease_until,lease_token,created_at,updated_at) + VALUES ('retained','trace','retained','vec_summary','unchanged','in_progress',2,6,?, + 'old-owner',?,'old-token',?,?)`).run(now, now + 9000, now, now); + db.db.exec(`CREATE TRIGGER refuse_counter_upgrade BEFORE INSERT ON schema_migrations + WHEN NEW.version=13 BEGIN SELECT RAISE(ABORT,'reject counter upgrade'); END`); + db.close(); db = undefined; + expect(() => { db = new MemoryDb({ path }); }).toThrow("reject counter upgrade"); + const backupPath = `${path}.pre-v13.bak`, originalBackup = readFileSync(backupPath); + raw = new Database(path); + const columns = (raw.prepare("PRAGMA table_info(embedding_retry_queue)").all() as Array<{ name: string }>).map(row => row.name); + for (const column of counters) expect(columns).not.toContain(column); + expect(raw.prepare("SELECT MAX(version) AS version FROM schema_migrations").get()).toEqual({ version: 12 }); + expect(raw.prepare("SELECT status,attempts,claimed_by,lease_token,source_text FROM embedding_retry_queue WHERE id='retained'").get()) + .toEqual({ status: "in_progress", attempts: 2, claimed_by: "old-owner", lease_token: "old-token", source_text: "unchanged" }); + raw.exec("DROP TRIGGER refuse_counter_upgrade"); raw.close(); raw = undefined; + db = new MemoryDb({ path }); expect(db.schemaVersion().version).toBe(13); + expect(new Repositories(db.db).runtime.getEmbeddingRetry("retained")).toMatchObject({ status: "in_progress", + attempts: 2, claimedBy: "old-owner", leaseToken: "old-token", leaseCount: 3, leaseRecoveryCount: 0 }); + expect(readFileSync(backupPath)).toEqual(originalBackup); + } finally { + raw?.close(); if (db?.db.open) db.close(); rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/Memory/tests/repository/fair-queue-scope-recovery.test.ts b/Memory/tests/repository/fair-queue-scope-recovery.test.ts new file mode 100644 index 000000000..fb638f1a7 --- /dev/null +++ b/Memory/tests/repository/fair-queue-scope-recovery.test.ts @@ -0,0 +1,172 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../src/storage/repositories.js"; +import { claimFairWorkerBatch, type FairClaimedWork } from "../../src/service/worker/fair-worker-queue.js"; +import { JOB_POLICY_VERSION } from "../../src/service/worker/job-policy-registry.js"; +import { resolveWorkerSchedulingOptions } from "../../src/service/worker/worker-scheduling-options.js"; +import type { QueueEligibilityOptions } from "../../src/storage/queue-eligibility.js"; + +const at = "2026-10-22T00:00:00.000Z", now = Date.parse(at); +const cursorKey = "worker.scheduler.wrr_cursor.v1"; +const resources: Array<{ db: MemoryDb; root: string }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "memmy-fair-scope-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ db, root }); + const repos = new Repositories(db.db), runtime = repos.runtime; + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "trace_summary", targetMemoryId = id) { + return runtime.enqueueJob({ id, jobType, targetMemoryId, userId: "user", status: "queued", payload: {}, + attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + function failed(id: string, jobType: EvolutionJobRecord["jobType"] = "import_summary") { + job(id, jobType); + const claim = runtime.claimQueuedJob(id, { at })!; + runtime.failJob(id, "transient", at, false, claim, { availableAt: at }); + // Simulate an active failed row imported from before policy projections. + db.db.prepare("UPDATE evolution_jobs SET policy_version=NULL WHERE id=?").run(id); + return runtime.getJob(id)!; + } + function claim(limit: number, filters: QueueEligibilityOptions & { includeRetries?: boolean } = {}) { + return claimFairWorkerBatch(repos, { at, limit, includeRetries: true, owner: "main", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({}, {}), ...filters }); + } + function finish(work: FairClaimedWork[]) { + for (const item of work) if (item.queue === "main") runtime.completeJob(item.job.id, at, item.job); + } + return { db, repos, runtime, job, failed, claim, finish }; +} +afterEach(() => { + for (const { db, root } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("fair queue scope cursors and failed recovery", () => { + it("keeps a filtered call from consuming the global round while retaining its own round", () => { + const h = harness(); + for (const prefix of ["local", "global"]) for (let i = 0; i < 8; i++) { + h.job(`${prefix}-i-${i}`, "trace_summary", prefix); + h.job(`${prefix}-r-${i}`, "decision_repair", prefix); + } + h.finish(h.claim(4)); + const global = h.runtime.getKv(cursorKey); + const local: string[] = []; + for (let i = 0; i < 5; i++) { + const work = h.claim(1, { targetMemoryIds: ["local"] }); + local.push(work[0]!.queue === "main" ? work[0]!.job.jobType : "retry"); h.finish(work); + } + expect(local).toEqual(["trace_summary", "trace_summary", "trace_summary", "trace_summary", "decision_repair"]); + expect(h.runtime.getKv(cursorKey)).toEqual(global); + }); + + it("normalizes target and type filter sets but preserves distinct scopes", () => { + const h = harness(); + for (let i = 0; i < 8; i++) { + h.job(`i-${i}`, "trace_summary", "a"); h.job(`r-${i}`, "decision_repair", "b"); + } + const types = ["trace_summary", "decision_repair"]; + for (let i = 0; i < 5; i++) { + const work = h.claim(1, { targetMemoryIds: i % 2 ? ["b", "a", "a"] : ["a", "b"], + allowedJobTypes: i % 2 ? [...types].reverse() : types, + excludedJobTypes: i % 2 ? ["reward", "reward"] : ["reward"] }); + expect(work[0]!.queue === "main" && work[0]!.job.jobType).toBe(i < 4 ? "trace_summary" : "decision_repair"); + h.finish(work); + } + expect(h.runtime.listKv(cursorKey)).toHaveLength(1); + h.finish(h.claim(1, { targetMemoryIds: ["a", "b"], allowedJobTypes: types, excludedJobTypes: ["reward"], includeRetries: false })); + h.finish(h.claim(1, { targetMemoryIds: ["a", "b"], allowedJobTypes: ["trace_summary"], excludedJobTypes: ["reward"] })); + expect(h.runtime.listKv(cursorKey)).toHaveLength(3); + expect(h.runtime.getKv(cursorKey)).toBeUndefined(); + }); + + it("hashes the entire target set including entries beyond one thousand", () => { + const h = harness(); h.job("one", "trace_summary", "selected"); h.job("two", "trace_summary", "selected"); + const targets = ["selected", ...Array.from({ length: 1_001 }, (_, i) => `unused-${String(i).padStart(4, "0")}`)]; + h.finish(h.claim(1, { targetMemoryIds: [...targets, "z-tail-one"] })); + h.finish(h.claim(1, { targetMemoryIds: [...targets, "z-tail-two"] })); + expect(h.runtime.listKv(cursorKey)).toHaveLength(2); + expect(h.runtime.getKv(cursorKey)).toBeUndefined(); + }); + + it("puts due failed work directly into lane FIFO before newer queued work", () => { + const h = harness(); const previous = h.failed("old-failed"); h.job("new-queued", "import_summary"); + const work = h.claim(1); + expect(work).toHaveLength(1); + expect(work[0]!.queue === "main" && work[0]!.job).toMatchObject({ id: "old-failed", status: "leased", + attempts: previous.attempts + 1, failureCount: 1, leaseCount: 2 }); + expect(h.runtime.getJob("new-queued")?.status).toBe("queued"); + expect(h.db.db.prepare("SELECT policy_version FROM evolution_jobs WHERE id='old-failed'").get()).toMatchObject({ policy_version: JOB_POLICY_VERSION }); + }); + + it("rechecks all failed eligibility fences and keeps legacy claims opt-in", () => { + const h = harness(); + for (const id of ["ready", "future", "config", "exhausted", "runAfter", "wrong-target", "wrong-type"]) h.failed(id); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='future'").run(new Date(now + 1).toISOString()); + h.db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='config'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET failure_count=max_attempts WHERE id='exhausted'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='runAfter'").run(JSON.stringify({ runAfter: new Date(now + 1).toISOString() })); + h.db.db.prepare("UPDATE evolution_jobs SET job_type='reflection' WHERE id='wrong-type'").run(); + expect(h.runtime.claimQueuedJob("ready", { at })).toBeUndefined(); + expect(h.runtime.leaseQueuedJobs(20)).toEqual([]); + const before = new Map(["future", "config", "exhausted", "runAfter", "wrong-target", "wrong-type"].map(id => [id, h.runtime.getJob(id)!])); + const work = h.claim(20, { targetMemoryIds: ["ready", "future", "config", "exhausted", "runAfter", "wrong-type"], allowedJobTypes: ["import_summary"] }); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["ready"]); + for (const id of ["future", "config", "exhausted", "runAfter", "wrong-target", "wrong-type"]) { + const prior = before.get(id)!; + // Metadata backfill includes delayed work, but never bypasses claim eligibility. + expect(h.runtime.getJob(id)).toMatchObject({ status: "failed", attempts: prior.attempts, + failureCount: prior.failureCount, leaseToken: prior.leaseToken, availableAt: prior.availableAt, + blockedReason: prior.blockedReason, payload: prior.payload }); + expect(h.db.db.prepare("SELECT policy_version FROM evolution_jobs WHERE id=?").get(id)).toMatchObject({ policy_version: JOB_POLICY_VERSION }); + } + expect(h.runtime.claimQueuedJob("wrong-target", { at, includeDueFailed: true, excludedJobTypes: ["import_summary"] })).toBeUndefined(); + }); + + it("fences the previous failed executor and preserves failure history when returning an unstarted claim", () => { + const h = harness(); h.job("retry"); + const old = h.runtime.claimQueuedJob("retry", { at, owner: "executor-a" })!; + h.runtime.failJob(old.id, "transient", at, false, old, { availableAt: at }); + const next = h.runtime.claimQueuedJob(old.id, { at, owner: "executor-b", includeDueFailed: true })!; + expect(next).toMatchObject({ attempts: 2, failureCount: 1, leaseCount: 2, leaseRecoveryCount: 0 }); + expect(next.leaseToken).not.toBe(old.leaseToken); + expect(h.runtime.completeJob(old.id, at, old)).toBeUndefined(); + expect(h.runtime.failJob(old.id, "stale error", at, true, old)).toBeUndefined(); + expect(h.runtime.releaseClaimedJobs([old], at)).toEqual([]); + expect(h.runtime.releaseClaimedJobs([next], at)[0]?.after).toMatchObject({ status: "queued", attempts: 1, + failureCount: 1, leaseCount: 2, leaseOwner: null, leaseToken: null }); + }); + + it("preserves failed dependency and business scope ordering", () => { + const h = harness(); + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,created_at,updated_at) + VALUES ('dependency',?,'user','text','L1',?,?)`).run(at, at, at); + h.db.db.prepare("INSERT INTO memory_processing_state(memory_id,state,stage,updated_at) VALUES ('dependency','embedding_pending','embedding',?)").run(at); + h.failed("reward", "reward"); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='reward'").run('{"l1MemoryId":"dependency","polarity":"positive"}'); + for (const [index, id] of ["scope-first", "scope-second"].entries()) { + h.job(id, "work_memory_extract"); + h.db.db.prepare("UPDATE evolution_jobs SET scope_key='scope',scope_seq=?,status='failed',failure_count=1,available_at=? WHERE id=?").run(index + 1, at, id); + } + const work = h.claim(10); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["scope-first"]); + expect(h.runtime.getJob("reward")?.status).toBe("failed"); + expect(h.runtime.getJob("scope-second")?.status).toBe("failed"); + h.finish(work); + expect(h.claim(1).map(item => item.queue === "main" && item.job.id)).toEqual(["scope-second"]); + }); + + it("rolls failed claim, lazy projection and scoped cursor back atomically", () => { + const h = harness(); const previous = h.failed("rollback"); + expect(() => h.repos.transaction(() => { + expect(h.claim(1, { targetMemoryIds: ["rollback"] })).toHaveLength(1); + throw new Error("rollback admission"); + })).toThrow("rollback admission"); + expect(h.runtime.getJob("rollback")).toEqual(previous); + expect(h.runtime.listKv(cursorKey)).toEqual([]); + expect(h.db.db.prepare("SELECT policy_version FROM evolution_jobs WHERE id='rollback'").get()).toMatchObject({ policy_version: null }); + }); +}); diff --git a/Memory/tests/repository/history-change-log.test.ts b/Memory/tests/repository/history-change-log.test.ts index 815d9a669..b5e1b9b6c 100644 --- a/Memory/tests/repository/history-change-log.test.ts +++ b/Memory/tests/repository/history-change-log.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { MemoryDb } from "../../src/storage/db.js"; import { Repositories } from "../../src/storage/repositories.js"; +import { withJobLeaseGuard } from "../../src/storage/job-lease-context.js"; import { compactHistoryChangeJson, compactHistoryChangePayload } from "../../src/storage/history-change-snapshot.js"; import { sha256Hex } from "../../src/contracts/memory-canonical-json.js"; import type { MemoryRow } from "../../src/types.js"; @@ -23,6 +24,34 @@ function history(state: "pending" | "processed" | "skipped" | undefined = "pendi } describe("History change log storage", () => { + it("lets independently scheduled History compaction finish after its originating task completes", async () => { + const root = mkdtempSync(join(tmpdir(), "memmy-history-compaction-lease-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + try { + const repos = new Repositories(db.db); + // Complete the initial empty startup pass so it cannot mask the pass + // deliberately scheduled from an active task context below. + await new Promise((resolve) => setImmediate(resolve)); + const current = new Date().toISOString(); + repos.runtime.enqueueJob({ id: "history-maintenance-origin", jobType: "reflection", status: "queued", + userId: "user", payload: {}, attempts: 0, maxAttempts: 3, createdAt: current, updatedAt: current }); + const [claim] = repos.runtime.leaseQueuedJobs(1); + const raw = JSON.stringify(history()); + db.db.prepare(`INSERT INTO memory_change_log + (memory_id, user_id, change_type, after_json, source, created_at) + VALUES ('observation_test', 'user', 'updated', ?, 'legacy', ?)`).run(raw, at); + withJobLeaseGuard(claim!, () => repos.runtime.isJobLeaseHeld(claim!), () => new Repositories(db.db)); + expect(repos.runtime.completeJob(claim!.id, undefined, claim)?.status).toBe("succeeded"); + await waitUntil(() => !(db.db.prepare("SELECT after_json FROM memory_change_log").get() as { after_json: string }) + .after_json.includes('"source_content":')); + expect((db.db.prepare("SELECT after_json FROM memory_change_log").get() as { after_json: string }) + .after_json).toBe(JSON.stringify(compactHistoryChangePayload(history()))); + } finally { + db.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it("omits input copies without mutating the current snapshot and records exact UTF-8 evidence", () => { const original = history(); const before = JSON.stringify(original); diff --git a/Memory/tests/repository/job-lease-context.test.ts b/Memory/tests/repository/job-lease-context.test.ts new file mode 100644 index 000000000..31f7183bc --- /dev/null +++ b/Memory/tests/repository/job-lease-context.test.ts @@ -0,0 +1,171 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + assertCurrentJobLease, + JobLeaseLostError, + withJobLeaseGuard +} from "../../src/storage/job-lease-context.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import type { EmbeddingJobProcessor } from "../../src/service/embedding/embedding-job-processor.js"; +import { + addAgentSourceImport, + createBatchReflectionLlm, + createCapturingEmbedder, + createMemoryServiceFixture +} from "../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +describe("worker lease write context", () => { + it("keeps ordinary writes and readonly queries available, but rejects stale prepared writes", () => { + const { db } = fixture.createTestService(); + db.db.exec("CREATE TABLE fence_test (id INTEGER PRIMARY KEY, value TEXT)"); + const insert = db.db.prepare("INSERT INTO fence_test (value) VALUES (?)"); + insert.run("foreground"); + let held = true; + withJobLeaseGuard({ id: "job-A" }, () => held, () => { + insert.run("valid-worker"); + held = false; + expect(() => insert.run("obsolete-worker")).toThrow(JobLeaseLostError); + expect(() => db.db.exec("DELETE FROM fence_test")).toThrow(JobLeaseLostError); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM fence_test").get()).toEqual({ count: 2 }); + }); + insert.run("foreground-after-context"); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM fence_test").get()).toEqual({ count: 3 }); + }); + + it.each(["get", "all", "iterate"] as const)("guards a RETURNING mutation executed using %s", (method) => { + const { db } = fixture.createTestService(); + db.db.exec("CREATE TABLE fence_returning (id INTEGER PRIMARY KEY, value TEXT)"); + const statement = db.db.prepare("INSERT INTO fence_returning (value) VALUES (?) RETURNING id"); + let held = true; + withJobLeaseGuard({ id: "returning-job" }, () => held, () => { + held = false; + expect(() => statement[method]("stale")).toThrow(JobLeaseLostError); + }); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM fence_returning").get()).toEqual({ count: 0 }); + }); + + it("rechecks a lazy mutating iterator before its first step", () => { + const { db } = fixture.createTestService(); + db.db.exec("CREATE TABLE fence_iterator (id INTEGER PRIMARY KEY, value TEXT)"); + let held = true; + withJobLeaseGuard({ id: "iterator-job" }, () => held, () => { + const iterator = db.db.prepare("INSERT INTO fence_iterator (value) VALUES ('stale') RETURNING id").iterate(); + held = false; + expect(() => iterator.next()).toThrow(JobLeaseLostError); + iterator.return?.(); + }); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM fence_iterator").get()).toEqual({ count: 0 }); + }); + + it("isolates overlapping async claims and rolls back a stale multi-statement transaction", async () => { + const { db } = fixture.createTestService(); + db.db.exec("CREATE TABLE fence_concurrent (value TEXT)"); + let release!: () => void; + const wait = new Promise((done) => { release = done; }); + let heldA = true; + const a = withJobLeaseGuard({ id: "A" }, () => heldA, async () => { + await wait; + db.db.prepare("INSERT INTO fence_concurrent VALUES ('A')").run(); + }); + const rejected = expect(a).rejects.toMatchObject({ name: "JobLeaseLostError", jobId: "A" }); + withJobLeaseGuard({ id: "B" }, () => true, () => { + db.db.prepare("INSERT INTO fence_concurrent VALUES ('B')").run(); + }); + heldA = false; + release(); + await rejected; + let checks = 0; + expect(() => withJobLeaseGuard({ id: "rollback" }, () => ++checks < 3, () => { + db.db.transaction(() => { + db.db.prepare("INSERT INTO fence_concurrent VALUES ('first')").run(); + db.db.prepare("INSERT INTO fence_concurrent VALUES ('second')").run(); + })(); + })).toThrow(JobLeaseLostError); + expect(db.db.prepare("SELECT value FROM fence_concurrent").all()).toEqual([{ value: "B" }]); + // The rejected worker's context must not leak into the test/foreground task. + expect(() => assertCurrentJobLease()).not.toThrow(); + }); + + it("does not persist a delayed imported summary or enqueue its embedding after another lease takes over", async () => { + let release!: () => void; + const gate = new Promise((done) => { release = done; }); + let entered!: () => void; + const enteredModel = new Promise((done) => { entered = done; }); + const llm = createBatchReflectionLlm([], "obsolete summary must never be saved"); + const original = llm.completeJson.bind(llm); + llm.completeJson = async (messages, options) => { + entered(); + await gate; + return original(messages, options); + }; + const { db, service } = fixture.createTestService({ llm }); + const repos = new Repositories(db.db); + const added = addAgentSourceImport(service, { + source: "codex", profileId: "fencing-test", userId: "fencing-user" + }, "retain the real user turn", "delayed-summary"); + const processor = (service as unknown as { embeddingJobs: EmbeddingJobProcessor }).embeddingJobs; + const [old] = repos.runtime.leaseQueuedJobs(1, 60, [added.id], false, ["import_summary"]); + expect(old).toBeDefined(); + const pending = withJobLeaseGuard(old!, () => repos.runtime.isJobLeaseHeld(old!), + () => processor.summarizeImportedTrace(old!)); + const rejected = expect(pending).rejects.toBeInstanceOf(JobLeaseLostError); + await enteredModel; + db.db.prepare("UPDATE evolution_jobs SET leased_until = ? WHERE id = ?") + .run("2000-01-01T00:00:00.000Z", old!.id); + const [replacement] = repos.runtime.leaseQueuedJobs(1, 60, [added.id], false, ["import_summary"]); + expect(replacement!.leaseToken).not.toBe(old!.leaseToken); + const before = repos.memories.get(added.id); + const jobCount = db.db.prepare("SELECT COUNT(*) AS count FROM evolution_jobs").get(); + release(); + await rejected; + expect(repos.memories.get(added.id)).toEqual(before); + expect(repos.runtime.getPendingJob(added.id, "embedding")).toBeUndefined(); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM evolution_jobs").get()).toEqual(jobCount); + expect(repos.runtime.getJob(old!.id)).toMatchObject({ status: "leased", leaseToken: replacement!.leaseToken }); + }); + + it("does not persist a delayed embedding vector or processing state after another lease takes over", async () => { + let release!: () => void; + const gate = new Promise((done) => { release = done; }); + let entered!: () => void; + const enteredModel = new Promise((done) => { entered = done; }); + const embedder = createCapturingEmbedder([]); + const original = embedder.embed.bind(embedder); + embedder.embed = async (texts, role) => { + entered(); + await gate; + return original(texts, role); + }; + const { db, service } = fixture.createTestService({ + llm: createBatchReflectionLlm([], "valid imported summary"), embedder + }); + const repos = new Repositories(db.db); + const added = addAgentSourceImport(service, { + source: "codex", profileId: "fencing-vector-test", userId: "fencing-vector-user" + }, "retain the real user turn and vector state", "delayed-vector"); + const processor = (service as unknown as { embeddingJobs: EmbeddingJobProcessor }).embeddingJobs; + const [summary] = repos.runtime.leaseQueuedJobs(1, 60, [added.id], false, ["import_summary"]); + await processor.summarizeImportedTrace(summary!); + const [old] = repos.runtime.leaseQueuedJobs(1, 60, [added.id], false, ["embedding"]); + expect(old).toBeDefined(); + const pending = withJobLeaseGuard(old!, () => repos.runtime.isJobLeaseHeld(old!), + () => processor.embedMemory(old!)); + const rejected = expect(pending).rejects.toBeInstanceOf(JobLeaseLostError); + await enteredModel; + db.db.prepare("UPDATE evolution_jobs SET leased_until = ? WHERE id = ?") + .run("2000-01-01T00:00:00.000Z", old!.id); + const [replacement] = repos.runtime.leaseQueuedJobs(1, 60, [added.id], false, ["embedding"]); + expect(replacement!.leaseToken).not.toBe(old!.leaseToken); + const before = repos.memories.get(added.id); + const processing = repos.processing.get(added.id); + release(); + await rejected; + expect(repos.memories.get(added.id)).toEqual(before); + expect(repos.processing.get(added.id)).toEqual(processing); + expect(repos.memories.hasVector(added.id, "vec_summary")).toBe(false); + expect(repos.runtime.countEmbeddingRetriesByStatus("pending")).toBe(0); + expect(repos.runtime.getJob(old!.id)).toMatchObject({ status: "leased", leaseToken: replacement!.leaseToken }); + }); +}); diff --git a/Memory/tests/repository/log-retention.test.ts b/Memory/tests/repository/log-retention.test.ts index 7c2b1d7c4..48e9a4ce4 100644 --- a/Memory/tests/repository/log-retention.test.ts +++ b/Memory/tests/repository/log-retention.test.ts @@ -4,10 +4,36 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { MemoryDb } from "../../src/index.js"; import { Repositories } from "../../src/storage/repositories.js"; +import { JobLeaseLostError, withJobLeaseGuard } from "../../src/storage/job-lease-context.js"; const LOG_RETENTION_LIMIT = 10_000; describe("runtime log retention", () => { + it("prunes task-generated logs after the originating task completes while still rejecting stale business writes", async () => { + const root = mkdtempSync(join(tmpdir(), "memmy-log-retention-lease-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + try { + const runtime = new Repositories(db.db).runtime; + seedChangeLogs(db, LOG_RETENTION_LIMIT); + const at = new Date().toISOString(); + runtime.enqueueJob({ id: "log-job", jobType: "reflection", status: "queued", userId: "user", + payload: {}, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + const [claim] = runtime.leaseQueuedJobs(1); + const append = () => runtime.appendChange({ memoryId: "job-memory", userId: "user", + changeType: "test", source: "worker", createdAt: isoAt(LOG_RETENTION_LIMIT) }); + withJobLeaseGuard(claim!, () => runtime.isJobLeaseHeld(claim!), append); + expect(countRows(db, "memory_change_log")).toBe(LOG_RETENTION_LIMIT + 1); + expect(runtime.completeJob(claim!.id, undefined, claim)?.status).toBe("succeeded"); + await waitForRowCount(db, "memory_change_log", LOG_RETENTION_LIMIT); + expect(db.db.prepare("SELECT seq FROM memory_change_log WHERE seq = 1").get()).toBeUndefined(); + expect(() => withJobLeaseGuard(claim!, () => runtime.isJobLeaseHeld(claim!), append)).toThrow(JobLeaseLostError); + expect(countRows(db, "memory_change_log")).toBe(LOG_RETENTION_LIMIT); + } finally { + db.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it("keeps only the latest 10000 rows in log tables and leaves other tables untouched", async () => { const root = mkdtempSync(join(tmpdir(), "mindock-log-retention-")); try { diff --git a/Memory/tests/repository/queue-admission-claim.test.ts b/Memory/tests/repository/queue-admission-claim.test.ts new file mode 100644 index 000000000..608351514 --- /dev/null +++ b/Memory/tests/repository/queue-admission-claim.test.ts @@ -0,0 +1,380 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../src/storage/repositories.js"; +import { claimFairWorkerBatch, createFairWorkerScan, releaseFairWorkerResources, FAIR_SCAN_CANDIDATE_LIMIT, type FairWorkerScan, type FairClaimedWork } from "../../src/service/worker/fair-worker-queue.js"; +import { resolveWorkerSchedulingOptions } from "../../src/service/worker/worker-scheduling-options.js"; + +const at = "2026-10-22T00:00:00.000Z", now = Date.parse(at); +const resources: Array<{ db: MemoryDb; root: string }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "memmy-admission-claim-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ db, root }); + const repos = new Repositories(db.db), runtime = repos.runtime; + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "trace_summary", targetMemoryId = id, dedupeKey?: string) { + return runtime.enqueueJob({ id, jobType, targetMemoryId, dedupeKey, userId: "user", status: "queued", payload: {}, + attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + function retry(id: string) { + return runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, vectorField: "vec_summary", sourceText: id, now }); + } + function claim(limit: number, admit: (work: FairClaimedWork) => { release(): void } | undefined, + algorithm: "wrr" | "legacy" = "wrr", priorityCohortOnly = false, scan?: FairWorkerScan) { + return claimFairWorkerBatch(repos, { at, limit, includeRetries: true, owner: "main", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({ algorithm }, {}), + admit, priorityCohortOnly, scan }); + } + return { db, repos, runtime, job, retry, claim }; +} +afterEach(() => { + vi.restoreAllMocks(); + for (const { db, root } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("atomic resource admission before queue claims", () => { + it.each(["wrr", "legacy"] as const)("resumes %s scans across bounded slices and still reaches the tail", algorithm => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 551; i++) h.job(`blocked-${i}`); + h.job("ready", "user_memory_embedding"); + const seen: string[] = []; + const admit = (work: FairClaimedWork) => { + const id = work.queue === "main" ? work.job.id : work.retry.id; + seen.push(id); + return id === "ready" ? { release() {} } : undefined; + }; + const work: FairClaimedWork[] = []; + do { + work.push(...h.claim(1, admit, algorithm, false, scan)); + expect(scan.inspected).toBeLessThanOrEqual(FAIR_SCAN_CANDIDATE_LIMIT); + } while (scan.more); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["ready"]); + expect(new Set(seen).size).toBe(552); + expect(seen).toHaveLength(552); + work[0]!.admission!.release(); + }); + + it.each(["delete", "ineligible"] as const)("keeps legacy tail reachable when examined rows become %s between slices", change => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 200; i++) h.job(`blocked-${i}`); + h.job("ready", "user_memory_embedding"); + const seen: string[] = []; + const admit = (work: FairClaimedWork) => { + const id = work.queue === "main" ? work.job.id : work.retry.id; + seen.push(id); + return id === "ready" ? { release() {} } : undefined; + }; + expect(h.claim(1, admit, "legacy", false, scan)).toEqual([]); + if (change === "delete") h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='blocked-0'").run(); + else h.db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE id='blocked-0'").run(); + const work: FairClaimedWork[] = []; + do { work.push(...h.claim(1, admit, "legacy", false, scan)); } while (scan.more); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["ready"]); + expect(seen).toHaveLength(201); + work[0]!.admission!.release(); + }); + + it("rechecks new legacy main jobs while continuing a blocked retry scan", () => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 130; i++) h.retry(`blocked-${i}`); + h.retry("tail-retry"); + const admit = (work: FairClaimedWork) => work.queue === "main" || work.retry.id === "tail-retry" ? { release() {} } : undefined; + expect(h.claim(1, admit, "legacy", false, scan)).toEqual([]); + expect(scan.more).toBe(true); + h.job("new-summary"); + const work = h.claim(1, admit, "legacy", false, scan); + expect(work.map(item => item.queue === "main" ? item.job.id : item.retry.id)).toEqual(["new-summary"]); + work[0]!.admission!.release(); + }); + + it("uses refreshed lane metadata when a deduped enqueue moves a buffered candidate", () => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 66; i++) h.job(`job-${i}`, "trace_summary", `job-${i}`, `dedupe-${i}`); + const admit = (work: FairClaimedWork) => work.queue === "main" && Number(work.job.id.split("-")[1]) < 64 + ? undefined : { release() {} }; + expect(h.claim(1, admit, "wrr", false, scan)).toEqual([]); + const previous = h.runtime.getJob("job-64")!; + const changed = h.runtime.enqueueJob({ ...previous, id: "new-id", payload: { source: "memory.processing.manual_retry" } }); + expect(changed.queueLane).toBe("repair_retry"); + const work: FairClaimedWork[] = []; + do { work.push(...h.claim(1, admit, "wrr", false, scan)); } while (!work.length && scan.more); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["job-65"]); + expect(h.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toMatchObject({ cursor: 1 }); + work[0]!.admission!.release(); + }); + + it("rewinds the lane heads when a buffered row moves before an already buffered new-lane head", () => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 66; i++) { + const job = h.job(`job-${i}`, "trace_summary", `job-${i}`, `dedupe-${i}`); + h.runtime.enqueueJob({ ...job, id: `deduped-${i}`, payload: { source: "memory.processing.manual_retry" } }); + } + h.job("later-interactive"); + // This case isolates metadata rewinds in an already spent modern round; + // cursor-only legacy state intentionally gets one upgrade advance instead. + h.runtime.setKv("worker.scheduler.wrr_cursor.v1", { weightsKey: "4/3/2/2/1", cursor: 4, + turns: { version: 1, interactive: { spent: true }, lanes: [ + { lane: "repair_retry", remaining: 3 }, { lane: "import", remaining: 2 }, { lane: "evolution", remaining: 2 }, + { lane: "maintenance", remaining: 1 }, { lane: "interactive", remaining: 4 } + ] } }, at); + const admit = (work: FairClaimedWork) => work.queue === "main" && Number(work.job.id.split("-")[1]) < 64 + ? undefined : { release() {} }; + expect(h.claim(1, admit, "wrr", false, scan)).toEqual([]); + // A first-report promotion can move an explicitly retried task to interactive. + // Ordinary background redelivery must now retain that task's earned urgency. + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,agent_id,created_at,updated_at) + VALUES ('job-64',?,'user','synthetic','L1','memmy-onboarding',?,?)`).run(at, at, at); + const work: FairClaimedWork[] = []; + do { work.push(...h.claim(1, admit, "wrr", false, scan)); } while (!work.length && scan.more); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["job-65"]); + expect(h.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toMatchObject({ cursor: 5 }); + work[0]!.admission!.release(); + }); + + it("reconsiders an already examined old sequence moved into another lane", () => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 64; i++) h.job(`interactive-${i}`, "trace_summary", `interactive-${i}`, `dedupe-${i}`); + for (let i = 0; i < 100; i++) h.job(`repair-${i}`, "decision_repair"); + h.job("maintenance-ready", "episode_idle_close"); + let changed = false; + const admit = (work: FairClaimedWork) => work.queue === "main" && + (work.job.id === "maintenance-ready" || (changed && work.job.id === "interactive-0")) ? { release() {} } : undefined; + expect(h.claim(1, admit, "wrr", false, scan)).toEqual([]); + new Repositories(h.db.db).runtime.enqueueJob({ ...h.runtime.getJob("interactive-0")!, id: "deduped", payload: { source: "memory.processing.manual_retry" } }); + changed = true; + const work: FairClaimedWork[] = []; + do { work.push(...h.claim(1, admit, "wrr", false, scan)); } while (!work.length && scan.more); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["interactive-0"]); + work[0]!.admission!.release(); + }); + + it("keeps deep keyset progress while new enqueues receive their initial policy", () => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 551; i++) h.job(`blocked-${i}`); + h.job("ready", "user_memory_embedding"); + const seen: string[] = [], work: FairClaimedWork[] = []; + const admit = (item: FairClaimedWork) => { + const id = item.queue === "main" ? item.job.id : item.retry.id; + seen.push(id); + return id === "ready" ? { release() {} } : undefined; + }; + for (let slice = 0; slice < 12 && !work.length; slice++) { + work.push(...h.claim(1, admit, "wrr", false, scan)); + h.job(`new-${slice}`); + } + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["ready"]); + expect(seen).toHaveLength(552); + expect(new Set(seen).size).toBe(552); + work[0]!.admission!.release(); + }); + + it("rejects stale buffered rows before requesting permits and still leases the next FIFO row", () => { + const h = harness(), scan = createFairWorkerScan(); + for (let i = 0; i < 100; i++) h.job(`job-${i}`); + expect(h.claim(1, () => undefined, "wrr", false, scan)).toEqual([]); + expect(scan.more).toBe(true); + h.db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE id='job-64'").run(); + const release = vi.fn(); + const work = h.claim(1, () => ({ release }), "wrr", false, scan); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["job-65"]); + expect(release).not.toHaveBeenCalled(); + expect(h.runtime.getJob("job-64")).toMatchObject({ status: "succeeded", attempts: 0 }); + work[0]!.admission!.release(); + expect(release).toHaveBeenCalledTimes(1); + }); + + it("distinguishes a deleted parked ID from its replacement before merging ready and new candidates", () => { + const h = harness(), scan = createFairWorkerScan(); + const first = h.job("reused"); + const input = { at, limit: 1, includeRetries: true, owner: "main", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({ algorithm: "wrr" }, {}), scan }; + claimFairWorkerBatch(h.repos, { ...input, admit: () => ({ blockedOn: "old-provider" }) }); + expect(scan.waiting.size).toBe(1); expect(scan.more).toBe(false); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='reused'").run(); + const replacement = h.job("reused"); + expect(replacement.enqueueSeq).toBeGreaterThan(first.enqueueSeq!); + releaseFairWorkerResources(scan, ["old-provider"]); + const acquired = vi.fn(() => ({ release() {} })); + const work = claimFairWorkerBatch(h.repos, { ...input, admit: acquired }); + expect(work.map(item => item.queue === "main" && item.job.enqueueSeq)).toEqual([replacement.enqueueSeq]); + expect(acquired).toHaveBeenCalledTimes(1); expect(scan.waiting.size).toBe(0); + work[0]!.admission!.release(); + }); + + it("persists unspent model turns while maintenance wraps the compatible legacy cursor", () => { + const h = harness(); + for (let i = 0; i < 4; i++) { h.job(`import-${i}`, "import_summary"); h.job(`evolution-${i}`, "reflection"); } + function finish(work: FairClaimedWork[]) { + for (const item of work) { + if (item.queue === "main") h.runtime.completeJob(item.job.id, at, item.job); + item.admission?.release(); + } + } + for (let i = 0; i < 2; i++) { + const work = h.claim(1, () => ({ release() {} })); + expect(work[0]?.queue === "main" && work[0].job.id).toBe(`import-${i}`); + finish(work); + } + for (let i = 0; i < 10; i++) { + h.job(`maintenance-${i}`, "episode_idle_close"); + const work = h.claim(1, item => item.queue === "main" && item.job.jobType === "episode_idle_close" ? { release() {} } : undefined); + expect(work[0]?.queue === "main" && work[0].job.id).toBe(`maintenance-${i}`); + finish(work); + } + const saved = h.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value; + expect(saved).toMatchObject({ cursor: 0, turns: { version: 1 } }); + const restored = new Repositories(h.db.db); + const work = claimFairWorkerBatch(restored, { at, limit: 1, includeRetries: true, owner: "restored", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({}, {}), admit: () => ({ release() {} }) }); + expect(work[0]?.queue === "main" && work[0].job.id).toBe("evolution-0"); + finish(work); + }); + + it.each([undefined, null, {}, { version: 9, lanes: [] }, { version: 1, lanes: [] }])( + "accepts old cursor state and ignores malformed optional lane turns: %j", turns => { + const h = harness(); h.job("import", "import_summary"); h.job("evolution", "reflection"); + h.runtime.setKv("worker.scheduler.wrr_cursor.v1", { weightsKey: "4/3/2/2/1", cursor: 8, turns }, at); + const work = h.claim(1, () => ({ release() {} })); + expect(work[0]?.queue === "main" && work[0].job.id).toBe("import"); + expect(h.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toMatchObject({ cursor: 9, turns: { version: 1 } }); + work[0]!.admission!.release(); + } + ); + + it("discards turns saved with a different weight configuration", () => { + const h = harness(); h.job("interactive"); h.job("import", "import_summary"); + h.runtime.setKv("worker.scheduler.wrr_cursor.v1", { weightsKey: "1/1/1/1/1", cursor: 8, + turns: { version: 1, lanes: ["import", "evolution", "maintenance", "interactive", "repair_retry"] + .map(lane => ({ lane, remaining: 1 })) } }, at); + const work = h.claim(1, () => ({ release() {} })); + expect(work[0]?.queue === "main" && work[0].job.id).toBe("interactive"); + work[0]!.admission!.release(); + }); + + it("does not claim, count an attempt or advance WRR when all permits are busy", () => { + const h = harness(); const before = h.job("busy"); h.retry("retry-busy"); + const admit = vi.fn((_work: FairClaimedWork) => undefined); + expect(h.claim(2, admit)).toEqual([]); + expect(admit).toHaveBeenCalledTimes(2); + expect(h.runtime.getJob("busy")).toEqual(before); + expect(h.runtime.getEmbeddingRetry("retry-busy")).toMatchObject({ status: "pending", claimedBy: null }); + expect(h.runtime.listKv("worker.scheduler.wrr_cursor")).toEqual([]); + }); + + it("finds a different resource beyond more than five hundred blocked heads in the same lane", () => { + const h = harness(); + for (let i = 0; i < 551; i++) h.job(`blocked-${i}`); + const before = h.job("free-embedding", "user_memory_embedding"); + const release = vi.fn(); + const admit = vi.fn((work: FairClaimedWork) => work.queue === "main" && work.job.jobType === "user_memory_embedding" ? { release } : undefined); + const work = h.claim(1, admit); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["free-embedding"]); + expect(work[0]?.admission).toBeDefined(); + expect(admit).toHaveBeenCalledTimes(552); + expect(h.runtime.getJob("blocked-0")).toMatchObject({ status: "queued", attempts: 0 }); + expect(h.runtime.getJob("free-embedding")?.enqueueSeq).toBe(before.enqueueSeq); + expect(release).not.toHaveBeenCalled(); + work[0]!.admission!.release(); + work[0]!.admission!.release(); + expect(release).toHaveBeenCalledTimes(1); + }); + + it("returns an acquired permit after main CAS loses and leaves the round intact", () => { + const h = harness(); h.job("lost-cas"); + vi.spyOn(h.runtime, "claimQueuedJob").mockReturnValueOnce(undefined); + const release = vi.fn(); + const admit = vi.fn((work: FairClaimedWork) => { + expect(work.queue === "main" && work.job).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + return { release }; + }); + expect(h.claim(1, admit)).toEqual([]); + expect(release).toHaveBeenCalledTimes(1); + expect(h.runtime.getJob("lost-cas")?.status).toBe("queued"); + expect(h.runtime.listKv("worker.scheduler.wrr_cursor")).toEqual([]); + }); + + it("keeps FIFO across both backing tables while paging past busy resources", () => { + const h = harness(); + for (let i = 0; i < 401; i++) { h.job(`repair-${i}`, "decision_repair"); h.retry(`blocked-retry-${i}`); } + h.retry("ready-retry"); + const seen: number[] = []; + const work = h.claim(1, item => { + seen.push((item.queue === "main" ? item.job.enqueueSeq : item.retry.enqueueSeq)!); + return item.queue === "embedding_retry" && item.retry.id === "ready-retry" ? { release() {} } : undefined; + }); + expect(work.map(item => item.queue === "embedding_retry" && item.retry.id)).toEqual(["ready-retry"]); + expect(seen).toHaveLength(803); + expect(seen).toEqual([...seen].sort((a, b) => a - b)); + }); + + it("returns a retry permit after retry CAS loses", () => { + const h = harness(); h.retry("lost-cas"); + vi.spyOn(h.runtime, "claimEmbeddingRetryById").mockReturnValueOnce(undefined); + const release = vi.fn(); + expect(h.claim(1, () => ({ release }))).toEqual([]); + expect(release).toHaveBeenCalledTimes(1); + expect(h.runtime.getEmbeddingRetry("lost-cas")?.status).toBe("pending"); + }); + + it("returns every acquired permit and rolls back every claim on a cursor write failure", () => { + const h = harness(); h.job("one"); h.job("two"); + const releases = [vi.fn(), vi.fn()]; let index = 0; + vi.spyOn(h.runtime, "setKv").mockImplementation(() => { throw new Error("cursor disk failure"); }); + expect(() => h.claim(2, () => ({ release: releases[index++]! }))).toThrow("cursor disk failure"); + expect(releases.map(release => release.mock.calls.length)).toEqual([1, 1]); + for (const id of ["one", "two"]) expect(h.runtime.getJob(id)).toMatchObject({ status: "queued", attempts: 0, leaseCount: 0 }); + }); + + it("rolls back earlier permits and claims if admission itself throws", () => { + const h = harness(); h.job("one"); h.job("two"); + const release = vi.fn(); let calls = 0; + expect(() => h.claim(2, () => { + if (calls++ > 0) throw new Error("admission failure"); + return { release }; + })).toThrow("admission failure"); + expect(release).toHaveBeenCalledTimes(1); + expect(h.runtime.getJob("one")).toMatchObject({ status: "queued", attempts: 0 }); + }); + + it("uses legacy priority order rather than WRR when admission is enabled in rollback mode", () => { + const h = harness(); + for (let i = 0; i < 6; i++) { h.job(`summary-${i}`); h.job(`repair-${i}`, "decision_repair"); } + const work = h.claim(5, () => ({ release() {} }), "legacy"); + expect(work.map(item => item.queue === "main" && item.job.jobType)).toEqual(Array(5).fill("trace_summary")); + expect(h.runtime.listKv("worker.scheduler.wrr_cursor")).toEqual([]); + }); + + it("scans legacy pages past blocked resources and keeps unclaimed retry surplus available", () => { + const h = harness(); + for (let i = 0; i < 551; i++) h.job(`blocked-${i}`); + h.job("free-main", "user_memory_embedding"); h.retry("free-retry"); h.retry("surplus"); + const release = vi.fn(); + const work = h.claim(2, item => item.queue === "embedding_retry" || item.job.id === "free-main" ? { release } : undefined, "legacy"); + expect(work.map(item => item.queue === "main" ? item.job.id : item.retry.id)).toEqual(["free-main", "free-retry"]); + expect(h.runtime.getEmbeddingRetry("surplus")?.status).toBe("pending"); + expect(h.runtime.getJob("blocked-550")?.status).toBe("queued"); + }); + + it("does not skip a later legacy page when earlier admitted rows leave the eligible result set", () => { + const h = harness(); + for (let i = 0; i < 199; i++) h.job(`blocked-before-${i}`); + h.job("free-one"); + for (let i = 0; i < 201; i++) h.job(`blocked-after-${i}`); + h.job("free-two"); + const work = h.claim(2, item => item.queue === "main" && item.job.id.startsWith("free-") ? { release() {} } : undefined, "legacy"); + expect(work.map(item => item.queue === "main" && item.job.id)).toEqual(["free-one", "free-two"]); + }); + + it("sets the legacy cohort from its first admitted main job and then uses spare capacity for retry", () => { + const h = harness(); h.job("blocked-summary"); h.job("free-import", "import_summary"); h.job("other-priority", "reflection"); h.retry("retry"); + const work = h.claim(3, item => item.queue === "main" && item.job.id === "blocked-summary" ? undefined : { release() {} }, "legacy", true); + expect(work.map(item => item.queue === "main" ? item.job.id : item.retry.id)).toEqual(["free-import", "retry"]); + expect(h.runtime.getJob("other-priority")?.status).toBe("queued"); + expect(h.runtime.getJob("blocked-summary")?.status).toBe("queued"); + }); +}); diff --git a/Memory/tests/repository/queue-scheduling-maintenance.test.ts b/Memory/tests/repository/queue-scheduling-maintenance.test.ts new file mode 100644 index 000000000..2ff73e818 --- /dev/null +++ b/Memory/tests/repository/queue-scheduling-maintenance.test.ts @@ -0,0 +1,201 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { JOB_POLICY_VERSION, jobPolicyRegistry } from "../../src/service/worker/job-policy-registry.js"; +import { pendingPolicyIndex } from "../../src/storage/queue-scheduling-schema.js"; + +const at = "2026-10-25T00:00:00.000Z", now = Date.parse(at); +const resources: Array<{ db: MemoryDb; root: string }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "memmy-policy-maintenance-")); + const h = { db: new MemoryDb({ path: join(root, "memory.sqlite") }), root }; + resources.push(h); + const runtime = () => new Repositories(h.db.db).runtime; + return { + get db() { return h.db.db; }, runtime, + reopen() { const path = h.db.path; h.db.close(); h.db = new MemoryDb({ path }); }, + memory(id = "target", tags: string[] = []) { + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,created_at,updated_at) + VALUES (?,?,'user','synthetic','L1',?,?,?)`).run(id, at, JSON.stringify(tags), at, at); + }, + job(id: string, source = "startup.placeholder_summary_repair", dedupeKey = id) { + return runtime().enqueueJob({ id, jobType: "import_summary", targetMemoryId: "target", userId: "user", + status: "queued", payload: { source }, dedupeKey, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + }, + retry(id = "retry", target = "target") { + return runtime().enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: target, + vectorField: "vec_summary", sourceText: "synthetic", now }); + } + }; +} +afterEach(() => { + vi.restoreAllMocks(); + for (const h of resources.splice(0)) { if (h.db.db.open) h.db.close(); rmSync(h.root, { recursive: true, force: true }); } +}); + +describe("event-driven policy maintenance", () => { + it("does no reclassification or scan invalidation on identical delivery or ordinary memory updates", () => { + const h = harness(); h.memory(); const main = h.job("original"); const retry = h.retry(); + const store = h.runtime().scheduling, revision = store.getSelectionRevision(); + const resolve = vi.spyOn(jobPolicyRegistry, "resolve"); + h.job("duplicate", "startup.placeholder_summary_repair", "original"); h.retry(); + h.db.prepare("UPDATE memories SET properties_json=?, tags_json=tags_json, agent_id=agent_id, memory_layer=memory_layer WHERE id='target'") + .run(JSON.stringify({ internal_info: { summary: "changed", processing_state: "ready" }, rating: 5 })); + expect(store.refreshActivePolicies({ maxRows: 64 })).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + expect(resolve).not.toHaveBeenCalled(); + expect(store.getSelectionRevision()).toBe(revision); + expect(h.runtime().getJob(main.id)).toMatchObject({ enqueueSeq: main.enqueueSeq, firstQueuedAt: main.firstQueuedAt }); + expect(h.runtime().getEmbeddingRetry(retry.id)).toMatchObject({ enqueueSeq: retry.enqueueSeq, firstQueuedAt: retry.firstQueuedAt }); + }); + + it.each(["capture_origin", "plugin_algorithm", "project_id", "projectId"])("still refreshes policy/admission when %s changes", field => { + const h = harness(); h.memory(); h.job("main"); h.retry(); + h.db.prepare("UPDATE memories SET properties_json=? WHERE id='target'") + .run(JSON.stringify({ internal_info: { [field]: "changed" } })); + const store = h.runtime().scheduling, revision = store.getSelectionRevision(); + expect(store.refreshActivePolicies({ maxRows: 64 }).refreshed).toBe(2); + expect(store.getSelectionRevision()).toBe(revision + 2); + }); + + it("uses only the pending indexes when both large queues already have valid classification", () => { + const h = harness(); + h.db.transaction(() => { for (let i = 0; i < 1_000; i++) { h.job(`main-${i}`); h.retry(`retry-${i}`, `target-${i}`); } })(); + const prepare = h.db.prepare.bind(h.db), plans: string[] = []; + const spy = vi.spyOn(h.db, "prepare").mockImplementation(sql => { + if (sql.includes("previous_queue_lane")) { + const rows = prepare(`EXPLAIN QUERY PLAN ${sql}`).all(64) as Array<{ detail: string }>; + plans.push(...rows.map(row => row.detail)); + } + return prepare(sql); + }); + const store = h.runtime().scheduling; + for (let i = 0; i < 10; i++) expect(store.refreshActivePolicies({ maxRows: 64 }).refreshed).toBe(0); + spy.mockRestore(); + expect(plans.filter(plan => plan.startsWith("SCAN q"))).toHaveLength(20); + expect(plans.filter(plan => plan.startsWith("SCAN q")).every(plan => plan.includes("policy_pending_v") && plan.includes("_active"))).toBe(true); + expect(plans.some(plan => /TEMP B-TREE|LIST SUBQUERY/.test(plan))).toBe(false); + }); + + it("backfills delayed and blocked old rows in bounded slices, resumes after reopening, and skips terminal history", () => { + const h = harness(); + h.db.transaction(() => { + for (let i = 0; i < 80; i++) { h.job(`main-${i}`); h.retry(`retry-${i}`, `target-${i}`); } + h.db.exec(`UPDATE evolution_jobs SET policy_version=1, available_at='2100-01-01T00:00:00.000Z', blocked_reason='configuration'; + UPDATE embedding_retry_queue SET policy_version=1, next_attempt_at=4102444800000; + UPDATE evolution_jobs SET status='succeeded' WHERE id='main-79'; + UPDATE embedding_retry_queue SET status='failed' WHERE id='retry-79'`); + })(); + const identity = h.runtime().getJob("main-0")!; + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 0 })).toMatchObject({ refreshed: 0, hasMore: true }); + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 })).toMatchObject({ refreshed: 64, hasMore: true }); + h.reopen(); + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 })).toMatchObject({ refreshed: 64, hasMore: true }); + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 })).toMatchObject({ refreshed: 30, hasMore: false }); + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 })).toMatchObject({ refreshed: 0, hasMore: false }); + expect(h.runtime().getJob("main-0")).toMatchObject({ enqueueSeq: identity.enqueueSeq, firstQueuedAt: identity.firstQueuedAt, + status: "queued", attempts: 0, leaseToken: null, availableAt: "2100-01-01T00:00:00.000Z", blockedReason: "configuration" }); + expect(h.runtime().getJob("main-79")!.policyVersion).toBe(1); + expect(h.runtime().getEmbeddingRetry("retry-79")!.policyVersion).toBe(1); + expect(h.runtime().getEmbeddingRetry("retry-0")).toMatchObject({ status: "pending", attempts: 0, nextAttemptAt: 4102444800000 }); + }); + + it("upgrades installed v2 metadata and broad triggers once without changing FIFO identity", () => { + const h = harness(); h.memory(); const original = h.job("main"); h.retry(); + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + h.db.exec(`DROP TRIGGER ${table}_policy_dirty; + DROP INDEX ${pendingPolicyIndex(table)}; DROP INDEX ${pendingPolicyIndex(table, true)}; + ALTER TABLE ${table} DROP COLUMN policy_promotion; + UPDATE ${table} SET policy_version=2; + CREATE INDEX ${table}_policy_pending_v2 ON ${table}(enqueue_seq) WHERE policy_version IS NULL OR policy_version != 2`); + const column = table === "evolution_jobs" ? "payload_json" : "source_text"; + h.db.exec(`CREATE TRIGGER ${table}_policy_dirty AFTER UPDATE OF ${column} ON ${table} BEGIN + UPDATE ${table} SET policy_version=NULL WHERE id=NEW.id; END`); + } + h.db.exec(`DROP TRIGGER memories_worker_policy_dirty; + CREATE TRIGGER memories_worker_policy_dirty AFTER UPDATE OF properties_json ON memories BEGIN + UPDATE evolution_jobs SET policy_version=NULL WHERE target_memory_id=NEW.id; + UPDATE embedding_retry_queue SET policy_version=NULL WHERE target_id=NEW.id; END`); + h.reopen(); + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 1 })).toMatchObject({ refreshed: 1, hasMore: true }); + h.reopen(); + expect(h.runtime().scheduling.refreshActivePolicies({ maxRows: 1 })).toMatchObject({ refreshed: 1, hasMore: false }); + expect(h.runtime().getJob("main")).toMatchObject({ policyVersion: JOB_POLICY_VERSION, enqueueSeq: original.enqueueSeq, firstQueuedAt: original.firstQueuedAt }); + expect(h.db.prepare("SELECT name FROM sqlite_master WHERE name LIKE '%policy_pending_v2%'").all()).toEqual([]); + h.db.exec("UPDATE memories SET properties_json='{\"summary\":\"ordinary update\"}' WHERE id='target'"); + expect(h.runtime().scheduling.pendingPolicyCount()).toBe(0); + }); + + it("keeps legacy raw writes discoverable, handles an empty retry gate, and resumes a terminal row as new work", () => { + const h = harness(); h.job("main"); h.retry(); + h.db.exec(`UPDATE evolution_jobs SET payload_json='{"source":"memory.processing.manual_retry"}'; + UPDATE embedding_retry_queue SET source_text='changed'`); + const store = h.runtime().scheduling; + expect(store.refreshActivePolicies({ maxRows: 64, includeRetries: false })).toMatchObject({ refreshed: 1, hasMore: false }); + expect(h.runtime().getEmbeddingRetry("retry")!.policyVersion).toBeNull(); + expect(store.refreshActivePolicies({ maxRows: 1 })).toMatchObject({ refreshed: 1, hasMore: false }); + h.db.exec(`UPDATE evolution_jobs SET payload_json='{"source":"startup.placeholder_summary_repair"}';`); + store.refreshActivePolicies({ maxRows: 64 }); + expect(h.runtime().getJob("main")).toMatchObject({ queueLane: "repair_retry", basePriority: 0 }); + h.db.exec("UPDATE evolution_jobs SET status='succeeded'; UPDATE evolution_jobs SET status='queued'"); + store.refreshActivePolicies({ maxRows: 64 }); + expect(h.runtime().getJob("main")).toMatchObject({ queueLane: "maintenance", basePriority: 90 }); + }); +}); + +describe("unfinished-task promotion lifetime", () => { + it.each(["queued", "leased", "failed"])("preserves explicit retry through background reuse and restart while %s", status => { + const h = harness(); h.memory(); const original = h.job("original"); + h.job("manual", "memory.processing.manual_retry", "original"); + h.db.prepare("UPDATE evolution_jobs SET status=?, lease_token='identity', leased_until=? WHERE id='original'") + .run(status, "2100-01-01T00:00:00.000Z"); + h.job("background", "startup.placeholder_summary_repair", "original"); + h.reopen(); + h.db.prepare("UPDATE evolution_jobs SET policy_version=? WHERE id='original'").run(JOB_POLICY_VERSION - 1); + h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 }); + expect(h.runtime().getJob(original.id)).toMatchObject({ queueLane: "repair_retry", basePriority: 0, + status, leaseToken: "identity", enqueueSeq: original.enqueueSeq, firstQueuedAt: original.firstQueuedAt }); + }); + + it.each(["succeeded", "dead_letter"])("does not pass urgency to a new main task after %s", status => { + const h = harness(); h.memory(); h.job("original", "memory.processing.manual_retry", "same"); + h.db.prepare("UPDATE evolution_jobs SET status=? WHERE id='original'").run(status); + const next = h.job("next", "startup.placeholder_summary_repair", "same"); + expect(next).toMatchObject({ id: "next", queueLane: "maintenance", basePriority: 90 }); + expect(next.enqueueSeq).toBeGreaterThan(h.runtime().getJob("original")!.enqueueSeq!); + }); + + it("preserves first-report urgency over a later manual retry or background reuse in both queues", () => { + const h = harness(); h.memory(); const original = h.job("main"); const retry = h.retry(); + h.job("manual", "memory.processing.manual_retry", "main"); + h.db.exec(`UPDATE memories SET tags_json='["onboarding-report"]' WHERE id='target'`); + h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 }); + h.db.exec("UPDATE memories SET tags_json='[]' WHERE id='target'"); + h.job("another-manual", "memory.processing.manual_retry", "main"); + h.job("background", "startup.placeholder_summary_repair", "main"); h.retry(); + h.reopen(); + for (const task of [h.runtime().getJob("main"), h.runtime().getEmbeddingRetry("retry")]) { + expect(task).toMatchObject({ queueLane: "interactive", basePriority: -100 }); + } + expect(h.runtime().getJob("main")).toMatchObject({ enqueueSeq: original.enqueueSeq, firstQueuedAt: original.firstQueuedAt }); + expect(h.runtime().getEmbeddingRetry("retry")).toMatchObject({ enqueueSeq: retry.enqueueSeq, firstQueuedAt: retry.firstQueuedAt }); + }); + + it.each(["failed", "succeeded"])("clears first-report promotion on a new retry work cycle after %s", status => { + const h = harness(); h.memory("target", ["onboarding-report"]); h.retry(); + h.db.exec("UPDATE memories SET tags_json='[]' WHERE id='target'"); + h.runtime().scheduling.refreshActivePolicies({ maxRows: 64 }); + h.db.prepare("UPDATE embedding_retry_queue SET status=? WHERE id='retry'").run(status); + h.retry(); + expect(h.runtime().getEmbeddingRetry("retry")).toMatchObject({ status: "pending", queueLane: "repair_retry", basePriority: 0 }); + }); + + it("captures a raw producer's explicit retry before background dedupe overwrites its source", () => { + const h = harness(); h.job("main"); + h.db.exec(`UPDATE evolution_jobs SET payload_json='{"source":"memory.processing.manual_retry"}', policy_version=NULL WHERE id='main'`); + h.job("background", "startup.placeholder_summary_repair", "main"); + expect(h.runtime().getJob("main")).toMatchObject({ queueLane: "repair_retry", basePriority: 0 }); + }); +}); diff --git a/Memory/tests/repository/queue-scheduling-store.test.ts b/Memory/tests/repository/queue-scheduling-store.test.ts new file mode 100644 index 000000000..91bb03859 --- /dev/null +++ b/Memory/tests/repository/queue-scheduling-store.test.ts @@ -0,0 +1,631 @@ +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { JOB_POLICY_VERSION, jobPolicyRegistry } from "../../src/service/worker/job-policy-registry.js"; +import { pendingPolicyIndex } from "../../src/storage/queue-scheduling-schema.js"; +import { SCHEMA_VERSION } from "../../src/storage/schema.js"; +import type { JobType } from "../../src/types.js"; + +const at = "2026-10-22T00:00:00.000Z"; +const now = Date.parse(at); +const resources: Array<{ db: MemoryDb; root: string }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "memmy-queue-metadata-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ db, root }); + const repos = new Repositories(db.db); + function memory(id: string, tags: string[] = []) { + db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,created_at,updated_at) + VALUES (?,?,'user','synthetic','L1',?,?,?)`).run(id, at, JSON.stringify(tags), at, at); + } + function job(id: string, type: JobType = "reflection", target?: string, payload: Record = {}, dedupeKey?: string) { + return repos.runtime.enqueueJob({ id, jobType: type, status: "queued", userId: "user", targetMemoryId: target, + dedupeKey, payload, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + function rawJob(id: string, type = "reflection", options: { at?: string; payload?: Record; target?: string; scopeKey?: string; scopeSeq?: number } = {}) { + db.db.prepare(`INSERT INTO evolution_jobs(id,job_type,status,user_id,target_memory_id,payload_json, + scope_key,scope_seq,created_at,updated_at) VALUES (?,?,'queued','user',?,?,?,?,?,?)`) + .run(id, type, options.target ?? null, JSON.stringify(options.payload ?? {}), options.scopeKey ?? null, + options.scopeSeq ?? null, options.at ?? at, options.at ?? at); + } + function retry(id: string, target = id, stamp = now) { + return repos.runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: target, + vectorField: "vec_summary", sourceText: "synthetic", now: stamp }); + } + function ids(options = {}) { + return repos.runtime.scheduling.candidates({ at, limit: 50, ...options }).map(c => c.id).sort(); + } + return { root, db, repos, runtime: repos.runtime, store: repos.runtime.scheduling, memory, job, rawJob, retry, ids }; +} +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + for (const { db, root } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +function removeSchedulingSchema(db: MemoryDb): void { + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + for (const suffix of ["enqueue_sequence", "policy_dirty"]) db.db.exec(`DROP TRIGGER ${table}_${suffix}`); + db.db.exec(`DROP INDEX ${pendingPolicyIndex(table)}`); + db.db.exec(`DROP INDEX ${pendingPolicyIndex(table, true)}`); + for (const suffix of ["enqueue_seq", "lane_due", "lane_fifo", "policy_dirty"]) db.db.exec(`DROP INDEX ${table}_${suffix}`); + } + for (const suffix of ["dirty", "insert", "delete"]) db.db.exec(`DROP TRIGGER memories_worker_policy_${suffix}`); + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + for (const column of ["queue_lane", "resource_class", "enqueue_seq", "first_queued_at", "base_priority", "policy_version", "policy_promotion"]) { + db.db.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); + } + } + db.db.exec("DROP TABLE worker_enqueue_counter"); +} + +describe("queue scheduling metadata and read-only candidates", () => { + it("retains a manual promotion when an unfinished task is redelivered by startup repair", () => { + const h = harness(); + const first = h.job("first", "import_summary", undefined, { source: "startup.placeholder_summary_repair" }, "same"); + h.job("manual", "import_summary", undefined, { source: "memory.processing.manual_retry" }, "same"); + const reused = h.job("background", "import_summary", undefined, { source: "startup.placeholder_summary_repair" }, "same"); + expect(reused).toMatchObject({ id: first.id, queueLane: "repair_retry", basePriority: 0, + enqueueSeq: first.enqueueSeq, firstQueuedAt: first.firstQueuedAt, + payload: { source: "startup.placeholder_summary_repair" } }); + }); + + it("does not dirty policies for changes to memory content unrelated to scheduling", () => { + const h = harness(); h.memory("target"); + h.job("summary", "trace_summary", "target"); h.retry("retry", "target"); + h.db.db.prepare("UPDATE memories SET properties_json=? WHERE id='target'") + .run(JSON.stringify({ internal_info: { processing_state: "ready" }, arbitrary: "updated" })); + expect(h.store.pendingPolicyCount()).toBe(0); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 64 }).refreshed).toBe(0); + }); + + it("migrates v11 with chronological order across both queues and remains stable after reopening", () => { + const h = harness(); + h.rawJob("late-main", "reflection", { at: "2026-10-22T00:00:03.000Z" }); + h.rawJob("first-main", "reflection", { at: "2026-10-22T00:00:01.000Z" }); + h.retry("middle-retry", "target", now + 2_000); + removeSchedulingSchema(h.db); + h.db.db.exec("DELETE FROM schema_migrations"); + h.db.db.prepare("INSERT INTO schema_migrations VALUES ('011_embedding_retry_lease_token',11,?,'legacy')").run(at); + const path = h.db.path; + h.db.close(); + const reopened = new MemoryDb({ path }); + resources.find(r => r.root === h.root)!.db = reopened; + const runtime = new Repositories(reopened.db).runtime; + expect(reopened.schemaVersion().version).toBe(SCHEMA_VERSION); + expect(existsSync(`${path}.pre-v${SCHEMA_VERSION}.bak`)).toBe(true); + expect(runtime.getJob("first-main")).toMatchObject({ enqueueSeq: 1, firstQueuedAt: "2026-10-22T00:00:01.000Z", status: "queued" }); + expect(runtime.getEmbeddingRetry("middle-retry")).toMatchObject({ enqueueSeq: 2, firstQueuedAt: "2026-10-22T00:00:02.000Z", status: "pending" }); + expect(runtime.getJob("late-main")).toMatchObject({ enqueueSeq: 3 }); + const counts = runtime.scheduling.refreshPolicies(); + expect(counts).toEqual({ refreshed: 3, unknown: [] }); + reopened.close(); + const again = new MemoryDb({ path }); + resources.find(r => r.root === h.root)!.db = again; + expect(new Repositories(again.db).runtime.getJob("first-main")!.enqueueSeq).toBe(1); + new Repositories(again.db).runtime.enqueueEmbeddingRetry({ id: "after-reopen", targetKind: "trace", targetId: "after", vectorField: "vec_summary", sourceText: "later", now }); + expect(new Repositories(again.db).runtime.getEmbeddingRetry("after-reopen")!.enqueueSeq).toBe(4); + }); + + it("assigns one global monotonic order to raw SQL and repository inserts without reusing a deleted maximum", () => { + const h = harness(); + const first = h.job("main-one"); + const second = h.retry("retry-two"); + h.rawJob("raw-three"); + const third = h.runtime.getJob("raw-three")!; + expect([first.enqueueSeq, second.enqueueSeq, third.enqueueSeq]).toEqual([1, 2, 3]); + expect(third).toMatchObject({ firstQueuedAt: expect.any(String), policyVersion: null }); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id = ?").run(third.id); + h.db.db.prepare(`INSERT INTO embedding_retry_queue(id,target_kind,target_id,vector_field,source_text,status,next_attempt_at,created_at,updated_at) + VALUES ('raw-four','trace','raw-target','vec_summary','raw','pending',?,?,?)`).run(now, now, now); + expect(h.runtime.getEmbeddingRetry("raw-four")).toMatchObject({ enqueueSeq: 4, firstQueuedAt: expect.any(String), policyVersion: null }); + expect(h.store.refreshPolicies().refreshed).toBe(1); + expect(h.runtime.getJob(first.id)).toMatchObject({ enqueueSeq: 1, queueLane: "evolution" }); + expect(h.runtime.getEmbeddingRetry("raw-four")).toMatchObject({ enqueueSeq: 4, queueLane: "repair_retry" }); + }); + + it("starts new waiting age at actual insertion even when a producer backdates creation to an old conversation", () => { + const h = harness(); + const before = Date.now(); + const old = "2001-01-01T00:00:00.000Z"; + h.rawJob("backdated-main", "import_summary", { at: old }); + h.retry("backdated-retry", "old-target", Date.parse(old)); + const after = Date.now(); + const main = h.runtime.getJob("backdated-main")!; + const retry = h.runtime.getEmbeddingRetry("backdated-retry")!; + expect(main.createdAt).toBe(old); + expect(retry.createdAt).toBe(Date.parse(old)); + for (const record of [main, retry]) { + expect(Date.parse(record.firstQueuedAt!)).toBeGreaterThanOrEqual(before); + expect(Date.parse(record.firstQueuedAt!)).toBeLessThanOrEqual(after); + } + }); + + it("returns no candidates for a zero-sized preview", () => { + const h = harness(); h.job("main"); h.retry("retry"); + expect(h.store.candidates({ at, limit: 0 })).toEqual([]); + expect(h.runtime.getJob("main")!.status).toBe("queued"); + expect(h.runtime.getEmbeddingRetry("retry")!.status).toBe("pending"); + }); + + it("keeps real enqueues successful and policy metadata dirty when optional projection fails", () => { + const h = harness(); + const resolve = vi.spyOn(jobPolicyRegistry, "resolve").mockImplementation(() => { throw new Error("projection-only failure"); }); + expect(h.job("main")).toMatchObject({ id: "main", status: "queued", policyVersion: null }); + expect(h.retry("retry")).toMatchObject({ id: "retry", status: "pending", policyVersion: null }); + expect(h.store.pendingPolicyCount()).toBe(2); + expect(h.ids()).toEqual([]); + resolve.mockRestore(); + expect(h.store.refreshPolicies()).toEqual({ refreshed: 2, unknown: [] }); + expect(h.ids()).toEqual(["main", "retry"]); + }); + + it("preserves FIFO identity through dedupe, temporary failure, configuration recovery and retry input updates", () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(now); + const h = harness(); + h.job("original", "trace_summary", undefined, {}, "dedupe"); + const first = h.runtime.getJob("original")!; + expect(h.job("duplicate", "trace_summary", undefined, { source: "memory.processing.manual_retry" }, "dedupe").id).toBe(first.id); + const [claim] = h.runtime.leaseQueuedJobs(1); + h.runtime.failJob(first.id, "configuration", at, false, claim, { blockedReason: "configuration", countFailure: false }); + h.runtime.resumeConfigurationBlockedJobs(at); + const [secondClaim] = h.runtime.leaseQueuedJobs(1); + h.runtime.failJob(first.id, "temporary", at, false, secondClaim, { availableAt: new Date(now + 5_000).toISOString() }); + expect(h.runtime.getJob(first.id)).toMatchObject({ enqueueSeq: first.enqueueSeq, firstQueuedAt: first.firstQueuedAt }); + const retry = h.retry("retry"); + h.runtime.claimDueEmbeddingRetries({ now, workerId: "worker", leaseUntil: now + 1_000 }); + h.runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "retry", vectorField: "vec_summary", sourceText: "changed", now: now + 1 }); + h.runtime.claimDueEmbeddingRetries({ now: now + 1, workerId: "worker", leaseUntil: now + 2_000 }); + h.runtime.requeueEmbeddingRetriesAfterRestart(now + 2_001); + expect(h.runtime.getEmbeddingRetry(retry.id)).toMatchObject({ enqueueSeq: retry.enqueueSeq, firstQueuedAt: retry.firstQueuedAt, status: "pending" }); + }); + + it("refreshes policy changes and target Onboarding tags without changing the sequence", () => { + const h = harness(); h.memory("target", ["agent-source"]); + const original = h.job("summary", "import_summary", "target"); + h.retry("retry", "target"); + expect(h.runtime.getJob(original.id)).toMatchObject({ queueLane: "import", policyVersion: JOB_POLICY_VERSION }); + h.db.db.prepare("UPDATE memories SET tags_json = ? WHERE id = 'target'").run('["agent-source","onboarding-report"]'); + expect(h.runtime.getJob(original.id)!.policyVersion).toBeNull(); + expect(h.runtime.getEmbeddingRetry("retry")!.policyVersion).toBeNull(); + expect(h.store.refreshPolicies().refreshed).toBe(2); + expect(h.runtime.getJob(original.id)).toMatchObject({ queueLane: "interactive", basePriority: -100, enqueueSeq: original.enqueueSeq }); + expect(h.runtime.getEmbeddingRetry("retry")).toMatchObject({ queueLane: "interactive", basePriority: -100 }); + h.db.db.prepare("UPDATE memories SET tags_json='[]' WHERE id='target'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json = ? WHERE id='summary'").run('{"source":"startup.placeholder_summary_repair"}'); + h.store.refreshPolicies(); + expect(h.runtime.getJob(original.id)).toMatchObject({ queueLane: "interactive", basePriority: -100, enqueueSeq: original.enqueueSeq }); + h.db.db.prepare("UPDATE evolution_jobs SET policy_version = ? WHERE id='summary'").run(JOB_POLICY_VERSION - 1); + expect(h.store.pendingPolicyCount()).toBe(1); + expect(h.store.refreshPolicies(1).refreshed).toBe(1); + }); + + it("projects native historical capture metadata on enqueue for both summary and subsequent embedding", () => { + const h = harness(); h.memory("native-history", ["database", "test"]); h.memory("live-hook", ["database", "test"]); + h.db.db.prepare("UPDATE memories SET agent_id='claude_code',properties_json=? WHERE id='native-history'") + .run(JSON.stringify({ internal_info: { plugin_algorithm: "capture.v7", capture_origin: "agent_source_scan" } })); + h.db.db.prepare("UPDATE memories SET agent_id='claude_code',properties_json=? WHERE id='live-hook'") + .run(JSON.stringify({ internal_info: { plugin_algorithm: "capture.v7" } })); + const summary = h.job("native-summary", "trace_summary", "native-history", { source: "turn.complete.capture", decideCapture: true }); + const embedding = h.job("native-embedding", "embedding", "native-history", { reason: "trace.summary.updated", sourceJobId: summary.id }); + expect(summary).toMatchObject({ jobType: "trace_summary", queueLane: "import", resourceClass: "summary", payload: { source: "turn.complete.capture", decideCapture: true } }); + expect(embedding).toMatchObject({ queueLane: "import", resourceClass: "embedding", payload: { reason: "trace.summary.updated" } }); + expect(h.retry("history-retry", "native-history").queueLane).toBe("repair_retry"); + expect(h.job("live-summary", "trace_summary", "live-hook").queueLane).toBe("interactive"); + expect(h.job("live-embedding", "embedding", "live-hook").queueLane).toBe("interactive"); + const memory = h.repos.memories.get("native-history")!; + expect(memory.tags).toEqual(["database", "test"]); + expect(memory.properties.internal_info.plugin_algorithm).toBe("capture.v7"); + }); + + it("refreshes an older policy version with explicit native history metadata while preserving FIFO and lease state", () => { + const h = harness(); h.memory("native-history"); h.memory("unmarked-native"); + h.db.db.prepare("UPDATE memories SET properties_json=? WHERE id='native-history'") + .run(JSON.stringify({ internal_info: { plugin_algorithm: "capture.v7", capture_origin: "agent_source_scan" } })); + const summary = h.job("history-summary", "trace_summary", "native-history"); + const unmarked = h.job("unmarked-summary", "trace_summary", "unmarked-native"); + h.db.db.prepare("UPDATE evolution_jobs SET policy_version=?,queue_lane='interactive' WHERE id IN (?,?)") + .run(JOB_POLICY_VERSION - 1, summary.id, unmarked.id); + expect(JOB_POLICY_VERSION).toBeGreaterThan(1); + expect(h.store.refreshPolicies()).toEqual({ refreshed: 2, unknown: [] }); + expect(h.runtime.getJob(summary.id)).toMatchObject({ queueLane: "import", policyVersion: JOB_POLICY_VERSION, + enqueueSeq: summary.enqueueSeq, firstQueuedAt: summary.firstQueuedAt, status: "queued", attempts: 0, leaseCount: 0, leaseToken: null }); + expect(h.runtime.getJob(unmarked.id)).toMatchObject({ queueLane: "interactive", enqueueSeq: unmarked.enqueueSeq }); + h.db.db.prepare("UPDATE memories SET properties_json=? WHERE id='native-history'") + .run(JSON.stringify({ internal_info: { plugin_algorithm: "capture.v7" } })); + expect(h.runtime.getJob(summary.id)!.policyVersion).toBeNull(); + h.store.refreshPolicies(); + expect(h.runtime.getJob(summary.id)).toMatchObject({ queueLane: "interactive", enqueueSeq: summary.enqueueSeq }); + }); + + it("marks unknown raw job types as explicitly unregistered and excludes them from lane previews", () => { + const h = harness(); h.rawJob("unknown", "future_job_type"); + expect(h.store.refreshPolicies()).toEqual({ refreshed: 1, unknown: [{ id: "unknown", jobType: "future_job_type" }] }); + expect(h.runtime.getJob("unknown")).toMatchObject({ queueLane: "unregistered", resourceClass: "unregistered", policyVersion: JOB_POLICY_VERSION, status: "queued" }); + expect(h.ids()).toEqual([]); + expect(h.store.pendingPolicyCount()).toBe(0); + }); + + it("uses the same main eligibility as legacy preview for future availability, config waits, lease and task filters", () => { + const h = harness(); + for (const id of ["ready", "future-available", "future-run", "configuration", "leased-valid", "leased-expired", "exhausted"]) h.rawJob(id, "reflection", { target: "target" }); + h.rawJob("future-run-2", "reflection", { payload: { runAfter: new Date(now + 60_000).toISOString() }, target: "target" }); + h.rawJob("different-type", "episode_title", { target: "other" }); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='future-available'").run(new Date(now + 60_000).toISOString()); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='future-run'").run(JSON.stringify({ runAfter: new Date(now + 1).toISOString() })); + h.db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='configuration'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='leased-valid'").run(new Date(now + 60_000).toISOString()); + h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='leased-expired'").run(at); + h.db.db.prepare("UPDATE evolution_jobs SET failure_count=max_attempts WHERE id='exhausted'").run(); + h.store.refreshPolicies(); + const filters = { allowedJobTypes: ["reflection"], excludedJobTypes: ["episode_title"], targetMemoryIds: ["target"] }; + expect(h.ids(filters)).toEqual(["leased-expired", "ready"]); + expect(h.runtime.previewQueuedJobs({ at, limit: 50, ...filters }).map(j => j.id).sort()).toEqual(h.ids(filters)); + expect(h.ids({ targetMemoryIds: [] })).toEqual([]); + expect(h.ids({ allowedJobTypes: [] })).toEqual([]); + expect(h.runtime.getJob("leased-expired")!.status).toBe("leased"); + }); + + it("skips a reward dependency and blocked scope positions, keeping other scopes runnable", () => { + const h = harness(); h.memory("dependency"); + h.db.db.prepare(`INSERT INTO memory_processing_state(memory_id,state,stage,updated_at) VALUES ('dependency','embedding_pending','embedding',?)`).run(at); + h.rawJob("reward-wait", "reward", { payload: { l1MemoryId: "dependency", polarity: "positive" } }); + h.rawJob("reward-negative", "reward", { payload: { l1MemoryId: "dependency", polarity: "negative" } }); + h.rawJob("scope-first", "work_memory_extract", { scopeKey: "scope-a", scopeSeq: 1, payload: { runAfter: new Date(now + 60_000).toISOString() } }); + h.rawJob("scope-next", "work_memory_extract", { scopeKey: "scope-a", scopeSeq: 2 }); + h.rawJob("other-scope", "work_memory_extract", { scopeKey: "scope-b", scopeSeq: 1 }); + h.rawJob("missing-scope", "work_memory_extract"); + h.store.refreshPolicies(); + expect(h.ids()).toEqual(["other-scope", "reward-negative"]); + expect(h.runtime.previewQueuedJobs({ at, limit: 50 }).map(j => j.id).sort()).toEqual(h.ids()); + }); + + it("shares retry due and target filtering and keeps previews free of claims", () => { + const h = harness(); + h.retry("ready", "target"); h.retry("later", "later-target", now + 1_000); h.retry("outside", "other"); + h.retry("valid", "valid-target"); h.retry("expired", "expired-target"); + h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by='fixture-worker',lease_until=? WHERE id='valid'").run(now + 1_000); + h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by='fixture-worker',lease_until=? WHERE id='expired'").run(now); + h.store.refreshPolicies(); + expect(h.ids()).toEqual(["expired", "outside", "ready"]); + expect(h.ids({ targetMemoryIds: ["target"] })).toEqual(["ready"]); + expect(h.ids({ includeRetries: false })).toEqual([]); + expect(h.runtime.previewDueEmbeddingRetries({ now, limit: 50 }).map(j => j.id).sort()).toEqual(h.ids()); + expect(h.runtime.getEmbeddingRetry("ready")!.status).toBe("pending"); + }); + + it("uses a target lookup for retry policy invalidation instead of scanning the whole retry backlog", () => { + const h = harness(); h.retry("target-retry", "target"); + const plan = h.db.db.prepare("EXPLAIN QUERY PLAN UPDATE embedding_retry_queue SET policy_version=NULL WHERE target_id=?") + .all("target") as Array<{ detail: string }>; + expect(plan.some(row => row.detail.includes("SEARCH") && row.detail.includes("target_id=?"))).toBe(true); + h.memory("target", ["onboarding-report"]); + expect(h.runtime.getEmbeddingRetry("target-retry")!.policyVersion).toBeNull(); + h.store.refreshPolicies(); + expect(h.runtime.getEmbeddingRetry("target-retry")!.queueLane).toBe("interactive"); + }); + + it("reads bounded candidates for every lane even behind 10000 imported tasks", () => { + const h = harness(); + h.db.db.transaction(() => { + for (let i = 0; i < 10_000; i++) h.rawJob(`import-${i}`, "import_summary"); + h.rawJob("native", "trace_summary"); + h.rawJob("repair", "decision_repair"); + h.rawJob("reflection", "reflection"); + h.rawJob("maintenance", "project_environment_profile"); + h.retry("retry"); + })(); + while (h.store.pendingPolicyCount() > 0) h.store.refreshPolicies(500); + const candidates = h.store.candidates({ at, limit: 2 }); + expect(candidates.filter(c => c.lane === "import").map(c => c.id)).toEqual(["import-0", "import-1"]); + expect(candidates.map(c => c.id)).toEqual(expect.arrayContaining(["native", "repair", "reflection", "maintenance", "retry"])); + expect(candidates).toHaveLength(7); + expect(h.runtime.getJob("native")!.status).toBe("queued"); + expect(h.runtime.getEmbeddingRetry("retry")!.status).toBe("pending"); + }); +}); + +describe("real-scheduler policy preparation", () => { + it("does not invalidate selection while initially classifying newly enqueued or raw jobs", () => { + const h = harness(); + expect(h.store.getSelectionRevision()).toBe(0); + h.job("new-main"); h.retry("new-retry"); + h.rawJob("raw-new"); + expect(h.store.refreshRunnablePolicies({ at }).refreshed).toBe(1); + expect(h.store.getSelectionRevision()).toBe(0); + }); + + it("invalidates selection for an existing deduplicated task even after its metadata is current", () => { + const h = harness(); + const original = h.job("original", "import_summary", undefined, {}, "same-job"); + const revision = h.store.getSelectionRevision(); + const updated = h.job("discarded-id", "import_summary", undefined, + { source: "startup.placeholder_summary_repair" }, "same-job"); + expect(updated).toMatchObject({ id: original.id, queueLane: "maintenance", policyVersion: JOB_POLICY_VERSION }); + expect(h.store.getSelectionRevision()).toBe(revision + 1); + h.store.refreshOne("main", original.id); + // Refreshing unchanged, current metadata must not reset a deep candidate scan. + expect(h.store.getSelectionRevision()).toBe(revision + 1); + h.job("another-id", "import_summary", undefined, { scope: "changed" }, "same-job"); + expect(h.store.getSelectionRevision()).toBe(revision + 2); + }); + + it.each(["full", "runnable"] as const)("shares %s refresh invalidations across both tables without counting first projections", mode => { + const h = harness(); h.job("old-main"); h.retry("old-retry"); h.rawJob("fresh-main"); + h.db.db.prepare("UPDATE evolution_jobs SET policy_version=NULL WHERE id='old-main'").run(); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL WHERE id='old-retry'").run(); + expect(h.store.getSelectionRevision()).toBe(0); + const result = mode === "full" ? h.store.refreshPolicies() : h.store.refreshRunnablePolicies({ at, maxRows: 64 }); + expect(result.refreshed).toBe(3); + expect(h.store.getSelectionRevision()).toBe(2); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 64 }).refreshed).toBe(0); + expect(h.store.getSelectionRevision()).toBe(2); + }); + + it("shares the revision between Repositories using the same SQLite connection", () => { + const h = harness(), second = new Repositories(h.db.db).runtime.scheduling; + h.job("shared-main"); h.retry("shared-retry"); + expect(second.getSelectionRevision()).toBe(0); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json='{\"scope\":\"changed\"}' WHERE id='shared-main'").run(); + second.refreshOne("main", "shared-main"); + expect(h.store.getSelectionRevision()).toBe(1); + h.db.db.prepare("UPDATE embedding_retry_queue SET source_text='changed' WHERE id='shared-retry'").run(); + h.store.refreshOne("embedding_retry", "shared-retry"); + expect(second.getSelectionRevision()).toBe(2); + expect(harness().store.getSelectionRevision()).toBe(0); + }); + + it("keeps invalidation monotonic when an outer transaction rolls a policy update back", () => { + const h = harness(); const original = h.job("rolled-back", "import_summary"); + const before = h.store.getSelectionRevision(); + expect(() => h.repos.transaction(() => { + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id=?") + .run(JSON.stringify({ source: "startup.placeholder_summary_repair" }), original.id); + h.store.refreshOne("main", original.id); + expect(h.runtime.getJob(original.id)?.queueLane).toBe("maintenance"); + throw new Error("fixture transaction rollback"); + })).toThrow("fixture transaction rollback"); + expect(h.runtime.getJob(original.id)?.queueLane).toBe("import"); + expect(h.store.getSelectionRevision()).toBeGreaterThan(before); + const afterRollback = h.store.getSelectionRevision(); + h.store.refreshOne("main", original.id); + expect(h.store.getSelectionRevision()).toBe(afterRollback); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json='{\"scope\":\"changed\"}' WHERE id=?").run(original.id); + h.store.refreshOne("main", original.id); + expect(h.store.getSelectionRevision()).toBeGreaterThan(afterRollback); + h.store.refreshOne("main", "missing"); + expect(h.store.getSelectionRevision()).toBe(afterRollback + 1); + }); + + it.each([0, 1, 64])("limits actual policy writes to maxRows=%i across both backing queues", maxRows => { + const h = harness(); + h.db.db.transaction(() => { + for (let i = 0; i < 65; i++) h.rawJob(`main-${i}`, "import_summary"); + for (let i = 0; i < 3; i++) h.retry(`retry-${i}`); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL").run(); + })(); + const input = { at, maxRows }; + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: maxRows, unknown: [], hasMore: true }); + const projected = h.db.db.prepare(`SELECT + (SELECT COUNT(*) FROM evolution_jobs WHERE policy_version=?) + + (SELECT COUNT(*) FROM embedding_retry_queue WHERE policy_version=?) AS n`) + .get(JOB_POLICY_VERSION, JOB_POLICY_VERSION) as { n: number }; + expect(projected.n).toBe(maxRows); + if (maxRows === 0) { + expect(h.store.pendingPolicyCount()).toBe(68); + expect(h.store.refreshRunnablePolicies({ at })).toEqual({ refreshed: 68, unknown: [] }); + } else { + let total = maxRows; + for (let round = 0; round < 70; round++) { + const result = h.store.refreshRunnablePolicies(input); + expect(result.refreshed).toBeLessThanOrEqual(maxRows); + total += result.refreshed; + if (result.hasMore === false) break; + } + expect(total).toBe(68); + } + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + }); + + it("shares the remaining row budget with retries instead of giving each table a separate allowance", () => { + const h = harness(); + h.rawJob("main"); + h.retry("retry-first"); h.retry("retry-second"); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL").run(); + const result = h.store.refreshRunnablePolicies({ at, maxRows: 2 }); + expect(result).toEqual({ refreshed: 2, unknown: [], hasMore: true }); + expect(h.runtime.getJob("main")?.policyVersion).toBe(JOB_POLICY_VERSION); + expect(h.runtime.getEmbeddingRetry("retry-first")?.policyVersion).toBe(JOB_POLICY_VERSION); + expect(h.runtime.getEmbeddingRetry("retry-second")?.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 2 })).toEqual({ refreshed: 1, unknown: [], hasMore: false }); + }); + + it("reports a precise exhausted boundary on the next empty slice and probes zero budget without writes", () => { + const h = harness(); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 0 })).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + h.rawJob("one"); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 0 })).toEqual({ refreshed: 0, unknown: [], hasMore: true }); + expect(h.runtime.getJob("one")?.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 1 })).toEqual({ refreshed: 1, unknown: [], hasMore: true }); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 1 })).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + }); + + it("preserves due, dependency, scope, configuration, terminal and retry eligibility in bounded slices", () => { + const h = harness(); + for (const id of ["ready", "due-failed", "future-failed", "configuration", "succeeded", "terminal", "leased-valid", "leased-expired"]) + h.rawJob(id); + h.rawJob("future-run", "reflection", { payload: { runAfter: new Date(now + 1000).toISOString() } }); + h.db.db.prepare("UPDATE evolution_jobs SET status='failed',available_at=? WHERE id='due-failed'").run(at); + h.db.db.prepare("UPDATE evolution_jobs SET status='failed',available_at=? WHERE id='future-failed'").run(new Date(now+1000).toISOString()); + h.db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='configuration'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE id='succeeded'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET status='dead_letter' WHERE id='terminal'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='leased-valid'").run(new Date(now+1000).toISOString()); + h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='leased-expired'").run(at); + h.memory("pending-target"); + h.repos.processing.save({ memoryId: "pending-target", state: "embedding_pending", stage: "embedding", activeJobId: null, + attemptCount: 0, manualRetryCount: 0, retryAction: "retry", updatedAt: at }); + h.rawJob("reward-blocked", "reward", { payload: { l1MemoryId: "pending-target" } }); + h.rawJob("scope-first", "l3_world_model_update", { scopeKey: "project", scopeSeq: 1 }); + h.rawJob("scope-blocked", "l3_world_model_update", { scopeKey: "project", scopeSeq: 2 }); + h.retry("retry-due"); h.retry("retry-future"); h.retry("retry-terminal"); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL").run(); + h.db.db.prepare("UPDATE embedding_retry_queue SET next_attempt_at=? WHERE id='retry-future'").run(now+1000); + h.db.db.prepare("UPDATE embedding_retry_queue SET status='succeeded' WHERE id='retry-terminal'").run(); + const input = { at, maxRows: 1, includeDueFailed: true }; + let total = 0; + for (let i = 0; i < 10; i++) { + const result = h.store.refreshRunnablePolicies(input); total += result.refreshed; + expect(result.refreshed).toBeLessThanOrEqual(1); + if (result.hasMore === false) break; + } + expect(total).toBe(5); + for (const id of ["ready", "due-failed", "leased-expired", "scope-first"]) + expect(h.runtime.getJob(id)?.policyVersion).toBe(JOB_POLICY_VERSION); + for (const id of ["future-failed", "configuration", "succeeded", "terminal", "leased-valid", "future-run", "reward-blocked", "scope-blocked"]) + expect(h.runtime.getJob(id)?.policyVersion).toBeNull(); + expect(h.runtime.getEmbeddingRetry("retry-due")?.policyVersion).toBe(JOB_POLICY_VERSION); + expect(h.runtime.getEmbeddingRetry("retry-future")?.policyVersion).toBeNull(); + expect(h.runtime.getEmbeddingRetry("retry-terminal")?.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies({ ...input, maxRows: 0 })).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + }); + + it("applies target/type/retry gates to the bounded zero-write probe as well as projection", () => { + const h = harness(); + h.rawJob("outside", "reflection", { target: "outside" }); + h.rawJob("wrong-type", "episode_title", { target: "selected" }); + h.retry("inside-retry", "selected"); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL").run(); + const input = { at, targetMemoryIds: ["selected"], allowedJobTypes: ["reflection"], includeRetries: false, maxRows: 0 }; + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + expect(h.store.refreshRunnablePolicies({ ...input, includeRetries: true })).toEqual({ refreshed: 0, unknown: [], hasMore: true }); + expect(h.store.refreshRunnablePolicies({ ...input, includeRetries: true, maxRows: 1 })).toEqual({ refreshed: 1, unknown: [], hasMore: true }); + expect(h.runtime.getJob("outside")?.policyVersion).toBeNull(); + expect(h.runtime.getJob("wrong-type")?.policyVersion).toBeNull(); + }); + + it("counts unknown-type quarantine against the same row budget and does not warn/project it again", () => { + const h = harness(); h.rawJob("unknown", "future_new_job"); h.rawJob("next"); + const input = { at, maxRows: 1 }; + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: 1, unknown: [{ id: "unknown", jobType: "future_new_job" }], hasMore: true }); + expect(h.runtime.getJob("unknown")).toMatchObject({ queueLane: "unregistered", attempts: 0, status: "queued" }); + expect(h.runtime.getJob("next")?.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: 1, unknown: [], hasMore: true }); + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + }); + + it("does not charge already-classified rows, exhausted failures or valid retry leases against a slice", () => { + const h = harness(); + h.job("already-classified"); + h.rawJob("exhausted"); + h.db.db.prepare("UPDATE evolution_jobs SET failure_count=max_attempts WHERE id='exhausted'").run(); + h.retry("valid-lease"); h.retry("expired-lease"); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL,status='in_progress',claimed_by='fixture-worker',lease_until=? WHERE id='valid-lease'").run(now+1000); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL,status='in_progress',claimed_by='fixture-worker',lease_until=? WHERE id='expired-lease'").run(now); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 1 })).toEqual({ refreshed: 1, unknown: [], hasMore: true }); + expect(h.runtime.getEmbeddingRetry("expired-lease")?.policyVersion).toBe(JOB_POLICY_VERSION); + expect(h.runtime.getEmbeddingRetry("valid-lease")?.policyVersion).toBeNull(); + expect(h.runtime.getJob("exhausted")?.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies({ at, maxRows: 1 })).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + }); + + it("retains type exclusion and the default failed-job gate during zero-write and nonzero slices", () => { + const h = harness(); + h.rawJob("excluded", "trace_summary"); h.rawJob("failure", "reflection"); + h.db.db.prepare("UPDATE evolution_jobs SET status='failed' WHERE id='failure'").run(); + const input = { at, maxRows: 0, excludedJobTypes: ["trace_summary"] }; + expect(h.store.refreshRunnablePolicies(input)).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + expect(h.store.refreshRunnablePolicies({ ...input, maxRows: 1 })).toEqual({ refreshed: 0, unknown: [], hasMore: false }); + expect(h.store.refreshRunnablePolicies({ ...input, maxRows: 1, includeDueFailed: true })) + .toEqual({ refreshed: 1, unknown: [], hasMore: true }); + expect(h.runtime.getJob("excluded")?.policyVersion).toBeNull(); + }); + + it.each([-1, 0.5, NaN, Infinity])("rejects invalid projection budget %s without writing rows", maxRows => { + const h = harness(); h.rawJob("untouched"); + expect(() => h.store.refreshRunnablePolicies({ at, maxRows })).toThrow(RangeError); + expect(h.runtime.getJob("untouched")?.policyVersion).toBeNull(); + }); + + it("finishes more than one projection batch before FIFO selection without touching terminal history", () => { + const h = harness(); + h.db.db.transaction(() => { + for (let i = 0; i < 501; i++) h.rawJob(`old-${i}`, "import_summary"); + for (const status of ["succeeded", "dead_letter", "failed"] as const) { + h.rawJob(`terminal-${status}`); + h.db.db.prepare("UPDATE evolution_jobs SET status=? WHERE id=?").run(status, `terminal-${status}`); + } + h.rawJob("future", "reflection", { payload: { runAfter: new Date(now + 1_000).toISOString() } }); + h.job("new-already-classified", "import_summary"); + h.retry("ready-retry"); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL WHERE id='ready-retry'").run(); + })(); + expect(h.store.refreshRunnablePolicies({ at })).toEqual({ refreshed: 502, unknown: [] }); + expect(h.store.pendingPolicyCount()).toBe(4); + const candidates = h.store.candidates({ at, limit: 5 }); + expect(candidates.filter(c => c.lane === "import").map(c => c.id)).toEqual(["old-0", "old-1", "old-2", "old-3", "old-4"]); + expect(candidates.map(c => c.id)).toContain("ready-retry"); + expect(h.runtime.getJob("terminal-succeeded")!.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies({ at }).refreshed).toBe(0); + expect(h.store.refreshRunnablePolicies({ at: new Date(now + 1_000).toISOString() }).refreshed).toBe(1); + }); + + it("limits policy preparation to the same runnable targets/types and retry gate", () => { + const h = harness(); + h.rawJob("inside", "reflection", { target: "selected" }); + h.rawJob("outside", "reflection", { target: "other" }); + h.rawJob("other-type", "episode_title", { target: "selected" }); + h.retry("retry", "selected"); + h.db.db.prepare("UPDATE embedding_retry_queue SET policy_version=NULL WHERE id='retry'").run(); + expect(h.store.refreshRunnablePolicies({ at, targetMemoryIds: ["selected"], allowedJobTypes: ["reflection"], includeRetries: false }).refreshed).toBe(1); + expect(h.runtime.getJob("outside")!.policyVersion).toBeNull(); + expect(h.runtime.getJob("other-type")!.policyVersion).toBeNull(); + expect(h.runtime.getEmbeddingRetry("retry")!.policyVersion).toBeNull(); + expect(h.store.refreshRunnablePolicies({ at, targetMemoryIds: [], includeRetries: true }).refreshed).toBe(0); + expect(h.store.refreshRunnablePolicies({ at, allowedJobTypes: [], includeRetries: true }).refreshed).toBe(1); + }); + + it("normalizes valid JSON with malformed metadata shapes so one row cannot poison the runnable set", () => { + const h = harness(); h.memory("bad-shapes"); + h.db.db.prepare("UPDATE memories SET tags_json=?, properties_json=? WHERE id='bad-shapes'") + .run('[null,2,"agent-source"]', '{"internal_info":{"plugin_algorithm":42}}'); + h.rawJob("malformed-payload", "embedding", { target: "bad-shapes" }); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json='null' WHERE id='malformed-payload'").run(); + h.rawJob("healthy", "reflection"); + expect(h.store.refreshRunnablePolicies({ at })).toEqual({ refreshed: 2, unknown: [] }); + expect(h.runtime.getJob("malformed-payload")).toMatchObject({ queueLane: "import", policyVersion: JOB_POLICY_VERSION }); + expect(h.ids()).toEqual(["healthy", "malformed-payload"]); + h.db.db.prepare("UPDATE memories SET tags_json='{}' WHERE id='bad-shapes'").run(); + expect(h.store.refreshRunnablePolicies({ at }).refreshed).toBe(1); + expect(h.runtime.getJob("malformed-payload")!.queueLane).toBe("interactive"); + }); + + it("projects historical origin during dirty runnable refresh and ignores malformed origins", () => { + const h = harness(); h.memory("native-history"); h.memory("malformed-history"); + h.db.db.prepare("UPDATE memories SET properties_json=? WHERE id='native-history'") + .run(JSON.stringify({ internal_info: { plugin_algorithm: "capture.v7", capture_origin: "agent_source_scan" } })); + h.db.db.prepare("UPDATE memories SET properties_json=? WHERE id='malformed-history'") + .run(JSON.stringify({ internal_info: { plugin_algorithm: "capture.v7", capture_origin: ["agent_source_scan"] } })); + h.rawJob("raw-native-embedding", "embedding", { target: "native-history" }); + h.rawJob("raw-malformed-summary", "trace_summary", { target: "malformed-history" }); + expect(h.store.refreshRunnablePolicies({ at, includeRetries: false })).toEqual({ refreshed: 2, unknown: [] }); + expect(h.runtime.getJob("raw-native-embedding")).toMatchObject({ queueLane: "import", policyVersion: JOB_POLICY_VERSION }); + expect(h.runtime.getJob("raw-malformed-summary")).toMatchObject({ queueLane: "interactive", policyVersion: JOB_POLICY_VERSION }); + }); + + it("exposes unknown jobs only through explicit maintenance quarantine admission", () => { + const h = harness(); h.rawJob("unknown", "not_registered_yet"); + expect(h.store.refreshRunnablePolicies({ at })).toEqual({ refreshed: 1, unknown: [{ id: "unknown", jobType: "not_registered_yet" }] }); + expect(h.store.candidates({ at, limit: 2 })).toEqual([]); + expect(h.store.candidates({ at, limit: 2, includeUnregistered: true })).toEqual([ + expect.objectContaining({ id: "unknown", queue: "main", lane: "maintenance", unregistered: true }) + ]); + expect(h.runtime.getJob("unknown")).toMatchObject({ queueLane: "unregistered", status: "queued", attempts: 0 }); + }); +}); diff --git a/Memory/tests/repository/queue-selected-claim.test.ts b/Memory/tests/repository/queue-selected-claim.test.ts new file mode 100644 index 000000000..a0a6303ce --- /dev/null +++ b/Memory/tests/repository/queue-selected-claim.test.ts @@ -0,0 +1,128 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../src/storage/repositories.js"; + +const at = "2026-10-22T00:00:00.000Z", now = Date.parse(at); +const resources: Array<{ db: MemoryDb; root: string }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "memmy-selected-claim-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ db, root }); + const repos = new Repositories(db.db), runtime = repos.runtime; + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection", targetMemoryId?: string) { + return runtime.enqueueJob({ id, jobType, targetMemoryId, userId: "user", status: "queued", payload: {}, + attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + function retry(id: string) { + return runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, vectorField: "vec_summary", sourceText: id, now }); + } + return { db, repos, runtime, job, retry }; +} +afterEach(() => { + for (const { db, root } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("selected-id claims for joint scheduling", () => { + it("claims the selected job instead of substituting the legacy highest-priority head", () => { + const h = harness(); h.job("legacy-first", "trace_summary"); h.job("chosen-reflection"); + const chosen = h.runtime.claimQueuedJob("chosen-reflection", { at, owner: "scheduler", leaseSeconds: 90 }); + expect(chosen).toMatchObject({ id: "chosen-reflection", status: "leased", leaseOwner: "scheduler", attempts: 1, + leaseCount: 1, failureCount: 0, leasedUntil: new Date(now + 90_000).toISOString() }); + expect(chosen!.leaseToken).toEqual(expect.any(String)); + expect(h.runtime.getJob("legacy-first")).toMatchObject({ status: "queued", attempts: 0 }); + expect(h.runtime.claimQueuedJob("chosen-reflection", { at })).toBeUndefined(); + expect(h.runtime.claimQueuedJob("' OR 1=1 --", { at })).toBeUndefined(); + }); + + it("rechecks target, type, deadlines, configuration and failure budget before claiming", () => { + const h = harness(); + for (const id of ["ready", "future", "runAfter", "config", "exhausted"]) h.job(id, "reflection", "target"); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='future'").run(new Date(now + 1_000).toISOString()); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='runAfter'").run(JSON.stringify({ runAfter: new Date(now + 1_000).toISOString() })); + h.db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='config'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET failure_count=max_attempts WHERE id='exhausted'").run(); + for (const id of ["future", "runAfter", "config", "exhausted"]) expect(h.runtime.claimQueuedJob(id, { at })).toBeUndefined(); + for (const filters of [{ targetMemoryIds: [] }, { targetMemoryIds: ["other"] }, { allowedJobTypes: [] }, + { allowedJobTypes: ["embedding"] }, { excludedJobTypes: ["reflection"] }]) { + expect(h.runtime.claimQueuedJob("ready", { at, ...filters })).toBeUndefined(); + } + expect(h.runtime.claimQueuedJob("ready", { at, targetMemoryIds: ["target"], allowedJobTypes: ["reflection"] })?.id).toBe("ready"); + }); + + it("retains dependency and scope ordering checks in the selected-id path", () => { + const h = harness(); + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,created_at,updated_at) + VALUES ('dependency',?,'user','text','L1',?,?)`).run(at, at, at); + h.db.db.prepare(`INSERT INTO memory_processing_state(memory_id,state,stage,updated_at) + VALUES ('dependency','embedding_pending','embedding',?)`).run(at); + h.job("reward", "reward"); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='reward'").run('{"l1MemoryId":"dependency","polarity":"positive"}'); + h.job("scope-first", "work_memory_extract"); h.job("scope-second", "work_memory_extract"); + h.db.db.prepare("UPDATE evolution_jobs SET scope_key='scope',scope_seq=1 WHERE id='scope-first'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET scope_key='scope',scope_seq=2 WHERE id='scope-second'").run(); + expect(h.runtime.claimQueuedJob("reward", { at })).toBeUndefined(); + expect(h.runtime.claimQueuedJob("scope-second", { at })).toBeUndefined(); + const first = h.runtime.claimQueuedJob("scope-first", { at })!; + expect(h.runtime.claimQueuedJob("scope-second", { at })).toBeUndefined(); + h.runtime.completeJob(first.id, at, first); + expect(h.runtime.claimQueuedJob("scope-second", { at })?.id).toBe("scope-second"); + }); + + it("recovers an expired main claim with a new generation without charging a business failure", () => { + const h = harness(); h.job("recover"); + const a = h.runtime.claimQueuedJob("recover", { at, owner: "same", leaseSeconds: 1 })!; + const later = new Date(now + 1_000).toISOString(); + const b = h.runtime.claimQueuedJob("recover", { at: later, owner: "same" })!; + expect(b.leaseToken).not.toBe(a.leaseToken); + expect(b).toMatchObject({ attempts: 1, leaseCount: 2, leaseRecoveryCount: 1, failureCount: 0 }); + expect(h.runtime.completeJob(a.id, later, a)).toBeUndefined(); + expect(h.runtime.failJob(a.id, "stale", later, false, a)).toBeUndefined(); + expect(h.runtime.completeJob(b.id, later, b)?.status).toBe("succeeded"); + }); + + it("claims only the chosen retry and rechecks readiness plus target filters", () => { + const h = harness(); h.retry("first"); h.retry("chosen"); + const input = { now, workerId: "worker", leaseUntil: now + 60_000 }; + expect(h.runtime.claimEmbeddingRetryById("chosen", { ...input, targetMemoryIds: [] })).toBeUndefined(); + expect(h.runtime.claimEmbeddingRetryById("chosen", { ...input, targetMemoryIds: ["first"] })).toBeUndefined(); + const chosen = h.runtime.claimEmbeddingRetryById("chosen", input)!; + expect(chosen).toMatchObject({ id: "chosen", status: "in_progress", claimedBy: "worker", attempts: 0 }); + expect(chosen.leaseToken).toEqual(expect.any(String)); + expect(h.runtime.getEmbeddingRetry("first")!.status).toBe("pending"); + expect(h.runtime.claimEmbeddingRetryById("chosen", input)).toBeUndefined(); + h.db.db.prepare("UPDATE embedding_retry_queue SET next_attempt_at=? WHERE id='first'").run(now + 1); + expect(h.runtime.claimEmbeddingRetryById("first", input)).toBeUndefined(); + expect(h.runtime.claimEmbeddingRetryById("first", { ...input, now: now + 1 })?.id).toBe("first"); + }); + + it("returns unstarted retry claims using current fencing credentials, preserving retry history", () => { + const h = harness(); h.retry("release"); + const input = { now, workerId: "worker", leaseUntil: now + 10_000 }; + const old = h.runtime.claimEmbeddingRetryById("release", input)!; + const renewed = h.runtime.renewEmbeddingRetryClaim(old.id, { workerId: old.claimedBy!, leaseUntil: old.leaseUntil!, leaseToken: old.leaseToken! }, now + 20_000, now + 1)!; + expect(h.runtime.releaseClaimedEmbeddingRetries([old], now + 1)).toEqual([]); + const changes = h.runtime.releaseClaimedEmbeddingRetries([renewed], now + 1); + expect(changes).toHaveLength(1); + expect(changes[0]!.after).toMatchObject({ status: "pending", attempts: 0, nextAttemptAt: now, leaseToken: null, claimedBy: null, leaseUntil: null }); + const again = h.runtime.claimEmbeddingRetryById(old.id, input)!; + expect(again.leaseToken).not.toBe(old.leaseToken); + expect(h.runtime.releaseClaimedEmbeddingRetries([old], now + 2)).toEqual([]); + }); + + it("participates in an outer transaction so joint claims roll back together", () => { + const h = harness(); h.job("main"); h.retry("retry"); + expect(() => h.repos.transaction(() => { + expect(h.runtime.claimQueuedJob("main", { at })).toBeDefined(); + expect(h.runtime.claimEmbeddingRetryById("retry", { now, workerId: "worker", leaseUntil: now + 60_000 })).toBeDefined(); + throw new Error("cancel joint admission"); + })).toThrow("cancel joint admission"); + expect(h.runtime.getJob("main")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null, leaseCount: 0 }); + expect(h.runtime.getEmbeddingRetry("retry")).toMatchObject({ status: "pending", leaseToken: null }); + }); +}); diff --git a/Memory/tests/repository/sqlite-schema.test.ts b/Memory/tests/repository/sqlite-schema.test.ts index cef16a698..c574c758c 100644 --- a/Memory/tests/repository/sqlite-schema.test.ts +++ b/Memory/tests/repository/sqlite-schema.test.ts @@ -703,6 +703,7 @@ describe("repository sqlite schema contract", () => { DROP TABLE l3_world_model_scopes; DROP INDEX uq_evolution_jobs_l3_immutable_dedupe; DROP INDEX uq_evolution_jobs_scope_seq; + DROP TRIGGER evolution_jobs_policy_dirty; ALTER TABLE evolution_jobs DROP COLUMN scope_key; ALTER TABLE evolution_jobs DROP COLUMN scope_seq; DELETE FROM schema_migrations; diff --git a/Memory/tests/repository/worker-lease-schema.test.ts b/Memory/tests/repository/worker-lease-schema.test.ts new file mode 100644 index 000000000..8b95b91f4 --- /dev/null +++ b/Memory/tests/repository/worker-lease-schema.test.ts @@ -0,0 +1,248 @@ +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Database from "better-sqlite3"; +import { describe, expect, it } from "vitest"; +import { pendingPolicyIndex } from "../../src/storage/queue-scheduling-schema.js"; +import { MemoryDb } from "../../src/storage/db.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { SCHEMA_VERSION } from "../../src/storage/schema.js"; + +describe("worker lease schema migration", () => { + it("keeps ordinary dedupe enqueues blocked but lets an explicit manual retry resume the original job", () => { + usingDatabase((runtime) => { + const at = new Date().toISOString(); + const original = { + id: "original-summary", jobType: "trace_summary" as const, status: "queued" as const, + dedupeKey: "summary:same-target", userId: "user", attempts: 0, maxAttempts: 3, + payload: { source: "automatic" }, createdAt: at, updatedAt: at + }; + runtime.enqueueJob(original); + const [claimed] = runtime.leaseQueuedJobs(1); + runtime.failJob(claimed!.id, "needs configuration", undefined, false, claimed, { + blockedReason: "configuration", availableAt: new Date(Date.now() + 60_000).toISOString() + }); + runtime.enqueueJob({ ...original, id: "ordinary-duplicate" }); + expect(runtime.getJob(original.id)).toMatchObject({ status: "failed", blockedReason: "configuration", failureCount: 1 }); + const resumed = runtime.enqueueJob({ ...original, id: "manual-duplicate", payload: { source: "memory.processing.manual_retry" } }); + expect(resumed).toMatchObject({ id: original.id, status: "queued", blockedReason: null, + availableAt: null, attempts: 0, failureCount: 0, leaseCount: 1 }); + }); + }); + + it("keeps an unchanged retry claim and preserves the pending retry deadline on duplicate enqueue", () => { + usingDatabase((runtime) => { + const now = Date.now(); + const retry = runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "same", vectorField: "vec_summary", sourceText: "unchanged", now }); + const [claim] = runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 60_000 }); + runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "same", vectorField: "vec_summary", sourceText: "unchanged", now: now + 100 }); + expect(runtime.getEmbeddingRetry(retry.id)).toMatchObject({ status: "in_progress", claimedBy: "A", leaseUntil: now + 60_000 }); + expect(runtime.isEmbeddingRetryClaimHeld(retry.id, { workerId: "A", leaseUntil: claim!.leaseUntil!, leaseToken: claim!.leaseToken! }, now + 100)).toBe(true); + runtime.markEmbeddingRetryRetryClaimed(retry.id, { + workerId: "A", leaseUntil: now + 60_000, leaseToken: claim!.leaseToken!, attempts: 1, nextAttemptAt: now + 5_000, + error: "temporary", now: now + 100 + }); + runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "same", vectorField: "vec_summary", sourceText: "unchanged", now: now + 200 }); + expect(runtime.getEmbeddingRetry(retry.id)).toMatchObject({ status: "pending", attempts: 1, nextAttemptAt: now + 5_000 }); + expect(runtime.claimDueEmbeddingRetries({ now: now + 200, workerId: "B", leaseUntil: now + 60_000 })).toEqual([]); + }); + }); + + it("invalidates an active retry claim when its input changes instead of leaving it orphaned in progress", () => { + usingDatabase((runtime) => { + const now = Date.now(); + const retry = runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "changed", vectorField: "vec_summary", sourceText: "old", now }); + const [claim] = runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 60_000 }); + runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "changed", vectorField: "vec_summary", sourceText: "new", now: now + 100 }); + expect(runtime.getEmbeddingRetry(retry.id)).toMatchObject({ status: "pending", claimedBy: null, leaseUntil: null, sourceText: "new" }); + expect(runtime.markEmbeddingRetrySucceededClaimed(retry.id, { workerId: "A", leaseUntil: now + 60_000, leaseToken: claim!.leaseToken!, now: now + 100 })).toBeUndefined(); + expect(runtime.claimDueEmbeddingRetries({ now: now + 100, workerId: "B", leaseUntil: now + 60_100 })).toHaveLength(1); + }); + }); + + it("resumes configuration-blocked retries without reviving permanent failures or consuming attempts", () => { + usingDatabase((runtime) => { + const now = Date.now(); + const configurable = runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "configurable", vectorField: "vec_summary", sourceText: "configurable", now }); + const terminal = runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "terminal", vectorField: "vec_summary", sourceText: "terminal", now }); + const claims = runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 60_000 }); + runtime.markEmbeddingRetryConfigurationBlockedClaimed(configurable.id, { workerId: "A", leaseUntil: now + 60_000, leaseToken: claims.find(c => c.id === configurable.id)!.leaseToken!, now: now + 100, error: "credentials missing" }); + runtime.markEmbeddingRetryFailedClaimed(terminal.id, { workerId: "A", leaseUntil: now + 60_000, leaseToken: claims.find(c => c.id === terminal.id)!.leaseToken!, now: now + 100, error: "invalid input", attempts: 1 }); + expect(runtime.getEmbeddingRetry(configurable.id)).toMatchObject({ status: "failed", blockedReason: "configuration", attempts: 0 }); + expect(runtime.requeueEmbeddingRetriesAfterRestart(now + 100)).toEqual([]); + const resumed = runtime.resumeConfigurationBlockedEmbeddingRetries(now + 100); + expect(resumed.map(({ after }) => after.id)).toEqual([configurable.id]); + expect(runtime.getEmbeddingRetry(configurable.id)).toMatchObject({ status: "pending", blockedReason: null, attempts: 0 }); + expect(runtime.getEmbeddingRetry(terminal.id)).toMatchObject({ status: "failed", attempts: 1 }); + }); + }); + + it("rejects every stale completion even when the same worker reclaims changed input in the same millisecond", () => { + usingDatabase((runtime) => { + const now = Date.now(); + const input = { targetKind: "trace" as const, targetId: "same-clock", vectorField: "vec_summary" as const, sourceText: "old", now }; + const retry = runtime.enqueueEmbeddingRetry(input); + const [old] = runtime.claimDueEmbeddingRetries({ now, workerId: "worker", leaseUntil: now + 60_000 }); + runtime.enqueueEmbeddingRetry({ ...input, sourceText: "new" }); + const [current] = runtime.claimDueEmbeddingRetries({ now, workerId: "worker", leaseUntil: now + 60_000 }); + expect(current!.leaseToken).toEqual(expect.any(String)); + expect(current!.leaseToken).not.toBe(old!.leaseToken); + const stale = { workerId: "worker", leaseUntil: old!.leaseUntil!, leaseToken: old!.leaseToken! }; + expect(runtime.isEmbeddingRetryClaimHeld(retry.id, stale, now)).toBe(false); + expect(runtime.renewEmbeddingRetryClaim(retry.id, stale, now + 90_000, now)).toBeUndefined(); + expect(runtime.markEmbeddingRetrySucceededClaimed(retry.id, { ...stale, now })).toBeUndefined(); + expect(runtime.markEmbeddingRetryFailedClaimed(retry.id, { ...stale, attempts: 1, error: "old failure", now })).toBeUndefined(); + expect(runtime.markEmbeddingRetryRetryClaimed(retry.id, { ...stale, attempts: 1, error: "old failure", now, nextAttemptAt: now + 10_000 })).toBeUndefined(); + expect(runtime.markEmbeddingRetryConfigurationBlockedClaimed(retry.id, { ...stale, error: "old config", now })).toBeUndefined(); + expect(runtime.getEmbeddingRetry(retry.id)).toMatchObject({ status: "in_progress", sourceText: "new", attempts: 0, leaseToken: current!.leaseToken }); + const fresh = { workerId: "worker", leaseUntil: current!.leaseUntil!, leaseToken: current!.leaseToken! }; + const renewed = runtime.renewEmbeddingRetryClaim(retry.id, fresh, now + 90_000, now + 1)!; + expect(renewed.leaseToken).toBe(current!.leaseToken); + expect(runtime.isEmbeddingRetryClaimHeld(retry.id, { ...fresh, leaseUntil: renewed.leaseUntil! }, now + 2)).toBe(true); + expect(runtime.markEmbeddingRetrySucceededClaimed(retry.id, { ...fresh, leaseUntil: renewed.leaseUntil!, now: now + 2 })).toMatchObject({ status: "succeeded", leaseToken: null }); + }); + }); + + it("gives each retry in one batch its own token and keeps that token on duplicate enqueue", () => { + usingDatabase((runtime) => { + const now = Date.now(); + for (const targetId of ["one", "two"]) runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId, vectorField: "vec_summary", sourceText: targetId, now }); + const claims = runtime.claimDueEmbeddingRetries({ now, workerId: "worker", leaseUntil: now + 60_000 }); + expect(new Set(claims.map(c => c.leaseToken)).size).toBe(2); + runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "one", vectorField: "vec_summary", sourceText: "one", now }); + expect(runtime.getEmbeddingRetryByTarget("trace", "one", "vec_summary")!.leaseToken).toBe(claims.find(c => c.targetId === "one")!.leaseToken); + }); + }); + + it("upgrades a v10 retry queue, preserves a valid old lease, and assigns a unique token on recovery", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-retry-token-migration-")); + const path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + try { + const now = Date.now(); + db = new MemoryDb({ path }); + const runtime = new Repositories(db.db).runtime; + const retry = runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "legacy-retry", vectorField: "vec_summary", sourceText: "preserved", now }); + runtime.claimDueEmbeddingRetries({ now, workerId: "legacy", leaseUntil: now + 1_000 }); + removeSchedulingSchema(db); + db.db.exec("ALTER TABLE embedding_retry_queue DROP COLUMN lease_token"); + db.db.exec("DELETE FROM schema_migrations"); + db.db.prepare("INSERT INTO schema_migrations VALUES ('010_worker_lease_safety', 10, ?, 'legacy')").run(new Date(now).toISOString()); + db.close(); + db = new MemoryDb({ path }); + const reopened = new Repositories(db.db).runtime; + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + expect(reopened.getEmbeddingRetry(retry.id)).toMatchObject({ status: "in_progress", sourceText: "preserved", claimedBy: "legacy", leaseToken: null }); + expect(reopened.requeueEmbeddingRetriesAfterRestart(now + 100)).toEqual([]); + expect(reopened.claimDueEmbeddingRetries({ now: now + 100, workerId: "new", leaseUntil: now + 10_000 })).toEqual([]); + const [recovered] = reopened.claimDueEmbeddingRetries({ now: now + 1_001, workerId: "new", leaseUntil: now + 10_000 }); + expect(recovered).toMatchObject({ status: "in_progress", sourceText: "preserved", attempts: 0, leaseToken: expect.any(String) }); + expect(existsSync(`${path}.pre-v${SCHEMA_VERSION}.bak`)).toBe(true); + } finally { + if (db?.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([9, 10])("upgrades legacy v%i without losing jobs, separates historic counters, and keeps a pre-migration backup", version => { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-lease-migration-")); + const path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + let backup: Database.Database | undefined; + try { + db = new MemoryDb({ path }); + // Restore the prior v9 task-table shape to exercise an actual old file. + removeSchedulingSchema(db); + db.db.exec("DROP INDEX idx_evolution_jobs_available"); + for (const column of ["lease_owner", "lease_token", "available_at", "blocked_reason", + "failure_count", "lease_count", "lease_recovery_count"]) { + db.db.exec(`ALTER TABLE evolution_jobs DROP COLUMN ${column}`); + } + db.db.exec("ALTER TABLE embedding_retry_queue DROP COLUMN blocked_reason"); + db.db.exec("ALTER TABLE embedding_retry_queue DROP COLUMN lease_token"); + db.db.exec("DELETE FROM schema_migrations"); + const at = "2026-10-22T00:00:00.000Z"; + db.db.prepare("INSERT INTO schema_migrations VALUES (?, ?, ?, 'legacy')").run( + version === 9 ? "009_token_usage_outbox" : "010_pending_episode_assignment", version, at); + const insert = db.db.prepare(`INSERT INTO evolution_jobs + (id, job_type, status, user_id, attempts, max_attempts, leased_until, created_at, updated_at) + VALUES (?, 'reflection', ?, 'migration-user', ?, 3, ?, ?, ?)`); + for (const [status, attempts] of [["queued", 2], ["failed", 2], ["leased", 3], + ["succeeded", 3], ["dead_letter", 3]] as const) { + insert.run(status, status, attempts, status === "leased" ? "2099-01-01T00:00:00.000Z" : null, at, at); + } + db.close(); + db = new MemoryDb({ path }); + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + const runtime = new Repositories(db.db).runtime; + expect(runtime.getJob("queued")).toMatchObject({ attempts: 2, failureCount: 2, leaseCount: 2, leaseRecoveryCount: 0 }); + expect(runtime.getJob("failed")).toMatchObject({ failureCount: 2, leaseCount: 2 }); + expect(runtime.getJob("leased")).toMatchObject({ failureCount: 2, leaseCount: 3, leaseToken: null }); + expect(runtime.getJob("succeeded")).toMatchObject({ failureCount: 2, leaseCount: 3 }); + expect(runtime.getJob("dead_letter")).toMatchObject({ failureCount: 3, leaseCount: 3 }); + // An old lease in the future is retained until it expires. + expect(runtime.requeueLeasedJobsAfterRestart(at)).toEqual([]); + const backupPath = `${path}.pre-v${SCHEMA_VERSION}.bak`; + expect(existsSync(backupPath)).toBe(true); + backup = new Database(backupPath, { readonly: true }); + expect(backup.prepare("SELECT COUNT(*) AS n FROM evolution_jobs").get()).toEqual({ n: 5 }); + const backupColumns = backup.prepare("PRAGMA table_info(evolution_jobs)").all() as Array<{ name: string }>; + expect(backupColumns.map((column) => column.name)).not.toContain("failure_count"); + } finally { + backup?.close(); + if (db?.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it("reopens the current schema idempotently without recalculating counters or clearing current lease metadata", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-lease-idempotent-")); + const path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + try { + db = new MemoryDb({ path }); + db.db.prepare(`INSERT INTO evolution_jobs + (id, job_type, status, user_id, attempts, max_attempts, lease_owner, lease_token, + leased_until, failure_count, lease_count, lease_recovery_count, available_at, blocked_reason, created_at, updated_at) + VALUES ('retained', 'reflection', 'leased', 'user', 2, 3, 'owner', 'token', + '2099-01-01T00:00:00.000Z', 1, 7, 5, NULL, NULL, '2026-01-01', '2026-01-01')`).run(); + db.close(); + db = new MemoryDb({ path }); + expect(new Repositories(db.db).runtime.getJob("retained")).toMatchObject({ + status: "leased", leaseOwner: "owner", leaseToken: "token", attempts: 2, + failureCount: 1, leaseCount: 7, leaseRecoveryCount: 5 + }); + } finally { + if (db?.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +function usingDatabase(run: (runtime: Repositories["runtime"]) => void): void { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-lease-storage-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + try { run(new Repositories(db.db).runtime); } + finally { db.close(); rmSync(root, { recursive: true, force: true }); } +} + +function removeSchedulingSchema(db: MemoryDb): void { + for (const table of ["evolution_jobs", "embedding_retry_queue"]) + for (const suffix of ["deadline_insert", "deadline_update"]) db.db.exec(`DROP TRIGGER ${table}_${suffix}`); + for (const index of ["worker_main_deadline", "worker_main_lease_deadline", "worker_retry_lease_deadline"]) + db.db.exec(`DROP INDEX ${index}`); + db.db.exec("DROP TABLE worker_deadline_revision"); + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + for (const suffix of ["enqueue_sequence", "policy_dirty"]) db.db.exec(`DROP TRIGGER ${table}_${suffix}`); + db.db.exec(`DROP INDEX ${pendingPolicyIndex(table)}`); + db.db.exec(`DROP INDEX ${pendingPolicyIndex(table, true)}`); + for (const suffix of ["enqueue_seq", "lane_due", "lane_fifo", "policy_dirty"]) db.db.exec(`DROP INDEX ${table}_${suffix}`); + } + for (const suffix of ["dirty", "insert", "delete"]) db.db.exec(`DROP TRIGGER memories_worker_policy_${suffix}`); + for (const table of ["evolution_jobs", "embedding_retry_queue"]) { + for (const column of ["queue_lane", "resource_class", "enqueue_seq", "first_queued_at", "base_priority", "policy_version", "policy_promotion"]) { + db.db.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); + } + } + db.db.exec("DROP TABLE worker_enqueue_counter"); +} diff --git a/Memory/tests/repository/worker-wake-deadlines.test.ts b/Memory/tests/repository/worker-wake-deadlines.test.ts new file mode 100644 index 000000000..432e6b3c4 --- /dev/null +++ b/Memory/tests/repository/worker-wake-deadlines.test.ts @@ -0,0 +1,145 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Database from "better-sqlite3"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { MAIN_DEADLINE, MAIN_TIMED, MAIN_LEASED, RETRY_DEADLINE, RETRY_LEASED } from "../../src/storage/queue-deadline-schema.js"; +import { Repositories } from "../../src/storage/repositories.js"; + +const at = "2026-10-22T00:00:00.000Z", now = Date.parse(at); +const resources: Array<{ root: string; db: MemoryDb }> = []; +function setup() { + const root = mkdtempSync(join(tmpdir(), "memmy-wake-deadline-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const runtime = new Repositories(db.db).runtime; + const iso = (offset: number) => new Date(now + offset).toISOString(); + function job(id: string, delay: number, target = id) { + runtime.enqueueJob({ id, jobType: "trace_summary", targetMemoryId: target, userId: "user", + payload: { runAfter: iso(delay) }, status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + function retry(id: string, delay: number, target = id) { + runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: target, + vectorField: "vec_summary", sourceText: "synthetic", now: now + delay }); + } + return { root, db, runtime, iso, job, retry }; +} +afterEach(() => { + for (const { root, db } of resources.splice(0)) { db.close(); rmSync(root, { recursive: true, force: true }); } +}); + +describe("future worker wake deadlines", () => { + it.each(["owner", "lease"])("uses retry backoff when an in-progress row has no %s", missing => { + const h = setup(); h.retry("orphan", 2000); + h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by=?,lease_until=? WHERE id='orphan'") + .run(missing === "owner" ? null : "old-worker", missing === "lease" ? null : now + 10000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 2000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now + 2000 })).toBeUndefined(); + expect(h.runtime.claimEmbeddingRetryById("orphan", { now: now + 2000, workerId: "new", leaseUntil: now + 30000 })) + .toMatchObject({ claimedBy: "new", leaseToken: expect.any(String) }); + }); + + it("upgrades the earlier lease-only index when reopening an existing database", () => { + const h = setup(); h.retry("orphan", 2000); + h.db.db.exec("DROP INDEX worker_retry_lease_deadline; CREATE INDEX worker_retry_lease_deadline ON embedding_retry_queue(MAX(next_attempt_at, lease_until)) WHERE status='in_progress' AND lease_until IS NOT NULL"); + h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by=NULL,lease_until=NULL WHERE id='orphan'").run(); + h.db.close(); + const reopened = new MemoryDb({ path: join(h.root, "memory.sqlite") }); + try { + expect(new Repositories(reopened.db).runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 2000); + expect((reopened.db.prepare("SELECT sql FROM sqlite_master WHERE name='worker_retry_lease_deadline'").get() as { sql: string }).sql) + .toContain(RETRY_DEADLINE); + } finally { reopened.close(); } + }); + + it("preserves tokens on reopen and observes writes from another connection", () => { + const h = setup(); h.job("timed", 5000); + const before = h.runtime.scheduling.getDeadlineRevision(); + const other = new Database(join(h.root, "memory.sqlite")); + try { + other.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='timed'").run(h.iso(8000)); + } finally { other.close(); } + const changed = h.runtime.scheduling.getDeadlineRevision(); + expect(changed.rescanRevision).toBe(before.rescanRevision + 1); + h.db.close(); + const reopened = new MemoryDb({ path: join(h.root, "memory.sqlite") }); + try { + const runtime = new Repositories(reopened.db).runtime; + expect(runtime.scheduling.getDeadlineRevision()).toEqual(changed); + expect(runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 8000); + } finally { reopened.close(); } + }); + it("uses ordered partial indexes rather than scanning ordinary queued rows", () => { + const h = setup(); h.job("timed", 5000); + const queries = [ + [`SELECT ${MAIN_DEADLINE} FROM evolution_jobs INDEXED BY worker_main_deadline WHERE ${MAIN_TIMED} AND ${MAIN_DEADLINE} > ? ORDER BY ${MAIN_DEADLINE} LIMIT 1`, at, "worker_main_deadline"], + [`SELECT leased_until FROM evolution_jobs INDEXED BY worker_main_lease_deadline WHERE ${MAIN_LEASED} AND leased_until > ? ORDER BY leased_until LIMIT 1`, at, "worker_main_lease_deadline"], + [`SELECT ${RETRY_DEADLINE} FROM embedding_retry_queue INDEXED BY worker_retry_lease_deadline WHERE ${RETRY_LEASED} AND ${RETRY_DEADLINE} > ? ORDER BY ${RETRY_DEADLINE} LIMIT 1`, now, "worker_retry_lease_deadline"] + ] as const; + for (const [sql, param, index] of queries) { + const plan = h.db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(param) as Array<{ detail: string }>; + expect(plan.some(row => row.detail.includes(index) && row.detail.includes("SEARCH"))).toBe(true); + expect(plan.some(row => row.detail.includes("TEMP B-TREE"))).toBe(false); + } + }); + + it("tracks raw timing/filter edits transactionally but ignores ordinary completion", () => { + const h = setup(), revision = () => h.runtime.scheduling.getDeadlineRevision(); + const initial = revision(); h.job("timed", 5000); + const inserted = revision(); + expect(inserted.revision).toBeGreaterThan(initial.revision); + expect(inserted.rescanRevision).toBe(initial.rescanRevision); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='timed'").run(h.iso(8000)); + const changed = revision(); expect(changed.rescanRevision).toBeGreaterThan(inserted.rescanRevision); + expect(() => h.db.db.transaction(() => { + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='timed'").run(at); + throw new Error("rollback"); + })()).toThrow("rollback"); + expect(revision()).toEqual(changed); + h.db.db.prepare("UPDATE evolution_jobs SET status='succeeded',available_at=NULL WHERE id='timed'").run(); + expect(revision()).toEqual(changed); + h.retry("retry", 5000); const retryBefore = revision(); + h.db.db.prepare("UPDATE embedding_retry_queue SET next_attempt_at=? WHERE id='retry'").run(now + 2000); + expect(revision().rescanRevision).toBeGreaterThan(retryBefore.rescanRevision); + }); + + it("ignores due blocked heads without hiding a later scheduled job", () => { + const h = setup(); h.job("already-due", -1); h.job("later", 5_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 5_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now + 5_000 })).toBeUndefined(); + }); + it("uses the later of failure backoff and producer runAfter", () => { + const h = setup(); h.job("delayed", 5_000); + h.db.db.prepare("UPDATE evolution_jobs SET status='failed',available_at=? WHERE id='delayed'").run(h.iso(10_000)); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 10_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now + 6_000 })).toBe(now + 10_000); + }); + it("honors target, type and configuration filters", () => { + const h = setup(); h.job("other-target", 1_000, "other"); h.job("wanted", 5_000, "wanted"); + h.job("configuration", 500, "wanted"); + h.db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='configuration'").run(); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now, targetMemoryIds: ["wanted"] })).toBe(now + 5_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now, targetMemoryIds: [] })).toBeUndefined(); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now, jobTypes: [], includeEmbeddingRetries: false })).toBeUndefined(); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now, excludedJobTypes: ["trace_summary"] })).toBeUndefined(); + }); + it("includes a future expired-lease opportunity rather than an already expired lease", () => { + const h = setup(); h.job("expired", -2_000); h.job("leased", -2_000); + h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='expired'").run(h.iso(-1_000)); + h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='leased'").run(h.iso(4_000)); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 4_000); + }); + it("merges future retry deadlines with main jobs and preserves target filtering", () => { + const h = setup(); h.retry("due-retry", -1); h.retry("later-retry", 3_000, "wanted"); + h.retry("other-retry", 1_000); h.job("main", 5_000, "wanted"); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now, targetMemoryIds: ["wanted"] })).toBe(now + 3_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now, targetMemoryIds: ["wanted"], includeEmbeddingRetries: false })).toBe(now + 5_000); + }); + it("waits for both retry backoff and an in-progress lease to expire", () => { + const h = setup(); h.retry("in-progress", 2_000); + h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by='worker-A',lease_until=? WHERE id='in-progress'").run(now + 6_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now })).toBe(now + 6_000); + expect(h.runtime.nextWorkerRunAt({ futureAfterMs: now + 6_000 })).toBeUndefined(); + }); +}); diff --git a/Memory/tests/scan-consolidation.test.ts b/Memory/tests/scan-consolidation.test.ts index 50aedb24a..091b817c3 100644 --- a/Memory/tests/scan-consolidation.test.ts +++ b/Memory/tests/scan-consolidation.test.ts @@ -1,3 +1,4 @@ +import { withUnrestrictedScanPermit } from "./fixtures/scan-admission-fixture.js"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -28,7 +29,7 @@ function adapter(messages = turn("one")): SourceAdapter { return { descriptor: { sourceId: "fixture", displayName: "Fixture", dataPath: "/fixture", builtin: true }, detect: async () => true, async *scan() { yield* messages; } }; } function service(overrides: Record = {}): MemoryService { - return { addMemory: vi.fn(() => ({ id: "memory-one", duplicate: false })), enqueuePendingImportSummaries: vi.fn(), ...overrides } as unknown as MemoryService; + return { withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory: vi.fn(() => ({ id: "memory-one", duplicate: false })), enqueuePendingImportSummaries: vi.fn(), ...overrides } as unknown as MemoryService; } function executor(directory: string, source = adapter(), memory = service(), extra = {}) { const result = createAgentSourceExecutor({ service: memory, sourceRegistry: createSourceRegistry([source]), configPath: join(directory, "config.yaml"), statePath: join(directory, "state.json"), scanStoreDirectory: join(directory, "scans"), resolveAgentSkillRoot: () => null, ...extra }); diff --git a/Memory/tests/scan-parity-regression.test.ts b/Memory/tests/scan-parity-regression.test.ts index 64386482e..2cbc051b6 100644 --- a/Memory/tests/scan-parity-regression.test.ts +++ b/Memory/tests/scan-parity-regression.test.ts @@ -1,3 +1,4 @@ +import { withUnrestrictedScanPermit } from "./fixtures/scan-admission-fixture.js"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -34,7 +35,7 @@ async function run(scan: ReturnType, sourceId = "fixture", mode: return scan.scanStatus(); } const state = (root: string) => JSON.parse(readFileSync(join(root, "state.json"), "utf8")); -const stub = (overrides = {}) => ({ addMemory: vi.fn((input: { requestId: string }) => ({ id: input.requestId, duplicate: false })), +const stub = (overrides = {}) => ({ withAgentSourceScanPermit: withUnrestrictedScanPermit, addMemory: vi.fn((input: { requestId: string }) => ({ id: input.requestId, duplicate: false })), enqueuePendingImportSummaries: vi.fn(), ...overrides }) as unknown as MemoryService; describe("backend scan behavior preserved in Memory", () => { diff --git a/Memory/tests/scan-pause-finalization.test.ts b/Memory/tests/scan-pause-finalization.test.ts index 916d86f24..124e59f26 100644 --- a/Memory/tests/scan-pause-finalization.test.ts +++ b/Memory/tests/scan-pause-finalization.test.ts @@ -1,3 +1,4 @@ +import { withUnrestrictedScanPermit } from "./fixtures/scan-admission-fixture.js"; import { mkdtempSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -22,7 +23,7 @@ it.each(["resume", "cancel"] as const)("keeps a pause during source publication publication.wait = new Promise(resolve => { release = resolve; }); const entered = new Promise(resolve => { publication.entered = resolve; }); const executor = createAgentSourceExecutor({ configPath: join(directory, "config.yaml"), statePath: publication.path, - service: {} as MemoryService, resolveAgentSkillRoot: () => null, + service: { withAgentSourceScanPermit: withUnrestrictedScanPermit } as MemoryService, resolveAgentSkillRoot: () => null, sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "fixture", displayName: "Fixture", builtin: true, dataPath: directory }, detect: async () => true, async *scan() {} }]) }); try { diff --git a/Memory/tests/scan-state-version.test.ts b/Memory/tests/scan-state-version.test.ts index 6f73e77e3..5f5766047 100644 --- a/Memory/tests/scan-state-version.test.ts +++ b/Memory/tests/scan-state-version.test.ts @@ -1,3 +1,4 @@ +import { withUnrestrictedScanPermit } from "./fixtures/scan-admission-fixture.js"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -18,7 +19,7 @@ it("orders pause/resume/cancel, keeps snapshots immutable and rejects controls f const directory = root(); let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); const detect = vi.fn(async () => { await gate; return true; }); - const executor = createAgentSourceExecutor({ configPath: join(directory, "config.yaml"), service: {} as MemoryService, + const executor = createAgentSourceExecutor({ configPath: join(directory, "config.yaml"), service: { withAgentSourceScanPermit: withUnrestrictedScanPermit } as MemoryService, sourceRegistry: createSourceRegistry([{ descriptor: { sourceId: "fixture", displayName: "Fixture", builtin: true, dataPath: directory }, detect, async *scan() {} }]), resolveAgentSkillRoot: () => null }); try { const idle = executor.scanStatus(); expect(idle.status).toBe("idle"); diff --git a/Memory/tests/scan-worker-admission.test.ts b/Memory/tests/scan-worker-admission.test.ts new file mode 100644 index 000000000..af7a7ce62 --- /dev/null +++ b/Memory/tests/scan-worker-admission.test.ts @@ -0,0 +1,286 @@ +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createAgentSourceExecutor, type AgentSourceExecutor } from "../src/agent-source/runtime.js"; +import { createSourceRegistry } from "../src/agent-source/adapters/source-registry.js"; +import { createMemoryHttpServer, closeMemoryHttpServer } from "../src/server/http.js"; +import { createMemoryServiceFixture } from "./fixtures/memory-service-fixture.js"; +import { WorkerConcurrencyController } from "../src/service/worker/worker-concurrency.js"; + +const fixture = createMemoryServiceFixture(); +const releases: Array<() => void> = []; +const executors: AgentSourceExecutor[] = []; +const closeServers: Array<() => Promise> = []; +afterEach(async () => { + releases.splice(0).forEach(release => release()); + await Promise.all(closeServers.splice(0).map(close => close())); + await Promise.all(executors.splice(0).map(executor => executor.dispose())); + vi.restoreAllMocks(); + fixture.cleanup(); +}); + +function deferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + releases.push(resolve); + return { promise, resolve }; +} + +async function harness(run: (source: string, call: number) => Promise, + concurrency = new WorkerConcurrencyController({}, {})) { + const { service, root } = fixture.createTestService({ concurrency }); + let active = 0, peak = 0; + const calls: string[] = []; + const createExecutor = (name = "primary") => { + const created = createAgentSourceExecutor({ service, + configPath: join(root, name, "config.yaml"), resolveAgentSkillRoot: () => null, + sourceRegistry: createSourceRegistry(["fixture-one", "fixture-two"].map(sourceId => ({ + descriptor: { sourceId, displayName: sourceId, builtin: true, dataPath: root }, + async detect() { return true; }, + async *scan() { + active++; peak = Math.max(peak, active); calls.push(sourceId); + try { await run(sourceId, calls.length); } finally { active--; } + } + }))) }); + executors.push(created); + return created; + }; + const executor = createExecutor(); + const server = createMemoryHttpServer({ service, agentSourceExecutor: executor, + auth: { mode: "dev", localServiceToken: "isolated-scan-test-token" } }); + closeServers.push(() => closeMemoryHttpServer(server)); + await new Promise((resolve, reject) => { + server.once("error", reject); server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const post = (entry: "local" | "viewer", suffix = "", body: unknown = { sourceId: "fixture-one", mode: "full" }) => + fetch(`http://127.0.0.1:${address.port}/api/v1/agent-sources/scan${suffix}`, { + method: "POST", headers: { "content-type": "application/json", + ...(entry === "local" ? { authorization: "Bearer isolated-scan-test-token" } : { "x-memmy-viewer": "1" }) }, + body: JSON.stringify(body) + }); + const done = async () => { + await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false)); + return executor.scanStatus(); + }; + return { service, concurrency, executor, createExecutor, post, done, calls, peak: () => peak, active: () => active }; +} + +describe("Memory-owned scan admission after Desktop scanner consolidation", () => { + it.each([["all", "fixture-one"], ["fixture-one", "all"]] as const)( + "reserves overlapping sources atomically across executors: %s then %s", async (firstSource, secondSource) => { + const entered = deferred(), finish = deferred(); + const h = await harness(async (_source, call) => { if (call === 1) { entered.resolve(); await finish.promise; } }, + new WorkerConcurrencyController({ resourceLimits: { scan: 2 } }, {})); + const second = h.createExecutor("secondary"); + await h.executor.startScan({ sourceId: firstSource, mode: "full" }); await entered.promise; + await second.startScan({ sourceId: secondSource, mode: "full" }); + await new Promise(resolve => setTimeout(resolve, 25)); + expect(h.calls).toEqual(["fixture-one"]); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 1, resources: { scan: 1 }, providers: {} }); + expect(h.concurrency.snapshot().scopeKeys.sort()).toEqual(firstSource === "all" + ? ["agent-source-scan:fixture-one", "agent-source-scan:fixture-two"] : ["agent-source-scan:fixture-one"]); + finish.resolve(); + await vi.waitFor(() => expect(second.scanStatus().running).toBe(false)); + await h.done(); + expect(second.scanStatus().error).toBeNull(); + expect(h.calls).toEqual(firstSource === "all" + ? ["fixture-one", "fixture-two", "fixture-one"] : ["fixture-one", "fixture-one", "fixture-two"]); + expect(h.peak()).toBe(1); + await vi.waitFor(() => expect(h.concurrency.snapshot().activeJobs).toBe(0)); + expect(h.concurrency.snapshot().scopeKeys).toEqual([]); + } + ); + + it("admits different sources from different executors when scan capacity is two", async () => { + const finish = deferred(); + const h = await harness(async () => { await finish.promise; }, + new WorkerConcurrencyController({ resourceLimits: { scan: 2 } }, {})); + const second = h.createExecutor("secondary"); + await h.executor.startScan({ sourceId: "fixture-one", mode: "full" }); + await second.startScan({ sourceId: "fixture-two", mode: "full" }); + await vi.waitFor(() => expect(h.calls).toHaveLength(2)); + expect(h.peak()).toBe(2); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 2, resources: { scan: 2 }, providers: {} }); + finish.resolve(); await h.done(); + await vi.waitFor(() => expect(second.scanStatus().running).toBe(false)); + await vi.waitFor(() => expect(h.concurrency.snapshot().activeJobs).toBe(0)); + }); + + it("cancels an overlapping scan-all waiter without releasing the holder, then admits a fresh scan after release", async () => { + const entered = deferred(), finish = deferred(); + const h = await harness(async (_source, call) => { if (call === 1) { entered.resolve(); await finish.promise; } }, + new WorkerConcurrencyController({ resourceLimits: { scan: 2 } }, {})); + const second = h.createExecutor("secondary"); + await h.executor.startScan({ sourceId: "fixture-one", mode: "full" }); await entered.promise; + await second.startScan({ sourceId: "all", mode: "full" }); + await new Promise(resolve => setImmediate(resolve)); + await second.cancelScan(); + expect(h.calls).toEqual(["fixture-one"]); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 1, resources: { scan: 1 }, + scopeKeys: ["agent-source-scan:fixture-one"] }); + finish.resolve(); await h.done(); + await second.startScan({ sourceId: "all", mode: "full" }); + await vi.waitFor(() => expect(second.scanStatus().running).toBe(false)); + expect(second.scanStatus().error).toBeNull(); + expect(h.calls).toEqual(["fixture-one", "fixture-one", "fixture-two"]); + await vi.waitFor(() => expect(h.concurrency.snapshot().activeJobs).toBe(0)); + expect(h.concurrency.snapshot().scopeKeys).toEqual([]); + }); + + it("counts a scan in the shared background and global budgets while retaining the interactive slot", async () => { + const entered = deferred(), finish = deferred(); + const h = await harness(async () => { entered.resolve(); await finish.promise; }); + expect((await h.post("local")).status).toBe(202); await entered.promise; + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 1, resources: { scan: 1 } }); + const held = Array.from({ length: 2 }, (_, index) => h.concurrency.tryAcquire({ + resourceClass: "summary", queueLane: "import", providers: [], scopeKeys: [`worker-${index}`] + })); + let interactive: ReturnType; + try { + expect(held.every(Boolean)).toBe(true); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, availableJobs: 1 }); + expect(h.concurrency.tryAcquire({ resourceClass: "summary", queueLane: "import", providers: [], scopeKeys: [] })).toBeUndefined(); + interactive = h.concurrency.tryAcquire({ resourceClass: "summary", queueLane: "interactive", providers: [], scopeKeys: [] }); + expect(interactive).toBeDefined(); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3, availableJobs: 0 }); + expect(h.concurrency.tryAcquire({ resourceClass: "summary", queueLane: "interactive", providers: [], scopeKeys: [] })).toBeUndefined(); + } finally { + interactive?.release(); held.forEach(permit => permit?.release()); + finish.resolve(); await h.done(); + } + await vi.waitFor(() => expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0 })); + }); + + it("keeps a scan waiting behind three background jobs, admits interactive work, and starts the scan when a background slot opens", async () => { + const concurrency = new WorkerConcurrencyController({}, {}), entered = deferred(), finish = deferred(); + const held = Array.from({ length: 3 }, (_, index) => concurrency.tryAcquire({ + resourceClass: "summary", queueLane: "import", providers: [], scopeKeys: [`worker-${index}`] + })!); + const h = await harness(async () => { entered.resolve(); await finish.promise; }, concurrency); + let interactive: ReturnType; + try { + expect(held.every(Boolean)).toBe(true); + expect((await h.post("viewer")).status).toBe(202); + await new Promise(resolve => setImmediate(resolve)); + expect(h.calls).toEqual([]); + expect(h.active()).toBe(0); + expect(concurrency.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, availableJobs: 1 }); + expect(concurrency.snapshot().scopeKeys).not.toContain("agent-source-scan:fixture-one"); + interactive = concurrency.tryAcquire({ resourceClass: "summary", queueLane: "interactive", providers: [], scopeKeys: ["fresh"] }); + expect(interactive).toBeDefined(); + expect(concurrency.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3 }); + expect(h.calls).toEqual([]); + held[0]!.release(); + await entered.promise; + expect(h.calls).toEqual(["fixture-one"]); + expect(concurrency.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3, resources: { summary: 3, scan: 1 } }); + expect(concurrency.snapshot().scopeKeys).toContain("fresh"); + expect(concurrency.snapshot().scopeKeys).toContain("agent-source-scan:fixture-one"); + } finally { + interactive?.release(); held.forEach(permit => permit?.release()); + finish.resolve(); await h.done(); + } + await vi.waitFor(() => expect(concurrency.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0, scopeKeys: [] })); + }); + + it("cancels a scan waiting for global capacity without starting its adapter or releasing another task", async () => { + const concurrency = new WorkerConcurrencyController({ maxActiveJobs: 1 }, {}); + const occupied = concurrency.tryAcquire({ resourceClass: "summary", providers: [], scopeKeys: [] })!; + const h = await harness(async () => {}, concurrency); + expect((await h.post("viewer")).status).toBe(202); + await new Promise(resolve => setImmediate(resolve)); + expect(h.calls).toEqual([]); + await h.executor.cancelScan(); + expect(h.calls).toEqual([]); + expect(concurrency.snapshot().activeJobs).toBe(1); + occupied.release(); + expect((await h.post("local")).status).toBe(202); + expect((await h.done()).error).toBeNull(); + expect(h.calls).toEqual(["fixture-one"]); + await vi.waitFor(() => expect(concurrency.snapshot().activeJobs).toBe(0)); + }); + + it.each([ + ["local", "viewer", "fixture-one"], ["viewer", "local", "all"], + ["local", "viewer", "fixture-two"] + ] as const)("shares the executor between %s and %s starts (second source %s)", async (first, second, sourceId) => { + const entered = deferred(), finish = deferred(); + const h = await harness(async () => { entered.resolve(); await finish.promise; }); + expect((await h.post(first)).status).toBe(202); + await entered.promise; + expect((await h.post(second, "", { sourceId, mode: "full" })).status).toBe(409); + expect(h.calls).toEqual(["fixture-one"]); + expect(h.peak()).toBe(1); + finish.resolve(); + expect((await h.done()).error).toBeNull(); + }); + + it("retains admission while cancellation waits for the old writer, then permits a replacement", async () => { + const entered = deferred(), finish = deferred(); + const h = await harness(async (_source, call) => { if (call === 1) { entered.resolve(); await finish.promise; } }); + expect((await h.post("viewer")).status).toBe(202); await entered.promise; + const cancellation = h.executor.cancelScan(); + const denied = await h.post("local"); + expect(denied.status).toBe(409); + expect(await denied.text()).toContain("Canceling"); + expect(h.calls).toHaveLength(1); + expect(h.active()).toBe(1); + finish.resolve(); await cancellation; + expect((await h.post("local")).status).toBe(202); + expect((await h.done()).error).toBeNull(); + expect(h.calls).toHaveLength(2); + expect(h.peak()).toBe(1); + }); + + it("keeps a paused scan exclusive and resumes it without opening a second adapter", async () => { + const entered = deferred(), finish = deferred(); + const h = await harness(async () => { entered.resolve(); await finish.promise; }); + expect((await h.post("local")).status).toBe(202); await entered.promise; + expect((await h.post("viewer", "/stop", {})).status).toBe(200); + expect(h.executor.scanStatus()).toMatchObject({ running: false, progress: { phase: "stopped" } }); + expect((await h.post("local", "", { sourceId: "fixture-two", mode: "full" })).status).toBe(409); + expect((await h.post("viewer")).status).toBe(202); + expect(h.calls).toHaveLength(1); + finish.resolve(); + expect((await h.done()).error).toBeNull(); + expect(h.peak()).toBe(1); + }); + + it("releases admission after adapter failure so the next API scan can succeed", async () => { + const h = await harness(async (_source, call) => { if (call === 1) throw new Error("isolated adapter failure"); }); + expect((await h.post("local")).status).toBe(202); + expect((await h.done()).error).toContain("isolated adapter failure"); + expect(h.active()).toBe(0); + expect((await h.post("viewer")).status).toBe(202); + expect((await h.done()).error).toBeNull(); + expect(h.calls).toHaveLength(2); + expect(h.peak()).toBe(1); + }); + + it("serializes sources within scan-all and rejects another scan while its first source is running", async () => { + const entered = deferred(), finish = deferred(); + const h = await harness(async (source, call) => { if (source === "fixture-one" && call === 1) { entered.resolve(); await finish.promise; } }); + expect((await h.post("viewer", "", { sourceId: "all", mode: "full" })).status).toBe(202); + await entered.promise; + expect((await h.post("local", "", { sourceId: "fixture-two", mode: "full" })).status).toBe(409); + expect(h.calls).toEqual(["fixture-one"]); + finish.resolve(); expect((await h.done()).error).toBeNull(); + expect(h.calls).toEqual(["fixture-one", "fixture-two"]); + expect(h.peak()).toBe(1); + }); + + it("waits for its active adapter on disposal and refuses starts during shutdown", async () => { + const entered = deferred(), finish = deferred(); + const h = await harness(async () => { entered.resolve(); await finish.promise; }); + expect((await h.post("local")).status).toBe(202); await entered.promise; + let disposed = false; + const disposal = Promise.resolve(h.executor.dispose()).then(() => { disposed = true; }); + const denied = await h.post("viewer"); + expect(denied.status).toBe(409); + expect(await denied.text()).toContain("shutting down"); + expect(disposed).toBe(false); + finish.resolve(); await disposal; + expect(h.active()).toBe(0); + }); +}); diff --git a/Memory/tests/service/embedding/embedding-processing.test.ts b/Memory/tests/service/embedding/embedding-processing.test.ts index d38e8e6b7..8b524549d 100644 --- a/Memory/tests/service/embedding/embedding-processing.test.ts +++ b/Memory/tests/service/embedding/embedding-processing.test.ts @@ -18,6 +18,7 @@ import { import { ModelHttpError } from "../../../src/model/http.js"; import { createEmbedder } from "../../../src/model/embedder.js"; import { Repositories } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; import { addAgentSourceImport, createBatchReflectionLlm, @@ -33,8 +34,9 @@ const { createTestService } = createMemoryServiceFixture(); -afterEach(() => { - cleanup(); +afterEach(async () => { + await cleanup(); + vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -46,12 +48,18 @@ describe("MemoryService / embedding / processing", () => { ["insufficient_quota", "Insufficient balance.", false], ["rate_limit_exceeded", "Rate limit exceeded.", true] ] as const)(`${queue} queue does not fan out shared HTTP 400 %s (${format})`, async (code, message, transient) => { - const fetchMock = vi.fn(async () => new Response(JSON.stringify({ error: { - code: format === "code" ? code : undefined, - message: format === "message" ? message : "Request rejected" - } }), { status: 400 })); + const inputs: string[][] = []; + const fetchMock = vi.fn(async (_url: unknown, init?: RequestInit) => { + inputs.push((JSON.parse(String(init?.body)) as { input: string[] }).input); + return new Response(JSON.stringify({ error: { + code: format === "code" ? code : undefined, + message: format === "message" ? message : "Request rejected" + } }), { status: 400 }); + }); vi.stubGlobal("fetch", fetchMock); const { db, service } = createTestService({ + // Four background members fit one request while the realtime slot remains reserved. + concurrency: new WorkerConcurrencyController({ maxActiveJobs: 5 }), embedder: createHttpTestEmbedder(), llm: createBatchReflectionLlm([], "Synthetic summary.") }); @@ -61,8 +69,9 @@ describe("MemoryService / embedding / processing", () => { const ids = await Promise.all(["GOOD_A", "GOOD_B", "GOOD_C"].map(async (answer, i) => (await service.completeTurn(`turn-${i}`, { sessionId: session.sessionId, query: "Remember this.", answer })).l1MemoryId)); - await service.runWorkerOnce(20, { priorityCohortOnly: true }); - await service.runWorkerOnce(20, { priorityCohortOnly: true }); + await prepareFreshEmbeddingBatch(service, ids); + expect(fetchMock).not.toHaveBeenCalled(); + await service.runWorkerOnce(ids.length, { targetMemoryIds: ids }); for (const id of ids) expect(repos.processing.get(id)).toMatchObject({ state: transient ? "embedding_pending" : "failed", errorCode: transient ? "transient_provider_error" : "model_configuration", @@ -78,11 +87,13 @@ describe("MemoryService / embedding / processing", () => { await service.runWorkerOnce(10); for (const retry of retries) { const stored = repos.runtime.getEmbeddingRetry(retry.id)!; - expect(stored).toMatchObject({ status: transient ? "pending" : "failed", attempts: 1 }); + expect(stored).toMatchObject({ status: transient ? "pending" : "failed", attempts: transient ? 1 : 0 }); if (transient) expect(stored.nextAttemptAt).toBeGreaterThan(Date.now()); + else expect(stored.blockedReason).toBe("configuration"); } } expect(fetchMock).toHaveBeenCalledTimes(1); + expect(inputs.map(input => input.length)).toEqual([3]); expect(db.db.prepare("SELECT count(*) AS n FROM memory_vector_entries").get()).toEqual({ n: 0 }); }); } @@ -91,8 +102,10 @@ describe("MemoryService / embedding / processing", () => { ["context_length_exceeded", "maximum context length exceeded"], ["20015", "The parameter is invalid."] ])(`${queue} queue preserves valid siblings through the HTTP adapter (%s)`, async (code, message) => { - const fetchMock = vi.fn(async (_url, init) => { + const inputs: string[][] = []; + const fetchMock = vi.fn(async (_url: unknown, init?: RequestInit) => { const { input } = JSON.parse(String(init?.body)) as { input: string[] }; + inputs.push(input); const bad = input.some(text => text.includes("BAD_EMBEDDING_ITEM")); return new Response(JSON.stringify(bad ? { code, message } : { data: input.map(() => ({ embedding: [1, 0, 0] })) @@ -100,6 +113,7 @@ describe("MemoryService / embedding / processing", () => { }); vi.stubGlobal("fetch", fetchMock); const { db, service } = createTestService({ embedder: createHttpTestEmbedder(), + concurrency: new WorkerConcurrencyController({ maxActiveJobs: 5 }), llm: createBatchReflectionLlm([], "Synthetic summary.") }); const repos = new Repositories(db.db); const texts = ["GOOD_A", "GOOD_B", "GOOD_C", "BAD_EMBEDDING_ITEM"]; @@ -108,8 +122,9 @@ describe("MemoryService / embedding / processing", () => { const ids = await Promise.all(texts.map(async (answer, i) => (await service.completeTurn(`turn-${i}`, { sessionId: session.sessionId, query: "Remember this.", answer })).l1MemoryId)); - await service.runWorkerOnce(20, { priorityCohortOnly: true }); - await service.runWorkerOnce(20, { priorityCohortOnly: true }); + await prepareFreshEmbeddingBatch(service, ids); + expect(fetchMock).not.toHaveBeenCalled(); + await service.runWorkerOnce(ids.length, { targetMemoryIds: ids }); expect(ids.map(id => repos.processing.get(id)?.state)).toEqual([ "ready", "ready", "ready", code === "context_length_exceeded" ? "ready_text_only" : "failed" ]); @@ -127,6 +142,8 @@ describe("MemoryService / embedding / processing", () => { } expect(db.db.prepare("SELECT count(*) AS n FROM memory_vector_entries").get()).toEqual({ n: 3 }); expect(fetchMock).toHaveBeenCalledTimes(5); + expect(inputs.map(input => input.length)).toEqual([4, 1, 1, 1, 1]); + for (const marker of texts) expect(inputs[0]!.filter(text => text.includes(marker))).toHaveLength(1); await service.runWorkerOnce(20, { priorityCohortOnly: true }); expect(fetchMock).toHaveBeenCalledTimes(5); }); @@ -200,6 +217,7 @@ describe("MemoryService / embedding / processing", () => { }); it("retries trace embedding jobs without leaving the processing state stuck", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); const root = createTestRoot("mindock-memory-embedding-retry-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") @@ -227,9 +245,11 @@ describe("MemoryService / embedding / processing", () => { .prepare(`SELECT version FROM memories WHERE id = ?`) .get(complete.l1MemoryId) as { version: number }; - const summaryRun = await service.runWorkerOnce(20, { priorityCohortOnly: true }); + // Use an explicit one-job budget to inspect the intermediate state before + // continuous admission can immediately start the generated embedding. + const summaryRun = await service.runWorkerOnce(1, { targetMemoryIds: [complete.l1MemoryId] }); expect(summaryRun.jobs.some((job) => job.jobType === "trace_summary" && job.status === "succeeded")).toBe(true); - const firstRun = await service.runWorkerOnce(20, { priorityCohortOnly: true }); + const firstRun = await service.runWorkerOnce(1, { targetMemoryIds: [complete.l1MemoryId] }); const failedEmbedding = firstRun.jobs.find((job) => job.jobType === "embedding" && job.status === "failed"); expect(failedEmbedding?.jobId).toBeTruthy(); const failedJob = db.db.prepare( @@ -257,6 +277,9 @@ describe("MemoryService / embedding / processing", () => { attemptCount: 1 }); + const retryAt = new Repositories(db.db).runtime.getJob(failedJob.id)!.availableAt!; + expect(Date.parse(retryAt)).toBeGreaterThan(Date.now()); + vi.setSystemTime(Date.parse(retryAt)); const secondRun = await service.runWorkerOnce(20, { priorityCohortOnly: true }); expect(secondRun.jobs).toEqual(expect.arrayContaining([ expect.objectContaining({ jobId: failedJob.id, jobType: "embedding", status: "succeeded" }) @@ -279,7 +302,10 @@ describe("MemoryService / embedding / processing", () => { .prepare(`SELECT version FROM memories WHERE id = ?`) .get(complete.l1MemoryId) as { version: number }; const reindexRun = await service.runWorkerOnce(20, { priorityCohortOnly: true }); - expect(reindexRun.jobs.some((job) => job.jobType === "embedding" && job.status === "succeeded")).toBe(true); + const afterReflection = [...reflectionRun.jobs, ...reindexRun.jobs]; + expect(afterReflection.some((job) => job.jobType === "embedding" && job.status === "succeeded")).toBe(true); + expect(afterReflection.findIndex(job => job.jobType === "reflection")) + .toBeLessThan(afterReflection.findIndex(job => job.jobType === "embedding")); expect(new Repositories(db.db).processing.get(complete.l1MemoryId)?.state).toBe("ready"); const memory = db.db .prepare( @@ -329,8 +355,9 @@ describe("MemoryService / embedding / processing", () => { answer: "BAD_EMBEDDING_ITEM" })); - await service.runWorkerOnce(20, { priorityCohortOnly: true }); - const embeddingRun = await service.runWorkerOnce(20, { priorityCohortOnly: true }); + const targets = { targetMemoryIds: [good.l1MemoryId, bad.l1MemoryId] }; + await service.runWorkerOnce(2, targets); + const embeddingRun = await service.runWorkerOnce(2, targets); const repositories = new Repositories(db.db); expect(embeddingRun.jobs).toEqual(expect.arrayContaining([ @@ -347,7 +374,7 @@ describe("MemoryService / embedding / processing", () => { db.close(); }); - it("terminates a legacy embedding retry after a deterministic provider failure", async () => { + it("holds a forbidden legacy embedding retry for configuration repair without consuming an attempt", async () => { const { db, service } = createTestService({ embedder: createForbiddenEmbedder() }); const repositories = new Repositories(db.db); const memory = skillMemory(); @@ -364,11 +391,12 @@ describe("MemoryService / embedding / processing", () => { const run = await service.runWorkerOnce(10); expect(run.embeddingRetries.items).toEqual([ - expect.objectContaining({ id: retry.id, status: "failed", attempts: 1 }) + expect.objectContaining({ id: retry.id, status: "failed", attempts: 0 }) ]); expect(repositories.runtime.getEmbeddingRetry(retry.id)).toMatchObject({ status: "failed", - attempts: 1, + attempts: 0, + blockedReason: "configuration", lastError: "Access to the configured model is forbidden." }); db.close(); @@ -603,14 +631,22 @@ describe("MemoryService / embedding / processing", () => { }); expect(recall.hits.some((hit) => hit.id === complete.l1MemoryId)).toBe(false); const openEpisodeRun = await service.runWorkerOnce(10, { priorityCohortOnly: true }); - expect(openEpisodeRun.jobs.map((job) => job.jobType)).toEqual(["trace_summary", "episode_assignment", "episode_finalize"]); + expect(openEpisodeRun.jobs).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "trace_summary", targetMemoryId: complete.l1MemoryId }) + ])); expect(llmCalls.filter((call) => call.options.operation === "capture.summarize")).toHaveLength(1); + expect(llmCalls.some((call) => call.options.operation === "capture.reflection.batch.v13")).toBe(false); const embeddingRun = await service.runWorkerOnce(10, { priorityCohortOnly: true }); - expect(embeddingRun.jobs.map((job) => job.jobType)).toEqual(["embedding"]); - const episodeRun = await service.runWorkerOnce(10, { priorityCohortOnly: true }); - expect(episodeRun.jobs.map((job) => job.jobType)).toEqual(["episode_idle_close"]); - const titleRun = await service.runWorkerOnce(10, { priorityCohortOnly: true }); - expect(titleRun.jobs.map((job) => job.jobType)).toEqual([]); + // Newly produced embedding work may finish in the original drain. + const completed = [...openEpisodeRun.jobs, ...embeddingRun.jobs]; + expect(completed).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "embedding", targetMemoryId: complete.l1MemoryId }) + ])); + const reflectionIndex = completed.findIndex((job) => job.jobType === "reflection"); + const embeddingIndex = completed.findIndex((job) => job.jobType === "embedding"); + if (reflectionIndex >= 0) expect(embeddingIndex).toBeLessThan(reflectionIndex); + const auxiliaryJobs = [...openEpisodeRun.jobs, ...embeddingRun.jobs].map((job) => job.jobType); + expect(auxiliaryJobs).toEqual(expect.arrayContaining(["episode_assignment", "episode_finalize", "episode_idle_close"])); expect(embeddingTexts).toHaveLength(1); expect(db.db.prepare( `SELECT COUNT(*) AS count FROM evolution_jobs @@ -839,6 +875,18 @@ function worldModelMemory(): MemoryRow { }; } +/** Prepare all compatible embedding jobs before draining, without relying on old priority cohorts. */ +async function prepareFreshEmbeddingBatch( + service: ReturnType["service"], ids: readonly string[] +): Promise { + for (const id of ids) { + const run = await service.runWorkerOnce(1, { targetMemoryIds: [id] }); + expect(run.jobs).toEqual([expect.objectContaining({ + jobType: "trace_summary", targetMemoryId: id, status: "succeeded" + })]); + } +} + function createHttpTestEmbedder(): Embedder { return createEmbedder({ ...DEFAULT_MEMMY_CONFIG.embedding, provider: "openai_compatible", model: "custom-alias", endpoint: "https://embedding.test/v1", cache: false, maxRetries: 0 }, diff --git a/Memory/tests/service/embedding/embedding-retry-backoff.test.ts b/Memory/tests/service/embedding/embedding-retry-backoff.test.ts new file mode 100644 index 000000000..0d1500706 --- /dev/null +++ b/Memory/tests/service/embedding/embedding-retry-backoff.test.ts @@ -0,0 +1,25 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { embeddingRetryBackoffMs } from "../../../src/service/embedding/embedding-pipeline.js"; + +afterEach(() => vi.restoreAllMocks()); + +describe("Embedding retry exponential backoff", () => { + it.each([[1, 60_000], [2, 120_000], [6, 1_920_000], [7, 3_600_000], [100, 3_600_000]])( + "keeps the existing baseline and one-hour ceiling at attempt %s", (attempt, expected) => { + vi.spyOn(Math, "random").mockReturnValue(0.5); + expect(embeddingRetryBackoffMs(attempt)).toBe(expected); + } + ); + + it("spreads equal-attempt retries around the unchanged baseline", () => { + vi.spyOn(Math, "random").mockReturnValueOnce(0).mockReturnValueOnce(1); + expect(embeddingRetryBackoffMs(1)).toBe(48_000); + expect(embeddingRetryBackoffMs(1)).toBe(72_000); + }); + + it("keeps jitter below the existing cap even after the exponential baseline reaches it", () => { + vi.spyOn(Math, "random").mockReturnValueOnce(0).mockReturnValueOnce(1); + expect(embeddingRetryBackoffMs(100)).toBe(2_880_000); + expect(embeddingRetryBackoffMs(100)).toBe(3_600_000); + }); +}); diff --git a/Memory/tests/service/evidence/history-log-compaction.test.ts b/Memory/tests/service/evidence/history-log-compaction.test.ts index a6cd2f550..cfbb51033 100644 --- a/Memory/tests/service/evidence/history-log-compaction.test.ts +++ b/Memory/tests/service/evidence/history-log-compaction.test.ts @@ -6,7 +6,7 @@ import { Repositories, type ChangeLogRecord } from "../../../src/storage/reposit import type { MemoryRow } from "../../../src/types.js"; import { isRecord } from "../../../src/utils/json.js"; import { observationMemoryState } from "../../../src/service/observation-memory-state.js"; -import { createCapturingEmbedder, createMemoryServiceFixture, runWorkerRounds } from "../../fixtures/memory-service-fixture.js"; +import { createCapturingEmbedder, createMemoryServiceFixture, runWorkerRounds, runWorkerRoundsWithDueRetries } from "../../fixtures/memory-service-fixture.js"; const { createTestService, cleanup } = createMemoryServiceFixture(); afterEach(cleanup); @@ -122,7 +122,7 @@ describe("History change-log compaction", () => { const priorSemantic = historySnapshots(updatedLogs, synced.id) .find((snapshot) => observationMemoryState(snapshot)?.status === "processed")!; expect(priorSemantic.memoryValue).toBe(processed.memoryValue); - await runWorkerRounds(service, 3); + await runWorkerRoundsWithDueRetries(service, db, [synced.id], 3); expect(service.memoryProcessingStatus([synced.id]).items[0]).toMatchObject({ state: "failed", stage: "summary" }); invalid = false; expect(service.retryMemoryProcessing(synced.id, { namespace }).job?.jobType).toBe("observation_extract"); @@ -140,7 +140,7 @@ describe("History change-log compaction", () => { const first = createTestService({ llm: model((input) => ({ keep: true, title: "Ungrounded result", summary: "Unsupported completion", items: [{ category: "progress", text: "Unsupported completion", evidence_quote: "Not present in the source.", source_memory_id: input.sources[0]!.id }] })) }); const source = first.service.syncExternalEvidence(evidence); - await runWorkerRounds(first.service, 3); + await runWorkerRoundsWithDueRetries(first.service, first.db, [source.id], 3); expect(first.service.memoryProcessingStatus([source.id]).items[0]?.state).toBe("failed"); const repos = new Repositories(first.db.db), current = repos.memories.get(source.id)!; const oldChange = repos.runtime.listChanges(undefined, 1000).find((change) => change.memoryId === source.id)!; diff --git a/Memory/tests/service/evidence/observation-memory-validation.test.ts b/Memory/tests/service/evidence/observation-memory-validation.test.ts index 45a7982bd..da497b8ff 100644 --- a/Memory/tests/service/evidence/observation-memory-validation.test.ts +++ b/Memory/tests/service/evidence/observation-memory-validation.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { get_encoding } from "tiktoken"; import { DEFAULT_MEMMY_CONFIG, type LlmClient } from "../../../src/index.js"; import type { LlmCompletionOptions, LlmMessage } from "../../../src/model/types.js"; @@ -7,7 +7,7 @@ import { observationMemoryState } from "../../../src/service/observation-memory- import { createMemoryServiceFixture, runWorkerRounds } from "../../fixtures/memory-service-fixture.js"; const { cleanup, createTestService } = createMemoryServiceFixture(); -afterEach(cleanup); +afterEach(() => { cleanup(); vi.useRealTimers(); }); const namespace = { source: "memmy", profileId: "default", userId: "validation-test-user" }; const evidence = { action: "upsert" as const, source: "computer_history", namespace, @@ -124,6 +124,7 @@ describe("observation validation correction", () => { }); it("leaves two invalid responses to the existing bounded worker retry without writing invalid items", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); const { llm, calls } = model(ungrounded); const { service, db } = createTestService({ llm }); const row = service.syncExternalEvidence(evidence); @@ -133,7 +134,13 @@ describe("observation validation correction", () => { state: "summary_pending", errorCode: "processing_failed", attemptCount: 1, errorMessage: "observation extraction item is not grounded in its source (quote_grounding) [item 1]" }); - await runWorkerRounds(service, 2); + for (let attempt = 0; attempt < 2; attempt++) { + const retry = db.db.prepare(`SELECT available_at FROM evolution_jobs + WHERE target_memory_id = ? AND status = 'failed'`).get(row.id) as { available_at: string }; + expect(Date.parse(retry.available_at)).toBeGreaterThan(Date.now()); + vi.setSystemTime(Date.parse(retry.available_at)); + await service.runWorkerOnce(1); + } expect(calls).toHaveLength(6); expect(service.memoryProcessingStatus([row.id]).items[0]).toMatchObject({ state: "failed", attemptCount: 3 }); const repos = new Repositories(db.db); diff --git a/Memory/tests/service/evidence/observation-memory.test.ts b/Memory/tests/service/evidence/observation-memory.test.ts index 90c1f6427..4a6b38437 100644 --- a/Memory/tests/service/evidence/observation-memory.test.ts +++ b/Memory/tests/service/evidence/observation-memory.test.ts @@ -1,14 +1,22 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { get_encoding } from "tiktoken"; -import { DEFAULT_MEMMY_CONFIG, type LlmClient } from "../../../src/index.js"; +import { DEFAULT_MEMMY_CONFIG, type LlmClient, type MemoryDb } from "../../../src/index.js"; import type { LlmMessage, LlmCompletionOptions } from "../../../src/model/types.js"; import { Repositories } from "../../../src/storage/repositories.js"; import { observationMemoryState } from "../../../src/service/observation-memory-state.js"; import { prepareObservationExtractionInput } from "../../../src/service/evidence/observation-memory-pipeline.js"; -import { createCapturingEmbedder, createMemoryServiceFixture, runWorkerRounds } from "../../fixtures/memory-service-fixture.js"; +import { addAgentSourceImport, createCapturingEmbedder, createMemoryServiceFixture, runWorkerRounds } from "../../fixtures/memory-service-fixture.js"; const { cleanup, createTestService } = createMemoryServiceFixture(); -afterEach(cleanup); +afterEach(() => { cleanup(); vi.useRealTimers(); }); + +function advanceToMemoryRetry(db: MemoryDb, memoryId: string): void { + const retry = db.db.prepare(`SELECT available_at FROM evolution_jobs + WHERE target_memory_id = ? AND status = 'failed' AND available_at IS NOT NULL LIMIT 1`) + .get(memoryId) as { available_at: string } | undefined; + expect(retry).toBeDefined(); + vi.setSystemTime(Date.parse(retry!.available_at)); +} const namespace = { source: "memmy", profileId: "default", userId: "history-user" }; const evidence = { action: "upsert" as const, source: "computer_history", namespace, @@ -202,8 +210,9 @@ describe("History meaningful-memory processing", () => { }); it("reports a queued first attempt as waiting and only a recorded failure as an automatic retry", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); let available = false; - const { service } = createTestService({ llm: model((input) => { + const { service, db } = createTestService({ llm: model((input) => { if (!available) throw new Error("429 temporarily rate limited"); return semantic(input); }) }); @@ -214,18 +223,20 @@ describe("History meaningful-memory processing", () => { expect(service.memoryProcessingStatus([row.id]).items[0]).toMatchObject({ state: "summary_pending", errorCode: "transient_provider_error", autoRetryScheduled: true }); available = true; + expect((await service.runWorkerOnce(1)).leased).toBe(0); + advanceToMemoryRetry(db, row.id); await runWorkerRounds(service, 2); expect(service.memoryProcessingStatus([row.id]).items[0]).toMatchObject({ state: "ready", autoRetryScheduled: false }); }); it("extracts a fresh History window before an older Agent-history backlog", async () => { const { service, db } = createTestService({ llm: model() }); - const session = service.openSession({ namespace: { ...namespace, source: "cursor" } }); - const backlog = await Promise.all(Array.from({ length: 3 }, async (_value, index) => (await service.completeTurn(`cursor-backfill-${index}`, { - sessionId: session.sessionId, query: `Old Cursor question ${index}`, answer: `Old Cursor answer ${index}` - })).l1MemoryId)); - const backdate = db.db.prepare("UPDATE memories SET created_at = ? WHERE id = ?"); - backlog.forEach((id, index) => backdate.run(`2026-08-18T0${index}:00:00.000Z`, id)); + const backlog = Array.from({ length: 3 }, (_value, index) => addAgentSourceImport( + service, { ...namespace, source: "cursor" }, `Old Cursor question ${index}`, `cursor-backfill-${index}`, + `2026-08-18T0${index}:00:00.000Z` + ).id); + const backlogPolicies = db.db.prepare("SELECT queue_lane FROM evolution_jobs WHERE target_memory_id IN (?, ?, ?)").all(...backlog); + expect(backlogPolicies).toEqual(Array.from({ length: 3 }, () => ({ queue_lane: "import" }))); const history = service.syncExternalEvidence(evidence); const run = await service.runWorkerOnce(1, { priorityCohortOnly: true }); expect(run.jobs).toEqual([expect.objectContaining({ jobType: "observation_extract", targetMemoryId: history.id })]); @@ -260,13 +271,17 @@ describe("History meaningful-memory processing", () => { }); it("rejects ungrounded output, then retries through the observation processor", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); let valid = false; - const { service } = createTestService({ llm: model((input) => valid ? semantic(input) : { + const { service, db } = createTestService({ llm: model((input) => valid ? semantic(input) : { keep: true, title: "Invented result", summary: "Purchase is complete", items: [{ category: "progress", text: "The purchase is complete.", source_memory_id: input.sources[0]!.id, evidence_quote: "The purchase is complete." }] }) }); const row = service.syncExternalEvidence(evidence); - await runWorkerRounds(service, 3); + for (let attempt = 0; attempt < 3; attempt++) { + await runWorkerRounds(service, 1); + if (attempt < 2) advanceToMemoryRetry(db, row.id); + } expect(service.memoryProcessingStatus([row.id]).items[0]).toMatchObject({ state: "failed", stage: "summary" }); valid = true; expect(service.retryMemoryProcessing(row.id, { namespace }).job?.jobType).toBe("observation_extract"); @@ -362,12 +377,19 @@ describe("History meaningful-memory processing", () => { const oldEmbedding = service.runWorkerOnce(1); await started; service.syncExternalEvidence({ ...evidence, revision: "2026-10-09T10:12:00.000Z", content: `${evidence.content} Extra window switch.` }); - await service.runWorkerOnce(1); + const newerSummary = service.runWorkerOnce(1); + try { + // A second generation can be queued while the old embedding is in flight, + // but it cannot create a second worker drain outside the shared boundary. + expect(service.memoryProcessingStatus([row.id]).items[0]?.state).toBe("summary_pending"); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE target_memory_id = ? AND job_type = 'embedding'").get(row.id)).toEqual({ n: 1 }); + } finally { release(); await oldEmbedding; } + expect(["summary_pending", "summarizing"]).toContain(service.memoryProcessingStatus([row.id]).items[0]?.state); + await newerSummary; const jobs = db.db.prepare("SELECT id, status FROM evolution_jobs WHERE target_memory_id = ? AND job_type = 'embedding'").all(row.id) as Array<{ id: string; status: string }>; expect(jobs).toHaveLength(2); expect(new Set(jobs.map((job) => job.id)).size).toBe(2); const newerJob = jobs.find((job) => job.status === "queued")!; - release(); await oldEmbedding; expect(service.memoryProcessingStatus([row.id]).items[0]).toMatchObject({ state: "embedding_pending", activeJobId: newerJob.id }); await service.runWorkerOnce(1); expect(service.memoryProcessingStatus([row.id]).items[0]?.state).toBe("ready"); @@ -474,13 +496,20 @@ describe("History meaningful-memory processing", () => { it.each(["model_unconfigured", "automatic_retry", "failed"] as const)( "does not hide a source %s behind normal Skill processing", async (state) => { + vi.useFakeTimers({ toFake: ["Date"] }); const configured = state !== "model_unconfigured"; - const { service } = createTestService({ llm: { ...model(() => { throw new Error("429 temporarily rate limited"); }), + const { service, db } = createTestService({ llm: { ...model(() => { throw new Error("429 temporarily rate limited"); }), isConfigured: () => configured } }); const first = service.syncExternalEvidence(evidence); const secondId = "2026-10-09T10-10-00-10min-summary"; service.syncExternalEvidence({ ...evidence, sourceRecordId: secondId }); - if (state !== "model_unconfigured") await runWorkerRounds(service, state === "failed" ? 3 : 1); + if (state !== "model_unconfigured") { + const attempts = state === "failed" ? 3 : 1; + for (let attempt = 0; attempt < attempts; attempt++) { + await runWorkerRounds(service, 1); + if (attempt < attempts - 1) advanceToMemoryRetry(db, first.id); + } + } expect(service.memoryProcessingStatus([first.id]).items[0]).toMatchObject(state === "model_unconfigured" ? { state: "summary_pending", errorCode: "summary_model_unconfigured", retryAction: "open_settings" } : state === "automatic_retry" ? { state: "summary_pending", errorCode: "transient_provider_error" } @@ -493,13 +522,14 @@ describe("History meaningful-memory processing", () => { ); it("generates the Skill after retrying a failed source without re-uploading ready sources", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); let available = false; let skillCalls = 0; let release!: () => void; let entered!: () => void; const started = new Promise((resolve) => { entered = resolve; }); const wait = new Promise((resolve) => { release = resolve; }); - const { service } = createTestService({ llm: model(async (input) => { + const { service, db } = createTestService({ llm: model(async (input) => { if (input.sources[0]!.content.includes("WINDOW_SWITCH_ONLY_SENTINEL")) { if (!available) throw new Error("observation extraction item is not grounded in its source"); entered(); @@ -516,7 +546,10 @@ describe("History meaningful-memory processing", () => { const second = service.syncExternalEvidence({ ...evidence, sourceRecordId: secondId, content: evidence.content.replace("WINDOW_SWITCH_ONLY_SENTINEL", "") }); const suggestion = skillSuggestion(secondId); - await runWorkerRounds(service, 3); + for (let attempt = 0; attempt < 3; attempt++) { + await runWorkerRounds(service, 1); + if (attempt < 2) advanceToMemoryRetry(db, first.id); + } expect(service.memoryProcessingStatus([first.id, second.id]).items).toEqual([ expect.objectContaining({ state: "failed", errorCode: "processing_failed", attemptCount: 3 }), expect.objectContaining({ state: "ready" }) diff --git a/Memory/tests/service/evidence/observation-recovery.test.ts b/Memory/tests/service/evidence/observation-recovery.test.ts index 7d30a801b..43647131d 100644 --- a/Memory/tests/service/evidence/observation-recovery.test.ts +++ b/Memory/tests/service/evidence/observation-recovery.test.ts @@ -5,13 +5,13 @@ import type { LlmMessage } from "../../../src/model/types.js"; import { closeMemoryHttpServer } from "../../../src/server/http.js"; import { observationMemoryState } from "../../../src/service/observation-memory-state.js"; import { Repositories } from "../../../src/storage/repositories.js"; -import { createMemoryServiceFixture, runWorkerRounds } from "../../fixtures/memory-service-fixture.js"; +import { createMemoryServiceFixture, runWorkerRoundsWithDueRetries } from "../../fixtures/memory-service-fixture.js"; const { cleanup, createTestMemoryService } = createMemoryServiceFixture(); const servers: Server[] = []; afterEach(async () => { for (const server of servers.splice(0)) await closeMemoryHttpServer(server); - cleanup(); + await cleanup(); }); const namespace = { source: "memmy", profileId: "default", userId: "history-recovery-user" }; @@ -49,7 +49,9 @@ describe("History recovery across runtime lifecycle", () => { startedAt: "2026-10-19T10:00:00.000Z", endedAt: "2026-10-19T10:10:00.000Z" }; const rows = sourceRecordIds.map((sourceRecordId, index) => service.syncExternalEvidence({ ...baseEvidence, sourceRecordId, content: `**Production constraint**: The A17 order review for source ${index} must retain locked production orders.` })); - await runWorkerRounds(service, 3); + // Lifecycle recovery starts from exhausted failures; make only these retries + // due explicitly instead of bypassing the production backoff rule. + await runWorkerRoundsWithDueRetries(service, db, rows.map(row => row.id), 3); for (const row of rows) { expect(service.memoryProcessingStatus([row.id]).items[0]).toMatchObject({ state: "failed", stage: "summary", attemptCount: 3, manualRetryCount: 0, diff --git a/Memory/tests/service/evolution/negative-experience.test.ts b/Memory/tests/service/evolution/negative-experience.test.ts index 8a88de01a..d951e43cd 100644 --- a/Memory/tests/service/evolution/negative-experience.test.ts +++ b/Memory/tests/service/evolution/negative-experience.test.ts @@ -159,20 +159,13 @@ describe("MemoryService / evolution / negative experience", () => { expect(feedback.jobs.map((job) => job.jobType)).not.toContain("reward"); service.closeSession(session.sessionId); - await service.runWorkerOnce(50); - expect(service.panelItems({ namespace, layer: "L2" }).items).toEqual([]); - expect(service.panelJobs({ namespace, status: "queued" }).items).toEqual( - expect.arrayContaining([ - expect.objectContaining({ jobType: "reward" }) - ]) - ); - - await service.runWorkerOnce(50); - expect(service.panelJobs({ namespace, status: "queued" }).items).toEqual( - expect.arrayContaining([ - expect.objectContaining({ jobType: "negative_experience" }) - ]) - ); + const run = await service.runWorkerOnce(50); + expect(run.jobs).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "reward", status: "succeeded" }), + expect.objectContaining({ jobType: "negative_experience", status: "succeeded" }) + ])); + expect(run.jobs.findIndex(job => job.jobType === "reward")) + .toBeLessThan(run.jobs.findIndex(job => job.jobType === "negative_experience")); const negativeJobCount = db.db.prepare( `SELECT COUNT(*) AS count FROM evolution_jobs @@ -181,7 +174,6 @@ describe("MemoryService / evolution / negative experience", () => { ).get(namespace.userId) as { count: number }; expect(negativeJobCount.count).toBe(1); - await service.runWorkerOnce(50); const policies = service.panelItems({ namespace, layer: "L2" }).items; expect(policies).toHaveLength(1); const detail = service.getMemory(policies[0]!.id, { namespace }); @@ -302,19 +294,13 @@ describe("MemoryService / evolution / negative experience", () => { }); service.closeSession(session.sessionId); - await service.runWorkerOnce(50); - expect(service.panelJobs({ namespace, status: "queued" }).items).toEqual( - expect.arrayContaining([ - expect.objectContaining({ jobType: "reward" }) - ]) - ); - await service.runWorkerOnce(50); - expect(service.panelJobs({ namespace, status: "queued" }).items).toEqual( - expect.arrayContaining([ - expect.objectContaining({ jobType: "negative_experience" }) - ]) - ); - await service.runWorkerOnce(50); + const run = await service.runWorkerOnce(50); + expect(run.jobs).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "reward", status: "succeeded" }), + expect.objectContaining({ jobType: "negative_experience", status: "succeeded" }) + ])); + expect(run.jobs.findIndex(job => job.jobType === "reward")) + .toBeLessThan(run.jobs.findIndex(job => job.jobType === "negative_experience")); const policies = service.panelItems({ namespace, layer: "L2" }).items; expect(policies).toHaveLength(1); diff --git a/Memory/tests/service/evolution/orchestration.test.ts b/Memory/tests/service/evolution/orchestration.test.ts index 2071d6eae..d3d47100b 100644 --- a/Memory/tests/service/evolution/orchestration.test.ts +++ b/Memory/tests/service/evolution/orchestration.test.ts @@ -851,6 +851,19 @@ describe("MemoryService / evolution / orchestration", () => { } expect(policyCreated).toBe(true); + // Finish upstream L2 work before promoting its output for the downstream + // prompt contract. WRR can expose a policy before other older L2 updates + // finish; those updates legitimately revise its eligibility metadata. + const upstreamPending = () => (db.db.prepare( + `SELECT COUNT(*) AS count FROM evolution_jobs + WHERE job_type IN ('l2_association', 'l2_induction') + AND target_memory_id IN (?, ?) AND status IN ('queued', 'leased', 'failed')` + ).get(first.l1MemoryId, second.l1MemoryId) as { count: number }).count; + for (let round = 0; round < 8 && upstreamPending() > 0; round++) { + await service.runWorkerOnce(100, { targetMemoryIds: [first.l1MemoryId, second.l1MemoryId] }); + } + expect(upstreamPending()).toBe(0); + const policyRow = db.db.prepare( `SELECT id, properties_json FROM memories diff --git a/Memory/tests/service/evolution/skill-cluster-admission.test.ts b/Memory/tests/service/evolution/skill-cluster-admission.test.ts new file mode 100644 index 000000000..322d745f3 --- /dev/null +++ b/Memory/tests/service/evolution/skill-cluster-admission.test.ts @@ -0,0 +1,121 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { SkillClusterPipeline } from "../../../src/service/evolution/skill-cluster-pipeline.js"; +import { evolutionJobDedupeKey } from "../../../src/service/worker/job-handlers.js"; +import { WorkerConcurrencyController, providerDescriptor } from "../../../src/service/worker/worker-concurrency.js"; +import { workerWorkRequirements } from "../../../src/service/worker/worker-work-requirements.js"; +import { jobPolicyRegistry } from "../../../src/service/worker/job-policy-registry.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createMemoryServiceFixture, createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(), runners: WorkerRunner[] = []; +afterEach(async () => { for (const runner of runners.splice(0)) await runner.stop(); fixture.cleanup(); }); +function deferred() { + let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; +} +function harness() { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db), controller = new WorkerConcurrencyController({}, {}); + const entered = deferred(), finishFirst = deferred(); let metaCalls = 0, nextJob = 0; + const config = { ...DEFAULT_MEMMY_CONFIG, algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, + skill: { ...DEFAULT_MEMMY_CONFIG.algorithm.skill, outcomeRTaskSuccessThreshold: 0.8, outcomeRTaskFailureThreshold: 0.2 } } }; + const modelConfig = { ...config.evolution, provider: "openai_compatible" as const, + endpoint: "https://cluster.test.invalid/v1", model: "evolve" }; + const embeddingConfig = { ...config.embedding, provider: "openai_compatible" as const, + endpoint: "https://embedding.test.invalid/v1", model: "embedding" }; + const evolutionProvider = providerDescriptor(modelConfig, controller.limits); + const embeddingProvider = providerDescriptor(embeddingConfig, controller.limits); + const llm: LlmClient = { config: modelConfig, isConfigured: () => true, complete: async () => "{}", + completeJson: async >(_messages: unknown, options: { operation: string }) => { + if (options.operation !== "skill.batch_evolve.meta") throw new Error(`Unexpected operation ${options.operation}`); + return controller.runRequest(evolutionProvider, async () => { + if (++metaCalls === 1) { entered.resolve(); await finishFirst.promise; } + return { meta_skill_content: `Reviewed cluster ${metaCalls}` } as unknown as T; + }); + }, status: () => ({ provider: modelConfig.provider, model: modelConfig.model, configured: true, remote: true }) }; + const enqueueJob: WorkerRunnerDeps["enqueueJob"] = input => repos.runtime.enqueueJob({ ...input, + id: `cluster-review-${++nextJob}`, payload: input.payload ?? {}, dedupeKey: input.dedupeKey ?? evolutionJobDedupeKey(input), status: "queued", + attempts: 0, maxAttempts: 3, createdAt: input.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }); + const pipeline = new SkillClusterPipeline({ repos, config, skillLlm: llm, + buildMemory: () => { throw new Error("Unexpected Skill creation for unknown-only evidence"); }, + upsertEvolutionMemory: () => { throw new Error("Unexpected Skill persistence"); }, + enqueueJob, namespaceIdFromMemory: () => "review", + queryVector: () => controller.runRequest(embeddingProvider, async () => [1, 0], true) }); + const prepare: NonNullable = work => { + const roles = jobPolicyRegistry.resolve({ jobType: work.queue === "main" ? work.job.jobType : "embedding_retry" }).concurrency.providerRoles; + return { providers: roles.flatMap(role => role === "embedding" ? embeddingProvider.providers : evolutionProvider.providers), + embedding: roles.includes("embedding"), run: operation => operation() }; + }; + const runner = new WorkerRunner({ repos, concurrency: controller, prepareExecution: prepare, + scheduling: { algorithm: "wrr" }, schedulingShadow: { enabled: false }, embedder: createCapturingEmbedder([]), + capture: { embedAfterCapture: true }, embeddingRetryWorkerId: "cluster-review", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "review", runWorkerNoWrite: async () => { throw new Error("Unexpected disabled worker"); }, + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob, enqueueEmbeddingRetry: () => { throw new Error("Unexpected embedding retry"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: job => job.jobType === "skill_cluster_assign" ? pipeline.assignCluster(job) : pipeline.evolveCluster(job) }, + embeddingJobs: { prepareEmbeddingJob: () => { throw new Error("Unexpected embedding job"); }, applyEmbeddingVector: () => {}, + persistEmbeddingVector: () => { throw new Error("Unexpected vector write"); }, enqueueEmbeddingRetryAfterFailure: () => false } + }); runners.push(runner); + async function assignment(index: number, projectId: string) { + const session = service.openSession({ namespace: { source: "codex", profileId: `review-${index}`, userId: "review-user", projectId } }); + const turn = await service.completeTurn(`review-turn-${index}`, { sessionId: session.sessionId, + query: "Review this code carefully", answer: "Reviewed this code" }); + expect(turn.episodeId).toBeDefined(); + repos.runtime.updateEpisodeReward(turn.episodeId!, { rTask: 0.5, rewardDetail: { source: "review" } }); + return enqueueJob({ jobType: "skill_cluster_assign", userId: "review-user", sessionId: session.sessionId, episodeId: turn.episodeId, payload: {} }); + } + async function initialCluster(job: EvolutionJobRecord) { + const claim = repos.runtime.claimQueuedJob(job.id)!; + const permit = controller.tryAcquire(workerWorkRequirements({ queue: "main", job: claim }, repos, prepare({ queue: "main", job: claim })))!; + try { await permit.run(() => pipeline.assignCluster(claim)); repos.runtime.completeJob(claim.id, new Date().toISOString(), claim); } + finally { permit.release(); } + return repos.runtime.getSkillClusterForEpisode(job.episodeId!)!; + } + return { db, repos, controller, runner, prepare, entered, finishFirst, assignment, initialCluster }; +} + +describe("Skill cluster assignment/evolution admission", () => { + it("keeps a new same-project assignment queued during evolution, preserves its new member and permits another project", async () => { + const h = harness(); + const first = await h.assignment(1, "project-A"); const cluster = await h.initialCluster(first); + const sameProject = await h.assignment(2, "project-A"), otherProject = await h.assignment(3, "project-B"); + h.db.db.prepare("UPDATE evolution_jobs SET available_at='2100-01-01T00:00:00.000Z' WHERE job_type NOT IN ('skill_cluster_assign','skill_batch_evolve')").run(); + const run = h.runner.runWorkerOnce(100); await h.entered.promise; + for (let i = 0; i < 500; i++) await Promise.resolve(); + const beforeSame = h.repos.runtime.getJob(sameProject.id), beforeOther = h.repos.runtime.getJob(otherProject.id); + h.finishFirst.resolve(); const result = await run; + expect(beforeSame).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(beforeOther?.status).toBe("succeeded"); + expect(result.failed).toBe(0); + expect(h.repos.runtime.getSkillCluster(cluster.id)?.memberCount).toBe(2); + expect(h.repos.runtime.listSkillClusterMembers(cluster.id)).toHaveLength(2); + const evolutions = h.repos.runtime.listJobs(undefined, 100).filter(job => job.jobType === "skill_batch_evolve" && job.payload.clusterId === cluster.id); + expect(evolutions).toHaveLength(2); + expect(evolutions.every(job => job.status === "succeeded")).toBe(true); + expect(h.controller.snapshot()).toEqual({ activeJobs: 0, availableJobs: 4, backgroundActiveJobs: 0, maxBackgroundJobs: 3, interactiveReservedSlots: 1, resources: {}, providers: {}, scopeKeys: [], embeddingRequests: 0 }); + }); + + it("resolves Skill scopes from authoritative episode/cluster metadata without a target or current session", async () => { + const h = harness(); const first = await h.assignment(1, "project-A"), cluster = await h.initialCluster(first); + const second = await h.assignment(2, "project-A"), other = await h.assignment(3, "project-B"); + const evolution = h.repos.runtime.listJobs(undefined, 100).find(job => job.jobType === "skill_batch_evolve" && job.payload.clusterId === cluster.id)!; + const evolveWithoutSession = { ...evolution, sessionId: undefined, episodeId: undefined }; + const assignWithoutSession = { ...second, sessionId: undefined }; + const requirements = (job: EvolutionJobRecord) => workerWorkRequirements({ queue: "main", job }, h.repos, h.prepare({ queue: "main", job })); + const owner = h.controller.tryAcquire(requirements(evolveWithoutSession))!; + const same = h.controller.tryAcquire(requirements(assignWithoutSession)); + const different = h.controller.tryAcquire(requirements(other)); + same?.release(); different?.release(); owner.release(); + expect(requirements(evolveWithoutSession).scopeKeys).toContain(JSON.stringify(["user_project", "review-user", "project-A"])); + expect(requirements(assignWithoutSession).scopeKeys).toContain(JSON.stringify(["user_project", "review-user", "project-A"])); + expect(same).toBeUndefined(); + expect(different).toBeDefined(); + }); +}); diff --git a/Memory/tests/service/evolution/span-big-turn.test.ts b/Memory/tests/service/evolution/span-big-turn.test.ts index 6300e103a..e35461143 100644 --- a/Memory/tests/service/evolution/span-big-turn.test.ts +++ b/Memory/tests/service/evolution/span-big-turn.test.ts @@ -2,6 +2,7 @@ import { assignedEpisodeId, settleCaptureAssignments } from "../../fixtures/memo import { LEGACY_TEST_CONFIG } from "../../fixtures/memory-service-fixture.js"; import { afterEach, describe, expect, it, vi } from "vitest"; import { type LlmClient, type LlmCompletionOptions, type LlmMessage } from "../../../src/index.js"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/index.js"; import { createCapturingEmbedder, createMemoryServiceFixture, @@ -511,6 +512,7 @@ describe("MemoryService / evolution / span big turn", () => { }); it("stores only the successful model result after a failed attempt", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); const calls: Array<{ messages: LlmMessage[]; options: LlmCompletionOptions }> = []; const llm = createSpanBigTurnLlm(calls, [ { @@ -572,6 +574,12 @@ describe("MemoryService / evolution / span big turn", () => { vi.setSystemTime(Date.now() + 31_000); await runWorkerRounds(service, 8); + const retry = db.db.prepare(`SELECT available_at FROM evolution_jobs + WHERE job_type = 'span_big_turn' AND status = 'failed'`).get() as { available_at: string }; + expect(Date.parse(retry.available_at)).toBeGreaterThan(Date.now()); + vi.setSystemTime(Date.parse(retry.available_at)); + await runWorkerRounds(service, 8); + const spans = service.panelItems({ namespace, layer: "L1" }).items.filter( (item) => item.kind === "span" ); @@ -652,6 +660,7 @@ describe("MemoryService / evolution / span big turn", () => { }); it("rejects overlapping spans without storing partial results", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); const calls: Array<{ messages: LlmMessage[]; options: LlmCompletionOptions }> = []; const llm = createSpanBigTurnLlm(calls, { shouldSplit: true, @@ -709,7 +718,15 @@ describe("MemoryService / evolution / span big turn", () => { }); vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(Date.now() + 31_000); - await runWorkerRounds(service, 8); + for (let attempt = 0; attempt < 3; attempt++) { + await runWorkerRounds(service, 8); + if (attempt < 2) { + const retry = db.db.prepare(`SELECT available_at FROM evolution_jobs + WHERE job_type = 'span_big_turn' AND status = 'failed'`).get() as { available_at: string }; + expect(Date.parse(retry.available_at)).toBeGreaterThan(Date.now()); + vi.setSystemTime(Date.parse(retry.available_at)); + } + } expect(calls.filter((call) => call.options.operation === "span.big_turn.v1")).toHaveLength(3); expect(service.panelItems({ namespace, layer: "L1" }).items.some( diff --git a/Memory/tests/service/facade/config-and-storage.test.ts b/Memory/tests/service/facade/config-and-storage.test.ts index 8ec52dd12..f86d0f190 100644 --- a/Memory/tests/service/facade/config-and-storage.test.ts +++ b/Memory/tests/service/facade/config-and-storage.test.ts @@ -6,17 +6,22 @@ import { MemoryDb, SCHEMA_MIGRATION_ID, SCHEMA_VERSION, - createStorageBackend + createStorageBackend, + type LlmCompletionOptions, + type LlmMessage } from "../../../src/index.js"; import { Repositories } from "../../../src/storage/repositories.js"; +import { withJobLeaseGuard } from "../../../src/storage/job-lease-context.js"; import { accountRuntimeConfig, addAgentSourceImport, configWithMemoryGates, createCapturingEmbedder, + createBatchReflectionLlm, createFailingLlm, createMemoryServiceFixture, runWorkerRounds, + runWorkerRoundsWithDueRetries, tableCounts } from "../../fixtures/memory-service-fixture.js"; @@ -30,6 +35,51 @@ const { afterEach(cleanup); describe("MemoryService / facade / config and storage", () => { + it("resumes a configuration block when the effective loader config changes without reload", async () => { + let currentConfig = { ...DEFAULT_MEMMY_CONFIG, summary: { ...DEFAULT_MEMMY_CONFIG.summary, model: "before-repair" } }; + let unauthorized = true; + const llm = createBatchReflectionLlm([]); + const completeJson = llm.completeJson.bind(llm); + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + if (unauthorized) throw new Error("unauthorized API key"); + return completeJson(messages, options); + }; + const { db, service } = createTestService({ config: currentConfig, configLoader: () => ({ config: currentConfig }), llm }); + const namespace = { source: "codex", profileId: "config-recovery", userId: "config-recovery-user" }; + const added = addAgentSourceImport(service, namespace, "config repair preserves this job", "effective-config"); + await service.runWorkerOnce(1); + const repos = new Repositories(db.db); + const blocked = repos.runtime.listJobs("failed").find((job) => job.targetMemoryId === added.id)!; + expect(blocked).toMatchObject({ blockedReason: "configuration", failureCount: 0 }); + expect(repos.processing.get(added.id)).toMatchObject({ state: "failed", retryAction: "open_settings" }); + unauthorized = false; + expect(service.reconcileWorkerStartup(100).restartedFailedProcessing).toBe(0); + expect((await service.runWorkerOnce(1)).leased).toBe(0); + currentConfig = { ...currentConfig, summary: { ...currentConfig.summary, model: "after-repair" } }; + await service.runWorkerOnce(1); + expect(repos.runtime.getJob(blocked.id)).toMatchObject({ status: "succeeded", failureCount: 0 }); + expect(repos.processing.get(added.id)?.state).toBe("embedding_pending"); + }); + + it("starts a new worker through the public service without inheriting an expired caller lease", async () => { + const { db, service } = createTestService({ llm: createBatchReflectionLlm([]) }); + const repos = new Repositories(db.db); + const at = new Date().toISOString(); + repos.runtime.enqueueJob({ id: "expired-caller", jobType: "reflection", status: "queued", userId: "detached-caller", + payload: {}, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + const [caller] = repos.runtime.leaseQueuedJobs(1, 60, undefined, false, ["reflection"]); + const child = addAgentSourceImport(service, { source: "codex", profileId: "detached", userId: "detached-caller" }, + "a fresh drain gets its own lease", "detached-drain"); + await withJobLeaseGuard(caller!, () => repos.runtime.isJobLeaseHeld(caller!), async () => { + db.db.prepare("UPDATE evolution_jobs SET leased_until = ? WHERE id = ?") + .run(new Date(Date.now() - 1_000).toISOString(), caller!.id); + const result = await service.runWorkerOnce(1, { targetMemoryIds: [child.id] }); + expect(result.succeeded).toBe(1); + }); + expect(repos.processing.get(child.id)?.state).toBe("embedding_pending"); + expect(repos.runtime.getJob(caller!.id)?.failureCount).toBe(0); + }); + it("migrates the local sqlite schema", () => { const { db, root } = createTestService(); const schema = db.schemaVersion(); @@ -258,7 +308,7 @@ describe("MemoryService / facade / config and storage", () => { }); const namespace = { source: "hermes", profileId: "default", userId: "reload-failure-user" }; const added = addAgentSourceImport(service, namespace, "retry after config reload", "reload-failure"); - await runWorkerRounds(service, 3, 1); + await runWorkerRoundsWithDueRetries(service, db, [added.id], 3, 1); const failed = service.memoryProcessingStatus([added.id], { namespace }).items[0]; expect(failed?.state).toBe("failed"); diff --git a/Memory/tests/service/feedback/feedback-worker-admission.test.ts b/Memory/tests/service/feedback/feedback-worker-admission.test.ts new file mode 100644 index 000000000..4857456cf --- /dev/null +++ b/Memory/tests/service/feedback/feedback-worker-admission.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { localCalendarDate } from "@memmy/agent-source-core"; +import type { Embedder, LlmClient } from "../../../src/model/types.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { memoryVector } from "../../../src/storage/memory-vector-state.js"; +import { MEMORY_BYOK_BUDGET_KV_KEY } from "../../../src/service/memory-token-budget-ledger.js"; +import { WorkerConcurrencyController, providerDescriptor } from "../../../src/service/worker/worker-concurrency.js"; +import { byokRuntimeConfig, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); + +describe("feedback experience worker model admission", () => { + it.each([false, true])("resumes deferred feedback through refinement and embedding with all permits (separate embedding provider=%s)", async separateProvider => { + const controller = new WorkerConcurrencyController({}, {}); + const resumed = byokRuntimeConfig({ tokenBudget: { dailyLimitM: 0, totalLimitM: 0 } }); + const embeddingConfig = { ...resumed.embedding, + ...(separateProvider ? { endpoint: "https://embedding.test.invalid/v1", sourceProvider: "review-embedding" } : {}) }; + const operations: string[] = [], embedded: Array<{ texts: string[]; role?: string }> = []; + const vector = [0.1, 0.2, 0.3]; + const llm: LlmClient = { + config: resumed.evolution, isConfigured: () => true, + complete: async () => "{}", + completeJson: async >(_messages: unknown, options: { operation: string }) => { + operations.push(options.operation); + const snapshot = controller.snapshot(); + expect(snapshot.activeJobs).toBe(1); + expect(snapshot.embeddingRequests).toBe(1); + return { title: "Validate issuer fields", trigger: "When parsing SEC 13F filings.", + procedure: "Read issuer names from filing fields rather than filenames.", + verification: "Match each issuer field to its CUSIP record.", caveats: ["Do not infer issuers from filenames."], confidence: 0.9 } as unknown as T; + }, + status: () => ({ provider: resumed.evolution.provider, model: resumed.evolution.model, configured: true, remote: true }) + }; + function recordEmbedding(texts: string[], role?: "query" | "document") { + embedded.push({ texts, role }); + const snapshot = controller.snapshot(); + expect(snapshot.activeJobs).toBe(1); + expect(snapshot.resources).toEqual({ evolution_llm: 1 }); + expect(snapshot.embeddingRequests).toBe(1); + for (const provider of providerDescriptor(embeddingConfig, controller.limits).providers) { + expect(snapshot.providers[provider.key]).toBe(1); + } + } + const embedder: Embedder = { + config: embeddingConfig, isRemote: () => true, + embed: async (texts, role) => { recordEmbedding(texts, role); return texts.map(() => [...vector]); }, + embedOne: async (text, role) => { recordEmbedding([text], role); return [...vector]; }, + status: () => ({ provider: embeddingConfig.provider, model: embeddingConfig.model, configured: true, remote: true }) + }; + const { db, service } = createTestService({ concurrency: controller, + config: byokRuntimeConfig({ tokenBudget: { dailyLimitM: 1, totalLimitM: 500 } }), + configLoader: () => ({ config: resumed }), llm, skillLlm: llm, embedder }); + const repos = new Repositories(db.db); + repos.runtime.setKv(MEMORY_BYOK_BUDGET_KV_KEY, { dailyUsed: 1_000_000, + lifetimeUsed: 1_000_000, dailyDate: localCalendarDate() }); + const session = service.openSession({ namespace: { source: "codex", profileId: "review", userId: "review-user" } }); + const complete = await service.completeTurn("feedback-admission-turn", { sessionId: session.sessionId, + query: "Parse a SEC 13F filing and extract issuer CUSIP holdings.", answer: "Parsed the filing and validated the issuer field." }); + const episode = repos.runtime.createEpisode({ id: "feedback-admission-episode", sessionId: session.sessionId, + userId: "review-user", status: "open", rawTurnIds: [complete.rawTurnId], l1MemoryIds: [complete.l1MemoryId], + feedbackIds: [], decisionRepairIds: [], l2PolicyIds: [], l3WorldModelIds: [], skillMemoryIds: [], + turnCount: 1, rewardDetail: {}, pipelineStatus: "idle", meta: {}, openedAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + const raw = repos.runtime.getRawTurn(complete.rawTurnId)!; + repos.runtime.updateRawTurn({ ...raw, episodeId: episode.id, + messagePayload: { ...raw.messagePayload, episode_assignment: { status: "assigned" } } }); + complete.episodeId = episode.id; + const feedback = await service.feedback({ sessionId: session.sessionId, episodeId: complete.episodeId, + l1MemoryId: complete.l1MemoryId, channel: "explicit", polarity: "positive", magnitude: 1, + rationale: "Verifier feedback: success. Next time keep the issuer name out of the filename.", rawPayload: { source: "verifier", score: 1 } }); + const queued = repos.runtime.listJobs(undefined, 100).filter(job => job.jobType === "feedback_experience"); + expect(queued).toHaveLength(1); + expect(operations).toEqual([]); + expect(embedded).toEqual([]); + // Isolate the feedback drain while retaining the real producer's record. + db.db.prepare("UPDATE evolution_jobs SET available_at='2100-01-01T00:00:00.000Z' WHERE job_type!='feedback_experience'").run(); + service.reloadConfig(); + const run = await service.runWorkerOnce(1); + expect(run).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(repos.runtime.getJob(queued[0]!.id)).toMatchObject({ status: "succeeded", attempts: 1, failureCount: 0, leaseToken: null }); + expect(operations).toEqual(["feedback.refine.v1"]); + expect(embedded).toEqual([{ texts: [expect.stringContaining("Read issuer names from filing fields")], role: "query" }]); + const policies = db.db.prepare("SELECT id,properties_json FROM memories WHERE user_id='review-user' AND memory_layer='L2'").all() as Array<{ id: string; properties_json: string }>; + expect(policies).toHaveLength(1); + const policy = JSON.parse(policies[0]!.properties_json).internal_info.policy; + expect(policy).toMatchObject({ title: "Success: Validate issuer fields", evidence_polarity: "positive", + source_feedback_ids: [feedback.feedbackId], source_trace_ids: [complete.l1MemoryId] }); + const savedVector = memoryVector(repos.memories.get(policies[0]!.id)!, "vec")!; + expect(savedVector).toHaveLength(vector.length); + vector.forEach((value, index) => expect(savedVector[index]).toBeCloseTo(value)); + expect(controller.snapshot()).toEqual({ activeJobs: 0, availableJobs: 4, backgroundActiveJobs: 0, maxBackgroundJobs: 3, interactiveReservedSlots: 1, resources: {}, providers: {}, scopeKeys: [], embeddingRequests: 0 }); + await service.stop(); + }); +}); diff --git a/Memory/tests/service/import/import-processing.test.ts b/Memory/tests/service/import/import-processing.test.ts index 6d25e22c6..2ec91e25c 100644 --- a/Memory/tests/service/import/import-processing.test.ts +++ b/Memory/tests/service/import/import-processing.test.ts @@ -18,6 +18,7 @@ import { createFailingLlm, createMemoryServiceFixture, runWorkerRounds, + runWorkerRoundsWithDueRetries, stableTestVector } from "../../fixtures/memory-service-fixture.js"; @@ -58,18 +59,30 @@ describe("MemoryService / import / processing", () => { expectAutomaticRetry(false); expect(repos.runtime.leaseQueuedJobs(1, 60, [added.id])).toHaveLength(1); expectAutomaticRetry(false); + // A restart must not revoke another executor's still-valid lease. + expect(repos.runtime.requeueLeasedJobsAfterRestart()).toEqual([]); + db.db.prepare("UPDATE evolution_jobs SET leased_until=? WHERE id=?") + .run("2000-01-01T00:00:00.000Z", repos.processing.get(added.id)!.activeJobId); const recovered = repos.runtime.requeueLeasedJobsAfterRestart(); - expect(recovered[0]?.after).toMatchObject({ status: "queued", attempts: 0, lastError: null }); + expect(recovered[0]?.after).toMatchObject({ status: "queued", attempts: 0, failureCount: 0, lastError: null }); expectAutomaticRetry(false); await service.runWorkerOnce(1); expect(repos.processing.get(added.id)).toMatchObject({ state: "summary_pending", errorCode: "transient_provider_error" }); expectAutomaticRetry(true); + // Failure visibility is immediate, but execution still respects backoff. + expect(repos.runtime.requeueFailedJobs(1, undefined, [added.id])).toEqual([]); + db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id=?") + .run("2000-01-01T00:00:00.000Z", repos.processing.get(added.id)!.activeJobId); expect(repos.runtime.requeueFailedJobs(1, undefined, [added.id])).toHaveLength(1); expectAutomaticRetry(true); expect(repos.runtime.leaseQueuedJobs(1, 60, [added.id])).toHaveLength(1); expectAutomaticRetry(false); - repos.runtime.requeueLeasedJobsAfterRestart(); + expect(repos.runtime.requeueLeasedJobsAfterRestart()).toEqual([]); + db.db.prepare("UPDATE evolution_jobs SET leased_until=? WHERE id=?") + .run("2000-01-01T00:00:00.000Z", repos.processing.get(added.id)!.activeJobId); + const retryRecovered = repos.runtime.requeueLeasedJobsAfterRestart(); + expect(retryRecovered[0]?.after).toMatchObject({ status: "queued", failureCount: 1 }); expectAutomaticRetry(true); failSummary = false; @@ -154,7 +167,9 @@ describe("MemoryService / import / processing", () => { { job_type: "import_summary", status: "queued", target_memory_id: added.id } ]); - const summaryRun = await service.runWorkerOnce(100); + // A one-job budget exposes summary_pending -> embedding_pending before + // continuous admission is allowed to complete the downstream embedding. + const summaryRun = await service.runWorkerOnce(1); expect(summaryRun.jobs.map((job) => job.jobType)).toEqual(["import_summary"]); expect(llmCalls.map((call) => call.options.operation)).toEqual(["capture.summarize"]); const summaryCall = llmCalls.find((call) => call.options.operation === "capture.summarize"); @@ -456,11 +471,7 @@ describe("MemoryService / import / processing", () => { "summary-fallback" ); - const runs = [ - await service.runWorkerOnce(100), - await service.runWorkerOnce(100), - await service.runWorkerOnce(100) - ]; + const runs = await runWorkerRoundsWithDueRetries(service, db, [added.id], 3); expect(runs.flatMap((run) => run.jobs).map((job) => job.jobType)).toEqual([ "import_summary", @@ -600,7 +611,7 @@ describe("MemoryService / import / processing", () => { expect(failed).toMatchObject({ state: "failed", stage: "summary", - attemptCount: 1, + attemptCount: 0, retryAction: "open_settings", errorCode: "model_configuration" }); @@ -642,8 +653,7 @@ describe("MemoryService / import / processing", () => { errorMessage: "429 second provider failure", autoRetryScheduled: true }); - await service.runWorkerOnce(1); - await service.runWorkerOnce(1); + await runWorkerRoundsWithDueRetries(service, db, [added.id], 2, 1); const secondFailure = service.memoryProcessingStatus([added.id], { namespace }).items[0]; expect(secondFailure).toMatchObject({ state: "failed", @@ -783,7 +793,7 @@ describe("MemoryService / import / processing", () => { expect(service.memoryProcessingStatus([added.id], { namespace }).items[0]).toMatchObject({ state: "failed", stage: "summary", - attemptCount: 1, + attemptCount: 0, retryAction: "open_settings", errorCode: "model_configuration", errorMessage: expect.stringContaining("HTTP 200") @@ -849,9 +859,7 @@ describe("MemoryService / import / processing", () => { const added = addAgentSourceImport(service, namespace, "retry only the vector stage", "embedding-stage"); await service.runWorkerOnce(1); - for (let attempt = 0; attempt < 6; attempt += 1) { - await service.runWorkerOnce(1); - } + await runWorkerRoundsWithDueRetries(service, db, [added.id], 6, 1); expect(service.memoryProcessingStatus([added.id], { namespace }).items[0]).toMatchObject({ state: "failed", stage: "embedding", @@ -1202,7 +1210,7 @@ describe("MemoryService / import / processing", () => { const first = addAgentSourceImport(service, namespace, "first targeted import", "targeted-first"); const unrelated = addAgentSourceImport(service, namespace, "unrelated import", "targeted-unrelated"); - const summaryRun = await service.runWorkerOnce(10, { targetMemoryIds: [first.id] }); + const summaryRun = await service.runWorkerOnce(1, { targetMemoryIds: [first.id] }); const embeddingRun = await service.runWorkerOnce(10, { targetMemoryIds: [first.id] }); expect(summaryRun.jobs.filter(job => !["episode_assignment", "episode_finalize"].includes(job.jobType))).toEqual([ @@ -1216,7 +1224,7 @@ describe("MemoryService / import / processing", () => { db.close(); }); - it("orders import summaries by newest panel memories before embedding jobs", async () => { + it("orders import work by enqueue FIFO regardless of panel memory recency", async () => { const root = createTestRoot("mindock-memory-import-order-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") @@ -1254,17 +1262,17 @@ describe("MemoryService / import / processing", () => { db.db.prepare(`UPDATE memories SET updated_at = ? WHERE id = ?`).run("2026-06-10T13:00:00.000Z", older.id); db.db.prepare(`UPDATE memories SET updated_at = ? WHERE id = ?`).run("2026-06-10T12:00:00.000Z", newer.id); - const summaryRun = await service.runWorkerOnce(10); - const embeddingRun = await service.runWorkerOnce(10); + const summaryRun = await service.runWorkerOnce(2); + const embeddingRun = await service.runWorkerOnce(2); - expect(summaryRun.jobs.map((job) => job.targetMemoryId)).toEqual([newer.id, older.id]); - expect(embeddingRun.jobs.map((job) => job.targetMemoryId)).toEqual([newer.id, older.id]); - expect(llmCalls[0]?.messages.find((message) => message.role === "user")?.content).toContain("newer memory query"); + expect(summaryRun.jobs.map((job) => job.targetMemoryId)).toEqual([older.id, newer.id]); + expect(embeddingRun.jobs.map((job) => job.targetMemoryId)).toEqual([older.id, newer.id]); + expect(llmCalls[0]?.messages.find((message) => message.role === "user")?.content).toContain("older memory query"); db.close(); }); - it("embeds summarized import memories before continuing later import summaries", async () => { + it("preserves FIFO within available import resources while downstream embeddings progress", async () => { const root = createTestRoot("mindock-memory-import-interleave-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") @@ -1286,34 +1294,36 @@ describe("MemoryService / import / processing", () => { userId: "user-import-interleave" }; + const importedIds: string[] = []; for (let index = 0; index < 25; index += 1) { - addAgentSourceImport( + const imported = addAgentSourceImport( service, namespace, `imported query ${index}`, `interleave-${index}`, new Date(Date.UTC(2026, 5, 10, 10, index)).toISOString() ); + importedIds.push(imported.id); } - const summaryRun = await service.runWorkerOnce(20); - const embeddingRun = await service.runWorkerOnce(20); - - expect(summaryRun.jobs.map((job) => job.jobType)).toEqual(Array.from({ length: 20 }, () => "import_summary")); - expect(embeddingRun.jobs.map((job) => job.jobType)).toEqual(Array.from({ length: 20 }, () => "embedding")); - expect(llmCalls.filter((call) => call.options.operation === "capture.summarize")).toHaveLength(20); - expect(embeddingTexts).toHaveLength(20); + // A busy summary provider does not block ready embedding work. The drain + // keeps filling available resources while each resource retains its FIFO. + const run = await service.runWorkerOnce(60); + expect(run.jobs).toHaveLength(50); + expect(run.jobs.filter(job => job.jobType === "import_summary").map(job => job.targetMemoryId)).toEqual(importedIds); + expect(run.jobs.filter(job => job.jobType === "embedding").map(job => job.targetMemoryId)).toEqual(importedIds); + expect(run.failed).toBe(0); + expect(llmCalls.filter((call) => call.options.operation === "capture.summarize")).toHaveLength(25); + const remainingSummaries = db.db.prepare( + `SELECT COUNT(*) AS count FROM evolution_jobs WHERE job_type = 'import_summary' AND status = 'queued'` + ).get() as { count: number }; + expect(remainingSummaries.count).toBe(0); + const finalRun = await service.runWorkerOnce(20); + expect(finalRun.jobs).toEqual([]); + expect(embeddingTexts).toHaveLength(25); expect(embeddingTexts.every((text) => text.includes("Summary: LLM batch summary"))).toBe(true); expect(embeddingTexts.every((text) => text.includes("Original exchange:"))).toBe(true); expect(embeddingTexts.every((text) => text.includes("imported query"))).toBe(true); - const remainingSummaries = db.db.prepare( - `SELECT COUNT(*) AS count - FROM evolution_jobs - WHERE job_type = 'import_summary' - AND status = 'queued'` - ).get() as { count: number }; - expect(remainingSummaries.count).toBe(5); - db.close(); }); @@ -1371,7 +1381,7 @@ describe("MemoryService / import / processing", () => { db.close(); }); - it("orders placeholder import embeddings by newest role heading", async () => { + it("keeps placeholder import embeddings in FIFO order and repairs their summaries first", async () => { const root = createTestRoot("mindock-memory-import-placeholder-order-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") @@ -1426,9 +1436,9 @@ describe("MemoryService / import / processing", () => { "2026-06-10T12:00:00.000Z" ); - const run = await service.runWorkerOnce(10); + const run = await service.runWorkerOnce(2); - expect(run.jobs.map((job) => job.targetMemoryId)).toEqual([newer.id, older.id]); + expect(run.jobs.map((job) => job.targetMemoryId)).toEqual([older.id, newer.id]); expect(embeddingTexts).toEqual([]); const queuedSummary = db.db.prepare( `SELECT job_type, status @@ -1440,7 +1450,7 @@ describe("MemoryService / import / processing", () => { db.close(); }); - it("finishes a new Memmy chat memory before draining scanned-memory backlog", async () => { + it("gives a new Memmy chat memory interactive opportunities alongside scanned backlog", async () => { const root = createTestRoot("mindock-memory-live-priority-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") @@ -1487,22 +1497,23 @@ describe("MemoryService / import / processing", () => { const embeddingRun = await service.runWorkerOnce(4, { priorityCohortOnly: true }); const scanRun = await service.runWorkerOnce(4, { priorityCohortOnly: true }); - expect(summaryRun.jobs.filter(job => !["episode_assignment", "episode_finalize"].includes(job.jobType))).toEqual([ - expect.objectContaining({ jobType: "trace_summary", targetMemoryId: live.l1MemoryId }) - ]); - expect(embeddingRun.jobs).toEqual([ - expect.objectContaining({ jobType: "embedding", targetMemoryId: live.l1MemoryId }) - ]); - expect(scanRun.jobs.map((job) => job.targetMemoryId)).toEqual([recentImport.id, oldImport.id]); - expect(scanRun.jobs.every((job) => job.jobType === "import_summary")).toBe(true); + expect(summaryRun.jobs.find(job => job.jobType === "trace_summary")).toMatchObject({ jobType: "trace_summary", targetMemoryId: live.l1MemoryId }); + const runs = [summaryRun, embeddingRun, scanRun]; + const completed = runs.flatMap((run) => run.jobs); + const imports = completed.filter((job) => job.jobType === "import_summary"); + expect(imports.map((job) => job.targetMemoryId)).toEqual([oldImport.id, recentImport.id]); + const embeddings = completed.filter((job) => job.jobType === "embedding"); + expect(embeddings.map((job) => job.targetMemoryId)).toEqual([live.l1MemoryId, oldImport.id, recentImport.id]); + for (const memoryId of [live.l1MemoryId, oldImport.id, recentImport.id]) { + expect(service.memoryProcessingStatus([memoryId], { namespace }).items[0]?.state).toBe("ready"); + } expect(llmCalls[0]?.messages.find((message) => message.role === "user")?.content) .toContain("new interactive preference"); - expect(embeddingTexts).toHaveLength(1); - + expect(embeddingTexts).toHaveLength(3); db.close(); }); - it("finishes the Memmy first-report memory before interactive and scanned-memory work", async () => { + it("keeps the first report interactive while preserving earlier interactive FIFO", async () => { const root = createTestRoot("mindock-memory-first-report-priority-"); const db = new MemoryDb({ path: join(root, "memory.sqlite") @@ -1559,18 +1570,22 @@ describe("MemoryService / import / processing", () => { const reportEmbeddingRun = await service.runWorkerOnce(4, { priorityCohortOnly: true }); const liveSummaryRun = await service.runWorkerOnce(4, { priorityCohortOnly: true }); - expect(reportSummaryRun.jobs).toEqual([ - expect.objectContaining({ jobType: "import_summary", targetMemoryId: firstReport.id }) - ]); - expect(reportEmbeddingRun.jobs).toEqual([ - expect.objectContaining({ jobType: "embedding", targetMemoryId: firstReport.id }) - ]); - expect(liveSummaryRun.jobs.filter(job => !["episode_assignment", "episode_finalize"].includes(job.jobType))).toEqual([ - expect.objectContaining({ jobType: "trace_summary", targetMemoryId: live.l1MemoryId }) - ]); - expect(liveSummaryRun.jobs.map((job) => job.targetMemoryId)).not.toContain(scanned.id); + const reportPolicy = db.db.prepare("SELECT queue_lane, base_priority FROM evolution_jobs WHERE target_memory_id = ? AND job_type = 'import_summary'").get(firstReport.id); + expect(reportPolicy).toEqual({ queue_lane: "interactive", base_priority: -100 }); + const completed = [reportSummaryRun, reportEmbeddingRun, liveSummaryRun].flatMap((run) => run.jobs); + // A database-only idle close can start while the local model is occupied. + // Check ordering among the actual model summaries, rather than timer jobs. + expect(completed.filter(job => job.jobType === "trace_summary" || job.jobType === "import_summary") + .map(job => job.targetMemoryId)).toEqual([live.l1MemoryId, firstReport.id, scanned.id]); + expect(completed).toEqual(expect.arrayContaining([expect.objectContaining({ jobType: "episode_finalize" })])); + const interactiveEmbeddings = completed.filter((job) => job.jobType === "embedding" && + [live.l1MemoryId, firstReport.id].includes(job.targetMemoryId ?? "")); + expect(interactiveEmbeddings.map((job) => job.targetMemoryId)).toEqual([live.l1MemoryId, firstReport.id]); + expect(completed).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "import_summary", targetMemoryId: scanned.id }) + ])); expect(service.memoryProcessingStatus([firstReport.id], { namespace }).items[0]?.state).toBe("ready"); - + expect(service.memoryProcessingStatus([live.l1MemoryId], { namespace }).items[0]?.state).toBe("ready"); db.close(); }); @@ -1603,7 +1618,7 @@ describe("MemoryService / import / processing", () => { )` ).run(namespace.userId, added.id, "2026-06-10T10:00:00.000Z", "2026-06-10T10:00:00.000Z"); - const firstRun = await service.runWorkerOnce(100); + const firstRun = await service.runWorkerOnce(1); expect(firstRun.jobs).toEqual([ expect.objectContaining({ diff --git a/Memory/tests/service/session/async-capture.test.ts b/Memory/tests/service/session/async-capture.test.ts index 425b8015b..34befe1f4 100644 --- a/Memory/tests/service/session/async-capture.test.ts +++ b/Memory/tests/service/session/async-capture.test.ts @@ -1,16 +1,22 @@ -import { afterEach, describe, expect, it } from "vitest"; -import { DEFAULT_MEMMY_CONFIG, MemoryDb } from "../../../src/index.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb, SCHEMA_VERSION } from "../../../src/index.js"; import { insertActiveSkillMemoryForTest } from "../../fixtures/evolution-fixture.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; import { Repositories } from "../../../src/storage/repositories.js"; import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; import { createTurnJudgeLlm, turnJudgeConfig } from "./turn-judge-llm.js"; const fixture = createMemoryServiceFixture(); -afterEach(fixture.cleanup); +afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function drain(service: ReturnType["service"], limit = 8) { for (let i = 0; i < limit; i++) { const result = await service.runWorkerOnce(30); - if (!result.leased) break; + if (!result.leased) { + const row = service["repos"].db.prepare("SELECT MIN(available_at) AS at FROM evolution_jobs WHERE status='failed' AND failure_count < max_attempts AND blocked_reason IS NULL").get() as { at: string | null }; + if (!row.at) break; + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Math.max(Date.now(), Date.parse(row.at))); + } } } @@ -209,7 +215,7 @@ describe("raw-first independent L1 and Episode processing", () => { if (options.operation === "episode.select.v1") { entered(); await blocked; } return complete(messages, options); }; - const { service, db } = fixture.createTestService({ llm, config: turnJudgeConfig() }); + const { service, db } = fixture.createTestService({ llm, config: turnJudgeConfig(), concurrency: new WorkerConcurrencyController({ localModelConcurrency: 2 }, {}) }); const { sessionId } = service.openSession({}); const result = await service.completeTurn("parallel", { sessionId, query: "Capture concurrently", answer: "Verified result" }); const running = service.runWorkerOnce(30); @@ -327,6 +333,7 @@ describe("raw-first independent L1 and Episode processing", () => { const repos = new Repositories(db.db); const leases = repos.runtime.leaseQueuedJobs(30, 60); expect(leases.some(j => j.jobType === "episode_assignment")).toBe(true); + db.db.prepare("UPDATE evolution_jobs SET leased_until=? WHERE status='leased'").run(new Date(Date.now() - 1).toISOString()); const restored = fixture.createTestMemoryService({ db: new MemoryDb({ path: db.path }), llm, config }); restored.reconcileWorkerStartup(); await drain(restored); expect(db.db.prepare("SELECT id,created_at,user_text,assistant_text FROM raw_turns").get()).toEqual(before); @@ -361,7 +368,7 @@ describe("raw-first independent L1 and Episode processing", () => { } db.db.exec("DELETE FROM schema_migrations; INSERT INTO schema_migrations(id,version,applied_at,checksum) VALUES('009_token_usage_outbox',9,'2026-01-01','old')"); const upgraded = new MemoryDb({ path: db.path }); - expect(upgraded.schemaVersion().version).toBe(10); + expect(upgraded.schemaVersion().version).toBe(SCHEMA_VERSION); expect(upgraded.db.prepare("SELECT rowid,id,user_text FROM raw_turns").get()).toEqual({ rowid: 77, id: old.rawTurnId, user_text: "Existing original" }); expect(upgraded.db.pragma("foreign_key_check")).toEqual([]); for (const table of ["raw_turns", "skill_trials"]) expect((upgraded.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string; notnull: number }>).find(c => c.name === "episode_id")?.notnull).toBe(0); diff --git a/Memory/tests/service/session/deferred-episode-finalization.test.ts b/Memory/tests/service/session/deferred-episode-finalization.test.ts index a8f47e1a0..a8a5822a6 100644 --- a/Memory/tests/service/session/deferred-episode-finalization.test.ts +++ b/Memory/tests/service/session/deferred-episode-finalization.test.ts @@ -100,6 +100,7 @@ describe("resume all deferred Episode finalization after Session attribution dra service.closeSession(sessionId); context.failSelection(recovery === "selection-failure"); if (recovery !== "normal") { + if (recovery === "leased-restart") db.db.prepare("UPDATE evolution_jobs SET available_at='2100-01-01T00:00:00.000Z' WHERE job_type='episode_assignment' AND json_extract(payload_json,'$.rawTurnId')=?").run(saved[2]!.rawTurnId); for (let i = 0; i < 12 && !repos.runtime.getRawTurn(saved[1]!.rawTurnId)?.episodeId; i++) await service.runWorkerOnce(30); expect(repos.runtime.getRawTurn(saved[1]!.rawTurnId)?.episodeId).toBeTruthy(); expect(repos.runtime.getRawTurn(saved[2]!.rawTurnId)?.episodeId).toBeUndefined(); @@ -110,7 +111,10 @@ describe("resume all deferred Episode finalization after Session attribution dra expect(db.db.prepare("SELECT status FROM evolution_jobs WHERE job_type='episode_assignment' AND json_extract(payload_json,'$.rawTurnId')=?") .get(saved[2]!.rawTurnId)).toEqual({ status: "dead_letter" }); } else { + db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE job_type='episode_assignment' AND json_extract(payload_json,'$.rawTurnId')=?").run(saved[2]!.rawTurnId); expect(repos.runtime.leaseQueuedJobs(30, 60).some(job => job.jobType === "episode_assignment")).toBe(true); + // The previous process is gone; recover only once its leases expire. + vi.setSystemTime(Date.now() + 60_001); } context.failSelection(false); service = fixture.createTestMemoryService({ db: new MemoryDb({ path: db.path }), llm, config }); diff --git a/Memory/tests/service/session/episode-relation.test.ts b/Memory/tests/service/session/episode-relation.test.ts index 227d72333..09c74f3b6 100644 --- a/Memory/tests/service/session/episode-relation.test.ts +++ b/Memory/tests/service/session/episode-relation.test.ts @@ -9,6 +9,7 @@ import { type LlmMessage } from "../../../src/index.js"; import { Repositories } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; import { accountRuntimeConfig, createBatchReflectionLlm, @@ -409,7 +410,8 @@ describe("MemoryService / session / episode relation", () => { WHERE episode_id = ? AND job_type IN ('reflection', 'reward')` ).all(assignedEpisodeId(service, first)) as Array<{ job_type: string; payload_json: string }>; - expect(episodeEvolutionJobs().map(job => job.job_type)).toEqual(["reflection"]); + // Continuous refill may complete reflection and enqueue reward before this drain returns. + expect(episodeEvolutionJobs().map(job => job.job_type)).toEqual(["reflection", "reward"]); await runWorkerRounds(service, 4, 50); @@ -1164,7 +1166,10 @@ describe("MemoryService / session / episode relation", () => { } } }; - const { db, service } = createTestService({ llm, config }); + // This case verifies orchestration overlap. Permit two local requests here; + // the production default of one is covered by concurrency admission tests. + const { db, service } = createTestService({ llm, config, + concurrency: new WorkerConcurrencyController({ localModelConcurrency: 2 }) }); const session = service.openSession({ namespace: { source: "codex", diff --git a/Memory/tests/service/session/episode-selection.test.ts b/Memory/tests/service/session/episode-selection.test.ts index 27d005fd7..5efa696e4 100644 --- a/Memory/tests/service/session/episode-selection.test.ts +++ b/Memory/tests/service/session/episode-selection.test.ts @@ -181,7 +181,7 @@ describe("episode selection before creation", () => { expect(repos.runtime.getRawTurn(accepted.rawTurnId)?.userText).toBe(request.query); expect(repos.runtime.listEpisodesForSession(session.sessionId)).toEqual([]); llm.completeJson = complete; - db.db.prepare("UPDATE evolution_jobs SET status='queued',attempts=0,leased_until=NULL WHERE status IN ('failed','dead_letter')").run(); + db.db.prepare("UPDATE evolution_jobs SET status='queued',attempts=0,failure_count=0,available_at=NULL,blocked_reason=NULL,leased_until=NULL,lease_owner=NULL,lease_token=NULL WHERE status IN ('failed','dead_letter')").run(); await settle(service); const first = await service.completeTurn("x", request); const retry = await service.completeTurn("x", request); diff --git a/Memory/tests/service/session/idle-sweep.test.ts b/Memory/tests/service/session/idle-sweep.test.ts index bc36ad569..754073e71 100644 --- a/Memory/tests/service/session/idle-sweep.test.ts +++ b/Memory/tests/service/session/idle-sweep.test.ts @@ -81,7 +81,12 @@ describe("MemoryService / session / idle sweep", () => { WHERE episode_id = ? AND job_type = 'reflection'` ).get(assignedEpisodeId(service, idleTurn)) as { status: string; payload_json: string } | undefined; - expect(reflection?.status).toBe("queued"); + // Continuous admission can finish the reflection produced by idle close in + // the same drain, rather than leaving it for a later timer round. + expect(reflection?.status).toBe("succeeded"); + expect(run.jobs).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "reflection", status: "succeeded" }) + ])); expect(JSON.parse(reflection!.payload_json)).toMatchObject({ trigger: "idle_timeout", targetKind: "episode" diff --git a/Memory/tests/service/session/source-turn-scheduling.test.ts b/Memory/tests/service/session/source-turn-scheduling.test.ts new file mode 100644 index 000000000..26bf0d233 --- /dev/null +++ b/Memory/tests/service/session/source-turn-scheduling.test.ts @@ -0,0 +1,293 @@ +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createAgentSourceExecutor } from "../../../src/agent-source/runtime.js"; +import { createSourceRegistry } from "../../../src/agent-source/adapters/source-registry.js"; +import { MemoryDb } from "../../../src/index.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import type { SourceTurnCompleteRequest } from "../../../src/types.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { + createMemoryServiceFixture, createBatchReflectionLlm, createCapturingEmbedder, settleCaptureAssignments, assignedEpisodeId +} from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +const namespace = { source: "codex", profileId: "default", userId: "scan-scheduling-user" }; + +function turn(index: number, overrides: Partial = {}): SourceTurnCompleteRequest { + return { + namespace, source: "codex", channel: "agent_source_scan", captureLegacyHistory: true, + sourceTurn: { + source: "codex", profileId: "default", conversationId: "history-conversation", turnId: `source-turn-${index}`, + startedAt: new Date(Date.UTC(2000, 0, 1, 10, index)).toISOString(), + completedAt: new Date(Date.UTC(2000, 0, 1, 10, index, 30)).toISOString(), + sequence: index, completionEvidence: `task_complete:${index}` + }, + query: `Implement database transaction test case ${index} and preserve rollback safety.`, + answer: `Implemented database transaction test case ${index}; rollback safety is verified.`, + tags: ["database", "test"], + ...overrides + }; +} + +function current(index: number, overrides: Partial = {}) { + const input = turn(index); + return turn(index, { ...input, channel: "hook", captureLegacyHistory: undefined, + sourceTurn: { ...input.sourceTurn, conversationId: "current-conversation", + startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:00:30.000Z" }, ...overrides }); +} + +function harness() { + const calls: Parameters[0] = []; + const embeddingTexts: string[] = []; + const h = fixture.createTestService({ + llm: createBatchReflectionLlm(calls), embedder: createCapturingEmbedder(embeddingTexts), + concurrency: new WorkerConcurrencyController({ maxActiveJobs: 1 }) + }); + return { ...h, calls, embeddingTexts, repos: new Repositories(h.db.db) }; +} + +function jobs(db: MemoryDb, id: string) { + return db.db.prepare(`SELECT id, job_type, queue_lane, base_priority, enqueue_seq, status, payload_json + FROM evolution_jobs WHERE target_memory_id = ? ORDER BY enqueue_seq`).all(id) as Array<{ + id: string; job_type: string; queue_lane: string; base_priority: number; enqueue_seq: number; + status: string; payload_json: string; + }>; +} + +async function capture(h: ReturnType, input: SourceTurnCompleteRequest) { + const result = await h.service.completeSourceTurn(input); + expect(result.status).toBe("stored"); + expect(result.result?.l1MemoryIds).toHaveLength(1); + return result.result!; +} + +describe("native source-turn historical scheduling through MemoryService", () => { + it.each([ + ["historical full/initial scan", (index: number) => turn(index), "import", "agent_source_scan"], + ["post-activation historical full/initial scan", (index: number) => current(index, { + channel: "agent_source_scan", captureLegacyHistory: true + }), "import", "agent_source_scan"], + ["current Hook", (index: number) => current(index), "interactive", undefined], + ["current incremental scan", (index: number) => current(index, { channel: "agent_source_scan" }), "interactive", undefined], + ["Hook with an irrelevant history flag", (index: number) => current(index, { captureLegacyHistory: true }), "interactive", undefined] + ] as const)("keeps %s on the native capture pipeline with the appropriate lane", async (_label, input, lane, origin) => { + const h = harness(); + const result = await capture(h, input(1)); + const memory = h.repos.memories.get(result.l1MemoryId)!; + expect(memory.properties.internal_info.plugin_algorithm).toBe("capture.v7"); + expect(memory.properties.internal_info.capture_origin).toBe(origin); + expect(memory.tags).toEqual(expect.arrayContaining(["database", "test"])); + expect(memory.tags).not.toContain("agent-source"); + expect(jobs(h.db, memory.id)).toEqual([expect.objectContaining({ job_type: "trace_summary", queue_lane: lane })]); + expect(JSON.parse(jobs(h.db, memory.id)[0]!.payload_json)).toMatchObject({ decideCapture: true }); + expect(h.repos.processing.get(memory.id)?.state).toBe("summary_pending"); + }); + + it("preserves historical origin for a detached out-of-order source turn", async () => { + const h = harness(); + const newer = await capture(h, turn(2)); + const older = await capture(h, turn(1)); + expect(older.sessionId).not.toBe(newer.sessionId); + expect(h.repos.memories.get(older.l1MemoryId)?.properties.internal_info.capture_origin).toBe("agent_source_scan"); + expect(jobs(h.db, older.l1MemoryId)[0]).toMatchObject({ job_type: "trace_summary", queue_lane: "import" }); + }); + + it.each(["history", "current"] as const)("runs the real %s trace summarizer and inherits provenance in the generated embedding", async kind => { + const h = harness(); + const result = await capture(h, kind === "history" ? turn(1) : current(1)); + const summary = jobs(h.db, result.l1MemoryId)[0]!; + await h.service.runWorkerOnce(1, { targetMemoryIds: [result.l1MemoryId] }); + expect(h.calls.some(call => call.options.operation === "capture.summarize")).toBe(true); + const afterSummary = jobs(h.db, result.l1MemoryId); + expect(afterSummary[0]).toMatchObject({ job_type: "trace_summary", status: "succeeded" }); + expect(afterSummary[1]).toMatchObject({ job_type: "embedding", queue_lane: kind === "history" ? "import" : "interactive" }); + expect(JSON.parse(afterSummary[1]!.payload_json)).toMatchObject({ reason: "trace.summary.updated", sourceJobId: summary.id }); + expect(h.repos.processing.get(result.l1MemoryId)?.state).toBe("embedding_pending"); + await h.service.runWorkerOnce(1, { targetMemoryIds: [result.l1MemoryId] }); + expect(h.embeddingTexts).toHaveLength(1); + expect(h.repos.processing.get(result.l1MemoryId)?.state).toBe("ready"); + expect(h.repos.memories.get(result.l1MemoryId)?.properties.internal_info.plugin_algorithm).toBe("capture.v7"); + expect(h.repos.memories.get(result.l1MemoryId)?.properties.internal_info.capture_origin) + .toBe(kind === "history" ? "agent_source_scan" : undefined); + }); + + it.each(["history", "mixed", "current"] as const)("keeps provisional and final titles in the proper %s lane", async kind => { + const h = harness(); + const input = kind === "current" ? current(1) : current(1, { channel: "agent_source_scan", captureLegacyHistory: true }); + const first = await capture(h, input); + if (kind === "mixed") { + const next = current(2); + next.sourceTurn = { ...next.sourceTurn, conversationId: input.sourceTurn.conversationId, + startedAt: "2099-01-01T10:01:00.000Z", completedAt: "2099-01-01T10:01:30.000Z" }; + const second = await capture(h, next); + expect(second.episodeId).toBe(first.episodeId); + } + await settleCaptureAssignments(h.service); + first.episodeId = assignedEpisodeId(h.service, first); + h.service["workerHandlers"].enqueueJob({ jobType: "episode_title", userId: namespace.userId, + sessionId: first.sessionId, episodeId: first.episodeId, payload: { stage: "provisional" } }); + const provisional = h.db.db.prepare("SELECT queue_lane,payload_json FROM evolution_jobs WHERE episode_id=? AND job_type='episode_title'") + .all(first.episodeId) as Array<{queue_lane: string;payload_json: string}>; + expect(provisional.length).toBeGreaterThan(0); + expect(provisional[0]!.queue_lane).toBe(kind === "history" ? "import" : "interactive"); + h.service.closeSession(first.sessionId); + // A final title is only produced once capture decisions finish. Exercise the + // same production enqueuer directly with the now-closed real episode. + const episode = h.repos.runtime.getEpisode(first.episodeId)!; + const inputJob = { jobType: "episode_title" as const, userId: episode.userId, + sessionId: episode.sessionId, episodeId: episode.id, payload: { stage: "final", captureOrigin: "agent_source_scan" } }; + // A distinct stage avoids reusing the existing provisional title. + const queued = h.service["workerHandlers"].enqueueJob(inputJob); + expect(queued.queueLane).toBe(kind === "history" ? "import" : "interactive"); + expect(queued.payload.captureOrigin).toBe(kind === "history" ? "agent_source_scan" : undefined); + }); + + it("keeps historical embedding behind an earlier runnable historical summary in the same FIFO lane", async () => { + const h = harness(); + const first = await capture(h, turn(1)); + const second = await capture(h, turn(2)); + await h.service.runWorkerOnce(1, { targetMemoryIds: [first.l1MemoryId] }); + const firstEmbedding = jobs(h.db, first.l1MemoryId).find(job => job.job_type === "embedding")!; + const secondSummary = jobs(h.db, second.l1MemoryId)[0]!; + expect(firstEmbedding.queue_lane).toBe("import"); + expect(secondSummary.queue_lane).toBe("import"); + expect(secondSummary.enqueue_seq).toBeLessThan(firstEmbedding.enqueue_seq); + const executed: string[] = []; + for (let round = 0; round < 8; round++) { + executed.push(...(await h.service.runWorkerOnce(1)).jobs.map(job => job.jobId)); + if (executed.includes(firstEmbedding.id)) break; + } + expect(executed).toContain(secondSummary.id); + expect(executed).toContain(firstEmbedding.id); + expect(executed.indexOf(secondSummary.id)).toBeLessThan(executed.indexOf(firstEmbedding.id)); + }); + + it.each([false, true])("finishes a current native summary and embedding before a historical backlog drains (separate conversations: %s)", async separate => { + const h = harness(); + const historicalIds: string[] = []; + for (let index = 1; index <= 48; index++) { + const input = turn(index); + if (separate) input.sourceTurn = { ...input.sourceTurn, conversationId: `history-conversation-${index}` }; + historicalIds.push((await capture(h, input)).l1MemoryId); + } + const live = await capture(h, current(100)); + const executed: Array<{ jobType: string; targetMemoryId?: string }> = []; + for (let round = 0; round < 24; round++) executed.push(...(await h.service.runWorkerOnce(1)).jobs); + expect(executed).toEqual(expect.arrayContaining([ + expect.objectContaining({ jobType: "trace_summary", targetMemoryId: live.l1MemoryId }), + expect.objectContaining({ jobType: "embedding", targetMemoryId: live.l1MemoryId }) + ])); + expect(h.repos.processing.get(live.l1MemoryId)?.state).toBe("ready"); + expect(historicalIds.some(id => jobs(h.db, id).some(job => job.job_type === "trace_summary" && job.status === "queued"))).toBe(true); + const historicalExecution = executed.filter(job => job.jobType === "trace_summary" && historicalIds.includes(job.targetMemoryId ?? "")); + expect(historicalExecution.length).toBeGreaterThan(0); + expect(historicalExecution.map(job => job.targetMemoryId)).toEqual(historicalIds.slice(0, historicalExecution.length)); + }); + + it.each(["hook-first", "scan-first"] as const)("keeps the first capture origin and FIFO position through %s replay and restart", async order => { + const h = harness(); + const live = current(1); + const historical = { ...live, channel: "agent_source_scan" as const, captureLegacyHistory: true }; + const firstInput = order === "hook-first" ? live : historical; + const secondInput = order === "hook-first" ? historical : live; + const first = await capture(h, firstInput); + const expectedOrigin = order === "hook-first" ? undefined : "agent_source_scan"; + const expectedLane = order === "hook-first" ? "interactive" : "import"; + const before = jobs(h.db, first.l1MemoryId); + expect(before[0]?.queue_lane).toBe(expectedLane); + const duplicate = await h.service.completeSourceTurn(secondInput); + expect(duplicate).toMatchObject({ status: "existing", result: { l1MemoryId: first.l1MemoryId, jobs: [], duplicate: true } }); + expect(jobs(h.db, first.l1MemoryId)).toEqual(before); + expect(h.repos.memories.get(first.l1MemoryId)?.properties.internal_info.capture_origin).toBe(expectedOrigin); + expect(await h.service.completeSourceTurn({ ...secondInput, answer: "A conflicting answer." })) + .toMatchObject({ status: "conflict", reason: "source_turn_content_conflict" }); + expect(jobs(h.db, first.l1MemoryId)).toEqual(before); + expect(h.repos.memories.get(first.l1MemoryId)?.properties.internal_info.capture_origin).toBe(expectedOrigin); + await h.service.stop(); + h.db.close(); + const reopened = new MemoryDb({ path: join(h.root, "memory.sqlite") }); + const restarted = fixture.createTestMemoryService({ db: reopened, llm: createBatchReflectionLlm([]) }); + expect(await restarted.completeSourceTurn(secondInput)).toMatchObject({ status: "existing", result: { l1MemoryId: first.l1MemoryId, jobs: [] } }); + expect(jobs(reopened, first.l1MemoryId)).toEqual(before); + expect(new Repositories(reopened.db).memories.get(first.l1MemoryId)?.properties.internal_info.capture_origin).toBe(expectedOrigin); + }); + + it("keeps a real onboarding report and its embedding interactive during historical native backlog", async () => { + const h = harness(); + for (let index = 1; index <= 16; index++) await capture(h, turn(index)); + const report = h.service.addMemory({ + namespace, adapterId: "agent-source:memmy-onboarding", requestId: "scan-backlog-report", layer: "L1", + source: "memmy-onboarding", tags: ["agent-source", "first-encounter-report", "onboarding-report"], + title: "Memmy First Encounter Report", turnId: "scan-backlog-report", deferProcessing: true, + content: "## user\n\nGenerate the first encounter report.\n\n## assistant\n\nThe report and next step are ready." + }); + h.service.enqueuePendingImportSummaries(10_000, [report.id]); + expect(jobs(h.db, report.id)[0]).toMatchObject({ job_type: "import_summary", queue_lane: "interactive", base_priority: -100 }); + for (let round = 0; round < 16; round++) await h.service.runWorkerOnce(1); + expect(jobs(h.db, report.id)).toEqual(expect.arrayContaining([ + expect.objectContaining({ job_type: "embedding", queue_lane: "interactive", base_priority: -100, status: "succeeded" }) + ])); + expect(h.repos.processing.get(report.id)?.state).toBe("ready"); + }); + + it.each(["manual", "startup"] as const)("keeps the %s recovery override ahead of historical origin", async mode => { + const h = harness(); + const result = await capture(h, turn(1)); + // Represent an interrupted/missing queue row without changing the captured + // memory. Recovery must use its existing API and retain native provenance. + h.db.db.prepare("DELETE FROM evolution_jobs WHERE target_memory_id = ?").run(result.l1MemoryId); + if (mode === "manual") { + h.repos.processing.update(result.l1MemoryId, { state: "failed", stage: "summary", activeJobId: null, + errorCode: "llm_request_failed", errorMessage: "fixture transient failure", retryAction: "retry", + updatedAt: new Date().toISOString() }); + expect(h.service.retryMemoryProcessing(result.l1MemoryId, { namespace }).accepted).toBe(true); + } else h.service.reconcileWorkerStartup(); + expect(jobs(h.db, result.l1MemoryId)[0]).toMatchObject({ + job_type: "trace_summary", queue_lane: mode === "manual" ? "repair_retry" : "maintenance" + }); + expect(h.repos.memories.get(result.l1MemoryId)?.properties.internal_info.capture_origin).toBe("agent_source_scan"); + }); + + it.each(["full", "initial_subset", "incremental"] as const)("passes native %s scan provenance from the actual executor into MemoryService", async mode => { + const h = harness(); + const input = current(1); + const sourceTurn = { ...input.sourceTurn, query: input.query, answer: input.answer, status: "succeeded" as const, + toolCalls: [], toolResults: [] }; + const executor = createAgentSourceExecutor({ + service: h.service, configPath: join(h.root, "config.yaml"), statePath: join(h.root, "source-state.json"), + sourceRegistry: createSourceRegistry([{ + descriptor: { sourceId: "codex", displayName: "Codex fixture", builtin: true, dataPath: h.root }, + async detect() { return true; }, + async *scan() { + for (const [ordinal, role] of ["user", "assistant"].entries()) yield { + messageId: `native-message-${ordinal}`, sourceId: "codex", conversationId: sourceTurn.conversationId, + role: role as "user" | "assistant", content: role === "user" ? input.query : input.answer, + createdAt: role === "user" ? sourceTurn.startedAt : sourceTurn.completedAt, + workspacePath: null, gitRoot: null, + rawMeta: { sourceTurnState: "complete", sourceTurnId: sourceTurn.turnId, sourceTurn } + }; + } + }]) + }); + try { + await executor.startScan({ sourceId: "codex", mode }); + await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false)); + expect(executor.scanStatus().error).toBeNull(); + const rows = h.db.db.prepare("SELECT target_memory_id FROM evolution_jobs WHERE job_type = 'trace_summary'") + .all() as Array<{ target_memory_id: string }>; + expect(rows).toHaveLength(1); + const id = rows[0]!.target_memory_id; + expect(jobs(h.db, id)[0]?.queue_lane).toBe(mode === "incremental" ? "interactive" : "import"); + expect(h.repos.memories.get(id)?.properties.internal_info.capture_origin) + .toBe(mode === "incremental" ? undefined : "agent_source_scan"); + expect(h.repos.memories.get(id)?.properties.internal_info.plugin_algorithm).toBe("capture.v7"); + const before = jobs(h.db, id); + await executor.startScan({ sourceId: "codex", mode }); + await vi.waitFor(() => expect(executor.scanStatus().running).toBe(false)); + expect(executor.scanStatus().error).toBeNull(); + expect(jobs(h.db, id)).toEqual(before); + } finally { await executor.dispose(); } + }); +}); diff --git a/Memory/tests/service/session/turn-judge-routing.test.ts b/Memory/tests/service/session/turn-judge-routing.test.ts index 87f49acd8..ca3453905 100644 --- a/Memory/tests/service/session/turn-judge-routing.test.ts +++ b/Memory/tests/service/session/turn-judge-routing.test.ts @@ -68,7 +68,13 @@ function jobsOf(db: Db, jobType: string, episodeId?: string): Array["service"], rounds = 4): Promise { - for (let round = 0; round < rounds; round += 1) await service.runWorkerOnce(30); + for (let round = 0; round < rounds; round += 1) { + const result = await service.runWorkerOnce(30); + if (!result.leased) { + const row = service["repos"].db.prepare("SELECT MIN(available_at) AS at FROM evolution_jobs WHERE status='failed' AND failure_count < max_attempts AND blocked_reason IS NULL").get() as { at: string | null }; + if (row.at) { vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(Math.max(Date.now(), Date.parse(row.at))); } + } + } } describe("turn.judge.v1 routing (switch on)", () => { @@ -488,7 +494,7 @@ describe("representative feedback for the reward fallback", () => { const jobs = repo.runtime.leaseQueuedJobs(10, 60, undefined, false, ["trace_summary"]); expect(jobs.map(job => job.targetMemoryId)).toEqual([second.l1MemoryId]); expect(repo.runtime.leaseQueuedJobs(10, 60, [third.l1MemoryId], false, ["trace_summary"])).toEqual([]); - repo.runtime.failJob(jobs[0]!.id, "transient model failure"); + repo.runtime.failJob(jobs[0]!.id, "transient model failure", new Date().toISOString(), false, jobs[0]); expect(repo.runtime.leaseQueuedJobs(10, 60, [third.l1MemoryId], false, ["trace_summary"])).toEqual([]); repo.runtime.requeueFailedJobs(); await settle(service, 8); diff --git a/Memory/tests/service/session/turn-judge.test.ts b/Memory/tests/service/session/turn-judge.test.ts index cc0a8c011..86add7c12 100644 --- a/Memory/tests/service/session/turn-judge.test.ts +++ b/Memory/tests/service/session/turn-judge.test.ts @@ -1,5 +1,5 @@ import { assignedEpisodeId } from "../../fixtures/memory-service-fixture.js"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; import { captureJob, @@ -13,7 +13,7 @@ import { const { cleanup, createTestService } = createMemoryServiceFixture(); -afterEach(cleanup); +afterEach(async () => { vi.useRealTimers(); await cleanup(); }); describe("turn.judge.v1 capture (switch on)", () => { it("keeps the capture-only decision while the switch is off", async () => { @@ -61,10 +61,10 @@ describe("turn.judge.v1 capture (switch on)", () => { const session = openSession(service, "turn-judge-cutoff"); (await service.completeTurn("t1", { sessionId: session.sessionId, query: "第一轮：写部署脚本", answer: "第一轮回答：脚本在此。" })); await service.runWorkerOnce(20); - (await service.completeTurn("t2", { sessionId: session.sessionId, query: "第二轮：加上回滚", answer: "第二轮回答。" })); + const secondTurn = await service.completeTurn("t2", { sessionId: session.sessionId, query: "第二轮：加上回滚", answer: "第二轮回答。" }); (await service.completeTurn("t3", { sessionId: session.sessionId, query: "第三轮：再加日志", answer: "第三轮回答。" })); - // Judgments of one session run in turn order: the third waits for the second. - await service.runWorkerOnce(20); + // A targeted one-job budget exposes the intermediate ordering boundary. + await service.runWorkerOnce(1, { targetMemoryIds: [secondTurn.l1MemoryId] }); expect(judgeCalls.map((call) => call.user.slice(0, 3))).toEqual(["第一轮", "第二轮"]); await service.runWorkerOnce(20); @@ -152,6 +152,9 @@ describe("turn.judge.v1 capture (switch on)", () => { const turn = (await service.completeTurn("t1", { sessionId: session.sessionId, query: "帮我写一个部署脚本", answer: "好的。" })); await service.runWorkerOnce(20); (await service.completeTurn("t2", { sessionId: session.sessionId, query: "另一轮", answer: "好的。" })); + const retry = db.db.prepare("SELECT available_at FROM evolution_jobs WHERE job_type='trace_summary' AND target_memory_id=?").get(turn.l1MemoryId) as { available_at: string }; + expect(Date.parse(retry.available_at)).toBeGreaterThan(Date.now()); + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(Date.parse(retry.available_at)); await service.runWorkerOnce(20); const firstTurnCalls = judgeCalls.filter((call) => call.user.startsWith("帮我")); @@ -247,6 +250,9 @@ describe("turn.judge.v1 capture (switch on)", () => { toolResults: [{ toolCallId: "call-1", output: "MemTotal: 32GB", success: true }] })); await service.runWorkerOnce(20); + const retry = db.db.prepare("SELECT available_at FROM evolution_jobs WHERE job_type='trace_summary' AND target_memory_id=?").get(turn.l1MemoryId) as { available_at: string }; + expect(Date.parse(retry.available_at)).toBeGreaterThan(Date.now()); + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(Date.parse(retry.available_at)); await service.runWorkerOnce(20); expect(judgeCalls).toHaveLength(2); diff --git a/Memory/tests/service/worker/admission-lane-queue-selector.test.ts b/Memory/tests/service/worker/admission-lane-queue-selector.test.ts new file mode 100644 index 000000000..b4b7bdc13 --- /dev/null +++ b/Memory/tests/service/worker/admission-lane-queue-selector.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from "vitest"; +import { AdmissionLaneQueueSelector } from "../../../src/service/worker/admission-lane-queue-selector.js"; +import { DEFAULT_LANE_WEIGHTS, type LaneQueueCandidate } from "../../../src/service/worker/lane-queue-selector.js"; +import { QUEUE_LANES, type QueueLane } from "../../../src/service/worker/job-policy-registry.js"; + +function candidate(lane: QueueLane, enqueueSeq = 1): LaneQueueCandidate { + return { queue: "main", id: `${lane}-${enqueueSeq}`, lane, enqueueSeq, basePriority: 0 }; +} +function claim(selector: AdmissionLaneQueueSelector, available: QueueLane[], blocked: QueueLane[] = [], interactive = false): QueueLane | undefined { + const pool = available.map((lane, i) => candidate(lane, i + 1)); + selector.dropEmpty(new Set(QUEUE_LANES.filter(lane => !available.includes(lane)))); + while (pool.length) { + const item = (interactive ? selector.chooseInteractive(pool) : undefined) ?? selector.choose(pool)!; + pool.splice(pool.indexOf(item), 1); + if (blocked.includes(item.lane)) continue; + selector.commit(item.lane); + return item.lane; + } + return undefined; +} + +describe("admission-aware weighted lane turns", () => { + it("keeps the original weighted sequence without resource contention", () => { + const selector = new AdmissionLaneQueueSelector(); + const sequence = Array.from({ length: 24 }, () => claim(selector, [...QUEUE_LANES])); + const round = QUEUE_LANES.flatMap(lane => Array(DEFAULT_LANE_WEIGHTS[lane]).fill(lane)); + expect(sequence).toEqual([...round, ...round]); + expect(selector.getCursor()).toBe(0); + }); + + it.each([false, true])("preserves model opportunities across fast DB work with another permanently blocked Provider: %s", hasBlockedProvider => { + const selector = new AdmissionLaneQueueSelector(); + const available: QueueLane[] = [...(hasBlockedProvider ? ["interactive"] as const : []), "import", "evolution", "maintenance"]; + const permanentlyBlocked: QueueLane[] = hasBlockedProvider ? ["interactive"] : []; + const sequence: QueueLane[] = []; + for (let done = 0; done < 100; done++) { + const served = claim(selector, available, permanentlyBlocked)!; + if (served !== "maintenance") sequence.push(served); + for (let tick = 0; tick < 10; tick++) expect(claim(selector, available, + [...permanentlyBlocked, "import", "evolution"])).toBe("maintenance"); + } + expect(sequence).toHaveLength(100); + expect(sequence.filter(lane => lane === "import")).toHaveLength(50); + expect(sequence.filter(lane => lane === "evolution")).toHaveLength(50); + }); + + it("restores remaining turns and consumes no opportunity for preview or a failed CAS", () => { + const selector = new AdmissionLaneQueueSelector(); + claim(selector, [...QUEUE_LANES]); + const saved = selector.snapshot(); + const restored = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, selector.getCursor(), saved); + const before = restored.snapshot(); + expect(restored.choose([candidate("interactive")])?.lane).toBe("interactive"); + expect(restored.snapshot()).toEqual(before); + expect(Array.from({ length: 4 }, () => claim(restored, [...QUEUE_LANES]))).toEqual([ + "interactive", "interactive", "interactive", "repair_retry" + ]); + expect(saved.lanes[0]?.remaining).toBe(3); + }); + + it("gives a previously empty interactive lane a bounded turn without accumulating historical debt", () => { + const selector = new AdmissionLaneQueueSelector(); + for (let i = 0; i < 1000; i++) claim(selector, ["import", "evolution", "maintenance"]); + const sequence = Array.from({ length: 24 }, () => claim(selector, [...QUEUE_LANES])); + expect(sequence.slice(0, 12)).toContain("interactive"); + expect(sequence.filter(lane => lane === "interactive")).toHaveLength(8); + expect(selector.snapshot().lanes.every(turn => turn.remaining <= DEFAULT_LANE_WEIGHTS[turn.lane])).toBe(true); + }); + + it("keeps an empty lane skip tentative until a successful claim confirms it", () => { + const selector = new AdmissionLaneQueueSelector(); + const before = selector.snapshot(); + selector.dropEmpty(new Set(["interactive", "repair_retry"])); + expect(selector.choose([candidate("import")])?.lane).toBe("import"); + expect(selector.snapshot()).toEqual(before); + selector.dropEmpty(new Set(["repair_retry"])); + expect(selector.choose([candidate("interactive"), candidate("import")])?.lane).toBe("interactive"); + selector.commit("interactive"); + expect(selector.getCursor()).toBe(1); + }); + + it("sanitizes old or malformed optional state using the old cursor", () => { + for (const state of [undefined, null, {}, { version: 2, lanes: [] }, { version: 1, lanes: [] }, + { version: 1, lanes: QUEUE_LANES.map(lane => ({ lane, remaining: 0 })) }, + { version: 1, lanes: QUEUE_LANES.map(() => ({ lane: "import", remaining: 1 })) }]) { + const selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, 8, state); + expect(Array.from({ length: 3 }, () => claim(selector, [...QUEUE_LANES]))).toEqual(["import", "evolution", "evolution"]); + } + }); + + it("keeps FIFO across backing tables, validates duplicates and leaves input untouched", () => { + const selector = new AdmissionLaneQueueSelector(); + const old = Object.freeze({ ...candidate("repair_retry", 1), queue: "embedding_retry" as const }); + const newer = Object.freeze(candidate("repair_retry", 2)); + const input = Object.freeze([newer, old]); + expect(selector.choose(input)).toBe(old); + expect(input).toEqual([newer, old]); + expect(() => selector.choose([old, old])).toThrow(/Duplicate/); + }); + + it("restores the bounded aging round and sanitizes malformed optional aging state", () => { + const selector = new AdmissionLaneQueueSelector(); + selector.commit("evolution", true); + const saved = selector.snapshot(); + const aged = { ...candidate("maintenance"), firstQueuedAt: "2026-10-22T00:00:00.000Z" }; + const now = Date.parse("2026-10-23T00:00:00.000Z"); + expect(new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, 0, saved).chooseAged([aged], now, 300_000)).toBeUndefined(); + for (const aging of [undefined, null, [], {}, { servedMask: -1, used: true }, + { servedMask: 32, used: true }, { servedMask: 0, used: "true" }]) { + const restored = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, 0, { ...saved, aging }); + expect(restored.chooseAged([aged], now, 300_000)).toBe(aged); + expect(restored.snapshot().lanes).toEqual(saved.lanes); + } + saved.aging!.used = false; + expect(selector.snapshot().aging?.used).toBe(true); + }); + + it("does not advance a lane that already consumed its complete turn this round", () => { + const selector = new AdmissionLaneQueueSelector(); + selector.commit("maintenance"); + const aged = { ...candidate("maintenance"), firstQueuedAt: "2026-10-22T00:00:00.000Z" }; + expect(selector.chooseAged([aged], Date.parse("2026-10-23T00:00:00.000Z"), 300_000)).toBeUndefined(); + expect(() => selector.commit("maintenance", true)).toThrow(/unspent/); + }); + + it("applies the same metadata validation to aging and ordinary selection", () => { + const selector = new AdmissionLaneQueueSelector(); + const old = { ...candidate("evolution"), firstQueuedAt: "2026-10-22T00:00:00.000Z" }; + const now = Date.parse("2026-10-23T00:00:00.000Z"); + expect(() => selector.chooseAged([old, old], now, 300_000)).toThrow(/Duplicate/); + expect(() => selector.chooseAged([{ ...old, enqueueSeq: 0 }], now, 300_000)).toThrow(/Invalid enqueue/); + }); + + it("prefers unspent interactive tickets after an empty skip while retaining complete weighted rounds", () => { + const selector = new AdmissionLaneQueueSelector(); + expect(claim(selector, ["import", "evolution", "maintenance"], [], true)).toBe("import"); + expect(selector.snapshot().aging!.servedMask & 1).toBe(1); + expect(claim(selector, [...QUEUE_LANES], [], true)).toBe("interactive"); + const steady = new AdmissionLaneQueueSelector(); + const sequence = Array.from({ length: 36 }, () => claim(steady, [...QUEUE_LANES], [], true)); + const expected = QUEUE_LANES.flatMap(lane => Array(DEFAULT_LANE_WEIGHTS[lane]).fill(lane)); + expect(sequence).toEqual([...expected, ...expected, ...expected]); + }); + + it("does not refresh spent interactive tickets on restart, repeated previews or fast unrelated work", () => { + let selector = new AdmissionLaneQueueSelector(); + for (let i = 0; i < 4; i++) expect(claim(selector, [...QUEUE_LANES], [], true)).toBe("interactive"); + expect(selector.snapshot().interactive?.spent).toBe(true); + selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, selector.getCursor(), selector.snapshot()); + const before = selector.snapshot(); + for (let i = 0; i < 20; i++) expect(selector.chooseInteractive([candidate("interactive")])).toBeUndefined(); + expect(selector.snapshot()).toEqual(before); + for (let i = 0; i < 100; i++) expect(claim(selector, [...QUEUE_LANES], + ["interactive", "repair_retry", "import", "evolution"], true)).toBe("maintenance"); + expect(selector.chooseInteractive([candidate("interactive")])).toBeUndefined(); + // Ordinary WRR must also honor the older unfulfilled model opportunities. + expect(claim(selector, [...QUEUE_LANES], [], true)).toBe("repair_retry"); + }); + + it.each([4, 5, 6, 7, 8, 9, 10, 11])("advances exactly one existing ticket when upgrading old cursor %s", cursor => { + let selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, cursor); + const item = candidate("interactive"); + expect(selector.chooseInteractive([item])).toBe(item); + const preview = selector.snapshot(); + // A lost CAS or aborted transaction discards the selector without commit. + selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, cursor); + expect(selector.snapshot()).toEqual(preview); + selector.commit("interactive"); + expect(selector.snapshot().lanes.find(turn => turn.lane === "interactive")?.remaining).toBe(3); + const restored = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, selector.getCursor(), selector.snapshot()); + expect(restored.chooseInteractive([item])).toBeUndefined(); + expect(restored.choose([...QUEUE_LANES].map(lane => candidate(lane)))?.lane).not.toBe("interactive"); + }); + + it.each([0, 1, 2, 3])("uses only the current remaining quota in old interactive cursor segment %s", cursor => { + const selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, cursor), item = candidate("interactive"); + expect(selector.snapshot().interactive?.upgradeAdvance).toBe(false); + for (let i = 0; i < 4 - cursor; i++) { expect(selector.chooseInteractive([item])).toBe(item); selector.commit("interactive"); } + expect(selector.chooseInteractive([item])).toBeUndefined(); + }); + + it("distinguishes old turns from explicit new state and never regenerates a consumed upgrade advance", () => { + const lanes = ["import", "evolution", "maintenance", "interactive", "repair_retry"] as QueueLane[]; + const old = { version: 1, lanes: lanes.map(lane => ({ lane, remaining: DEFAULT_LANE_WEIGHTS[lane] })), + aging: { servedMask: 3, used: false } }; + const item = candidate("interactive"); + const selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, 7, old); + expect(selector.chooseInteractive([item])).toBe(item); + selector.commit("interactive"); + const spent = selector.snapshot(); + for (let restart = 0; restart < 20; restart++) { + const restored = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, selector.getCursor(), spent); + expect(restored.chooseInteractive([item])).toBeUndefined(); + } + expect(new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, 7, + { ...old, interactive: { spent: true } }).chooseInteractive([item])).toBeUndefined(); + const empty = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, 7, + { ...old, interactive: { spent: false } }); + for (let i = 0; i < 4; i++) { expect(empty.chooseInteractive([item])).toBe(item); empty.commit("interactive"); } + expect(empty.chooseInteractive([item])).toBeUndefined(); + }); + + it.each([4, 7, 9, 11])("retains every lane's long-term weighted share after one upgrade advance at cursor %s", cursor => { + const selector = new AdmissionLaneQueueSelector(DEFAULT_LANE_WEIGHTS, cursor); + const sequence = Array.from({ length: 144 }, () => claim(selector, [...QUEUE_LANES], [], true)); + const steady = sequence.slice(-120); + for (const lane of QUEUE_LANES) expect(steady.filter(value => value === lane)).toHaveLength(DEFAULT_LANE_WEIGHTS[lane] * 10); + }); +}); diff --git a/Memory/tests/service/worker/embedding-maintenance-history.test.ts b/Memory/tests/service/worker/embedding-maintenance-history.test.ts index e7636d9da..a11264d3e 100644 --- a/Memory/tests/service/worker/embedding-maintenance-history.test.ts +++ b/Memory/tests/service/worker/embedding-maintenance-history.test.ts @@ -23,7 +23,11 @@ describe("embedding maintenance current failures", () => { } await service.rebuildEmbeddings("repair"); call.mockRejectedValue(new Error("embedding provider unavailable")); - for (let attempt = 0; attempt < 6; attempt++) await service.runWorkerOnce(10); + // Make retries due explicitly; consecutive immediate drains preserve backoff. + for (let attempt = 0; attempt < 6; attempt++) { + db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE status='failed'").run(); + await service.runWorkerOnce(10); + } expect(db.db.prepare("SELECT attempts,status FROM evolution_jobs WHERE job_type='user_memory_embedding'").all()) .toEqual([{ attempts: 6, status: "dead_letter" }, { attempts: 6, status: "dead_letter" }]); call.mockResolvedValue([1, 0, 0]); @@ -57,7 +61,11 @@ describe("embedding maintenance current failures", () => { expect(changed).toMatchObject({ missing: 1, retryQueue: { failed: 0 } }); await service.rebuildEmbeddings("repair"); call.mockRejectedValue(new Error("new content still fails")); - for (let attempt = 0; attempt < 6; attempt++) await service.runWorkerOnce(10); + // Make retries due explicitly; consecutive immediate drains preserve backoff. + for (let attempt = 0; attempt < 6; attempt++) { + db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE status='failed'").run(); + await service.runWorkerOnce(10); + } expect(await service.embeddingMaintenanceStats()).toMatchObject({ missing: 1, retryQueue: { failed: 1 } }); }); diff --git a/Memory/tests/service/worker/embedding-retry-reproduction.test.ts b/Memory/tests/service/worker/embedding-retry-reproduction.test.ts index cacbdbd73..86021c760 100644 --- a/Memory/tests/service/worker/embedding-retry-reproduction.test.ts +++ b/Memory/tests/service/worker/embedding-retry-reproduction.test.ts @@ -315,6 +315,7 @@ describe("v1.2.6 embedding retry reproduction", () => { expect(runtime.markEmbeddingRetrySucceededClaimed(original.id, { workerId: "worker-a", leaseUntil: 10_000, + leaseToken: claimed!.leaseToken!, now: 3_000 })).toMatchObject({ status: "succeeded" }); expect(runtime.claimDueEmbeddingRetries({ @@ -336,11 +337,12 @@ describe("v1.2.6 embedding retry reproduction", () => { "UPDATE embedding_retry_queue SET status = 'in_progress', claimed_by = NULL, lease_until = NULL WHERE id = ?" ).run(retry.id); expect(runtime.nextWorkerRunAt()).toBe(1_000); - expect(runtime.claimDueEmbeddingRetries({ + const claims = runtime.claimDueEmbeddingRetries({ now: 2_000, workerId: "worker-b", leaseUntil: 10_000, limit: 1 - })).toMatchObject([{ id: retry.id, claimedBy: "worker-b", leaseUntil: 10_000 }]); + }); + expect(claims).toMatchObject([{ id: retry.id, claimedBy: "worker-b", leaseUntil: 10_000 }]); expect(runtime.markEmbeddingRetrySucceededClaimed(retry.id, { - workerId: "worker-b", leaseUntil: 10_000, now: 3_000 + workerId: "worker-b", leaseUntil: 10_000, leaseToken: claims[0]!.leaseToken!, now: 3_000 })).toMatchObject({ status: "succeeded" }); }); @@ -359,7 +361,7 @@ describe("v1.2.6 embedding retry reproduction", () => { "UPDATE embedding_retry_queue SET status = 'in_progress', claimed_by = ?, lease_until = ? WHERE id = ?" ).run("worker-a", 10_000, active.id); db.db.prepare( - "UPDATE embedding_retry_queue SET status = 'in_progress', claimed_by = ?, lease_until = ? WHERE id = ?" + "UPDATE embedding_retry_queue SET status = 'in_progress', claimed_by = ?, lease_until = ?, lease_token = 'expired-lease' WHERE id = ?" ).run("worker-old", 1_500, expired.id); expect(runtime.embeddingRetryQueueStats(2_000)).toEqual({ @@ -369,7 +371,7 @@ describe("v1.2.6 embedding retry reproduction", () => { now: 2_000, workerId: "worker-b", leaseUntil: 12_000, limit: 2 })).toMatchObject([{ id: expired.id, claimedBy: "worker-b" }]); expect(runtime.markEmbeddingRetrySucceededClaimed(expired.id, { - workerId: "worker-old", leaseUntil: 1_500, now: 3_000 + workerId: "worker-old", leaseUntil: 1_500, leaseToken: "expired-lease", now: 3_000 })).toBeUndefined(); expect(runtime.embeddingRetryQueueStats(2_000)).toEqual({ pending: 0, active: 2, stale: 0, failed: 0 diff --git a/Memory/tests/service/worker/episode-title-origin-projection.test.ts b/Memory/tests/service/worker/episode-title-origin-projection.test.ts new file mode 100644 index 000000000..19559b8fd --- /dev/null +++ b/Memory/tests/service/worker/episode-title-origin-projection.test.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMemoryServiceFixture, createBatchReflectionLlm, settleCaptureAssignments, assignedEpisodeId } from "../../fixtures/memory-service-fixture.js"; +import { Repositories } from "../../../src/storage/repositories.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => { vi.restoreAllMocks(); fixture.cleanup(); }); + +async function harness() { + const h = fixture.createTestService({ llm: createBatchReflectionLlm([]) }); + const captured = await h.service.completeSourceTurn({ + namespace: { source: "codex", profileId: "default", userId: "projection-user" }, + source: "codex", channel: "agent_source_scan", captureLegacyHistory: true, + sourceTurn: { source: "codex", profileId: "default", conversationId: "projection-conversation", turnId: "turn-1", + startedAt: "2000-01-01T10:00:00.000Z", completedAt: "2000-01-01T10:00:30.000Z", + completionEvidence: "complete:1" }, + query: "Implement a transaction test.", answer: "The transaction test is complete." + }); + expect(captured.status).toBe("stored"); + const result = captured.result!; + await settleCaptureAssignments(h.service); + result.episodeId = assignedEpisodeId(h.service, result); + const repos = new Repositories(h.db.db); + function memory(id: string, origin: unknown = "agent_source_scan", agentId = "codex", tags: string[] = [], deletedAt: string | null = null) { + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,agent_id,tags_json,properties_json, + deleted_at,created_at,updated_at) VALUES (?,?,'projection-user',?,'L1',?,?,?,?,?,?)`) + .run(id, "2000-01-01", "Long synthetic conversation text. ".repeat(2048), agentId, JSON.stringify(tags), + JSON.stringify({ internal_info: { capture_origin: origin, trace: { userText: "Synthetic private trace body. ".repeat(2048) } } }), + deletedAt, "2000-01-01", "2000-01-01"); + } + return { ...h, repos, result, memory }; +} + +describe("episode title provenance uses metadata without memory/vector hydration", async () => { + it.each([ + { name: "wholly historical", ids: ["history"], expected: "import" }, + { name: "mixed historical/current", ids: ["history", "current"], expected: "interactive" }, + { name: "current only", ids: ["current"], expected: "interactive" }, + { name: "unknown origin", ids: ["unknown"], expected: "interactive" }, + { name: "origin casing is not canonical", ids: ["uppercase"], expected: "interactive" }, + { name: "empty episode", ids: [], expected: "interactive" }, + { name: "missing memory", ids: ["history", "missing"], expected: "interactive" }, + { name: "soft-deleted memory", ids: ["history", "deleted"], expected: "interactive" }, + { name: "duplicate historical ids", ids: ["history", "history"], expected: "import" }, + { name: "duplicate current ids", ids: ["current", "current"], expected: "interactive" }, + { name: "onboarding agent", ids: ["history", "onboarding-agent"], expected: "interactive" }, + { name: "first report tag", ids: ["history", "first-report"], expected: "interactive" }, + { name: "onboarding report tag", ids: ["history", "onboarding-report"], expected: "interactive" }, + { name: "ordinary tag", ids: ["ordinary"], expected: "import" } + ])("keeps $name semantics without loading bodies", async ({ ids, expected }) => { + const h = await harness(); + h.memory("history"); h.memory("current", null); h.memory("unknown", "hook"); + h.memory("uppercase", "AGENT_SOURCE_SCAN"); h.memory("deleted", "agent_source_scan", "codex", [], "2020-01-01"); + h.memory("onboarding-agent", "agent_source_scan", "MeMmY-OnBoArDiNg"); + h.memory("first-report", "agent_source_scan", "codex", ["FiRsT-EnCoUnTeR-RePoRt"]); + h.memory("onboarding-report", "agent_source_scan", "codex", ["ONBOARDING-REPORT"]); + h.memory("ordinary", "agent_source_scan", "codex", ["database", "test"]); + h.db.db.prepare("UPDATE episodes SET l1_memory_ids_json = ? WHERE id = ?").run(JSON.stringify(ids), h.result.episodeId); + const memoryHydration = vi.spyOn(h.service["repos"].memories, "getMany"); + const vectorHydration = vi.spyOn(h.service["repos"].vectors, "getMany"); + const sql = vi.spyOn(h.db.db, "prepare"); + const queued = h.service["workerHandlers"].enqueueJob({ jobType: "episode_title", userId: "projection-user", + sessionId: h.result.sessionId, episodeId: h.result.episodeId, + payload: { stage: "final", captureOrigin: "agent_source_scan" } }); + expect(queued.queueLane).toBe(expected); + expect(queued.payload.captureOrigin).toBe(expected === "import" ? "agent_source_scan" : undefined); + expect(memoryHydration).not.toHaveBeenCalled(); + expect(vectorHydration).not.toHaveBeenCalled(); + expect(sql.mock.calls.map(call => call[0]).join("\n")).not.toMatch(/SELECT\s+\*\s+FROM\s+memories|FROM\s+memory_vector_entries/i); + }); + + it("does not hydrate a growing historical episode when refreshing a deduplicated provisional title", async () => { + const h = await harness(); + const episode = h.repos.runtime.getEpisode(h.result.episodeId)!; + h.db.db.prepare("UPDATE episodes SET l1_memory_ids_json = ? WHERE id = ?") + .run(JSON.stringify(Array.from({ length: 2000 }, () => h.result.l1MemoryId)), episode.id); + h.service["workerHandlers"].enqueueJob({ jobType: "episode_title", userId: episode.userId, + sessionId: episode.sessionId, episodeId: episode.id, payload: { stage: "provisional" } }); + const original = h.db.db.prepare("SELECT id,enqueue_seq FROM evolution_jobs WHERE episode_id = ? AND job_type = 'episode_title'") + .get(episode.id) as { id: string; enqueue_seq: number }; + const memoryHydration = vi.spyOn(h.service["repos"].memories, "getMany"); + const vectorHydration = vi.spyOn(h.service["repos"].vectors, "getMany"); + for (let index = 0; index < 3; index++) { + const job = h.service["workerHandlers"].enqueueJob({ jobType: "episode_title", userId: episode.userId, + sessionId: episode.sessionId, episodeId: episode.id, payload: { stage: "provisional" } }); + expect(job).toMatchObject({ id: original.id, enqueueSeq: original.enqueue_seq, queueLane: "import" }); + } + expect(memoryHydration).not.toHaveBeenCalled(); + expect(vectorHydration).not.toHaveBeenCalled(); + }); + + it("matches memory hydration for malformed legacy tags without treating an object as an onboarding tag list", async () => { + const h = await harness(); + h.db.db.prepare("UPDATE memories SET tags_json = ? WHERE id = ?") + .run(JSON.stringify({ label: "first-encounter-report" }), h.result.l1MemoryId); + expect(h.repos.memories.getMany([h.result.l1MemoryId])[0]?.tags).not.toContain("first-encounter-report"); + expect(h.repos.memories.areAllHistoricalCaptureMemories([h.result.l1MemoryId])).toBe(true); + }); + + it.each(["info_json", "properties_json"] as const)("keeps report tags stored only in %s interactive", async column => { + const h = await harness(); + h.db.db.prepare(`UPDATE memories SET ${column} = json_set(${column}, '$.tags', json(?)) WHERE id = ?`) + .run(JSON.stringify(["FIRST-ENCOUNTER-REPORT"]), h.result.l1MemoryId); + expect(h.repos.memories.getMany([h.result.l1MemoryId])[0]?.tags).toContain("FIRST-ENCOUNTER-REPORT"); + const hydration = vi.spyOn(h.service["repos"].memories, "getMany"); + const queued = h.service["workerHandlers"].enqueueJob({ jobType: "episode_title", userId: "projection-user", + sessionId: h.result.sessionId, episodeId: h.result.episodeId, payload: { stage: "final" } }); + expect(queued.queueLane).toBe("interactive"); + expect(queued.payload.captureOrigin).toBeUndefined(); + expect(hydration).not.toHaveBeenCalled(); + }); + + it("ignores a caller-supplied historical origin when the title has no verified episode", async () => { + const h = await harness(); + const queued = h.service["workerHandlers"].enqueueJob({ jobType: "episode_title", userId: "projection-user", + payload: { stage: "final", captureOrigin: "agent_source_scan" } }); + expect(queued.queueLane).toBe("interactive"); + expect(queued.payload.captureOrigin).toBeUndefined(); + }); +}); diff --git a/Memory/tests/service/worker/job-policy-registry.test.ts b/Memory/tests/service/worker/job-policy-registry.test.ts new file mode 100644 index 000000000..a4dda7346 --- /dev/null +++ b/Memory/tests/service/worker/job-policy-registry.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "vitest"; +import { MEMORY_WORKER_JOB_TYPES } from "@memmy/agent-source-core"; +import { + JOB_POLICY_VERSION, JobPolicyRegistry, REGISTERED_JOB_TYPES, UnknownJobPolicyError, isImportedPolicyTarget +} from "../../../src/service/worker/job-policy-registry.js"; + +const registry = new JobPolicyRegistry(); + +describe("job policy registry", () => { + it("covers every shared worker type, locally scheduled idle flush, and the retry adapter", () => { + const types = [...MEMORY_WORKER_JOB_TYPES, "work_memory_idle_flush", "embedding_retry"]; + expect(new Set(REGISTERED_JOB_TYPES)).toEqual(new Set(types)); + for (const jobType of types) { + expect(registry.resolve({ jobType }).policyVersion).toBe(JOB_POLICY_VERSION); + } + }); + + it("admits episode assignment as model work and finalization as local work, retaining scan provenance", () => { + expect(registry.resolve({ jobType: "episode_assignment" })).toMatchObject({ + queueLane: "interactive", resourceClass: "summary", concurrency: { providerRoles: ["summary"] } + }); + expect(registry.resolve({ jobType: "episode_finalize" })).toMatchObject({ + queueLane: "interactive", resourceClass: "database", concurrency: { providerRoles: [] } + }); + for (const jobType of ["episode_assignment", "episode_finalize"]) { + expect(registry.resolve({ jobType, payload: { captureOrigin: "agent_source_scan" } }).queueLane).toBe("import"); + } + }); + + it("keeps unknown and inherited property names explicit errors", () => { + for (const jobType of ["unregistered_task", "toString", "__proto__", ""]) { + expect(() => registry.resolve({ jobType })).toThrow(UnknownJobPolicyError); + } + }); + + it("separates normal historical import, startup bulk repair, and user retry", () => { + expect(registry.resolve({ jobType: "import_summary" }).queueLane).toBe("import"); + for (const source of ["startup.processing_repair", "startup.placeholder_summary_repair"]) { + expect(registry.resolve({ jobType: "import_summary", payload: { source } }).queueLane).toBe("maintenance"); + } + expect(registry.resolve({ + jobType: "import_summary", payload: { source: "memory.processing.manual_retry" } + }).queueLane).toBe("repair_retry"); + expect(registry.resolve({ + jobType: "import_summary", payload: { source: "memory.processing.lifecycle_retry" } + }).queueLane).toBe("import"); + }); + + it("recognizes the actual startup embedding reason and observation backfill producers", () => { + for (const reason of ["startup.processing_repair", "startup.retrieval_document_v2"]) { + expect(registry.resolve({ jobType: "embedding", payload: { reason }, target: { memoryLayer: "L1" } }).queueLane) + .toBe("maintenance"); + } + expect(registry.resolve({ jobType: "observation_extract", payload: { source: "startup.observation_backfill" } }).queueLane) + .toBe("maintenance"); + expect(registry.resolve({ + jobType: "embedding", payload: { reason: "startup.processing_repair" }, target: { agentId: "memmy-onboarding" } + }).queueLane).toBe("interactive"); + }); + + it("classifies historical summaries by origin without demoting fresh Computer History", () => { + for (const jobType of ["trace_summary", "import_summary", "observation_extract"]) { + expect(registry.resolve({ jobType, target: { tags: ["agent-source"] } }).queueLane).toBe("import"); + expect(registry.resolve({ jobType, payload: { source: "agent_source.scan.summary_stage" } }).queueLane).toBe("import"); + } + expect(registry.resolve({ + jobType: "embedding", payload: { reason: "import.summary.updated" } + }).queueLane).toBe("import"); + expect(registry.resolve({ + jobType: "observation_extract", payload: { source: "evidence.sync" }, target: { tags: ["external-evidence", "computer_history"] } + }).queueLane).toBe("interactive"); + }); + + it.each([ + { agentId: "Memmy-Onboarding" }, + { tags: ["First-Encounter-Report"] }, + { tags: ["ONBOARDING-REPORT"] } + ])("retains first-report urgency through import and startup repair: %j", (target) => { + const policy = registry.resolve({ + jobType: "import_summary", target, + payload: { source: "startup.placeholder_summary_repair" } + }); + expect(policy).toMatchObject({ queueLane: "interactive", basePriority: -100 }); + // The report's later reflection is background evolution, not a first-report result. + expect(registry.resolve({ jobType: "reflection", target }).queueLane).toBe("evolution"); + }); + + it("classifies embedding from a narrow memory projection without reading the content", () => { + expect(registry.resolve({ jobType: "embedding", target: { memoryLayer: "L1" } }).queueLane).toBe("interactive"); + for (const target of [ + { memoryLayer: "L1", tags: ["Agent-Source"] }, + { memoryLayer: "L1", pluginAlgorithm: "memory.add.import_async.v2" } + ]) { + expect(registry.resolve({ jobType: "embedding", target }).queueLane).toBe("import"); + } + expect(registry.resolve({ jobType: "embedding", target: { memoryLayer: "L2" } }).queueLane).toBe("evolution"); + expect(registry.resolve({ jobType: "embedding_retry" }).queueLane).toBe("repair_retry"); + expect(registry.resolve({ jobType: "user_memory_embedding" }).queueLane).toBe("interactive"); + }); + + it.each(["trace_summary", "import_summary", "observation_extract", "embedding"])( + "routes native historical %s by scheduling provenance without changing import-pipeline identity", jobType => { + const target = { memoryLayer: "L1", tags: ["database", "test"], pluginAlgorithm: "capture.v7", captureOrigin: "agent_source_scan" }; + expect(isImportedPolicyTarget(target)).toBe(false); + expect(registry.resolve({ jobType, target })).toMatchObject({ queueLane: "import", basePriority: 2 }); + expect(registry.resolve({ jobType, target }).resourceClass).toBe(registry.resolve({ jobType }).resourceClass); + expect(registry.resolve({ jobType, target }).concurrency).toEqual(registry.resolve({ jobType }).concurrency); + } + ); + + it("preserves first-report, manual retry, startup repair and retry-queue policy over historical provenance", () => { + const target = { memoryLayer: "L1", captureOrigin: "agent_source_scan" }; + expect(registry.resolve({ jobType: "trace_summary", target: { ...target, agentId: "memmy-onboarding" } })) + .toMatchObject({ queueLane: "interactive", basePriority: -100 }); + expect(registry.resolve({ jobType: "trace_summary", target, payload: { source: "memory.processing.manual_retry" } }).queueLane).toBe("repair_retry"); + expect(registry.resolve({ jobType: "embedding", target, payload: { reason: "startup.processing_repair" } }).queueLane).toBe("maintenance"); + expect(registry.resolve({ jobType: "embedding_retry", target }).queueLane).toBe("repair_retry"); + expect(registry.resolve({ jobType: "reflection", target }).queueLane).toBe("evolution"); + expect(registry.resolve({ jobType: "user_memory_embedding", target }).queueLane).toBe("interactive"); + }); + + it("does not infer historical provenance from a native agent, payload hint, or an inexact origin", () => { + for (const target of [ + { agentId: "claude_code", pluginAlgorithm: "capture.v7" }, + { captureOrigin: "hook" }, { captureOrigin: "agent_source_scan " }, { captureOrigin: "AGENT_SOURCE_SCAN" } + ]) { + expect(registry.resolve({ jobType: "trace_summary", target }).queueLane).toBe("interactive"); + expect(registry.resolve({ jobType: "embedding", target: { ...target, memoryLayer: "L1" } }).queueLane).toBe("interactive"); + } + expect(registry.resolve({ jobType: "trace_summary", payload: { captureOrigin: "agent_source_scan" } }).queueLane).toBe("interactive"); + }); + + it("accepts explicit operation modes without trusting arbitrary payload fields", () => { + expect(registry.resolve({ jobType: "embedding", operationMode: "historical_import" }).queueLane).toBe("import"); + expect(registry.resolve({ jobType: "embedding_retry", operationMode: "maintenance" }).queueLane).toBe("maintenance"); + expect(registry.resolve({ jobType: "embedding", operationMode: "manual_retry" }).queueLane).toBe("repair_retry"); + expect(registry.resolve({ jobType: "import_summary", operationMode: "onboarding_first_report" }).queueLane).toBe("interactive"); + expect(registry.resolve({ jobType: "reflection", payload: { queueLane: "interactive" } }).queueLane).toBe("evolution"); + }); + + it("describes actual model/resource families and scoped work without changing execution", () => { + expect(registry.resolve({ jobType: "work_memory_extract" })).toMatchObject({ + queueLane: "evolution", resourceClass: "summary", + concurrency: { providerRoles: ["summary", "embedding"], scope: "user_project", batchable: false } + }); + expect(registry.resolve({ jobType: "l3_world_model_update" })).toMatchObject({ + resourceClass: "evolution_llm", concurrency: { scope: "user_project" } + }); + expect(registry.resolve({ jobType: "project_environment_profile" }).queueLane).toBe("maintenance"); + expect(registry.resolve({ jobType: "decision_repair" }).queueLane).toBe("repair_retry"); + expect(registry.resolve({ jobType: "negative_experience" }).concurrency.providerRoles).toContain("evolution"); + expect(registry.resolve({ jobType: "episode_idle_close" }).concurrency.providerRoles).toEqual([]); + }); + + it("retains current lease durations and keeps batch size independent from request admission", () => { + const main = registry.resolve({ jobType: "embedding" }); + const retry = registry.resolve({ jobType: "embedding_retry" }); + expect(main.lease).toEqual({ durationSec: 60, heartbeatMs: 20_000 }); + expect(retry.lease).toEqual({ durationSec: 300, heartbeatMs: 20_000 }); + expect(main.concurrency.batchable).toBe(true); + expect(retry.concurrency.batchable).toBe(true); + expect(Object.isFrozen(main)).toBe(true); + expect(Object.isFrozen(main.concurrency.providerRoles)).toBe(true); + }); +}); diff --git a/Memory/tests/service/worker/lane-queue-selector.test.ts b/Memory/tests/service/worker/lane-queue-selector.test.ts new file mode 100644 index 000000000..554fec2a5 --- /dev/null +++ b/Memory/tests/service/worker/lane-queue-selector.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { QUEUE_LANES, type QueueLane } from "../../../src/service/worker/job-policy-registry.js"; +import { + LaneQueueSelector, DEFAULT_LANE_WEIGHTS, validateLaneWeights, type LaneQueueCandidate +} from "../../../src/service/worker/lane-queue-selector.js"; + +function pool() { + return QUEUE_LANES.flatMap((lane, laneIndex) => Array.from({ length: 20 }, (_, index) => ({ + queue: "main" as const, lane, enqueueSeq: laneIndex * 20 + index + 1, + id: `${lane}-${index}`, basePriority: 0, extra: `business-${lane}` + }))); +} +function remove(pending: readonly T[], selected: readonly T[]): T[] { + const ids = new Set(selected.map(item => `${item.queue}:${item.id}`)); + return pending.filter(item => !ids.has(`${item.queue}:${item.id}`)); +} + +describe("production lane selection", () => { + it("preserves a weighted round across single-choice drains and a restored cursor", () => { + const first = new LaneQueueSelector(); + let pending = pool(); + const totals = Object.fromEntries(QUEUE_LANES.map(lane => [lane, 0])) as Record; + for (let round = 0; round < 12; round++) { + const selector = new LaneQueueSelector(DEFAULT_LANE_WEIGHTS, first.getCursor()); + const selected = selector.select(pending, 1); + first.setCursor(selected.nextCursor); + totals[selected.selected[0]!.lane]++; + pending = remove(pending, selected.selected); + } + expect(totals).toEqual(DEFAULT_LANE_WEIGHTS); + expect(first.getCursor()).toBe(0); + }); + + it("allows a transaction preview without consuming the durable cursor position", () => { + const selector = new LaneQueueSelector(); + const input = pool(); + const preview = selector.preview(input, 5); + expect(preview.nextCursor).toBe(5); + expect(selector.getCursor()).toBe(0); + expect(selector.preview(input, 5)).toEqual(preview); + const committed = selector.select(input, 5); + expect(committed).toEqual(preview); + expect(selector.getCursor()).toBe(5); + expect(committed.selected[0]!.extra).toBe("business-interactive"); + }); + + it("keeps an explicit non-advancing selection isolated from a later committed selection", () => { + const selector = new LaneQueueSelector(DEFAULT_LANE_WEIGHTS, 4); + expect(selector.select(pool(), 3, { advanceCursor: false }).selected.every(item => item.lane === "repair_retry")).toBe(true); + expect(selector.getCursor()).toBe(4); + selector.select(pool(), 1); + expect(selector.getCursor()).toBe(5); + }); + + it("uses runnable FIFO across backing tables and skips absent lanes without reserving slots", () => { + const selector = new LaneQueueSelector(); + const oldest = Object.freeze({ queue: "embedding_retry" as const, id: "old", lane: "repair_retry" as const, enqueueSeq: 1, basePriority: 100 }); + const newest = Object.freeze({ queue: "main" as const, id: "new", lane: "repair_retry" as const, enqueueSeq: 20, basePriority: -100 }); + const input = Object.freeze([newest, oldest]); + expect(selector.select(input, 4).selected).toEqual([oldest, newest]); + expect(input).toEqual([newest, oldest]); + expect(selector.select([], 4).selected).toEqual([]); + }); + + it("normalizes persisted offsets after a weight change and rejects corrupt cursors", () => { + const selector = new LaneQueueSelector(DEFAULT_LANE_WEIGHTS, 14); + expect(selector.getCursor()).toBe(2); + selector.setCursor(12); + expect(selector.getCursor()).toBe(0); + for (const bad of [-1, 0.5, Number.NaN, Number.POSITIVE_INFINITY]) expect(() => selector.setCursor(bad)).toThrow(RangeError); + }); + + it("detaches weights and requires exactly five positive integer weights", () => { + const weights = { ...DEFAULT_LANE_WEIGHTS }; + const selector = new LaneQueueSelector(weights); + weights.interactive = 1; + expect(selector.select(pool(), 4).selected.every(item => item.lane === "interactive")).toBe(true); + expect(Object.isFrozen(validateLaneWeights(DEFAULT_LANE_WEIGHTS))).toBe(true); + for (const bad of [null, [], {}, { ...DEFAULT_LANE_WEIGHTS, extra: 1 }, + { ...DEFAULT_LANE_WEIGHTS, import: "2" }, { ...DEFAULT_LANE_WEIGHTS, import: 0 }, + { ...DEFAULT_LANE_WEIGHTS, import: 1025 }, { ...DEFAULT_LANE_WEIGHTS, import: Number.NaN }]) { + expect(() => validateLaneWeights(bad)).toThrow(); + } + }); +}); diff --git a/Memory/tests/service/worker/queue-scheduling-baseline.repro.test.ts b/Memory/tests/service/worker/queue-scheduling-baseline.repro.test.ts new file mode 100644 index 000000000..8df92f644 --- /dev/null +++ b/Memory/tests/service/worker/queue-scheduling-baseline.repro.test.ts @@ -0,0 +1,363 @@ +/** + * Stage 1 diagnostics, with repaired cases converted to stage 2 regressions. + * + * Scheduling diagnostics explicitly select the legacy rollback algorithm. + * Lease/recovery/backoff cases assert the repaired stage 2 behavior. + * Cross-entry overlap is a repaired stage 4 regression. Stage 6 keeps the legacy + * selection diagnostic but asserts safe admission and continuous execution. + * SQLite, schema, repositories and WorkerRunner are real. Business handlers and + * model requests are controlled local substitutes so queue timing is repeatable. + */ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ root: string; db: MemoryDb }> = []; +const observations: Array<{ scenario: string; [key: string]: unknown }> = []; +const epoch = Date.parse("2026-10-22T00:00:00.000Z"); + +function record(scenario: string, values: Record): void { + observations.push({ scenario, ...values }); +} + +async function flushWorker() { for (let i = 0; i < 100; i++) await Promise.resolve(); } + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +function harness(processJob: WorkerRunnerDeps["jobHandlers"]["processJob"] = async () => {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-queue-baseline-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const deps: WorkerRunnerDeps = { + repos, + scheduling: { algorithm: "legacy" }, + embedder: createCapturingEmbedder([]), + capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "baseline-runner", + memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, + memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, + nowIso: () => new Date().toISOString(), + encodeChangeCursor: String, + namespaceIdFromMemory: () => "baseline", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ + orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] + }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement retry"); }, + appendJobChange: () => {}, + appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding job"); }, + applyEmbeddingVector: () => {}, + // Only the queue-state transition is relevant to these tests. + persistEmbeddingVector: (input) => { + const target = repos.memories.get(input.memoryId); + if (!target) throw new Error("missing synthetic retry target"); + input.finalize?.(target, false, new Date().toISOString()); + return target; + }, + enqueueEmbeddingRetryAfterFailure: () => false + } + }; + const runner = new WorkerRunner(deps); + function memory(id: string, createdAt = new Date(epoch).toISOString(), imported = true): void { + db.db.prepare(`INSERT INTO memories + (id, timeline, user_id, memory_value, memory_layer, tags_json, created_at, updated_at) + VALUES (?, ?, 'baseline-user', ?, 'L1', ?, ?, ?)`) + .run(id, createdAt, `synthetic memory ${id}`, imported ? '["agent-source"]' : '[]', createdAt, createdAt); + } + function job(id: string, jobType: EvolutionJobRecord["jobType"], options: { + target?: string; source?: string; createdAt?: string; runAfter?: string; + } = {}): void { + const at = options.createdAt ?? new Date(epoch).toISOString(); + repos.runtime.enqueueJob({ + id, jobType, status: "queued", userId: "baseline-user", targetMemoryId: options.target, + payload: { ...(options.source ? { source: options.source } : {}), + ...(options.runAfter ? { runAfter: options.runAfter } : {}) }, + attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at + }); + } + function retry(id: string, target: string): void { + repos.runtime.enqueueEmbeddingRetry({ + id, targetKind: "trace", targetId: target, vectorField: "vec_summary", + sourceText: `synthetic memory ${target}`, now: Date.now() + }); + } + return { db, repos, runner, memory, job, retry }; +} + +afterEach(() => { + vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +afterAll(() => { + const path = process.env.MEMMY_QUEUE_BASELINE_RESULTS; + if (path) { + // The same diagnostics become regression tests as implementation advances. + // A caller-supplied label must never masquerade as the code actually run. + let testedCommit = "unavailable"; + let workingTreeDirty: boolean | null = null; + try { + testedCommit = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + workingTreeDirty = execFileSync("git", ["status", "--porcelain"], { encoding: "utf8" }).trim().length > 0; + } catch { /* An exported source archive may not include Git metadata. */ } + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, JSON.stringify({ + testedCommit, + workingTreeDirty, + referenceLabel: process.env.MEMMY_QUEUE_BASELINE_REF ?? null, + schedulingAlgorithm: "legacy", + method: "real SQLite and WorkerRunner; local substitute business/model handlers", + observations + }, null, 2)); + } +}); + +describe("legacy selection diagnostics and current execution safety", () => { + it("leaves runnable reflection and embedding retry untouched across 50 full import rounds", async () => { + const h = harness(); + h.db.db.transaction(() => { + for (let i = 0; i < 10_000; i++) { + h.memory(`import-${i}`); + h.job(`job-import-${i}`, "import_summary", { target: `import-${i}` }); + } + })(); + h.memory("retry-target"); + h.retry("retry-waiting", "retry-target"); + h.job("reflection-waiting", "reflection"); + h.job("maintenance-waiting", "import_summary", { + source: "startup.placeholder_summary_repair" + }); + const start = performance.now(); + const rounds = []; + for (let i = 0; i < 50; i++) rounds.push(await h.runner.runWorkerOnce(4, { priorityCohortOnly: true })); + expect(rounds.every((r) => r.leased === 4 && r.succeeded === 4 && r.embeddingRetries.leased === 0)).toBe(true); + expect(h.repos.runtime.getJob("reflection-waiting")).toMatchObject({ status: "queued", attempts: 0 }); + expect(h.repos.runtime.getJob("maintenance-waiting")).toMatchObject({ status: "queued", attempts: 0 }); + expect(h.repos.runtime.getEmbeddingRetry("retry-waiting")).toMatchObject({ status: "pending", attempts: 0 }); + const runnable = h.repos.runtime.claimDueEmbeddingRetries({ + now: Date.now(), workerId: "runnable-probe", leaseUntil: Date.now() + 60_000, limit: 1 + }); + expect(runnable.map((r) => r.id)).toEqual(["retry-waiting"]); + const reflectionProbe = h.repos.runtime.leaseQueuedJobs(1, 60, undefined, false, ["reflection"]); + expect(reflectionProbe.map((j) => j.id)).toEqual(["reflection-waiting"]); + // Control: a new native task still outranks the import backlog. + h.memory("native-target", undefined, false); + h.job("interactive-probe", "trace_summary", { target: "native-target" }); + const interactive = await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + expect(interactive.jobs[0]?.jobId).toBe("interactive-probe"); + expect(interactive.leased).toBe(4); + record("10000_import_backlog", { + seededImports: 10_000, rounds: 50, completedImports: 200, remainingImports: 9_800, + retryLeased: 0, reflectionStarted: 0, maintenanceStarted: 0, retryDirectClaimControl: true, + reflectionDirectClaimControl: true, + nativeTaskControl: "starts first, then fills remaining execution opportunities", elapsedMs: Math.round(performance.now() - start) + }); + }); + + it("runs a due embedding retry with free capacity (control)", async () => { + const h = harness(); + h.memory("free-target"); + h.retry("free-retry", "free-target"); + const result = await h.runner.runWorkerOnce(4); + expect(result).toMatchObject({ leased: 0, embeddingRetries: { leased: 1, succeeded: 1, failed: 0 } }); + record("retry_free_capacity_control", { leased: 1, succeeded: 1 }); + }); + + it("already places startup repair after reflection in the current target branch (control)", () => { + const h = harness(); + for (let i = 0; i < 10_000; i++) h.job(`startup-${i}`, "import_summary", { + source: "startup.placeholder_summary_repair" + }); + h.job("normal-reflection", "reflection"); + const leased = h.repos.runtime.leaseQueuedJobs(4, 60, undefined, true); + expect(leased.map((j) => j.id)).toEqual(["normal-reflection"]); + record("startup_repair_priority_control", { startupBacklog: 10_000, firstClaim: "reflection", leased: 1 }); + }); + + it("keeps an onboarding first report ahead of native and imported summaries (control)", () => { + const h = harness(); + h.memory("onboarding"); h.memory("native", undefined, false); h.memory("imported"); + h.db.db.prepare("UPDATE memories SET tags_json = '[\"first-encounter-report\"]' WHERE id = 'onboarding'").run(); + h.job("onboarding-report", "import_summary", { target: "onboarding" }); + h.job("native-summary", "trace_summary", { target: "native" }); + h.job("import-summary", "import_summary", { target: "imported" }); + const leased = h.repos.runtime.leaseQueuedJobs(4, 60, undefined, true); + expect(leased.map((j) => j.id)).toEqual(["onboarding-report"]); + record("onboarding_priority_control", { firstClaim: "onboarding-first-report" }); + }); + + it("lets continuously newer imports overtake an older queued import", () => { + const h = harness(); + h.memory("old-memory"); + h.job("old-job", "import_summary", { target: "old-memory" }); + const selected = []; + for (let i = 1; i <= 25; i++) { + const at = new Date(epoch + i * 1_000).toISOString(); + h.memory(`new-memory-${i}`, at); + h.job(`new-job-${i}`, "import_summary", { target: `new-memory-${i}`, createdAt: at }); + const [claimed] = h.repos.runtime.leaseQueuedJobs(1); + selected.push(claimed!.id); + h.repos.runtime.completeJob(claimed!.id, undefined, claimed); + } + expect(selected.every((id) => id !== "old-job")).toBe(true); + expect(h.repos.runtime.getJob("old-job")).toMatchObject({ status: "queued", attempts: 0 }); + record("new_imports_overtake_old", { rounds: 25, newerCompleted: 25, oldestAttempts: 0 }); + }); + + it("also prefers a newer memory over an older due embedding retry", () => { + const h = harness(); + h.memory("retry-old"); h.retry("old-retry", "retry-old"); + h.memory("retry-new", new Date(epoch + 60_000).toISOString()); h.retry("new-retry", "retry-new"); + const [claimed] = h.repos.runtime.claimDueEmbeddingRetries({ + now: Date.now(), workerId: "probe", leaseUntil: Date.now() + 60_000, limit: 1 + }); + expect(claimed!.id).toBe("new-retry"); + expect(h.repos.runtime.getEmbeddingRetry("old-retry")!.status).toBe("pending"); + record("retry_order_control", { firstClaim: "newer-memory", olderRetry: "pending" }); + }); + + it("fills lower-tier work after the first legacy priority cohort finishes", async () => { + const h = harness(); + h.job("one-summary", "trace_summary"); + for (let i = 0; i < 3; i++) h.job(`waiting-reflection-${i}`, "reflection"); + const result = await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + expect(result).toMatchObject({ leased: 4, succeeded: 4 }); + expect(h.repos.runtime.getJob("waiting-reflection-0")!.status).toBe("succeeded"); + record("priority_cohort_refill", { limit: 4, leased: 4, otherRunnableInitially: 3 }); + }); + + it("shares one execution boundary across two concurrent worker entry calls", async () => { + const gate = deferred(); + let active = 0; let peak = 0; + const h = harness(async () => { active++; peak = Math.max(peak, active); await gate.promise; active--; }); + for (let i = 0; i < 8; i++) h.job(`parallel-${i}`, "trace_summary"); + const a = h.runner.runWorkerOnce(4); + const b = h.runner.runWorkerOnce(4); + try { + await flushWorker(); + expect(active).toBe(4); expect(peak).toBe(4); + expect(h.repos.runtime.getJob("parallel-4")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); await Promise.all([a, b]); } + expect(peak).toBe(4); + record("multiple_entries_concurrency", { calls: 2, perCallLimit: 4, peakHandlers: peak }); + }); + + it("starts the fifth summary while the first slow one is still running", async () => { + const fast = deferred(); const slow = deferred(); + const started: string[] = []; + let active = 0; + const h = harness(async (j) => { + started.push(j.id); active++; + await (j.id === "summary-0" ? slow.promise : fast.promise); + active--; + }); + for (let i = 0; i < 8; i++) h.job(`summary-${i}`, "trace_summary"); + const run = h.runner.runWorkerOnce(8); + try { + await flushWorker(); + expect(started).toHaveLength(4); + expect(h.repos.runtime.getJob("summary-4")!.status).toBe("queued"); + fast.resolve(); + await flushWorker(); + expect(active).toBe(1); + expect(started).toHaveLength(8); + expect(h.repos.runtime.getJob("summary-4")!.status).toBe("succeeded"); + record("summary_continuous_refill", { limit: 8, finished: 7, active: 1, unstartedButLeased: 0, fifthStarted: true }); + } finally { fast.resolve(); slow.resolve(); await run; } + expect(started).toHaveLength(8); + }); + + it("renews only the active serial-scope job while waiting jobs stay unclaimed", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); const started: string[] = []; + const h = harness(async (j) => { started.push(j.id); if (j.id === "serial-0") await gate.promise; }); + for (let i = 0; i < 4; i++) h.job(`serial-${i}`, "reflection", { target: "shared-target" }); + const run = h.runner.runWorkerOnce(4); + try { + await flushWorker(); + expect(started).toEqual(["serial-0"]); + expect(h.repos.runtime.getJob("serial-3")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + await vi.advanceTimersByTimeAsync(61_000); + expect(h.repos.runtime.claimQueuedJob("serial-0")).toBeUndefined(); + expect(h.repos.runtime.getJob("serial-0")!.failureCount).toBe(0); + record("serial_admission_renewal", { actualStarted: 1, leased: 1, waitingQueued: 3, elapsedClockSeconds: 61, reclaimableActive: 0 }); + } finally { gate.resolve(); await run; } + }); + + it("rejects an old failure after a newer lease has succeeded", () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("stale-result", "reflection"); + const [a] = h.repos.runtime.leaseQueuedJobs(1); + vi.setSystemTime(epoch + 61_000); + const [b] = h.repos.runtime.leaseQueuedJobs(1); + expect(a!.id).toBe(b!.id); expect(b!.leaseToken).not.toBe(a!.leaseToken); + h.runner.completeLeasedWorkerJob(b!); + expect(h.repos.runtime.getJob(b!.id)!.status).toBe("succeeded"); + h.runner.failLeasedWorkerJob(a!, new Error("late error from old execution")); + expect(h.repos.runtime.getJob(b!.id)!.status).toBe("succeeded"); + record("stale_execution_overwrite", { newExecution: "succeeded", afterOldFailure: "succeeded", leaseGenerationGuard: true }); + }); + + it("does not retry temporary failures before their backoff expires", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(async () => { throw new Error("temporary timeout"); }); + h.job("temporary-failure", "reflection"); + const attempts = []; + for (let i = 0; i < 3; i++) { + await h.runner.runWorkerOnce(1); + attempts.push(h.repos.runtime.getJob("temporary-failure")!.attempts); + } + expect(attempts).toEqual([1, 1, 1]); + expect(h.repos.runtime.getJob("temporary-failure")).toMatchObject({ status: "failed", failureCount: 1 }); + record("generic_failure_backoff", { attempts, clockAdvancedMs: 0, terminal: "failed" }); + }); + + it("startup recovery preserves valid main and embedding retry leases", () => { + const h = harness(); h.job("valid-main", "reflection"); + h.memory("valid-retry-target"); h.retry("valid-retry", "valid-retry-target"); + h.repos.runtime.leaseQueuedJobs(1, 600); + h.repos.runtime.claimDueEmbeddingRetries({ now: Date.now(), workerId: "active-owner", leaseUntil: Date.now() + 600_000 }); + const result = h.runner.reconcileWorkerStartup(100); + expect(result).toMatchObject({ requeuedJobs: 0, requeuedEmbeddingRetries: 0 }); + expect(h.repos.runtime.getJob("valid-main")!.status).toBe("leased"); + expect(h.repos.runtime.getEmbeddingRetry("valid-retry")!.status).toBe("in_progress"); + record("restart_valid_leases", { validMainLease: "leased", validRetryLease: "in_progress", remainingSecondsBeforeRecovery: 600 }); + }); + + it("already rejects an obsolete embedding retry owner after a replacement claim (control)", () => { + const h = harness(); h.memory("fenced-target"); h.retry("fenced-retry", "fenced-target"); + const now = Date.now(); + const [oldClaim] = h.repos.runtime.claimDueEmbeddingRetries({ now, workerId: "A", leaseUntil: now + 1_000 }); + h.repos.runtime.claimDueEmbeddingRetries({ now: now + 1_001, workerId: "B", leaseUntil: now + 60_000 }); + const oldCompletion = h.repos.runtime.markEmbeddingRetrySucceededClaimed("fenced-retry", { + workerId: "A", leaseUntil: now + 1_000, leaseToken: oldClaim!.leaseToken!, now: now + 1_002 + }); + expect(oldCompletion).toBeUndefined(); + expect(h.repos.runtime.getEmbeddingRetry("fenced-retry")).toMatchObject({ status: "in_progress", claimedBy: "B" }); + record("embedding_existing_claim_guard", { staleCompletionRejected: true, currentOwner: "B" }); + }); +}); diff --git a/Memory/tests/service/worker/shadow-queue-selector.test.ts b/Memory/tests/service/worker/shadow-queue-selector.test.ts new file mode 100644 index 000000000..6e9e4e5bf --- /dev/null +++ b/Memory/tests/service/worker/shadow-queue-selector.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_LANE_WEIGHTS, ShadowQueueSelector, type ShadowQueueCandidate +} from "../../../src/service/worker/shadow-queue-selector.js"; +import { QUEUE_LANES, type QueueLane } from "../../../src/service/worker/job-policy-registry.js"; + +function candidate(lane: QueueLane, enqueueSeq: number, queue: ShadowQueueCandidate["queue"] = "main"): ShadowQueueCandidate { + return { queue, lane, enqueueSeq, basePriority: 0, id: `${lane}-${enqueueSeq}` }; +} + +describe("read-only shadow WRR and runnable FIFO", () => { + it("gives every runnable lane its weighted opportunities across repeated small drains", () => { + const selector = new ShadowQueueSelector(); + let pending = QUEUE_LANES.flatMap((lane) => Array.from({ length: 100 }, (_, i) => candidate(lane, i + 1))); + const totals = Object.fromEntries(QUEUE_LANES.map((lane) => [lane, 0])) as Record; + for (let drain = 0; drain < 9; drain += 1) { + const { selected } = selector.select(pending, 4); + expect(selected).toHaveLength(4); + for (const item of selected) totals[item.lane] += 1; + const identities = new Set(selected.map((item) => `${item.queue}:${item.id}`)); + pending = pending.filter((item) => !identities.has(`${item.queue}:${item.id}`)); + } + // 36 selections span exactly three 12-slot weighted rounds. + for (const lane of QUEUE_LANES) expect(totals[lane]).toBe(DEFAULT_LANE_WEIGHTS[lane] * 3); + }); + + it("uses stable enqueue order within a lane regardless of priority or source table", () => { + const selector = new ShadowQueueSelector(); + const newer = { ...candidate("repair_retry", 30), basePriority: -100 }; + const oldRetry = candidate("repair_retry", 10, "embedding_retry"); + const middle = candidate("repair_retry", 20); + expect(selector.select([newer, middle, oldRetry], 3).selected).toEqual([oldRetry, middle, newer]); + }); + + it("keeps evolution and retry progressing under a ten-thousand import backlog", () => { + const selector = new ShadowQueueSelector(); + const imports = Array.from({ length: 10_000 }, (_, i) => candidate("import", i + 1)); + const retry = candidate("repair_retry", 10_001, "embedding_retry"); + const reflection = candidate("evolution", 10_002); + const maintenance = candidate("maintenance", 10_003); + const firstRound = selector.select([...imports, retry, reflection, maintenance], 12).selected; + expect(firstRound).toEqual(expect.arrayContaining([retry, reflection, maintenance])); + expect(firstRound).toHaveLength(12); + }); + + it("fills available slots by skipping empty lanes and accepts an already-filtered blocked head", () => { + const selector = new ShadowQueueSelector(); + // The store omitted blocked seq 1; it can reappear later with its original sequence. + const newer = candidate("maintenance", 2); + expect(selector.select([newer], 4).selected).toEqual([newer]); + const previouslyBlocked = candidate("maintenance", 1); + expect(selector.select([candidate("maintenance", 3), previouslyBlocked], 4).selected[0]).toEqual(previouslyBlocked); + }); + + it("does not modify candidates, invoke jobs, or move the cursor for a zero-sized/empty preview", () => { + const selector = new ShadowQueueSelector(); + const input = Object.freeze([ + Object.freeze(candidate("interactive", 2)), Object.freeze(candidate("interactive", 1)) + ]); + expect(selector.select(input, 0).selected).toEqual([]); + expect(selector.select([], 4).selected).toEqual([]); + expect(selector.getCursor()).toBe(0); + expect(selector.select(input, 2).selected.map((item) => item.enqueueSeq)).toEqual([1, 2]); + expect(input.map((item) => item.enqueueSeq)).toEqual([2, 1]); + }); + + it("validates configuration and sequence identities instead of silently corrupting fairness", () => { + expect(() => new ShadowQueueSelector({ ...DEFAULT_LANE_WEIGHTS, import: 0 })).toThrow(RangeError); + expect(() => new ShadowQueueSelector({ ...DEFAULT_LANE_WEIGHTS, import: Number.POSITIVE_INFINITY })).toThrow(RangeError); + const selector = new ShadowQueueSelector(); + expect(() => selector.select([], -1)).toThrow(RangeError); + expect(() => selector.select([candidate("import", 0)], 1)).toThrow(RangeError); + const duplicate = candidate("import", 1); + expect(() => selector.select([duplicate, duplicate], 2)).toThrow("Duplicate queue candidate"); + }); +}); diff --git a/Memory/tests/service/worker/worker-aging.test.ts b/Memory/tests/service/worker/worker-aging.test.ts new file mode 100644 index 000000000..ac2663fc3 --- /dev/null +++ b/Memory/tests/service/worker/worker-aging.test.ts @@ -0,0 +1,130 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { claimFairWorkerBatch, type FairClaimedWork } from "../../../src/service/worker/fair-worker-queue.js"; +import { resolveWorkerSchedulingOptions } from "../../../src/service/worker/worker-scheduling-options.js"; +import { QUEUE_LANES } from "../../../src/service/worker/job-policy-registry.js"; +import { DEFAULT_LANE_WEIGHTS } from "../../../src/service/worker/lane-queue-selector.js"; + +const at = "2026-10-23T00:00:00.000Z", now = Date.parse(at); +const roots: Array<{ root: string; db: MemoryDb }> = []; +const types = { interactive: "trace_summary", repair_retry: "decision_repair", import: "import_summary", + evolution: "reflection", maintenance: "episode_idle_close" } as const; +function harness() { + const root = mkdtempSync(join(tmpdir(), "worker-aging-")), db = new MemoryDb({ path: join(root, "memory.sqlite") }); + roots.push({ root, db }); + const repos = new Repositories(db.db); + function job(id: string, type: EvolutionJobRecord["jobType"], ageMs = 0) { + repos.runtime.enqueueJob({ id, jobType: type, targetMemoryId: id, userId: "aging", payload: {}, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + db.db.prepare("UPDATE evolution_jobs SET first_queued_at=? WHERE id=?").run(new Date(now - ageMs).toISOString(), id); + } + function claim(admit: (work: FairClaimedWork) => { release(): void } | undefined = () => ({ release() {} }), algorithm: "wrr" | "legacy" = "wrr") { + return claimFairWorkerBatch(repos, { at, limit: 1, includeRetries: true, owner: "aging-worker", retryOwner: "aging-retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({ algorithm }, {}), admit }); + } + function finish(work: FairClaimedWork[]) { + for (const item of work) { if (item.queue === "main") repos.runtime.completeJob(item.job.id, at, item.job); item.admission?.release(); } + } + return { root, db, repos, job, claim, finish }; +} +afterEach(() => { vi.restoreAllMocks(); for (const { root, db } of roots.splice(0)) { db.close(); rmSync(root, { recursive: true, force: true }); } }); + +describe("bounded runnable queue aging", () => { + it("advances one overdue FIFO lane head at the configured five-minute threshold", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("old", "reflection", 300_000); + const work = h.claim(); + expect(work[0]?.queue === "main" && work[0].job.id).toBe("old"); h.finish(work); + expect(work[0]?.selectionReason).toBe("aging"); + }); + + it("does not age a task before the threshold or replace lane FIFO by timestamp ordering", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("head", "reflection", 299_999); h.job("older-tail", "reflection", 900_000); + const first = h.claim(); expect(first[0]?.queue === "main" && first[0].job.id).toBe("current"); h.finish(first); + const second = h.claim(); expect(second[0]?.queue === "main" && second[0].job.id).toBe("head"); h.finish(second); + }); + + it("reorders at most one assigned ticket per complete round and retains the 4/3/2/2/1 shares", () => { + const h = harness(); + for (const lane of QUEUE_LANES) for (let i = 0; i < 20; i++) h.job(`${lane}-${i}`, types[lane], lane === "maintenance" ? 900_000 : 400_000); + const sequence: string[] = []; + for (let i = 0; i < 24; i++) { const work = h.claim(); sequence.push(work[0]!.queue === "main" ? work[0]!.job.queueLane! : "repair_retry"); h.finish(work); } + expect(sequence[4]).toBe("maintenance"); expect(sequence[16]).toBe("maintenance"); + for (const chunk of [sequence.slice(0, 12), sequence.slice(12)]) { + expect(Object.fromEntries(QUEUE_LANES.map(lane => [lane, chunk.filter(item => item === lane).length]))).toEqual(DEFAULT_LANE_WEIGHTS); + expect(chunk.slice(0, 4)).toEqual(Array(4).fill("interactive")); + } + }); + + it("spends neither the aging opportunity nor a lane ticket when the old candidate cannot acquire a permit", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("old", "reflection", 900_000); + const first = h.claim(work => work.queue === "main" && work.job.id === "old" ? undefined : { release() {} }); + expect(first[0]?.queue === "main" && first[0].job.id).toBe("current"); h.finish(first); + expect(h.repos.runtime.getJob("old")).toMatchObject({ status: "queued", attempts: 0 }); + const second = h.claim(); expect(second[0]?.queue === "main" && second[0].job.id).toBe("old"); h.finish(second); + }); + + it("retains the aging opportunity when the promoted candidate loses its lease CAS", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("old", "reflection", 900_000); + const original = h.repos.runtime.claimQueuedJob.bind(h.repos.runtime); + vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementationOnce(() => undefined).mockImplementation(original); + const first = h.claim(); expect(first[0]?.queue === "main" && first[0].job.id).toBe("current"); h.finish(first); + const second = h.claim(); expect(second[0]?.queue === "main" && second[0].job.id).toBe("old"); h.finish(second); + }); + + it("persists the used promotion across a reconstructed repository and does not spend it twice", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("old-0", "reflection", 900_000); h.job("old-1", "reflection", 900_000); + const first = h.claim(); expect(first[0]?.queue === "main" && first[0].job.id).toBe("old-0"); h.finish(first); + const restored = new Repositories(h.db.db); + const second = claimFairWorkerBatch(restored, { at, limit: 1, includeRetries: true, owner: "restarted", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({}, {}), admit: () => ({ release() {} }) }); + expect(second[0]?.queue === "main" && second[0].job.id).toBe("current"); h.finish(second); + }); + + it.each(["future", "configuration"] as const)("does not promote a %s-blocked old task", reason => { + const h = harness(); h.job("current", "decision_repair"); h.job("old", "reflection", 900_000); + if (reason === "future") h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='old'").run(new Date(now + 1).toISOString()); + else h.db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='old'").run(); + const work = h.claim(); expect(work[0]?.queue === "main" && work[0].job.id).toBe("current"); h.finish(work); + expect(h.repos.runtime.getJob("old")).toMatchObject({ status: "queued", attempts: 0 }); + }); + + it("includes the oldest independent embedding retry in the same bounded aging policy", () => { + const h = harness(); h.job("current", "reflection"); + h.repos.runtime.enqueueEmbeddingRetry({ id: "old-retry", targetKind: "trace", targetId: "target", vectorField: "vec_summary", + sourceText: "synthetic", now: now - 900_000 }); + h.db.db.prepare("UPDATE embedding_retry_queue SET first_queued_at=? WHERE id='old-retry'").run(new Date(now - 900_000).toISOString()); + const work = h.claim(); expect(work[0]?.queue === "embedding_retry" && work[0].retry.id).toBe("old-retry"); h.finish(work); + }); + + it("rolls both the promoted claim and the used opportunity back on KV failure", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("old", "reflection", 900_000); + vi.spyOn(h.repos.runtime, "setKv").mockImplementationOnce(() => { throw new Error("disk failed"); }); + expect(() => h.claim()).toThrow("disk failed"); + expect(h.repos.runtime.getJob("old")).toMatchObject({ status: "queued", attempts: 0 }); + const work = h.claim(); expect(work[0]?.queue === "main" && work[0].job.id).toBe("old"); h.finish(work); + }); + + it("does not enable aging in the explicit legacy rollback", () => { + const h = harness(); h.job("current", "trace_summary"); h.job("old", "reflection", 900_000); + const work = h.claim(undefined, "legacy"); expect(work[0]?.queue === "main" && work[0].job.id).toBe("current"); h.finish(work); + expect(work[0]?.selectionReason).toBe("legacy"); + }); + + it("honors explicit aging disable and resets saved aging state when weights change", () => { + const h = harness(); h.job("current", "decision_repair"); h.job("old-0", "reflection", 900_000); h.job("old-1", "reflection", 900_000); + const first = h.claim(); expect(first[0]?.queue === "main" && first[0].job.id).toBe("old-0"); h.finish(first); + const disabled = claimFairWorkerBatch(h.repos, { at, limit: 1, includeRetries: true, owner: "disabled", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({ aging: { enabled: false } }, {}), + admit: () => ({ release() {} }) }); + expect(disabled[0]?.queue === "main" && disabled[0].job.id).toBe("current"); h.finish(disabled); + h.job("new-current", "decision_repair"); + const changed = claimFairWorkerBatch(h.repos, { at, limit: 1, includeRetries: true, owner: "changed", retryOwner: "retry", + leaseSeconds: 60, retryLeaseMs: 60_000, options: resolveWorkerSchedulingOptions({ laneWeights: { ...DEFAULT_LANE_WEIGHTS, interactive: 5 } }, {}), + admit: () => ({ release() {} }) }); + expect(changed[0]?.queue === "main" && changed[0].job.id).toBe("old-1"); h.finish(changed); + }); +}); diff --git a/Memory/tests/service/worker/worker-capability-reload.test.ts b/Memory/tests/service/worker/worker-capability-reload.test.ts new file mode 100644 index 000000000..f2745efde --- /dev/null +++ b/Memory/tests/service/worker/worker-capability-reload.test.ts @@ -0,0 +1,184 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; +import type { LlmClient } from "../../../src/model/types.js"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController, providerDescriptor } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerRunner, type WorkerRunnerDeps, type WorkerRunSummary } from "../../../src/service/worker/worker-runner.js"; +import { addAgentSourceImport, createBatchReflectionLlm, createCapturingEmbedder, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +const resources: Array<{ db: MemoryDb; root: string; runner: WorkerRunner; controller: WorkerConcurrencyController }> = []; +afterEach(async () => { + for (const { db, root, runner, controller } of resources.splice(0)) { + await runner.stop(); await controller.close(); if (db.db.open) db.close(); rmSync(root, { recursive: true, force: true }); + } + fixture.cleanup(); + vi.restoreAllMocks(); +}); +function deferred() { + let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; +} +function harness(process?: (job: EvolutionJobRecord) => Promise) { + const root = mkdtempSync(join(tmpdir(), "worker-capability-reload-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }), repos = new Repositories(db.db); + const controller = new WorkerConcurrencyController({ maxActiveJobs: 1 }, {}), executed: string[] = [], embedded: string[] = []; + let enabled = true, reconcileAt: number | undefined; + const noWrite = (): WorkerRunSummary => ({ leased: 0, succeeded: 0, failed: 0, jobs: [], + embeddingRetries: { leased: 0, succeeded: 0, failed: 0, items: [] }, changeSeq: 0, syncCursor: "0", serverTime: new Date().toISOString() }); + const deps: WorkerRunnerDeps = { repos, concurrency: controller, scheduling: { algorithm: "wrr" }, schedulingShadow: { enabled: false }, + embedder: createCapturingEmbedder(embedded), capture: { embedAfterCapture: true }, embeddingRetryWorkerId: "capability-review", + memoryAddEnabled: () => enabled, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + memoryBudgetNextReconcileAtMs: () => reconcileAt, summaryModelConfigured: () => true, + nowIso: () => new Date().toISOString(), encodeChangeCursor: String, namespaceIdFromMemory: () => "review", + runWorkerNoWrite: async () => noWrite(), restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: () => { throw new Error("Unexpected startup enqueue"); }, enqueueEmbeddingRetry: () => { throw new Error("Unexpected retry enqueue"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async job => { executed.push(job.id); await process?.(job); } }, + embeddingJobs: { prepareEmbeddingJob: () => { throw new Error("Unexpected embedding job"); }, applyEmbeddingVector: () => {}, + persistEmbeddingVector: input => { const memory = repos.memories.get(input.memoryId)!; input.finalize?.(memory, false, new Date().toISOString()); return memory; }, + enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); resources.push({ db, root, runner, controller }); + function job(id: string) { + return repos.runtime.enqueueJob({ id, jobType: "l2_association", userId: "review-user", payload: {}, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + } + return { db, repos, runner, deps, controller, executed, embedded, job, + setEnabled(value: boolean) { enabled = value; }, setReconcile(value: number) { reconcileAt = value; } }; +} + +describe("worker capability changes during a drain", () => { + it("wakes an unclaimed automatic run on real reload without releasing the foreground provider", async () => { + const foregroundEntered = deferred(), finishForeground = deferred(), denied = deferred(); + const calls: Parameters[0] = []; + const llm = createBatchReflectionLlm(calls); + const controller = new WorkerConcurrencyController({ maxActiveJobs: 1 }, {}); + let config = { ...DEFAULT_MEMMY_CONFIG, algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, + capture: { ...DEFAULT_MEMMY_CONFIG.algorithm.capture, embedAfterCapture: false } } }; + const { db, service } = fixture.createTestService({ config, configLoader: () => ({ config }), llm, skillLlm: llm, + concurrency: controller }); + const repos = new Repositories(db.db), abort = new AbortController(); + addAgentSourceImport(service, { source: "codex", profileId: "review", userId: "review-user" }, + "Remember this provider-blocked setting", "capability-provider-blocked"); + const job = repos.runtime.listJobs("queued", 100).find(row => row.jobType === "import_summary")!; + const foreground = controller.runRequest(providerDescriptor(llm.config, controller.limits), async () => { + foregroundEntered.resolve(); await finishForeground.promise; + }); + await foregroundEntered.promise; + const acquire = controller.tryAcquire.bind(controller); + vi.spyOn(controller, "tryAcquire").mockImplementation((...args) => { + const permit = acquire(...args); if (!permit) denied.resolve(); return permit; + }); + const run = service.runWorkerOnce(4, {}, { continuous: true, signal: abort.signal }); + let exitedWhileForegroundHeld = false; + let waiting: EvolutionJobRecord | undefined; + let result: WorkerRunSummary | undefined; + try { + await denied.promise; + expect(controller.snapshot().activeJobs).toBe(0); + expect(Object.values(controller.snapshot().providers)).toContain(1); + config = { ...config, algorithm: { ...config.algorithm, enableMemoryAdd: false } }; + service.reloadConfig({ restartFailedProcessing: false }); + let timeout: ReturnType | undefined; + try { + exitedWhileForegroundHeld = await Promise.race([ + run.then(value => { result = value; return true; }), + new Promise(resolve => { timeout = setTimeout(() => resolve(false), 100); }) + ]); + } finally { if (timeout !== undefined) clearTimeout(timeout); } + waiting = repos.runtime.getJob(job.id); + expect(controller.snapshot().activeJobs).toBe(0); + expect(Object.values(controller.snapshot().providers)).toContain(1); + } finally { + abort.abort(); finishForeground.resolve(); await foreground; await run; await service.stop(); + } + expect(exitedWhileForegroundHeld).toBe(true); + expect(result).toMatchObject({ leased: 0, succeeded: 0, failed: 0 }); + expect(waiting).toMatchObject({ status: "queued", attempts: 0, leaseCount: 0, leaseToken: null }); + expect(calls).toEqual([]); + }); + + it("uses real config reload to finish only the admitted summary and resumes untouched jobs on a new run", async () => { + const entered = deferred(), finish = deferred(); let first = true; + const calls: Parameters[0] = []; + const llm = createBatchReflectionLlm(calls), originalCompleteJson = llm.completeJson; + llm.completeJson = async >(...args: Parameters): Promise => { + if (first) { first = false; entered.resolve(); await finish.promise; } + return originalCompleteJson(...args); + }; + let config = { ...DEFAULT_MEMMY_CONFIG, algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, + capture: { ...DEFAULT_MEMMY_CONFIG.algorithm.capture, embedAfterCapture: false } } }; + const { db, service } = fixture.createTestService({ config, configLoader: () => ({ config }), llm, skillLlm: llm, + concurrency: new WorkerConcurrencyController({ maxActiveJobs: 1 }, {}) }); + const repos = new Repositories(db.db); + for (let index = 0; index < 8; index++) addAgentSourceImport(service, + { source: "codex", profileId: "review", userId: "review-user" }, `Remember build setting ${index}`, `capability-${index}`); + const jobs = repos.runtime.listJobs("queued", 100).filter(job => job.jobType === "import_summary"); + expect(jobs).toHaveLength(8); + const run = service.runWorkerOnce(4, {}, { continuous: true }); await entered.promise; + config = { ...config, algorithm: { ...config.algorithm, enableMemoryAdd: false } }; + service.reloadConfig({ restartFailedProcessing: false }); finish.resolve(); + try { + const stopped = await run; + expect(stopped).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + const waiting = jobs.map(job => repos.runtime.getJob(job.id)!).filter(job => job.status === "queued"); + expect(waiting).toHaveLength(7); + waiting.forEach(job => expect(job).toMatchObject({ attempts: 0, leaseToken: null, leaseCount: 0 })); + expect(calls.filter(call => call.options.operation === "capture.summarize")).toHaveLength(1); + config = { ...config, algorithm: { ...config.algorithm, enableMemoryAdd: true } }; + service.reloadConfig({ restartFailedProcessing: false }); + const resumed = await service.runWorkerOnce(100); + expect(resumed).toMatchObject({ leased: 7, succeeded: 7, failed: 0 }); + expect(jobs.every(job => repos.runtime.getJob(job.id)?.status === "succeeded")).toBe(true); + } finally { finish.resolve(); await service.stop(); } + }); + + it("rechecks the capability after awaiting reconciliation inside a continuous run", async () => { + const entered = deferred(), finish = deferred(); + const h = harness(async job => { if (job.id === "first") h.setReconcile(Date.now() - 1); }); + let beforeCalls = 0; + h.deps.beforeRun = async () => { if (++beforeCalls === 2) { entered.resolve(); await finish.promise; } }; + h.job("first"); h.job("second"); + const run = h.runner.runWorkerOnce(4, {}, { continuous: true }); await entered.promise; + h.setEnabled(false); finish.resolve(); const stopped = await run; + expect(stopped).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.executed).toEqual(["first"]); + expect(h.repos.runtime.getJob("second")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + }); + + it("honors a timer-driven disable while short tasks yield to the event loop", async () => { + const h = harness(), count = 128; + for (let index = 0; index < count; index++) h.job(`short-${index}`); + const changed = new Promise(resolve => setTimeout(() => { h.setEnabled(false); resolve(); }, 0)); + const stopped = await h.runner.runWorkerOnce(4, {}, { continuous: true }); await changed; + expect(h.executed.length).toBeGreaterThan(0); + expect(h.executed.length).toBeLessThan(count); + expect(stopped.leased).toBe(h.executed.length); + const waiting = h.repos.runtime.listJobs("queued", count); + expect(waiting).toHaveLength(count - h.executed.length); + waiting.forEach(job => expect(job).toMatchObject({ attempts: 0, leaseToken: null, leaseCount: 0 })); + }); + + it("keeps retry-only work pending while disabled and processes it after re-enabling", async () => { + const h = harness(), at = new Date().toISOString(); + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,created_at,updated_at) + VALUES ('retry-target',?,'review-user','synthetic','L1',?,?)`).run(at, at, at); + h.repos.runtime.enqueueEmbeddingRetry({ id: "retry", targetKind: "trace", targetId: "retry-target", + vectorField: "vec_summary", sourceText: "synthetic", now: Date.now() }); + h.setEnabled(false); + const disabled = await h.runner.runEmbeddingRetryOnce(1); + expect(disabled.leased).toBe(0); + expect(h.embedded).toEqual([]); + expect(h.repos.runtime.getEmbeddingRetry("retry")).toMatchObject({ status: "pending", attempts: 0, leaseToken: null, claimedBy: null }); + h.setEnabled(true); + const resumed = await h.runner.runEmbeddingRetryOnce(1); + expect(resumed).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.embedded).toEqual(["synthetic"]); + }); +}); diff --git a/Memory/tests/service/worker/worker-capacity-wake.test.ts b/Memory/tests/service/worker/worker-capacity-wake.test.ts new file mode 100644 index 000000000..78be0044b --- /dev/null +++ b/Memory/tests/service/worker/worker-capacity-wake.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it, vi } from "vitest"; +import { WorkerConcurrencyController, workerCapacityKey, type WorkerConcurrencyOptions, type WorkerRequirements } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerAdmissionWaitSet } from "../../../src/service/worker/worker-admission-wait-set.js"; +import type { SchedulingCandidate } from "../../../src/storage/queue-scheduling-store.js"; + +const requirements = (changes: Partial = {}): WorkerRequirements => ({ + resourceClass: "summary", providers: [], scopeKeys: [], queueLane: "interactive", ...changes +}); + +describe("resource-specific worker wakeups", () => { + it.each([ + { kind: "global", jobs: 4 }, + { kind: "background", jobs: 3, held: { queueLane: "import" }, waiting: { queueLane: "import" } }, + { kind: "resource", key: "summary", options: { resourceLimits: { summary: 1 } } }, + { kind: "scope", key: "same-target", held: { scopeKeys: ["same-target"] }, waiting: { scopeKeys: ["same-target"] } }, + { kind: "embedding_request", held: { resourceClass: "embedding", embedding: true }, waiting: { resourceClass: "embedding", embedding: true } }, + { kind: "provider", key: "A", held: { providers: [{ key: "A", limit: 1 }] }, waiting: { providers: [{ key: "A", limit: 1 }, { key: "B", limit: 1 }] } } + ] satisfies Array<{ kind: Parameters[0]; key?: string; jobs?: number; + options?: WorkerConcurrencyOptions; held?: Partial; waiting?: Partial }>) + ("wakes only on the actual released $kind capacity", async row => { + const spec: { kind: Parameters[0]; key?: string; jobs?: number; + options?: WorkerConcurrencyOptions; held?: Partial; waiting?: Partial } = row; + const c = new WorkerConcurrencyController(spec.options, {}); + const held = c.tryAcquire(requirements(spec.held), spec.jobs ?? 1)!; + const releases: string[][] = []; + const unsubscribe = c.subscribe(keys => releases.push([...keys])); + const blocked: string[] = []; + const before = c.snapshot(); + expect(c.tryAcquire(requirements(spec.waiting), 1, undefined, key => blocked.push(key))).toBeUndefined(); + expect(blocked).toEqual([workerCapacityKey(spec.kind, spec.key)]); + expect(c.snapshot()).toEqual(before); // No partial permit, including Provider B. + await c.runRequest({ local: false, providers: [{ key: "unrelated", limit: 1 }] }, async () => {}); + expect(releases.flat()).not.toContain(blocked[0]); + held.release(); + expect(releases.at(-1)).toContain(blocked[0]); + const permit = c.tryAcquire(requirements(spec.waiting)); + expect(permit).toBeDefined(); + permit!.release(); + expect(c.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0, providers: {}, scopeKeys: [], embeddingRequests: 0 }); + unsubscribe(); await c.close(); + }); + + it("does not announce a shared Provider or Embedding slot before its last active request ends", async () => { + const c = new WorkerConcurrencyController({}, {}); + const descriptor = { local: false, providers: [{ key: "embedding-model", limit: 1 }] }; + const req = requirements({ resourceClass: "embedding", embedding: true, providers: descriptor.providers }); + const first = c.tryAcquire(req)!, second = c.tryAcquire(req, 1, first)!; + const releases: string[][] = []; + c.subscribe(keys => releases.push([...keys])); + let finish!: () => void, started!: () => void; + const entered = new Promise(resolve => { started = resolve; }); + const gate = new Promise(resolve => { finish = resolve; }); + const running = first.run(() => c.runRequest(descriptor, async () => { started(); await gate; }, true)); + try { + await entered; + first.release(); second.release(); + expect(releases.flat()).not.toContain(workerCapacityKey("provider", "embedding-model")); + expect(releases.flat()).not.toContain(workerCapacityKey("embedding_request")); + expect(c.tryAcquire(req)).toBeUndefined(); + } finally { finish(); await running; first.release(); second.release(); } + expect(releases.at(-1)).toEqual(expect.arrayContaining([ + workerCapacityKey("provider", "embedding-model"), workerCapacityKey("embedding_request") + ])); + await c.close(); + }); + + it("bounds stale-head cleanup across many resources and continues it on later fills", () => { + const waiting = new WorkerAdmissionWaitSet(); + for (let i = 0; i < 200; i++) waiting.park({ id: `job-${i}`, queue: "main", jobType: "trace_summary", + lane: "import", enqueueSeq: i + 1, basePriority: 2 }, `provider-${i}`); + const pending = vi.fn(() => true); + waiting.pruneHeads(pending, 64); + expect(pending).toHaveBeenCalledTimes(64); + pending.mockImplementation(() => false); + for (let i = 0; i < 6; i++) { + pending.mockClear(); waiting.pruneHeads(pending, 64); + expect(pending.mock.calls.length).toBeLessThanOrEqual(64); + } + expect(waiting.size).toBe(0); + expect(waiting.release(["provider-0"])).toEqual(new Set()); + }); + + it("merges reawakened main and retry rows by their original FIFO sequence, including repeated blocking", () => { + const waiting = new WorkerAdmissionWaitSet(); + const candidates: SchedulingCandidate[] = Array.from({ length: 257 }, (_, i) => ({ + id: `job-${i}`, queue: i % 2 ? "main" : "embedding_retry", jobType: "embedding", lane: "import", + enqueueSeq: i + 1, basePriority: 2, firstQueuedAt: "2000-01-01T00:00:00.000Z" + })); + // A permutation forces out-of-order insertion, as when a second limiter blocks a woken row. + for (let i = 0; i < candidates.length; i++) { + const candidate = candidates[(i * 31) % candidates.length]!; + waiting.park(candidate, candidate.enqueueSeq % 2 ? "A" : "B"); + } + expect(waiting.heads()).toEqual([]); + expect(waiting.release(["unrelated"]).size).toBe(0); + expect(waiting.release(["A"])).toEqual(new Set(["import"])); + const oldest = waiting.heads()[0]!; + expect(oldest.enqueueSeq).toBe(1); + waiting.take(oldest); waiting.park(oldest, "C"); + expect(waiting.isBlocked(oldest)).toBe(true); + waiting.release(["B", "C"]); + const order: number[] = []; + while (waiting.heads().length) { + const candidate = waiting.heads()[0]!; + order.push(candidate.enqueueSeq); waiting.take(candidate); + } + expect(order).toEqual(candidates.map(candidate => candidate.enqueueSeq)); + expect(waiting.size).toBe(0); + }); +}); diff --git a/Memory/tests/service/worker/worker-concurrency.test.ts b/Memory/tests/service/worker/worker-concurrency.test.ts new file mode 100644 index 000000000..f2d892b50 --- /dev/null +++ b/Memory/tests/service/worker/worker-concurrency.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { WorkerConcurrencyController, providerDescriptor, workerModelBatchKey, type WorkerRequirements } from "../../../src/service/worker/worker-concurrency.js"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; +import { createLlmClient } from "../../../src/model/llm.js"; +import { createEmbedder } from "../../../src/model/embedder.js"; +import { MemoryModelTaskRouter } from "../../../src/model/task-routing.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; +const req = (changes: Partial = {}): WorkerRequirements => ({ resourceClass: "summary", providers: [], scopeKeys: [], ...changes }); +const tick = () => new Promise(resolve => setImmediate(resolve)); +describe("atomic worker admission and real model requests", () => { + it("shares the Mistral service root across native embeddings and compatible chat without merging other bases", () => { + const chat = providerDescriptor({ provider: "openai_compatible", endpoint: "https://api.mistral.ai/v1", model: "chat" }); + const native = providerDescriptor({ provider: "mistral", model: "mistral-embed" }); + const explicit = providerDescriptor({ provider: "mistral", endpoint: "https://api.mistral.ai/v1/embeddings", model: "mistral-embed" }); + expect(native.providers[0]).toEqual(chat.providers[0]); + expect(explicit.providers[0]).toEqual(chat.providers[0]); + expect(native.providers[1]!.key).not.toBe(chat.providers[1]!.key); + const custom = providerDescriptor({ provider: "mistral", endpoint: "https://proxy.example.test/tenant/v1/embeddings" }); + expect(custom.providers[0]).toEqual(providerDescriptor({ provider: "openai_compatible", endpoint: "https://proxy.example.test/tenant/v1" }).providers[0]); + expect(custom.providers[0]).not.toEqual(chat.providers[0]); + expect(custom.providers[0]).not.toEqual(providerDescriptor({ provider: "openai_compatible", endpoint: "https://proxy.example.test/other/v1" }).providers[0]); + }); + + it.each([undefined, "https://api.mistral.ai/v1/embeddings"])("caps real Mistral chat and native embedding calls together (embedding endpoint %s)", async endpoint => { + const summary = createLlmClient({ ...DEFAULT_MEMMY_CONFIG.summary, provider: "openai_compatible", + sourceProvider: "mistral", endpoint: "https://api.mistral.ai/v1", model: "mistral-small-latest", + apiKey: "synthetic-test-key", maxRetries: 0, malformedRetries: 0 }); + const embedding = createEmbedder({ ...DEFAULT_MEMMY_CONFIG.embedding, provider: "mistral", mode: "custom", + endpoint, model: "mistral-embed", apiKey: "synthetic-test-key", maxRetries: 0, cache: false }); + const c = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => ({ config: DEFAULT_MEMMY_CONFIG, summary, evolution: summary, embedding }), c); + let active = 0, maximum = 0; + const urls: string[] = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = async input => { + const url = String(input); + urls.push(url); maximum = Math.max(maximum, ++active); + await tick(); active--; + const response = url.endsWith("/embeddings") ? { data: [{ embedding: [1, 0, 0] }] } + : { choices: [{ message: { content: "OK" }, finish_reason: "stop" }] }; + return new Response(JSON.stringify(response), { status: 200, headers: { "content-type": "application/json" } }); + }; + try { + await Promise.all([ + router.client("summary").complete([{ role: "user", content: "first" }], { operation: "test.provider-limit" }), + router.client("summary").complete([{ role: "user", content: "second" }], { operation: "test.provider-limit" }), + router.embedder().embed(["document"]) + ]); + expect(urls.filter(url => url === "https://api.mistral.ai/v1/chat/completions")).toHaveLength(2); + expect(urls.filter(url => url === "https://api.mistral.ai/v1/embeddings")).toHaveLength(1); + expect(maximum).toBe(c.limits.cloudModelConcurrency); + expect(c.snapshot()).toMatchObject({ providers: {}, embeddingRequests: 0 }); + } finally { + globalThis.fetch = originalFetch; + await Promise.all([router.stop(), c.close()]); + } + }); + + it("shares Anthropic admission for equivalent actual request URLs", () => { + const missing = providerDescriptor({ provider: "anthropic", model: "same" }); + const bare = providerDescriptor({ provider: "anthropic", model: "same", endpoint: "https://api.anthropic.com" }); + const full = providerDescriptor({ provider: "anthropic", model: "same", endpoint: "https://api.anthropic.com/v1/messages" }); + expect(missing.providers).toEqual(bare.providers); + expect(missing.providers).toEqual(full.providers); + // HttpLlmClient appends /v1/messages unless already present. A /v1 + // endpoint therefore really sends /v1/v1/messages and must stay distinct. + expect(providerDescriptor({ provider: "anthropic", endpoint: "https://api.anthropic.com/v1", model: "same" }).providers) + .not.toEqual(missing.providers); + }); + + it("caps real Anthropic calls across default and explicit base endpoints at cloud concurrency", async () => { + const config = { ...DEFAULT_MEMMY_CONFIG.summary, provider: "anthropic" as const, + model: "claude-3-haiku-20240307", apiKey: "synthetic-test-key", maxRetries: 0, malformedRetries: 0 }; + const summary = createLlmClient({ ...config, endpoint: undefined }); + const evolution = createLlmClient({ ...config, endpoint: "https://api.anthropic.com" }); + const c = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => ({ config: DEFAULT_MEMMY_CONFIG, summary, evolution, + embedding: createCapturingEmbedder([]) }), c); + let active = 0, maximum = 0; + const urls: string[] = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = async input => { + urls.push(String(input)); maximum = Math.max(maximum, ++active); + await tick(); active--; + return new Response(JSON.stringify({ content: [{ type: "text", text: "OK" }], stop_reason: "end_turn" }), + { status: 200, headers: { "content-type": "application/json" } }); + }; + try { + await Promise.all((["summary", "evolution", "summary", "evolution"] as const).map(role => + router.client(role).complete([{ role: "user", content: "test" }], { operation: "test.provider-limit", maxTokens: 64 }))); + expect([...new Set(urls)]).toEqual(["https://api.anthropic.com/v1/messages"]); + expect(maximum).toBe(c.limits.cloudModelConcurrency); + expect(c.snapshot().providers).toEqual({}); + } finally { + globalThis.fetch = originalFetch; + await Promise.all([router.stop(), c.close()]); + } + }); + + it("separates batches after credential/header reload without exposing secret values", () => { + const first = { endpoint: "https://example.test/v1", model: "same", apiKey: "private-key-one", + extraHeaders: { Authorization: "Bearer private-header-one", "X-Account": "first" }, extraBody: { nested: { beta: 2, alpha: 1 } } }; + const hash = workerModelBatchKey([first]); + expect(hash).toMatch(/^[a-f0-9]{64}$/); + expect(hash).not.toContain("private"); + expect(workerModelBatchKey([{ ...first, apiKey: "private-key-two" }])).not.toBe(hash); + expect(workerModelBatchKey([{ ...first, extraHeaders: { ...first.extraHeaders, Authorization: "Bearer private-header-two" } }])).not.toBe(hash); + expect(workerModelBatchKey([{ ...first, extraBody: { nested: { alpha: 1, beta: 3 } } }])).not.toBe(hash); + expect(workerModelBatchKey([{ ...first, extraBody: { nested: { alpha: 1, beta: 2 } } }])).toBe(hash); + }); + it("hashes complete credentials/configuration without length or entry truncation", () => { + const prefix = "x".repeat(25_000); + expect(workerModelBatchKey([{ apiKey: `${prefix}a` }])).not.toBe(workerModelBatchKey([{ apiKey: `${prefix}b` }])); + const headers = Object.fromEntries(Array.from({ length: 1_100 }, (_, index) => [`key-${index}`, "same"])); + expect(workerModelBatchKey([{ extraHeaders: headers }])).not.toBe(workerModelBatchKey([{ extraHeaders: { ...headers, "key-1099": "changed" } }])); + }); + it("never holds partial resources when provider, scope, resource or global admission fails", () => { + const c = new WorkerConcurrencyController({}, {}); + const first = c.tryAcquire(req({ providers: [{ key: "root", limit: 1 }], scopeKeys: ["a"] }))!; + expect(c.tryAcquire(req({ providers: [{ key: "other", limit: 1 }, { key: "root", limit: 1 }], scopeKeys: ["b"] }))).toBeUndefined(); + expect(c.availableJobs()).toBe(3); expect(c.snapshot().providers).toEqual({ root: 1 }); + expect(c.tryAcquire(req({ scopeKeys: ["a"] }))).toBeUndefined(); + const other = c.tryAcquire(req({ providers: [{ key: "other", limit: 1 }], scopeKeys: ["b"] }))!; + first.release(); first.release(); other.release(); + expect(c.availableJobs()).toBe(4); expect(c.snapshot().activeJobs).toBe(0); + }); + it("counts logical microbatch members independently while sharing one provider/request reservation", () => { + const c = new WorkerConcurrencyController({}, {}); + const r = req({ resourceClass: "embedding", embedding: true, providers: [{ key: "root", limit: 1 }] }); + const a = c.tryAcquire({ ...r, scopeKeys: ["a"] })!; + const b = c.tryAcquire({ ...r, scopeKeys: ["b"] }, 1, a)!; + expect(c.snapshot()).toMatchObject({ activeJobs: 2, embeddingRequests: 1, providers: { root: 1 } }); + a.release(); a.release(); + expect(c.snapshot()).toMatchObject({ activeJobs: 1, embeddingRequests: 1, providers: { root: 1 } }); + expect(c.tryAcquire(r)).toBeUndefined(); + b.release(); expect(c.snapshot()).toMatchObject({ activeJobs: 0, embeddingRequests: 0, providers: {} }); + expect(c.limits.embeddingBatchSize).toBe(16); + }); + it("rejects sharing other resources/providers or released permits without consuming anything", () => { + const c = new WorkerConcurrencyController({}, {}); + const r = req({ resourceClass: "embedding", embedding: true, providers: [{ key: "root", limit: 1 }] }); + const p = c.tryAcquire(r)!; + expect(c.tryAcquire({ ...r, resourceClass: "summary" }, 1, p)).toBeUndefined(); + expect(c.tryAcquire({ ...r, providers: [{ key: "another", limit: 1 }] }, 1, p)).toBeUndefined(); + p.release(); expect(c.tryAcquire(r, 1, p)).toBeUndefined(); expect(c.availableJobs()).toBe(4); + }); + it("enforces resource and global job limits before a request starts", () => { + const c = new WorkerConcurrencyController({ resourceLimits: { summary: 1 } }, {}); + const p = c.tryAcquire(req())!; + expect(c.tryAcquire(req())).toBeUndefined(); + const other = c.tryAcquire(req({ resourceClass: "embedding", queueLane: "interactive" }), 3)!; + expect(c.availableJobs()).toBe(0); expect(c.tryAcquire(req({ resourceClass: "database" }))).toBeUndefined(); + p.release(); other.release(); + }); + it.each([0, -1, 1.2, NaN, Infinity, 1025])("rejects invalid limits %s", value => { + expect(() => new WorkerConcurrencyController({ maxActiveJobs: value }, {})).toThrow(); + }); + it("rejects bad environment JSON, unknown resources and invalid provider limits", () => { + expect(() => new WorkerConcurrencyController({}, { MEMMY_WORKER_RESOURCE_LIMITS: "invalid" })).toThrow(); + expect(() => new WorkerConcurrencyController({}, { MEMMY_WORKER_MAX_ACTIVE_JOBS: "2x" })).toThrow(); + expect(() => new WorkerConcurrencyController({ resourceLimits: { unknown: 1 } as never }, {})).toThrow(); + expect(new WorkerConcurrencyController({}, { MEMMY_WORKER_CLOUD_MODEL_CONCURRENCY: "3" }).limits.cloudModelConcurrency).toBe(3); + expect(() => new WorkerConcurrencyController({}, {}).tryAcquire(req({ providers: [{ key: "p", limit: 0 }] }))).toThrow(); + }); + it("uses a common local root across models and omits endpoint credentials from identities", () => { + const c = new WorkerConcurrencyController({}, {}); + const a = providerDescriptor({ provider: "openai_compatible", sourceProvider: "lmstudio", endpoint: "http://user:secret@localhost:1234/v1?api_key=secret", model: "a" }, c.limits); + const b = providerDescriptor({ provider: "openai_compatible", sourceProvider: "another-label", endpoint: "http://127.0.0.1:1234/v1/", model: "b" }, c.limits); + expect(a.providers[0]).toEqual(b.providers[0]); expect(a.providers[0]!.limit).toBe(1); + expect(a.providers[1]!.key).not.toBe(b.providers[1]!.key); expect(JSON.stringify(a)).not.toContain("secret"); + expect(providerDescriptor({ provider: "openai_compatible", endpoint: "https://api.example.test/v1", model: "a" }, c.limits).providers[0]!.limit).toBe(2); + }); + it("resolves real default endpoints and normalizes explicit model operation paths", () => { + const defaulted = providerDescriptor({ provider: "openai_compatible", sourceProvider: "openai", model: "a" }); + const explicit = providerDescriptor({ provider: "openai_compatible", endpoint: "https://api.openai.com/v1/embeddings", model: "b" }); + const chat = providerDescriptor({ provider: "openai_compatible", endpoint: "https://api.openai.com/v1/chat/completions", model: "c" }); + expect(defaulted.providers[0]).toEqual(explicit.providers[0]); + expect(defaulted.providers[0]).toEqual(chat.providers[0]); + const account = providerDescriptor({ provider: "openai_compatible", endpoint: "http://localhost:19000/proxy", model: "cloud", actualModelContext: { source: "account" } }); + expect(account.local).toBe(false); expect(account.providers[0]!.limit).toBe(2); + }); + it("serializes actual parallel calls inside a reserved provider without reacquiring its own slot", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "local", model: "a" }); + const p = c.tryAcquire(req({ providers: d.providers }))!; + let active = 0, maximum = 0; + const invoke = () => c.runRequest(d, async () => { maximum = Math.max(maximum, ++active); await tick(); active--; return 7; }); + expect(await p.run(() => Promise.all([invoke(), invoke(), invoke()]))).toEqual([7, 7, 7]); + expect(maximum).toBe(1); expect(c.snapshot().activeJobs).toBe(1); p.release(); + }); + it("shares internal serialization between embedding group members", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "local", model: "a" }); + const r = req({ resourceClass: "embedding", providers: d.providers, embedding: true }); + const a = c.tryAcquire(r)!, b = c.tryAcquire(r, 1, a)!; + let active = 0, maximum = 0; + const invoke = () => c.runRequest(d, async () => { maximum = Math.max(maximum, ++active); await tick(); active--; }, true); + await Promise.all([a.run(invoke), b.run(invoke)]); expect(maximum).toBe(1); a.release(); b.release(); + }); + it("caps actual non-worker embedding requests and releases on rejection without taking logical job slots", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "openai_compatible", endpoint: "https://example.test", model: "e" }); + let active = 0, maximum = 0; + const invoke = () => c.runRequest(d, async () => { maximum = Math.max(maximum, ++active); await tick(); active--; }, true); + await Promise.all([invoke(), invoke(), invoke()]); expect(maximum).toBe(1); + await expect(c.runRequest(d, async () => { throw new Error("remote failed"); }, true)).rejects.toThrow("remote failed"); + expect(c.snapshot()).toMatchObject({ activeJobs: 0, embeddingRequests: 0, providers: {} }); + }); + it("permits exactly two actual cloud requests across independent tasks", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "openai_compatible", endpoint: "https://example.test", model: "chat" }); + let active = 0, maximum = 0; + const invoke = () => c.runRequest(d, async () => { maximum = Math.max(maximum, ++active); await tick(); active--; }); + await Promise.all([invoke(), invoke(), invoke(), invoke()]); + expect(maximum).toBe(2); expect(c.snapshot().providers).toEqual({}); + }); + it("refuses a model that was not part of the worker admission snapshot", async () => { + const c = new WorkerConcurrencyController({}, {}); + const a = providerDescriptor({ provider: "local", model: "a" }), b = providerDescriptor({ provider: "local", model: "b" }); + const p = c.tryAcquire(req({ providers: a.providers }))!; let called = false; + await expect(p.run(() => c.runRequest(b, async () => { called = true; }))).rejects.toThrow(/admitted/); + expect(called).toBe(false); p.release(); + }); + it("enforces the microbatch upper bound even when logical capacity is larger", () => { + const c = new WorkerConcurrencyController({ maxActiveJobs: 32, resourceLimits: { embedding: 32 } }, {}); + const r = req({ resourceClass: "embedding", embedding: true }); + expect(c.tryAcquire(r, 17)).toBeUndefined(); + const p = c.tryAcquire(r, 16)!; + expect(c.tryAcquire(r, 1, p)).toBeUndefined(); p.release(); + }); + it("notifies subscribers on release and supports unsubscribe and idempotent release", () => { + const c = new WorkerConcurrencyController({}, {}); let calls = 0; + const off = c.subscribe(() => { calls++; }); + const p = c.tryAcquire(req())!; p.release(); p.release(); expect(calls).toBe(1); + off(); c.tryAcquire(req())!.release(); expect(calls).toBe(1); + }); + it("does not start a queued request after its own permit has been released", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "local", model: "a" }); + const p = c.tryAcquire(req({ providers: d.providers }))!; let called = false; + const request = p.run(() => c.runRequest(d, async () => { called = true; })); + p.release(); await expect(request).rejects.toThrow(/released/); expect(called).toBe(false); + }); + it("keeps the provider reserved until an already started request settles after early task release", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "local", model: "a" }); + const r = req({ providers: d.providers }); + const p = c.tryAcquire(r)!; + let finish!: () => void; + const request = p.run(() => c.runRequest(d, () => new Promise(resolve => { finish = resolve; }))); + await tick(); p.release(); + expect(c.availableJobs()).toBe(4); + expect(c.tryAcquire(r)).toBeUndefined(); + finish(); await request; + const next = c.tryAcquire(r)!; expect(next).toBeDefined(); next.release(); + }); + it("closes foreground admission, rejects waiters, and waits for admitted work to finish", async () => { + const c = new WorkerConcurrencyController({}, {}), d = providerDescriptor({ provider: "local", model: "a" }); + const p = c.tryAcquire(req({ providers: d.providers }))!; + let called = false, closed = false; + const waiting = c.runRequest(d, async () => { called = true; }); + const rejected = expect(waiting).rejects.toThrow(/clos/i); + const close = c.close().then(() => { closed = true; }); + await tick(); expect(called).toBe(false); expect(closed).toBe(false); + expect(c.tryAcquire(req())).toBeUndefined(); + expect(await p.run(() => c.runRequest(d, async () => "admitted-finishes"))).toBe("admitted-finishes"); + p.release(); await Promise.all([close, rejected]); + expect(closed).toBe(true); expect(c.snapshot().providers).toEqual({}); + await expect(c.runRequest(d, async () => {})).rejects.toThrow(/clos/i); + }); +}); diff --git a/Memory/tests/service/worker/worker-configuration-race.test.ts b/Memory/tests/service/worker/worker-configuration-race.test.ts new file mode 100644 index 000000000..b8ad3657a --- /dev/null +++ b/Memory/tests/service/worker/worker-configuration-race.test.ts @@ -0,0 +1,82 @@ +import { afterEach, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, MemoryDb, type LlmCompletionOptions, type LlmMessage } from "../../../src/index.js"; +import { ModelHttpError } from "../../../src/model/http.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { addAgentSourceImport, createBatchReflectionLlm, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(fixture.cleanup); + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +it("resumes an old in-flight configuration failure after the replacement fingerprint was observed", async () => { + const entered = deferred(), release = deferred(); + let currentConfig = { ...DEFAULT_MEMMY_CONFIG, summary: { ...DEFAULT_MEMMY_CONFIG.summary, model: "config-A" } }; + let unauthorized = true; + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + if (unauthorized) { + entered.resolve(); + await release.promise; + throw new Error("unauthorized API key from config-A request"); + } + return complete(messages, options); + }; + const { db, service } = fixture.createTestService({ config: currentConfig, configLoader: () => ({ config: currentConfig }), llm }); + const item = addAgentSourceImport(service, { source: "codex", profileId: "review", userId: "review-user" }, "configuration repair race", "review-race"); + const oldRun = service.runWorkerOnce(1); + await entered.promise; + currentConfig = { ...currentConfig, summary: { ...currentConfig.summary, model: "config-B-repaired" } }; + service.reloadConfig({ reason: "model_settings_saved" }); + release.resolve(); + await oldRun; + unauthorized = false; + const retry = await service.runWorkerOnce(1); + const repos = new Repositories(db.db); + const job = repos.runtime.listJobs("succeeded").find((job) => job.targetMemoryId === item.id); + expect(job).toMatchObject({ status: "succeeded", blockedReason: null, failureCount: 0 }); + expect(retry.leased).toBe(1); +}); + +it("preserves configuration failure and manual retry across a real database reopen", async () => { + const llm = createBatchReflectionLlm([]); + llm.completeJson = async () => { throw new Error("unauthorized API key"); }; + const { db, service } = fixture.createTestService({ llm }); + const namespace = { source: "codex", profileId: "review", userId: "review-startup" }; + const item = addAgentSourceImport(service, namespace, "configuration block on restart", "review-startup"); + await service.runWorkerOnce(1); + const before = service.memoryProcessingStatus([item.id], { namespace }).items[0]; + expect(before).toMatchObject({ state: "failed", retryAction: "open_settings", errorCode: "model_configuration" }); + await service.stop(); + db.close(); + const reopened = new MemoryDb({ path: db.path }); + const nextService = fixture.createTestMemoryService({ db: reopened, mode: "dev", llm }); + nextService.reconcileWorkerStartup(100); + const after = nextService.memoryProcessingStatus([item.id], { namespace }).items[0]; + const retry = nextService.retryMemoryProcessing(item.id, { namespace }); + expect(after).toMatchObject({ state: "failed", retryAction: "open_settings", errorCode: "model_configuration" }); + expect(retry.accepted).toBe(true); +}); + +it("does not recreate permanently invalid imported summaries on restart", async () => { + const llm = createBatchReflectionLlm([]); + llm.completeJson = async () => { throw new ModelHttpError("invalid input", "test", 400, undefined, "invalid request body"); }; + const { db, service } = fixture.createTestService({ llm }); + const namespace = { source: "codex", profileId: "permanent", userId: "permanent-restart" }; + const item = addAgentSourceImport(service, namespace, "permanent invalid summary input", "permanent-restart"); + await service.runWorkerOnce(1); + expect(service.memoryProcessingStatus([item.id], { namespace }).items[0]).toMatchObject({ state: "failed", retryAction: "none" }); + const before = new Repositories(db.db).runtime.listJobs("dead_letter").filter((job) => job.targetMemoryId === item.id); + expect(before).toHaveLength(1); + await service.stop(); db.close(); + const reopened = new MemoryDb({ path: db.path }); + const nextService = fixture.createTestMemoryService({ db: reopened, mode: "dev", llm }); + expect(nextService.reconcileWorkerStartup(100).enqueuedImportSummaries).toBe(0); + expect(nextService.memoryProcessingStatus([item.id], { namespace }).items[0]).toMatchObject({ state: "failed", retryAction: "none" }); + expect((await nextService.runWorkerOnce(1)).leased).toBe(0); + expect(new Repositories(reopened.db).runtime.listJobs("dead_letter").filter((job) => job.targetMemoryId === item.id)).toHaveLength(1); +}); diff --git a/Memory/tests/service/worker/worker-continuous-admission.test.ts b/Memory/tests/service/worker/worker-continuous-admission.test.ts new file mode 100644 index 000000000..922642422 --- /dev/null +++ b/Memory/tests/service/worker/worker-continuous-admission.test.ts @@ -0,0 +1,830 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { getEventListeners } from "node:events"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController, type PreparedWorkerExecution } from "../../../src/service/worker/worker-concurrency.js"; +import type { FairClaimedWork } from "../../../src/service/worker/fair-worker-queue.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const epoch = Date.parse("2026-10-22T00:00:00.000Z"); +const at = new Date(epoch).toISOString(); +const resources: Array<{ root: string; db: MemoryDb }> = []; +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} +type HarnessOptions = { concurrency?: WorkerConcurrencyController; algorithm?: "wrr" | "legacy"; process?: (job: EvolutionJobRecord) => Promise }; + +/** Real SQLite and claim transitions; business/model work is a local substitute. */ +function harness(options: HarnessOptions = {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-pump-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const executed: string[] = []; + const deps: WorkerRunnerDeps = { + repos, embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "pump-test", memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), summaryModelConfigured: () => true, + schedulingShadow: { enabled: false }, + ...(options.algorithm ? { scheduling: { algorithm: options.algorithm } } : {}), + nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "synthetic", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement retry"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async (job) => { executed.push(job.id); await options.process?.(job); } }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding job"); }, + applyEmbeddingVector: () => {}, enqueueEmbeddingRetryAfterFailure: () => false, + persistEmbeddingVector: (input) => { + const memory = repos.memories.get(input.memoryId)!; + executed.push(`retry:${input.memoryId}`); + input.finalize?.(memory, false, new Date().toISOString()); + return memory; + } + } + }; + const concurrency = options.concurrency ?? new WorkerConcurrencyController({}, {}); + deps.concurrency = concurrency; + const runner = new WorkerRunner(deps); + function memory(id: string, createdAt = at, tags: string[] = ["agent-source"]) { + db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,created_at,updated_at) + VALUES (?,?,'user','synthetic','L1',?,?,?)`).run(id, createdAt, JSON.stringify(tags), createdAt, createdAt); + } + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection", input: { + target?: string; source?: string; createdAt?: string; payload?: Record; + scopeKey?: string; scopeSeq?: number; episodeId?: string; sessionId?: string; + } = {}) { + return repos.runtime.enqueueJob({ id, jobType, userId: "user", targetMemoryId: input.target, + scopeKey: input.scopeKey, scopeSeq: input.scopeSeq, episodeId: input.episodeId, sessionId: input.sessionId, + payload: { ...input.payload, ...(input.source ? { source: input.source } : {}) }, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: input.createdAt ?? at, updatedAt: at }); + } + function retry(id: string, createdAt = at) { + memory(id, createdAt); + return repos.runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, + vectorField: "vec_summary", sourceText: "synthetic", now: epoch }); + } + return { root, db, repos, deps, runner, job, memory, retry, executed, concurrency }; +} +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + vi.stubEnv("MEMMY_WORKER_SCHEDULER_ALGORITHM", "wrr"); +}); +afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + + +async function flushPump() { + for (let i = 0; i < 200; i++) await Promise.resolve(); +} +function mainRows(h: ReturnType) { + return h.db.db.prepare("SELECT id,status,attempts,lease_token FROM evolution_jobs ORDER BY enqueue_seq").all() as + Array<{ id: string; status: string; attempts: number; lease_token: string | null }>; +} + +function expectNoPermits(h: ReturnType) { + expect(h.concurrency.snapshot()).toEqual({ activeJobs: 0, availableJobs: h.concurrency.limits.maxActiveJobs, + backgroundActiveJobs: 0, maxBackgroundJobs: h.concurrency.limits.maxBackgroundJobs, + interactiveReservedSlots: h.concurrency.limits.interactiveReservedSlots, resources: {}, providers: {}, scopeKeys: [], embeddingRequests: 0 }); +} +function prepareProviders(h: ReturnType, prepare: (work: FairClaimedWork) => PreparedWorkerExecution) { + h.deps.prepareExecution = prepare; +} + +describe("continuous worker admission with real SQLite", () => { + it("rechecks multiple required Providers in turn without losing FIFO or reserving partial capacity", async () => { + const h = harness(); + const holds = ["A", "B"].map(key => h.concurrency.tryAcquire({ resourceClass: "summary", + queueLane: "interactive", providers: [{ key, limit: 1 }], scopeKeys: [] })!); + prepareProviders(h, () => ({ providers: [{ key: "A", limit: 1 }, { key: "B", limit: 1 }], run: operation => operation() })); + h.job("oldest", "trace_summary"); h.job("next", "trace_summary"); + const positions = mainRows(h).map(row => row.id); + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(h.executed).toEqual([]); + holds[0]!.release(); await flushPump(); + expect(h.executed).toEqual([]); + expect(h.concurrency.snapshot().providers).toEqual({ B: 1 }); + expect(mainRows(h).every(row => row.status === "queued" && row.attempts === 0 && row.lease_token === null)).toBe(true); + holds[1]!.release(); + expect((await run).jobs.map(job => job.jobId)).toEqual(positions); + expect(h.executed).toEqual(["oldest", "next"]); + } finally { holds.forEach(permit => permit.release()); await run; } + expectNoPermits(h); + }); + + it("finishes a drained request while an unrelated foreground Provider is still held", async () => { + const h = harness(), gate = deferred(), entered = deferred(), abort = new AbortController(); + const foreground = h.concurrency.runRequest({ local: false, providers: [{ key: "unrelated", limit: 1 }] }, async () => { + entered.resolve(); await gate.promise; + }); + await entered.promise; + prepareProviders(h, () => ({ providers: [{ key: "shared-worker", limit: 1 }], run: operation => operation() })); + h.job("first", "trace_summary"); h.job("second", "trace_summary"); + let finished = false; + const run = h.runner.runWorkerOnce(100, {}, { signal: abort.signal }).then(result => { finished = true; return result; }); + try { + await flushPump(); + expect(finished).toBe(true); + expect(h.executed).toEqual(["first", "second"]); + expect(h.concurrency.snapshot().providers).toEqual({ unrelated: 1 }); + } finally { abort.abort(); gate.resolve(); await foreground; await run; } + expectNoPermits(h); + }); + + it("reawakens parked embedding retries without changing their FIFO positions or attempts before admission", async () => { + const h = harness(); + const held = h.concurrency.tryAcquire({ resourceClass: "embedding", embedding: true, + providers: [{ key: "embedding-provider", limit: 1 }], scopeKeys: [] })!; + prepareProviders(h, () => ({ providers: [{ key: "embedding-provider", limit: 1 }], embedding: true, run: operation => operation() })); + h.retry("retry-oldest"); h.retry("retry-next"); + const run = h.runner.runEmbeddingRetryOnce(2); + try { + await flushPump(); + expect(h.executed).toEqual([]); + expect(h.repos.runtime.getEmbeddingRetry("retry-oldest")).toMatchObject({ status: "pending", attempts: 0 }); + held.release(); + const result = await run; + expect(result.items.map(item => item.id)).toEqual(["retry-oldest", "retry-next"]); + expect(result).toMatchObject({ leased: 2, succeeded: 2, failed: 0 }); + } finally { held.release(); await run; } + expectNoPermits(h); + }); + + it("keeps limit=100 as a drain budget rather than leasing waiting tasks ahead of capacity", async () => { + const gate = deferred(); + const started: string[] = []; + const h = harness({ process: async job => { started.push(job.id); await gate.promise; } }); + for (let i = 0; i < 12; i++) h.job(`summary-${i}`, "trace_summary"); + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(started.length).toBeGreaterThan(0); + expect(started.length).toBeLessThanOrEqual(4); + const rows = mainRows(h); + const leased = rows.filter(row => row.status === "leased"); + expect(leased.map(row => row.id).sort()).toEqual([...started].sort()); + expect(leased.length).toBeLessThanOrEqual(4); + for (const row of rows.filter(row => !started.includes(row.id))) { + expect(row).toMatchObject({ status: "queued", attempts: 0, lease_token: null }); + } + } finally { gate.resolve(); await run; } + }); + + it("refills a slot after a fast task completes while an earlier slow task is still running", async () => { + const slow = deferred(); + let slowFinished = false; + const h = harness({ process: async job => { + if (job.id === "summary-0") { await slow.promise; slowFinished = true; } + } }); + for (let i = 0; i < 12; i++) h.job(`summary-${i}`, "trace_summary"); + const run = h.runner.runWorkerOnce(12); + try { + await flushPump(); + expect(slowFinished).toBe(false); + expect(h.executed).toContain("summary-4"); + expect(h.repos.runtime.getJob("summary-4")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("summary-0")!.status).toBe("leased"); + } finally { slow.resolve(); await run; } + }); + + it("shares the global logical cap across task kinds and admits another resource while summaries wait", async () => { + const gate = deferred(); + const active = new Set(); + const types = new Set(); + let maximum = 0; + const h = harness({ concurrency: new WorkerConcurrencyController({ resourceLimits: { summary: 2 } }, {}), process: async job => { + active.add(job.id); types.add(job.jobType); maximum = Math.max(maximum, active.size); + try { await gate.promise; } finally { active.delete(job.id); } + } }); + for (let i = 0; i < 8; i++) h.job(`summary-${i}`, "trace_summary"); + for (let i = 0; i < 3; i++) { + h.job(`reflection-${i}`, "reflection"); + h.job(`database-${i}`, "l2_association"); + h.job(`maintenance-${i}`, "project_environment_profile", { payload: { projectId: `project-${i}` } }); + } + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(maximum).toBeLessThanOrEqual(4); + expect(types.size).toBeGreaterThan(1); + expect(mainRows(h).filter(row => row.status === "leased")).toHaveLength(active.size); + } finally { gate.resolve(); await run; } + expect(active.size).toBe(0); + expectNoPermits(h); + expect(mainRows(h).every(row => row.status === "succeeded")).toBe(true); + }); + + it("does not lease a second task for the same target while the target is already executing", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + h.memory("shared-target", at, []); + for (let i = 0; i < 4; i++) h.job(`target-${i}`, "trace_summary", { target: "shared-target" }); + const run = h.runner.runWorkerOnce(4); + try { + await flushPump(); + expect(h.executed).toEqual(["target-0"]); + expect(h.repos.runtime.getJob("target-1")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); await run; } + expect(h.executed).toEqual(["target-0", "target-1", "target-2", "target-3"]); + }); + + it("keeps same-episode tasks queued while their episode scope is occupied", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + h.job("episode-title", "episode_title", { episodeId: "episode-A" }); + h.job("episode-reward", "reward", { episodeId: "episode-A" }); + const run = h.runner.runWorkerOnce(2); + try { + await flushPump(); + expect(h.executed).toEqual(["episode-title"]); + expect(h.repos.runtime.getJob("episode-reward")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); await run; } + expect(h.executed).toEqual(["episode-title", "episode-reward"]); + }); + + it("preserves L3 user/project serialization without keeping later scope jobs leased", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + h.job("L3-A-1", "l3_world_model_update", { scopeKey: "user/project-A", scopeSeq: 1, payload: { projectId: "project-A" } }); + h.job("L3-A-2", "l3_world_model_update", { scopeKey: "user/project-A", scopeSeq: 2, payload: { projectId: "project-A" } }); + h.job("L3-B-1", "l3_world_model_update", { scopeKey: "user/project-B", scopeSeq: 1, payload: { projectId: "project-B" } }); + const run = h.runner.runWorkerOnce(3); + try { + await flushPump(); + expect(h.executed).toContain("L3-A-1"); + expect(h.repos.runtime.getJob("L3-A-2")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); await run; } + expect(h.repos.runtime.getJob("L3-A-2")!.status).toBe("succeeded"); + }); + + it("bounds an Embedding microbatch by both batch size and free logical capacity", async () => { + const gate = deferred(); + const calls: string[][] = []; + const h = harness(); + h.deps.embeddingJobs.prepareEmbeddingJob = job => ({ job, memory: h.repos.memories.get(job.targetMemoryId!)!, + text: job.id, role: "document", vectorField: "vec_summary" }); + h.deps.embedder.embed = async texts => { calls.push([...texts]); await gate.promise; return texts.map(() => [1, 0, 0]); }; + for (let i = 0; i < 20; i++) { + h.memory(`embedding-${i}`, at, []); + h.job(`embedding-${i}`, "embedding", { target: `embedding-${i}` }); + } + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(calls).toHaveLength(1); + expect(calls[0]!.length).toBeLessThanOrEqual(16); + expect(calls[0]!.length).toBeLessThanOrEqual(4); + const rows = mainRows(h); + expect(rows.filter(row => row.status === "leased")).toHaveLength(calls[0]!.length); + for (const row of rows.filter(row => row.status === "queued")) { + expect(row).toMatchObject({ attempts: 0, lease_token: null }); + } + } finally { gate.resolve(); await run; } + expect(calls.flat()).toHaveLength(20); + expect(h.repos.runtime.listJobs("succeeded")).toHaveLength(20); + }); + + + it("uses the configured batch limit of sixteen when logical capacity is raised to thirty-two", async () => { + const gate = deferred(); + const calls: string[][] = []; + const h = harness({ concurrency: new WorkerConcurrencyController({ maxActiveJobs: 32, + resourceLimits: { embedding: 32 }, embeddingBatchSize: 16, embeddingRequestConcurrency: 1 }, {}) }); + h.deps.embeddingJobs.prepareEmbeddingJob = job => ({ job, memory: h.repos.memories.get(job.targetMemoryId!)!, + text: job.id, role: "document", vectorField: "vec_summary" }); + h.deps.embedder.embed = async texts => { calls.push([...texts]); await gate.promise; return texts.map(() => [1, 0, 0]); }; + for (let i = 0; i < 40; i++) { + h.memory(`embedding-${i}`, at, []); h.job(`embedding-${i}`, "embedding", { target: `embedding-${i}` }); + } + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(calls).toHaveLength(1); + expect(calls[0]).toHaveLength(16); + expect(mainRows(h).filter(row => row.status === "leased")).toHaveLength(16); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 16, embeddingRequests: 1 }); + expect(mainRows(h).filter(row => row.status === "queued").every(row => row.attempts === 0 && row.lease_token === null)).toBe(true); + } finally { gate.resolve(); await run; } + expect(calls.map(batch => batch.length)).toEqual([16, 16, 8]); + expectNoPermits(h); + }); + + it.each([1, 2])("admits only the provider cap %s while other ready rows remain unclaimed", async limit => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + prepareProviders(h, () => ({ providers: [{ key: "shared-model", limit }], run: operation => operation() })); + for (let i = 0; i < 8; i++) h.job(`summary-${i}`, "trace_summary"); + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(h.executed).toHaveLength(limit); + expect(mainRows(h).filter(row => row.status === "leased")).toHaveLength(limit); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: limit, providers: { "shared-model": limit } }); + expect(mainRows(h).filter(row => row.status === "queued").every(row => row.attempts === 0 && row.lease_token === null)).toBe(true); + } finally { gate.resolve(); await run; } + expect(h.executed).toHaveLength(8); + expectNoPermits(h); + }); + + it("does not reserve any of a blocked task's providers and lets an independent provider progress", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + prepareProviders(h, work => { + const id = work.queue === "main" ? work.job.id : work.retry.id; + const keys = id === "A+B" ? ["A", "B"] : [id]; + return { providers: keys.map(key => ({ key, limit: 1 })), run: operation => operation() }; + }); + h.job("A", "trace_summary"); h.job("A+B", "trace_summary"); h.job("B", "trace_summary"); + const run = h.runner.runWorkerOnce(3); + try { + await flushPump(); + expect(h.executed).toEqual(["A", "B"]); + expect(h.repos.runtime.getJob("A+B")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 2, providers: { A: 1, B: 1 } }); + } finally { gate.resolve(); await run; } + expect(h.repos.runtime.getJob("A+B")!.status).toBe("succeeded"); + expectNoPermits(h); + }); + + it("returns every permit after handler failures and permits later successful work", async () => { + const h = harness({ process: async job => { if (job.id === "bad") throw new Error("temporary timeout"); } }); + prepareProviders(h, () => ({ providers: [{ key: "shared-model", limit: 1 }], run: operation => operation() })); + h.job("bad", "trace_summary"); h.job("good", "trace_summary"); + expect(await h.runner.runWorkerOnce(2)).toMatchObject({ leased: 2, succeeded: 1, failed: 1 }); + expect(h.repos.runtime.getJob("good")!.status).toBe("succeeded"); + expectNoPermits(h); + }); + + it("returns reservations when a selected lease CAS fails", async () => { + const h = harness(); + h.job("missed", "trace_summary"); h.job("good", "trace_summary"); + const original = h.repos.runtime.claimQueuedJob.bind(h.repos.runtime); + const claim = vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementation((id, input) => id === "missed" ? undefined : original(id, input)); + try { + await h.runner.runWorkerOnce(2); + expect(h.repos.runtime.getJob("missed")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(h.repos.runtime.getJob("good")!.status).toBe("succeeded"); + expectNoPermits(h); + } finally { claim.mockRestore(); } + }); + + it("returns reservations on claim exceptions and can run again after the database recovers", async () => { + const h = harness(); h.job("recover", "trace_summary"); + const claim = vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementationOnce(() => { throw new Error("synthetic SQLITE_BUSY"); }); + try { + await expect(h.runner.runWorkerOnce(1)).rejects.toThrow("SQLITE_BUSY"); + expectNoPermits(h); + } finally { claim.mockRestore(); } + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expectNoPermits(h); + }); + + it("keeps automatic, manual target and retry drains behind one execution boundary", async () => { + const gate = deferred(); + const h = harness({ process: async job => { if (job.id === "slow") await gate.promise; } }); + h.job("slow", "trace_summary", { target: "target-A" }); + h.job("manual", "trace_summary", { target: "target-B" }); + h.retry("retry"); + const first = h.runner.runWorkerOnce(1, { targetMemoryIds: ["target-A"] }); + const manual = h.runner.runWorkerOnce(1, { targetMemoryIds: ["target-B"] }); + const retry = h.runner.runEmbeddingRetryOnce(1); + try { + await flushPump(); + expect(h.executed).toEqual(["slow"]); + expect(h.repos.runtime.getJob("manual")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(h.repos.runtime.getEmbeddingRetry("retry")).toMatchObject({ status: "pending", leaseToken: null }); + } finally { gate.resolve(); await Promise.all([first, manual, retry]); } + expect(h.repos.runtime.getJob("manual")!.status).toBe("succeeded"); + expect(h.repos.runtime.getEmbeddingRetry("retry")!.status).toBe("succeeded"); + expectNoPermits(h); + }); + + it("cleans permits on shutdown and rejects newly submitted drains", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + prepareProviders(h, () => ({ providers: [{ key: "shared-model", limit: 2 }], run: operation => operation() })); + for (let i = 0; i < 6; i++) h.job(`summary-${i}`, "trace_summary"); + const run = h.runner.runWorkerOnce(100); + await flushPump(); + const stopped = h.runner.stop(); + const rejected = expect(h.runner.runWorkerOnce(1)).rejects.toThrow(/stopped/i); + gate.resolve(); + await Promise.all([run, stopped, rejected]); + expectNoPermits(h); + }); + + + it("shares target exclusion between primary jobs and the retry backing table", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + h.memory("shared", at, []); + h.job("summary", "trace_summary", { target: "shared" }); + h.repos.runtime.enqueueEmbeddingRetry({ id: "retry", targetKind: "trace", targetId: "shared", + vectorField: "vec_summary", sourceText: "synthetic", now: epoch }); + const embed = vi.spyOn(h.deps.embedder, "embed"); + const run = h.runner.runWorkerOnce(2); + try { + await flushPump(); + expect(h.executed).toEqual(["summary"]); + expect(embed).not.toHaveBeenCalled(); + expect(h.repos.runtime.getEmbeddingRetry("retry")).toMatchObject({ status: "pending", attempts: 0, + claimedBy: null, leaseToken: null }); + } finally { gate.resolve(); await run; } + expect(h.repos.runtime.getEmbeddingRetry("retry")!.status).toBe("succeeded"); + expectNoPermits(h); + }); + + it("uses the same user/project exclusion for L3 and Work Memory without blocking another project", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + h.job("L3-A", "l3_world_model_update", { scopeKey: "user/project-A", scopeSeq: 1, payload: { projectId: "project-A" } }); + h.job("Work-A", "work_memory_extract", { scopeKey: "user/project-A", scopeSeq: 2, payload: { projectId: "project-A" } }); + h.job("L3-B", "l3_world_model_update", { scopeKey: "user/project-B", scopeSeq: 1, payload: { projectId: "project-B" } }); + const run = h.runner.runWorkerOnce(3); + try { + await flushPump(); + expect(h.executed).toEqual(["L3-A", "L3-B"]); + expect(h.repos.runtime.getJob("Work-A")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); await run; } + expect(h.repos.runtime.getJob("Work-A")!.status).toBe("succeeded"); + expectNoPermits(h); + }); + + it("counts existing running jobs when building an Embedding microbatch", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + const calls: string[][] = []; + h.deps.embeddingJobs.prepareEmbeddingJob = job => ({ job, memory: h.repos.memories.get(job.targetMemoryId!)!, + text: job.id, role: "document", vectorField: "vec_summary" }); + h.deps.embedder.embed = async texts => { calls.push([...texts]); await gate.promise; return texts.map(() => [1, 0, 0]); }; + h.job("summary-0", "trace_summary"); h.job("summary-1", "trace_summary"); + for (let i = 0; i < 8; i++) { + h.memory(`embedding-${i}`, at, []); h.job(`embedding-${i}`, "embedding", { target: `embedding-${i}` }); + } + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(h.executed).toEqual(["summary-0", "summary-1"]); + expect(calls).toHaveLength(1); + expect(calls[0]).toHaveLength(2); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 4, embeddingRequests: 1 }); + expect(mainRows(h).filter(row => row.status === "leased")).toHaveLength(4); + } finally { gate.resolve(); await run; } + expectNoPermits(h); + }); + + + it("keeps already running work protected when a later refill claim throws", async () => { + const gate = deferred(); + const h = harness({ process: async job => { if (job.id === "summary-0") await gate.promise; } }); + for (let i = 0; i < 6; i++) h.job(`summary-${i}`, "trace_summary"); + const original = h.repos.runtime.claimQueuedJob.bind(h.repos.runtime); + let refillFailed = false; + const claim = vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementation((id, input) => { + if (id === "summary-4") { refillFailed = true; throw new Error("refill SQLITE_BUSY"); } + return original(id, input); + }); + let settled = false; + const run = h.runner.runWorkerOnce(6); + void run.then(() => { settled = true; }, () => { settled = true; }); + try { + await flushPump(); + expect(refillFailed).toBe(true); + expect(settled).toBe(false); + expect(h.repos.runtime.getJob("summary-0")!.status).toBe("leased"); + expect(h.concurrency.snapshot().activeJobs).toBe(1); + } finally { + gate.resolve(); + await expect(run).rejects.toThrow("refill SQLITE_BUSY"); + claim.mockRestore(); + } + expectNoPermits(h); + expect(h.repos.runtime.getJob("summary-0")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("summary-4")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + }); + + + it("keeps individual attempt results when the same task becomes due again during a long drain", async () => { + const gate = deferred(); + let flakyAttempts = 0; + const h = harness({ process: async job => { + if (job.id === "slow") await gate.promise; + if (job.id === "flaky" && flakyAttempts++ === 0) throw new Error("temporary timeout"); + } }); + h.job("flaky", "trace_summary"); h.job("slow", "trace_summary"); + const run = h.runner.runWorkerOnce(3); + try { + await flushPump(); + expect(h.repos.runtime.getJob("flaky")).toMatchObject({ status: "failed", failureCount: 1 }); + vi.setSystemTime(epoch + 10_000); + h.runner.notifyWorkAvailable(); + await flushPump(); + expect(flakyAttempts).toBe(2); + expect(h.repos.runtime.getJob("flaky")!.status).toBe("succeeded"); + } finally { gate.resolve(); } + const result = await run; + expect(result).toMatchObject({ leased: 3, succeeded: 2, failed: 1 }); + expect(result.jobs).toEqual([ + expect.objectContaining({ jobId: "flaky", status: "failed" }), + expect.objectContaining({ jobId: "slow", status: "succeeded" }), + expect.objectContaining({ jobId: "flaky", status: "succeeded" }) + ]); + expectNoPermits(h); + }); + + + it("admits newly queued independent work before an existing slow task completes", async () => { + const gate = deferred(); + const h = harness({ process: async job => { if (job.id === "slow") await gate.promise; } }); + prepareProviders(h, work => ({ providers: [{ key: work.queue === "main" && work.job.id === "slow" ? "model-A" : "model-B", limit: 1 }], + run: operation => operation() })); + h.job("slow", "trace_summary"); + const run = h.runner.runWorkerOnce(2); + try { + await flushPump(); + expect(h.executed).toEqual(["slow"]); + h.job("late-database", "l2_association"); + h.runner.notifyWorkAvailable(); + await flushPump(); + expect(h.executed).toEqual(["slow", "late-database"]); + expect(h.repos.runtime.getJob("late-database")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("slow")!.status).toBe("leased"); + } finally { gate.resolve(); await run; } + expectNoPermits(h); + }); + + it("wakes a drain when an external foreground request releases its provider", async () => { + const gate = deferred(), entered = deferred(); + const h = harness(); + const descriptor = { local: false, providers: [{ key: "external-model", limit: 1 }] }; + prepareProviders(h, () => ({ providers: descriptor.providers, run: operation => operation() })); + const foreground = h.concurrency.runRequest(descriptor, async () => { entered.resolve(); await gate.promise; }); + await entered.promise; + h.job("waiting-summary", "trace_summary"); + const run = h.runner.runWorkerOnce(1); + try { + await flushPump(); + expect(h.executed).toEqual([]); + expect(h.repos.runtime.getJob("waiting-summary")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + gate.resolve(); await foreground; + await flushPump(); + expect(h.executed).toEqual(["waiting-summary"]); + } finally { gate.resolve(); await foreground; h.runner.notifyWorkAvailable(); await run; } + expectNoPermits(h); + }); + + it("keeps the last valid Embedding share group when a middle row fails its first lease CAS", async () => { + const h = harness(); + const calls: string[][] = []; + h.deps.embeddingJobs.prepareEmbeddingJob = job => ({ job, memory: h.repos.memories.get(job.targetMemoryId!)!, + text: job.id, role: "document", vectorField: "vec_summary" }); + h.deps.embedder.embed = async texts => { calls.push([...texts]); return texts.map(() => [1, 0, 0]); }; + for (let i = 0; i < 3; i++) { + h.memory(`embedding-${i}`, at, []); h.job(`embedding-${i}`, "embedding", { target: `embedding-${i}` }); + } + const original = h.repos.runtime.claimQueuedJob.bind(h.repos.runtime); + let missed = false; + const claim = vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementation((id, input) => { + if (id === "embedding-1" && !missed) { missed = true; return undefined; } + return original(id, input); + }); + try { + expect(await h.runner.runWorkerOnce(3)).toMatchObject({ leased: 3, succeeded: 3, failed: 0 }); + expect(calls).toEqual([["embedding-0", "embedding-2"], ["embedding-1"]]); + expect(h.repos.runtime.getJob("embedding-1")).toMatchObject({ status: "succeeded", attempts: 1, failureCount: 0 }); + expectNoPermits(h); + } finally { claim.mockRestore(); } + }); + + + it("uses persisted session and L3 batch projects when production payloads omit projectId", async () => { + const gate = deferred(); + const h = harness({ process: async () => { await gate.promise; } }); + h.db.db.prepare(`INSERT INTO sessions + (id,user_id,project_id,source,profile_id,opened_at,last_seen_at,updated_at) + VALUES ('work-session','user','real-project','codex','test',?,?,?)`).run(at, at, at); + const scope = h.repos.l3WorldModels.ensureScope("user", "real-project", at); + h.db.db.prepare(`INSERT INTO l3_world_model_evidence_batches + (id,scope_key,scope_seq,user_id,project_id,session_id,trigger,start_trace_seq,end_trace_seq, + l1_memory_ids_json,raw_turn_ids_json,feedback_ids_json,payload_hash,created_at,updated_at) + VALUES ('real-batch',?,1,'user','real-project','work-session','session_close',1,1, + '[]','[]','[]','synthetic-hash',?,?)`).run(scope.scopeKey, at, at); + const l3 = h.job("production-L3", "l3_world_model_update", { scopeKey: "opaque-l3-field-hash", scopeSeq: 1, + payload: { batchId: "real-batch", targetField: "project_contract" } }); + const work = h.job("production-Work", "work_memory_extract", { scopeKey: "different-opaque-work-hash", scopeSeq: 1, + sessionId: "work-session", payload: { trajectoryHash: "synthetic-trajectory", qa: [] } }); + expect(l3.payload).not.toHaveProperty("projectId"); + expect(work.payload).not.toHaveProperty("projectId"); + expect(h.repos.runtime.getSession(work.sessionId!)!.projectId).toBe("real-project"); + expect(h.repos.l3WorldModels.getBatch(String(l3.payload.batchId))!.projectId).toBe("real-project"); + const run = h.runner.runWorkerOnce(2); + try { + await flushPump(); + expect(h.executed).toEqual(["production-L3"]); + expect(h.repos.runtime.getJob("production-Work")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); await run; } + expect(h.executed).toEqual(["production-L3", "production-Work"]); + expectNoPermits(h); + }); + + it("does not busy-loop on synchronous CAS-release wakes while still waiting for an external provider", async () => { + const gate = deferred(), entered = deferred(); + const h = harness(); + const descriptor = { local: false, providers: [{ key: "external-model", limit: 1 }] }; + prepareProviders(h, work => ({ providers: work.queue === "main" && work.job.id === "cas-miss" ? [] : descriptor.providers, + run: operation => operation() })); + const foreground = h.concurrency.runRequest(descriptor, async () => { entered.resolve(); await gate.promise; }); + await entered.promise; + h.job("cas-miss", "trace_summary"); h.job("provider-waiting", "trace_summary"); + const original = h.repos.runtime.claimQueuedJob.bind(h.repos.runtime); + let misses = 0; + const claim = vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementation((id, input) => { + if (id === "cas-miss") { misses++; return undefined; } + return original(id, input); + }); + const run = h.runner.runWorkerOnce(1); + try { + await flushPump(); + expect(misses).toBe(1); + expect(h.executed).toEqual([]); + gate.resolve(); await foreground; + await flushPump(); + expect(h.executed).toEqual(["provider-waiting"]); + } finally { gate.resolve(); await foreground; h.runner.notifyWorkAvailable(); await run; claim.mockRestore(); } + expectNoPermits(h); + }); + + + it("wakes naturally at runAfter while an unrelated slow task still occupies one slot", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + const h = harness({ process: async job => { if (job.id === "slow") await gate.promise; } }); + h.job("slow", "trace_summary"); + h.job("scheduled-database", "l2_association", { payload: { runAfter: new Date(epoch + 5_000).toISOString() } }); + const run = h.runner.runWorkerOnce(100); + try { + await flushPump(); + expect(h.executed).toEqual(["slow"]); + await vi.advanceTimersByTimeAsync(5_001); + expect(h.executed).toEqual(["slow", "scheduled-database"]); + expect(h.repos.runtime.getJob("scheduled-database")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("slow")!.status).toBe("leased"); + } finally { gate.resolve(); await run; } + expectNoPermits(h); + expect(vi.getTimerCount()).toBe(0); + }); + + it.each(["wrr", "legacy"] as const)("wakes naturally when a failed main task's backoff expires without an explicit notify (%s)", async algorithm => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + let attempts = 0; + const h = harness({ algorithm, process: async job => { + if (job.id === "slow") await gate.promise; + if (job.id === "retry-after-backoff" && attempts++ === 0) throw new Error("temporary timeout"); + } }); + h.job("retry-after-backoff", "trace_summary"); h.job("slow", "trace_summary"); + const run = h.runner.runWorkerOnce(3, {}, { continuous: true }); + let result; + try { + await flushPump(); + const failed = h.repos.runtime.getJob("retry-after-backoff")!; + expect(failed).toMatchObject({ status: "failed", failureCount: 1 }); + await vi.advanceTimersByTimeAsync(Date.parse(failed.availableAt!) - epoch + 1); + expect(attempts).toBe(2); + expect(h.repos.runtime.getJob("retry-after-backoff")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("slow")!.status).toBe("leased"); + } finally { gate.resolve(); result = await run; } + expect(result).toMatchObject({ leased: 3, succeeded: 2, failed: 1 }); + expectNoPermits(h); + expect(vi.getTimerCount()).toBe(0); + }); + + it("wakes naturally when an Embedding retry reaches nextAttemptAt", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + const h = harness({ process: async job => { if (job.id === "slow") await gate.promise; } }); + h.job("slow", "trace_summary"); const retry = h.retry("delayed-retry"); + h.db.db.prepare("UPDATE embedding_retry_queue SET next_attempt_at=? WHERE id=?").run(epoch + 5_000, retry.id); + const run = h.runner.runWorkerOnce(2); + try { + await flushPump(); + expect(h.repos.runtime.getEmbeddingRetry(retry.id)).toMatchObject({ status: "pending", attempts: 0, leaseToken: null }); + await vi.advanceTimersByTimeAsync(5_001); + expect(h.repos.runtime.getEmbeddingRetry(retry.id)!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("slow")!.status).toBe("leased"); + } finally { gate.resolve(); await run; } + expectNoPermits(h); + expect(vi.getTimerCount()).toBe(0); + }); + + it("wakes at an expired foreign lease without waiting for an unrelated running task", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + const h = harness({ process: async job => { if (job.id === "slow") await gate.promise; } }); + h.job("recover-expired", "trace_summary"); + const foreign = h.repos.runtime.leaseQueuedJobs(1, 5)[0]!; + h.job("slow", "trace_summary"); + const run = h.runner.runWorkerOnce(2); + try { + await flushPump(); + expect(h.executed).toEqual(["slow"]); + await vi.advanceTimersByTimeAsync(5_001); + expect(h.executed).toEqual(["slow", "recover-expired"]); + expect(h.repos.runtime.getJob("recover-expired")).toMatchObject({ status: "succeeded", leaseRecoveryCount: 1, + failureCount: 0, attempts: foreign.attempts }); + } finally { gate.resolve(); await run; } + expectNoPermits(h); + expect(vi.getTimerCount()).toBe(0); + }); + + + it("reconciles a naturally due budget deadline during continuous execution before the slow task finishes", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + let paused = true, reconcileCalls = 0; + const h = harness({ process: async job => { if (job.id === "slow-database") await gate.promise; } }); + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "trace_summary"; + h.deps.memoryBudgetModelSources = () => ({ memory_summary: { source: "byok" }, memory_evolution: { source: "byok" }, + embedding: { source: "byok", mode: "remote" } }); + h.deps.memoryBudgetNextWakeAtMs = () => epoch + 86_400_000; + h.deps.memoryBudgetNextReconcileAtMs = () => epoch + 5_000; + h.deps.beforeRun = async () => { + reconcileCalls++; + if (Date.now() >= epoch + 5_000) paused = false; + }; + h.job("slow-database", "l2_association"); h.job("budget-held-summary", "trace_summary"); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true }); + try { + await flushPump(); + expect(h.executed).toEqual(["slow-database"]); + expect(reconcileCalls).toBe(1); + await vi.advanceTimersByTimeAsync(5_001); + expect(h.executed).toEqual(["slow-database", "budget-held-summary"]); + expect(h.repos.runtime.getJob("budget-held-summary")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("slow-database")!.status).toBe("leased"); + expect(reconcileCalls).toBe(2); + // A still-stale callback deadline must not cause a reconcile busy loop. + await vi.advanceTimersByTimeAsync(1_000); + expect(reconcileCalls).toBe(2); + } finally { gate.resolve(); await run; } + expectNoPermits(h); + expect(vi.getTimerCount()).toBe(0); + }); + + + it("does not admit more work when continuous execution is aborted during an awaited budget reconciliation", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const slow = deferred(), reconcile = deferred(), reconciling = deferred(); + const abort = new AbortController(); + let paused = true; + const h = harness({ process: async job => { if (job.id === "slow-database") await slow.promise; } }); + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "trace_summary"; + h.deps.memoryBudgetModelSources = () => ({ memory_summary: { source: "byok" }, memory_evolution: { source: "byok" }, + embedding: { source: "byok", mode: "remote" } }); + h.deps.memoryBudgetNextWakeAtMs = () => epoch + 86_400_000; + h.deps.memoryBudgetNextReconcileAtMs = () => epoch + 5_000; + h.deps.beforeRun = async () => { + if (Date.now() >= epoch + 5_000) { + reconciling.resolve(); await reconcile.promise; paused = false; + } + }; + h.job("slow-database", "l2_association"); h.job("not-admitted-after-abort", "trace_summary"); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + await flushPump(); + await vi.advanceTimersByTimeAsync(5_001); + await reconciling.promise; + abort.abort(); reconcile.resolve(); + await flushPump(); + expect(h.executed).toEqual(["slow-database"]); + expect(h.repos.runtime.getJob("not-admitted-after-abort")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(h.repos.runtime.getJob("slow-database")!.status).toBe("leased"); + } finally { abort.abort(); reconcile.resolve(); slow.resolve(); await run; } + expectNoPermits(h); + expect(getEventListeners(abort.signal, "abort")).toHaveLength(0); + expect(vi.getTimerCount()).toBe(0); + }); + +}); diff --git a/Memory/tests/service/worker/worker-embedding-budget-boundary.test.ts b/Memory/tests/service/worker/worker-embedding-budget-boundary.test.ts new file mode 100644 index 000000000..d31b63ced --- /dev/null +++ b/Memory/tests/service/worker/worker-embedding-budget-boundary.test.ts @@ -0,0 +1,242 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ModelHttpError } from "../../../src/model/http.js"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const epoch = Date.parse("2026-10-22T00:00:00.000Z"); +const at = new Date(epoch).toISOString(); +const resources: Array<{ root: string; db: MemoryDb }> = []; +type HarnessOptions = { algorithm?: "wrr" | "legacy"; process?: (job: EvolutionJobRecord) => Promise }; + +/** Real SQLite and claim transitions; business/model work is a local substitute. */ +function harness(options: HarnessOptions = {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-fairness-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const executed: string[] = []; + const deps: WorkerRunnerDeps = { + repos, embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "fairness-test", memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), summaryModelConfigured: () => true, + schedulingShadow: { enabled: false }, + ...(options.algorithm ? { scheduling: { algorithm: options.algorithm } } : {}), + nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "synthetic", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement retry"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async (job) => { executed.push(job.id); await options.process?.(job); } }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding job"); }, + applyEmbeddingVector: () => {}, enqueueEmbeddingRetryAfterFailure: () => false, + persistEmbeddingVector: (input) => { + const memory = repos.memories.get(input.memoryId)!; + executed.push(`retry:${input.memoryId}`); + input.finalize?.(memory, false, new Date().toISOString()); + return memory; + } + } + }; + const runner = new WorkerRunner(deps); + function memory(id: string, createdAt = at, tags: string[] = ["agent-source"]) { + db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,created_at,updated_at) + VALUES (?,?,'user','synthetic','L1',?,?,?)`).run(id, createdAt, JSON.stringify(tags), createdAt, createdAt); + } + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection", input: { + target?: string; source?: string; createdAt?: string; payload?: Record; + scopeKey?: string; scopeSeq?: number; + } = {}) { + return repos.runtime.enqueueJob({ id, jobType, userId: "user", targetMemoryId: input.target, + scopeKey: input.scopeKey, scopeSeq: input.scopeSeq, + payload: { ...input.payload, ...(input.source ? { source: input.source } : {}) }, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: input.createdAt ?? at, updatedAt: at }); + } + function retry(id: string, createdAt = at) { + memory(id, createdAt); + return repos.runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, + vectorField: "vec_summary", sourceText: "synthetic", now: epoch }); + } + return { root, db, repos, deps, runner, job, memory, retry, executed }; +} +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + vi.stubEnv("MEMMY_WORKER_SCHEDULER_ALGORITHM", "wrr"); +}); +afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +function embeddingRoleHarness(algorithm: "wrr" | "legacy", roles: Array<"document" | "query">) { + const h = harness({ algorithm }); + const calls: Array<{ role: "document" | "query" | undefined; texts: string[] }> = []; + const applied: string[] = []; + h.deps.embeddingJobs.prepareEmbeddingJob = (job) => ({ + job, memory: h.repos.memories.get(job.targetMemoryId!)!, text: job.id, + role: job.payload.role as "document" | "query", vectorField: "vec_summary" + }); + h.deps.embeddingJobs.applyEmbeddingVector = (item) => { applied.push(item.job.id); }; + h.deps.embedder.embed = async (texts, role) => { + calls.push({ role, texts: [...texts] }); + return texts.map(() => [1, 0, 0]); + }; + roles.forEach((role, index) => { + const id = `embedding-${index}`; + h.memory(id, at, []); + h.job(id, "embedding", { target: id, payload: { role } }); + }); + return { ...h, calls, applied }; +} + + +function inputTooLong() { + return new ModelHttpError("maximum context length exceeded", "openai_compatible", 400, + "context_length_exceeded", "maximum input length exceeded"); +} +function saveProcessing(h: ReturnType, id: string, attemptCount = 0) { + h.repos.processing.save({ memoryId: id, state: "embedding_pending", stage: "embedding", + activeJobId: id, attemptCount, manualRetryCount: 0, retryAction: "retry", updatedAt: at }); +} + +describe("Embedding budget checks between requests", () => { + it("preserves legacy SQL admission order while holding subsequent main roles after a pause", async () => { + const h = embeddingRoleHarness("legacy", ["query", "document", "query"]); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + for (let i = 0; i < 3; i++) saveProcessing(h, `embedding-${i}`); + const originalEmbed = h.deps.embedder.embed; + h.deps.embedder.embed = async (texts, role) => { + const result = await originalEmbed(texts, role); + paused = true; + return result; + }; + expect(await h.runner.runWorkerOnce(3)).toMatchObject({ leased: 3, succeeded: 1, failed: 0 }); + expect(h.calls).toEqual([{ role: "query", texts: ["embedding-0"] }]); + for (const id of ["embedding-1", "embedding-2"]) { + expect(h.repos.runtime.getJob(id)).toMatchObject({ status: "queued", attempts: 0, failureCount: 0 }); + expect(h.repos.processing.get(id)).toMatchObject({ state: "embedding_pending", attemptCount: 0 }); + } + }); + + it("preserves legacy document-first retry ordering while holding the next role after a pause", async () => { + const h = harness({ algorithm: "legacy" }); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + h.retry("document-retry"); + const query = h.retry("query-retry"); + h.db.db.prepare("UPDATE embedding_retry_queue SET embed_role = 'query' WHERE id = ?").run(query.id); + const roles: Array<"document" | "query" | undefined> = []; + h.deps.embedder.embed = async (texts, role) => { roles.push(role); paused = true; return texts.map(() => [1, 0, 0]); }; + expect(await h.runner.runEmbeddingRetryOnce(2)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(roles).toEqual(["document"]); + expect(h.repos.runtime.getEmbeddingRetry(query.id)).toMatchObject({ status: "pending", attempts: 0, + enqueueSeq: query.enqueueSeq, nextAttemptAt: query.nextAttemptAt, claimedBy: null, leaseToken: null, leaseUntil: null }); + }); + + it("returns later role batches to their original FIFO and waiting processing state, then resumes", async () => { + const h = embeddingRoleHarness("wrr", ["query", "document", "query"]); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + for (let i = 0; i < 3; i++) { + saveProcessing(h, `embedding-${i}`, 2); + h.db.db.prepare("UPDATE evolution_jobs SET attempts = 2, failure_count = 1 WHERE id = ?").run(`embedding-${i}`); + } + const originals = [1, 2].map(i => h.repos.runtime.getJob(`embedding-${i}`)!); + const originalEmbed = h.deps.embedder.embed; + h.deps.embedder.embed = async (texts, role) => { + const result = await originalEmbed(texts, role); + if (h.calls.length === 1) paused = true; + return result; + }; + const first = await h.runner.runWorkerOnce(3); + expect(h.calls).toEqual([{ role: "query", texts: ["embedding-0"] }]); + expect(first).toMatchObject({ leased: 3, succeeded: 1, failed: 0 }); + expect(first.jobs.map(job => job.status)).toEqual(["succeeded", "queued", "queued"]); + for (const original of originals) { + expect(h.repos.runtime.getJob(original.id)).toMatchObject({ status: "queued", attempts: 2, + failureCount: 1, enqueueSeq: original.enqueueSeq, firstQueuedAt: original.firstQueuedAt, + leasedUntil: null, leaseOwner: null, leaseToken: null }); + expect(h.repos.processing.get(original.targetMemoryId!)).toMatchObject({ state: "embedding_pending", + stage: "embedding", activeJobId: original.id, attemptCount: 2 }); + } + paused = false; + expect(await h.runner.runWorkerOnce(3)).toMatchObject({ leased: 2, succeeded: 2, failed: 0 }); + expect(h.applied).toEqual(["embedding-0", "embedding-1", "embedding-2"]); + }); + + it.each([false, true])("checks main item fallback boundaries when pause starts before first item=%s", async pauseBeforeFirst => { + const h = embeddingRoleHarness("wrr", ["document", "document", "document"]); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + for (let i = 0; i < 3; i++) saveProcessing(h, `embedding-${i}`); + h.deps.embedder.embed = async () => { paused = pauseBeforeFirst; throw inputTooLong(); }; + const singles: string[] = []; + h.deps.embedder.embedOne = async text => { singles.push(text); paused = true; return [1, 0, 0]; }; + const result = await h.runner.runWorkerOnce(3); + expect(singles).toEqual(pauseBeforeFirst ? [] : ["embedding-0"]); + expect(result).toMatchObject({ leased: 3, succeeded: pauseBeforeFirst ? 0 : 1, failed: 0 }); + for (let i = pauseBeforeFirst ? 0 : 1; i < 3; i++) { + expect(h.repos.runtime.getJob(`embedding-${i}`)).toMatchObject({ status: "queued", attempts: 0, failureCount: 0 }); + expect(h.repos.processing.get(`embedding-${i}`)).toMatchObject({ state: "embedding_pending", attemptCount: 0 }); + } + }); + + it.each([false, true])("checks retry item fallback boundaries when pause starts before first item=%s", async pauseBeforeFirst => { + const h = harness(); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + const originals = [0, 1, 2].map(i => h.retry(`retry-${i}`)); + h.deps.embedder.embed = async () => { paused = pauseBeforeFirst; throw inputTooLong(); }; + const singles: string[] = []; + h.deps.embedder.embedOne = async text => { singles.push(text); paused = true; return [1, 0, 0]; }; + const result = await h.runner.runWorkerOnce(3); + expect(singles).toHaveLength(pauseBeforeFirst ? 0 : 1); + expect(result.embeddingRetries).toMatchObject({ leased: 3, succeeded: pauseBeforeFirst ? 0 : 1, failed: 0 }); + for (const original of originals.slice(pauseBeforeFirst ? 0 : 1)) { + expect(h.repos.runtime.getEmbeddingRetry(original.id)).toMatchObject({ status: "pending", attempts: 0, + enqueueSeq: original.enqueueSeq, nextAttemptAt: original.nextAttemptAt, + claimedBy: null, leaseToken: null, leaseUntil: null }); + } + }); + + it("does not restore processing owned by a replacement lease when returning a later role", async () => { + const h = embeddingRoleHarness("wrr", ["query", "document"]); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + saveProcessing(h, "embedding-1"); + h.job("replacement-job", "embedding", { target: "embedding-1" }); + const originalEmbed = h.deps.embedder.embed; + h.deps.embedder.embed = async (texts, role) => { + const result = await originalEmbed(texts, role); + h.db.db.prepare("UPDATE evolution_jobs SET lease_owner = 'replacement', lease_token = 'replacement-token' WHERE id = 'embedding-1'").run(); + h.repos.processing.update("embedding-1", { activeJobId: "replacement-job", state: "embedding" }); + paused = true; + return result; + }; + await h.runner.runWorkerOnce(2); + expect(h.calls).toHaveLength(1); + expect(h.repos.runtime.getJob("embedding-1")).toMatchObject({ status: "leased", leaseOwner: "replacement", leaseToken: "replacement-token" }); + expect(h.repos.processing.get("embedding-1")).toMatchObject({ activeJobId: "replacement-job", state: "embedding" }); + }); +}); diff --git a/Memory/tests/service/worker/worker-episode-scope.test.ts b/Memory/tests/service/worker/worker-episode-scope.test.ts new file mode 100644 index 000000000..1088dab60 --- /dev/null +++ b/Memory/tests/service/worker/worker-episode-scope.test.ts @@ -0,0 +1,89 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { workerWorkRequirements } from "../../../src/service/worker/worker-work-requirements.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ root: string; db: MemoryDb; runner?: WorkerRunner }> = []; +function deferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} +function harness(process: (job: EvolutionJobRecord) => Promise = async () => {}) { + const root = mkdtempSync(join(tmpdir(), "worker-episode-scope-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const resource: typeof resources[number] = { root, db }; resources.push(resource); + const repos = new Repositories(db.db), concurrency = new WorkerConcurrencyController({}, {}); + const deps: WorkerRunnerDeps = { + repos, concurrency, scheduling: { algorithm: "wrr" }, schedulingShadow: { enabled: false }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: true }, embeddingRetryWorkerId: "episode-review", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "review", runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, enqueueEmbeddingRetry: () => { throw new Error("unexpected embedding retry"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, jobHandlers: { processJob: process }, + embeddingJobs: { prepareEmbeddingJob: () => { throw new Error("unexpected embedding"); }, + applyEmbeddingVector: () => {}, persistEmbeddingVector: () => { throw new Error("unexpected embedding vector"); }, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); resource.runner = runner; + function job(id: string, jobType: "reflection" | "reward", episodeId: string) { + return repos.runtime.enqueueJob({ id, jobType, userId: "review-user", episodeId, + ...(jobType === "reflection" ? { targetMemoryId: `trace-${episodeId}` } : {}), + payload: jobType === "reward" ? { l1MemoryId: `trace-${episodeId}`, polarity: "positive" } : { targetKind: "episode" }, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + } + return { repos, concurrency, runner, job }; +} +afterEach(async () => { + for (const { db, root, runner } of resources.splice(0)) { + await runner?.stop(); if (db.db.open) db.close(); rmSync(root, { recursive: true, force: true }); + } +}); + +describe("reflection and reward episode admission", () => { + it("shares the episode permit for production reflection/reward metadata, retaining target protection", () => { + const h = harness(); + const reflection = h.job("reflection-A", "reflection", "episode-A"); + const reward = h.job("reward-A", "reward", "episode-A"); + expect(reward.targetMemoryId).toBeUndefined(); + const reflectionRequirements = workerWorkRequirements({ queue: "main", job: reflection }, h.repos); + const rewardRequirements = workerWorkRequirements({ queue: "main", job: reward }, h.repos); + expect(reflectionRequirements.scopeKeys).toContain(JSON.stringify(["target", "trace-episode-A"])); + const first = h.concurrency.tryAcquire(reflectionRequirements)!; + const conflicting = h.concurrency.tryAcquire(rewardRequirements); + conflicting?.release(); first.release(); + expect(conflicting).toBeUndefined(); + const next = h.concurrency.tryAcquire(rewardRequirements); + expect(next).toBeDefined(); next!.release(); + }); + + it("does not claim same-episode reward while reflection runs, but runs another episode in parallel", async () => { + const gate = deferred(), entered = deferred(); const executed: string[] = []; + const h = harness(async job => { + executed.push(job.id); + if (job.id === "reflection-A") { entered.resolve(); await gate.promise; } + }); + h.job("reflection-A", "reflection", "episode-A"); h.job("reward-A", "reward", "episode-A"); h.job("reward-B", "reward", "episode-B"); + const run = h.runner.runWorkerOnce(3); await entered.promise; + for (let i = 0; i < 100; i++) await Promise.resolve(); + const heldReward = h.repos.runtime.getJob("reward-A"); + const otherReward = h.repos.runtime.getJob("reward-B"); + const beforeRelease = [...executed]; + gate.resolve(); const result = await run; + expect(heldReward).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(otherReward?.status).toBe("succeeded"); + expect(beforeRelease).toEqual(["reflection-A", "reward-B"]); + expect(executed).toEqual(["reflection-A", "reward-B", "reward-A"]); + expect(result).toMatchObject({ leased: 3, succeeded: 3, failed: 0 }); + expect(h.concurrency.snapshot()).toEqual({ activeJobs: 0, availableJobs: 4, backgroundActiveJobs: 0, maxBackgroundJobs: 3, interactiveReservedSlots: 1, resources: {}, providers: {}, scopeKeys: [], embeddingRequests: 0 }); + }); +}); diff --git a/Memory/tests/service/worker/worker-event-loop-yield.test.ts b/Memory/tests/service/worker/worker-event-loop-yield.test.ts new file mode 100644 index 000000000..e4d630e88 --- /dev/null +++ b/Memory/tests/service/worker/worker-event-loop-yield.test.ts @@ -0,0 +1,959 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setImmediate as nextTurn } from "node:timers/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { WorkerConcurrencyController, type WorkerProviderDescriptor } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerAdmissionWaitSet } from "../../../src/service/worker/worker-admission-wait-set.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ root: string; db: MemoryDb; runner: WorkerRunner; concurrency: WorkerConcurrencyController }> = []; +const total = 200; +function harness(maxActiveJobs = 4) { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-event-loop-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db), concurrency = new WorkerConcurrencyController({ maxActiveJobs }, {}); + const processed: string[] = []; + let peakJobs = 0; + const unexpected = () => { throw new Error("Unexpected model or startup operation"); }; + const deps: WorkerRunnerDeps = { + repos, concurrency, scheduling: { algorithm: "wrr" }, schedulingShadow: { enabled: false }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: false }, embeddingRetryWorkerId: "event-loop-test", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "event-loop-test", runWorkerNoWrite: async () => unexpected(), + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: unexpected, enqueueEmbeddingRetry: unexpected, appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async job => { + peakJobs = Math.max(peakJobs, concurrency.snapshot().activeJobs); + processed.push(job.id); + repos.runtime.setKv("event-loop.database-progress", { id: job.id }, new Date().toISOString()); + } }, + embeddingJobs: { prepareEmbeddingJob: unexpected, applyEmbeddingVector: unexpected, + persistEmbeddingVector: unexpected, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); + resources.push({ root, db, runner, concurrency }); + function job(id: string, type: EvolutionJobRecord["jobType"] = "l2_association", target = id, delayed = false) { + const at = new Date().toISOString(); + const queued = repos.runtime.enqueueJob({ id, jobType: type, userId: "event-loop-user", targetMemoryId: target, + payload: {}, status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + if (delayed) db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id=?") + .run("2100-01-01T00:00:00.000Z", id); + return queued; + } + function jobs(type: EvolutionJobRecord["jobType"] = "l2_association") { + repos.transaction(() => { for (let i = 0; i < total; i++) job(`auto-${i}`, type); }); + } + function expectClean() { + expect(concurrency.snapshot()).toEqual({ activeJobs: 0, availableJobs: maxActiveJobs, backgroundActiveJobs: 0, + maxBackgroundJobs: concurrency.limits.maxBackgroundJobs, interactiveReservedSlots: concurrency.limits.interactiveReservedSlots, resources: {}, providers: {}, scopeKeys: [], embeddingRequests: 0 }); + const queued = db.db.prepare("SELECT attempts,lease_token FROM evolution_jobs WHERE status='queued'").all(); + expect(queued.every(row => (row as { attempts: number; lease_token: string | null }).attempts === 0 + && (row as { lease_token: string | null }).lease_token === null)).toBe(true); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get()).toEqual({ n: 0 }); + } + return { db, repos, concurrency, runner, deps, processed, job, jobs, expectClean, peakJobs: () => peakJobs }; +} +function timerEvent(operation: () => void): Promise { + return new Promise((resolve, reject) => setTimeout(() => { try { operation(); resolve(); } catch (error) { reject(error); } }, 0)); +} +afterEach(async () => { + for (const { root, db, runner, concurrency } of resources.splice(0)) { + await runner.stop(); await concurrency.close(); + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } + vi.useRealTimers(); vi.restoreAllMocks(); +}); + +function blockedBacklog(h: ReturnType, count = 1000) { + const provider = [{ key: "blocked-local", limit: 1 }]; + const held = h.concurrency.tryAcquire({ resourceClass: "summary", providers: provider, scopeKeys: [] })!; + let prepared = 0; + h.deps.prepareExecution = work => { + prepared++; + return { providers: work.queue === "main" && work.job.jobType === "trace_summary" ? provider : [], + run: operation => operation() }; + }; + h.repos.transaction(() => { + for (let i = 0; i < count; i++) h.job(`blocked-${i}`, "trace_summary"); + h.job("late-database"); + h.db.db.prepare("UPDATE evolution_jobs SET queue_lane='import'").run(); + }); + return { held, prepared: () => prepared, count }; +} + +describe("continuous worker event-loop fairness", () => { + it("resumes scheduled maintenance inside an open worker entry and preserves its target context", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const h = harness(), abort = new AbortController(), backlog = blockedBacklog(h, 1); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + let clock = Date.now(), maintenanceAt: number | undefined = clock + 100; + h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + h.deps.maintenanceNextRunAtMs = () => maintenanceAt; + const before = vi.fn(async (request?: { targetMemoryIds?: readonly string[] }) => { + expect(request).toEqual({}); + if (maintenanceAt !== undefined && maintenanceAt <= clock) { + maintenanceAt = undefined; h.job("maintenance-ready", "user_memory_embedding"); + } + }); + h.deps.beforeRun = before; + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 12; i++) await nextTurn(); + expect(before).toHaveBeenCalledTimes(1); + clock += 100; await vi.advanceTimersByTimeAsync(100); + for (let i = 0; i < 15; i++) await nextTurn(); + expect(before).toHaveBeenCalledTimes(2); + expect(h.processed).toEqual(["maintenance-ready"]); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + describe.each(["available", "runAfter", "mainLease", "retryTime", "retryLease", "failed"] as const)("refreshing an old %s deadline", kind => { + it.each([0, 1])("retains the deadline when a new insert's snapshot crosses it by %i ms", async crossedBy => { + const h = harness(), abort = new AbortController(); + let clock = Date.now(); + h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + const due = clock + 3_600_000, future = new Date(due).toISOString(); + const retry = kind === "retryTime" || kind === "retryLease"; + if (retry) { + h.repos.runtime.enqueueEmbeddingRetry({ id: "old-deadline", targetKind: "trace", targetId: "missing", + vectorField: "vec_summary", sourceText: "synthetic", now: due }); + if (kind === "retryLease") h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by='old-worker',next_attempt_at=?,lease_until=? WHERE id='old-deadline'").run(clock - 1, due); + } else { + h.job("old-deadline", "user_memory_embedding"); + if (kind === "available" || kind === "failed") h.db.db.prepare("UPDATE evolution_jobs SET available_at=?,status=?,failure_count=? WHERE id='old-deadline'") + .run(future, kind === "failed" ? "failed" : "queued", kind === "failed" ? 1 : 0); + else if (kind === "runAfter") h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='old-deadline'").run(JSON.stringify({ runAfter: future })); + else h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='old-deadline'").run(future); + } + const backlog = blockedBacklog(h, 1); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + if (retry) { + h.repos.runtime.enqueueEmbeddingRetry({ id: "later-blocked-retry", targetKind: "trace", targetId: "other-missing", + vectorField: "vec_summary", sourceText: "synthetic", now: clock - 1 }); + const prepare = h.deps.prepareExecution!; + h.deps.prepareExecution = work => work.queue === "embedding_retry" && work.retry.id === "later-blocked-retry" + ? { providers: [{ key: "blocked-local", limit: 1 }], run: operation => operation() } : prepare(work); + } + h.repos.runtime.scheduling.refreshPolicies(100); + h.db.db.prepare("UPDATE evolution_jobs SET queue_lane='import'").run(); + h.db.db.prepare("UPDATE embedding_retry_queue SET queue_lane='import'").run(); + const claim = vi.spyOn(h.repos.runtime, "claimEmbeddingRetryById"); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 15; i++) await nextTurn(); + expect(h.processed).toEqual([]); + expect(backlog.prepared()).toBe(1); + // The insert arrives one millisecond before the old deadline. SQLite + // crosses that deadline after the expiry check, without changing rows. + clock = due - 1; + const read = h.repos.runtime.scheduling.readSnapshot.bind(h.repos.runtime.scheduling); + const snapshot = vi.spyOn(h.repos.runtime.scheduling, "readSnapshot").mockImplementationOnce(callback => { + clock = due + crossedBy; + return read(callback); + }); + h.job("fresh-live", "user_memory_embedding"); h.runner.notifyWorkAvailable(); + for (let i = 0; i < 30; i++) await nextTurn(); + expect(snapshot).toHaveBeenCalled(); + expect(h.processed).toContain("fresh-live"); + if (retry) expect(claim.mock.calls.some(([id]) => id === "old-deadline")).toBe(true); + else expect(h.processed).toContain("old-deadline"); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + }); + + it("invalidates old cursors when a timing edit arrives between the change probe and its snapshot", async () => { + const h = harness(), abort = new AbortController(); + h.job("old-delayed", "user_memory_embedding", "old-delayed", true); + const backlog = blockedBacklog(h, 1); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 12; i++) await nextTurn(); + expect(h.processed).toEqual([]); + const read = h.repos.runtime.scheduling.readSnapshot.bind(h.repos.runtime.scheduling); + const snapshot = vi.spyOn(h.repos.runtime.scheduling, "readSnapshot").mockImplementationOnce(callback => { + // A new enqueue caused the change probe; a producer then edits an + // already-scanned older row before the deadline snapshot begins. + h.db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE id='old-delayed'").run(); + return read(callback); + }); + h.job("new-task", "user_memory_embedding"); h.runner.notifyWorkAvailable(); + for (let i = 0; i < 20; i++) await nextTurn(); + expect(snapshot).toHaveBeenCalled(); + expect(h.processed).toContain("old-delayed"); + expect(h.processed).toContain("new-task"); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it.each(["main", "retry"] as const)("discovers an earlier edited %s deadline while idle without another notification", async queue => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const h = harness(), abort = new AbortController(); + let clock = Date.now(); + h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + if (queue === "main") { + h.job("old-delayed", "user_memory_embedding"); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='old-delayed'").run(new Date(clock + 10_000).toISOString()); + } else h.repos.runtime.enqueueEmbeddingRetry({ id: "old-delayed", targetKind: "trace", targetId: "missing", + vectorField: "vec_summary", sourceText: "synthetic", now: clock + 10_000 }); + const backlog = blockedBacklog(h, 1); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const claim = vi.spyOn(h.repos.runtime, "claimEmbeddingRetryById"); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 12; i++) await nextTurn(); + if (queue === "main") h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='old-delayed'").run(new Date(clock + 100).toISOString()); + else h.db.db.prepare("UPDATE embedding_retry_queue SET next_attempt_at=? WHERE id='old-delayed'").run(clock + 100); + h.runner.notifyWorkAvailable(); + for (let i = 0; i < 12; i++) await nextTurn(); + expect(h.processed).toEqual([]); expect(claim).not.toHaveBeenCalled(); + clock += 100; + await vi.advanceTimersByTimeAsync(100); + for (let i = 0; i < 12; i++) await nextTurn(); + if (queue === "main") expect(h.processed).toEqual(["old-delayed"]); + else expect(claim.mock.calls.some(([id]) => id === "old-delayed")).toBe(true); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it("discovers a delayed task derived by a successful handler without a producer notification", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const h = harness(), abort = new AbortController(), backlog = blockedBacklog(h, 1); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + let clock = Date.now(); h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + h.job("parent", "user_memory_embedding"); + const process = h.deps.jobHandlers.processJob; + h.deps.jobHandlers.processJob = async job => { + await process(job); + if (job.id === "parent") { + h.job("child", "user_memory_embedding"); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='child'").run(new Date(clock + 100).toISOString()); + } + }; + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 15; i++) await nextTurn(); + expect(h.processed).toEqual(["parent"]); + clock += 100; await vi.advanceTimersByTimeAsync(100); + for (let i = 0; i < 15; i++) await nextTurn(); + expect(h.processed).toEqual(["parent", "child"]); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it.each(["available", "runAfter", "mainLease", "retryTime", "retryLease"] as const)("rechecks an old %s row edited during a deep scan", async kind => { + const h = harness(), abort = new AbortController(); + const clock = Date.now(), future = new Date(clock + 10_000).toISOString(), past = new Date(clock - 1).toISOString(); + h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + const retry = kind === "retryTime" || kind === "retryLease"; + if (retry) { + h.repos.runtime.enqueueEmbeddingRetry({ id: "old-timed", targetKind: "trace", targetId: "missing", + vectorField: "vec_summary", sourceText: "synthetic", now: clock + 10_000 }); + if (kind === "retryLease") h.db.db.prepare("UPDATE embedding_retry_queue SET status='in_progress',claimed_by='old-worker',next_attempt_at=?,lease_until=? WHERE id='old-timed'").run(clock - 1, clock + 10_000); + } else { + h.job("old-timed", "user_memory_embedding"); + if (kind === "available") h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='old-timed'").run(future); + else if (kind === "runAfter") h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='old-timed'").run(JSON.stringify({ runAfter: future })); + else h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='old-timed'").run(future); + } + const backlog = blockedBacklog(h, 2000); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const claim = vi.spyOn(h.repos.runtime, "claimEmbeddingRetryById"); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + await nextTurn(); expect(backlog.prepared()).toBeLessThan(2000); + if (kind === "available") h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='old-timed'").run(past); + else if (kind === "runAfter") h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='old-timed'").run(JSON.stringify({ runAfter: past })); + else if (kind === "mainLease") h.db.db.prepare("UPDATE evolution_jobs SET leased_until=? WHERE id='old-timed'").run(past); + else if (kind === "retryTime") h.db.db.prepare("UPDATE embedding_retry_queue SET next_attempt_at=? WHERE id='old-timed'").run(clock - 1); + else h.db.db.prepare("UPDATE embedding_retry_queue SET lease_until=? WHERE id='old-timed'").run(clock - 1); + h.runner.notifyWorkAvailable(); + for (let i = 0; i < 60; i++) await nextTurn(); + if (retry) expect(claim.mock.calls.some(([id]) => id === "old-timed")).toBe(true); + else expect(h.processed).toEqual(["old-timed"]); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it("reuses deadline state for ordinary completions and unrelated capacity wakes", async () => { + const h = harness(); h.jobs(); + const queries = vi.spyOn(h.repos.runtime, "nextWorkerRunAt"); + const process = h.deps.jobHandlers.processJob; + h.deps.jobHandlers.processJob = async job => { + const permit = h.concurrency.tryAcquire({ resourceClass: "summary", providers: [{ key: "unrelated", limit: 1 }], scopeKeys: [] }); + permit?.release(); + await process(job); + }; + const result = await h.runner.runWorkerOnce(total); + expect(result.succeeded).toBe(total); + expect(queries.mock.calls.length).toBeLessThanOrEqual(2); + h.expectClean(); + }); + + it.each([1, 10_000, 50_000])("bounds repeated candidate reads with %i blocked rows and ongoing arrivals", async count => { + const h = harness(), backlog = blockedBacklog(h, count), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const enqueued = new Map(), waits: number[] = []; + let pageCalls = 0, rowsRead = 0; + const inspections = vi.spyOn(WorkerAdmissionWaitSet.prototype, "isBlocked"); + const page = h.repos.runtime.scheduling.candidatePage.bind(h.repos.runtime.scheduling); + vi.spyOn(h.repos.runtime.scheduling, "candidatePage").mockImplementation(input => { + const rows = page(input); pageCalls++; rowsRead += rows.length; return rows; + }); + const process = h.deps.jobHandlers.processJob; + h.deps.jobHandlers.processJob = async job => { + waits.push(performance.now() - enqueued.get(job.id)!); + await new Promise(resolve => setTimeout(resolve, 1)); + await process(job); + }; + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 49; i++) { + await new Promise(resolve => setTimeout(resolve, 100)); + const id = `live-${i}`; enqueued.set(id, performance.now()); + h.job(id, "user_memory_embedding"); h.runner.notifyWorkAvailable(); + } + await new Promise(resolve => setTimeout(resolve, 2000)); + const sample = { backlog: count, added: enqueued.size, started: waits.length, notStarted: enqueued.size - waits.length, + meanWaitMs: waits.reduce((a,b) => a+b, 0) / waits.length, maxWaitMs: Math.max(...waits), pageCalls, rowsRead, candidateChecks: inspections.mock.calls.length }; + console.log("CONTINUOUS_ARRIVALS", JSON.stringify(sample)); + expect(waits).toHaveLength(49); + expect(Math.max(...waits)).toBeLessThan(1000); + expect(rowsRead).toBeLessThanOrEqual(count + 49 + 200); + expect(inspections.mock.calls.length).toBeLessThanOrEqual(count + 49 + 200); + expect(pageCalls).toBeLessThan(count / 64 * 12 + 2000); + expect(backlog.prepared()).toBeLessThanOrEqual(count + 49); + expect(backlog.prepared()).toBeGreaterThan(49); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + // A later finite request starts from durable FIFO, not the cancelled scan. + h.deps.jobHandlers.processJob = process; + const resumed = await h.runner.runWorkerOnce(Math.min(3, count)); + expect(resumed.jobs.map(job => job.jobId)).toEqual(Array.from({ length: Math.min(3, count) }, (_, i) => `blocked-${i}`)); + h.expectClean(); + }, 30_000); + + it.each(["available", "runAfter", "expiredLease"] as const)("discovers a new delayed row behind the retained cursor when %s becomes due", async boundary => { + const h = harness(), backlog = blockedBacklog(h, 100), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + let clock = Date.now(); + h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 8; i++) await nextTurn(); + h.job("new-delayed", "user_memory_embedding"); + const future = new Date(clock + 1000).toISOString(); + if (boundary === "available") h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='new-delayed'").run(future); + else if (boundary === "runAfter") h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='new-delayed'").run(JSON.stringify({ runAfter: future })); + else h.db.db.prepare("UPDATE evolution_jobs SET status='leased', leased_until=? WHERE id='new-delayed'").run(future); + h.job("new-ready", "user_memory_embedding"); h.runner.notifyWorkAvailable(); + for (let i = 0; i < 12; i++) await nextTurn(); + expect(h.processed).toEqual(["new-ready"]); + clock += 2000; h.runner.notifyWorkAvailable(); + for (let i = 0; i < 12; i++) await nextTurn(); + expect(h.processed).toEqual(["new-ready", "new-delayed"]); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it("finishes a bounded dependency sweep after one wake and then sleeps", async () => { + const h = harness(), backlog = blockedBacklog(h, 1), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const at = new Date().toISOString(); + h.repos.transaction(() => { + for (let i = 0; i < 200; i++) { + const id = `dependency-${i}`; + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,created_at,updated_at) + VALUES (?,?,'event-loop-user','synthetic','L1',?,?)`).run(id, at, at, at); + h.db.db.prepare(`INSERT INTO memory_processing_state(memory_id,state,stage,updated_at) + VALUES (?,'embedding_pending','embedding',?)`).run(id, at); + h.job(`reward-${i}`, "reward"); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id=?") + .run(JSON.stringify({ l1MemoryId: id, polarity: "positive" }), `reward-${i}`); + h.repos.runtime.scheduling.refreshOne("main", `reward-${i}`); + } + }); + const pending = vi.spyOn(h.repos.runtime.scheduling, "isPendingCandidate"); + const eligible = vi.spyOn(h.repos.runtime.scheduling, "isEligibleCandidate"); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 40; i++) await nextTurn(); + expect(h.processed).toEqual([]); + const idleChecks = pending.mock.calls.length; + for (let i = 0; i < 10; i++) await nextTurn(); + expect(pending.mock.calls.length).toBe(idleChecks); + eligible.mockClear(); h.runner.notifyWorkAvailable(); + for (let i = 0; i < 20; i++) await nextTurn(); + const target = eligible.mock.calls.filter(([candidate]) => candidate.id.startsWith("reward-")).at(-1)![0].id; + h.db.db.prepare("UPDATE memory_processing_state SET state='ready' WHERE memory_id=?") + .run(target.replace("reward-", "dependency-")); + h.runner.notifyWorkAvailable(); + for (let i = 0; i < 40; i++) await nextTurn(); + expect(h.processed).toEqual([target]); + const completedChecks = pending.mock.calls.length; + for (let i = 0; i < 10; i++) await nextTurn(); + expect(pending.mock.calls.length).toBe(completedChecks); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it.each(["available", "runAfter", "expiredLease", "failed"] as const)("retains a %s deadline crossed inside the final discovery page", async boundary => { + const h = harness(), abort = new AbortController(); + let clock = Date.now(); + h.deps.nowIso = () => new Date(clock).toISOString(); h.deps.nowMs = () => clock; + h.job("due-during-query", "trace_summary"); + const future = new Date(clock + 10).toISOString(); + if (boundary === "available" || boundary === "failed") { + h.db.db.prepare("UPDATE evolution_jobs SET available_at=?,status=? WHERE id='due-during-query'") + .run(future, boundary === "failed" ? "failed" : "queued"); + } else if (boundary === "runAfter") { + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='due-during-query'").run(JSON.stringify({ runAfter: future })); + } else h.db.db.prepare("UPDATE evolution_jobs SET status='leased',leased_until=? WHERE id='due-during-query'").run(future); + const backlog = blockedBacklog(h, 1); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + h.db.db.prepare("UPDATE evolution_jobs SET queue_lane='interactive'").run(); + const prepare = h.deps.prepareExecution!; + h.deps.prepareExecution = work => work.queue === "main" && work.job.id === "due-during-query" + ? { providers: [], run: operation => operation() } : prepare(work); + const page = h.repos.runtime.scheduling.candidatePage.bind(h.repos.runtime.scheduling); + let crossed = false; + vi.spyOn(h.repos.runtime.scheduling, "candidatePage").mockImplementation(input => { + const rows = page(input); + if (!crossed && rows.some(row => row.id === "blocked-0")) { clock += 20; crossed = true; } + return rows; + }); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 20; i++) await nextTurn(); + expect(crossed).toBe(true); + expect(h.processed).toEqual(["due-during-query"]); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it("does not reread a fully scanned backlog after idle wakeups and a normal refill", async () => { + const h = harness(), backlog = blockedBacklog(h, 1000), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const page = h.repos.runtime.scheduling.candidatePage.bind(h.repos.runtime.scheduling); + let rowsRead = 0; + vi.spyOn(h.repos.runtime.scheduling, "candidatePage").mockImplementation(input => { + const rows = page(input); rowsRead += rows.length; return rows; + }); + let ended = false; + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }).then(result => { ended = true; return result; }); + try { + for (let i = 0; i < 50; i++) await nextTurn(); + expect(backlog.prepared()).toBe(1000); expect(rowsRead).toBe(1000); + for (let i = 0; i < 10; i++) { h.runner.notifyWorkAvailable(); await nextTurn(); } + h.job("new-live", "user_memory_embedding"); h.runner.notifyWorkAvailable(); + for (let i = 0; i < 10; i++) await nextTurn(); + expect(h.processed).toEqual(["new-live"]); expect(rowsRead).toBe(1001); + // Old/raw producers classify at the next slice, without rewinding older + // resource-blocked rows merely because this new sequence had no policy. + const at = new Date().toISOString(); + h.db.db.prepare(`INSERT INTO evolution_jobs(id,job_type,status,user_id,payload_json,created_at,updated_at) + VALUES ('raw-live','user_memory_embedding','queued','event-loop-user','{}',?,?)`).run(at, at); + h.runner.notifyWorkAvailable(); + for (let i = 0; i < 10; i++) await nextTurn(); + expect(h.processed).toEqual(["new-live", "raw-live"]); expect(rowsRead).toBe(1002); + expect(ended).toBe(false); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it("keeps dependency-blocked rows parked through unrelated completions and rediscovers them after an external dependency change", async () => { + const h = harness(), backlog = blockedBacklog(h, 1), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const at = new Date().toISOString(); + h.db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,created_at,updated_at) + VALUES ('dependency',?,'event-loop-user','synthetic','L1',?,?)`).run(at, at, at); + h.db.db.prepare(`INSERT INTO memory_processing_state(memory_id,state,stage,updated_at) + VALUES ('dependency','embedding_pending','embedding',?)`).run(at); + h.repos.transaction(() => { + for (let i = 0; i < 500; i++) { + const job = h.job(`reward-${i}`, "reward"); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id=?") + .run(JSON.stringify({ l1MemoryId: "dependency", polarity: "positive" }), job.id); + h.repos.runtime.scheduling.refreshOne("main", job.id); + } + }); + let rowsRead = 0; + const page = h.repos.runtime.scheduling.candidatePage.bind(h.repos.runtime.scheduling); + vi.spyOn(h.repos.runtime.scheduling, "candidatePage").mockImplementation(input => { + const rows = page(input); rowsRead += rows.length; return rows; + }); + const process = h.deps.jobHandlers.processJob; + h.deps.jobHandlers.processJob = async job => { + await process(job); + if (job.id === "reward-2") { abort.abort(); h.runner.notifyWorkAvailable(); } + }; + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + for (let i = 0; i < 40; i++) await nextTurn(); + expect(rowsRead).toBe(501); expect(h.processed).toEqual([]); + for (let i = 0; i < 5; i++) { + h.job(`independent-${i}`, "user_memory_embedding"); h.runner.notifyWorkAvailable(); + for (let j = 0; j < 5; j++) await nextTurn(); + } + expect(h.processed).toEqual(Array.from({ length: 5 }, (_, i) => `independent-${i}`)); + expect(rowsRead).toBe(506); + // This update does not change policy revision or emit a model release. + h.db.db.prepare("UPDATE memory_processing_state SET state='ready' WHERE memory_id='dependency'").run(); + h.runner.notifyWorkAvailable(); await run; + expect(h.processed.filter(id => id.startsWith("reward-")).slice(0, 3)).toEqual(["reward-0", "reward-1", "reward-2"]); + expect(rowsRead).toBe(506); + } finally { abort.abort(); h.runner.notifyWorkAvailable(); await run; backlog.held.release(); } + h.expectClean(); + }); + + it("reaches an independent evolution Provider behind 20,000 blocked jobs during ongoing interference", async () => { + const h = harness(), backlog = blockedBacklog(h, 20_000), abort = new AbortController(); + h.deps.scheduling = { algorithm: "wrr", aging: { enabled: true, thresholdMs: 1 } }; + h.db.db.prepare("UPDATE evolution_jobs SET job_type='reflection' WHERE id='late-database'").run(); + h.repos.runtime.scheduling.refreshOne("main", "late-database"); + h.db.db.prepare("UPDATE evolution_jobs SET queue_lane='import', first_queued_at='2000-01-01T00:00:00.000Z'").run(); + const prepare = h.deps.prepareExecution!; + h.deps.prepareExecution = work => { + const execution = prepare(work); + return work.queue === "main" && work.job.id === "late-database" + ? { ...execution, providers: [{ key: "independent-evolution", limit: 1 }] } : execution; + }; + let releases = 0; + const interference = (async () => { + for (let turn = 0; turn < 400 && !abort.signal.aborted; turn++) { + await nextTurn(); + await h.concurrency.runRequest({ local: false, providers: [{ key: "noise", limit: 1 }] }, async () => {}); + releases++; + } + abort.abort(); + })(); + try { + const result = await h.runner.runWorkerOnce(1, {}, { signal: abort.signal }); + expect(result.jobs.map(job => job.jobId)).toEqual(["late-database"]); + expect(releases).toBeGreaterThan(0); + expect(releases).toBeLessThan(400); + expect(backlog.prepared()).toBe(20_001); + expect(h.db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='queued' AND attempts=0 AND lease_token IS NULL").get()).toEqual({ n: 20_000 }); + } finally { abort.abort(); await interference; backlog.held.release(); } + h.expectClean(); + }, 30_000); + + it("does not re-prepare already blocked rows after a full scan on unrelated wakeups", async () => { + const h = harness(), backlog = blockedBacklog(h, 100), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + const run = h.runner.runWorkerOnce(2, {}, { signal: abort.signal }); + try { + for (let i = 0; i < 8; i++) await nextTurn(); + expect(backlog.prepared()).toBe(100); + for (let i = 0; i < 8; i++) { + await h.concurrency.runRequest({ local: false, providers: [{ key: "noise", limit: 1 }] }, async () => {}); + await nextTurn(); + } + expect(backlog.prepared()).toBe(100); + backlog.held.release(); + expect((await run).jobs.map(job => job.jobId)).toEqual(["blocked-0", "blocked-1"]); + } finally { abort.abort(); backlog.held.release(); await run; } + h.expectClean(); + }); + + it("rechecks a parked row after its stored execution metadata changes", async () => { + const h = harness(), backlog = blockedBacklog(h), abort = new AbortController(); + const prepare = h.deps.prepareExecution!; + h.deps.prepareExecution = work => { + const execution = prepare(work); + return work.queue === "main" && work.job.payload.independent === true + ? { ...execution, providers: [] } : execution; + }; + const changed = (async () => { + await nextTurn(); + h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='blocked-0'") + .run(JSON.stringify({ independent: true })); + h.repos.runtime.scheduling.refreshOne("main", "blocked-0"); + h.db.db.prepare("UPDATE evolution_jobs SET queue_lane='import' WHERE id='blocked-0'").run(); + h.runner.notifyWorkAvailable(); + })(); + try { + const result = await h.runner.runWorkerOnce(1, {}, { signal: abort.signal }); + expect(result.jobs.map(job => job.jobId)).toEqual(["blocked-0"]); + expect(h.repos.runtime.getJob("late-database")).toMatchObject({ status: "queued", attempts: 0 }); + } finally { abort.abort(); await changed; backlog.held.release(); } + h.expectClean(); + }); + + it("does not keep a finite drain waiting on parked records that were deleted", async () => { + const h = harness(), backlog = blockedBacklog(h, 8), abort = new AbortController(); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + let finished = false; + const run = h.runner.runWorkerOnce(1, {}, { signal: abort.signal }).then(result => { finished = true; return result; }); + try { + for (let i = 0; i < 4; i++) await nextTurn(); + expect(backlog.prepared()).toBe(8); + h.db.db.prepare("DELETE FROM evolution_jobs").run(); + h.runner.notifyWorkAvailable(); + for (let i = 0; i < 4; i++) await nextTurn(); + expect(finished).toBe(true); + expect((await run).leased).toBe(0); + } finally { abort.abort(); backlog.held.release(); await run; } + h.expectClean(); + }); + + it.each([false, true])("makes progress through unrelated Provider releases (aging=%s)", async aging => { + const h = harness(), backlog = blockedBacklog(h, 2_000), abort = new AbortController(); + h.deps.scheduling = { algorithm: "wrr", aging: { enabled: aging, thresholdMs: 1 } }; + h.db.db.prepare("UPDATE evolution_jobs SET first_queued_at='2000-01-01T00:00:00.000Z'").run(); + const noise = { local: false, providers: [{ key: "unrelated-foreground", limit: 1 }] }; + let releases = 0; + const interference = (async () => { + for (let turn = 0; turn < 80 && !abort.signal.aborted; turn++) { + await nextTurn(); + await h.concurrency.runRequest(noise, async () => {}); + releases++; + } + abort.abort(); + })(); + try { + const result = await h.runner.runWorkerOnce(1, {}, { signal: abort.signal }); + expect(releases).toBeGreaterThan(0); + expect(result.jobs.map(job => job.jobId)).toEqual(["late-database"]); + expect(backlog.prepared()).toBe(backlog.count + 1); + expect(h.repos.runtime.getJob("blocked-0")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { abort.abort(); await interference; backlog.held.release(); } + h.expectClean(); + }); + + it("yields while projecting an upgraded dirty queue and admits no row before projection completes", async () => { + const h = harness(), backlog = blockedBacklog(h), abort = new AbortController(); + h.db.db.prepare("UPDATE evolution_jobs SET policy_version=0").run(); + let dirtyAtTimer = -1; + const timer = timerEvent(() => { + dirtyAtTimer = Number((h.db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE policy_version=0").get() as { n: number }).n); + abort.abort(); + }); + try { + const result = await h.runner.runWorkerOnce(1, {}, { signal: abort.signal }); + await timer; + expect(dirtyAtTimer).toBeGreaterThan(0); + expect(dirtyAtTimer).toBeGreaterThanOrEqual(backlog.count + 1 - 128); + expect(result.leased).toBe(0); + expect(backlog.prepared()).toBe(0); + expect(h.processed).toEqual([]); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("finishes bounded dirty-policy projection before selecting the runnable FIFO tail", async () => { + const h = harness(), backlog = blockedBacklog(h); + h.db.db.prepare("UPDATE evolution_jobs SET policy_version=0").run(); + let projectedAtFirstAdmission = -1; + const prepare = h.deps.prepareExecution!; + h.deps.prepareExecution = work => { + if (projectedAtFirstAdmission < 0) projectedAtFirstAdmission = Number((h.db.db.prepare( + "SELECT COUNT(*) AS n FROM evolution_jobs WHERE policy_version=0").get() as { n: number }).n); + return prepare(work); + }; + try { + const result = await h.runner.runWorkerOnce(1); + expect(projectedAtFirstAdmission).toBe(0); + expect(result.jobs.map(job => job.jobId)).toEqual(["late-database"]); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("yields while inspecting blocked candidates so a timer can abort before the runnable tail", async () => { + const h = harness(), backlog = blockedBacklog(h), abort = new AbortController(); + let atAbort = -1; + const timer = timerEvent(() => { atAbort = backlog.prepared(); abort.abort(); }); + try { + const result = await h.runner.runWorkerOnce(1, {}, { signal: abort.signal }); + await timer; + expect(atAbort).toBeGreaterThan(0); + expect(atAbort).toBeLessThanOrEqual(128); + expect(backlog.prepared()).toBe(atAbort); + expect(result.leased).toBe(0); + expect(h.processed).toEqual([]); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("continues beyond all blocked slices to a runnable database tail without starving timers", async () => { + const h = harness(), backlog = blockedBacklog(h); + let atTimer = -1; + const timer = timerEvent(() => { atTimer = backlog.prepared(); }); + try { + const result = await h.runner.runWorkerOnce(1); + await timer; + expect(atTimer).toBeGreaterThan(0); + expect(atTimer).toBeLessThanOrEqual(128); + expect(backlog.prepared()).toBe(backlog.count + 1); + expect(result.jobs.map(job => job.jobId)).toEqual(["late-database"]); + expect(h.processed).toEqual(["late-database"]); + expect(h.repos.runtime.getJob("blocked-0")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("honors stop between blocked candidate slices without claiming the runnable tail", async () => { + const h = harness(), backlog = blockedBacklog(h); + let atStop = -1, stopped: Promise | undefined; + const timer = timerEvent(() => { atStop = backlog.prepared(); stopped = h.runner.stop(); }); + try { + const result = await h.runner.runWorkerOnce(1); + await timer; await stopped; + expect(atStop).toBeGreaterThan(0); + expect(atStop).toBeLessThanOrEqual(128); + expect(backlog.prepared()).toBe(atStop); + expect(result.leased).toBe(0); + expect(h.processed).toEqual([]); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it.each(["available", "runAfter", "expiredLease"] as const)("rechecks an older FIFO row becoming due by %s between slices", async boundary => { + const h = harness(); + let clock = Date.now(); + h.deps.nowIso = () => new Date(clock).toISOString(); + h.deps.nowMs = () => clock; + const future = new Date(clock + 1000).toISOString(); + h.job("older-due"); + if (boundary === "available") h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='older-due'").run(future); + else if (boundary === "runAfter") h.db.db.prepare("UPDATE evolution_jobs SET payload_json=? WHERE id='older-due'").run(JSON.stringify({ runAfter: future })); + else h.db.db.prepare("UPDATE evolution_jobs SET status='leased', leased_until=? WHERE id='older-due'").run(future); + h.repos.runtime.scheduling.refreshOne("main", "older-due"); + const backlog = blockedBacklog(h); + const timer = timerEvent(() => { clock += 2000; }); + try { + const result = await h.runner.runWorkerOnce(1); + await timer; + expect(result.jobs.map(job => job.jobId)).toEqual(["older-due"]); + expect(h.repos.runtime.getJob("late-database")).toMatchObject({ status: "queued", attempts: 0 }); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("admits new interactive work arriving during an import continuation", async () => { + const h = harness(), backlog = blockedBacklog(h); + const timer = timerEvent(() => { h.job("new-interactive", "user_memory_embedding"); }); + try { + const result = await h.runner.runWorkerOnce(1); + await timer; + expect(result.jobs.map(job => job.jobId)).toEqual(["new-interactive"]); + expect(h.repos.runtime.getJob("late-database")).toMatchObject({ status: "queued", attempts: 0 }); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("refreshes a changed model snapshot between slices while admitted work keeps its own snapshot", async () => { + const h = harness(); h.job("first-active", "trace_summary"); + const backlog = blockedBacklog(h); + let configuration = "old", releaseFirst!: () => void; + const firstGate = new Promise(resolve => { releaseFirst = resolve; }); + const executions: string[] = []; + h.deps.configurationFingerprint = () => configuration; + h.deps.prepareExecutionBatch = () => { + const snapshot = configuration; + return work => ({ + providers: work.queue === "main" && work.job.id === "first-active" + ? [{ key: "first-provider", limit: 1 }] + : work.queue === "main" && work.job.jobType === "trace_summary" + ? [{ key: snapshot === "old" ? "blocked-local" : "new-provider", limit: 1 }] : [], + run: operation => { executions.push(snapshot); return operation(); } + }); + }; + const process = h.deps.jobHandlers.processJob; + h.deps.jobHandlers.processJob = async job => { + if (job.id === "first-active") await firstGate; + else releaseFirst(); + await process(job); + }; + const timer = timerEvent(() => { configuration = "new"; h.runner.notifyWorkAvailable(); }); + try { + const result = await h.runner.runWorkerOnce(2); + await timer; + expect(result.jobs.map(job => job.jobId)).toEqual(["first-active", "blocked-0"]); + expect(executions).toEqual(["old", "new"]); + } finally { releaseFirst(); backlog.held.release(); } + h.expectClean(); + }); + + it.each([64, 128])("retains blocked admission state after exactly %s candidates and waits for a released permit", async count => { + const h = harness(), backlog = blockedBacklog(h, count); + h.db.db.prepare("DELETE FROM evolution_jobs WHERE id='late-database'").run(); + let released = false; + const release = (async () => { + for (let turn = 0; turn < 4; turn++) await nextTurn(); + released = true; backlog.held.release(); + })(); + try { + const result = await h.runner.runWorkerOnce(1); + expect(released).toBe(true); + expect(result.jobs.map(job => job.jobId)).toEqual(["blocked-0"]); + } finally { await release; backlog.held.release(); } + h.expectClean(); + }); + + it("rechecks skipped FIFO heads when a held Provider is released between slices", async () => { + const h = harness(), backlog = blockedBacklog(h); + const timer = timerEvent(() => backlog.held.release()); + const result = await h.runner.runWorkerOnce(1); + await timer; + expect(result.jobs.map(job => job.jobId)).toEqual(["blocked-0"]); + expect(h.repos.runtime.getJob("late-database")).toMatchObject({ status: "queued", attempts: 0 }); + h.expectClean(); + }); + + it("starts work admitted in a partial slice before yielding to a timer", async () => { + const h = harness(); h.job("early-database"); + const backlog = blockedBacklog(h), abort = new AbortController(); + let processedAtTimer: string[] = []; + const timer = timerEvent(() => { processedAtTimer = [...h.processed]; abort.abort(); }); + try { + const result = await h.runner.runWorkerOnce(4, {}, { signal: abort.signal }); + await timer; + expect(processedAtTimer).toEqual(["early-database"]); + expect(result.jobs.map(job => job.jobId)).toEqual(["early-database"]); + expect(h.processed).toEqual(processedAtTimer); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("hands a blocked automatic scan to a manual request arriving between slices", async () => { + const h = harness(), backlog = blockedBacklog(h); + h.job("manual", "l2_association", "manual-target", true); + let manual: ReturnType | undefined; + const timer = timerEvent(() => { + h.db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE id='manual'").run(); + manual = h.runner.runWorkerOnce(1, { targetMemoryIds: ["manual-target"] }); + }); + try { + const automatic = await h.runner.runWorkerOnce(4, {}, { continuous: true }); + await timer; + const result = await manual!; + expect(automatic.leased).toBe(0); + expect(result.jobs.map(job => job.jobId)).toEqual(["manual"]); + expect(h.processed).toEqual(["manual"]); + } finally { backlog.held.release(); } + h.expectClean(); + }); + + it("lets an already scheduled zero-delay abort run before synchronous database work is drained", async () => { + const h = harness(), abort = new AbortController(); h.jobs(); + let atAbort = -1; + const event = timerEvent(() => { atAbort = h.processed.length; abort.abort(); }); + const run = await h.runner.runWorkerOnce(4, {}, { continuous: true, signal: abort.signal }); + await event; + expect(atAbort).toBeGreaterThan(0); + // Allow a timers/check phase turn; do not assert an exact refill boundary. + expect(atAbort).toBeLessThanOrEqual(64); + expect(h.processed).toHaveLength(atAbort); + expect(run.leased).toBe(atAbort); + expect(run.failed).toBe(0); h.expectClean(); + }); + + it("admits a manual request posted from a timer before exhausting the automatic backlog", async () => { + const h = harness(); h.jobs(); h.job("manual", "l2_association", "manual-target", true); + let beforeManual = -1, manual: ReturnType | undefined; + const event = timerEvent(() => { + beforeManual = h.processed.length; + h.db.db.prepare("UPDATE evolution_jobs SET available_at=NULL WHERE id='manual'").run(); + manual = h.runner.runWorkerOnce(1, { targetMemoryIds: ["manual-target"] }); + }); + const automatic = await h.runner.runWorkerOnce(4, {}, { continuous: true }); + await event; const result = await manual!; + expect(beforeManual).toBeLessThanOrEqual(64); + expect(automatic.leased).toBe(beforeManual); + expect(result.jobs.map(job => job.jobId)).toEqual(["manual"]); + expect(h.processed.slice(-1)).toEqual(["manual"]); + expect(h.processed).toHaveLength(beforeManual + 1); h.expectClean(); + }); + + it("honors stop issued from a timer and leaves unadmitted records untouched", async () => { + const h = harness(); h.jobs(); + let atStop = -1, stopped: Promise | undefined; + const event = timerEvent(() => { atStop = h.processed.length; stopped = h.runner.stop(); }); + const result = await h.runner.runWorkerOnce(4, {}, { continuous: true }); + await event; await stopped; + expect(atStop).toBeLessThanOrEqual(64); + expect(h.processed).toHaveLength(atStop); + expect(result.leased).toBe(atStop); + await expect(h.runner.runWorkerOnce(1)).rejects.toThrow(/stopped/i); + h.expectClean(); + }); + + it("retains the global and actual Provider request limits across macro-task turns", async () => { + const h = harness(); h.jobs("trace_summary"); + const descriptors: WorkerProviderDescriptor[] = ["provider-a", "provider-b"].map(key => ({ + local: false, providers: [{ key, limit: 2 }, { key: `${key}:model`, limit: 2 }] + })); + const descriptorFor = (id: string) => descriptors[Number(id.split("-")[1]) % 2]!; + h.deps.prepareExecution = work => ({ + providers: descriptorFor(work.queue === "main" ? work.job.id : work.retry.id).providers, + run: operation => operation() + }); + const process = h.deps.jobHandlers.processJob; + const activeByProvider = [0, 0], peakByProvider = [0, 0]; + let activeRequests = 0, peakRequests = 0, atTimer = -1; + h.deps.jobHandlers.processJob = async job => { + const index = Number(job.id.split("-")[1]) % 2; + await h.concurrency.runRequest(descriptors[index]!, async () => { + peakRequests = Math.max(peakRequests, ++activeRequests); + peakByProvider[index] = Math.max(peakByProvider[index]!, ++activeByProvider[index]!); + try { await Promise.resolve(); await process(job); } + finally { activeRequests--; activeByProvider[index]!--; } + }); + }; + const event = timerEvent(() => { atTimer = h.processed.length; }); + const result = await h.runner.runWorkerOnce(4, {}, { continuous: true }); + await event; + expect(atTimer).toBeGreaterThan(0); expect(atTimer).toBeLessThanOrEqual(64); + expect(result).toMatchObject({ leased: total, succeeded: total, failed: 0 }); + expect(h.processed).toHaveLength(total); + // Both Providers can run concurrently, while neither exceeds its own cap. + expect(peakRequests).toBe(4); expect(peakByProvider).toEqual([2, 2]); + expect(h.peakJobs()).toBe(4); + expect(activeRequests).toBe(0); expect(activeByProvider).toEqual([0, 0]); h.expectClean(); + }); + + it("clears deadline and heartbeat timers when cancellation arrives during a macro-task turn", async () => { + // Keep native immediates live: the production yield must not need a test + // clock advance. Track only the owned deadline/heartbeat timer handles. + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval"] }); + const schedule = vi.spyOn(globalThis, "setTimeout"); + const h = harness(1), abort = new AbortController(); h.jobs(); h.job("future", "l2_association", "future", true); + let atAbort = -1; + const event = nextTurn().then(() => { atAbort = h.processed.length; abort.abort(); }); + const run = await h.runner.runWorkerOnce(4, {}, { continuous: true, signal: abort.signal }); + await event; + expect(atAbort).toBeGreaterThan(0); expect(atAbort).toBeLessThanOrEqual(64); + expect(h.processed).toHaveLength(atAbort); expect(run.leased).toBe(atAbort); + expect(schedule.mock.calls.some(([, delay]) => Number(delay) > 1000)).toBe(true); + expect(vi.getTimerCount()).toBe(0); h.expectClean(); + }); + + it("clears timer handles and permits if admission throws after the event loop gets a turn", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval"] }); + const schedule = vi.spyOn(globalThis, "setTimeout"); + const h = harness(1); h.jobs(); h.job("future", "l2_association", "future", true); + let failAdmission = false, completedBeforeFailure = -1; + h.deps.prepareExecution = () => { + if (failAdmission) throw new Error("admission failed after macro-task turn"); + return { providers: [], run: operation => operation() }; + }; + const event = nextTurn().then(() => { completedBeforeFailure = h.processed.length; failAdmission = true; }); + const rejected = expect(h.runner.runWorkerOnce(4, {}, { continuous: true })) + .rejects.toThrow("admission failed after macro-task turn"); + await Promise.all([event, rejected]); + expect(completedBeforeFailure).toBeGreaterThan(0); expect(completedBeforeFailure).toBeLessThanOrEqual(64); + expect(h.processed).toHaveLength(completedBeforeFailure); + expect(schedule.mock.calls.some(([, delay]) => Number(delay) > 1000)).toBe(true); + expect(vi.getTimerCount()).toBe(0); h.expectClean(); + }); +}); diff --git a/Memory/tests/service/worker/worker-fair-scheduling.test.ts b/Memory/tests/service/worker/worker-fair-scheduling.test.ts new file mode 100644 index 000000000..b5136ca58 --- /dev/null +++ b/Memory/tests/service/worker/worker-fair-scheduling.test.ts @@ -0,0 +1,538 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { QUEUE_LANES, type QueueLane } from "../../../src/service/worker/job-policy-registry.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const epoch = Date.parse("2026-10-22T00:00:00.000Z"); +const at = new Date(epoch).toISOString(); +const resources: Array<{ root: string; db: MemoryDb }> = []; +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} +type HarnessOptions = { concurrency?: WorkerConcurrencyController; algorithm?: "wrr" | "legacy"; process?: (job: EvolutionJobRecord) => Promise }; + +/** Real SQLite and claim transitions; business/model work is a local substitute. */ +function harness(options: HarnessOptions = {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-fairness-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const executed: string[] = []; + const deps: WorkerRunnerDeps = { + repos, embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "fairness-test", memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), summaryModelConfigured: () => true, + schedulingShadow: { enabled: false }, concurrency: options.concurrency, + ...(options.algorithm ? { scheduling: { algorithm: options.algorithm } } : {}), + nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "synthetic", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement retry"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async (job) => { executed.push(job.id); await options.process?.(job); } }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding job"); }, + applyEmbeddingVector: () => {}, enqueueEmbeddingRetryAfterFailure: () => false, + persistEmbeddingVector: (input) => { + const memory = repos.memories.get(input.memoryId)!; + executed.push(`retry:${input.memoryId}`); + input.finalize?.(memory, false, new Date().toISOString()); + return memory; + } + } + }; + const runner = new WorkerRunner(deps); + function memory(id: string, createdAt = at, tags: string[] = ["agent-source"]) { + db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,created_at,updated_at) + VALUES (?,?,'user','synthetic','L1',?,?,?)`).run(id, createdAt, JSON.stringify(tags), createdAt, createdAt); + } + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection", input: { + target?: string; source?: string; createdAt?: string; payload?: Record; + scopeKey?: string; scopeSeq?: number; + } = {}) { + return repos.runtime.enqueueJob({ id, jobType, userId: "user", targetMemoryId: input.target, + scopeKey: input.scopeKey, scopeSeq: input.scopeSeq, + payload: { ...input.payload, ...(input.source ? { source: input.source } : {}) }, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: input.createdAt ?? at, updatedAt: at }); + } + function retry(id: string, createdAt = at) { + memory(id, createdAt); + return repos.runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, + vectorField: "vec_summary", sourceText: "synthetic", now: epoch }); + } + return { root, db, repos, deps, runner, job, memory, retry, executed }; +} +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + vi.stubEnv("MEMMY_WORKER_SCHEDULER_ALGORITHM", "wrr"); +}); +afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("production WRR and runnable FIFO", () => { + it("keeps global lane opportunities when targeted maintenance drains interleave", async () => { + // Keep all four selected jobs admissible: this case isolates cursor sharing, + // while the dedicated reservation tests exercise background saturation. + const h = harness({ concurrency: new WorkerConcurrencyController({ maxActiveJobs: 5 }, {}) }); + for (let i = 0; i < 30; i++) { + h.job(`native-${i}`, "trace_summary"); h.job(`repair-${i}`, "decision_repair"); + h.job(`import-${i}`, "import_summary"); h.job(`evolution-${i}`); + h.job(`maintenance-${i}`, "project_environment_profile"); + } + for (let i = 0; i < 12; i++) h.job(`targeted-${i}`, "import_summary", { + target: `manual-${i}`, source: "startup.placeholder_summary_repair" + }); + const counts = Object.fromEntries(QUEUE_LANES.map(lane => [lane, 0])) as Record; + for (let round = 0; round < 12; round++) { + const auto = await h.runner.runWorkerOnce(4); + for (const ref of auto.jobs) counts[h.repos.runtime.getJob(ref.jobId)!.queueLane as QueueLane]++; + const targeted = await h.runner.runWorkerOnce(1, { targetMemoryIds: [`manual-${round}`] }); + expect(targeted.jobs.map(ref => ref.jobId)).toEqual([`targeted-${round}`]); + } + expect(counts).toEqual({ interactive: 16, repair_retry: 12, import: 8, evolution: 8, maintenance: 4 }); + }); + + it("preserves a targeted lane round without consuming the global cursor", async () => { + const h = harness(); + for (let i = 0; i < 8; i++) { + h.job(`global-${i}`, "trace_summary"); + h.job(`target-native-${i}`, "trace_summary", { target: "target" }); + h.job(`target-evolution-${i}`, "reflection", { target: "target" }); + } + await h.runner.runWorkerOnce(2); + const globalCursor = h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")!.value; + const selected: string[] = []; + for (let i = 0; i < 6; i++) { + selected.push((await h.runner.runWorkerOnce(1, { targetMemoryIds: ["target"] })).jobs[0]!.jobId); + expect(h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")!.value).toEqual(globalCursor); + } + expect(selected.filter(id => id.startsWith("target-native"))).toHaveLength(4); + expect(selected.filter(id => id.startsWith("target-evolution"))).toHaveLength(2); + }); + + it("does not let a public retry-only drain consume global repair opportunities", async () => { + const h = harness(); + for (let i = 0; i < 8; i++) h.job(`native-${i}`, "trace_summary"); + h.job("manual-repair", "decision_repair"); + h.retry("retry-first"); h.retry("retry-second"); + await h.runner.runWorkerOnce(4); + const globalCursor = h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")!.value; + expect((await h.runner.runEmbeddingRetryOnce(1)).leased).toBe(1); + expect(h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")!.value).toEqual(globalCursor); + expect((await h.runner.runWorkerOnce(1)).jobs[0]?.jobId).toBe("manual-repair"); + }); + + it("retries the older due failure before newer failures in the same lane", async () => { + const h = harness(); + h.memory("older", "2026-10-20T00:00:00.000Z"); + h.memory("newer", "2026-10-21T00:00:00.000Z"); + const older = h.job("older-job", "import_summary", { target: "older" }); + h.job("newer-job", "import_summary", { target: "newer" }); + h.db.db.prepare("UPDATE evolution_jobs SET status='failed',failure_count=1,attempts=1,available_at=?").run(at); + const result = await h.runner.runWorkerOnce(1); + expect(result.jobs.map(job => job.jobId)).toEqual(["older-job"]); + expect(h.repos.runtime.getJob("older-job")).toMatchObject({ status: "succeeded", enqueueSeq: older.enqueueSeq, + failureCount: 1, attempts: 2 }); + expect(h.repos.runtime.getJob("newer-job")).toMatchObject({ status: "failed", failureCount: 1, attempts: 1 }); + }); + + it("gives every lane due-failure opportunities before old-priority recovery can hide them", async () => { + const h = harness(); + for (let i = 0; i < 16; i++) h.job(`native-${i}`, "trace_summary"); + h.job("repair", "decision_repair"); h.job("import", "import_summary"); + h.job("evolution"); h.job("maintenance", "project_environment_profile"); + h.db.db.prepare("UPDATE evolution_jobs SET status='failed',failure_count=1,attempts=1,available_at=?").run(at); + for (let i = 0; i < 12; i++) await h.runner.runWorkerOnce(1); + for (const id of ["repair", "import", "evolution", "maintenance"]) { + expect(h.repos.runtime.getJob(id)).toMatchObject({ status: "succeeded", failureCount: 1, attempts: 2 }); + } + }); + + it("gives retry, reflection and maintenance real execution under ten thousand imported summaries", async () => { + const h = harness(); + h.repos.transaction(() => { + for (let i = 0; i < 10_000; i++) h.job(`import-${i}`, "import_summary"); + h.job("native", "trace_summary"); h.job("reflection"); + h.job("maintenance", "import_summary", { source: "startup.placeholder_summary_repair" }); + h.retry("repair"); + }); + for (let round = 0; round < 3; round++) await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + for (const id of ["native", "reflection", "maintenance"]) { + expect(h.repos.runtime.getJob(id)).toMatchObject({ status: "succeeded", failureCount: 0 }); + } + expect(h.repos.runtime.getEmbeddingRetry("repair")).toMatchObject({ status: "succeeded" }); + expect(h.executed.some(id => id.startsWith("import-"))).toBe(true); + expect(h.repos.runtime.getJob("import-9999")).toMatchObject({ status: "queued", attempts: 0 }); + }, 15_000); + + it("preserves a 4/3/2/2/1 complete round across twelve one-item drains", async () => { + const h = harness(), lanes = new Map(); + for (let i = 0; i < 8; i++) { + for (const [lane, jobType, source] of [ + ["interactive", "trace_summary", undefined], ["repair_retry", "decision_repair", undefined], + ["import", "import_summary", undefined], ["evolution", "reflection", undefined], + ["maintenance", "import_summary", "startup.placeholder_summary_repair"] + ] as const) { + const id = `${lane}-${i}`; h.job(id, jobType, { source }); lanes.set(id, lane); + } + } + for (let i = 0; i < 12; i++) await h.runner.runWorkerOnce(1); + const totals = Object.fromEntries(QUEUE_LANES.map(lane => [lane, 0])) as Record; + for (const id of h.executed) totals[lanes.get(id)!]++; + expect(totals).toEqual({ interactive: 4, repair_retry: 3, import: 2, evolution: 2, maintenance: 1 }); + expect(h.executed).toHaveLength(12); + }); + + it("continues the weighted round after the database and runner reopen", async () => { + const h = harness(); + for (let i = 0; i < 5; i++) h.job(`native-${i}`, "trace_summary"); + h.job("repair", "decision_repair"); h.job("import", "import_summary"); + h.job("evolution"); h.job("maintenance", "project_environment_profile"); + for (let i = 0; i < 4; i++) await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["native-0", "native-1", "native-2", "native-3"]); + const path = h.db.path; + h.db.close(); + const reopened = new MemoryDb({ path }); + resources.find(resource => resource.root === h.root)!.db = reopened; + h.deps.repos = new Repositories(reopened.db); + const nextRunner = new WorkerRunner(h.deps); + expect((await nextRunner.runWorkerOnce(1)).jobs[0]?.jobId).toBe("repair"); + }); + + it("rolls back claims and weighted position when the claim transaction fails", async () => { + const h = harness(); + for (let i = 0; i < 4; i++) h.job(`native-${i}`, "trace_summary"); + h.job("repair", "decision_repair"); + const runtime = h.repos.runtime; + const original = runtime.claimQueuedJob.bind(runtime); + let claims = 0; + const failure = vi.spyOn(runtime, "claimQueuedJob").mockImplementation((...args) => { + if (++claims === 2) throw new Error("controlled claim transaction failure"); + return original(...args); + }); + await expect(h.runner.runWorkerOnce(4)).rejects.toThrow("controlled claim transaction failure"); + expect(h.repos.runtime.getJob("native-0")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + expect(h.executed).toEqual([]); + failure.mockRestore(); + expect((await h.runner.runWorkerOnce(4)).jobs.map(job => job.jobId)).toEqual(["native-0", "native-1", "native-2", "native-3"]); + expect(h.repos.runtime.getJob("repair")!.status).toBe("queued"); + }); + + it("uses explicitly configured positive lane weights", async () => { + vi.stubEnv("MEMMY_WORKER_LANE_WEIGHTS", JSON.stringify({ interactive: 1, repair_retry: 1, import: 1, evolution: 1, maintenance: 1 })); + const h = harness(); + for (let i = 0; i < 3; i++) { + h.job(`native-${i}`, "trace_summary"); h.job(`repair-${i}`, "decision_repair"); + h.job(`import-${i}`, "import_summary"); h.job(`evolution-${i}`); + h.job(`maintenance-${i}`, "project_environment_profile"); + } + for (let i = 0; i < 5; i++) await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["native-0", "repair-0", "import-0", "evolution-0", "maintenance-0"]); + }); + + it("does not let newer memory creation times overtake an older queued import", async () => { + const h = harness(); h.memory("old"); h.job("old-job", "import_summary", { target: "old" }); + for (let i = 0; i < 5; i++) { + h.memory(`new-${i}`, new Date(epoch + (i + 1) * 60_000).toISOString()); + h.job(`new-job-${i}`, "import_summary", { target: `new-${i}` }); + } + await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["old-job"]); + expect(h.repos.runtime.getJob("new-job-4")!.status).toBe("queued"); + }); + + it("uses retry enqueue order instead of target recency", async () => { + const h = harness(); h.retry("old-retry"); h.retry("new-retry", new Date(epoch + 60_000).toISOString()); + expect((await h.runner.runWorkerOnce(1)).embeddingRetries).toMatchObject({ leased: 1, succeeded: 1 }); + expect(h.executed).toEqual(["retry:old-retry"]); + expect(h.repos.runtime.getEmbeddingRetry("new-retry")!.status).toBe("pending"); + }); + + it("starts explicit repair jobs and embedding retry requests in the same lane FIFO", async () => { + const started: string[] = []; + const h = harness({ process: async job => { started.push(job.id); } }); + h.deps.embedder.embed = async texts => { started.push("retry:older-retry"); return texts.map(() => [1, 0, 0]); }; + h.retry("older-retry"); h.job("manual-summary", "trace_summary", { source: "memory.processing.manual_retry" }); + await h.runner.runWorkerOnce(2); + expect(started).toEqual(["retry:older-retry", "manual-summary"]); + expect(h.executed).toEqual(expect.arrayContaining(["retry:older-retry", "manual-summary"])); + }); + it("fills a request across priority cohorts instead of throwing away three runnable slots", async () => { + const h = harness(); h.job("summary", "trace_summary"); + for (let i = 0; i < 3; i++) h.job(`reflection-${i}`); + const result = await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + expect(result).toMatchObject({ leased: 4, succeeded: 4 }); + expect(h.executed).toHaveLength(4); + }); + + it("skips delayed and configuration-blocked heads, then restores the original FIFO position", async () => { + const h = harness(); + const original = h.job("backoff", "import_summary"); + h.job("scheduled", "import_summary", { payload: { runAfter: new Date(epoch + 60_000).toISOString() } }); + h.job("config", "import_summary"); h.job("younger", "import_summary"); + h.db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE id='backoff'").run(new Date(epoch + 60_000).toISOString()); + h.db.db.exec("UPDATE evolution_jobs SET blocked_reason='configuration' WHERE id='config'"); + await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["younger"]); + h.job("newest", "import_summary"); + vi.setSystemTime(epoch + 60_000); + await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["younger", "backoff"]); + expect(h.repos.runtime.getJob("backoff")!.enqueueSeq).toBe(original.enqueueSeq); + expect(h.repos.runtime.getJob("config")!.attempts).toBe(0); + }); + + it("skips reward dependencies and blocked scope positions without blocking another scope", async () => { + const h = harness(); h.memory("parent", at, []); + h.db.db.prepare(`INSERT INTO memory_processing_state(memory_id,state,stage,updated_at) + VALUES ('parent','embedding_pending','embedding',?)`).run(at); + h.job("blocked-reward", "reward", { payload: { l1MemoryId: "parent", polarity: "positive" } }); + h.job("scope-first", "work_memory_extract", { scopeKey: "scope-A", scopeSeq: 1, payload: { runAfter: new Date(epoch + 60_000).toISOString() } }); + h.job("scope-next", "work_memory_extract", { scopeKey: "scope-A", scopeSeq: 2 }); + h.job("other-scope", "work_memory_extract", { scopeKey: "scope-B", scopeSeq: 1 }); + h.job("free-reflection"); + await h.runner.runWorkerOnce(4); + expect(h.executed).toEqual(["other-scope", "free-reflection"]); + for (const id of ["blocked-reward", "scope-first", "scope-next"]) expect(h.repos.runtime.getJob(id)!.attempts).toBe(0); + h.db.db.exec("UPDATE memory_processing_state SET state='ready',stage=NULL WHERE memory_id='parent'"); + await h.runner.runWorkerOnce(1); + expect(h.executed.at(-1)).toBe("blocked-reward"); + }); + + it("keeps first reports in interactive while respecting earlier runnable work in that lane", async () => { + const h = harness(); h.job("earlier-native", "trace_summary"); + h.memory("first-report", at, ["first-encounter-report"]); + h.job("first-report-job", "import_summary", { target: "first-report" }); + for (let i = 0; i < 20; i++) h.job(`history-${i}`, "import_summary"); + await h.runner.runWorkerOnce(2); + expect(h.executed).toEqual(["earlier-native", "first-report-job"]); + expect(h.repos.runtime.getJob("first-report-job")).toMatchObject({ queueLane: "interactive", status: "succeeded" }); + }); + + it("continues weighted opportunities after a temporary failure instead of retrying it immediately", async () => { + const h = harness({ process: async job => { if (job.id === "failing-native") throw new Error("temporary request timeout"); } }); + h.job("failing-native", "trace_summary"); + for (let i = 0; i < 15; i++) h.job(`native-${i}`, "trace_summary"); + h.job("reflection"); h.retry("retry"); h.job("maintenance", "project_environment_profile"); + for (let i = 0; i < 12; i++) await h.runner.runWorkerOnce(1); + expect(h.repos.runtime.getJob("failing-native")).toMatchObject({ status: "failed", failureCount: 1, attempts: 1 }); + expect(h.repos.runtime.getJob("reflection")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("maintenance")!.status).toBe("succeeded"); + expect(h.repos.runtime.getEmbeddingRetry("retry")!.status).toBe("succeeded"); + }); + + it("refreshes a dirty active backlog before an already classified younger task can overtake it", async () => { + const h = harness(); + h.repos.transaction(() => { + const insert = h.db.db.prepare(`INSERT INTO evolution_jobs + (id,job_type,status,user_id,payload_json,attempts,max_attempts,created_at,updated_at) + VALUES (?,?,'queued','user','{}',0,3,?,?)`); + for (let i = 0; i < 1_000; i++) insert.run(`dirty-maintenance-${i}`, "project_environment_profile", at, at); + insert.run("dirty-older-import", "import_summary", at, at); + h.job("classified-younger-import", "import_summary"); + }); + expect(h.repos.runtime.getJob("dirty-older-import")!.policyVersion).toBeNull(); + expect(h.repos.runtime.getJob("classified-younger-import")!.policyVersion).not.toBeNull(); + await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["dirty-older-import"]); + expect(h.repos.runtime.getJob("classified-younger-import")).toMatchObject({ status: "queued", attempts: 0 }); + }); + + it("quarantines an unknown task without invoking its business handler or hiding valid work", async () => { + const h = harness(); + h.db.db.prepare(`INSERT INTO evolution_jobs + (id,job_type,status,user_id,payload_json,attempts,max_attempts,created_at,updated_at) + VALUES ('unsupported','future_unregistered_job','queued','user','{}',0,3,?,?)`).run(at, at); + h.job("native", "trace_summary"); h.job("reflection"); + await h.runner.runWorkerOnce(4); + expect(h.executed).toEqual(["native", "reflection"]); + expect(h.repos.runtime.getJob("unsupported")).toMatchObject({ status: "dead_letter" }); + expect(h.repos.runtime.getJob("native")!.status).toBe("succeeded"); + expect(h.repos.runtime.getJob("reflection")!.status).toBe("succeeded"); + expect(h.runner.nextWorkerRunAt()).toBeUndefined(); + }); + + it("keeps a capacity-blocked retry unclaimed while the active main task renews its lease", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const entered = deferred(), gate = deferred(); + const h = harness({ concurrency: new WorkerConcurrencyController({ maxActiveJobs: 1 }, {}), + process: async () => { entered.resolve(); await gate.promise; } }); + h.job("slow-summary", "trace_summary"); const original = h.retry("waiting-retry"); + const run = h.runner.runWorkerOnce(2); + await entered.promise; + const initialMain = h.repos.runtime.getJob("slow-summary")!; + try { + expect(h.repos.runtime.getEmbeddingRetry(original.id)).toMatchObject({ status: "pending", leaseToken: null }); + await vi.advanceTimersByTimeAsync(310_000); + expect(h.repos.runtime.getEmbeddingRetry(original.id)).toMatchObject({ status: "pending", attempts: 0, + enqueueSeq: original.enqueueSeq, claimedBy: null, leaseToken: null, leaseUntil: null }); + const activeMain = h.repos.runtime.getJob("slow-summary")!; + expect(activeMain).toMatchObject({ status: "leased", leaseToken: initialMain.leaseToken }); + expect(Date.parse(activeMain.leasedUntil!)).toBeGreaterThan(Date.now()); + expect(h.executed).toEqual(["slow-summary"]); + } finally { gate.resolve(); } + expect((await run).embeddingRetries).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.repos.runtime.getEmbeddingRetry(original.id)!.status).toBe("succeeded"); + expect(vi.getTimerCount()).toBe(0); + }); + it("leaves a waiting retry unclaimed when an active task pauses the budget, preserving its FIFO position", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const entered = deferred(), gate = deferred(); + let paused = false; + const h = harness({ concurrency: new WorkerConcurrencyController({ maxActiveJobs: 1 }, {}), + process: async () => { entered.resolve(); await gate.promise; paused = true; } }); + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = type => type === "embedding"; + h.job("slow-summary", "trace_summary"); const original = h.retry("budget-held-retry"); + const run = h.runner.runWorkerOnce(2); + await entered.promise; + try { + await vi.advanceTimersByTimeAsync(310_000); + expect(h.repos.runtime.getEmbeddingRetry(original.id)).toMatchObject({ status: "pending", leaseUntil: null }); + } finally { gate.resolve(); } + expect((await run).embeddingRetries).toMatchObject({ leased: 0, succeeded: 0, failed: 0 }); + expect(h.executed).toEqual(["slow-summary"]); + expect(h.repos.runtime.getEmbeddingRetry(original.id)).toMatchObject({ + status: "pending", attempts: 0, enqueueSeq: original.enqueueSeq, + nextAttemptAt: original.nextAttemptAt, leaseToken: null, claimedBy: null, leaseUntil: null + }); + expect((await h.runner.runWorkerOnce(1)).embeddingRetries.leased).toBe(0); + paused = false; + expect((await h.runner.runWorkerOnce(1)).embeddingRetries).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(vi.getTimerCount()).toBe(0); + }); + it("keeps target-limited requests from consuming jobs in another target", async () => { + const h = harness(); h.job("allowed", "import_summary", { target: "A" }); h.job("outside", "trace_summary", { target: "B" }); + h.retry("B"); + const result = await h.runner.runWorkerOnce(4, { targetMemoryIds: ["A"] }); + expect(result).toMatchObject({ leased: 1, succeeded: 1, embeddingRetries: { leased: 0 } }); + expect(h.executed).toEqual(["allowed"]); + expect(h.repos.runtime.getJob("outside")!.attempts).toBe(0); + expect(h.repos.runtime.getEmbeddingRetry("B")!.status).toBe("pending"); + }); +}); + +describe("legacy scheduling rollback", () => { + it("retains legacy SQL priority and spare retry admission while continuous refill uses remaining capacity", async () => { + vi.stubEnv("MEMMY_WORKER_SCHEDULER_ALGORITHM", "legacy"); + const h = harness(); h.job("one-summary", "trace_summary"); + for (let i = 0; i < 4; i++) h.job(`import-${i}`, "import_summary"); + h.job("reflection"); h.retry("retry"); + expect(h.repos.runtime.previewLegacyJobPage({ at, limit: 10 }).map(row => row.job.id)) + .toEqual(["one-summary", "import-0", "import-1", "import-2", "import-3", "reflection"]); + const first = await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + expect(first).toMatchObject({ leased: 3, succeeded: 3, embeddingRetries: { leased: 1 } }); + expect(first.jobs.map(job => job.jobId)).toEqual(["one-summary", "import-0", "import-1"]); + expect(h.repos.runtime.getJob("reflection")!.attempts).toBe(0); + const second = await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + expect(second).toMatchObject({ leased: 3, succeeded: 3, embeddingRetries: { leased: 0 } }); + expect(second.jobs.map(job => job.jobId)).toEqual(["import-2", "import-3", "reflection"]); + }); + it("keeps the old memory-recency order when legacy is explicitly selected", async () => { + const h = harness({ algorithm: "legacy" }); + h.memory("older"); h.job("older-job", "import_summary", { target: "older" }); + h.memory("newer", new Date(epoch + 60_000).toISOString()); h.job("newer-job", "import_summary", { target: "newer" }); + await h.runner.runWorkerOnce(1); + expect(h.executed).toEqual(["newer-job"]); + }); +}); + + +describe("scheduler configuration validation", () => { + it("fails construction explicitly for an unsupported algorithm", () => { + vi.stubEnv("MEMMY_WORKER_SCHEDULER_ALGORITHM", "unsupported-algorithm"); + expect(() => harness()).toThrow(RangeError); + }); + + it("fails construction explicitly for malformed lane-weight JSON", () => { + vi.stubEnv("MEMMY_WORKER_LANE_WEIGHTS", "{broken-json"); + expect(() => harness()).toThrow(TypeError); + }); + + it("fails construction explicitly for a missing or non-positive lane weight", () => { + vi.stubEnv("MEMMY_WORKER_LANE_WEIGHTS", JSON.stringify({ interactive: 4, repair_retry: 3, import: 2, evolution: 2 })); + expect(() => harness()).toThrow(RangeError); + vi.stubEnv("MEMMY_WORKER_LANE_WEIGHTS", JSON.stringify({ interactive: 4, repair_retry: 3, import: 2, evolution: 2, maintenance: 0 })); + expect(() => harness()).toThrow(RangeError); + }); +}); + +function embeddingRoleHarness(algorithm: "wrr" | "legacy", roles: Array<"document" | "query">) { + const h = harness({ algorithm }); + const calls: Array<{ role: "document" | "query" | undefined; texts: string[] }> = []; + const applied: string[] = []; + h.deps.embeddingJobs.prepareEmbeddingJob = (job) => ({ + job, memory: h.repos.memories.get(job.targetMemoryId!)!, text: job.id, + role: job.payload.role as "document" | "query", vectorField: "vec_summary" + }); + h.deps.embeddingJobs.applyEmbeddingVector = (item) => { applied.push(item.job.id); }; + h.deps.embedder.embed = async (texts, role) => { + calls.push({ role, texts: [...texts] }); + return texts.map(() => [1, 0, 0]); + }; + roles.forEach((role, index) => { + const id = `embedding-${index}`; + h.memory(id, at, []); + h.job(id, "embedding", { target: id, payload: { role } }); + }); + return { ...h, calls, applied }; +} + +describe("main Embedding FIFO across role boundaries", () => { + it("executes query then document then query in the same order as WRR admission", async () => { + const h = embeddingRoleHarness("wrr", ["query", "document", "query"]); + const result = await h.runner.runWorkerOnce(3); + expect(h.calls).toEqual([ + { role: "query", texts: ["embedding-0"] }, + { role: "document", texts: ["embedding-1"] }, + { role: "query", texts: ["embedding-2"] } + ]); + expect(h.applied).toEqual(["embedding-0", "embedding-1", "embedding-2"]); + expect(result).toMatchObject({ leased: 3, succeeded: 3, failed: 0 }); + expect(result.jobs.map(job => job.status)).toEqual(["succeeded", "succeeded", "succeeded"]); + }); + + it("retains a batch for adjacent document jobs without overtaking the older query", async () => { + const h = embeddingRoleHarness("wrr", ["query", "document", "document", "query"]); + expect(await h.runner.runWorkerOnce(4)).toMatchObject({ leased: 4, succeeded: 4, failed: 0 }); + expect(h.calls).toEqual([ + { role: "query", texts: ["embedding-0"] }, + { role: "document", texts: ["embedding-1", "embedding-2"] }, + { role: "query", texts: ["embedding-3"] } + ]); + expect(h.applied).toEqual(["embedding-0", "embedding-1", "embedding-2", "embedding-3"]); + expect(h.repos.runtime.listJobs("succeeded")).toHaveLength(4); + }); + + it("retains legacy SQL admission order without restoring a cross-role execution barrier", async () => { + const h = embeddingRoleHarness("legacy", ["query", "document", "document", "query"]); + expect(h.repos.runtime.previewLegacyJobPage({ at, limit: 4 }).map(row => row.job.id)) + .toEqual(["embedding-0", "embedding-1", "embedding-2", "embedding-3"]); + expect(await h.runner.runWorkerOnce(4)).toMatchObject({ leased: 4, succeeded: 4, failed: 0 }); + expect(h.calls).toEqual([ + { role: "query", texts: ["embedding-0"] }, + { role: "document", texts: ["embedding-1", "embedding-2"] }, + { role: "query", texts: ["embedding-3"] } + ]); + expect(h.applied).toEqual(["embedding-0", "embedding-1", "embedding-2", "embedding-3"]); + expect(h.repos.runtime.listJobs("succeeded")).toHaveLength(4); + });}); diff --git a/Memory/tests/service/worker/worker-interactive-priority.test.ts b/Memory/tests/service/worker/worker-interactive-priority.test.ts new file mode 100644 index 000000000..0527140ab --- /dev/null +++ b/Memory/tests/service/worker/worker-interactive-priority.test.ts @@ -0,0 +1,138 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; +import type { WorkerObservation } from "../../../src/service/worker/worker-observability.js"; + +const resources: Array<{ root: string; db: MemoryDb; runner: WorkerRunner }> = []; +function gate() { let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } +function harness(process: WorkerRunnerDeps["jobHandlers"]["processJob"], cap = 1, algorithm: "wrr" | "legacy" = "wrr") { + const root = mkdtempSync("/private/tmp/worker-interactive-"), db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db), concurrency = new WorkerConcurrencyController({}, {}); + const events: WorkerObservation[] = []; + const unexpected = () => { throw new Error("Unexpected fixture dependency"); }; + const deps: WorkerRunnerDeps = { + repos, concurrency, observer: { record: event => events.push(event) }, + scheduling: { algorithm, aging: { enabled: false } }, schedulingShadow: { enabled: false }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: false }, embeddingRetryWorkerId: "interactive", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "interactive", runWorkerNoWrite: async () => unexpected(), + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: unexpected, enqueueEmbeddingRetry: unexpected, appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + prepareExecution: work => ({ providers: work.queue === "main" && work.job.jobType === "episode_idle_close" ? [] : [{ key: "shared-model", limit: cap }], + run: operation => operation() }), + jobHandlers: { processJob: process }, embeddingJobs: { prepareEmbeddingJob: unexpected, applyEmbeddingVector: unexpected, + persistEmbeddingVector: unexpected, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); resources.push({ root, db, runner }); + function job(id: string, jobType: EvolutionJobRecord["jobType"]) { + const at = new Date().toISOString(); repos.runtime.enqueueJob({ id, jobType, targetMemoryId: id, userId: "fixture", payload: {}, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + return { repos, db, runner, concurrency, job, events, deps }; +} +afterEach(async () => { vi.restoreAllMocks(); for (const { root, db, runner } of resources.splice(0)) { await runner.stop(); db.close(); rmSync(root, { recursive: true, force: true }); } }); + +describe("bounded fresh interactive preference", () => { + it.each(["new", "old-cursor", "old-turns"] as const)("starts new interactive work at the next free shared model slot from %s state", async state => { + const entered = gate(), release = gate(), abort = new AbortController(), calls: string[] = []; + const h = harness(async job => { + calls.push(job.id); + if (job.id === "import-0") { entered.resolve(); await release.promise; } + if (calls.length >= 3) abort.abort(); + }); + h.job("import-0", "import_summary"); h.job("import-1", "import_summary"); + h.job("reflection-0", "reflection"); h.job("reflection-1", "reflection"); + if (state !== "new") h.repos.runtime.setKv("worker.scheduler.wrr_cursor.v1", { weightsKey: "4/3/2/2/1", cursor: 7, + ...(state === "old-turns" ? { turns: { version: 1, lanes: [ + { lane: "import", remaining: 2 }, { lane: "evolution", remaining: 2 }, { lane: "maintenance", remaining: 1 }, + { lane: "interactive", remaining: 4 }, { lane: "repair_retry", remaining: 3 } + ] } } : {}) }, new Date().toISOString()); + const run = h.runner.runWorkerOnce(4, {}, { continuous: true, signal: abort.signal }); + await entered.promise; + h.job("native", "trace_summary"); h.runner.notifyWorkAvailable(); release.resolve(); + await run; + expect(calls.slice(0, 2)).toEqual(["import-0", "native"]); + expect(h.events.filter(event => event.type === "claimed").slice(0, 2)).toMatchObject([ + { work: { lane: "import", selectionReason: "wrr" } }, + { work: { lane: "interactive", selectionReason: "interactive" } } + ]); + expect(h.concurrency.availableJobs()).toBe(4); + }); + + it.each([1, 2])("continues all model lanes and maintenance under sustained interactive input with shared cap %s", async cap => { + const abort = new AbortController(), model: string[] = []; + const types = ["trace_summary", "decision_repair", "import_summary", "reflection", "episode_idle_close"] as const; + let seq = 0, active = 0, peak = 0, maintained = 0; + const h = harness(async job => { + if (job.jobType === "episode_idle_close") maintained++; + else { + peak = Math.max(peak, ++active); + try { await new Promise(resolve => setTimeout(resolve, 2)); model.push(job.jobType); } + finally { active--; } + if (model.length >= 90) abort.abort(); + } + if (!abort.signal.aborted && job.jobType !== "episode_idle_close") { + h.job(`new-${seq++}`, job.jobType); + // Keep maintenance arriving throughout every model wave, without + // turning this concurrency test into an unbounded zero-time DB load. + h.job(`maintenance-${seq++}`, "episode_idle_close"); + } + }, cap); + for (const type of types) for (let i = 0; i < (type === "episode_idle_close" ? 1 : 8); i++) h.job(`${type}-${i}`, type); + const emergency = setTimeout(() => abort.abort(), 5000); + try { await h.runner.runWorkerOnce(4, {}, { continuous: true, signal: abort.signal }); } + finally { clearTimeout(emergency); abort.abort(); } + expect(model.length).toBeGreaterThanOrEqual(90); expect(peak).toBe(cap); expect(maintained).toBeGreaterThan(0); + for (let offset = 0; offset < 90; offset += 30) for (const type of types.slice(0, 4)) expect(model.slice(offset, offset + 30)).toContain(type); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, scopeKeys: [] }); + }); + + it("retains the fourth ticket on rollback, then persists exhaustion across notifications and a new runner", async () => { + const calls: string[] = [], h = harness(async job => { calls.push(job.id); }); + for (let i = 0; i < 8; i++) h.job(`interactive-${i}`, "trace_summary"); + h.job("repair", "decision_repair"); h.job("import", "import_summary"); h.job("evolution", "reflection"); h.job("maintenance", "episode_idle_close"); + for (let i = 0; i < 3; i++) await h.runner.runWorkerOnce(1); + vi.spyOn(h.repos.runtime, "setKv").mockImplementationOnce(() => { throw new Error("quota commit failed"); }); + await expect(h.runner.runWorkerOnce(1)).rejects.toThrow("quota commit failed"); + expect(h.repos.runtime.getJob("interactive-3")).toMatchObject({ status: "queued", attempts: 0 }); + expect(h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toMatchObject({ + turns: { interactive: { spent: false }, lanes: expect.arrayContaining([{ lane: "interactive", remaining: 1 }]) } + }); + await h.runner.runWorkerOnce(1); + expect(h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toMatchObject({ turns: { interactive: { spent: true } } }); + for (let i = 0; i < 20; i++) h.runner.notifyWorkAvailable(); + await h.runner.stop(); + const restarted = new WorkerRunner(h.deps); + try { await restarted.runWorkerOnce(1); } + finally { await restarted.stop(); } + expect(calls).toEqual(["interactive-0", "interactive-1", "interactive-2", "interactive-3", "repair"]); + expect(h.repos.runtime.getJob("interactive-4")).toMatchObject({ status: "queued", attempts: 0 }); + }); + + it("persists the one-ticket upgrade advance only after a committed lease and does not restore it on restart", async () => { + const calls: string[] = [], h = harness(async job => { calls.push(job.id); }); + h.job("native-0", "trace_summary"); h.job("native-1", "trace_summary"); h.job("import", "import_summary"); h.job("evolution", "reflection"); + const previous = { weightsKey: "4/3/2/2/1", cursor: 7 }; + h.repos.runtime.setKv("worker.scheduler.wrr_cursor.v1", previous, new Date().toISOString()); + vi.spyOn(h.repos.runtime, "setKv").mockImplementationOnce(() => { throw new Error("upgrade commit failed"); }); + await expect(h.runner.runWorkerOnce(1)).rejects.toThrow("upgrade commit failed"); + expect(h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toEqual(previous); + expect(h.repos.runtime.getJob("native-0")).toMatchObject({ status: "queued", attempts: 0 }); + await h.runner.runWorkerOnce(1); + expect(h.repos.runtime.getKv("worker.scheduler.wrr_cursor.v1")?.value).toMatchObject({ turns: { + interactive: { spent: true, upgradeAdvance: false }, lanes: expect.arrayContaining([{ lane: "interactive", remaining: 3 }]) + } }); + await h.runner.stop(); const restarted = new WorkerRunner(h.deps); + try { await restarted.runWorkerOnce(1); } finally { await restarted.stop(); } + expect(calls).toEqual(["native-0", "import"]); + expect(h.repos.runtime.getJob("native-1")).toMatchObject({ status: "queued", attempts: 0 }); + }); +}); diff --git a/Memory/tests/service/worker/worker-interactive-reservation.test.ts b/Memory/tests/service/worker/worker-interactive-reservation.test.ts new file mode 100644 index 000000000..617c18d03 --- /dev/null +++ b/Memory/tests/service/worker/worker-interactive-reservation.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { WorkerConcurrencyController, type WorkerPermit, type WorkerRequirements } from "../../../src/service/worker/worker-concurrency.js"; + +const requirements = (queueLane?: string, extra: Partial = {}): WorkerRequirements => + ({ resourceClass: "summary", providers: [], scopeKeys: [], queueLane, ...extra }); + +describe("interactive global slot reservation", () => { + it("leaves one global slot for interactive work when background jobs remain blocked", async () => { + const c = new WorkerConcurrencyController({}, {}); + const background = Array.from({ length: 3 }, () => c.tryAcquire(requirements("import"))!); + const before = c.snapshot(); + const denied = c.tryAcquire(requirements("maintenance", { + providers: [{ key: "must-not-reserve", limit: 1 }], scopeKeys: ["must-not-reserve"] + })); + let interactive: WorkerPermit | undefined; + try { + expect(background.every(Boolean)).toBe(true); + expect(denied).toBeUndefined(); + expect(c.snapshot()).toEqual(before); + interactive = c.tryAcquire(requirements("interactive"))!; + expect(interactive).toBeDefined(); + expect(c.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3 }); + expect(c.tryAcquire(requirements("interactive"))).toBeUndefined(); + interactive.release(); + expect(c.tryAcquire(requirements("import"))).toBeUndefined(); + } finally { + interactive?.release(); denied?.release(); background.forEach(p => p?.release()); + } + expect(c.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0 }); + await c.close(); + }); + + it.each([undefined, "unregistered", "repair_retry", "evolution", "maintenance"])("treats lane %s as background", lane => { + const c = new WorkerConcurrencyController({}, {}); + const p = c.tryAcquire(requirements("import"), 3)!; + expect(c.tryAcquire(requirements(lane))).toBeUndefined(); + p.release(); p.release(); + const next = c.tryAcquire(requirements(lane))!; + expect(next).toBeDefined(); next.release(); + expect(c.snapshot().backgroundActiveJobs).toBe(0); + }); + + it("allows interactive jobs to use all global slots", () => { + const c = new WorkerConcurrencyController({}, {}); + const p = c.tryAcquire(requirements("interactive"), 4)!; + expect(p).toBeDefined(); expect(c.availableJobs()).toBe(0); + expect(c.snapshot().backgroundActiveJobs).toBe(0); p.release(); + }); + + it.each([1, 2, 4, 8])("retains background progress at global limit %s", limit => { + const c = new WorkerConcurrencyController({ maxActiveJobs: limit, resourceLimits: { summary: limit } }, {}); + const backgroundLimit = Math.max(1, limit - 1); + const p = c.tryAcquire(requirements("import"), backgroundLimit)!; + expect(p).toBeDefined(); expect(c.tryAcquire(requirements("import"))).toBeUndefined(); + expect(c.limits.maxBackgroundJobs).toBe(backgroundLimit); + expect(c.limits.interactiveReservedSlots).toBe(limit === 1 ? 0 : 1); + p.release(); + }); + + it("charges mixed embedding group members individually and cannot borrow interactive privilege", () => { + const c = new WorkerConcurrencyController({}, {}); + const extra = { resourceClass: "embedding" as const, embedding: true, providers: [{ key: "model", limit: 1 }] }; + const a = c.tryAcquire(requirements("interactive", extra))!; + const b = c.tryAcquire(requirements("import", extra), 3, a)!; + expect(b).toBeDefined(); + a.release(); + expect(c.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, embeddingRequests: 1 }); + expect(c.tryAcquire(requirements("import", extra), 1, b)).toBeUndefined(); + const d = c.tryAcquire(requirements("interactive", extra), 1, b)!; + expect(d).toBeDefined(); + b.release(); b.release(); d.release(); + expect(c.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0, embeddingRequests: 0, providers: {} }); + }); + + it("does not bypass provider limits to use the reserved slot", () => { + const c = new WorkerConcurrencyController({}, {}); + const p = c.tryAcquire(requirements("import", { providers: [{ key: "local", limit: 1 }] }))!; + expect(c.tryAcquire(requirements("interactive", { providers: [{ key: "local", limit: 1 }] }))).toBeUndefined(); + p.release(); + const next = c.tryAcquire(requirements("interactive", { providers: [{ key: "local", limit: 1 }] }))!; + expect(next).toBeDefined(); next.release(); + }); + + it("keeps foreground requests independent from logical global reservation", async () => { + const c = new WorkerConcurrencyController({}, {}); + const p = c.tryAcquire(requirements("import"), 3)!; + expect(await c.runRequest({ providers: [{ key: "foreground", limit: 1 }], local: true }, async () => "done")).toBe("done"); + expect(c.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, providers: {} }); + p.release(); await c.close(); + }); +}); diff --git a/Memory/tests/service/worker/worker-lease-options.test.ts b/Memory/tests/service/worker/worker-lease-options.test.ts new file mode 100644 index 000000000..71e941888 --- /dev/null +++ b/Memory/tests/service/worker/worker-lease-options.test.ts @@ -0,0 +1,106 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { resolveWorkerLeaseOptions, type WorkerLeaseOptions } from "../../../src/service/worker/worker-lease-options.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ db: MemoryDb; root: string; runner: WorkerRunner }> = []; +afterEach(async () => { + for (const { db, root, runner } of resources.splice(0)) { + await runner.stop(); db.close(); rmSync(root, { recursive: true, force: true }); + } + vi.useRealTimers(); vi.unstubAllEnvs(); +}); +function deferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} +function harness(leases: WorkerLeaseOptions, processJob: WorkerRunnerDeps["jobHandlers"]["processJob"], algorithm: "legacy" | "wrr" = "wrr") { + const root = mkdtempSync("/private/tmp/memmy-lease-options-"), db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db), bad = () => { throw new Error("Unexpected dependency"); }; + const deps: WorkerRunnerDeps = { repos, leases, scheduling: { algorithm }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: true }, embeddingRetryWorkerId: "lease-options", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "test", runWorkerNoWrite: async () => bad(), restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + reconcileOrphanedPolicies: () => 0, enqueueJob: bad, enqueueEmbeddingRetry: bad, appendJobChange: () => {}, + appendEmbeddingRetryChange: () => {}, jobHandlers: { processJob }, embeddingJobs: { + prepareEmbeddingJob: bad, applyEmbeddingVector: () => {}, enqueueEmbeddingRetryAfterFailure: () => false, + persistEmbeddingVector: input => { const memory = repos.memories.get(input.memoryId)!; input.finalize?.(memory, false, new Date().toISOString()); return memory; } + } + }; + const runner = new WorkerRunner(deps); resources.push({ db, root, runner }); + return { db, repos, deps, runner, job(type: "trace_summary" | "reflection") { + const at = new Date().toISOString(); + repos.runtime.enqueueJob({ id: "job", jobType: type, status: "queued", userId: "test", payload: {}, attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } }; +} +async function flushUntil(check: () => boolean) { + for (let turn = 0; turn < 30 && !check(); turn++) await new Promise(resolve => setImmediate(resolve)); + expect(check()).toBe(true); +} + +describe("resource-specific worker lease settings", () => { + it("retains defaults and resolves dependency overrides field by field", () => { + const options = resolveWorkerLeaseOptions({ resources: { summary: { durationSec: 90 } } }, { + MEMMY_WORKER_LEASE_OPTIONS: JSON.stringify({ resources: { summary: { durationSec: 120, heartbeatMs: 25_000 } }, embeddingRetry: { durationSec: 400 } }) + }); + expect(options.resources.summary).toEqual({ durationSec: 90, heartbeatMs: 25_000 }); + expect(options.resources.database).toEqual({ durationSec: 60, heartbeatMs: 20_000 }); + expect(options.embeddingRetry).toEqual({ durationSec: 400, heartbeatMs: 20_000 }); + }); + it.each(["null", "[]", "true", "{", '{"resources":{"typo":{}}}', '{"embeddingRetry":{"durationSec":0}}', + '{"resources":{"summary":{"durationSec":10,"heartbeatMs":6000}}}']) + ("rejects unsafe environment settings before claiming: %s", value => { + expect(() => resolveWorkerLeaseOptions({}, { MEMMY_WORKER_LEASE_OPTIONS: value })).toThrow(); + }); + it.each(["wrr", "legacy"] as const)("uses custom resource duration and heartbeat in %s mode", async algorithm => { + vi.useFakeTimers({ toFake: ["Date", "setInterval", "clearInterval"] }); + vi.setSystemTime(Date.parse("2026-10-23T00:00:00Z")); + const gate = deferred(), h = harness({ resources: { summary: { durationSec: 6, heartbeatMs: 2000 } } }, () => gate.promise, algorithm); + h.job("trace_summary"); const run = h.runner.runWorkerOnce(1); + try { + await flushUntil(() => h.repos.runtime.getJob("job")?.status === "leased"); + const before = h.repos.runtime.getJob("job")!; + expect(Date.parse(before.leasedUntil!) - Date.now()).toBe(6000); + await vi.advanceTimersByTimeAsync(2000); + const after = h.repos.runtime.getJob("job")!; + expect(after.leaseToken).toBe(before.leaseToken); + expect(Date.parse(after.leasedUntil!) - Date.now()).toBe(6000); + } finally { gate.resolve(); } + expect(await run).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + }); + it("uses the evolution resource policy independently of summary", async () => { + const h = harness({ resources: { evolution_llm: { durationSec: 12, heartbeatMs: 4000 } } }, async job => { + expect(Date.parse(job.leasedUntil!) - Date.now()).toBeGreaterThan(11_000); + expect(Date.parse(job.leasedUntil!) - Date.now()).toBeLessThanOrEqual(12_000); + }); + h.job("reflection"); expect(await h.runner.runWorkerOnce(1)).toMatchObject({ succeeded: 1, failed: 0 }); + }); + it("applies retry duration and heartbeat to the same fenced claim", async () => { + vi.useFakeTimers({ toFake: ["Date", "setInterval", "clearInterval"] }); + vi.setSystemTime(Date.parse("2026-10-23T00:00:00Z")); + const gate = deferred(), h = harness({ embeddingRetry: { durationSec: 8, heartbeatMs: 2000 } }, async () => {}); + const at = new Date().toISOString(); + h.db.db.prepare("INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,created_at,updated_at) VALUES('target',?,'test','synthetic','L1','[]',?,?)").run(at, at, at); + h.repos.runtime.enqueueEmbeddingRetry({ id: "retry", targetKind: "trace", targetId: "target", vectorField: "vec_summary", sourceText: "synthetic", now: Date.now() }); + h.deps.embedder.embed = async texts => { await gate.promise; return texts.map(() => [1, 0, 0]); }; + const run = h.runner.runWorkerOnce(1); + try { + await flushUntil(() => h.repos.runtime.getEmbeddingRetry("retry")?.status === "in_progress"); + const before = h.repos.runtime.getEmbeddingRetry("retry")!; + expect(before.leaseUntil! - Date.now()).toBe(8000); + await vi.advanceTimersByTimeAsync(2000); + const after = h.repos.runtime.getEmbeddingRetry("retry")!; + expect(after.leaseToken).toBe(before.leaseToken); + expect(after.leaseUntil! - Date.now()).toBe(8000); + } finally { gate.resolve(); } + expect((await run).embeddingRetries).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + }); +}); diff --git a/Memory/tests/service/worker/worker-lease-safety.test.ts b/Memory/tests/service/worker/worker-lease-safety.test.ts new file mode 100644 index 000000000..c4d17ab35 --- /dev/null +++ b/Memory/tests/service/worker/worker-lease-safety.test.ts @@ -0,0 +1,538 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ModelHttpError } from "../../../src/model/http.js"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { classifyProcessingError } from "../../../src/service/worker/job-handlers.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ root: string; db: MemoryDb }> = []; +const epoch = Date.parse("2026-10-22T00:00:00.000Z"); + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +/** Real SQLite and worker lifecycle; only model/business work is substituted. */ +function harness(processJob: WorkerRunnerDeps["jobHandlers"]["processJob"] = async () => {}, concurrency?: WorkerConcurrencyController) { + const root = mkdtempSync(join(tmpdir(), "memmy-lease-safety-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const deps: WorkerRunnerDeps = { + repos, concurrency, + embedder: createCapturingEmbedder([]), + capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "lease-safety-runner", + memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, + memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, + nowIso: () => new Date().toISOString(), + encodeChangeCursor: String, + namespaceIdFromMemory: () => "lease-safety", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ + orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] + }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement retry"); }, + appendJobChange: () => {}, + appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding"); }, + applyEmbeddingVector: () => {}, + persistEmbeddingVector: (input) => { + const target = repos.memories.get(input.memoryId); + if (!target) throw new Error("missing synthetic target"); + input.finalize?.(target, false, new Date().toISOString()); + return target; + }, + enqueueEmbeddingRetryAfterFailure: () => false + } + }; + const runner = new WorkerRunner(deps); + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection"): void { + const at = new Date().toISOString(); + repos.runtime.enqueueJob({ + id, jobType, status: "queued", userId: "lease-safety-user", payload: {}, + attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at + }); + } + function retry(id: string): void { + const at = new Date().toISOString(); + db.db.prepare(`INSERT INTO memories + (id, timeline, user_id, memory_value, memory_layer, created_at, updated_at) + VALUES (?, ?, 'lease-safety-user', 'retry text', 'L1', ?, ?)`) + .run(id, at, at, at); + repos.runtime.enqueueEmbeddingRetry({ + id, targetKind: "trace", targetId: id, vectorField: "vec_summary", sourceText: "retry text", now: Date.now() + }); + } + return { db, repos, runner, job, retry, deps }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("worker lease ownership and recovery", () => { + it("rejects old completion and failure after a replacement lease succeeds", () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("replacement"); + const [a] = h.repos.runtime.leaseQueuedJobs(1); + vi.setSystemTime(epoch + 61_000); + const [b] = h.repos.runtime.leaseQueuedJobs(1); + expect(a!.leaseToken).toBeTruthy(); + expect(b!.leaseToken).not.toBe(a!.leaseToken); + const at = new Date().toISOString(); + expect(h.repos.runtime.completeJob(b!.id, at, b)!.status).toBe("succeeded"); + expect(h.repos.runtime.completeJob(a!.id, at, a)).toBeUndefined(); + expect(h.repos.runtime.failJob(a!.id, "late failure", at, false, a)).toBeUndefined(); + expect(h.repos.runtime.getJob(a!.id)).toMatchObject({ status: "succeeded", failureCount: 0 }); + }); + + it("rejects writes and renewal from an expired lease even before anyone reclaims it", () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("expired"); + const [a] = h.repos.runtime.leaseQueuedJobs(1); + vi.setSystemTime(epoch + 60_001); + const at = new Date().toISOString(); + expect(h.repos.runtime.isJobLeaseHeld(a!, at)).toBe(false); + expect(h.repos.runtime.completeJob(a!.id, at, a)).toBeUndefined(); + expect(h.repos.runtime.failJob(a!.id, "late failure", at, false, a)).toBeUndefined(); + h.repos.runtime.renewJobLease(a!, 60, at); + expect(h.repos.runtime.isJobLeaseHeld(a!, at)).toBe(false); + expect(h.repos.runtime.getJob(a!.id)).toMatchObject({ status: "leased", failureCount: 0 }); + }); + + it("does not let legacy id-only completion or failure bypass a current lease", () => { + const h = harness(); h.job("requires-claim"); + const [claim] = h.repos.runtime.leaseQueuedJobs(1); + expect(h.repos.runtime.completeJob(claim!.id)).toBeUndefined(); + expect(h.repos.runtime.failJob(claim!.id, "without lease credentials")).toBeUndefined(); + expect(h.repos.runtime.getJob(claim!.id)).toMatchObject({ status: "leased", leaseToken: claim!.leaseToken }); + }); + + it("renews active work while capacity-blocked tasks remain queued without lease tokens", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(), entered = deferred(); const started: string[] = []; + const h = harness(async (job) => { + started.push(job.id); + if (job.id === "serial-0") { entered.resolve(); await gate.promise; } + }, new WorkerConcurrencyController({ maxActiveJobs: 1 }, {})); + for (let i = 0; i < 4; i++) h.job(`serial-${i}`); + const run = h.runner.runWorkerOnce(4); + await entered.promise; + try { + const initial = h.repos.runtime.getJob("serial-0")!; + await vi.advanceTimersByTimeAsync(125_000); + expect(started).toEqual(["serial-0"]); + const active = h.repos.runtime.getJob("serial-0")!; + expect(active).toMatchObject({ status: "leased", leaseToken: initial.leaseToken }); + expect(Date.parse(active.leasedUntil!)).toBeGreaterThan(Date.now()); + expect(h.repos.runtime.claimQueuedJob("serial-0", { owner: "competing-worker" })).toBeUndefined(); + for (let i = 1; i < 4; i++) { + expect(h.repos.runtime.getJob(`serial-${i}`)).toMatchObject({ status: "queued", attempts: 0, + leaseToken: null, leasedUntil: null, leaseOwner: null }); + } + } finally { gate.resolve(); await run; } + expect(started).toHaveLength(4); + expect(vi.getTimerCount()).toBe(0); + expect(h.repos.runtime.getJob("serial-0")).toMatchObject({ status: "succeeded", failureCount: 0 }); + }); + it("clears its heartbeat after a handler fails", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + const h = harness(async () => { await gate.promise; throw new Error("temporary timeout"); }); + h.job("fails-after-renewal"); + const run = h.runner.runWorkerOnce(1); + await vi.advanceTimersByTimeAsync(65_000); + gate.resolve(); + const result = await run; + expect(result.failed).toBe(1); + expect(h.repos.runtime.getJob("fails-after-renewal")).toMatchObject({ status: "failed", failureCount: 1 }); + expect(vi.getTimerCount()).toBe(0); + }); + + it("keeps concurrent calls isolated when one call finishes before the other", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const a = deferred(); const b = deferred(); const started: string[] = []; + const h = harness(async (j) => { started.push(j.id); await (j.id === "A" ? a.promise : b.promise); }); + h.job("A"); h.job("B"); + const runA = h.runner.runWorkerOnce(1); + const runB = h.runner.runWorkerOnce(1); + try { + await vi.advanceTimersByTimeAsync(45_000); + a.resolve(); await runA; + await vi.advanceTimersByTimeAsync(85_000); + expect(started.sort()).toEqual(["A", "B"]); + expect(h.repos.runtime.getJob("A")!.status).toBe("succeeded"); + const pending = h.repos.runtime.getJob("B")!; + expect(pending.status).toBe("leased"); + expect(Date.parse(pending.leasedUntil!)).toBeGreaterThan(Date.now()); + expect(h.repos.runtime.leaseQueuedJobs(1)).toEqual([]); + } finally { a.resolve(); b.resolve(); await Promise.all([runA, runB]); } + expect(vi.getTimerCount()).toBe(0); + }); + + it("renews an embedding retry during a request lasting beyond its original five minute lease", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + const h = harness(); + const at = new Date().toISOString(); + h.db.db.prepare(`INSERT INTO memories + (id, timeline, user_id, memory_value, memory_layer, created_at, updated_at) + VALUES ('retry-target', ?, 'lease-safety-user', 'retry text', 'L1', ?, ?)`) + .run(at, at, at); + h.repos.runtime.enqueueEmbeddingRetry({ + id: "long-retry", targetKind: "trace", targetId: "retry-target", vectorField: "vec_summary", + sourceText: "retry text", now: epoch + }); + h.deps.embedder.embed = async (texts) => { await gate.promise; return texts.map(() => [1, 0, 0]); }; + const run = h.runner.runEmbeddingRetryOnce(1); + try { + await vi.advanceTimersByTimeAsync(310_000); + const retry = h.repos.runtime.getEmbeddingRetry("long-retry")!; + expect(retry.status).toBe("in_progress"); + expect(retry.leaseUntil).toBeGreaterThan(Date.now()); + expect(h.repos.runtime.claimDueEmbeddingRetries({ + now: Date.now(), workerId: "competing-runner", leaseUntil: Date.now() + 60_000, limit: 1 + })).toEqual([]); + } finally { gate.resolve(); await run; } + expect(h.repos.runtime.getEmbeddingRetry("long-retry")!.status).toBe("succeeded"); + expect(vi.getTimerCount()).toBe(0); + }); + + it("treats lost ownership as recovery, not a successful job or a business failure", async () => { + vi.useFakeTimers(); vi.setSystemTime(epoch); + const gate = deferred(); + const h = harness(async () => { await gate.promise; }); h.job("lost"); + const renew = vi.spyOn(h.repos.runtime, "renewJobLease"); + const run = h.runner.runWorkerOnce(1); + await Promise.resolve(); + h.db.db.prepare(`UPDATE evolution_jobs + SET lease_owner='replacement-owner', lease_token='replacement-token' + WHERE id='lost'`).run(); + gate.resolve(); + const result = await run; + expect(result).toMatchObject({ succeeded: 0, failed: 0 }); + expect(h.repos.runtime.getJob("lost")).toMatchObject({ + status: "leased", leaseOwner: "replacement-owner", leaseToken: "replacement-token", failureCount: 0 + }); + const renewalCalls = renew.mock.calls.length; + await vi.advanceTimersByTimeAsync(60_000); + expect(renew).toHaveBeenCalledTimes(renewalCalls); + }); + + it("preserves valid startup leases and recovers only expired main and retry leases", () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("valid-main"); + const [valid] = h.repos.runtime.leaseQueuedJobs(1, 600); + h.job("expired-main"); + h.repos.runtime.leaseQueuedJobs(1, 60); + for (const id of ["valid-retry", "expired-retry"]) h.repos.runtime.enqueueEmbeddingRetry({ + id, targetKind: "trace", targetId: id, vectorField: "vec_summary", sourceText: id, now: epoch + }); + h.repos.runtime.claimDueEmbeddingRetries({ + now: epoch, workerId: "valid", leaseUntil: epoch + 600_000, targetMemoryIds: ["valid-retry"] + }); + h.repos.runtime.claimDueEmbeddingRetries({ + now: epoch, workerId: "expired", leaseUntil: epoch + 60_000, targetMemoryIds: ["expired-retry"] + }); + vi.setSystemTime(epoch + 61_000); + const recovered = h.runner.reconcileWorkerStartup(100); + expect(recovered).toMatchObject({ requeuedJobs: 1, requeuedEmbeddingRetries: 1 }); + expect(h.repos.runtime.getJob("valid-main")).toMatchObject({ + status: "leased", leaseToken: valid!.leaseToken, failureCount: 0 + }); + expect(h.repos.runtime.getJob("expired-main")).toMatchObject({ + status: "queued", failureCount: 0, leaseRecoveryCount: 1 + }); + expect(h.repos.runtime.getEmbeddingRetry("valid-retry")!.status).toBe("in_progress"); + expect(h.repos.runtime.getEmbeddingRetry("expired-retry")!.status).toBe("pending"); + const [again] = h.repos.runtime.leaseQueuedJobs(1); + expect(again).toMatchObject({ id: "expired-main", failureCount: 0, leaseCount: 2, leaseRecoveryCount: 1 }); + }); +}); + +describe("worker business failure policy", () => { + it.each([408, 429, 500, 503])("backs off HTTP %s even when its response mentions an API key", async httpStatus => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + vi.spyOn(Math, "random").mockReturnValue(0.5); + let calls = 0; + const h = harness(async () => { + if (++calls === 1) throw new ModelHttpError("provider request failed", "test", httpStatus, + "temporary_provider_error", "Temporary API key service limit; please retry shortly"); + }); + h.deps.configurationFingerprint = () => "unchanged-config"; + h.job("transient-key-response", "trace_summary"); + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, failed: 1 }); + const failed = h.repos.runtime.getJob("transient-key-response")!; + expect(failed).toMatchObject({ status: "failed", blockedReason: null, failureCount: 1 }); + expect(Date.parse(failed.availableAt!)).toBe(epoch + 5_000); + expect(h.runner.nextWorkerRunAt()).toBe(epoch + 5_000); + vi.setSystemTime(epoch + 4_999); + expect((await h.runner.runWorkerOnce(1)).leased).toBe(0); + vi.setSystemTime(epoch + 5_000); + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(calls).toBe(2); + }); + + it.each([ + [401, "invalid API key", "model_configuration", "open_settings"], + [400, "invalid request body", "invalid_model_request", "none"], + [400, "maximum context length exceeded", "model_input_too_long", "none"] + ] as const)("preserves the HTTP %s classification for %s", (status, detail, code, retryAction) => { + expect(classifyProcessingError(new ModelHttpError("provider request failed", "test", status, undefined, detail))) + .toEqual({ code, retryAction }); + }); + + it("keeps an explicit missing-model configuration error waiting for configuration", () => { + expect(classifyProcessingError(new Error("model is not configured: missing API key"))) + .toEqual({ code: "model_configuration", retryAction: "open_settings" }); + }); + + it("preserves the explicit 40309 business error before retryable HTTP status classification", () => { + expect(classifyProcessingError(new ModelHttpError("provider request failed", "test", 429, "40309", "API key quota exhausted"))) + .toEqual({ code: "40309", retryAction: "open_settings" }); + }); + + it("does not apply a transient HTTP failure after the execution loses its lease", async () => { + const h = harness(async job => { + h.db.db.prepare("UPDATE evolution_jobs SET lease_owner='replacement', lease_token='replacement-token' WHERE id=?") + .run(job.id); + throw new ModelHttpError("provider request failed", "test", 429, undefined, "Temporary API key rate limit"); + }); + h.job("lost-before-http-failure"); + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 0, failed: 0 }); + expect(h.repos.runtime.getJob("lost-before-http-failure")).toMatchObject({ status: "leased", + leaseOwner: "replacement", leaseToken: "replacement-token", failureCount: 0, availableAt: null, blockedReason: null }); + }); + + it("schedules a temporary failure, skips early retries, then runs it when due", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + vi.spyOn(Math, "random").mockReturnValue(0.5); + let calls = 0; + const h = harness(async () => { if (++calls === 1) throw new Error("temporary timeout"); }); + h.job("retry-later"); + const first = await h.runner.runWorkerOnce(1); + expect(first.failed).toBe(1); + const failed = h.repos.runtime.getJob("retry-later")!; + expect(failed).toMatchObject({ status: "failed", failureCount: 1 }); + const available = Date.parse(failed.availableAt!); + expect(available - epoch).toBeGreaterThanOrEqual(4_000); + expect(available - epoch).toBeLessThanOrEqual(6_000); + expect(h.runner.nextWorkerRunAt()).toBe(available); + vi.setSystemTime(available - 1); + expect((await h.runner.runWorkerOnce(1)).leased).toBe(0); + expect(calls).toBe(1); + vi.setSystemTime(available); + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.repos.runtime.getJob("retry-later")).toMatchObject({ status: "succeeded", failureCount: 1, leaseCount: 2 }); + }); + + it("counts temporary business failures separately from lease recoveries", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(async () => { throw new Error("temporary timeout"); }); + h.job("eventual-dead-letter"); + for (let i = 1; i <= 3; i++) { + await h.runner.runWorkerOnce(1); + const row = h.repos.runtime.getJob("eventual-dead-letter")!; + expect(row.failureCount).toBe(i); + if (i < 3) { + expect(row.status).toBe("failed"); + vi.setSystemTime(Date.parse(row.availableAt!)); + } else { + expect(row.status).toBe("dead_letter"); + } + } + }); + + it("holds configuration failures without burning business failure budget", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + let calls = 0; + const h = harness(async () => { calls++; throw new Error("model is not configured: missing API key"); }); + h.job("config-blocked", "trace_summary"); + await h.runner.runWorkerOnce(1); + expect(h.repos.runtime.getJob("config-blocked")).toMatchObject({ + status: "failed", blockedReason: "configuration", failureCount: 0 + }); + vi.setSystemTime(epoch + 3_600_000); + expect((await h.runner.runWorkerOnce(1)).leased).toBe(0); + expect(calls).toBe(1); + }); + + it("resumes the original blocked job when the model configuration changes", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + let valid = false; + let fingerprint = "before"; + const h = harness(async () => { if (!valid) throw new Error("unauthorized API key"); }); + h.deps.configurationFingerprint = () => fingerprint; + h.job("config-restored", "trace_summary"); + await h.runner.runWorkerOnce(1); + expect(h.repos.runtime.getJob("config-restored")).toMatchObject({ blockedReason: "configuration", failureCount: 0 }); + valid = true; fingerprint = "after"; + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.repos.runtime.getJob("config-restored")).toMatchObject({ + id: "config-restored", status: "succeeded", failureCount: 0, blockedReason: null + }); + }); + + it("dead-letters permanent invalid input immediately", async () => { + const h = harness(async () => { + throw new ModelHttpError("bad input", "test", 400, undefined, "invalid request body"); + }); + h.job("permanent", "trace_summary"); + const result = await h.runner.runWorkerOnce(1); + expect(result).toMatchObject({ succeeded: 0, failed: 1 }); + expect(h.repos.runtime.getJob("permanent")).toMatchObject({ status: "dead_letter", failureCount: 1 }); + expect((await h.runner.runWorkerOnce(1)).leased).toBe(0); + }); +}); + +describe("embedding retry configuration recovery", () => { + it.each([429, 503])("retries HTTP %s embedding failures mentioning API keys without waiting for configuration", async httpStatus => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.retry("transient-key-retry"); + let calls = 0; + h.deps.embedder.embed = async texts => { + if (++calls === 1) throw new ModelHttpError("provider request failed", "test", httpStatus, undefined, + "Temporary API key service limit; please retry shortly"); + return texts.map(() => [1, 0, 0]); + }; + h.deps.configurationFingerprint = () => "unchanged-config"; + expect((await h.runner.runWorkerOnce(1)).embeddingRetries).toMatchObject({ leased: 1, failed: 1 }); + const retry = h.repos.runtime.getEmbeddingRetry("transient-key-retry")!; + expect(retry).toMatchObject({ status: "pending", attempts: 1, blockedReason: null }); + expect(retry.nextAttemptAt).toBeGreaterThan(epoch); + vi.setSystemTime(retry.nextAttemptAt); + expect((await h.runner.runWorkerOnce(1)).embeddingRetries).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(calls).toBe(2); + }); + + it("holds unauthorized embedding retries without consuming their failure budget", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.retry("unauthorized-retry"); + let requests = 0; + h.deps.embedder.embed = async () => { + requests++; + throw new ModelHttpError("unauthorized", "test", 401, undefined, "invalid API key"); + }; + h.deps.configurationFingerprint = () => "same-config"; + await h.runner.runWorkerOnce(4); + expect(h.repos.runtime.getEmbeddingRetry("unauthorized-retry")).toMatchObject({ + status: "failed", blockedReason: "configuration", attempts: 0 + }); + vi.setSystemTime(epoch + 3_600_000); + const held = await h.runner.runWorkerOnce(4); + expect(held.embeddingRetries.leased).toBe(0); + expect(requests).toBe(1); + }); + + it("resumes the same embedding retry only after the configuration fingerprint changes", async () => { + const h = harness(); h.retry("configuration-restored-retry"); + let valid = false; let fingerprint = "invalid"; + h.deps.configurationFingerprint = () => fingerprint; + h.deps.embedder.embed = async (texts) => { + if (!valid) throw new ModelHttpError("unauthorized", "test", 401, undefined, "invalid API key"); + return texts.map(() => [1, 0, 0]); + }; + await h.runner.runWorkerOnce(4); + expect(h.repos.runtime.getEmbeddingRetry("configuration-restored-retry")).toMatchObject({ + blockedReason: "configuration", attempts: 0 + }); + valid = true; + expect((await h.runner.runWorkerOnce(4)).embeddingRetries.leased).toBe(0); + fingerprint = "valid"; + const resumed = await h.runner.runWorkerOnce(4); + expect(resumed.embeddingRetries).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.repos.runtime.getEmbeddingRetry("configuration-restored-retry")).toMatchObject({ + id: "configuration-restored-retry", status: "succeeded", blockedReason: null, attempts: 0 + }); + }); + + it("does not revive permanently invalid embedding input on configuration changes or restart", async () => { + const h = harness(); h.retry("invalid-input-retry"); + let fingerprint = "before"; + h.deps.configurationFingerprint = () => fingerprint; + h.deps.embedder.embed = async () => { + throw new ModelHttpError("invalid input", "test", 400, undefined, "invalid request body"); + }; + await h.runner.runWorkerOnce(4); + expect(h.repos.runtime.getEmbeddingRetry("invalid-input-retry")).toMatchObject({ + status: "failed", attempts: 1 + }); + fingerprint = "after"; + expect((await h.runner.runWorkerOnce(4)).embeddingRetries.leased).toBe(0); + expect(h.runner.reconcileWorkerStartup(100).requeuedEmbeddingRetries).toBe(0); + expect(h.repos.runtime.getEmbeddingRetry("invalid-input-retry")).toMatchObject({ status: "failed", attempts: 1 }); + }); +}); + +describe("in-flight main-job configuration changes", () => { + it("does not resume another job that failed under the current configuration", async () => { + const gate = deferred(); + let fingerprint = "config-A"; + const h = harness(async (job) => { + if (job.id === "old-config-job") { + // Requests may still overlap inside one admitted parallel batch. The + // second item starts after the configuration change, under config B. + fingerprint = "config-B"; + h.runner.resumeJobsAfterConfigurationChange(); + await gate.promise; + } + throw new Error("unauthorized API key"); + }); + h.deps.configurationFingerprint = () => fingerprint; + h.job("old-config-job", "trace_summary"); + h.job("current-config-job", "trace_summary"); + const run = h.runner.runWorkerOnce(2); + try { + await vi.waitFor(() => expect(h.repos.runtime.getJob("current-config-job")) + .toMatchObject({ status: "failed", blockedReason: "configuration" })); + } finally { gate.resolve(); await run; } + expect(h.repos.runtime.getJob("old-config-job")).toMatchObject({ status: "queued", blockedReason: null }); + expect(h.repos.runtime.getJob("current-config-job")).toMatchObject({ status: "failed", blockedReason: "configuration" }); + }); +}); + +describe("in-flight embedding configuration changes", () => { + it("requeues a late old-config failure once and holds failures from the current config", async () => { + const h = harness(); h.retry("old-config-retry"); + let fingerprint = "config-A"; + h.deps.configurationFingerprint = () => fingerprint; + const gate = deferred(); const entered = deferred(); let calls = 0; + h.deps.embedder.embed = async () => { + calls++; + if (calls === 1) { entered.resolve(); await gate.promise; } + throw new ModelHttpError("unauthorized", "test", 401, undefined, "invalid API key"); + }; + const oldRun = h.runner.runWorkerOnce(1); + await entered.promise; + fingerprint = "config-B"; + h.runner.resumeJobsAfterConfigurationChange(); + gate.resolve(); await oldRun; + expect(h.repos.runtime.getEmbeddingRetry("old-config-retry")).toMatchObject({ status: "pending", blockedReason: null, attempts: 0 }); + expect((await h.runner.runWorkerOnce(4)).embeddingRetries.leased).toBe(1); + expect(h.repos.runtime.getEmbeddingRetry("old-config-retry")).toMatchObject({ status: "failed", blockedReason: "configuration", attempts: 0 }); + expect((await h.runner.runWorkerOnce(4)).embeddingRetries.leased).toBe(0); + expect(calls).toBe(2); + }); +}); diff --git a/Memory/tests/service/worker/worker-manual-retry-yield.test.ts b/Memory/tests/service/worker/worker-manual-retry-yield.test.ts new file mode 100644 index 000000000..c0d7e2881 --- /dev/null +++ b/Memory/tests/service/worker/worker-manual-retry-yield.test.ts @@ -0,0 +1,54 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createMemoryServiceFixture, createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(fixture.cleanup); +function deferred() { + let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; +} + +describe("manual retry yields a waiting continuous drain", () => { + it("wakes an idle automatic drain without waiting for an unrelated foreground provider request", async () => { + const { db } = fixture.createTestService(); + const repos = new Repositories(db.db), controller = new WorkerConcurrencyController({}, {}); + const foregroundGate = deferred(), denied = deferred(), abort = new AbortController(); + const providers = [{ key: "review-provider", limit: 1 }]; + const foreground = controller.runRequest({ providers, local: false }, () => foregroundGate.promise); + const deps: WorkerRunnerDeps = { repos, concurrency: controller, + prepareExecution: () => { denied.resolve(); return { providers, run: operation => operation() }; }, + scheduling: { algorithm: "wrr" }, schedulingShadow: { enabled: false }, embedder: createCapturingEmbedder([]), + capture: { embedAfterCapture: true }, embeddingRetryWorkerId: "manual-retry-review", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "review", runWorkerNoWrite: async () => { throw new Error("Unexpected disabled worker"); }, + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: () => { throw new Error("Unexpected startup enqueue"); }, enqueueEmbeddingRetry: () => { throw new Error("Unexpected retry enqueue"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, jobHandlers: { processJob: async () => {} }, + embeddingJobs: { prepareEmbeddingJob: () => { throw new Error("Unexpected embedding job"); }, applyEmbeddingVector: () => {}, + persistEmbeddingVector: () => { throw new Error("Unexpected vector write"); }, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); + repos.runtime.enqueueJob({ id: "provider-blocked", jobType: "trace_summary", userId: "review-user", payload: {}, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + const automatic = runner.runWorkerOnce(4, {}, { continuous: true, signal: abort.signal }); + await denied.promise; + let manualFinished = false; + const manual = runner.runEmbeddingRetryOnce(1).then(result => { manualFinished = true; return result; }); + for (let i = 0; i < 200; i++) await Promise.resolve(); + const beforeForegroundRelease = manualFinished; + try { + abort.abort(); + const [, result] = await Promise.all([automatic, manual]); + expect(beforeForegroundRelease).toBe(true); + expect(result.leased).toBe(0); + expect(repos.runtime.getJob("provider-blocked")).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { + abort.abort(); foregroundGate.resolve(); await foreground; await runner.stop(); await controller.close(); + } + }); +}); diff --git a/Memory/tests/service/worker/worker-model-admission.test.ts b/Memory/tests/service/worker/worker-model-admission.test.ts new file mode 100644 index 000000000..c71cf4497 --- /dev/null +++ b/Memory/tests/service/worker/worker-model-admission.test.ts @@ -0,0 +1,32 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; +import type { FairClaimedWork } from "../../../src/service/worker/fair-worker-queue.js"; +import type { PreparedWorkerExecution } from "../../../src/service/worker/worker-concurrency.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); +describe("model admission fill snapshots", () => { + it("loads config once per fill and separates same-provider batches after credential-only reload", () => { + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.summary = { ...config.summary, provider: "openai_compatible", endpoint: "https://provider.test/v1", model: "same", apiKey: "old-secret" }; + let loads = 0; + const { service } = createTestService({ configLoader: () => { loads++; return { config }; } }); + const internal = service as unknown as { + prepareWorkerExecutionBatch(): (work: FairClaimedWork) => PreparedWorkerExecution; + }; + const candidate = { queue: "main", job: { jobType: "embedding" } } as FairClaimedWork; + const before = loads; + const firstFill = internal.prepareWorkerExecutionBatch(); + const first = firstFill(candidate); + for (let i = 0; i < 1_000; i++) expect(firstFill(candidate).batchKey).toBe(first.batchKey); + expect(loads).toBe(before + 1); + config.summary.apiKey = "reloaded-secret"; + expect(firstFill(candidate).batchKey).toBe(first.batchKey); + const second = internal.prepareWorkerExecutionBatch()(candidate); + expect(loads).toBe(before + 2); + expect(second.providers).toEqual(first.providers); + expect(second.batchKey).not.toBe(first.batchKey); + expect(second.batchKey).toMatch(/^[a-f0-9]{64}$/); + expect(second.batchKey).not.toContain("secret"); + }); +}); diff --git a/Memory/tests/service/worker/worker-model-shutdown.test.ts b/Memory/tests/service/worker/worker-model-shutdown.test.ts new file mode 100644 index 000000000..31c50ace2 --- /dev/null +++ b/Memory/tests/service/worker/worker-model-shutdown.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it } from "vitest"; +import type { MemoryModelTaskRouter } from "../../../src/model/task-routing.js"; +import { createBatchReflectionLlm, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; +const { cleanup, createTestService } = createMemoryServiceFixture(); +afterEach(cleanup); +const tick = () => new Promise(resolve => setImmediate(resolve)); +describe("MemoryService model business shutdown", () => { + it("rejects foreground shutdown reentry without stopping the service", async () => { + const llm = createBatchReflectionLlm([]); + const { service } = createTestService({ llm }); + const tasks = (service as unknown as { modelTasks: MemoryModelTaskRouter }).modelTasks; + llm.complete = async () => { + await expect(service.stop()).rejects.toThrow(/foreground model task/); + return "finished after rejected stop"; + }; + expect(await tasks.run(() => tasks.client("summary").complete([], { operation: "test.reentry" }))) + .toBe("finished after rejected stop"); + expect(await tasks.run(() => Promise.resolve("still accepting foreground"))).toBe("still accepting foreground"); + await service.stop(); + }); + + it("keeps storage open until foreground post-model processing finishes", async () => { + let respond!: () => void, finishBusiness!: () => void; + const response = new Promise(resolve => { respond = resolve; }); + const businessGate = new Promise(resolve => { finishBusiness = resolve; }); + const llm = createBatchReflectionLlm([]); + llm.complete = async () => { await response; return "response"; }; + const { db, service } = createTestService({ llm }); + db.db.exec("CREATE TABLE shutdown_probe(value INTEGER NOT NULL)"); + // Exercise the same task boundary used by search/startTurn/feedback, with a + // controlled post-response DB step rather than timing a remote provider. + const tasks = (service as unknown as { modelTasks: MemoryModelTaskRouter }).modelTasks; + const business = tasks.run(async () => { + await tasks.client("summary").complete([], { operation: "test.shutdown" }); + await businessGate; + db.db.prepare("INSERT INTO shutdown_probe(value) VALUES (1)").run(); + }); + await tick(); + let stopped = false; + const stop = service.stop().then(() => { stopped = true; }); + try { + await tick(); expect(stopped).toBe(false); + respond(); await tick(); expect(stopped).toBe(false); + finishBusiness(); await Promise.all([business, stop]); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM shutdown_probe").get()).toEqual({ count: 1 }); + } finally { + respond(); finishBusiness(); await Promise.all([business, stop]); + } + }); +}); diff --git a/Memory/tests/service/worker/worker-observability.test.ts b/Memory/tests/service/worker/worker-observability.test.ts new file mode 100644 index 000000000..b26839f63 --- /dev/null +++ b/Memory/tests/service/worker/worker-observability.test.ts @@ -0,0 +1,289 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { join } from "node:path"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { MemoryModelTaskRouter, type MemoryModelTaskContext } from "../../../src/model/task-routing.js"; +import { ModelHttpError } from "../../../src/model/http.js"; +import { WorkerConcurrencyController, providerDescriptor, type WorkerRequirements } from "../../../src/service/worker/worker-concurrency.js"; +import { observeWorker, readWorkerQueueMetrics, WorkerObservability, type WorkerObservation, type WorkerObserver } from "../../../src/service/worker/worker-observability.js"; +import { createBatchReflectionLlm, createCapturingEmbedder, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); await fixture.cleanup(); }); +const at = Date.parse("2026-10-23T00:00:00Z"); +const work = { lane: "import", queue: "main" as const, firstQueuedAt: new Date(at - 1_000).toISOString() }; +const req = (changes: Partial = {}): WorkerRequirements => ({ resourceClass: "summary", providers: [], scopeKeys: [], ...changes }); +const asSnapshot = (metrics: WorkerObservability) => metrics.snapshot() as any; +function routed(metrics: WorkerObserver, c: WorkerConcurrencyController) { + const summary = createBatchReflectionLlm([]); + const context: MemoryModelTaskContext = { config: DEFAULT_MEMMY_CONFIG, summary, evolution: summary, embedding: createCapturingEmbedder([]) }; + const router = new MemoryModelTaskRouter(() => context, c, metrics); + return { router, context }; +} + +describe("bounded worker diagnostics", () => { + it("reports background occupancy, reservation and admission rejection without a full queue census", () => { + let now = at; + const metrics = new WorkerObservability({ now: () => now }); + const c = new WorkerConcurrencyController({}, {}), detach = c.observe(metrics); + const background = c.tryAcquire(req({ queueLane: "import" }), 3)!; + expect(c.tryAcquire(req({ queueLane: "maintenance" }))).toBeUndefined(); + const interactive = c.tryAcquire(req({ queueLane: "interactive" }))!; + expect(asSnapshot(metrics).concurrency).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3, + maxBackgroundJobs: 3, interactiveReservedSlots: 1, peaks: { global: 4, background: 3 } }); + expect(asSnapshot(metrics).counters["admission_rejected.background"]).toBe(1); + // Unchanged occupancy must remain visible after the event's original window expires. + now += 310_001; + expect(asSnapshot(metrics).concurrency.peaks.background).toBe(3); + background.release(); background.release(); interactive.release(); + expect(asSnapshot(metrics).concurrency).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0 }); + detach(); metrics.stop(); + }); + + it("reports started wait percentiles, completion rate and time-due counts without claiming they are runnable", () => { + let now = at; + const metrics = new WorkerObservability({ now: () => now, queueSnapshot: () => [{ lane: "import", queued: 7, failed: 1, + leased: 2, dueTimeOnly: 6, oldestWaitingAt: new Date(at - 100_000).toISOString() }] }); + for (const wait of [100, 200, 300, 400, 900]) { + const item = { ...work, firstQueuedAt: new Date(now - wait).toISOString() }; + metrics.record({ type: "claimed", work: item }); metrics.record({ type: "started", work: item }); + metrics.record({ type: "finished", work: item, outcome: "succeeded" }); + } + metrics.record({ type: "skipped", lane: "import", reason: "scope" }); now += 60_000; + const result = asSnapshot(metrics); + expect(result.lanes.import).toMatchObject({ queued: 7, failed: 1, leased: 2, dueTimeOnly: 6, + oldestWaitingAgeMs: 160_000, waitMs: { sampleCount: 5, p50: 300, p95: 900 }, admitted: 5, succeeded: 5, completedPerMinute: 5 }); + expect(result.dueBasis).toContain("not_dependency"); expect(result.runnableBasis).toContain("not_a_full_queue"); + expect(result.counters["skip.import.scope"]).toBe(1); + }); + + it("bounds lane samples and model cardinality, then expires old samples and counters", () => { + let now = at; const metrics = new WorkerObservability({ now: () => now }); + for (let i = 0; i < 2_000; i++) metrics.record({ type: "started", work }); + for (let i = 0; i < 100; i++) metrics.record({ type: "model_request", providerKey: `provider-secret-${i}`, + modelKey: `model-secret-${i}`, kind: "embedding", durationMs: i, outcome: "succeeded", batchSize: 16 }); + let result = asSnapshot(metrics); + expect(result.lanes.import.waitMs.sampleCount).toBe(512); expect(result.models).toHaveLength(33); + expect(result.models.find((model: any) => model.id === "overflow").latencyMs.sampleCount).toBe(68); + expect(JSON.stringify(result)).not.toMatch(/provider-secret|model-secret/); + now += 310_001; result = asSnapshot(metrics); + expect(result.lanes.import.waitMs.sampleCount).toBe(0); expect(result.counters).toEqual({}); + expect(result.models.every((model: any) => model.latencyMs.sampleCount === 0)).toBe(true); + }); + + it("reuses time buckets across two thousand wall-clock reversals and excludes future observations", () => { + let now = at; const metrics = new WorkerObservability({ now: () => now }); + for (let i = 0; i < 2_000; i++) { now = at + (i % 2) * 10_000; metrics.record({ type: "claimed", work }); } + expect((metrics as unknown as { buckets: unknown[] }).buckets).toHaveLength(2); + expect(asSnapshot(metrics).counters["lane.import.claimed"]).toBe(2_000); + now = at; expect(asSnapshot(metrics).counters["lane.import.claimed"]).toBe(1_000); + // Many distinct backwards positions also obey the hard capacity. + for (let i = 0; i < 1_000; i++) { now = at - i * 10_000; metrics.record({ type: "claimed", work }); } + expect((metrics as unknown as { buckets: unknown[] }).buckets.length).toBeLessThanOrEqual(32); + }); + + it.each([0, -1, NaN, Infinity, 0.5, 3_600_001])("rejects invalid metrics timer/window configuration %s", invalid => { + expect(() => new WorkerObservability({ windowMs: invalid })).toThrow(RangeError); + expect(() => new WorkerObservability({ logIntervalMs: invalid })).toThrow(RangeError); + }); + + it("logs at most once per interval, flushes final once, and removes its unref timer", () => { + vi.useFakeTimers(); vi.setSystemTime(at); + const write = vi.fn(); const before = vi.getTimerCount(); const metrics = new WorkerObservability({ write }); + metrics.start(); metrics.start(); expect(vi.getTimerCount()).toBe(before + 1); + metrics.flush(); expect(write).not.toHaveBeenCalled(); + vi.advanceTimersByTime(30_000); expect(write).toHaveBeenCalledTimes(1); + metrics.stop(); metrics.stop(); expect(write).toHaveBeenCalledTimes(2); + expect(write.mock.calls[1]![0]).toMatchObject({ final: true }); expect(vi.getTimerCount()).toBe(before); + vi.advanceTimersByTime(60_000); expect(write).toHaveBeenCalledTimes(2); + }); + + it("isolates snapshot, writer and observer failures from business execution", async () => { + const metrics = new WorkerObservability({ queueSnapshot: () => { throw Error("private body"); }, write: () => { throw Error("private key"); } }); + expect(asSnapshot(metrics)).toMatchObject({ queueSnapshotError: true, queueSnapshotAvailable: false, + lanes: { import: { queued: null, dueTimeOnly: null } } }); expect(() => metrics.stop()).not.toThrow(); + const c = new WorkerConcurrencyController({}, {}); c.observe({ record: () => { throw Error("observer failed"); } }); + const { router } = routed({ record: () => { throw Error("observer failed"); } }, c); + const permit = c.tryAcquire(req())!; expect(permit).toBeDefined(); permit.release(); + await expect(router.embedder().embed(["safe synthetic text"])).resolves.toEqual([[1, 0, 0]]); + expect(() => observeWorker({ record: () => { throw Error("observer failed"); } }, { type: "started", work })).not.toThrow(); + await Promise.all([router.stop(), c.close()]); expect(c.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, embeddingRequests: 0 }); + }); + + it("records lease transitions separately from claims with a historical recovery count", () => { + const metrics = new WorkerObservability({ now: () => at }); + metrics.record({ type: "claimed", work: { ...work, queue: "embedding_retry", leaseCount: 8, leaseRecoveryCount: 3 } }); + for (const action of ["renewed", "renew_failed", "fenced", "expired", "recovered"] as const) metrics.record({ type: "lease", queue: "embedding_retry", action }); + const counters = asSnapshot(metrics).counters; + expect(counters["queue.embedding_retry.claimed"]).toBe(1); expect(counters["queue.embedding_retry.claims_with_recovery_history"]).toBe(1); + expect(counters["lease.embedding_retry.recovered"]).toBe(1); + }); + + it("aggregates finite selection reasons only from committed claim observations", () => { + const metrics = new WorkerObservability(); + for (const selectionReason of ["wrr", "aging", "interactive", "legacy"] as const) { + metrics.record({ type: "claimed", work: { ...work, selectionReason } }); + metrics.record({ type: "started", work: { ...work, selectionReason } }); + } + const counters = asSnapshot(metrics).counters; + for (const reason of ["wrr", "aging", "interactive", "legacy"]) expect(counters[`selection.import.${reason}`]).toBe(1); + expect(counters["lane.import.claimed"]).toBe(4); + }); + + it.each([ + ["global", { maxActiveJobs: 1 }, req(), req()], + ["resource", { resourceLimits: { summary: 1 } }, req(), req()], + ["scope", {}, req({ scopeKeys: ["secret-scope"] }), req({ scopeKeys: ["secret-scope"] })], + ["provider", {}, req({ providers: [{ key: "secret-provider", limit: 1 }] }), req({ providers: [{ key: "secret-provider", limit: 1 }] })], + ["embedding_request", {}, req({ embedding: true }), req({ embedding: true })] + ] as const)("reports exact %s denial without holding partial permits or private identities", async (reason, options, first, second) => { + const c = new WorkerConcurrencyController(options, {}), metrics = new WorkerObservability(); c.observe(metrics); + const permit = c.tryAcquire(first)!; expect(c.tryAcquire(second)).toBeUndefined(); let result = asSnapshot(metrics); + expect(result.counters[`admission_rejected.${reason}`]).toBe(1); + expect(result.concurrency.activeJobs).toBe(1); expect(result.concurrency.peaks.global).toBe(1); + expect(JSON.stringify(result)).not.toMatch(/secret-scope|secret-provider/); + permit.release(); await c.close(); result = asSnapshot(metrics); + expect(result.concurrency.activeJobs).toBe(0); expect(result.concurrency.resources).toMatchObject({ summary: 0 }); + expect(result.concurrency.maxActiveJobs).toBe(c.limits.maxActiveJobs); + }); + + it("reports batch, invalid sharing and closing as separate rejection reasons", async () => { + const c = new WorkerConcurrencyController({ embeddingBatchSize: 1 }, {}), events: WorkerObservation[] = []; c.observe({ record: event => events.push(event) }); + const permit = c.tryAcquire(req({ resourceClass: "embedding", embedding: true }))!; + expect(c.tryAcquire(req({ resourceClass: "embedding", embedding: true }), 1, permit)).toBeUndefined(); + expect(c.tryAcquire(req(), 1, permit)).toBeUndefined(); permit.release(); await c.close(); expect(c.tryAcquire(req())).toBeUndefined(); + expect(events.filter(event => event.type === "admission_rejected").map(event => event.reason)).toEqual(["batch", "share_invalid", "closing"]); + }); + + it("keeps active occupancy in the window peak even without transitions during a long-running job", async () => { + let now = at; const metrics = new WorkerObservability({ now: () => now }), c = new WorkerConcurrencyController({}, {}); + c.observe(metrics); const permit = c.tryAcquire(req())!; + now += 310_001; + expect(asSnapshot(metrics).concurrency).toMatchObject({ activeJobs: 1, peaks: { global: 1, "resource.summary": 1 }, + utilization: { global: 0.25, resources: { summary: 0.25 } } }); + permit.release(); await c.close(); + }); + + it("bounds provider labels across repeated endpoint changes, including the rolling peak map", async () => { + const metrics = new WorkerObservability(), c = new WorkerConcurrencyController({}, {}); c.observe(metrics); + for (let i = 0; i < 100; i++) { + const permit = c.tryAcquire(req({ providers: [{ key: `private-endpoint-${i}`, limit: 2 }] }))!; + permit.release(); + } + const snapshot = asSnapshot(metrics); + expect(Object.keys(snapshot.concurrency.peaks).filter(key => key.startsWith("provider."))).toHaveLength(33); + expect(JSON.stringify(snapshot)).not.toContain("private-endpoint"); await c.close(); + }); + + it("starts model latency only after physical admission, then records failures without error messages", async () => { + let clock = 0; vi.spyOn(performance, "now").mockImplementation(() => clock); + const metrics = new WorkerObservability(), c = new WorkerConcurrencyController({}, {}); const { router, context } = routed(metrics, c); + let finish!: () => void; const held = new Promise(resolve => { finish = resolve; }); let calls = 0; + context.summary.complete = async () => { calls++; if (calls === 1) { await held; clock += 5; return "synthetic"; } + clock += 7; throw new ModelHttpError("private response and credential", "host", 429, undefined, "private detail"); }; + const descriptor = providerDescriptor(context.summary.config, c.limits); + const p = c.tryAcquire(req({ providers: descriptor.providers.map(provider => ({ ...provider, limit: 1 })) }))!; + const first = p.run(() => router.runWith(context, () => router.client("summary").complete([], { operation: "isolated" }))); + await Promise.resolve(); await Promise.resolve(); + const second = router.client("summary").complete([], { operation: "isolated" }); const rejected = expect(second).rejects.toMatchObject({ httpStatus: 429 }); + clock = 1_000; finish(); await first; p.release(); await rejected; + const result = asSnapshot(metrics), model = result.models[0]; + expect(model.latencyMs).toMatchObject({ sampleCount: 2, p50: 7, p95: 1_005 }); expect(model.rateLimited).toBe(1); + expect(model.failed).toBe(1); expect(model.reportedTokens).toBeNull(); expect(JSON.stringify(result)).not.toMatch(/private response|private detail|credential/); + await Promise.all([router.stop(), c.close()]); + }); + + it("records per-slice batch sizes, reported tokens and timeout outcomes", async () => { + const metrics = new WorkerObservability(), c = new WorkerConcurrencyController({}, {}); const { router, context } = routed(metrics, c); + context.embedding.embed = async texts => texts.map(() => [1]); + await router.embedder().embed(Array.from({ length: 37 }, () => "private text"), "query"); const descriptor = providerDescriptor(context.embedding.config, c.limits); + metrics.record({ type: "tokens", providerKey: descriptor.providers[0]!.key, modelKey: descriptor.providers.at(-1)!.key, + kind: "embedding", inputTokens: 12, outputTokens: 0, totalTokens: 12 }); + context.embedding.embedOne = async () => { throw new DOMException("private endpoint", "TimeoutError"); }; + await expect(router.embedder().embedOne("private text")).rejects.toMatchObject({ name: "TimeoutError" }); const result = asSnapshot(metrics); + expect(result.models[0]).toMatchObject({ requests: 4, usageReports: 1, reportedTokens: { input: 12, output: 0, total: 12 }, + timedOut: 1, embeddingBatchSize: { sampleCount: 4, p50: 5, p95: 16 } }); expect(JSON.stringify(result)).not.toMatch(/private text|private endpoint/); + await Promise.all([router.stop(), c.close()]); + }); + + it("reads queue counts and oldest timestamps without hydrating bodies or claiming blocked jobs", async () => { + const root = fixture.createTestRoot(), db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const service = fixture.createTestMemoryService({ db, observability: new WorkerObservability({ write: () => {} }) }); const repos = new Repositories(db.db); + const job = repos.runtime.enqueueJob({ id: "isolated", jobType: "import_summary", userId: "isolated", status: "queued", payload: {}, + attempts: 0, maxAttempts: 3, createdAt: new Date(at).toISOString(), updatedAt: new Date(at).toISOString() }); + db.db.prepare("UPDATE evolution_jobs SET blocked_reason='configuration', available_at=? WHERE id=?").run(new Date(at).toISOString(), job.id); + const retry = repos.runtime.enqueueEmbeddingRetry({ id: "isolated-retry", targetKind: "trace", targetId: "missing-target", + vectorField: "vec_summary", sourceText: "PRIVATE_LARGE_BODY", now: at + 60_000 }); + const get = vi.spyOn(repos.memories, "get").mockImplementation(() => { throw Error("must not hydrate"); }); + const prepare = vi.spyOn(db.db, "prepare"); const result = readWorkerQueueMetrics(db.db, at); + expect(prepare.mock.calls.map(([sql]) => sql).join(" ")).not.toMatch(/memory_value|source_text|payload_json|properties_json|embedding_json/); + expect(result.find(row => row.lane === job.queueLane)).toMatchObject({ queued: 1, dueTimeOnly: 1, oldestWaitingAt: job.firstQueuedAt }); + expect(result.find(row => row.lane === retry.queueLane)).toMatchObject({ queued: 1, dueTimeOnly: 0 }); expect(get).not.toHaveBeenCalled(); + expect(JSON.stringify(result)).not.toContain("PRIVATE_LARGE_BODY"); expect(repos.runtime.getJob(job.id)).toMatchObject({ status: "queued", leaseToken: null }); await service.stop(); + }); + + it("collects actual client usage through MemoryService and flushes before the DB closes", async () => { + const root = fixture.createTestRoot(), db = new MemoryDb({ path: join(root, "memory.sqlite") }); const writes: Record[] = []; + const metrics = new WorkerObservability({ queueSnapshot: at => readWorkerQueueMetrics(db.db, at), write: snapshot => writes.push(snapshot) }); + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.summary = { ...config.summary, provider: "openai_compatible", sourceProvider: undefined, endpoint: "https://isolated.test/v1", + model: "isolated-summary", apiKey: "PRIVATE_TEST_KEY", maxRetries: 0, malformedRetries: 0 }; config.algorithm.capture.embedAfterCapture = false; + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ choices: [{ message: { content: JSON.stringify({ + title: "Synthetic build", summary: "Use a reproducible build pipeline." }) }, finish_reason: "stop" }], + usage: { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 } }), { status: 200, headers: { "content-type": "application/json" } }))); + const service = fixture.createTestMemoryService({ db, config, observability: metrics }); + service.addMemory({ namespace: { userId: "isolated", source: "codex", profileId: "default" }, adapterId: "agent-source:codex", + requestId: "isolated", layer: "L1", source: "codex", tags: ["agent-source", "codex"], turnId: "isolated-turn", + content: "## user\n\nUse a reproducible build pipeline.\n\n## assistant\n\nPipeline documented." }); + expect(await service.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); const snapshot = asSnapshot(metrics); + expect(snapshot.models).toEqual(expect.arrayContaining([expect.objectContaining({ kind: "summary", requests: 1, + usageReports: 1, reportedTokens: { input: 11, output: 7, total: 18 } })])); + expect(JSON.stringify(snapshot)).not.toMatch(/PRIVATE_TEST_KEY|isolated.test|reproducible build/); await service.stop(); + expect(writes).toHaveLength(1); expect(writes[0]).toMatchObject({ final: true, queueSnapshotError: false }); + expect(asSnapshot(metrics).concurrency.activeJobs).toBe(0); + }); + + it("fixture cleanup waits for services and only closes the resources captured before its await", async () => { + const local = createMemoryServiceFixture(); + const first = local.createTestService(); + const stop = first.service.stop.bind(first.service); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + vi.spyOn(first.service, "stop").mockImplementation(async () => { await stop(); await gate; }); + const cleaning = local.cleanup(); + const next = local.createTestService(); + try { + await Promise.resolve(); expect(first.db.db.open).toBe(true); expect(next.db.db.open).toBe(true); + release(); await cleaning; + expect(first.db.db.open).toBe(false); expect(next.db.db.open).toBe(true); + } finally { release(); await cleaning; await local.cleanup(); } + }); + + it("fixture cleanup reports a failed stop after closing its captured resources", async () => { + const local = createMemoryServiceFixture(), first = local.createTestService(); + const stop = first.service.stop.bind(first.service), failure = new Error("isolated stop failure"); + vi.spyOn(first.service, "stop").mockImplementation(async () => { await stop(); throw failure; }); + await expect(local.cleanup()).rejects.toBe(failure); expect(first.db.db.open).toBe(false); + }); + + it("fixture cleanup still stops other services if one stop throws synchronously", async () => { + const local = createMemoryServiceFixture(), first = local.createTestService(), next = local.createTestService(); + await first.service.stop(); const failure = new Error("isolated synchronous stop failure"); + vi.spyOn(first.service, "stop").mockImplementation(() => { throw failure; }); + const nextStop = vi.spyOn(next.service, "stop"); + await expect(local.cleanup()).rejects.toBe(failure); + expect(nextStop).toHaveBeenCalledOnce(); expect(first.db.db.open).toBe(false); expect(next.db.db.open).toBe(false); + }); + + it("fixture shutdown completes with fake timers without requiring clock advancement", async () => { + vi.stubEnv("MEMMY_LOG_LEVEL", "silent"); vi.useFakeTimers(); vi.setSystemTime(at); + const timers = vi.getTimerCount(), local = createMemoryServiceFixture(); local.createTestService(); + await local.cleanup(); expect(Date.now()).toBe(at); + // RuntimeRepository schedules a pre-existing best-effort compaction + // immediate on construction. It notices the closed DB and returns; this + // zero-time turn must not hide any leaked 30s metrics interval. + await vi.advanceTimersByTimeAsync(0); + expect(Date.now()).toBe(at); expect(vi.getTimerCount()).toBe(timers); + }); +}); diff --git a/Memory/tests/service/worker/worker-policy-backlog-benchmark.test.ts b/Memory/tests/service/worker/worker-policy-backlog-benchmark.test.ts new file mode 100644 index 000000000..aef746dea --- /dev/null +++ b/Memory/tests/service/worker/worker-policy-backlog-benchmark.test.ts @@ -0,0 +1,71 @@ +import { appendFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +// Opt-in comparison: fixed work budget, varying backlog, real SQLite and scheduler. +// Handlers deliberately do no model/network work. These are NOT model throughput figures. +it.skipIf(process.env.MEMMY_QUEUE_POLICY_BENCHMARK !== "1")("benchmarks 1,000 completions against classified backlogs", async () => { + for (const backlog of [1_000, 10_000, 50_000]) { + const root = mkdtempSync(join(tmpdir(), "memmy-policy-benchmark-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db); + const concurrency = new WorkerConcurrencyController({ maxActiveJobs: 4 }, {}); + const processed: string[] = []; + const unexpected = () => { throw new Error("Unexpected model or startup operation"); }; + const deps: WorkerRunnerDeps = { + repos, concurrency, scheduling: { algorithm: "wrr", aging: { enabled: false } }, schedulingShadow: { enabled: false }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: false }, embeddingRetryWorkerId: "benchmark", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "benchmark", runWorkerNoWrite: async () => unexpected(), + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: unexpected, enqueueEmbeddingRetry: unexpected, appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async job => { processed.push(job.id); } }, + embeddingJobs: { prepareEmbeddingJob: unexpected, applyEmbeddingVector: unexpected, + persistEmbeddingVector: unexpected, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); + const delay = monitorEventLoopDelay({ resolution: 10 }); + let policyMs = 0, policyCalls = 0; + const original = repos.runtime.scheduling.refreshActivePolicies.bind(repos.runtime.scheduling); + const spy = vi.spyOn(repos.runtime.scheduling, "refreshActivePolicies").mockImplementation(input => { + const start = performance.now(); + try { return original(input); } finally { policyMs += performance.now() - start; policyCalls++; } + }); + try { + const at = new Date().toISOString(); + const insert = db.db.prepare(`INSERT INTO evolution_jobs(id,job_type,status,user_id,payload_json,created_at,updated_at) + VALUES (?,'l2_association','queued','benchmark','{}',?,?)`); + repos.transaction(() => { + for (let i = 0; i < backlog; i++) insert.run(`job-${i}`, at, at); + while (repos.runtime.scheduling.refreshPolicies(500).refreshed > 0) { /* setup excluded from timing */ } + }); + expect(repos.runtime.scheduling.pendingPolicyCount()).toBe(0); + delay.enable(); + const start = performance.now(); + const result = await runner.runWorkerOnce(1_000); + const elapsedMs = performance.now() - start; + delay.disable(); + expect(result.jobs).toHaveLength(1_000); + expect(processed).toEqual(Array.from({ length: 1_000 }, (_, i) => `job-${i}`)); + expect(result.jobs.every(job => job.status === "succeeded")).toBe(true); + const sample = JSON.stringify({ backlog, completed: processed.length, + elapsedMs: Math.round(elapsedMs), policyCalls, policyMs: Math.round(policyMs), + eventLoopP99Ms: Math.round(delay.percentile(99) / 1e6) }); + if (process.env.MEMMY_QUEUE_POLICY_BENCHMARK_OUTPUT) appendFileSync(process.env.MEMMY_QUEUE_POLICY_BENCHMARK_OUTPUT, `${sample}\n`); + console.log("POLICY_BENCHMARK", sample); + } finally { + delay.disable(); spy.mockRestore(); await runner.stop(); await concurrency.close(); db.close(); + rmSync(root, { recursive: true, force: true }); + } + } +}, 120_000); diff --git a/Memory/tests/service/worker/worker-provider-lane-fairness.test.ts b/Memory/tests/service/worker/worker-provider-lane-fairness.test.ts new file mode 100644 index 000000000..4abb18286 --- /dev/null +++ b/Memory/tests/service/worker/worker-provider-lane-fairness.test.ts @@ -0,0 +1,181 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { providerDescriptor, WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +type ModelLane = "import" | "evolution"; +const providerCompletions = 60; +const waveSize = 20; + +/** Real claims and timers: fast database work must not reset a busy model's turn. */ +async function runSharedProvider(options: { + delayMs: number; + replenishMaintenance: boolean; + waitForMaintenance?: boolean; + cloud?: boolean; + blockedInteractive?: boolean; + blockedInteractiveScope?: boolean; +}) { + const root = mkdtempSync(join(tmpdir(), "memmy-provider-lane-fairness-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db); + const concurrency = new WorkerConcurrencyController({ maxActiveJobs: 4 }, {}); + const abort = new AbortController(); + const starts: ModelLane[] = []; + const completed: ModelLane[] = []; + const descriptor = providerDescriptor({ provider: "openai_compatible", model: "shared-model", + endpoint: options.cloud ? "https://shared.example.test/v1" : "http://127.0.0.1:1234/v1", + mode: options.cloud ? "cloud" : "local" }, concurrency.limits); + const providers = descriptor.providers; + const providerLimit = options.cloud ? 2 : 1; + const blockedProvider = [{ key: "independent-blocked-model", limit: 1 }]; + const held = options.blockedInteractiveScope + ? concurrency.tryAcquire({ queueLane: "interactive", resourceClass: "summary", providers: [], scopeKeys: [JSON.stringify(["target", "busy-scope"])] })! + : options.blockedInteractive + ? concurrency.tryAcquire({ queueLane: "interactive", resourceClass: "summary", providers: blockedProvider, scopeKeys: [] })! : undefined; + const maintenanceWaiters = new Set<() => void>(); + const wakeMaintenance = () => { for (const wake of maintenanceWaiters) wake(); maintenanceWaiters.clear(); }; + abort.signal.addEventListener("abort", wakeMaintenance); + let maintenanceCompleted = 0, maintenanceSequence = 0, activeModelRequests = 0, peakModelRequests = 0; + const unexpected = () => { throw new Error("Unexpected startup or embedding operation"); }; + function enqueue(id: string, jobType: EvolutionJobRecord["jobType"]) { + const at = new Date().toISOString(); + repos.runtime.enqueueJob({ id, jobType, userId: "provider-fairness", episodeId: `episode-${id}`, + targetMemoryId: options.blockedInteractiveScope && jobType === "trace_summary" ? "busy-scope" : `memory-${id}`, + payload: {}, status: "queued", attempts: 0, maxAttempts: 3, + createdAt: at, updatedAt: at }); + } + function enqueueMaintenance() { enqueue(`maintenance-${maintenanceSequence++}`, "episode_idle_close"); } + function enqueueBlockedInteractive() { enqueue(`interactive-${maintenanceSequence}`, "trace_summary"); } + const deps: WorkerRunnerDeps = { + repos, concurrency, scheduling: { algorithm: "wrr" }, schedulingShadow: { enabled: false }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: false }, embeddingRetryWorkerId: "provider-fairness", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "provider-fairness", runWorkerNoWrite: async () => unexpected(), + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: unexpected, enqueueEmbeddingRetry: unexpected, appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + prepareExecution: work => ({ + providers: work.queue === "main" && work.job.jobType === "episode_idle_close" ? [] + : work.queue === "main" && work.job.jobType === "trace_summary" && !options.blockedInteractiveScope ? blockedProvider : providers, + run: operation => operation() + }), + jobHandlers: { processJob: async job => { + if (job.jobType === "episode_idle_close") { + maintenanceCompleted++; + wakeMaintenance(); + if (!abort.signal.aborted) { + enqueueMaintenance(); + if (options.blockedInteractive || options.blockedInteractiveScope) enqueueBlockedInteractive(); + } + return; + } + if (job.jobType === "trace_summary") unexpected(); + const lane: ModelLane = job.jobType === "import_summary" ? "import" : "evolution"; + const maintenanceAtStart = maintenanceCompleted; + starts.push(lane); + activeModelRequests++; + peakModelRequests = Math.max(peakModelRequests, activeModelRequests); + try { + await new Promise(resolve => setTimeout(resolve, options.delayMs)); + // Exercise the real interleaving that caused starvation: the next model + // permit becomes free after unrelated database work advanced its lane. + if (options.waitForMaintenance && maintenanceCompleted <= maintenanceAtStart && !abort.signal.aborted) { + await new Promise(resolve => maintenanceWaiters.add(resolve)); + } + completed.push(lane); + if (completed.length >= providerCompletions) abort.abort(); + } finally { activeModelRequests--; } + } }, + embeddingJobs: { prepareEmbeddingJob: unexpected, applyEmbeddingVector: unexpected, + persistEmbeddingVector: unexpected, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); + // Both queues remain nonempty throughout all measured Provider opportunities. + repos.transaction(() => { + for (let i = 0; i < 200; i++) { + enqueue(`import-${i}`, "import_summary"); + enqueue(`evolution-${i}`, "reflection"); + } + if (options.replenishMaintenance) enqueueMaintenance(); + if (options.blockedInteractive || options.blockedInteractiveScope) enqueueBlockedInteractive(); + }); + // A broken scheduler must still terminate and release every permit/lease. + const emergencyStop = setTimeout(() => abort.abort(), 5000); + try { + const result = await runner.runWorkerOnce(4, { priorityCohortOnly: true }, { continuous: true, signal: abort.signal }); + expect(result.failed).toBe(0); + expect(completed.length).toBeGreaterThanOrEqual(providerCompletions); + expect(completed.length).toBeLessThanOrEqual(providerCompletions + providerLimit - 1); + expect([...starts].sort()).toEqual([...completed].sort()); + expect(peakModelRequests).toBe(providerLimit); + expect(activeModelRequests).toBe(0); + held?.release(); + expect(concurrency.snapshot()).toEqual({ activeJobs: 0, availableJobs: 4, backgroundActiveJobs: 0, maxBackgroundJobs: 3, interactiveReservedSlots: 1, resources: {}, providers: {}, scopeKeys: [], embeddingRequests: 0 }); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE status='leased'").get()).toEqual({ n: 0 }); + const waiting = db.db.prepare("SELECT attempts,lease_token FROM evolution_jobs WHERE status='queued'").all() as Array<{ attempts: number; lease_token: string | null }>; + expect(waiting.every(row => row.attempts === 0 && row.lease_token === null)).toBe(true); + return { starts, completed, maintenanceCompleted }; + } finally { + clearTimeout(emergencyStop); + abort.abort(); + wakeMaintenance(); + await runner.stop(); + held?.release(); + await concurrency.close(); + abort.signal.removeEventListener("abort", wakeMaintenance); + db.close(); + rmSync(root, { recursive: true, force: true }); + } +} + +function expectRepeatedModelProgress(completed: readonly ModelLane[]) { + // Do not demand an exact WRR ratio or accept one lucky start at the beginning. + // Each successive set of real Provider completions must make both lanes progress. + for (let offset = 0; offset < providerCompletions; offset += waveSize) { + const wave = completed.slice(offset, offset + waveSize); + expect(wave, `Provider completions ${offset + 1}-${offset + waveSize}`).toContain("import"); + expect(wave, `Provider completions ${offset + 1}-${offset + waveSize}`).toContain("evolution"); + } +} + +describe("shared Provider lane fairness under fast database maintenance", () => { + it("makes repeated progress in both model lanes without maintenance as a control", async () => { + const result = await runSharedProvider({ delayMs: 10, replenishMaintenance: false }); + expect(result.maintenanceCompleted).toBe(0); + expectRepeatedModelProgress(result.completed); + }, 10_000); + + it.each([5, 10])("keeps both model lanes moving while maintenance replenishes with %s ms model work", async delayMs => { + const result = await runSharedProvider({ delayMs, replenishMaintenance: true }); + expect(result.maintenanceCompleted).toBeGreaterThan(0); + expectRepeatedModelProgress(result.completed); + }, 10_000); + + it("preserves model opportunities when a local request completes after maintenance", async () => { + const result = await runSharedProvider({ delayMs: 5, replenishMaintenance: true, waitForMaintenance: true }); + expect(result.maintenanceCompleted).toBeGreaterThan(0); + expectRepeatedModelProgress(result.completed); + }, 10_000); + + it.each([false, true])("keeps the cloud Provider moving with an independent blocked interactive Provider=%s", async blockedInteractive => { + const result = await runSharedProvider({ delayMs: 10, replenishMaintenance: true, + waitForMaintenance: true, cloud: true, blockedInteractive }); + expect(result.maintenanceCompleted).toBeGreaterThan(0); + expectRepeatedModelProgress(result.completed); + }, 10_000); + + it("skips a held business scope without withholding the shared cloud Provider from other lanes", async () => { + const result = await runSharedProvider({ delayMs: 10, replenishMaintenance: true, + waitForMaintenance: true, cloud: true, blockedInteractiveScope: true }); + expect(result.maintenanceCompleted).toBeGreaterThan(0); + expectRepeatedModelProgress(result.completed); + }, 10_000); +}); diff --git a/Memory/tests/service/worker/worker-reserved-slot-runner.test.ts b/Memory/tests/service/worker/worker-reserved-slot-runner.test.ts new file mode 100644 index 000000000..46a0ae32d --- /dev/null +++ b/Memory/tests/service/worker/worker-reserved-slot-runner.test.ts @@ -0,0 +1,232 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { workerWorkRequirements } from "../../../src/service/worker/worker-work-requirements.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +function gate() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} +async function until(condition: () => boolean) { + const deadline = Date.now() + 3000; + while (!condition()) { + if (Date.now() > deadline) throw new Error("Worker did not reach the expected admission state"); + await new Promise(resolve => setImmediate(resolve)); + } +} +const resources: Array<{ root: string; db: MemoryDb; runner: WorkerRunner }> = []; +function harness(process: WorkerRunnerDeps["jobHandlers"]["processJob"] = async () => {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-reserved-slot-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db), concurrency = new WorkerConcurrencyController({}, {}); + const unexpected = () => { throw new Error("Unexpected fixture dependency"); }; + const deps: WorkerRunnerDeps = { + repos, concurrency, scheduling: { algorithm: "wrr", aging: { enabled: false } }, + schedulingShadow: { enabled: false }, embedder: createCapturingEmbedder([]), + capture: { embedAfterCapture: true }, embeddingRetryWorkerId: "reserved-slot", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), summaryModelConfigured: () => true, + nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "reserved-slot", runWorkerNoWrite: async () => unexpected(), + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: unexpected, enqueueEmbeddingRetry: unexpected, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + prepareExecution: work => ({ providers: [], embedding: work.queue === "main" && work.job.jobType === "embedding", + run: operation => operation() }), + jobHandlers: { processJob: process }, + embeddingJobs: { prepareEmbeddingJob: unexpected, applyEmbeddingVector: unexpected, + persistEmbeddingVector: unexpected, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); + resources.push({ root, db, runner }); + function memory(id: string, tags: string[] = [], agentId?: string) { + const at = new Date().toISOString(); + db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,tags_json,agent_id,created_at,updated_at) + VALUES (?,?,'fixture','synthetic','L1',?,?,?,?)`).run(id, at, JSON.stringify(tags), agentId ?? null, at, at); + } + function job(id: string, jobType: EvolutionJobRecord["jobType"], targetMemoryId?: string, payload: Record = {}) { + const at = new Date().toISOString(); + return repos.runtime.enqueueJob({ id, jobType, targetMemoryId, userId: "fixture", payload, + status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + return { db, repos, concurrency, deps, runner, memory, job }; +} +afterEach(async () => { + for (const { root, db, runner } of resources.splice(0)) { + await runner.stop(); db.close(); rmSync(root, { recursive: true, force: true }); + } +}); + +describe("reserved interactive slot through real SQLite worker claims", () => { + it.each(["native", "onboarding"] as const)("starts %s while three background jobs remain active, without leasing blocked background rows", async kind => { + const background = gate(), interactive = gate(), abort = new AbortController(); + const entered: string[] = []; + const h = harness(async job => { + entered.push(job.id); + await (job.id === "fresh" ? interactive.promise : background.promise); + }); + h.job("import-0", "import_summary"); h.job("import-1", "import_summary"); h.job("reflection", "reflection"); + // More than one admission scan page; every row must retain its FIFO lease state. + for (let i = 0; i < 70; i++) { + h.memory(`history-${i}`, ["agent-source"]); + h.job(`history-${i}`, "trace_summary", `history-${i}`, { queueLane: "interactive" }); + } + h.job("startup", "import_summary", undefined, { source: "startup.placeholder_summary_repair", queueLane: "interactive" }); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + await until(() => entered.length >= 3); + // Give the pump a complete extra event-loop turn to try a fourth background row. + await new Promise(resolve => setImmediate(resolve)); + expect(entered).toHaveLength(3); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, availableJobs: 1 }); + h.memory("fresh", [], kind === "onboarding" ? "memmy-onboarding" : undefined); + h.job("fresh", kind === "onboarding" ? "import_summary" : "trace_summary", "fresh", + kind === "onboarding" ? { source: "startup.placeholder_summary_repair" } : {}); + h.runner.notifyWorkAvailable(); + await until(() => entered.includes("fresh")); + expect(entered).toHaveLength(4); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3 }); + expect(h.repos.runtime.getJob("fresh")).toMatchObject({ status: "leased", queueLane: "interactive", attempts: 1 }); + for (let i = 0; i < 70; i++) { + const row = h.repos.runtime.getJob(`history-${i}`)!; + expect(row.queueLane).toBe("import"); + if (!entered.includes(row.id)) expect(row).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } + expect(h.repos.runtime.getJob("startup")).toMatchObject({ status: "queued", queueLane: "maintenance", attempts: 0, leaseToken: null }); + } finally { + abort.abort(); background.resolve(); interactive.resolve(); await run; + } + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0, providers: {}, scopeKeys: [] }); + }); + + it("uses the persisted lane instead of re-inferring privilege from a job type or payload", () => { + const h = harness(); + h.memory("imported", ["agent-source"]); + h.memory("report", [], "memmy-onboarding"); + const records = [ + h.job("historical", "trace_summary", "imported", { queueLane: "interactive" }), + h.job("startup", "import_summary", undefined, { source: "startup.placeholder_summary_repair", queueLane: "interactive" }), + h.job("report", "import_summary", "report", { source: "startup.placeholder_summary_repair" }) + ]; + expect(records.map(job => workerWorkRequirements({ queue: "main", job }, h.repos).queueLane)) + .toEqual(["import", "maintenance", "interactive"]); + }); + + it("discovers interactive work inserted at a real scan-slice boundary while three background jobs stay active", async () => { + const background = gate(), interactive = gate(), abort = new AbortController(); + const entered: string[] = []; + const h = harness(async job => { + entered.push(job.id); + await (job.id === "fresh-during-scan" ? interactive.promise : background.promise); + }); + const backlog = 300; + for (let i = 0; i < 3; i++) h.job(`active-${i}`, "import_summary"); + for (let i = 0; i < backlog; i++) h.job(`blocked-${i}`, "import_summary"); + let rejected = 0, rejectedAtInsertion = -1, inserted = false; + let insertionError: unknown; + const stopObserving = h.concurrency.observe({ record: event => { + if (event.type !== "admission_rejected" || event.reason !== "background") return; + rejected++; + if (rejected !== 1) return; + // Admission is synchronous inside SQLite's claim transaction. This callback + // runs only once the current bounded slice has actually yielded to the loop. + setImmediate(() => { + try { + rejectedAtInsertion = rejected; + h.memory("fresh-during-scan"); + h.job("fresh-during-scan", "trace_summary", "fresh-during-scan"); + h.runner.notifyWorkAvailable(); + } catch (error) { insertionError = error; } + inserted = true; + }); + } }); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + await until(() => inserted); + expect(insertionError).toBeUndefined(); + expect(rejectedAtInsertion).toBeGreaterThan(0); + expect(rejectedAtInsertion).toBeLessThanOrEqual(64); + expect(rejectedAtInsertion).toBeLessThan(backlog); + await until(() => entered.includes("fresh-during-scan")); + expect(entered).toEqual(["active-0", "active-1", "active-2", "fresh-during-scan"]); + expect(rejected).toBeLessThan(backlog); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3 }); + for (let i = 0; i < backlog; i++) expect(h.repos.runtime.getJob(`blocked-${i}`)) + .toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { + stopObserving(); abort.abort(); background.resolve(); interactive.resolve(); await run; + } + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0 }); + }); + + it("wakes an exhausted background scan for newly enqueued interactive work without claiming a fourth background job", async () => { + const background = gate(), interactive = gate(), abort = new AbortController(); + const entered: string[] = []; + const h = harness(async job => { + entered.push(job.id); + await (job.id === "fresh-after-wait" ? interactive.promise : background.promise); + }); + const backlog = 130; + for (let i = 0; i < 3; i++) h.job(`active-${i}`, "import_summary"); + for (let i = 0; i < backlog; i++) h.job(`blocked-${i}`, "import_summary"); + let rejected = 0; + const stopObserving = h.concurrency.observe({ record: event => { + if (event.type === "admission_rejected" && event.reason === "background") rejected++; + } }); + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + await until(() => rejected === backlog); + // The last partial slice exhausts the queue. Several turns with no new + // rejection demonstrate that the pump is waiting rather than polling it. + for (let turn = 0; turn < 3; turn++) await new Promise(resolve => setImmediate(resolve)); + expect(rejected).toBe(backlog); + expect(entered).toEqual(["active-0", "active-1", "active-2"]); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, availableJobs: 1 }); + h.memory("fresh-after-wait"); + h.job("fresh-after-wait", "trace_summary", "fresh-after-wait"); + h.runner.notifyWorkAvailable(); + await until(() => entered.includes("fresh-after-wait")); + expect(entered).toEqual(["active-0", "active-1", "active-2", "fresh-after-wait"]); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 4, backgroundActiveJobs: 3 }); + expect(h.repos.runtime.getJob("fresh-after-wait")).toMatchObject({ status: "leased", attempts: 1 }); + for (let i = 0; i < backlog; i++) expect(h.repos.runtime.getJob(`blocked-${i}`)) + .toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { + stopObserving(); abort.abort(); background.resolve(); interactive.resolve(); await run; + } + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0 }); + }); + + it("limits a shared background embedding batch to three logical members and leaves other rows unleased", async () => { + const embedding = gate(), abort = new AbortController(), batches: string[][] = []; + const h = harness(); + h.deps.embeddingJobs.prepareEmbeddingJob = job => ({ job, memory: h.repos.memories.get(job.targetMemoryId!)!, + text: job.id, role: "document", vectorField: "vec_summary" }); + h.deps.embeddingJobs.applyEmbeddingVector = () => {}; + h.deps.embedder.embed = async texts => { + batches.push([...texts]); await embedding.promise; return texts.map(() => [1, 0, 0]); + }; + for (let i = 0; i < 8; i++) { + h.memory(`embedding-${i}`, ["agent-source"]); + h.job(`embedding-${i}`, "embedding", `embedding-${i}`); + } + const run = h.runner.runWorkerOnce(100, {}, { continuous: true, signal: abort.signal }); + try { + await until(() => batches.length > 0); + expect(batches).toHaveLength(1); expect(batches[0]).toHaveLength(3); + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 3, backgroundActiveJobs: 3, embeddingRequests: 1 }); + for (let i = 3; i < 8; i++) expect(h.repos.runtime.getJob(`embedding-${i}`)) + .toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { abort.abort(); embedding.resolve(); await run; } + expect(h.concurrency.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0, embeddingRequests: 0 }); + }); +}); diff --git a/Memory/tests/service/worker/worker-responses-provider.test.ts b/Memory/tests/service/worker/worker-responses-provider.test.ts new file mode 100644 index 000000000..c5922e8ea --- /dev/null +++ b/Memory/tests/service/worker/worker-responses-provider.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi, afterEach } from "vitest"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; +import { createLlmClient } from "../../../src/model/llm.js"; +import { MemoryModelTaskRouter } from "../../../src/model/task-routing.js"; +import { WorkerConcurrencyController, providerDescriptor } from "../../../src/service/worker/worker-concurrency.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +afterEach(() => vi.unstubAllGlobals()); +describe("Responses and Chat share actual Provider capacity after upstream protocol integration", () => { + it.each([undefined, "https://api.openai.com/v1", "https://api.openai.com/v1/responses", "https://api.openai.com/v1/responses/"])("normalizes the Responses endpoint %s to the actual shared service", endpoint => { + const chat = providerDescriptor({ provider: "openai_compatible", model: "same" }); + expect(providerDescriptor({ provider: "openai_responses", model: "same", endpoint }).providers).toEqual(chat.providers); + expect(providerDescriptor({ provider: "openai_responses", endpoint: "https://proxy.test/other/v1/responses", model: "same" }).providers[0]) + .not.toEqual(chat.providers[0]); + }); + + it("caps mixed real protocol calls at cloud concurrency instead of creating independent pools", async () => { + const common = { ...DEFAULT_MEMMY_CONFIG.summary, apiKey: "synthetic-key", endpoint: undefined, model: "synthetic", + maxRetries: 0, malformedRetries: 0 }; + const summary = createLlmClient({ ...common, provider: "openai_responses" }); + const evolution = createLlmClient({ ...common, provider: "openai_compatible" }); + const c = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => ({ config: DEFAULT_MEMMY_CONFIG, summary, evolution, embedding: createCapturingEmbedder([]) }), c); + let active = 0, peak = 0; + const urls: string[] = []; + vi.stubGlobal("fetch", async (input: unknown) => { + const url = String(input); urls.push(url); peak = Math.max(peak, ++active); + await new Promise(resolve => setImmediate(resolve)); active--; + return new Response(JSON.stringify(url.endsWith("/responses") + ? { status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }] } + : { choices: [{ message: { content: "OK" }, finish_reason: "stop" }] }), { status: 200 }); + }); + try { + await Promise.all((["summary", "evolution", "summary", "evolution"] as const).map(role => + router.client(role).complete([{ role: "user", content: "synthetic" }], { operation: "test.protocol-admission" }))); + expect(urls.filter(url => url.endsWith("/responses"))).toHaveLength(2); + expect(urls.filter(url => url.endsWith("/chat/completions"))).toHaveLength(2); + expect(peak).toBe(2); + expect(c.snapshot()).toMatchObject({ activeJobs: 0, backgroundActiveJobs: 0, providers: {} }); + } finally { await router.stop(); await c.close(); } + }); +}); diff --git a/Memory/tests/service/worker/worker-runner-observability.test.ts b/Memory/tests/service/worker/worker-runner-observability.test.ts new file mode 100644 index 000000000..c1f507389 --- /dev/null +++ b/Memory/tests/service/worker/worker-runner-observability.test.ts @@ -0,0 +1,140 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { WorkerConcurrencyController } from "../../../src/service/worker/worker-concurrency.js"; +import type { WorkerObservation } from "../../../src/service/worker/worker-observability.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ root: string; db: MemoryDb; runner: WorkerRunner }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "worker-observer-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db), events: WorkerObservation[] = []; + const concurrency = new WorkerConcurrencyController({}, {}); + const unexpected = () => { throw new Error("Unexpected fixture operation"); }; + const deps: WorkerRunnerDeps = { + repos, concurrency, observer: { record: event => events.push(event) }, + embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: false }, embeddingRetryWorkerId: "observer", + memoryAddEnabled: () => true, memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "fixture", runWorkerNoWrite: async () => unexpected(), + restartFailedProcessing: () => 0, reconcileOrphanedPolicies: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + enqueueJob: unexpected, enqueueEmbeddingRetry: unexpected, appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + schedulingShadow: { enabled: false }, jobHandlers: { processJob: async () => {} }, + embeddingJobs: { prepareEmbeddingJob: unexpected, applyEmbeddingVector: unexpected, + persistEmbeddingVector: unexpected, enqueueEmbeddingRetryAfterFailure: () => false } + }; + const runner = new WorkerRunner(deps); resources.push({ root, db, runner }); + function job(id = "private-id") { + const at = new Date().toISOString(); + return repos.runtime.enqueueJob({ id, jobType: "reflection", userId: "private-user", targetMemoryId: "private-target", + payload: { body: "private-content" }, status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + return { db, repos, runner, deps, events, concurrency, job }; +} +afterEach(async () => { + vi.restoreAllMocks(); + for (const { root, db, runner } of resources.splice(0)) { await runner.stop(); db.close(); rmSync(root, { recursive: true, force: true }); } +}); + +describe("real worker lifecycle observations", () => { + it("reports committed claims, actual starts and successes without body or identity fields", async () => { + const h = harness(); h.job(); + expect((await h.runner.runWorkerOnce(1)).succeeded).toBe(1); + expect(h.events.map(event => event.type)).toEqual(["claimed", "started", "finished"]); + expect(h.events[0]).toMatchObject({ type: "claimed", work: { lane: "evolution", queue: "main", leaseCount: 1, leaseRecoveryCount: 0 } }); + expect(h.events[1]).toMatchObject({ type: "started", work: { firstQueuedAt: expect.any(String) } }); + expect(h.events[2]).toMatchObject({ type: "finished", outcome: "succeeded" }); + const serialized = JSON.stringify(h.events); + for (const secret of ["private-id", "private-user", "private-target", "private-content"]) expect(serialized).not.toContain(secret); + }); + + it("separates business failure from a fenced old executor", async () => { + const h = harness(); h.job("failure"); + h.deps.jobHandlers.processJob = async () => { throw new Error("unsupported job type"); }; + expect((await h.runner.runWorkerOnce(1)).failed).toBe(1); + expect(h.events.filter(event => event.type === "finished")).toMatchObject([{ outcome: "failed" }]); + h.events.length = 0; h.job("fenced"); + h.deps.jobHandlers.processJob = async job => { + h.db.db.prepare("UPDATE evolution_jobs SET lease_token='replacement-token' WHERE id=?").run(job.id); + }; + expect((await h.runner.runWorkerOnce(1)).failed).toBe(0); + expect(h.events.filter(event => event.type === "finished")).toMatchObject([{ outcome: "fenced" }]); + expect(h.events.filter(event => event.type === "lease")).toEqual([{ type: "lease", queue: "main", action: "fenced" }]); + expect(h.repos.runtime.getJob("fenced")).toMatchObject({ status: "leased", leaseToken: "replacement-token" }); + }); + + it("records a known scope refusal without guessing a Provider refusal", async () => { + const h = harness(); h.job(); + const held = h.concurrency.tryAcquire({ resourceClass: "scan", providers: [], scopeKeys: [JSON.stringify(["target", "private-target"])] })!; + const controller = new AbortController(); + const abort = setTimeout(() => controller.abort(), 10); + try { expect((await h.runner.runWorkerOnce(1, {}, { signal: controller.signal })).leased).toBe(0); } + finally { clearTimeout(abort); held.release(); } + expect(h.events).toContainEqual({ type: "skipped", lane: "evolution", reason: "scope" }); + expect(h.events.some(event => event.type === "claimed")).toBe(false); + expect(h.events.some(event => event.type === "skipped" && event.reason === "provider")).toBe(false); + expect(h.repos.runtime.getJob("private-id")).toMatchObject({ status: "queued", attempts: 0 }); + }); + + it("does not let a failing metrics sink change a successful durable result", async () => { + const h = harness(); h.job(); h.deps.observer = { record() { throw new Error("metrics unavailable"); } }; + expect((await h.runner.runWorkerOnce(1)).succeeded).toBe(1); + expect(h.repos.runtime.getJob("private-id")?.status).toBe("succeeded"); + expect(h.concurrency.availableJobs()).toBe(4); + }); + + it("does not report a claim rolled back by a failed cursor commit", async () => { + const h = harness(); h.job(); + vi.spyOn(h.repos.runtime, "setKv").mockImplementationOnce(() => { throw new Error("cursor storage failed"); }); + await expect(h.runner.runWorkerOnce(1)).rejects.toThrow("cursor storage failed"); + expect(h.events).toEqual([]); + expect(h.repos.runtime.getJob("private-id")).toMatchObject({ status: "queued", attempts: 0 }); + expect(h.concurrency.availableJobs()).toBe(4); + }); + + it("reports an expired lease recovery only on its successful replacement claim", async () => { + const h = harness(); h.job(); + const oldAt = new Date(Date.now() - 120_000).toISOString(); + h.repos.runtime.claimQueuedJob("private-id", { at: oldAt, leaseSeconds: 1 }); + expect((await h.runner.runWorkerOnce(1)).succeeded).toBe(1); + expect(h.events.filter(event => event.type === "lease")).toEqual([ + { type: "lease", queue: "main", action: "expired", count: 1 }, + { type: "lease", queue: "main", action: "recovered", count: 1 } + ]); + expect(h.events.find(event => event.type === "claimed")).toMatchObject({ work: { leaseCount: 2, leaseRecoveryCount: 1 } }); + }); + + it("distinguishes a committed claim returned by a budget pause from actual execution", async () => { + const h = harness(); h.job(); + let paused = false; + h.deps.memoryBudgetPaused = () => paused; + h.deps.memoryBudgetJobConsumes = () => true; + h.deps.prepareExecution = () => { paused = true; return { providers: [], run: operation => operation() }; }; + const result = await h.runner.runWorkerOnce(1); + expect(result).toMatchObject({ leased: 1, succeeded: 0, failed: 0 }); + expect(h.events.filter(event => event.type === "claimed")).toHaveLength(1); + expect(h.events.filter(event => event.type === "started")).toHaveLength(0); + expect(h.events.filter(event => event.type === "finished")).toMatchObject([{ outcome: "released" }]); + expect(h.events).toContainEqual({ type: "skipped", lane: "evolution", reason: "budget" }); + expect(h.repos.runtime.getJob("private-id")).toMatchObject({ status: "queued", attempts: 0 }); + }); + + it("records retry claim/start/failure exactly once when a real retry targets missing data", async () => { + const h = harness(); + h.repos.runtime.enqueueEmbeddingRetry({ id: "retry-private", targetKind: "trace", targetId: "missing-private", + vectorField: "vec_summary", sourceText: "secret retry body", now: Date.now() }); + const result = await h.runner.runEmbeddingRetryOnce(1); + expect(result.failed).toBe(1); + expect(h.events.filter(event => event.type === "claimed")).toHaveLength(1); + expect(h.events.filter(event => event.type === "started")).toHaveLength(1); + expect(h.events.filter(event => event.type === "finished")).toMatchObject([{ outcome: "failed", work: { queue: "embedding_retry", lane: "repair_retry" } }]); + expect(JSON.stringify(h.events)).not.toContain("secret retry body"); + }); +}); diff --git a/Memory/tests/service/worker/worker-runtime.test.ts b/Memory/tests/service/worker/worker-runtime.test.ts index f8abf105e..2d12a96f9 100644 --- a/Memory/tests/service/worker/worker-runtime.test.ts +++ b/Memory/tests/service/worker/worker-runtime.test.ts @@ -68,7 +68,7 @@ describe("MemoryService / worker / runtime", () => { const leased = runtime.leaseQueuedJobs(10, 60, undefined, true).map((job) => job.id); expect(leased).toEqual(["job_normal_reflection_before_startup_repairs"]); - runtime.completeJob("job_normal_reflection_before_startup_repairs"); + runtime.completeJob("job_normal_reflection_before_startup_repairs", undefined, runtime.getJob("job_normal_reflection_before_startup_repairs")); const repairs = runtime.leaseQueuedJobs(10, 60, undefined, true); expect(repairs).toHaveLength(2); expect(repairs.every((job) => job.jobType === "import_summary")).toBe(true); @@ -118,7 +118,7 @@ describe("MemoryService / worker / runtime", () => { expect(firstLease.map((job) => job.id).sort()).toEqual(["contract-1", "knowledge-1"]); expect(repos.runtime.leaseQueuedJobs(10, 60)).toEqual([]); - repos.runtime.completeJob("contract-1"); + repos.runtime.completeJob("contract-1", undefined, firstLease.find((job) => job.id === "contract-1")); expect(repos.runtime.leaseQueuedJobs(10, 60).map((job) => job.id)).toEqual(["contract-2"]); db.close(); @@ -149,12 +149,12 @@ describe("MemoryService / worker / runtime", () => { } expect(repos.runtime.leaseQueuedJobs(10, 60).map((job) => job.id)).toEqual(["field-1"]); - expect(repos.runtime.failJob("field-1", "retry")?.status).toBe("failed"); + expect(repos.runtime.failJob("field-1", "retry", undefined, false, repos.runtime.getJob("field-1"))?.status).toBe("failed"); expect(repos.runtime.leaseQueuedJobs(10, 60)).toEqual([]); repos.runtime.requeueFailedJobs(); expect(repos.runtime.leaseQueuedJobs(10, 60).map((job) => job.id)).toEqual(["field-1"]); - expect(repos.runtime.failJob("field-1", "terminal")?.status).toBe("dead_letter"); + expect(repos.runtime.failJob("field-1", "terminal", undefined, false, repos.runtime.getJob("field-1"))?.status).toBe("dead_letter"); expect(repos.runtime.leaseQueuedJobs(10, 60).map((job) => job.id)).toEqual(["field-2"]); db.close(); @@ -236,22 +236,26 @@ describe("MemoryService / worker / runtime", () => { }) ])); const row = db.db.prepare( - `SELECT status, attempts, leased_until + `SELECT status, attempts, leased_until, failure_count, lease_recovery_count FROM evolution_jobs WHERE id = ?` ).get(jobId) as { status: string; attempts: number; leased_until: string | null; + failure_count: number; + lease_recovery_count: number; }; expect(row.status).toBe("succeeded"); - expect(row.attempts).toBe(2); + expect(row.attempts).toBe(1); expect(row.leased_until).toBeNull(); + expect(row.failure_count).toBe(0); + expect(row.lease_recovery_count).toBe(1); db.close(); }); - it("persists retryable worker failures before requeueing them on the next tick", async () => { + it("dead-letters an unsupported job immediately instead of retrying it every tick", async () => { const { db, service } = createTestService(); const jobId = "job_retryable_failure"; const createdAt = new Date(Date.now() - 60_000).toISOString(); @@ -266,7 +270,7 @@ describe("MemoryService / worker / runtime", () => { expect(firstRun.jobs).toEqual([ expect.objectContaining({ jobId, - status: "failed" + status: "dead_letter" }) ]); const failedRow = db.db.prepare( @@ -279,18 +283,13 @@ describe("MemoryService / worker / runtime", () => { leased_until: string | null; last_error: string | null; }; - expect(failedRow.status).toBe("failed"); + expect(failedRow.status).toBe("dead_letter"); expect(failedRow.attempts).toBe(1); expect(failedRow.leased_until).toBeNull(); - expect(failedRow.last_error).toContain("unsupported job type"); + expect(failedRow.last_error).toContain("No scheduling policy registered for job type"); const secondRun = await service.runWorkerOnce(1); - expect(secondRun.jobs).toEqual([ - expect.objectContaining({ - jobId, - status: "failed" - }) - ]); + expect(secondRun.jobs).toEqual([]); const retriedRow = db.db.prepare( `SELECT status, attempts, leased_until FROM evolution_jobs @@ -300,8 +299,8 @@ describe("MemoryService / worker / runtime", () => { attempts: number; leased_until: string | null; }; - expect(retriedRow.status).toBe("failed"); - expect(retriedRow.attempts).toBe(2); + expect(retriedRow.status).toBe("dead_letter"); + expect(retriedRow.attempts).toBe(1); expect(retriedRow.leased_until).toBeNull(); const ops = db.db.prepare( `SELECT op @@ -309,7 +308,7 @@ describe("MemoryService / worker / runtime", () => { WHERE entity_id = ? ORDER BY seq ASC` ).all(jobId) as Array<{ op: string }>; - expect(ops.map((row) => row.op)).toEqual(["leased", "failed", "queued", "leased", "failed"]); + expect(ops.map((row) => row.op)).toEqual(["leased", "dead_letter"]); db.close(); }); @@ -345,7 +344,7 @@ describe("MemoryService / worker / runtime", () => { expect(row.status).toBe("dead_letter"); expect(row.attempts).toBe(1); expect(row.leased_until).toBeNull(); - expect(row.last_error).toContain("unsupported job type"); + expect(row.last_error).toContain("No scheduling policy registered for job type"); const ops = db.db.prepare( `SELECT op diff --git a/Memory/tests/service/worker/worker-scheduler-boundary.test.ts b/Memory/tests/service/worker/worker-scheduler-boundary.test.ts new file mode 100644 index 000000000..9b4d44450 --- /dev/null +++ b/Memory/tests/service/worker/worker-scheduler-boundary.test.ts @@ -0,0 +1,197 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { WorkerSchedulerReentryError, WorkerSchedulerStoppedError } from "../../../src/service/worker/worker-scheduler.js"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const resources: Array<{ root: string; db: MemoryDb }> = []; + +async function flushWorker() { for (let i = 0; i < 100; i++) await Promise.resolve(); } + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +/** Real SQLite and worker lifecycle; only model/business work is substituted. */ +function harness(processJob: WorkerRunnerDeps["jobHandlers"]["processJob"] = async () => {}) { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-entry-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const deps: WorkerRunnerDeps = { + repos, + embedder: createCapturingEmbedder([]), + capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "lease-safety-runner", + memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, + memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), + summaryModelConfigured: () => true, + nowIso: () => new Date().toISOString(), + encodeChangeCursor: String, + namespaceIdFromMemory: () => "lease-safety", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ + orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] + }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement retry"); }, + appendJobChange: () => {}, + appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding"); }, + applyEmbeddingVector: () => {}, + persistEmbeddingVector: (input) => { + const target = repos.memories.get(input.memoryId); + if (!target) throw new Error("missing synthetic target"); + input.finalize?.(target, false, new Date().toISOString()); + return target; + }, + enqueueEmbeddingRetryAfterFailure: () => false + } + }; + const runner = new WorkerRunner(deps); + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection"): void { + const at = new Date().toISOString(); + repos.runtime.enqueueJob({ + id, jobType, status: "queued", userId: "lease-safety-user", payload: {}, + attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at + }); + } + function retry(id: string): void { + const at = new Date().toISOString(); + db.db.prepare(`INSERT INTO memories + (id, timeline, user_id, memory_value, memory_layer, created_at, updated_at) + VALUES (?, ?, 'lease-safety-user', 'retry text', 'L1', ?, ?)`) + .run(id, at, at, at); + repos.runtime.enqueueEmbeddingRetry({ + id, targetKind: "trace", targetId: id, vectorField: "vec_summary", sourceText: "retry text", now: Date.now() + }); + } + return { db, repos, runner, job, retry, deps }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("unified worker execution boundary", () => { + it("keeps independent target requests queued without preclaiming their jobs", async () => { + const gate = deferred(); const started: string[] = []; + const h = harness(async (job) => { started.push(job.id); if (job.id === "A") await gate.promise; }); + for (const id of ["A", "B", "C"]) { + h.job(id, "trace_summary"); + h.db.db.prepare("UPDATE evolution_jobs SET target_memory_id = ? WHERE id = ?").run(id, id); + } + const a = h.runner.runWorkerOnce(1, { targetMemoryIds: ["A"], priorityCohortOnly: true }); + const b = h.runner.runWorkerOnce(1, { targetMemoryIds: ["B"] }); + const c = h.runner.runWorkerOnce(1, { targetMemoryIds: ["C"] }); + try { + await flushWorker(); + expect(started).toEqual(["A"]); + for (const id of ["B", "C"]) expect(h.repos.runtime.getJob(id)).toMatchObject({ status: "queued", attempts: 0, leaseToken: null }); + } finally { gate.resolve(); } + const results = await Promise.all([a, b, c]); + expect(results.map((r) => r.jobs.map((j) => j.jobId))).toEqual([["A"], ["B"], ["C"]]); + expect(started).toEqual(["A", "B", "C"]); + }); + + it("shares the boundary with direct retry drains and still runs the internal retry path", async () => { + const gate = deferred(); + const h = harness(async () => { await gate.promise; }); + h.job("main", "trace_summary"); h.retry("retry"); + const main = h.runner.runWorkerOnce(1); + const retry = h.runner.runEmbeddingRetryOnce(1); + try { expect(h.repos.runtime.getEmbeddingRetry("retry")).toMatchObject({ status: "pending", leaseToken: null }); } + finally { gate.resolve(); } + expect((await main).leased).toBe(1); + expect(await retry).toMatchObject({ leased: 1, succeeded: 1 }); + h.retry("internal-retry"); + expect((await h.runner.runWorkerOnce(1)).embeddingRetries).toMatchObject({ leased: 1, succeeded: 1 }); + }); + + it("releases the entry boundary after an unexpected leasing error", async () => { + const h = harness(); h.job("survivor"); + vi.spyOn(h.repos.runtime, "claimQueuedJob").mockImplementationOnce(() => { throw new Error("synthetic database unavailable"); }); + const failed = h.runner.runWorkerOnce(1); + const next = h.runner.runWorkerOnce(1); + await expect(failed).rejects.toThrow("synthetic database unavailable"); + expect(await next).toMatchObject({ leased: 1, succeeded: 1 }); + }); + + it.each(["main", "retry"] as const)("rejects nested %s execution instead of awaiting the gate it already owns", async (kind) => { + let rejected: unknown; + const h = harness(async () => { + try { + if (kind === "main") await h.runner.runWorkerOnce(1); + else await h.runner.runEmbeddingRetryOnce(1); + } catch (error) { rejected = error; throw error; } + }); + h.job("nested"); + expect((await h.runner.runWorkerOnce(1)).failed).toBe(1); + expect(rejected).toBeInstanceOf(WorkerSchedulerReentryError); + }); + + it("rejects pending and new requests on stop, while letting the active guarded job finish", async () => { + const gate = deferred(); + const h = harness(async () => { await gate.promise; }); + h.job("active"); h.job("waiting"); + const running = h.runner.runWorkerOnce(1); + const pending = h.runner.runWorkerOnce(1); + const rejected = expect(pending).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); + const stopped = h.runner.stop(); + let settled = false; void stopped.then(() => { settled = true; }); + await rejected; + await expect(h.runner.runWorkerOnce(1)).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); + expect(settled).toBe(false); + expect(h.repos.runtime.getJob("waiting")).toMatchObject({ status: "queued", attempts: 0 }); + gate.resolve(); await running; await stopped; + expect(h.repos.runtime.getJob("active")!.status).toBe("succeeded"); + expect(settled).toBe(true); + }); + + it("keeps the default limit of 100 as work budget inside one admitted drain", async () => { + const gate = deferred(); let active = 0; let peak = 0; + const h = harness(async () => { active++; peak = Math.max(peak, active); await gate.promise; active--; }); + for (let i = 0; i < 100; i++) h.job(`summary-${i}`, "trace_summary"); + const run = h.runner.runWorkerOnce(); + try { await flushWorker(); expect(active).toBe(4); expect(peak).toBe(4); } + finally { gate.resolve(); } + expect(await run).toMatchObject({ leased: 100, succeeded: 100 }); + expect(peak).toBe(4); + }); +}); + +it("keeps lifecycle preparation and post-run reconciliation inside the same boundary", async () => { + const before = deferred(); const entered = deferred(); const events: string[] = []; + const h = harness(async (job) => { events.push(job.id); }); + let calls = 0; + h.deps.beforeRun = async () => { + events.push(`before-${++calls}`); + if (calls === 1) { entered.resolve(); await before.promise; } + }; + h.deps.afterRun = () => { events.push(`after-${calls}`); }; + h.job("first"); h.job("second"); + const a = h.runner.runWorkerOnce(1), b = h.runner.runWorkerOnce(1); + await entered.promise; + expect(events).toEqual(["before-1"]); + expect(h.repos.runtime.getJob("first")!.status).toBe("queued"); + before.resolve(); await Promise.all([a, b]); await h.runner.stop(); + expect(events).toEqual(["before-1", "first", "after-1", "before-2", "second", "after-2"]); +}); diff --git a/Memory/tests/service/worker/worker-scheduler.test.ts b/Memory/tests/service/worker/worker-scheduler.test.ts new file mode 100644 index 000000000..c4d997777 --- /dev/null +++ b/Memory/tests/service/worker/worker-scheduler.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; +import { + WorkerScheduler, WorkerSchedulerReentryError, WorkerSchedulerStoppedError, withoutWorkerSchedulerContext +} from "../../../src/service/worker/worker-scheduler.js"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + return { promise, resolve, reject }; +} + +describe("single worker scheduling boundary", () => { + it("starts queued requests FIFO and preserves each result", async () => { + const scheduler = new WorkerScheduler(), gate = deferred(); + const started: string[] = []; + const a = scheduler.run(async () => { started.push("A"); await gate.promise; return "result-A"; }); + const b = scheduler.run(() => { started.push("B"); return "result-B"; }); + const c = scheduler.run(async () => { started.push("C"); return "result-C"; }); + expect(started).toEqual(["A"]); + gate.resolve(); + expect(await Promise.all([a, b, c])).toEqual(["result-A", "result-B", "result-C"]); + expect(started).toEqual(["A", "B", "C"]); + }); + + it.each(["synchronous", "asynchronous"] as const)("releases the boundary after %s failure", async (mode) => { + const scheduler = new WorkerScheduler(), gate = deferred(); + const error = new Error(`controlled ${mode} failure`); + const first = scheduler.run(async () => { await gate.promise; }); + const failing = scheduler.run(mode === "synchronous" ? () => { throw error; } : async () => { throw error; }); + const failureObserved = expect(failing).rejects.toBe(error); + const later = scheduler.run(() => "later-result"); + gate.resolve(); + await first; + await failureObserved; + expect(await later).toBe("later-result"); + }); + + it("rejects nested run and close instead of deadlocking the active request", async () => { + const scheduler = new WorkerScheduler(); + await scheduler.run(async () => { + await expect(scheduler.run(() => "nested")).rejects.toBeInstanceOf(WorkerSchedulerReentryError); + expect(() => scheduler.close()).toThrow(WorkerSchedulerReentryError); + }); + expect(await scheduler.run(() => "still-open")).toBe("still-open"); + await scheduler.close(); + }); + + it("lets an old async descendant submit after its parent execution has ended", async () => { + const scheduler = new WorkerScheduler(), afterParent = deferred(), childDone = deferred(); + await scheduler.run(() => { + void afterParent.promise.then(async () => { + try { childDone.resolve(await scheduler.run(() => "descendant-result")); } + catch (error) { childDone.reject(error); } + }); + }); + afterParent.resolve(); + expect(await childDone.promise).toBe("descendant-result"); + }); + + it("rejects pending requests on close but waits for active work and rejects new requests", async () => { + const scheduler = new WorkerScheduler(), gate = deferred(); + let pendingStarted = false, stopped = false; + const active = scheduler.run(async () => { await gate.promise; return "active-result"; }); + const pending = scheduler.run(() => { pendingStarted = true; }); + const pendingRejected = expect(pending).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); + const stopping = scheduler.close().then(() => { stopped = true; }); + const stoppingAgain = scheduler.close(); + await pendingRejected; + expect(pendingStarted).toBe(false); + expect(stopped).toBe(false); + await expect(scheduler.run(() => "new")).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); + gate.resolve(); + expect(await active).toBe("active-result"); + await Promise.all([stopping, stoppingAgain]); + expect(stopped).toBe(true); + }); + + it("settles close even if its active work fails", async () => { + const scheduler = new WorkerScheduler(), gate = deferred(); + const error = new Error("failure while stopping"); + const active = scheduler.run(async () => { await gate.promise; throw error; }); + const failed = expect(active).rejects.toBe(error); + const stopping = scheduler.close(); + gate.resolve(); + await failed; + await stopping; + }); + + it("closes an unused scheduler idempotently", async () => { + const scheduler = new WorkerScheduler(); + await scheduler.close(); await scheduler.close(); + await expect(scheduler.run(() => 1)).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); + }); +}); + +it("accepts an independent wake request from an active job without overlapping execution", async () => { + const scheduler = new WorkerScheduler(), gate = deferred(); + const started: string[] = []; + let wake!: Promise; + const active = scheduler.run(async () => { + started.push("active"); + wake = withoutWorkerSchedulerContext(() => scheduler.run(() => { started.push("wake"); return "woken"; })); + await gate.promise; + }); + expect(started).toEqual(["active"]); + gate.resolve(); await active; + expect(await wake).toBe("woken"); + expect(started).toEqual(["active", "wake"]); +}); diff --git a/Memory/tests/service/worker/worker-scheduling-options.test.ts b/Memory/tests/service/worker/worker-scheduling-options.test.ts new file mode 100644 index 000000000..2c9642905 --- /dev/null +++ b/Memory/tests/service/worker/worker-scheduling-options.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { DEFAULT_LANE_WEIGHTS } from "../../../src/service/worker/lane-queue-selector.js"; +import { + resolveWorkerSchedulingOptions, type WorkerSchedulingOptions +} from "../../../src/service/worker/worker-scheduling-options.js"; + +describe("worker scheduling configuration", () => { + it("defaults to real WRR and the confirmed 4/3/2/2/1 weights", () => { + const actual = resolveWorkerSchedulingOptions(undefined, {}); + expect(actual).toEqual({ algorithm: "wrr", laneWeights: DEFAULT_LANE_WEIGHTS, aging: { enabled: true, thresholdMs: 300_000 } }); + expect(Object.isFrozen(actual)).toBe(true); + expect(Object.isFrozen(actual.laneWeights)).toBe(true); + }); + + it("provides an explicit legacy rollback through the environment", () => { + expect(resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_SCHEDULER_ALGORITHM: " legacy " }).algorithm).toBe("legacy"); + expect(resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_SCHEDULER_ALGORITHM: "WRR" }).algorithm).toBe("wrr"); + expect(resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_SCHEDULER_ALGORITHM: " " }).algorithm).toBe("wrr"); + }); + + it("reads complete runtime weight objects and keeps override precedence field by field", () => { + const custom = { ...DEFAULT_LANE_WEIGHTS, import: 5 }; + expect(resolveWorkerSchedulingOptions({ algorithm: "legacy" }, { + MEMMY_WORKER_SCHEDULER_ALGORITHM: "wrr", MEMMY_WORKER_LANE_WEIGHTS: JSON.stringify(custom) + })).toEqual({ algorithm: "legacy", laneWeights: custom, aging: { enabled: false, thresholdMs: 300_000 } }); + expect(resolveWorkerSchedulingOptions({ laneWeights: custom }, { + MEMMY_WORKER_SCHEDULER_ALGORITHM: "legacy", MEMMY_WORKER_LANE_WEIGHTS: "invalid ignored override" + })).toEqual({ algorithm: "legacy", laneWeights: custom, aging: { enabled: false, thresholdMs: 300_000 } }); + expect(resolveWorkerSchedulingOptions({ algorithm: "wrr", laneWeights: custom }, { + MEMMY_WORKER_SCHEDULER_ALGORITHM: "invalid ignored override", MEMMY_WORKER_LANE_WEIGHTS: "{}" + })).toEqual({ algorithm: "wrr", laneWeights: custom, aging: { enabled: true, thresholdMs: 300_000 } }); + }); + + it("configures bounded aging explicitly while keeping the legacy rollback disabled", () => { + expect(resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_AGING_ENABLED: "false", MEMMY_WORKER_AGING_THRESHOLD_MS: "600000" }).aging) + .toEqual({ enabled: false, thresholdMs: 600_000 }); + expect(resolveWorkerSchedulingOptions({ aging: { enabled: true, thresholdMs: 1000 } }, { + MEMMY_WORKER_AGING_ENABLED: "invalid override", MEMMY_WORKER_AGING_THRESHOLD_MS: "invalid override" + }).aging).toEqual({ enabled: true, thresholdMs: 1000 }); + expect(resolveWorkerSchedulingOptions({ algorithm: "legacy", aging: { enabled: true } }, {}).aging.enabled).toBe(false); + }); + + it.each(["", "maybe", "yes", "2"])("rejects malformed aging flags %j", flag => { + expect(() => resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_AGING_ENABLED: flag })).toThrow(); + }); + + it.each(["", "0", "-1", "1.1", "NaN", "Infinity", "bad"])("rejects malformed aging thresholds %j", threshold => { + expect(() => resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_AGING_THRESHOLD_MS: threshold })).toThrow(); + }); + + it.each([null, [], { enabled: "true" }, { enabled: null }, { thresholdMs: null }, { thresholdMs: -1 }])( + "rejects invalid dependency aging values %j", aging => { + expect(() => resolveWorkerSchedulingOptions({ aging } as unknown as WorkerSchedulingOptions, {})).toThrow(); + } + ); + + it.each(["fifo", "shadow", "wrrr", "false"])("rejects an unknown algorithm %s instead of silently selecting a policy", (algorithm) => { + expect(() => resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_SCHEDULER_ALGORITHM: algorithm })).toThrow("must be wrr or legacy"); + }); + + it.each(["not-json", "null", "[]", "{}", '{"interactive":4}', + JSON.stringify({ ...DEFAULT_LANE_WEIGHTS, maintenance: 0 }), + JSON.stringify({ ...DEFAULT_LANE_WEIGHTS, future: 1 }) + ])("rejects incomplete or malformed environment weights %s", (encoded) => { + expect(() => resolveWorkerSchedulingOptions(undefined, { MEMMY_WORKER_LANE_WEIGHTS: encoded })).toThrow(); + }); + + it("validates dependency values at runtime instead of relying solely on TypeScript", () => { + expect(() => resolveWorkerSchedulingOptions({ algorithm: "bad" } as unknown as WorkerSchedulingOptions, {})).toThrow(); + expect(() => resolveWorkerSchedulingOptions({ laneWeights: null } as unknown as WorkerSchedulingOptions, {})).toThrow(); + }); +}); diff --git a/Memory/tests/service/worker/worker-scheduling-shadow.test.ts b/Memory/tests/service/worker/worker-scheduling-shadow.test.ts new file mode 100644 index 000000000..7e8d68a52 --- /dev/null +++ b/Memory/tests/service/worker/worker-scheduling-shadow.test.ts @@ -0,0 +1,186 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { jobTypeConsumesMemoryBudget } from "@memmy/agent-source-core"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { Repositories, type EvolutionJobRecord } from "../../../src/storage/repositories.js"; +import { WorkerRunner, type WorkerRunnerDeps } from "../../../src/service/worker/worker-runner.js"; +import { WorkerSchedulingShadow, type WorkerSchedulingComparison } from "../../../src/service/worker/worker-scheduling-shadow.js"; +import { createCapturingEmbedder } from "../../fixtures/memory-service-fixture.js"; + +const at = "2026-10-22T00:00:00.000Z"; +const epoch = Date.parse(at); +const resources: Array<{ root: string; db: MemoryDb }> = []; +function harness() { + const root = mkdtempSync(join(tmpdir(), "memmy-worker-shadow-")); + const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + resources.push({ root, db }); + const repos = new Repositories(db.db); + const executed: string[] = [], comparisons: WorkerSchedulingComparison[] = []; + const deps: WorkerRunnerDeps = { + repos, embedder: createCapturingEmbedder([]), capture: { embedAfterCapture: true }, + embeddingRetryWorkerId: "shadow-test", memoryAddEnabled: () => true, + memoryBudgetPaused: () => false, memoryBudgetJobConsumes: () => false, + memoryBudgetModelSources: () => ({ embedding: { mode: "local" } }), + memoryBudgetNextWakeAtMs: () => Date.now(), summaryModelConfigured: () => true, + schedulingShadow: { enabled: true, onComparison: (value) => { comparisons.push(value); } }, + nowIso: () => new Date().toISOString(), encodeChangeCursor: String, + namespaceIdFromMemory: () => "synthetic", + runWorkerNoWrite: async () => { throw new Error("unexpected disabled worker"); }, + restartFailedProcessing: () => 0, + previewPolicyEvidenceReconciliation: () => ({ orphanPolicyIds: [], affectedWorldModelIds: [], affectedSkillIds: [], restorablePolicyIds: [] }), + reconcileOrphanedPolicies: () => 0, + enqueueJob: () => { throw new Error("unexpected startup enqueue"); }, + enqueueEmbeddingRetry: () => { throw new Error("unexpected replacement"); }, + appendJobChange: () => {}, appendEmbeddingRetryChange: () => {}, + jobHandlers: { processJob: async (job) => { executed.push(job.id); } }, + embeddingJobs: { + prepareEmbeddingJob: () => { throw new Error("unexpected main embedding"); }, + applyEmbeddingVector: () => {}, enqueueEmbeddingRetryAfterFailure: () => false, + persistEmbeddingVector: (input) => { + const memory = repos.memories.get(input.memoryId)!; + input.finalize?.(memory, false, new Date().toISOString()); + return memory; + } + } + }; + const runner = new WorkerRunner(deps); + const shadow = new WorkerSchedulingShadow(repos.runtime); + function job(id: string, jobType: EvolutionJobRecord["jobType"] = "reflection", targetMemoryId?: string, payload: Record = {}) { + return repos.runtime.enqueueJob({ id, jobType, userId: "user", targetMemoryId, + payload, status: "queued", attempts: 0, maxAttempts: 3, createdAt: at, updatedAt: at }); + } + function retry(id: string) { + db.db.prepare(`INSERT INTO memories(id,timeline,user_id,memory_value,memory_layer,created_at,updated_at) + VALUES (?,?,'user','synthetic','L1',?,?)`).run(id, at, at, at); + return repos.runtime.enqueueEmbeddingRetry({ id, targetKind: "trace", targetId: id, + vectorField: "vec_summary", sourceText: "synthetic", now: epoch }); + } + function state() { + const metadata = new Set(["queue_lane", "resource_class", "enqueue_seq", "first_queued_at", "base_priority", "policy_version", "policy_promotion"]); + const rows = (table: string) => (db.db.prepare(`SELECT * FROM ${table} ORDER BY id`).all() as Record[]) + .map((row) => Object.fromEntries(Object.entries(row).filter(([key]) => !metadata.has(key)))); + return { main: rows("evolution_jobs"), retry: rows("embedding_retry_queue"), + memories: rows("memories"), processing: (db.db.prepare("SELECT * FROM memory_processing_state ORDER BY memory_id").all()), + changeSeq: repos.runtime.latestChangeSeq() }; + } + return { db, repos, deps, runner, shadow, job, retry, comparisons, executed, state }; +} +afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); + for (const { root, db } of resources.splice(0)) { + if (db.db.open) db.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("worker shadow scheduling integration", () => { + it("previews WRR including retries without mutating task facts or business data", () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); + for (let i = 0; i < 4; i++) h.job(`import-${i}`, "import_summary"); + h.job("reflection"); h.job("maintenance", "project_environment_profile"); h.retry("retry"); + // Force the same bounded projection refresh used after a schema upgrade. + h.db.db.exec("UPDATE evolution_jobs SET policy_version = NULL"); + const before = h.state(); + const comparison = h.shadow.sample({ at, limit: 4 })!; + expect(comparison.legacySelection).toEqual(["main:import-0", "main:import-1", "main:import-2", "main:import-3"]); + expect(comparison.shadowSelection).toEqual(["embedding_retry:retry", "main:import-0", "main:import-1", "main:reflection"]); + expect(comparison).toMatchObject({ sameSelection: false, coverageComplete: true, pendingPolicyCount: 0 }); + expect(h.state()).toEqual(before); + }); + + it("executes the legacy selection even when shadow recommends a retry and evolution", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.deps.scheduling = { algorithm: "legacy" }; + for (let i = 0; i < 4; i++) h.job(`import-${i}`, "import_summary"); + h.job("reflection"); h.retry("retry"); + const result = await h.runner.runWorkerOnce(4, { priorityCohortOnly: true }); + expect(result).toMatchObject({ leased: 4, succeeded: 4, failed: 0, embeddingRetries: { leased: 0 } }); + expect(h.executed).toEqual(["import-0", "import-1", "import-2", "import-3"]); + expect(h.comparisons[0]).toMatchObject({ sameSelection: false }); + expect(h.comparisons[0]!.shadowSelection).toContain("embedding_retry:retry"); + expect(h.repos.runtime.getJob("reflection")!.status).toBe("queued"); + expect(h.repos.runtime.getEmbeddingRetry("retry")!.status).toBe("pending"); + }); + + it.each(["preview", "observer"] as const)("isolates %s failures from actual work", async (failure) => { + const h = harness(); h.job("still-runs"); + if (failure === "preview") vi.spyOn(h.repos.runtime.scheduling, "candidates").mockImplementationOnce(() => { throw new Error("synthetic preview failure"); }); + else h.deps.schedulingShadow!.onComparison = () => { throw new Error("synthetic observer failure"); }; + expect(await h.runner.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(h.executed).toEqual(["still-runs"]); + }); + + it.each(["environment", "dependency"] as const)("does not sample when disabled by %s", async (mode) => { + const h = harness(); h.job("still-runs"); + if (mode === "environment") { h.deps.schedulingShadow = undefined; vi.stubEnv("MEMMY_WORKER_SCHEDULER_SHADOW", "0"); } + else { h.deps.schedulingShadow!.enabled = false; vi.stubEnv("MEMMY_WORKER_SCHEDULER_SHADOW", "1"); } + const preview = vi.spyOn(h.repos.runtime.scheduling, "refreshPolicies"); + expect((await h.runner.runWorkerOnce(1)).succeeded).toBe(1); + expect(preview).not.toHaveBeenCalled(); expect(h.comparisons).toHaveLength(0); + }); + + it("samples no more than once per five seconds and caps comparisons at 32 choices", () => { + const h = harness(); + for (let i = 0; i < 40; i++) h.job(`reflection-${i.toString().padStart(2, "0")}`); + const refresh = vi.spyOn(h.repos.runtime.scheduling, "refreshPolicies"); + const first = h.shadow.sample({ at, limit: 100 })!; + expect(first.sampleLimit).toBe(32); expect(first.shadowSelection).toHaveLength(32); expect(first.legacySelection).toHaveLength(32); + expect(h.shadow.sample({ at: new Date(epoch + 4_999).toISOString(), limit: 100 })).toBeUndefined(); + expect(refresh).toHaveBeenCalledTimes(1); + expect(h.shadow.sample({ at: new Date(epoch + 5_000).toISOString(), limit: 100 })!.sampleLimit).toBe(32); + expect(refresh).toHaveBeenCalledTimes(2); expect(refresh).toHaveBeenLastCalledWith(500); + }); + + it("keeps other lanes visible behind ten thousand import candidates", () => { + const h = harness(); + h.repos.transaction(() => { + for (let i = 0; i < 10_000; i++) h.job(`import-${i.toString().padStart(5, "0")}`, "import_summary"); + h.job("reflection"); h.job("maintenance", "project_environment_profile"); h.retry("retry"); + }); + const comparison = h.shadow.sample({ at, limit: 8 })!; + expect(comparison.coverageComplete).toBe(true); + expect(comparison.legacySelection.every((id) => id.startsWith("main:import-"))).toBe(true); + expect(comparison.shadowSelection).toEqual(expect.arrayContaining(["main:reflection", "main:maintenance", "embedding_retry:retry"])); + expect(comparison.candidatePoolSize).toBe(11); + expect(comparison.shadowSelection).toHaveLength(8); + }); + + it("applies target filters to both preview queues and the legacy execution", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("target-job", "import_summary", "target-A"); h.job("other-job", "import_summary", "target-B"); h.retry("target-B"); + expect((await h.runner.runWorkerOnce(4, { targetMemoryIds: ["target-A"] })).leased).toBe(1); + expect(h.comparisons[0]!.legacySelection).toEqual(["main:target-job"]); + expect(h.comparisons[0]!.shadowSelection).toEqual(["main:target-job"]); + expect(h.executed).toEqual(["target-job"]); + expect(h.repos.runtime.getJob("other-job")!.status).toBe("queued"); + expect(h.repos.runtime.getEmbeddingRetry("target-B")!.status).toBe("pending"); + }); + + it("holds summary jobs in both selectors while allowing embedding retries", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("held-summary", "import_summary"); h.job("reflection"); h.retry("retry"); + h.deps.summaryModelConfigured = () => false; + const result = await h.runner.runWorkerOnce(4); + expect(result).toMatchObject({ leased: 1, succeeded: 1, embeddingRetries: { leased: 1, succeeded: 1 } }); + expect(h.comparisons[0]!.legacySelection).toEqual(["main:reflection", "embedding_retry:retry"]); + expect(h.comparisons[0]!.shadowSelection).toEqual(["embedding_retry:retry", "main:reflection"]); + expect(h.repos.runtime.getJob("held-summary")!.status).toBe("queued"); + }); + + it("keeps budget-gated models and retries out of previews and execution", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(epoch); + const h = harness(); h.job("held-summary", "import_summary"); h.job("held-evolution"); h.job("database-only", "l2_association"); h.retry("held-retry"); + const sources = { memory_summary: { source: "byok" }, memory_evolution: { source: "byok" }, embedding: { source: "byok", mode: "remote" } }; + h.deps.memoryBudgetPaused = () => true; + h.deps.memoryBudgetModelSources = () => sources; + h.deps.memoryBudgetJobConsumes = (type) => jobTypeConsumesMemoryBudget(type, sources); + expect(await h.runner.runWorkerOnce(4)).toMatchObject({ leased: 1, succeeded: 1, embeddingRetries: { leased: 0 } }); + expect(h.comparisons[0]!.legacySelection).toEqual(["main:database-only"]); + expect(h.comparisons[0]!.shadowSelection).toEqual(["main:database-only"]); + expect(h.executed).toEqual(["database-only"]); + expect(h.repos.runtime.getEmbeddingRetry("held-retry")!.status).toBe("pending"); + }); +}); diff --git a/Memory/tests/service/worker/worker-service-lifecycle.test.ts b/Memory/tests/service/worker/worker-service-lifecycle.test.ts new file mode 100644 index 000000000..489c2a937 --- /dev/null +++ b/Memory/tests/service/worker/worker-service-lifecycle.test.ts @@ -0,0 +1,80 @@ +import { afterEach, expect, it } from "vitest"; +import type { LlmCompletionOptions, LlmMessage } from "../../../src/model/types.js"; +import { WorkerSchedulerReentryError, WorkerSchedulerStoppedError } from "../../../src/service/worker/worker-scheduler.js"; +import { addAgentSourceImport, createBatchReflectionLlm, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(fixture.cleanup); +const namespace = { source: "codex", profileId: "scheduler-lifecycle", userId: "scheduler-lifecycle-user" }; +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +it("rejects a worker's attempt to await its own service shutdown without poisoning later shutdown", async () => { + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + let stopInside!: () => Promise; + let refused = 0; + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + await expect(stopInside()).rejects.toBeInstanceOf(WorkerSchedulerReentryError); + refused++; + return complete(messages, options); + }; + const { service } = fixture.createTestService({ llm }); + stopInside = () => service.stop(); + const memory = addAgentSourceImport(service, namespace, "shutdown reentry does not close the service", "reentry"); + expect(await service.runWorkerOnce(1)).toMatchObject({ leased: 1, succeeded: 1 }); + expect(refused).toBeGreaterThan(0); + await service.runWorkerOnce(4); + expect(service.memoryProcessingStatus([memory.id], { namespace }).items[0]?.state).toBe("ready"); + await service.stop(); + await expect(service.runWorkerOnce(1)).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); +}); + +it("waits for admitted service work before storage may close and cancels unstarted requests", async () => { + const entered = deferred(), gate = deferred(); + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + entered.resolve(); await gate.promise; return complete(messages, options); + }; + const { service, db } = fixture.createTestService({ llm }); + const active = addAgentSourceImport(service, namespace, "active summary", "active"); + const waiting = addAgentSourceImport(service, namespace, "waiting summary", "waiting"); + const run = service.runWorkerOnce(1, { targetMemoryIds: [active.id] }); + await entered.promise; + const pending = service.runWorkerOnce(1, { targetMemoryIds: [waiting.id] }); + const pendingRejected = expect(pending).rejects.toBeInstanceOf(WorkerSchedulerStoppedError); + await Promise.resolve(); + const stop = service.stop(), repeatedStop = service.stop(); + let closed = false; + const closeStorage = stop.then(() => { db.close(); closed = true; }); + try { + await pendingRejected; + expect(closed).toBe(false); + expect(service.memoryProcessingStatus([waiting.id], { namespace }).items[0]?.state).toBe("summary_pending"); + } finally { gate.resolve(); } + await Promise.all([stop, repeatedStop, closeStorage]); + expect(await run).toMatchObject({ leased: 1, succeeded: 1 }); + expect(closed).toBe(true); +}); + +it("does not let an active job await an already requested external shutdown", async () => { + const entered = deferred(), gate = deferred(); + const llm = createBatchReflectionLlm([]), complete = llm.completeJson.bind(llm); + let stopInside!: () => Promise; + llm.completeJson = async >(messages: LlmMessage[], options: LlmCompletionOptions): Promise => { + entered.resolve(); await gate.promise; + await expect(stopInside()).rejects.toBeInstanceOf(WorkerSchedulerReentryError); + return complete(messages, options); + }; + const { service } = fixture.createTestService({ llm }); + stopInside = () => service.stop(); + addAgentSourceImport(service, namespace, "nested shutdown after external close", "external-close"); + const active = service.runWorkerOnce(1); + await entered.promise; + const stopped = service.stop(); + gate.resolve(); + expect(await active).toMatchObject({ leased: 1, succeeded: 1 }); + await stopped; +}); diff --git a/Memory/tests/storage/token-usage-outbox.test.ts b/Memory/tests/storage/token-usage-outbox.test.ts index fd5e05618..874b1a741 100644 --- a/Memory/tests/storage/token-usage-outbox.test.ts +++ b/Memory/tests/storage/token-usage-outbox.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { MemoryDb } from "../../src/index.js"; +import { MemoryDb, SCHEMA_MIGRATION_ID, SCHEMA_VERSION } from "../../src/index.js"; import { Repositories } from "../../src/storage/repositories.js"; import { TokenUsageOutbox } from "../../src/storage/token-usage-outbox.js"; @@ -64,8 +64,8 @@ describe("token usage outbox", () => { const upgraded = new MemoryDb({ path: dbPath }); expect(upgraded.schemaVersion()).toMatchObject({ - version: 10, - lastMigrationId: "010_pending_episode_assignment" + version: SCHEMA_VERSION, + lastMigrationId: SCHEMA_MIGRATION_ID }); expect(upgraded.db.prepare(`SELECT id FROM memories`).get()).toEqual({ id: "memory-keep" }); expect(upgraded.db.prepare( @@ -75,7 +75,7 @@ describe("token usage outbox", () => { upgraded.close(); const reopened = new MemoryDb({ path: dbPath }); - expect(reopened.schemaVersion().version).toBe(10); + expect(reopened.schemaVersion().version).toBe(SCHEMA_VERSION); expect(reopened.db.prepare(`SELECT COUNT(*) AS count FROM memories`).get()).toEqual({ count: 1 }); reopened.close(); }); diff --git a/Memory/tests/task-routing-embedding-batches.test.ts b/Memory/tests/task-routing-embedding-batches.test.ts new file mode 100644 index 000000000..a197c2895 --- /dev/null +++ b/Memory/tests/task-routing-embedding-batches.test.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_MEMMY_CONFIG } from "../src/config/index.js"; +import { MemoryModelTaskRouter, type MemoryModelTaskContext } from "../src/model/task-routing.js"; +import { WorkerConcurrencyController, providerDescriptor } from "../src/service/worker/worker-concurrency.js"; +import type { Embedder, LlmClient } from "../src/model/types.js"; +import { vectorEmbeddingIdentity } from "../src/model/embedding-identity.js"; +import { Repositories } from "../src/storage/repositories.js"; +import { createMemoryServiceFixture } from "./fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => fixture.cleanup()); + +function context(embed: Embedder["embed"]): MemoryModelTaskContext { + const llm: LlmClient = { + config: { ...DEFAULT_MEMMY_CONFIG.summary, provider: "openai_compatible", endpoint: "https://isolated.test/v1", model: "summary" }, + isConfigured: () => true, complete: async () => "unused", + completeJson: async >() => ({}) as T, + status: () => ({ provider: "openai_compatible", model: "summary", configured: true, remote: true }) + }; + const embedding: Embedder = { + config: { ...DEFAULT_MEMMY_CONFIG.embedding, provider: "openai_compatible", mode: "custom", + endpoint: "https://isolated.test/v1", model: "embedding", cache: false }, + isRemote: () => true, embed, + embedOne: async (text, role) => (await embed([text], role))[0]!, + status: () => ({ provider: "openai_compatible", model: "embedding", configured: true, remote: true }) + }; + return { config: DEFAULT_MEMMY_CONFIG, summary: llm, evolution: llm, embedding }; +} + +describe("routed embedding request batches", () => { + it("preserves the request's original model identity across batches and an asynchronous config change", async () => { + const controller = new WorkerConcurrencyController({ embeddingBatchSize: 2 }, {}); + const task = context(async texts => { + await new Promise(resolve => setImmediate(resolve)); + task.embedding.config.model = "later-model"; + return texts.map(() => [1]); + }); + const router = new MemoryModelTaskRouter(() => task, controller); + try { + const vectors = await router.embedder().embed(["one", "two", "three"]); + expect(vectors.map(vectorEmbeddingIdentity)).toEqual(Array.from({ length: 3 }, () => ({ provider: "openai_compatible", model: "embedding" }))); + task.embedding.config.model = "single-model"; + expect(vectorEmbeddingIdentity(await router.embedder().embedOne("single"))) + .toEqual({ provider: "openai_compatible", model: "single-model" }); + expect(controller.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, embeddingRequests: 0 }); + } finally { await Promise.all([router.stop(), controller.close()]); } + }); + + it.each(["query", "document", undefined] as const)("caps every request at 16 and preserves %s role and vector order", async role => { + const calls: Array<{ texts: string[]; role?: string }> = []; + let active = 0, peak = 0; + const task = context(async (texts, requestRole) => { + calls.push({ texts: [...texts], role: requestRole }); peak = Math.max(peak, ++active); + await new Promise(resolve => setImmediate(resolve)); active--; + return texts.map(text => [Number(text)]); + }); + const controller = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => task, controller); + try { + expect(await router.embedder().embed(Array.from({ length: 37 }, (_, i) => String(i)), role)) + .toEqual(Array.from({ length: 37 }, (_, i) => [i])); + expect(calls.map(call => call.texts.length)).toEqual([16, 16, 5]); + expect(calls.map(call => call.role)).toEqual([role, role, role]); + expect(peak).toBe(1); + expect(controller.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, embeddingRequests: 0 }); + } finally { await Promise.all([router.stop(), controller.close()]); } + }); + + it("honors a configured batch limit without charging text count as worker jobs", async () => { + const sizes: number[] = [], logical: number[] = []; + const controller = new WorkerConcurrencyController({ embeddingBatchSize: 3 }, {}); + const task = context(async texts => { + sizes.push(texts.length); logical.push(controller.snapshot().activeJobs); + return texts.map(text => [Number(text)]); + }); + const router = new MemoryModelTaskRouter(() => task, controller); + const permit = controller.tryAcquire({ resourceClass: "summary", embedding: true, + providers: providerDescriptor(task.embedding.config, controller.limits).providers, scopeKeys: [] })!; + try { + expect(await permit.run(() => router.runWith(task, () => router.embedder().embed(["0", "1", "2", "3", "4", "5", "6"], "query")))) + .toEqual([[0], [1], [2], [3], [4], [5], [6]]); + expect(sizes).toEqual([3, 3, 1]); expect(logical).toEqual([1, 1, 1]); + } finally { permit.release(); await Promise.all([router.stop(), controller.close()]); } + }); + + it("does not call a provider for empty input", async () => { + let calls = 0; + const task = context(async () => { calls++; return []; }); + const controller = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => task, controller); + try { expect(await router.embedder().embed([], "document")).toEqual([]); expect(calls).toBe(0); } + finally { await Promise.all([router.stop(), controller.close()]); } + }); + + it("stops before later slices after an error and releases request permits", async () => { + const sizes: number[] = []; + const task = context(async texts => { + sizes.push(texts.length); + if (sizes.length === 2) throw new Error("isolated second batch failure"); + return texts.map(() => [1]); + }); + const controller = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => task, controller); + try { + await expect(router.embedder().embed(Array.from({ length: 37 }, (_, i) => String(i)))) + .rejects.toThrow("isolated second batch failure"); + expect(sizes).toEqual([16, 16]); + expect(controller.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, embeddingRequests: 0 }); + } finally { await Promise.all([router.stop(), controller.close()]); } + }); + + it("shares the single request limit across simultaneous business calls and their slices", async () => { + let active = 0, peak = 0; + const sizes: number[] = []; + const task = context(async texts => { + peak = Math.max(peak, ++active); sizes.push(texts.length); + await new Promise(resolve => setImmediate(resolve)); active--; + return texts.map(text => [Number(text)]); + }); + const controller = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => task, controller); + try { + const input = Array.from({ length: 33 }, (_, i) => String(i)); + const results = await Promise.all([router.embedder().embed(input, "query"), router.embedder().embed(input, "document")]); + for (const vectors of results) expect(vectors).toEqual(input.map(text => [Number(text)])); + expect(sizes).toHaveLength(6); expect(Math.max(...sizes)).toBe(16); expect(peak).toBe(1); + } finally { await Promise.all([router.stop(), controller.close()]); } + }); + + it("batches actual Work Memory candidate retrieval while running one logical worker job", async () => { + const sizes: number[] = [], roles: unknown[] = [], logical: number[] = []; + const concurrency = new WorkerConcurrencyController({}, {}); + const task = context(async (texts, role) => { + sizes.push(texts.length); roles.push(role); logical.push(concurrency.snapshot().activeJobs); + return texts.map(() => [1, 0, 0]); + }); + task.summary.completeJson = async >(_messages: unknown, options: { operation: string }) => { + if (options.operation === "capture.work_memory_extract") return { work_memories: Array.from({ length: 37 }, (_, i) => ({ + requirement: `Requirement ${i}`, reason: `Reproducibility reason ${i}` })) } as unknown as T; + if (options.operation === "capture.work_memory_merge") return { decisions: Array.from({ length: 37 }, (_, i) => ({ + candidate_ref: `new_${i + 1}`, action: "create", matched_memory_ref: null, work_topic: `Topic ${i}` })) } as unknown as T; + throw new Error(`Unexpected test LLM operation ${options.operation}`); + }; + const config = structuredClone(DEFAULT_MEMMY_CONFIG); config.algorithm.capture.embedAfterCapture = false; + const { service, db } = fixture.createTestService({ config, llm: task.summary, skillLlm: task.summary, + embedder: task.embedding, concurrency }); + const namespace = { source: "codex", profileId: "default", userId: "isolated-work-memory-batch", sessionKey: "batch-session" }; + const opened = service.openSession({ namespace, l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only", + workspaceUri: "file:///tmp/isolated-work-memory-batch", workspaceHostId: "a".repeat(64) }); + const completed = await service.completeTurn("batch-turn", { sessionId: opened.sessionId, + query: "Use the reproducible build pipeline.", answer: "The build pipeline is documented." }); + service.l3WorldModelBoundary(opened.sessionId, { requestId: "isolated-batch-boundary", adapterId: "codex-memory", source: "codex", + namespace: { ...namespace, projectId: opened.projectId ?? undefined }, trigger: "token_compaction", throughL1MemoryId: completed.l1MemoryId }); + db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE job_type != 'work_memory_extract'").run(); + const result = await service.runWorkerOnce(1); + expect(result).toMatchObject({ leased: 1, succeeded: 1, failed: 0 }); + expect(result.jobs[0]?.jobType).toBe("work_memory_extract"); + expect(sizes).toEqual([16, 16, 5]); expect(roles).toEqual(["query", "query", "query"]); + expect(logical).toEqual([1, 1, 1]); + const repos = new Repositories(db.db); + expect(repos.memories.list({ memoryLayer: "L1", memoryKind: "work_memory" }, 100)).toHaveLength(37); + expect(concurrency.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, embeddingRequests: 0 }); + }); +}); diff --git a/Memory/tests/task-routing.test.ts b/Memory/tests/task-routing.test.ts index 83840b58f..02447829e 100644 --- a/Memory/tests/task-routing.test.ts +++ b/Memory/tests/task-routing.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { MemoryModelTaskRouter, type MemoryModelTaskContext } from "../src/model/task-routing.js"; +import { WorkerConcurrencyController, providerDescriptor } from "../src/service/worker/worker-concurrency.js"; import type { Embedder, LlmClient, LlmCompletionOptions, LlmMessage } from "../src/model/types.js"; function client(name: string): LlmClient { @@ -65,6 +66,142 @@ function embedder(name: string): Embedder { } describe("MemoryModelTaskRouter", () => { + it.each(["foreground", "worker"] as const)("distinguishes new foreground business from a released %s request callback", async kind => { + const c = new WorkerConcurrencyController({}, {}); + const task = context("callback"); + const router = new MemoryModelTaskRouter(() => task, c); + let finish!: (result: { value?: string; error?: unknown }) => void; + const callback = new Promise<{ value?: string; error?: unknown }>(resolve => { finish = resolve; }); + let calls = 0; + task.summary.complete = async () => { + calls++; + if (calls === 1) setImmediate(() => { + void router.client("summary").complete([], { operation: "test.new-business" }) + .then(value => finish({ value }), error => finish({ error })); + }); + return `call-${calls}`; + }; + const first = () => router.client("summary").complete([], { operation: "test.original" }); + const permit = kind === "worker" ? c.tryAcquire({ resourceClass: "summary", + providers: providerDescriptor(task.summary.config, c.limits).providers, scopeKeys: [] })! : undefined; + try { + expect(await (permit ? permit.run(() => router.runWith(task, first)) : router.run(first))).toBe("call-1"); + permit?.release(); + const result = await callback; + if (kind === "foreground") { + expect(result.error).toBeUndefined(); expect(result.value).toBe("call-2"); expect(calls).toBe(2); + } else { + // A late callback from a finished worker remains fenced; it cannot turn + // itself into an unleased foreground model request after token release. + expect(result.error).toBeInstanceOf(Error); expect(calls).toBe(1); + } + expect(c.snapshot()).toMatchObject({ activeJobs: 0, providers: {}, embeddingRequests: 0 }); + } finally { permit?.release(); await Promise.all([router.stop(), c.close()]); } + }); + + it("allows shutdown from an inherited callback after its foreground task completed", async () => { + const router = new MemoryModelTaskRouter(() => context("completed")); + let continueCallback!: () => void; + const gate = new Promise(resolve => { continueCallback = resolve; }); + let completeCallback!: () => void, rejectCallback!: (error: unknown) => void; + const completed = new Promise((resolve, reject) => { completeCallback = resolve; rejectCallback = reject; }); + await router.run(async () => { + setImmediate(() => { + void (async () => { await gate; await router.stop(); })().then(completeCallback, rejectCallback); + }); + }); + continueCallback(); + await expect(completed).resolves.toBeUndefined(); + }); + + it("rejects new work from an inherited completed-task callback after shutdown", async () => { + const router = new MemoryModelTaskRouter(() => context("completed")); + let continueCallback!: () => void; + const gate = new Promise(resolve => { continueCallback = resolve; }); + let completeCallback!: (error: unknown) => void; + const completed = new Promise(resolve => { completeCallback = resolve; }); + let invoked = false; + await router.run(async () => { + setImmediate(() => { + void (async () => { + await gate; + try { router.run(() => { invoked = true; }); completeCallback(undefined); } + catch (error) { completeCallback(error); } + })(); + }); + }); + await router.stop(); continueCallback(); + expect(await completed).toBeInstanceOf(Error); + expect(invoked).toBe(false); + }); + + it("waits for complete business tasks after model responses while stopping new foreground tasks", async () => { + const router = new MemoryModelTaskRouter(() => context("closing")); + let response!: () => void, processed!: () => void; + const responseGate = new Promise(resolve => { response = resolve; }); + const processingGate = new Promise(resolve => { processed = resolve; }); + let postRequestWrite = false, stopped = false; + const business = router.run(async () => { await responseGate; await processingGate; postRequestWrite = true; }); + const stop = router.stop().then(() => { stopped = true; }); + response(); await Promise.resolve(); + expect(stopped).toBe(false); expect(postRequestWrite).toBe(false); + expect(() => router.run(() => "new task")).toThrow(/clos|stop/i); + processed(); await Promise.all([business, stop]); + expect(postRequestWrite).toBe(true); expect(stopped).toBe(true); + }); + + it("allows an already admitted worker to enter its queued snapshot after router shutdown begins", async () => { + const c = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => context("claimed"), c); + const snapshot = router.snapshot(); + const p = c.tryAcquire({ resourceClass: "summary", providers: providerDescriptor(snapshot.summary.config, c.limits).providers, scopeKeys: [] })!; + await router.stop(); const close = c.close(); + expect(await p.run(() => router.runWith(snapshot, () => router.client("summary").complete([], { operation: "finish" })))).toBe("claimed-summary"); + p.release(); await close; + }); + + it("runs the admission snapshot after configuration reload without resolving a new provider", async () => { + let current = context("admitted"), resolutions = 0; + const controller = new WorkerConcurrencyController({}, {}); + const router = new MemoryModelTaskRouter(() => { resolutions++; return current; }, controller); + const snapshot = router.snapshot(); + const providers = [snapshot.summary, snapshot.evolution].flatMap(delegate => providerDescriptor(delegate.config, controller.limits).providers); + const permit = controller.tryAcquire({ resourceClass: "summary", providers, scopeKeys: [] })!; + current = context("reloaded"); + try { + const result = await permit.run(() => router.runWith(snapshot, async () => [ + await router.client("summary").complete([], { operation: "test" }), + await router.client("evolution").complete([], { operation: "test" }) + ])); + expect(result).toEqual(["admitted-summary", "admitted-evolution"]); + expect(resolutions).toBe(1); expect(Object.isFrozen(snapshot)).toBe(true); + } finally { permit.release(); } + }); + + it("guards real routed calls to distinct models sharing a local endpoint", async () => { + const task = context("shared"); + const controller = new WorkerConcurrencyController({}, {}); + let active = 0, maximum = 0; + for (const delegate of [task.summary, task.evolution]) { + delegate.config.endpoint = "http://localhost:1234/v1"; + delegate.complete = async () => { + maximum = Math.max(maximum, ++active); + await new Promise(resolve => setImmediate(resolve)); + active--; return "ok"; + }; + } + const router = new MemoryModelTaskRouter(() => task, controller); + const providers = [task.summary, task.evolution].flatMap(delegate => providerDescriptor(delegate.config, controller.limits).providers); + const permit = controller.tryAcquire({ resourceClass: "summary", providers, scopeKeys: [] })!; + try { + await permit.run(() => router.runWith(task, () => Promise.all([ + router.client("summary").complete([], { operation: "one" }), + router.client("evolution").complete([], { operation: "two" }) + ]))); + expect(maximum).toBe(1); + } finally { permit.release(); } + }); + it("keeps one immutable model snapshot across all LLM calls in a task", async () => { let resolutions = 0; const router = new MemoryModelTaskRouter(() => context(`task-${++resolutions}`)); diff --git a/Migrations/src/migrations/v1.0.7/0001-normalize-runtime-model-catalog.ts b/Migrations/src/migrations/v1.0.7/0001-normalize-runtime-model-catalog.ts index 64a2be586..42e8b17bc 100644 --- a/Migrations/src/migrations/v1.0.7/0001-normalize-runtime-model-catalog.ts +++ b/Migrations/src/migrations/v1.0.7/0001-normalize-runtime-model-catalog.ts @@ -9,6 +9,7 @@ import { flattenLegacyMemoryModelConfig } from "./legacy-memory-model-config.js" import { isPlatformMemoryConnection } from "../../platform-memory-connection.js"; const MIGRATION_ID = "v1.0.7/0001-normalize-runtime-model-catalog"; +const BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID = "memmy-builtin-local-embedding"; export type CatalogCapability = | "agent" @@ -24,6 +25,7 @@ export type LegacyCatalogConnection = { model: string; sourceProvider?: string; protocol?: string; + inheritProviderDefaults?: boolean; apiKey?: string; extraHeaders?: Record; extraBody?: Record; @@ -182,10 +184,11 @@ function normalizedAuthValue(key: "apiKey" | "extraHeaders" | "extraBody", value } function effectiveAuth(provider: JsonObject, endpoint: JsonObject): JsonObject { + const defaults = endpoint.inheritProviderDefaults === false ? {} : provider; return { - apiKey: normalizedAuthValue("apiKey", endpoint.apiKey ?? provider.apiKey), - extraHeaders: normalizedAuthValue("extraHeaders", endpoint.extraHeaders ?? provider.extraHeaders), - extraBody: normalizedAuthValue("extraBody", endpoint.extraBody ?? provider.extraBody), + apiKey: normalizedAuthValue("apiKey", endpoint.apiKey ?? defaults.apiKey), + extraHeaders: normalizedAuthValue("extraHeaders", { ...objectAt(defaults, "extraHeaders"), ...objectAt(endpoint, "extraHeaders") }), + extraBody: normalizedAuthValue("extraBody", { ...objectAt(defaults, "extraBody"), ...objectAt(endpoint, "extraBody") }), }; } @@ -201,6 +204,7 @@ function endpointIdentity(provider: JsonObject, endpoint: JsonObject): string { return stableJson({ protocol: endpoint.protocol, apiBase: normalizeApiBase(String(endpoint.apiBase ?? "")), + inheritProviderDefaults: endpoint.inheritProviderDefaults !== false, auth: effectiveAuth(provider, endpoint), }); } @@ -401,15 +405,26 @@ function ensureEndpoint( connection: LegacyCatalogConnection, ): string { const endpoints = objectAt(provider, "endpoints") ?? {}; + const memoryCapability = ["memory_summary", "memory_evolution", "embedding"].includes(capability); + const independent = memoryCapability && connection.inheritProviderDefaults === false; const providerApiKey = stringAt(provider, "apiKey"); - if (!providerApiKey && connection.apiKey) provider.apiKey = connection.apiKey; + if (!independent && !providerApiKey && connection.apiKey) provider.apiKey = connection.apiKey; const endpoint: JsonObject = { apiBase: normalizeApiBase(connection.apiBase), - protocol: (capability === "agent" || capability.startsWith("memory_")) && connection.protocol + protocol: (capability === "agent" || memoryCapability) && connection.protocol ? connection.protocol : protocolFor(canonicalProviderId(connection.provider), capability), }; + if (independent) { + endpoint.inheritProviderDefaults = false; + endpoint.apiKey = connection.apiKey ?? ""; + endpoint.extraHeaders = structuredClone(connection.extraHeaders ?? {}); + endpoint.extraBody = structuredClone(connection.extraBody ?? {}); + } if (connection.apiKey && stringAt(provider, "apiKey") !== connection.apiKey) endpoint.apiKey = connection.apiKey; + // Standalone Memory connections did not inherit another catalog provider's + // key. Materialize no-auth so endpoint reuse and later runs keep that meaning. + if (memoryCapability && connection.apiKey === "") endpoint.apiKey = ""; if (connection.extraHeaders) endpoint.extraHeaders = structuredClone(connection.extraHeaders); if (connection.extraBody) endpoint.extraBody = structuredClone(connection.extraBody); const identity = endpointIdentity(provider, endpoint); @@ -430,6 +445,7 @@ function ensurePreset( config: JsonObject, capability: CatalogCapability, connection: LegacyCatalogConnection, + extendExistingCapabilities = true, ): string { const providerId = canonicalProviderId(connection.provider); const providers = objectAt(config, "providers") ?? {}; @@ -443,7 +459,8 @@ function ensurePreset( value.provider === providerId && value.endpoint === endpointId && value.model === connection.model && - value.source === "byok", + value.source === "byok" && + (extendExistingCapabilities || (Array.isArray(value.capabilities) && value.capabilities.includes(capability))), ); if (existing && isObject(existing[1])) { const capabilities = Array.isArray(existing[1].capabilities) @@ -469,6 +486,48 @@ function ensurePreset( return presetId; } +/** Forward-only repair: inspect active marked Memory fields, never renormalize + * unrelated providers, presets, assignments, or application configuration. */ +export function repairIndependentMemoryConnections(config: RuntimeConfigDocument): void { + if (objectAt(config, "app")?.userMode !== "byok") return; + const memory = objectAt(config, "memmyMemory"); + if (!memory) return; + const routing = objectAt(memory, "roleRouting") ?? {}; + const assignments = objectAt(config, "modelAssignments") ?? {}; + const byok = objectAt(assignments, "byok") ?? {}; + for (const [capability, role, field] of [ + ["memory_summary", "summary", "memorySummary"], + ["memory_evolution", "evolution", "memoryEvolution"], + ["embedding", "embedding", "embedding"], + ] as const) { + const raw = objectAt(memory, role); + if (!raw) continue; + if (role === "embedding") { + // A missing/null/builtin-local selection deliberately means local, even + // if a stale custom connection remains in the file. Do not activate it. + const selected = byok.embedding; + if (raw.mode !== "custom" || typeof selected !== "string" || selected === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID + || !objectAt(objectAt(config, "modelPresets") ?? {}, selected)) continue; + } else if (routing[role] !== "fixed") continue; + const source = role === "embedding" ? objectAt(raw, "custom") ?? raw : raw; + const connection = connectionFrom(source, undefined, true); + if (!connection || !preservesIndependentMemoryConnection(capability, connection) + || isPlatformMemoryConnection(config, capability, connection)) continue; + const independent = { ...connection, provider: connection.provider === "memmy_account" ? "openai_compatible" : connection.provider, + inheritProviderDefaults: false, protocol: connection.protocol === "memmy-account" ? undefined : connection.protocol }; + const presetId = ensurePreset(config, capability, independent, false); + if (isValidByokPresetForCapability(config, presetId, capability)) { + byok[field] = presetId; + assignments.byok = byok; + config.modelAssignments = assignments; + } + if (source.provider === "memmy_account") { + source.provider = "openai_compatible"; + source.sourceProvider ??= "memmy_account"; + } + } +} + export function mergeLegacyByokCatalog(config: RuntimeConfigDocument, legacy: LegacyByokCatalog): void { const assignments = ensureAssignments(config); for (const capability of [ @@ -482,17 +541,30 @@ export function mergeLegacyByokCatalog(config: RuntimeConfigDocument, legacy: Le const connection = legacy[capability]; if (!connection?.provider || !connection.apiBase || !connection.model) continue; if (isPlatformMemoryConnection(config, capability, connection)) continue; - const presetId = ensurePreset(config, capability, connection); + // A retained platform marker is no longer authoritative once the + // real connection was edited. Import it as BYOK, not an account-only preset. + const byokConnection = (connection.provider === "memmy_account" || connection.sourceProvider === "memmy_account") + && ["memory_summary", "memory_evolution", "embedding"].includes(capability) + ? { ...connection, provider: connection.provider === "memmy_account" ? "openai_compatible" : connection.provider, + inheritProviderDefaults: false, + protocol: connection.protocol === "memmy-account" ? undefined : connection.protocol } + : connection; + const presetId = ensurePreset(config, capability, byokConnection); assignPreset(assignments, "byok", capability, presetId); } } -function connectionFrom(value: JsonObject | null, fallbackProvider?: string | null): LegacyCatalogConnection | null { +function preservesIndependentMemoryConnection(capability: CatalogCapability, connection: LegacyCatalogConnection): boolean { + return ["memory_summary", "memory_evolution", "embedding"].includes(capability) + && (connection.provider === "memmy_account" || connection.sourceProvider === "memmy_account"); +} + +function connectionFrom(value: JsonObject | null, fallbackProvider?: string | null, preserveMemoryAuth = false): LegacyCatalogConnection | null { const provider = stringAt(value, "provider", "vendor") ?? fallbackProvider; const apiBase = stringAt(value, "apiBase", "api_base", "baseUrl", "endpoint"); const model = stringAt(value, "model", "modelId", "model_id"); if (!provider || !apiBase || !model) return null; - const apiKey = stringAt(value, "apiKey", "api_key"); + const apiKey = stringAt(value, "apiKey", "api_key") ?? (preserveMemoryAuth ? "" : null); const sourceProvider = stringAt(value, "sourceProvider"); const legacyProtocol = stringAt(value, "protocol", "apiType", "api_type"); const extraHeaders = value && isObject(value.extraHeaders ?? value.extra_headers) @@ -505,9 +577,10 @@ function connectionFrom(value: JsonObject | null, fallbackProvider?: string | nu provider, apiBase, model, + ...(typeof value?.inheritProviderDefaults === "boolean" ? { inheritProviderDefaults: value.inheritProviderDefaults } : {}), ...(sourceProvider ? { sourceProvider } : {}), ...(legacyProtocol ? { protocol: normalizeProtocol(legacyProtocol, canonicalProviderId(provider)) } : {}), - ...(apiKey ? { apiKey } : {}), + ...(apiKey !== null ? { apiKey } : {}), ...(extraHeaders ? { extraHeaders } : {}), ...(extraBody ? { extraBody } : {}), }; @@ -543,15 +616,15 @@ function captureLegacyConnections(config: JsonObject): LegacyByokCatalog { if (agent) legacy.agent = agent; const memory = objectAt(config, "memmyMemory"); - const summary = connectionFrom(memory ? objectAt(memory, "summary") : null); - const evolution = connectionFrom(memory ? objectAt(memory, "evolution") : null); + const summary = connectionFrom(memory ? objectAt(memory, "summary") : null, undefined, true); + const evolution = connectionFrom(memory ? objectAt(memory, "evolution") : null, undefined, true); if (summary) legacy.memory_summary = summary; if (evolution) legacy.memory_evolution = evolution; const embedding = memory ? objectAt(memory, "embedding") : null; const customEmbedding = embedding && isObject(embedding.custom) ? embedding.custom : embedding; const embeddingConnection = stringAt(embedding, "mode") === "local" ? null - : connectionFrom(customEmbedding); + : connectionFrom(customEmbedding, undefined, true); if (embeddingConnection) legacy.embedding = embeddingConnection; const tools = objectAt(config, "tools"); @@ -883,6 +956,7 @@ export function isValidByokPresetForCapability( presetId: string, capability: CatalogCapability, ): boolean { + if (capability === "embedding" && presetId === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID) return true; const presets = objectAt(config, "modelPresets"); const preset = presets && isObject(presets[presetId]) ? presets[presetId] : null; if (!preset || preset.source !== "byok") return false; @@ -983,7 +1057,27 @@ function hasCurrentAccountCatalog(config: RuntimeConfigDocument): boolean { function normalizeRuntimeCatalog(config: RuntimeConfigDocument, migrationId: string): void { flattenLegacyMemoryModelConfig(config); liftLegacyRoots(config); + const existingAssignments = objectAt(config, "modelAssignments"); + const selectedEmbedding = existingAssignments ? objectAt(existingAssignments, "byok")?.embedding : undefined; + const preserveLocalEmbedding = existingAssignments !== null && objectAt(config, "app")?.userMode === "byok" + && (selectedEmbedding === undefined || selectedEmbedding === null || selectedEmbedding === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID); const legacy = captureLegacyConnections(config); + const memory = objectAt(config, "memmyMemory"); + const independentMemory: JsonObject = {}; + for (const [capability, role] of [["memory_summary", "summary"], ["memory_evolution", "evolution"], ["embedding", "embedding"]] as const) { + const connection = legacy[capability], raw = memory ? objectAt(memory, role) : null; + if (raw && connection && preservesIndependentMemoryConnection(capability, connection) + && !isPlatformMemoryConnection(config, capability, connection)) { + independentMemory[role] = { ...structuredClone(raw), + ...(raw.provider === "memmy_account" ? { provider: "openai_compatible", sourceProvider: raw.sourceProvider ?? "memmy_account" } : {}) }; + } + } + // A real pre-catalog custom connection can be imported for the first time. + // An existing catalog's local choice must not be replaced by leftover raw + // custom fields when these historical migrations run or are repaired. + if (preserveLocalEmbedding && legacy.embedding && preservesIndependentMemoryConnection("embedding", legacy.embedding)) { + delete legacy.embedding; + } const assignmentsBefore = objectAt(config, "modelAssignments"); const accountBefore = assignmentsBefore && isObject(assignmentsBefore.account) ? structuredClone(assignmentsBefore.account) @@ -1005,6 +1099,10 @@ function normalizeRuntimeCatalog(config: RuntimeConfigDocument, migrationId: str ensureAccountCatalog(config); mergeLegacyByokCatalog(config, legacy); removeLegacyRuntimeModelFields(config); + // BYOK fixed/custom roles still read these established independent fields. + // Keep edited platform projections executable without inheriting catalog + // defaults or requiring a settings save to recreate the connection. + if (memory) Object.assign(memory, independentMemory); if (preserveDormantAccount && accountBefore) { const assignments = objectAt(config, "modelAssignments") ?? {}; assignments.account = accountBefore; diff --git a/Migrations/src/migrations/v1.2.4/0001-remove-platform-memory-duplicates.ts b/Migrations/src/migrations/v1.2.4/0001-remove-platform-memory-duplicates.ts index 4d6af0a56..a77590915 100644 --- a/Migrations/src/migrations/v1.2.4/0001-remove-platform-memory-duplicates.ts +++ b/Migrations/src/migrations/v1.2.4/0001-remove-platform-memory-duplicates.ts @@ -23,9 +23,10 @@ function duplicateCapability(config: RuntimeConfigDocument, id: string, preset: const provider = record(record(config.providers).openai); const endpoint = record(record(provider.endpoints)[endpointId]); if (endpoint.protocol !== (capability === "embedding" ? "openai-embeddings" : "openai-chat-completions")) return null; - if (Object.keys(endpoint).some((key) => !["apiBase", "protocol", "apiKey", "extraHeaders", "extraBody"].includes(key))) return null; + if (Object.keys(endpoint).some((key) => !["apiBase", "protocol", "apiKey", "extraHeaders", "extraBody", "inheritProviderDefaults"].includes(key))) return null; return isPlatformMemoryConnection(config, capability, { - provider: "openai", model: preset.model, apiBase: endpoint.apiBase, + provider: "openai", model: preset.model, apiBase: endpoint.apiBase, protocol: endpoint.protocol, + inheritProviderDefaults: endpoint.inheritProviderDefaults, apiKey: endpoint.apiKey ?? provider.apiKey, extraHeaders: { ...record(provider.extraHeaders), ...record(endpoint.extraHeaders) }, extraBody: { ...record(provider.extraBody), ...record(endpoint.extraBody) }, diff --git a/Migrations/src/migrations/v1.2.6/0001-preserve-independent-memory-connections.ts b/Migrations/src/migrations/v1.2.6/0001-preserve-independent-memory-connections.ts new file mode 100644 index 000000000..7c8c00f1d --- /dev/null +++ b/Migrations/src/migrations/v1.2.6/0001-preserve-independent-memory-connections.ts @@ -0,0 +1,18 @@ +import { repairIndependentMemoryConnections } from "../v1.0.7/0001-normalize-runtime-model-catalog.js"; +import { mutateRuntimeConfig, mutateRuntimeConfigLockHeld } from "../../runtime-config-writer.js"; +import type { MigrationDefinition } from "../../types.js"; + +export const preserveIndependentMemoryConnectionsV126: MigrationDefinition = { + id: "v1.2.6/0001-preserve-independent-memory-connections", + introducedIn: "1.2.6", + scope: "runtime-config", + description: "Preserve active edited platform Memory connections using independent catalog endpoints", + async up(context) { + const options = { createIfMissing: false as const }; + const result = context.runtimeConfigLock + ? await mutateRuntimeConfigLockHeld(context.runtimeConfigLock, repairIndependentMemoryConnections, options) + : await mutateRuntimeConfig(context.runtimeConfigFile, repairIndependentMemoryConnections, options); + if (!result.sourceExists) return { scanned: 0, changed: 0, ignored: 0, deferred: true }; + return { scanned: 1, changed: result.changed ? 1 : 0, ignored: result.changed ? 0 : 1 }; + }, +}; diff --git a/Migrations/src/platform-memory-connection.ts b/Migrations/src/platform-memory-connection.ts index 7b2addd7b..b6db4b4ef 100644 --- a/Migrations/src/platform-memory-connection.ts +++ b/Migrations/src/platform-memory-connection.ts @@ -30,10 +30,21 @@ export function isPlatformMemoryConnection( capability: string, connection: RuntimeConfigDocument, ): boolean { + if (connection.inheritProviderDefaults === false) return false; if (!MODELS[capability] || connection.model !== MODELS[capability]) return false; - if (connection.sourceProvider === "memmy_account" || connection.provider === "memmy_account") return true; - if (!["openai", "openai_compatible"].includes(String(connection.provider))) return false; + const protocol = connection.protocol; + const compatibleProtocol = capability === "embedding" ? "openai-embeddings" : "openai-chat-completions"; + if (protocol !== undefined && protocol !== "memmy-account" && protocol !== compatibleProtocol) return false; + // Editing a projected platform connection can leave either source marker + // behind. Only its effective connection, not that marker, proves identity. + if (!["openai", "openai_compatible", "memmy_account"].includes(String(connection.provider))) return false; const account = record(record(config.providers).memmy_account); + const activeUserId = record(config.app).userId; + const activeOwner = typeof activeUserId === "string" ? activeUserId.trim() : ""; + const providerOwner = typeof account.ownerAccountId === "string" ? account.ownerAccountId.trim() : ""; + // Legacy files may omit ownership, but an explicit different owner must + // never authorize discarding the current user's connection. + if (activeOwner && providerOwner && activeOwner !== providerOwner) return false; const key = connection.apiKey; if (typeof key !== "string" || !key.trim()) return false; const endpoints = Object.values(record(account.endpoints)); diff --git a/Migrations/src/registry.ts b/Migrations/src/registry.ts index 323e9cf31..e673b54fc 100644 --- a/Migrations/src/registry.ts +++ b/Migrations/src/registry.ts @@ -10,6 +10,7 @@ import { removeLegacyMemoryEmbeddingV119 } from "./migrations/v1.1.9/0001-remove import { renameMemmyComputerUseMcpV119 } from "./migrations/v1.1.9/0002-rename-memmy-computer-use-mcp.js"; import { removePlatformMemoryDuplicatesV124 } from "./migrations/v1.2.4/0001-remove-platform-memory-duplicates.js"; import { removeReintroducedPlatformMemoryDuplicatesV124 } from "./migrations/v1.2.4/0002-remove-reintroduced-platform-memory-duplicates.js"; +import { preserveIndependentMemoryConnectionsV126 } from "./migrations/v1.2.6/0001-preserve-independent-memory-connections.js"; import { MigrationError, type MigrationDefinition } from "./types.js"; const STABLE_SEMVER_PATTERN = @@ -36,6 +37,7 @@ export const migrations: readonly MigrationDefinition[] = [ renameMemmyComputerUseMcpV119, removePlatformMemoryDuplicatesV124, removeReintroducedPlatformMemoryDuplicatesV124, + preserveIndependentMemoryConnectionsV126, ]; function definitionError(message: string, migrationId: string | null = null): never { diff --git a/Migrations/tests/independent-memory-applied-repair.test.ts b/Migrations/tests/independent-memory-applied-repair.test.ts new file mode 100644 index 000000000..d43428337 --- /dev/null +++ b/Migrations/tests/independent-memory-applied-repair.test.ts @@ -0,0 +1,156 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { migrations } from "../src/registry.js"; +import { runMigrationsForTest } from "../src/runner.js"; +import { loadMemmyConfig } from "../../Memory/src/config/index.js"; +import { createLlmClient } from "../../Memory/src/model/llm.js"; +import { createEmbedder } from "../../Memory/src/model/embedder.js"; + +const repairId = "v1.2.6/0001-preserve-independent-memory-connections"; +const roots: string[] = []; +afterEach(async () => { + vi.unstubAllGlobals(); + await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); +}); +async function fixture(applied = true) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "memory-applied-repair-")); roots.push(root); + const file = path.join(root, "config.yaml"), context = { profileWorkspace: root, runtimeConfigFile: file, + sessionsDir: path.join(root, "sessions"), sessionDagDir: path.join(root, "dag"), logger: { info() {}, warn() {}, error() {} } }; + const options = { targets: { agentWorkspace: root, runtimeConfigFile: file, sessionDagDir: context.sessionDagDir }, logger: context.logger }; + await fs.writeFile(file, YAML.stringify({ app: { userMode: "byok", userId: "owner-a" }, + providers: { openai: { apiKey: "cloud-token", extraHeaders: { Authorization: "Bearer cloud-header" }, extraBody: { model: "unrelated-cloud-model" }, + endpoints: { cloud: { apiBase: "https://cloud.example.test/v1", protocol: "openai-chat-completions" } } } }, + modelPresets: { cloud: { provider: "openai", endpoint: "cloud", model: "cloud-model", source: "byok", capabilities: ["agent"] } }, + modelAssignments: { byok: { agent: { candidates: ["cloud"], default: "cloud" } } }, + memmyMemory: { storage: { sqlitePath: "do-not-change.sqlite" } } })); + // Establish actual on-disk applied records before the old writer saves the + // custom connections. Updating the historical definitions cannot rerun them. + if (applied) await runMigrationsForTest(options, { definitions: migrations.filter(item => item.id !== repairId) }); + const read = async (): Promise> => YAML.parse(await fs.readFile(file, "utf8")); + return { root, file, context, options, read }; +} + +describe("forward repair of independent Memory connections after historical migrations", () => { + it("repairs existing applied installations without rerunning normalization or changing unrelated configuration", async () => { + const f = await fixture(), before = await f.read(); + before.customUnrelated = { preserve: "byte-for-byte values", nested: [1, false, null] }; + before.modelPresets.cloud.label = "User label must stay"; + before.memmyMemory.roleRouting = { summary: "fixed", evolution: "fixed" }; + before.providers.openai.endpoints.oldEmbedding = { apiBase: "https://cloud.example.test/v1", protocol: "openai-embeddings" }; + before.modelPresets.oldEmbedding = { provider: "openai", endpoint: "oldEmbedding", model: "old-embedding", source: "byok", capabilities: ["embedding"] }; + before.modelAssignments.byok.embedding = "oldEmbedding"; + for (const role of ["summary", "evolution", "embedding"]) before.memmyMemory[role] = { + provider: "openai_compatible", sourceProvider: "memmy_account", endpoint: "http://127.0.0.1:1234/v1", + model: role === "embedding" ? "embedding" : `memory_${role}`, apiKey: "", extraHeaders: { "X-Own": "retained" }, extraBody: { seed: 1 }, + ...(role === "embedding" ? { mode: "custom" } : {}), + }; + await fs.writeFile(f.file, YAML.stringify(before)); + const result = await runMigrationsForTest(f.options, {}); + expect(result.skipped).toContain("v1.0.7/0001-normalize-runtime-model-catalog"); + expect(result.skipped).toContain("v1.0.9/0001-repair-runtime-model-catalog"); + expect(result.applied.map(item => item.id)).toEqual([repairId]); + const saved = await f.read(); + expect(saved.customUnrelated).toEqual(before.customUnrelated); + expect(saved.modelPresets.cloud).toEqual(before.modelPresets.cloud); + expect(saved.app).toEqual(before.app); + expect(saved.providers.openai.apiKey).toBe(before.providers.openai.apiKey); + expect(saved.providers.openai.extraHeaders).toEqual(before.providers.openai.extraHeaders); + expect(saved.providers.openai.extraBody).toEqual(before.providers.openai.extraBody); + expect(saved.providers.openai.endpoints.cloud).toEqual(before.providers.openai.endpoints.cloud); + expect(saved.memmyMemory).toEqual(before.memmyMemory); + for (const field of ["memorySummary", "memoryEvolution", "embedding"]) { + const preset = saved.modelPresets[saved.modelAssignments.byok[field]]; + expect(saved.providers[preset.provider].endpoints[preset.endpoint]).toMatchObject({ inheritProviderDefaults: false, apiKey: "" }); + } + const requests: Array<{ url: string; headers: Headers; body: Record }> = []; + vi.stubGlobal("fetch", async (input: unknown, init: RequestInit) => { + const url = String(input); requests.push({ url, headers: new Headers(init.headers), body: JSON.parse(String(init.body)) }); + return new Response(JSON.stringify(url.endsWith("/embeddings") ? { data: [{ embedding: [1, 0, 0] }] } + : { choices: [{ message: { content: "OK" }, finish_reason: "stop" }] }), { status: 200, headers: { "content-type": "application/json" } }); + }); + const config = loadMemmyConfig(f.file).config; + for (const role of ["summary", "evolution"] as const) await createLlmClient({ ...config[role], maxRetries: 0 }) + .complete([{ role: "user", content: "synthetic" }], { operation: "test.forward-repair" }); + expect(config.embedding).toMatchObject({ provider: "openai_compatible", mode: "custom" }); + await createEmbedder({ ...config.embedding, cache: false, maxRetries: 0 }).embed(["synthetic"]); + expect(requests).toHaveLength(3); + for (const request of requests) { + expect(request.url.startsWith("http://127.0.0.1:1234/v1")).toBe(true); + expect(request.headers.has("authorization")).toBe(false); + expect(request.body.model).not.toBe("unrelated-cloud-model"); + } + const once = await fs.readFile(f.file, "utf8"); + expect((await runMigrationsForTest(f.options, {})).applied).toHaveLength(0); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + const repair = migrations.find(item => item.id === repairId)!; + expect(await repair.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(["null", "missing", "builtin-local", "unmarked", "platform", "follow", "account"])( + "does not reinterpret unrelated or inactive choices (%s)", async choice => { + const f = await fixture(), before = await f.read(); + const connection = { provider: "openai_compatible", sourceProvider: "memmy_account", endpoint: "http://127.0.0.1:1234/v1", + model: "memory_summary", apiKey: "", extraHeaders: { "X-Own": "retained" } }; + if (["null", "missing", "builtin-local"].includes(choice)) { + before.memmyMemory.embedding = { ...connection, model: "embedding", mode: "custom" }; + if (choice === "missing") delete before.modelAssignments.byok.embedding; + else before.modelAssignments.byok.embedding = choice === "null" ? null : "memmy-builtin-local-embedding"; + } else { + before.memmyMemory.roleRouting = { summary: choice === "follow" ? "follow" : "fixed" }; + before.memmyMemory.summary = connection; + if (choice === "unmarked") delete before.memmyMemory.summary.sourceProvider; + if (choice === "account") before.app.userMode = "account"; + if (choice === "platform") { + before.memmyMemory.summary.apiKey = "platform-token"; + before.memmyMemory.summary.extraHeaders = {}; + before.providers.memmy_account = { ownerAccountId: "owner-a", apiKey: "platform-token", + endpoints: { platform: { apiBase: connection.endpoint, protocol: "memmy-account" } } }; + } + } + await fs.writeFile(f.file, YAML.stringify(before)); + const once = await fs.readFile(f.file, "utf8"), result = await runMigrationsForTest(f.options, {}); + expect(result.applied.map(item => item.id)).toEqual([repairId]); + expect(result.applied[0]!.result).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + if (["null", "missing", "builtin-local"].includes(choice)) expect(loadMemmyConfig(f.file).config.embedding.provider).toBe("local"); + }); + + it.each(["openai-responses", "unsupported-memory-protocol"])("retains %s without enabling unsupported assignments", async protocol => { + const f = await fixture(), before = await f.read(); + before.memmyMemory.roleRouting = { summary: "fixed" }; + before.memmyMemory.summary = { provider: "openai_compatible", sourceProvider: "memmy_account", endpoint: "http://127.0.0.1:1234/v1", + model: "memory_summary", apiKey: "", protocol }; + await fs.writeFile(f.file, YAML.stringify(before)); await runMigrationsForTest(f.options, {}); + const saved = await f.read(); + const entry = Object.entries(saved.modelPresets).find(([, item]: any) => item.capabilities.includes("memory_summary")); + expect(entry).toBeDefined(); + const [id, preset] = entry! as [string, Record]; + expect(saved.providers[preset.provider].endpoints[preset.endpoint]).toMatchObject({ protocol, inheritProviderDefaults: false }); + expect(saved.modelAssignments.byok.memorySummary).toBe(protocol === "openai-responses" ? id : null); + const once = await fs.readFile(f.file, "utf8"); + expect(await migrations.find(item => item.id === repairId)!.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it("is a no-op after fresh normalization already created the independent connection", async () => { + const f = await fixture(false), before = await f.read(); + before.memmyMemory.roleRouting = { summary: "fixed" }; + before.memmyMemory.summary = { provider: "openai_compatible", sourceProvider: "memmy_account", endpoint: "http://127.0.0.1:1234/v1", + model: "memory_summary", apiKey: "" }; + await fs.writeFile(f.file, YAML.stringify(before)); + const result = await runMigrationsForTest(f.options, {}); + expect(result.applied.find(item => item.id === repairId)!.result).toMatchObject({ changed: 0 }); + const saved = await f.read(); + expect(Object.values(saved.modelPresets).filter((item: any) => item.capabilities.includes("memory_summary"))).toHaveLength(1); + }); + + it("defers without creating a missing configuration file", async () => { + const f = await fixture(); await fs.unlink(f.file); + expect(await migrations.find(item => item.id === repairId)!.up(f.context)).toMatchObject({ deferred: true, changed: 0 }); + await expect(fs.stat(f.file)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/Migrations/tests/platform-memory-legacy-preservation.test.ts b/Migrations/tests/platform-memory-legacy-preservation.test.ts new file mode 100644 index 000000000..afd12bbd5 --- /dev/null +++ b/Migrations/tests/platform-memory-legacy-preservation.test.ts @@ -0,0 +1,240 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { normalizeRuntimeModelCatalogV107 } from "../src/migrations/v1.0.7/0001-normalize-runtime-model-catalog.js"; +import { repairRuntimeModelCatalogV109 } from "../src/migrations/v1.0.9/0001-repair-runtime-model-catalog.js"; +import { removePlatformMemoryDuplicatesV124 } from "../src/migrations/v1.2.4/0001-remove-platform-memory-duplicates.js"; +import { loadMemmyConfig } from "../../Memory/src/config/index.js"; + +const roots: string[] = []; +const platformAddress = "https://platform.example.test/api/agentExternal/v1"; +const roles = [ + { role: "summary", capability: "memory_summary", assignment: "memorySummary" }, + { role: "evolution", capability: "memory_evolution", assignment: "memoryEvolution" }, + { role: "embedding", capability: "embedding", assignment: "embedding" }, +] as const; + +function initialConfig(): Record { + return { + app: { userMode: "byok", userId: "owner-a" }, + providers: { memmy_account: { ownerAccountId: "owner-a", apiKey: "platform-fixture-token", + endpoints: { platform: { apiBase: platformAddress, protocol: "memmy-account" } } } }, + modelAssignments: { account: { ownerAccountId: "owner-a" } }, + memmyMemory: { storage: { sqlitePath: "preserved-memory.sqlite" }, + roleRouting: { summary: "fixed", evolution: "fixed" } }, + }; +} + +function legacyConnection(model: string): Record { + return { provider: "openai_compatible", sourceProvider: "memmy_account", endpoint: platformAddress, + model, apiKey: "platform-fixture-token", ...(model === "embedding" ? { mode: "custom" } : {}) }; +} + +async function fixture(value: unknown) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "memmy-platform-legacy-")); roots.push(root); + const file = path.join(root, "config.yaml"); + await fs.writeFile(file, YAML.stringify(value)); + return { file, read: async (): Promise> => YAML.parse(await fs.readFile(file, "utf8")), context: { + profileWorkspace: root, sessionsDir: path.join(root, "sessions"), runtimeConfigFile: file, + sessionDagDir: path.join(root, "dag"), logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + } }; +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); +}); + +describe.each([normalizeRuntimeModelCatalogV107, repairRuntimeModelCatalogV109])("legacy platform connection preservation: $id", migration => { + it.each(["null", "missing", "builtin-local"])("keeps an existing catalog's local embedding choice (%s) despite stale raw custom fields", async choice => { + const initial = initialConfig(); + initial.modelAssignments.byok = {}; + if (choice !== "missing") initial.modelAssignments.byok.embedding = choice === "null" ? null : "memmy-builtin-local-embedding"; + initial.memmyMemory.embedding = { ...legacyConnection("embedding"), endpoint: "http://127.0.0.1:1234/v1", apiKey: "" }; + const f = await fixture(initial); + expect(loadMemmyConfig(f.file).config.embedding.provider).toBe("local"); + await migration.up(f.context); + const saved = await f.read(); + expect(saved.modelAssignments.byok.embedding).toBe(choice === "builtin-local" ? "memmy-builtin-local-embedding" : null); + expect(loadMemmyConfig(f.file).config.embedding.provider).toBe("local"); + expect(saved.memmyMemory.embedding).toEqual(initial.memmyMemory.embedding); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(roles.flatMap(role => ["endpoint", "model", "apiKey"].map(field => ({ ...role, field }))))( + "preserves edited $role $field despite its old platform source marker", async ({ role, capability, assignment, field }) => { + const initial = initialConfig(), connection = legacyConnection(capability); + if (role === "embedding") delete initial.modelAssignments; + connection[field] = field === "endpoint" ? "https://custom.example.test/v1" : `custom-${field}`; + initial.memmyMemory[role] = connection; + const f = await fixture(initial); + await migration.up(f.context); + const saved = await f.read(), presetId = saved.modelAssignments.byok[assignment]; + expect(presetId).toEqual(expect.any(String)); + const preset = saved.modelPresets[presetId], provider = saved.providers[preset.provider]; + const endpoint = provider.endpoints[preset.endpoint]; + expect(preset).toMatchObject({ source: "byok", provider: "openai", model: connection.model }); + expect(endpoint.apiBase).toBe(connection.endpoint); + expect(endpoint.inheritProviderDefaults).toBe(false); + expect(endpoint.apiKey ?? provider.apiKey).toBe(connection.apiKey); + expect(saved.memmyMemory.storage).toEqual(initial.memmyMemory.storage); + // Both the repeated normalization and the later cleanup must retain it. + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + await removePlatformMemoryDuplicatesV124.up(f.context); + expect((await f.read()).modelAssignments.byok[assignment]).toBe(presetId); + }); + + it.each([{ provider: "memmy_account", marked: true }, { provider: "memmy_account", marked: false }, { provider: "openai_compatible", marked: true }])( + "imports edited $provider (source marker=$marked) with its old account protocol as valid BYOK", async ({ provider, marked }) => { + const initial = initialConfig(); + initial.memmyMemory.summary = { ...legacyConnection("memory_summary"), provider, + protocol: "memmy-account", endpoint: "https://custom.example.test/v1", apiKey: "custom-fixture-token" }; + if (!marked) delete initial.memmyMemory.summary.sourceProvider; + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(), id = saved.modelAssignments.byok.memorySummary; + expect(saved.modelPresets[id]).toMatchObject({ source: "byok", provider: "openai", model: "memory_summary" }); + const preset = saved.modelPresets[id]; + expect(saved.providers.openai.endpoints[preset.endpoint].protocol).toBe("openai-chat-completions"); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(["extraHeaders", "extraBody"])("preserves edited %s despite the platform source marker", async field => { + const initial = initialConfig(), connection = legacyConnection("memory_summary"); + connection[field] = { custom: "retained" }; + initial.memmyMemory.summary = connection; + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(), id = saved.modelAssignments.byok.memorySummary; + const preset = saved.modelPresets[id]; + expect(preset).toMatchObject({ source: "byok", provider: "openai", model: "memory_summary" }); + expect(saved.providers.openai.endpoints[preset.endpoint][field]).toEqual(connection[field]); + await removePlatformMemoryDuplicatesV124.up(f.context); + expect((await f.read()).modelAssignments.byok.memorySummary).toBe(id); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(roles.flatMap(role => ["missing", "empty"].flatMap(key => [false, true].map(existing => ({ ...role, key, existing })))))( + "preserves no-auth $role with $key key and existing endpoint=$existing", async ({ role, capability, assignment, key, existing }) => { + const initial = initialConfig(), connection = legacyConnection(capability); + if (role === "embedding") delete initial.modelAssignments; + connection.endpoint = "http://127.0.0.1:1234/v1"; + if (key === "missing") delete connection.apiKey; else connection.apiKey = ""; + const protocol = role === "embedding" ? "openai-embeddings" : "openai-chat-completions"; + initial.providers.openai = { apiKey: "unrelated-openai-fixture-token", endpoints: { + cloud: { apiBase: "https://api.openai.com/v1", protocol: "openai-chat-completions" }, + ...(existing ? { authenticated: { apiBase: connection.endpoint, protocol } } : {}), + } }; + initial.memmyMemory[role] = connection; + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(), id = saved.modelAssignments.byok[assignment], preset = saved.modelPresets[id]; + const provider = saved.providers[preset.provider], endpoint = provider.endpoints[preset.endpoint]; + expect(endpoint.inheritProviderDefaults).toBe(false); + expect(endpoint.apiKey).toBe(""); + expect(endpoint.apiKey ?? provider.apiKey).toBe(""); + expect(provider.apiKey).toBe("unrelated-openai-fixture-token"); + if (existing) { + expect(preset.endpoint).not.toBe("authenticated"); + expect(provider.endpoints.authenticated.apiKey ?? provider.apiKey).toBe("unrelated-openai-fixture-token"); + } + await removePlatformMemoryDuplicatesV124.up(f.context); + expect((await f.read()).modelAssignments.byok[assignment]).toBe(id); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(roles.flatMap(role => ["openai-responses", "custom-memory-protocol"].map(protocol => ({ ...role, protocol }))))( + "preserves explicit $protocol for $role instead of deleting a same-address connection", async ({ role, capability, assignment, protocol }) => { + const initial = initialConfig(); + if (role === "embedding") delete initial.modelAssignments; + initial.memmyMemory[role] = { ...legacyConnection(capability), protocol }; + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(); + const entry = Object.entries(saved.modelPresets).find(([, value]: any) => value.source === "byok" + && value.capabilities.includes(capability) && saved.providers[value.provider].endpoints[value.endpoint].protocol === protocol); + expect(entry).toBeDefined(); + const [id, preset] = entry! as [string, Record]; + const endpoint = saved.providers[preset.provider].endpoints[preset.endpoint]; + // Unsupported protocols are retained for correction, not activated. + expect(saved.modelAssignments.byok[assignment]).toBe(role !== "embedding" && protocol === "openai-responses" ? id : null); + expect(endpoint.protocol).toBe(protocol); + await removePlatformMemoryDuplicatesV124.up(f.context); + expect((await f.read()).modelPresets[id]).toEqual(preset); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(["memmy-account", "role-default"])("still omits equivalent explicit %s platform projections", async protocol => { + const initial = initialConfig(); + for (const { role, capability } of roles) initial.memmyMemory[role] = { ...legacyConnection(capability), + protocol: protocol === "role-default" ? role === "embedding" ? "openai-embeddings" : "openai-chat-completions" : protocol }; + const f = await fixture(initial); await migration.up(f.context); + expect(Object.values((await f.read()).modelPresets ?? {}).filter((preset: any) => preset.source === "byok")).toEqual([]); + }); + + it("preserves a connection matching another account's dormant platform catalog", async () => { + const initial = initialConfig(); initial.app.userId = "owner-b"; + initial.memmyMemory.summary = legacyConnection("memory_summary"); + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(), id = saved.modelAssignments.byok.memorySummary; + expect(saved.modelPresets[id]).toMatchObject({ source: "byok", provider: "openai" }); + expect(saved.providers.memmy_account.ownerAccountId).toBe("owner-a"); + }); + + it("preserves a marked custom connection when no account catalog can verify it", async () => { + const initial = initialConfig(); delete initial.providers.memmy_account; + initial.memmyMemory.summary = legacyConnection("memory_summary"); + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(), id = saved.modelAssignments.byok.memorySummary; + expect(saved.modelPresets[id]).toMatchObject({ source: "byok", provider: "openai" }); + }); + + it("keeps independent endpoints distinct even when their current auth matches inherited defaults", async () => { + const initial = initialConfig(); + initial.providers.openai = { apiKey: "platform-fixture-token", endpoints: { + inherited: { apiBase: platformAddress, protocol: "openai-chat-completions" }, + } }; + initial.memmyMemory.summary = { ...legacyConnection("memory_summary"), inheritProviderDefaults: false }; + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(), id = saved.modelAssignments.byok.memorySummary, preset = saved.modelPresets[id]; + expect(preset.endpoint).not.toBe("inherited"); + const endpoint = saved.providers.openai.endpoints[preset.endpoint]; + expect(endpoint).toMatchObject({ inheritProviderDefaults: false, apiKey: "platform-fixture-token", extraHeaders: {}, extraBody: {} }); + expect(saved.providers.openai.endpoints.inherited).toEqual(initial.providers.openai.endpoints.inherited); + await removePlatformMemoryDuplicatesV124.up(f.context); + expect((await f.read()).modelPresets[id]).toEqual(preset); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); + + it.each(["current", "legacy-owner-fields", "provider-level-address"])( + "still omits unchanged platform connections with %s configuration", async shape => { + const initial = initialConfig(); + if (shape === "legacy-owner-fields") { + delete initial.app.userId; delete initial.providers.memmy_account.ownerAccountId; + delete initial.modelAssignments.account.ownerAccountId; + } + if (shape === "provider-level-address") { + initial.providers.memmy_account.apiBase = platformAddress; + delete initial.providers.memmy_account.endpoints; + } + for (const { role, capability } of roles) initial.memmyMemory[role] = legacyConnection(capability); + const f = await fixture(initial); await migration.up(f.context); + const saved = await f.read(); + expect(Object.values(saved.modelPresets ?? {}).filter((preset: any) => preset.source === "byok")).toEqual([]); + expect(saved.memmyMemory.storage).toEqual(initial.memmyMemory.storage); + const once = await fs.readFile(f.file, "utf8"); + expect(await migration.up(f.context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(f.file, "utf8")).toBe(once); + }); +}); diff --git a/Migrations/tests/platform-memory-request-preservation.test.ts b/Migrations/tests/platform-memory-request-preservation.test.ts new file mode 100644 index 000000000..5d16c0c59 --- /dev/null +++ b/Migrations/tests/platform-memory-request-preservation.test.ts @@ -0,0 +1,120 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { normalizeRuntimeModelCatalogV107 } from "../src/migrations/v1.0.7/0001-normalize-runtime-model-catalog.js"; +import { repairRuntimeModelCatalogV109 } from "../src/migrations/v1.0.9/0001-repair-runtime-model-catalog.js"; +import { removePlatformMemoryDuplicatesV124 } from "../src/migrations/v1.2.4/0001-remove-platform-memory-duplicates.js"; +import { DEFAULT_MEMMY_CONFIG, loadMemmyConfig } from "../../Memory/src/config/index.js"; +import { resolveAssignedModel, type RuntimeModelCatalog } from "../../Memory/src/contracts/model-catalog-resolver.js"; +import { createLlmClient } from "../../Memory/src/model/llm.js"; +import { createEmbedder } from "../../Memory/src/model/embedder.js"; + +const roots: string[] = []; +const localBase = "http://127.0.0.1:1234/v1"; +type CapturedRequest = { url: string; headers: Record; body: Record }; +afterEach(async () => { + vi.unstubAllGlobals(); + await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); +}); + +async function captureRequests(file: string, useCatalog = false): Promise { + const calls: CapturedRequest[] = []; + vi.stubGlobal("fetch", async (input: unknown, init: RequestInit) => { + const url = String(input); + calls.push({ url, headers: Object.fromEntries(new Headers(init.headers)), body: JSON.parse(String(init.body)) }); + return new Response(JSON.stringify(url.endsWith("/embeddings") ? { data: [{ embedding: [1, 0, 0] }] } + : { choices: [{ message: { content: "OK" }, finish_reason: "stop" }] }), + { status: 200, headers: { "content-type": "application/json" } }); + }); + try { + const config = loadMemmyConfig(file).config; + const raw = YAML.parse(await fs.readFile(file, "utf8")); + const connection = (capability: "memory_summary" | "memory_evolution" | "embedding") => { + const selected = resolveAssignedModel({ catalog: raw as RuntimeModelCatalog, mode: "byok", activeAccountId: "owner-a", capability }); + expect(selected.ok).toBe(true); + if (!selected.ok) throw new Error(`Migrated ${capability} assignment unavailable`); + return { endpoint: selected.provider.apiBase, model: selected.context.model, apiKey: selected.provider.apiKey, + extraHeaders: { ...selected.provider.extraHeaders }, extraBody: { ...selected.provider.extraBody } }; + }; + for (const role of ["summary", "evolution"] as const) { + const client = createLlmClient({ ...config[role], ...(useCatalog ? connection(`memory_${role}`) : {}), maxRetries: 0 }); + expect(client.isConfigured()).toBe(true); + await client.complete([{ role: "user", content: "synthetic migration check" }], { operation: "test.migration" }); + } + expect(config.embedding).toMatchObject({ provider: "openai_compatible", mode: "custom", endpoint: localBase }); + await createEmbedder({ ...config.embedding, ...(useCatalog ? connection("embedding") : {}), cache: false, maxRetries: 0 }).embed(["synthetic migration check"]); + const cloud = raw.modelAssignments + ? resolveAssignedModel({ catalog: raw as RuntimeModelCatalog, mode: "byok", activeAccountId: "owner-a", capability: "agent" }) + : undefined; + if (raw.modelAssignments) expect(cloud?.ok).toBe(true); + const provider = cloud?.ok ? cloud.provider : { ...raw.providers.openai, ...raw.providers.openai.endpoints.cloud }; + const cloudModel = cloud?.ok ? cloud.context.model : raw.agents.defaults.model; + await createLlmClient({ ...DEFAULT_MEMMY_CONFIG.summary, provider: "openai_compatible", sourceProvider: "openai", + endpoint: provider.apiBase, model: cloudModel, apiKey: provider.apiKey, + extraHeaders: { ...provider.extraHeaders }, extraBody: { ...provider.extraBody }, maxRetries: 0 }) + .complete([{ role: "user", content: "synthetic cloud check" }], { operation: "test.migration" }); + return calls; + } finally { vi.unstubAllGlobals(); } +} + +describe.each([normalizeRuntimeModelCatalogV107, repairRuntimeModelCatalogV109])("real Memory request preservation: $id", migration => { + it.each(["missing-key", "empty-key", "custom-options", "provider-only-options"])("keeps fixed/custom requests executable and isolated (%s)", async shape => { + const noAuth = shape === "missing-key" || shape === "empty-key"; + const cloud: Record = { apiKey: "unrelated-cloud-fixture-token", endpoints: { + cloud: { apiBase: "https://cloud.example.test/v1", protocol: "openai-chat-completions" }, + sameAddress: { apiBase: localBase, protocol: "openai-chat-completions" }, + } }; + if (!noAuth) { + cloud.extraHeaders = { Authorization: "Bearer unrelated-header-token", "X-Cloud-Account": "other" }; + cloud.extraBody = { model: "unrelated-model", temperature: 0.9 }; + } + const memory: Record = { storage: { sqlitePath: "untouched-memory.sqlite" }, roleRouting: { summary: "fixed", evolution: "fixed" } }; + for (const role of ["summary", "evolution", "embedding"]) { + memory[role] = { provider: "openai_compatible", sourceProvider: "memmy_account", endpoint: localBase, + model: role === "embedding" ? "embedding" : `memory_${role}`, + ...(shape === "empty-key" ? { apiKey: "" } : noAuth ? {} : { apiKey: "synthetic-local-key" }), + ...(shape === "custom-options" ? { extraHeaders: { "X-User": "retained" }, extraBody: { seed: 1 } } : {}), + ...(role === "embedding" ? { mode: "custom" } : {}) }; + } + const initial = { app: { userMode: "byok", userId: "owner-a" }, providers: { openai: cloud }, + modelPresets: { cloud: { provider: "openai", endpoint: "cloud", model: "cloud-model", source: "byok", capabilities: ["agent"] } }, + agents: { defaults: { provider: "openai", model: "cloud-model", apiBase: "https://cloud.example.test/v1" } }, memmyMemory: memory }; + const root = await fs.mkdtemp(path.join(os.tmpdir(), "memory-migration-requests-")); roots.push(root); + const file = path.join(root, "config.yaml"); await fs.writeFile(file, YAML.stringify(initial)); + const before = await captureRequests(file); + const context = { profileWorkspace: root, sessionsDir: path.join(root, "sessions"), runtimeConfigFile: file, + sessionDagDir: path.join(root, "dag"), logger: { info() {}, warn() {}, error() {} } }; + await migration.up(context); + const saved = YAML.parse(await fs.readFile(file, "utf8")); + expect(saved.providers.openai).toMatchObject(cloud); + expect(saved.memmyMemory).toEqual(memory); + const { endpoints: originalEndpoints, ...originalDefaults } = cloud; + const { endpoints: savedEndpoints, ...savedDefaults } = saved.providers.openai; + expect(savedDefaults).toEqual(originalDefaults); + for (const [id, endpoint] of Object.entries(originalEndpoints)) expect(savedEndpoints[id]).toEqual(endpoint); + for (const assignment of ["memorySummary", "memoryEvolution", "embedding"]) { + const id = saved.modelAssignments.byok[assignment]; + expect(id).toEqual(expect.any(String)); + const preset = saved.modelPresets[id]; + expect(preset.provider).toBe("openai"); + expect(savedEndpoints[preset.endpoint].inheritProviderDefaults).toBe(false); + } + const after = await captureRequests(file); + expect(after).toEqual(before); + expect(await captureRequests(file, true)).toEqual(before); + expect(after).toHaveLength(4); + for (const request of after.slice(0, 3)) { + expect(request.url.startsWith(localBase)).toBe(true); + expect(request.headers.authorization).toBe(noAuth ? undefined : "Bearer synthetic-local-key"); + expect(request.headers["x-cloud-account"]).toBeUndefined(); + expect(request.body.model).not.toBe("unrelated-model"); + } + await removePlatformMemoryDuplicatesV124.up(context); + expect(await captureRequests(file)).toEqual(before); + const once = await fs.readFile(file, "utf8"); + expect(await migration.up(context)).toMatchObject({ changed: 0 }); + expect(await fs.readFile(file, "utf8")).toBe(once); + }); +}); diff --git a/Migrations/tests/runner.test.ts b/Migrations/tests/runner.test.ts index 630d88242..43df24fa8 100644 --- a/Migrations/tests/runner.test.ts +++ b/Migrations/tests/runner.test.ts @@ -117,9 +117,10 @@ describe("migration runner", () => { "v1.1.9/0002-rename-memmy-computer-use-mcp", "v1.2.4/0001-remove-platform-memory-duplicates", "v1.2.4/0002-remove-reintroduced-platform-memory-duplicates", + "v1.2.6/0001-preserve-independent-memory-connections", ]); expect(first.deferred).toEqual(["v1.0.7/0002-import-legacy-app-state-model-config"]); - expect(first.results).toEqual({ scanned: 10, changed: 4, ignored: 6 }); + expect(first.results).toEqual({ scanned: 11, changed: 4, ignored: 7 }); expect(second).toEqual({ applied: [], skipped: [ @@ -134,6 +135,7 @@ describe("migration runner", () => { "v1.1.9/0002-rename-memmy-computer-use-mcp", "v1.2.4/0001-remove-platform-memory-duplicates", "v1.2.4/0002-remove-reintroduced-platform-memory-duplicates", + "v1.2.6/0001-preserve-independent-memory-connections", ], deferred: ["v1.0.7/0002-import-legacy-app-state-model-config"], results: { scanned: 0, changed: 0, ignored: 0 }, @@ -235,6 +237,12 @@ describe("migration runner", () => { key: runtimeConfigTargetKey(configPath), }, }, + { + id: "v1.2.6/0001-preserve-independent-memory-connections", + introducedIn: "1.2.6", + appliedAt: expect.stringMatching(/Z$/), + target: { type: "runtime-config", key: runtimeConfigTargetKey(configPath) }, + }, ]); }); diff --git a/docs/qa/memory-queue-baseline-20261022.json b/docs/qa/memory-queue-baseline-20261022.json new file mode 100644 index 000000000..2575cdbad --- /dev/null +++ b/docs/qa/memory-queue-baseline-20261022.json @@ -0,0 +1,100 @@ +{ + "baseline": "upstream/v1.2.6@3a5f5c44bd5cc557a529c2c715ce3f08cbb9fa93", + "method": "real SQLite and WorkerRunner; local substitute business/model handlers", + "observations": [ + { + "scenario": "10000_import_backlog", + "seededImports": 10000, + "rounds": 50, + "completedImports": 200, + "remainingImports": 9800, + "retryLeased": 0, + "reflectionStarted": 0, + "maintenanceStarted": 0, + "retryDirectClaimControl": true, + "reflectionDirectClaimControl": true, + "nativeTaskControl": "runs next round", + "elapsedMs": 542 + }, + { + "scenario": "retry_free_capacity_control", + "leased": 1, + "succeeded": 1 + }, + { + "scenario": "startup_repair_priority_control", + "startupBacklog": 10000, + "firstClaim": "reflection", + "leased": 1 + }, + { + "scenario": "onboarding_priority_control", + "firstClaim": "onboarding-first-report" + }, + { + "scenario": "new_imports_overtake_old", + "rounds": 25, + "newerCompleted": 25, + "oldestAttempts": 0 + }, + { + "scenario": "retry_order_control", + "firstClaim": "newer-memory", + "olderRetry": "pending" + }, + { + "scenario": "priority_cohort_underfill", + "limit": 4, + "leased": 1, + "otherRunnable": 3 + }, + { + "scenario": "multiple_entries_concurrency", + "calls": 2, + "perCallLimit": 4, + "peakHandlers": 8 + }, + { + "scenario": "summary_batch_barrier", + "limit": 8, + "finished": 3, + "active": 1, + "unstartedButLeased": 4, + "fifthStarted": false + }, + { + "scenario": "serial_prefetch_expiry", + "actualStarted": 1, + "leased": 4, + "elapsedClockSeconds": 61, + "reclaimable": 4 + }, + { + "scenario": "stale_execution_overwrite", + "newExecution": "succeeded", + "afterOldFailure": "failed", + "leaseGenerationGuard": false + }, + { + "scenario": "generic_failure_backoff", + "attempts": [ + 1, + 2, + 3 + ], + "clockAdvancedMs": 0, + "terminal": "dead_letter" + }, + { + "scenario": "restart_valid_leases", + "validMainLease": "requeued", + "validRetryLease": "pending", + "remainingSecondsBeforeRecovery": 600 + }, + { + "scenario": "embedding_existing_claim_guard", + "staleCompletionRejected": true, + "currentOwner": "B" + } + ] +} \ No newline at end of file diff --git a/docs/qa/memory-queue-baseline-20261022.md b/docs/qa/memory-queue-baseline-20261022.md new file mode 100644 index 000000000..4d0bf7de8 --- /dev/null +++ b/docs/qa/memory-queue-baseline-20261022.md @@ -0,0 +1,96 @@ +# Memory 队列改造：阶段 1 复现与基线 + +日期：2026-10-22（Asia/Shanghai）。本记录对应桌面《Memmy后台任务队列调度机制工作方案》的阶段 1。 + +## 开发基线与范围 + +- PR 目标：`upstream/v1.2.6`；执行前已 fetch；按用户最新要求，从该分支直接创建新的个人任务分支。旧 v1.2.4 与新目标存在分叉，未将旧分支的其他代码合入。 +- 起点 commit：`3a5f5c44bd5cc557a529c2c715ce3f08cbb9fa93`。 +- 个人分支：`test-memory-queue-v1.2.6`。 +- 独立工作区：`/private/tmp/memmy-memory-queue-v124`。 +- 环境：macOS arm64，Node.js 22.23.3，npm 10.9.9。 +- 该 commit 中 Desktop/package.json 版本为 **1.2.3**，Memory/package.json 为 **2.1.6**；目标分支名不等于运行时版本。 +- 本阶段仅新增复现测试和本记录；未修改生产实现，未打包，未推送或提交 PR。 + +## 方法及证据边界 + +测试使用项目真实 `MemoryDb`、SQLite schema、`Repositories` 和 `WorkerRunner`。业务处理器使用可控 Promise，本地 Embedding 替身返回固定向量；队列查询、领取、重排、完成与失败状态更新均调用当前生产实现。测试数据库各自放在临时目录，结束后清理。 + +这是调度层的确定性复现，未接入当前 Memmy 窗口，未调用付费模型，未改动真实用户数据。不能将替身处理器的耗时换算成产品吞吐、模型延迟或 UI 等待时间。租约过期用例将模拟 Date 时钟推进 61 秒，并未在真实时间中等待一分钟。未重放用户 Windows 环境的连续六小时日志，未声称已复现 UI 卡顿。 + +新增测试：`Memory/tests/service/worker/queue-scheduling-baseline.repro.test.ts`。测试明确标记为**修复前诊断基线**：部分断言确认当前不理想行为确实发生，不能把这些断言通过当作修复验收。后续修复时应将对应断言转换为预期行为的回归测试。 + +## 已复现的问题 + +| 编号 | 复现操作 | 实际结果 | 修复后的验收方向 | +| --- | --- | --- | --- | +| R1 | 入队 10,000 个普通 import_summary，同时放入可运行 reflection、已到期 Embedding retry 和一条启动维护任务；按自动 worker 配置连续执行 50 轮，每轮 limit=4、priorityCohortOnly=true。 | 200 个导入任务完成，剩余 9,800 个；50 轮 retryLeased 全为 0，reflection 和维护任务 attempts 均为 0。直接按目标队列领取可拿到 retry 与 reflection，排除了任务本身不可运行。 | 混合积压中各可运行 Lane 都有执行机会；retry 不依赖主队列剩余容量。 | +| R2 | 放入一条旧导入摘要；连续 25 次各插入一个目标记忆较新的同类摘要，然后领取一条并完成。 | 25 条较新任务均先完成，旧任务仍 queued、attempts=0。 | 同 Lane 对可运行任务按稳定入队顺序调度。 | +| R3 | 给两条不同创建时间的导入记忆各创建一个已到期 retry，先创建旧 retry，再创建新 retry，领取一条。 | 先领取新记忆对应 retry，旧 retry 继续 pending。 | retry 候选同样需要公平次序，不能只修改主任务表排序。 | +| R4 | 放入 1 个高优摘要和 3 个低优 reflection，limit=4、cohort=true。 | 只领取 1 条；另外 3 条可运行任务未补入本轮。 | 空 Lane 或不可运行候选不应使可用执行容量闲置。 | +| R5 | 入队 8 个同类摘要；第一次 runWorkerOnce(4) 启动并阻塞处理器，再同时调用第二次 runWorkerOnce(4)。 | 两次调用合计同时进入 8 个处理器。单次 limit=4 不是服务全局并发 4。 | 所有入口共享同一全局、资源和 Provider 并发边界。 | +| R6 | 入队 8 个摘要，单次 limit=8。前 4 个中保留 1 个慢任务，让另外 3 个结束。 | 仅剩 1 个处理器活跃，第 5 条仍未开始；后 4 条已经 leased。必须等待慢任务，才能开始下一组。 | 一个任务完成后即可补位，并只在有执行容量时领取。 | +| R7 | 入队 4 个串行 reflection，阻塞第一条；将模拟时钟推进 61 秒，再调用真实仓储领取。 | 首轮仅启动 1 个处理器，但 4 条都已 leased；过期后这 4 条均可被再次领取，attempts 变为 2。 | 未执行任务不提前持有租约；执行中任务续租；分别记录领取和业务失败次数。 | +| R8 | A 领取任务；模拟过期后 B 重新领取并成功；随后通过真实 runner 提交 A 的迟到失败。 | B 的 succeeded 被 A 改成 failed。 | 按 job id、当前状态、owner 和唯一 token 验证写入；拒绝旧执行者。 | +| R9 | 普通 reflection 处理器连续抛临时超时；固定 Date 时钟不变，连续执行三轮。 | attempts 为 1、2、3，同一时刻耗尽尝试并进入 dead_letter，无通用退避。 | 临时错误退避后重试，并与永久错误、配置阻塞和租约丢失区分。 | +| R10 | 分别创建未来 600 秒仍有效的主任务租约和 retry 租约，调用真实启动对账。 | 主任务立即 queued，retry 立即 pending；有效租约被清除。 | 启动恢复遵守文档的租约有效性规则，避免无条件重排。 | + +R1 证明了这 50 轮中的饥饿现象，不能单凭有限轮次宣称任务在所有情况下永不完成。但当前“先主任务、后剩余名额”的规则意味着，只要每轮主任务持续占满，retry 就持续没有名额；同理，严格优先级可持续挤压低优任务。 + +R5 的直接执行证据位于共享 runner 层。代码审查确认自动排空器、历史导入、Onboarding 和 HTTP worker/run 可以进入该 runner；本阶段未用 HTTP 多端口端到端重放这些入口。 + +## 对照用例：当前已有能力 + +| 编号 | 操作 | 当前结果及含义 | +| --- | --- | --- | +| C1 | 仅放入一个到期 retry，执行 limit=4 的 worker。 | leased=1、succeeded=1。retry 机制可执行，R1 的 0 不是因为未到时间或完全不能运行。这里只验证队列成功转换，未验证真实向量模型与索引。 | +| C2 | 放入 10,000 个 source=startup.placeholder_summary_repair 的摘要修复，再放普通 reflection。 | reflection 先领取；当前目标分支已将启动修复降到优先级 90。历史截图中“启动修复优先于反思”的具体路径已修正。 | +| C3 | 同时放入 Onboarding 首次报告、原生摘要和导入摘要。 | Onboarding 首次报告先领取，现有高优先级可保留为改造回归要求。 | +| C4 | retry 由 A 领取，模拟过期后 B 重新领取，再由 A 提交完成。 | 旧 owner 的完成被拒绝，B 的 in_progress 保留。retry 已有 workerId+leaseUntil 校验，不应笼统认定所有队列都没有租约保护。 | + +R1 还加入原生 trace_summary 对照：在导入积压仍存在时，原生摘要下一轮可以领取。它说明当前严格优先级能保护部分新摘要，同时牺牲其他类别；WRR 的作用是同时保证其余 Lane 的进展。 + +## 最新代码与方案描述的差异 + +1. 最新目标分支的并行白名单为 trace_summary、import_summary、observation_extract、embedding、l3_world_model_update、project_environment_profile 六类，不能继续描述为只有摘要、Embedding 并行。 +2. 默认 limit=100 代表最多预先领取 100 个任务。普通可并行任务在单次调用中每 4 个一组，不能解释为 100 个模型请求同时执行。 +3. 启动摘要修复已被降为低优先级，但普通导入积压、retry 剩余名额规则及其他租约问题仍在。 +4. 文档第六部分 Lane 权重为 4/3/2/2/1，第十三部分参数表为 4/2/2/2/1。实施策略注册表与 WRR 前需统一；不影响阶段 1 或阶段 2。当前未擅自修改原文档。 + +## 验证与重跑 + +下列结果是阶段 1 提交 `726468f4` 的历史记录。后续阶段已修改实现和部分断言，在当前分支重跑不能重现全部修复前结果。 + +若要复核原始 14 项诊断，应从 `726468f4` 创建独立工作区，并在该工作区按 README 准备依赖，再运行以下命令。不要在当前开发分支切换或回退代码。原始证据 JSON 保留不变。 + +```sh +git worktree add --detach /private/tmp/memmy-queue-stage1-replay 726468f4 +``` + +在上述历史工作区根目录运行： + +```sh +node node_modules/typescript/bin/tsc -p AgentSourceCore/tsconfig.json +MEMMY_QUEUE_BASELINE_REF=upstream/v1.2.6@3a5f5c44bd5cc557a529c2c715ce3f08cbb9fa93 \ +MEMMY_QUEUE_BASELINE_RESULTS=/private/tmp/memmy-queue-baseline-v126-20261022/results.json \ + node node_modules/vitest/vitest.mjs run \ + Memory/tests/service/worker/queue-scheduling-baseline.repro.test.ts \ + Memory/tests/service/worker/worker-runtime.test.ts --reporter=verbose +node node_modules/typescript/bin/tsc -p Memory/tsconfig.json --noEmit +``` + +当前分支的结果导出另外记录实际 `testedCommit` 和 `workingTreeDirty`；`MEMMY_QUEUE_BASELINE_REF` 仅作 `referenceLabel`，不能证明运行的是历史代码。历史提交的旧导出器没有这项检查，复核时必须另外记录 `git rev-parse HEAD` 和工作区状态。 + +- 新增诊断及对照 14 项、现有 worker-runtime 19 项：**33/33 通过**。 +- Memory TypeScript 检查：通过。 +- 原始结果副本：`docs/qa/memory-queue-baseline-20261022.json`。 +- 完整执行日志：`/private/tmp/memmy-queue-baseline-v126-20261022/test.log`。 +- 此时尚未进行仓库全量回归、lint、打包或安装验证；这些属于后续整体交付阶段。 + +## 下一阶段及确认点 + +改为 upstream/v1.2.6 后已重跑全部 33 项测试和 Memory 类型检查，结果仍成立。旧基线原件备份在 `/private/tmp/memmy-queue-baseline-20261022/v124-original`；原 Pages 工作文档仅三处目标分支引用更新为 v1.2.6。 + +阶段 1 已得到可重复、可核对的基线。按此前约定，向用户展示本记录，确认后进入阶段 2：先修主队列 owner/token fencing、长任务续租、过期恢复及失败退避，再改调度策略和提高受控并发。 + +租约安全与重试计数会被后续调度器使用；统一执行入口为四层并发提供控制边界；策略字段和注册表为 WRR、FIFO 提供依据。这些阶段按依赖顺序推进。整体改造完成后统一回归、macOS 打包测试，经用户确认后向 v1.2.6 提交一个 PR，标题使用 fix: 中文说明。 diff --git a/docs/qa/memory-queue-delivery-v126-20261023-results.json b/docs/qa/memory-queue-delivery-v126-20261023-results.json new file mode 100644 index 000000000..9fcf6a915 --- /dev/null +++ b/docs/qa/memory-queue-delivery-v126-20261023-results.json @@ -0,0 +1,382 @@ +{ + "date": "2026-10-23", + "branch": "test-memory-queue-v1.2.6", + "targetBranch": "upstream/v1.2.6", + "codeRevision": "a960abe84e523aeb6865eead836e482b05758096", + "targetRevision": "843d9d8712e3969b362bf1bb17af7872f2f2e080", + "status": "PR preparation only; user acceptance needed for observed latency and exit cleanup limits. No push or PR.", + "package": { + "desktopVersion": "1.2.3", + "runtime": { + "version": "2.1.6", + "buildId": "f06ace64abaa318259c5f1263f575c249c756aca894910ed83fe116dbaaaa3ce", + "schemaVersion": 12, + "protocolVersion": 1, + "target": "darwin-arm64", + "entrypoint": "dist/src/server/index.js", + "viewer": "dist/viewer/index.html" + }, + "dmgSha256": "eeeea9c25639d24cda7ab455c28e32b2824a833a6ca76c63548fc025925b76dd", + "diskImageVerified": true, + "memoryModules": [ + { + "file": "src/service/session/session-turn-service.js", + "sourceBuild": "5d3d4de66d1f402e9fc2d2a2fcd26b4c5d0e59a4cb5dd7034167bc91d4ecd5e1", + "installed": "5d3d4de66d1f402e9fc2d2a2fcd26b4c5d0e59a4cb5dd7034167bc91d4ecd5e1" + }, + { + "file": "src/service/worker/job-handlers.js", + "sourceBuild": "62c9a0b440c0f1f9519458d049723f2fe2132146f7b7b5a25aec60fe1471235c", + "installed": "62c9a0b440c0f1f9519458d049723f2fe2132146f7b7b5a25aec60fe1471235c" + }, + { + "file": "src/service/worker/job-policy-registry.js", + "sourceBuild": "c622e38df1c596cf9094397186b53aad3a533dda7a3e8e08e07cd5c91ee6226c", + "installed": "c622e38df1c596cf9094397186b53aad3a533dda7a3e8e08e07cd5c91ee6226c" + }, + { + "file": "src/service/worker/fair-worker-queue.js", + "sourceBuild": "c8bf14f1a995191df49d50f60771af156f512856fa90d4d4f8dddd1a7c6a01b9", + "installed": "c8bf14f1a995191df49d50f60771af156f512856fa90d4d4f8dddd1a7c6a01b9" + }, + { + "file": "src/service/worker/admission-lane-queue-selector.js", + "sourceBuild": "6afa4f7bc31b1af2d7298c908d2c0ae57417623345ba07288e3b5ad427a9c4b2", + "installed": "6afa4f7bc31b1af2d7298c908d2c0ae57417623345ba07288e3b5ad427a9c4b2" + }, + { + "file": "src/service/worker/worker-runner.js", + "sourceBuild": "a827b43938a960ef70f0183d53390c97a4550f9f0544dccc60580650f3b5ad6e", + "installed": "a827b43938a960ef70f0183d53390c97a4550f9f0544dccc60580650f3b5ad6e" + }, + { + "file": "src/storage/queue-scheduling-store.js", + "sourceBuild": "febe2752b960f2b3225119d4e6b1ad46bb5490848c5629abaa05fc9b644ab006", + "installed": "febe2752b960f2b3225119d4e6b1ad46bb5490848c5629abaa05fc9b644ab006" + }, + { + "file": "src/storage/repositories.js", + "sourceBuild": "11ceda0a9e06aad8d4ae12b340743c7fcb37cb3f1c692b97329e4d55099691fe", + "installed": "11ceda0a9e06aad8d4ae12b340743c7fcb37cb3f1c692b97329e4d55099691fe" + } + ], + "agentModules": [ + { + "file": "core/agent-runtime/tools/parameter-aliases.js", + "sourceBuild": "e008f6715483eef7d3c327b33b647258cd5e28a4529fcb41513984fa2ff86724", + "installed": "e008f6715483eef7d3c327b33b647258cd5e28a4529fcb41513984fa2ff86724" + }, + { + "file": "core/agent-runtime/tools/exec-session.js", + "sourceBuild": "7b5902ec206da677bc1af53dc11be717b47a815ba38632c7176c9cb6b810455e", + "installed": "7b5902ec206da677bc1af53dc11be717b47a815ba38632c7176c9cb6b810455e" + }, + { + "file": "core/agent-runtime/tools/filesystem.js", + "sourceBuild": "b0da296ef3a68a201448d016609ac3b4372cd3ce233a006ece12954326c1ce86", + "installed": "b0da296ef3a68a201448d016609ac3b4372cd3ce233a006ece12954326c1ce86" + }, + { + "file": "core/agent-runtime/tools/shell.js", + "sourceBuild": "45c765e05b60d04df86246d94566884391d236eddcf7d0db46e2c936ff746256", + "installed": "45c765e05b60d04df86246d94566884391d236eddcf7d0db46e2c936ff746256" + }, + { + "file": "providers/factory.js", + "sourceBuild": "f9f1217601723e88b01f7db8729bd454f33ed23b64b0aaa42b2879c2b76c1f50", + "installed": "f9f1217601723e88b01f7db8729bd454f33ed23b64b0aaa42b2879c2b76c1f50" + }, + { + "file": "providers/anthropic-provider.js", + "sourceBuild": "7944e520b695f52e4737edc948ea1defb2828786343bc4b43d9220dee2908a17", + "installed": "7944e520b695f52e4737edc948ea1defb2828786343bc4b43d9220dee2908a17" + }, + { + "file": "providers/openai-compat-provider.js", + "sourceBuild": "bd51ae8133713cab12336a576278d7541aca219c97f3e4588c180235eb02da91", + "installed": "bd51ae8133713cab12336a576278d7541aca219c97f3e4588c180235eb02da91" + } + ] + }, + "automated": { + "lint": "passed", + "typecheck": "passed", + "agentTypecheck": "passed", + "rootTest": "passed", + "agent": { + "passed": 7256, + "skipped": 80 + }, + "memory": { + "passed": 1812, + "skipped": 1 + }, + "migrations": { + "passed": 279, + "skipped": 1 + }, + "backend": 1109, + "frontend": 2452, + "electron": { + "passed": 1259, + "skipped": 126 + } + }, + "controlledCloudLoad": { + "initial": { + "main": 10193, + "retry": 64 + }, + "window": 36, + "bodyBytes": 32768, + "counts": { + "interactive": 12, + "repair_retry": 9, + "import": 6, + "evolution": 6, + "maintenance": 3 + }, + "elapsedMs": 743.46, + "timerMs": 158.72, + "peakJobs": 4, + "peakRequests": 2, + "peakEmbeddingRequests": 1, + "badQueued": 0 + }, + "controlledSharedProvider": { + "backlog": 200, + "delay": 10, + "done": { + "import": 50, + "evolution": 50 + }, + "maintenance": 48, + "completed": 100, + "milestones": [ + { + "completed": 10, + "import": 6, + "evolution": 4, + "maintenance": 4 + }, + { + "completed": 20, + "import": 10, + "evolution": 10, + "maintenance": 7 + }, + { + "completed": 30, + "import": 16, + "evolution": 14, + "maintenance": 11 + }, + { + "completed": 40, + "import": 20, + "evolution": 20, + "maintenance": 16 + }, + { + "completed": 50, + "import": 26, + "evolution": 24, + "maintenance": 21 + }, + { + "completed": 60, + "import": 30, + "evolution": 30, + "maintenance": 26 + }, + { + "completed": 70, + "import": 36, + "evolution": 34, + "maintenance": 31 + }, + { + "completed": 80, + "import": 40, + "evolution": 40, + "maintenance": 36 + }, + { + "completed": 90, + "import": 46, + "evolution": 44, + "maintenance": 42 + }, + { + "completed": 100, + "import": 50, + "evolution": 50, + "maintenance": 48 + } + ] + }, + "actualLocalModel": { + "passed": true, + "imports": 2, + "summarySucceeded": 2, + "embeddingSucceeded": 2, + "embeddingDimensions": 768, + "requestPeak": 1, + "elapsedMs": 50563.34 + }, + "installedScanAndLive": { + "revision": "a960abe84e523aeb6865eead836e482b05758096", + "checkedAt": "2026-10-23T06:36:00.130Z", + "initialHistory": 13000, + "finalHistory": 13000, + "scan": { + "running": false, + "jobId": "agent-scan-mvkldsbj-02b8de52", + "sourceId": "claude_code", + "mode": "full", + "progress": { + "sourceId": "claude_code", + "phase": "done", + "current": 26000, + "total": 26000, + "message": "Imported 0 memories and 0 skills" + }, + "startedAt": "2026-10-23T06:31:41.215Z", + "completedAt": "2026-10-23T06:34:07.986Z", + "error": null + }, + "id": "trace_15326b81430eee449b1f", + "processing": { + "state": "ready", + "stage": null, + "error_code": null + }, + "jobs": [ + { + "job_type": "trace_summary", + "queue_lane": "interactive", + "status": "succeeded", + "attempts": 1 + }, + { + "job_type": "embedding", + "queue_lane": "interactive", + "status": "succeeded", + "attempts": 1 + }, + { + "job_type": "reflection", + "queue_lane": "evolution", + "status": "queued", + "attempts": 0 + } + ], + "vectors": [ + { + "vector_field": "vec_summary", + "embedding_dim": 384 + } + ], + "readyObservedAfterMs": 125779.3, + "backlogAtReady": 11944, + "healthLatencyMs": { + "samples": 60, + "median": 29.14, + "p95": 1067.46, + "max": 1600.52 + }, + "originalDriverPassed": false, + "originalDriverError": "AssertionError [ERR_ASSERTION]: Full rescan did not finish", + "observation": "The original combined scan+live driver exhausted its 60 polling rounds before full rescan completion. The same run later completed without resetting data, reconfiguring delays or rerunning. Native ready was observed after125.779s; full rescan completed after146.771s. This is completion evidence, not an assertion of a126s latency target.", + "completionChecksPassed": true + }, + "retention": { + "finalHistory": 13000, + "quickCheck": "ok", + "oldCompleteSeedHash": "df625239a98a67faf03de2cef9e84bdf3946913d11ddabddd117a35ef54e1f37", + "oldCompleteSeedIdentical": true, + "rawSourceSamples": [ + { + "id": "trace_ddfc77f3fae39eb9c0d7", + "comparedToPrePackageBackup": true, + "rawTurns": 1, + "sourceTextHash": "1df2da248f3f491b7acde5afc3c45e6a4b6c06e8a37b9e165691f02ba6656a80", + "identical": true + }, + { + "id": "trace_a4b69831d9e5850172f1", + "comparedToPrePackageBackup": true, + "rawTurns": 1, + "sourceTextHash": "b5840c0e704e5f248a4aea587ea321b93a4fdd27cd1d0c1a49564049cb1f8931", + "identical": true + }, + { + "id": "trace_e783a65ec75de0309b5c", + "comparedToPrePackageBackup": true, + "rawTurns": 1, + "sourceTextHash": "92a69df8924592eae6b9f31f5a7e468add59506b61dc984ed5b0d4ebb80586a6", + "identical": true + }, + { + "id": "trace_32b734a530bc0fe32a62", + "comparedToPrePackageBackup": true, + "rawTurns": 1, + "sourceTextHash": "3c25694e54b950d993faeb07a483e5d323baa1356dc75537edc88a2de96bca12", + "identical": true + }, + { + "id": "trace_15326b81430eee449b1f", + "comparedToOriginalApiInput": true, + "rawTurns": 1, + "sourceTextHash": "aa099a844366aee5e976b9b15387bda194f1cd30fd54575856fc54f1ef098856", + "identical": true + } + ], + "rawSourcePassed": true, + "latestMemory": { + "id": "trace_15326b81430eee449b1f", + "state": { + "state": "ready", + "stage": null, + "error_code": null + }, + "vectors": [ + { + "vector_field": "vec_summary", + "embedding_dim": 384 + } + ] + }, + "independentModelAudit": "passed" + }, + "postReflectionRebuild": { + "id": "trace_e783a65ec75de0309b5c", + "beforeState": "embedding_pending", + "after": { + "state": "ready", + "stage": null, + "error_code": null + }, + "leased": 1, + "succeeded": 1, + "failed": 0, + "vectors": [ + { + "vector_field": "vec_summary", + "embedding_dim": 384, + "embedding_model": "stage8-embedding-384" + } + ], + "passed": true, + "method": "Actual packaged desktop Memory HTTP worker/run target filter; no direct database write, no priority or delay changes" + }, + "knownLimits": [ + "WRR counts opportunities and is non-preemptive. One installed scenario reached ready after125.779s under multiple42s reflection jobs; no product latency SLO agreed.", + "Original combined monitoring exhausted its60-round budget before same-run rescan completion; recorded as failed, with subsequent completion observation separate.", + "Desktop quit5s cleanup fallback reproduced with Cmd-Q only, without window-reopen interference.", + "Derived memory_value/summary and vectors can legitimately change after reflection; original raw input checks and post-reflection rebuild checked separately from static seed preservation.", + "No Aging promotion or full queue metrics/ETA UI; no cancelled producer backpressure work restored.", + "Only macOSarm64 and actual local models tested; no Windows,x64 or real cloud Provider.", + "Existing FTS/long-conversation costs and cosmetic DMG background warning remain." + ] +} diff --git a/docs/qa/memory-queue-delivery-v126-20261023.md b/docs/qa/memory-queue-delivery-v126-20261023.md new file mode 100644 index 000000000..777ea6644 --- /dev/null +++ b/docs/qa/memory-queue-delivery-v126-20261023.md @@ -0,0 +1,63 @@ +# Memory 队列改造：v1.2.6 交付准备复核 + +后续复查及最终退出补修包的最新结果见 [队列方案复查与打包验证](memory-queue-plan-recheck-v126-20261023.md)。以下保留此前候选包的历史结果。 + +日期：2026-10-23。个人分支 `test-memory-queue-v1.2.6`，目标 `MemTensor/memmy-agent:v1.2.6`。本记录承接阶段 8；工作文档没有新的开发阶段。本轮只完成同步、补测、重新打包及 PR 说明准备，没有推送或创建 PR。 + +## 当前结论 + +同步后的自动化套件全部通过。最终包的受控公平性、并发上限、实际本地模型及数据保留检查已有通过证据；但实际重扫中的组合监控未在原预算内完成，不能写成“全部实机测试通过”。同一轮随后完成，新记忆在约 126 秒达到 ready，全量重扫在约 147 秒结束。此前约 35 秒的场景不是固定等待保证。 + +提交前需用户确认的边界是：是否接受当前按任务机会分配且不抢占长任务的 WRR 等待表现，以及保留 Desktop 退出清理超时。没有为了消除这些观察而修改权重、并发、模型延迟或扩大 Desktop 生命周期范围。 + +## 上游同步及自动检查 + +获取最新上游后，`v1.2.6` 从 `f19bd810` 更新到 `843d9d8712e3969b362bf1bb17af7872f2f2e080`，新增 9 个 Agent 工具参数别名、输出限制及测试文件。使用 merge 无冲突合入，代码打包提交为 `a960abe84e523aeb6865eead836e482b05758096`，没有 rebase、强推或合并他人个人分支；这些上游文件不额外计入本 PR 的交付差异。 + +| 检查 | 结果 | +| --- | --- | +| 根目录 lint、typecheck、显式 Agent typecheck | 均通过 | +| Agent 全量 | 7,256 通过、80 原有跳过 | +| Memory 全量 | 1,812 通过、1 原有跳过（根流程中执行两次） | +| Migrations / Backend / Frontend | 279 通过、1 跳过 / 1,109 通过 / 2,452 通过 | +| Electron | 1,259 通过、126 原有跳过 | +| release / packaging guards / contracts / AgentSource / Knowledge / local contracts / TUI | 30/5 跳过、122/1、8、62、48、32、5 | +| 相对目标分支差异检查 | 无空白错误或冲突标记 | + +日志 `/private/tmp/memmy-delivery-sync-{lint,typecheck,agent-typecheck,agent-test,root-test}.log`。按顺序构建及测试，避免共享 dist 重建竞态。没有放宽生产回归断言或新增跳过；没有新增依赖、锁文件、版本号、用户数据库或安装包到 Git。 + +## 最终安装包 + +官方 arm64 国内未签名打包于 14:29:57 完成，3 分 31 秒、退出码 0。桌面 1.2.3、Memory 2.1.6，目标分支号不等于产品包内版本。DMG SHA-256：`eeeea9c25639d24cda7ab455c28e32b2824a833a6ca76c63548fc025925b76dd`，hdiutil 校验 VALID。固定副本 `/private/tmp/memmy-delivery/Memmy-1.2.3-arm64-a960abe8.dmg`；前轮 299a4a14 的包另保留为 `/private/tmp/memmy-stage8-package/Memmy-1.2.3-arm64-299a4a14.dmg`，不混淆同名 release 路径后续重打包的内容。 + +只读 DMG 复制到 `/private/tmp/memmy-delivery-installed/Memmy.app`。八个 Memory 模块及七个 ASAR 内 Agent 模块与源码编译产物逐个 SHA-256 相同，包括新上游 parameter-aliases 及本轮模型 Provider 兼容文件。Memory 源码未随本次上游合并改变，buildId 仍为 `f06ace64abaa318259c5f1263f575c249c756aca894910ed83fe116dbaaaa3ce`；桌面复用同 buildId 的 Memory 缓存，Electron 宿主路径可能沿用此前安装，但执行的 Memory 模块身份一致。证据 `/private/tmp/memmy-delivery/code-manifest.json`。 + +包内 10,000 条/32 KiB 受控混合负载完成 36 个逻辑成员，Lane 为 12/9/6/6/3，峰值逻辑任务 4、Provider 请求 2、embedding 请求 1，无未启动 attempts 和遗留许可，FIFO/补位通过。窗口 743.46ms、首次定时回调 158.72ms，只是观察值。共享模型、10ms 模型与持续 maintenance 探针完成摘要/反思 50/50，maintenance 48，失败 0。实际 LM Studio 再验 2 summary + 2 embedding 成功、2 ready、768 维向量、请求峰值 1，约 50.56 秒。 + +## 实际重扫及长任务等待 + +同一个独立 HOME/profile 保留 13,000 条 Claude L1，14:31:41.215 发起 full scan，14:34:07.986 完成，读取 26,000 条解析消息，导入 0 memory/0 Skill，数据库仍精确 13,000；这是去重重扫，不是从空库再次导入 13,000 条。 + +同时创建 `trace_15326b81430eee449b1f`。监控在 125,779ms 观察到 ready，摘要与 embedding 均 interactive、attempts1、succeeded，有 384 维向量；当时仍有 11,944 条历史摘要 queued。reflection 仍 queued，不能写成全演化完成。 + +原组合驱动仅执行 60 次轮询，退出时重扫还未结束，因此保留 `Full rescan did not finish` 失败。未放宽断言、重置数据或调快模型；同一轮后续结束后单独验证完成、计数及任务状态。证据 `live-check.log`、`post-run-observation.json`。60 个健康样本中位 29.14ms、P95 1,067.46ms、最大 1,600.52ms；受控模型每请求 5 秒，embedding 100ms，不是实际云 Provider 的测试。 + +Worker 日志记录反思：14:31:15.351→14:31:58.153，14:32:09.477→14:32:51.610，14:32:51.654→14:33:34.715；新摘要于 14:33:34.760 开始，14:33:40.216 成功，embedding 于 14:33:45.701 成功。它证明期间各任务推进、没有永久饥饿，也显示 WRR 按任务机会分配并不承诺短的墙钟等待时间。不能把 126 秒全部归于一个正在运行的任务，等待中也包含其他 Lane 的未兑现机会；未发现排队入口异常的充分证据。若产品需要更严格的实时等待目标，需先定义目标，再评估任务粒度、模型资源及调度政策，不凭一次结果擅自改变方案。 + +GUI 可搜索并打开新记忆，原始用户/助手正文与输入一致，截图 `/private/tmp/memmy-delivery/live-detail.png`。 + +## 重启、反思后的向量更新及退出观察 + +旧 schema9 种子的完整 record + vector 指纹保留；三种独立模型连接核对通过，13,000 条历史记录保留。首次退出和重启检查中，五条样本原始对话逐项与换包前备份或本轮原始 API 输入比对一致（`raw-source-retention.json`）。 + +不能要求后台演化运行时所有派生摘要和向量始终与冻结值相同。重启后 `trace_e783a65ec75de0309b5c` 的既有 reflection 于 14:41:04 完成；SpanPipeline 按 `reflection.updated`、`clearExistingVector:true` 更新派生摘要、清除旧向量并创建 embedding 任务。此时原“全部样本必须保持 ready、派生内容和向量 hash 不变”的临时断言失败，证据保留为 `restart-processing-investigation.json` 和 `body-retention-investigation.json`，没有据此修改产品。 + +随后通过最终桌面包的正式 `/api/v1/worker/run`，以 target filter 领取该持久化 embedding 任务，leased1/succeeded1/failed0，重新 ready 且有 384 维向量。没有直接改数据库、改优先级或删除任务。见 `post-reflection-worker.json`。这是业务演化后的重建恢复，不是把原重启不变断言重建 baseline 后记为通过；初始源对话、记录计数和未变化的旧种子另作校验。 + +一次退出后立即读取窗口触发 `quit:reopen-requested-during-quit`，自动重开不能当成受控重启证据。随后只发送 Cmd-Q、不读取关闭中的窗口，确认端口关闭后显式启动，已排除此工具干扰。最后一次单独 Cmd-Q 于 14:41:38.760 开始清理，仍捕获 `quit cleanup timed out; forcing app exit`，退出码 0、端口关闭且无重开事件；因此退出清理警告确实仍保留，不宣称完全正常退出，亦未定位或归因到队列。原日志 `/private/tmp/memmy-stage8-package/queue-delivery-restart-console.log`。 + +## 提交范围及未覆盖项 + +提交一个整体 PR，标题拟为 `fix: 统一记忆后台任务调度并修复历史任务阻塞`。包含此前用户批准的独立模型迁移兼容补修。仅推送个人 fork，正式 PR 创建前再次确认；当前未推送、未创建 PR。 + +没有恢复已取消的生产背压/批量取消，没有新增 Aging 提升或完整队列统计/ETA 界面。macOS x64、Windows、真实云 Provider 未验证，既有 FTS 写入放大、超长单会话成本和 DMG 背景图外观警告仍保留。自动化通过、最终完成与产品延迟验收应分别陈述，不以“最终执行”代替一个尚未定义的实时等待目标。 diff --git a/docs/qa/memory-queue-plan-recheck-v126-20261023-results.json b/docs/qa/memory-queue-plan-recheck-v126-20261023-results.json new file mode 100644 index 000000000..7bafeed71 --- /dev/null +++ b/docs/qa/memory-queue-plan-recheck-v126-20261023-results.json @@ -0,0 +1,215 @@ +{ + "codeRevision": "02a300e304d492cf32a7b1c507b73310e9e9061b", + "targetCommit": "b04f5c88c2000a1c7a936230a70401178aca4842", + "package": { + "exitCode": 0, + "elapsedSeconds": 210.254, + "commit": "02a300e304d492cf32a7b1c507b73310e9e9061b" + }, + "dmgPath": "/private/tmp/memmy-queue-final-20261023/accepted/Memmy-1.2.3-arm64-02a300e3.dmg", + "dmgSha256": "8ce459793ec7fb108058b04cc0b33c760d6381963e1e6f1e9fb079f561b66ef6", + "runtime": { + "version": "2.1.6", + "buildId": "72dcf73d5d1b64c2e7db7b26262f0227d520900b7d68525ca11c5e495622f3a2", + "schemaVersion": 13, + "protocolVersion": 1, + "target": "darwin-arm64", + "entrypoint": "dist/src/server/index.js", + "viewer": "dist/viewer/index.html" + }, + "identity": { + "memoryModulesCompared": 230, + "memoryIdenticalToPriorValidatedPackage": true, + "asarModulesCompared": 5 + }, + "repositoryChecks": [ + { + "name": "lint", + "command": [ + "npm", + "run", + "lint" + ], + "exitCode": 0, + "elapsedSeconds": 7.841, + "startedAtEpoch": 1792745886.0027 + }, + { + "name": "typecheck", + "command": [ + "npm", + "run", + "typecheck" + ], + "exitCode": 0, + "elapsedSeconds": 40.041, + "startedAtEpoch": 1792745893.843649 + }, + { + "name": "root-test", + "command": [ + "npm", + "test" + ], + "exitCode": 0, + "elapsedSeconds": 133.594, + "startedAtEpoch": 1792745933.884634 + } + ], + "backendAfterShutdownFix": { + "testFiles": 121, + "passed": 992, + "typecheck": 0, + "lint": 0 + }, + "rootTestComponents": { + "release": { + "passed": 30, + "skipped": 5 + }, + "packagingGuards": { + "passed": 124, + "skipped": 1 + }, + "contracts": 8, + "Memory": { + "passed": 2150, + "skipped": 1, + "executions": 2 + }, + "Migrations": { + "passed": 279, + "skipped": 1 + }, + "AgentSourceCore": 62, + "Knowledge": 48, + "BackendBeforeShutdownFix": 987, + "LocalContracts": 44, + "Frontend": 2505, + "Desktop": { + "passed": 1287, + "skipped": 126 + }, + "tui": 5, + "AgentSeparate": { + "passed": 7260, + "skipped": 80 + } + }, + "packagedCloudProbe": { + "counts": { + "interactive": 12, + "repair_retry": 9, + "import": 6, + "evolution": 6, + "maintenance": 3 + }, + "peakJobs": 4, + "peakRequests": 2, + "peakEmbeddingRequests": 1, + "badQueued": 0, + "elapsedMs": 654.79 + }, + "live": { + "historyBefore": 13000, + "historyAfter": 13000, + "backlogBefore": 11915, + "backlogAfter": 11870, + "nativeReadyMs": 12369.12, + "summaryStartObservedMs": 2175.45, + "scanSeconds": 315.984, + "scanMessages": 26000, + "scanResult": "Imported 0 memories and 0 skills", + "passed": true, + "healthSamples": 153, + "healthMedianMs": 21.17, + "healthP95Ms": 111.5, + "healthMaxMs": 662.95 + }, + "shutdown": { + "portsOpen": { + "19100": false, + "54477": false + }, + "cleanup": [ + "[quit] prepared-update:start", + "[quit] prepared-update:finished elapsedMs=0", + "[quit] runtime-services:start", + "[quit] runtime-services:finished elapsedMs=1146", + "[quit] native-history-agent:start", + "[quit] native-history-agent:finished elapsedMs=24", + "[quit] backend:start", + "[quit] backend:finished elapsedMs=3", + "[quit] renderer:start", + "[quit] renderer:finished elapsedMs=1", + "[quit] analytics:start", + "[quit] analytics:finished elapsedMs=0" + ], + "forcedTimeout": false + }, + "retention": { + "passed": true, + "rawCount": 13005, + "rawSha256": "8f8b8f63113ed26940719943da49a46bd60a62a6e55ec69992b76d87e68d3e65", + "sourceCount": 13000, + "sourceIdentitySha256": "60beb096e1eff5d8829701cc462a09efe693a3b8f95abd6471323e7aca717a34", + "seedSha256": "1d2893b8ef0b58080a795382b9c2468d6ad3ec145242de414886696a35c0d106", + "quickCheck": { + "quick_check": "ok" + } + }, + "nativeAfterRestart": { + "id": "trace_0b5aab651dcfccfc3f23", + "processing": { + "state": "ready", + "stage": null, + "error_code": null + }, + "vectors": [ + { + "vector_field": "vec_summary", + "embedding_model": "stage8-embedding-384", + "embedding_dim": 384 + } + ], + "passed": true + }, + "seedVectorBytes": [ + { + "profile": "/private/tmp/memmy-queue-rereview-before-schema13", + "dimension": 384, + "bytes": 1536, + "sha256": "a98f5ec30cd21611c4db6e7db17a7a7a289ce4b9f3a4b77c7394170091332291" + }, + { + "profile": "/private/tmp/memmy-queue-stage8-profile", + "dimension": 384, + "bytes": 1536, + "sha256": "a98f5ec30cd21611c4db6e7db17a7a7a289ce4b9f3a4b77c7394170091332291" + } + ], + "localModelEvidence": { + "runtimeIdentity": "all 230 Memory modules identical to final package", + "firstAttempt": "60 second timeout; preserved", + "retryPassed": true, + "retryElapsedMs": 64438.73, + "peak": 1, + "vectors": [ + { + "vector_field": "vec_summary", + "embedding_model": "text-embedding-nomic-embed-text-v1.5", + "embedding_provider": "openai_compatible", + "embedding_dim": 768, + "n": 2 + } + ] + }, + "limits": [ + "Mac arm64 only; Windows and Intel not validated", + "Cloud model scheduling uses controlled handlers, not a real cloud quota", + "No fixed seconds SLO; a running task can hold the provider permit across multiple HTTP requests", + "No independent background cap of 3; shared global cap is 4 per prose section 7.3", + "No exact full-queue runnable census or per-resource distinct waiting population; metric bases are explicit" + ], + "noPushOrPr": true +} diff --git a/docs/qa/memory-queue-plan-recheck-v126-20261023.md b/docs/qa/memory-queue-plan-recheck-v126-20261023.md new file mode 100644 index 000000000..5a967e37e --- /dev/null +++ b/docs/qa/memory-queue-plan-recheck-v126-20261023.md @@ -0,0 +1,59 @@ +# Memmy 队列方案复查与打包验证（2026-10-23） + +代码分支 `test-memory-queue-v1.2.6`，目标 `upstream/v1.2.6`。本轮已 merge 上游 `b04f5c88`；最后网络拉取中断后，以 GitHub API 核对目标 SHA 仍相同。未 rebase、强推、推送或创建 PR。 + +对照桌面 Pages 方案的正文和表格复查，保留用户确认的 Lane 权重 4/3/2/2/1；不恢复已取消的生产背压和批量取消部分。用户允许等待已有不可抢占任务释放模型。实时优先使用本轮未消耗配额，不能刷新配额使其他 Lane 饿死。旧 KV 首次升级最多提前兑现现有的一票，持久化后不重复获得。 + +本轮补修了含 API key 字样的 429/408/5xx 被误判配置错误、自动和手动 drain 交接丢唤醒、扫描生产者未纳入全局预算、全量和单来源扫描 scope 冲突、内部向量请求超过 16 条、向量重试缺少抖动与独立租约计数等问题。同时补齐可配置租约策略、Aging 和进程内滚动观测。Schema 13 为向量重试增加领取和恢复计数，历史领取数只回填可证明的下限，不伪造历史恢复次数。 + +首次候选包仍出现 5 秒退出强退。分段日志确认运行时服务 1202ms、原生辅助进程 154ms 均结束，卡在后端。真实 socket 红回归确认前端 SSE 和 MCP GET 通知流未服务端收尾；补修 preClose 收尾及关闭期空闲连接清理，普通写请求和 MCP 工具 POST 完整返回。保留原退出顺序及 5 秒预算。辅助进程另有独立、已复现的无界等待，已增加明确终止截止及迟到事件保护。 + +| 检查 | 结果 | +| --- | --- | +| 根目录 lint、typecheck | 通过 | +| 根目录 npm test | 完整流程通过；Memory 2150（流程中执行两次）、Frontend 2505、Desktop 1287，其他模块与 guard 详见结果 JSON | +| Agent 单独全量 | 7260 通过、80 原有跳过；后续未改该模块 | +| 后端退出补修后全量 | 121 文件、992 项通过；完整类型、规范检查通过 | +| 退出真实 socket 回归 | 5 项；覆盖双事件流、空闲连接、在途普通写、迟到认证、MCP GET 和工具 POST | + +Mac arm64 官方国内未签名打包，公开构建配置来自 `.env.example`；原生库沿用已验证的 macOS 12 arm64 本地依赖。最初两次因构建环境缺少云地址、原生库路径中止，日志保留；未靠删减 guard 绕过。产品版本仍为桌面 1.2.3、Memory 2.1.6，目标分支名称不等于包内版本。 + +初次候选包 `4bc41066` 的包内调度验证：一万条、每条 32KiB，36 次执行窗口五 Lane 分别为 12/9/6/6/3；逻辑峰值 4，受控云 Provider 请求峰值 2，embedding 请求峰值 1，FIFO、未启动 attempts 和许可/租约释放通过。持续 maintenance 负载下，100 次模型完成中 import/reflection 为 50/50，maintenance 同时完成 50 次。 + +同机受控 A/B：原始 `3a5f5c44` 对比队列代码 `4bc41066`，均一万条历史摘要、200 条反思、向量重试和维护任务，共享一个模型槽位，摘要5秒、反思42秒、向量100ms。 + +| 观察值 | 原始 | 新队列 | +| --- | ---: | ---: | +| 实时任务等待模型 | 20.071 秒 | 5.040 秒 | +| 实时摘要自身执行 | 5.001 秒 | 5.001 秒 | +| 实时摘要完成总耗时 | 25.072 秒 | 10.041 秒 | +| 首次反思启动 | 120 秒窗口未启动 | 15.269 秒 | +| 首次向量重试启动 | 25.092 秒 | 0.026 秒 | +| 首次维护启动 | 120 秒窗口未启动 | 1.768 秒 | + +120 秒时原始完成22条import、没有reflection/maintenance；新队列完成4条import、2条reflection及各1条retry/maintenance。调度公平需要分配模型时间，不宣称历史导入吞吐也提高。窗口截止后等待已有任务完成才清理；不能将整个进程时长当窗口结果。此探针是真实 SQLite/WorkerRunner 配受控处理器，非真实模型速度保证。 + +候选包实际全量扫描：325 段合成 Claude 会话、26,000 条消息、13,000 条历史记忆，约301.576秒完成，0重复导入，结束仍有11919条历史摘要排队。新增实时记忆约54.393秒ready：已有reflection任务持有模型许可约42秒，释放后约46ms开始摘要，随后标题和embedding完成。优先级按任务准入，不会抢占一个任务内部已持有的模型许可；多次HTTP请求组成的任务仍可能造成较长等待。没有定义数值SLO，不能承诺任意负载在固定秒数内可用。 + +实际界面记忆列表显示651页，跨Agent页加载后显示运行中和已采集26,000条。146个健康样本均成功，最长765.21ms。这是此环境观察值。 + +升级保留逐条核对原有13,005条raw turn、13,000条历史记录身份和固定种子。raw只允许明确新增的completed memory_capture派生结果，除此之外payload和原始文本必须相同；旧状态中共18条新增这类标记。后台reflection会改变memory_value中的Reflection/Alpha、摘要和content_hash，因此不能要求所有派生字段冻结。种子完整记录及实际向量字节指纹保持，数据库quick_check为ok。 + +新包内运行时配本机真实 LM Studio Gemma/Nomic：第一次60秒超时，日志保留；扫描结束后相同超时和断言重跑，2摘要+2向量成功，两条ready、各768维，请求峰值1，总64.439秒。不将该重试写成首次通过，也不保证繁忙时模型不会超时。 + +观测 JSON 在实际包内日志中完整解析成功，包含Lane、等待分位、并发、模型、租约、跳过原因。它是有界、进程内、滚动窗口指标；due仅时间条件，admitted不等于整队列runnable总数，缺失token用量保持unknown，不把观测尝试数解释为独立任务数。 + + +最终退出补修包为 `02a300e304d492cf32a7b1c507b73310e9e9061b`，官方打包210.254秒、退出0，DMG校验VALID。包内230个Memory模块与源码编译产物及先前已测包逐个一致，5个ASAR模块与源码一致，包含两个后端关闭补修文件。Memory buildId保持 `72dcf73d5d1b64c2e7db7b26262f0227d520900b7d68525ca11c5e495622f3a2`、schema13；复用同buildId运行时缓存不表示执行旧Memory代码。 + +最终包再次跑完一万条/32KiB混合探针，Lane为12/9/6/6/3，逻辑4、云请求2、向量请求1上限和FIFO通过；持续补流模型完成100次为50/50。实际再次扫描26,000消息，315.984秒完成、0重复导入，仍13,000历史记忆；新增记忆12.369秒ready。153次健康请求均成功，中位21.17ms、P95 111.50ms、最大662.95ms。两次实机耗时的起始活动任务不同，不能用54秒和12秒直接声称算法再次提速。 + +保持跨Agent页事件连接后Cmd-Q，最终清理为：运行时1146ms、原生辅助24ms、后端3ms、renderer1ms、analytics0ms，所有步骤完成；退出码0、端口关闭，没有五秒强退或自动重开。显式重新启动同一个包后，原有13,005条对话、13,000条历史记录身份及固定种子保留，quick_check为ok；固定384维向量的1536字节SHA保持相同，本轮新记忆ready且实际向量字节存在。界面概览13,007条（历史13,000+本机合成原生7条）、列表651页。 + +本轮仅验证macOS arm64。本地真实模型证据来自与最终包逐字节一致的230个Memory模块；最终包再次实测了HTTP扫描、队列、数据、界面及退出重启。Windows、Intel和真实云Provider限额仍未实测。实际模型首轮超时不能据重试通过当作消失。 + +与方案仍需收敛的口径：正文7.3的共享全局4槽已实现；实施表另写“后台最多全局减1”，当前没有独立后台3槽硬限制。观测提供due时间条件、成功准入和跳过尝试，并非全队列精确runnable计数或每种资源的独立等待任务人数。以上没有伪装成已完成；若将实施表这两点作为严格交付条件，还需确定规则后补实现与测试。本报告仅确认当前代码已执行的测试结果，不标记所有方案建议完全落地。 + +最终DMG：`/private/tmp/memmy-queue-final-20261023/accepted/Memmy-1.2.3-arm64-02a300e3.dmg`。SHA-256：`8ce459793ec7fb108058b04cc0b33c760d6381963e1e6f1e9fb079f561b66ef6`。 + +结果汇总见同目录 `memory-queue-plan-recheck-v126-20261023-results.json`；原始日志和前后候选包保留在 `/private/tmp/memmy-queue-final-20261023/`。最终截图 `accepted/memories-after-restart.png`。本地测试应用保持打开，合成数据后台处理已暂停，未覆盖日常应用及数据。未推送或提交PR。 diff --git a/docs/qa/memory-queue-pr-sync-v126-20261024-results.json b/docs/qa/memory-queue-pr-sync-v126-20261024-results.json new file mode 100644 index 000000000..2a7e26b1f --- /dev/null +++ b/docs/qa/memory-queue-pr-sync-v126-20261024-results.json @@ -0,0 +1,137 @@ +{ + "date": "2026-10-24", + "branch": "test-memory-queue-v1.2.6", + "targetBranch": "v1.2.6", + "upstreamCommit": "68697053ab4e1d0a47f1f38431a7b810c20a94f1", + "sourceCommit": "fdc58fd9d4e64679464f9c9f06307dbd03aad0ed", + "mergeUsed": true, + "rebaseUsed": false, + "defaultLimits": { + "global": 4, + "background": 3, + "reservedInteractive": 1, + "localProvider": 1, + "cloudProvider": 2, + "embeddingRequests": 1, + "embeddingBatchSize": 16 + }, + "versions": { + "desktop": "1.2.4", + "memory": "2.1.7", + "schema": 13 + }, + "checks": { + "rootLint": "passed", + "rootTypecheck": "passed", + "rootNpmTest": "passed", + "memory": { + "passedEachRun": 2279, + "skippedEachRun": 1, + "runs": 2 + }, + "backend": { + "passed": 1030 + }, + "frontend": { + "passed": 2717 + }, + "desktop": { + "passed": 1308, + "skipped": 126 + }, + "agentFull": { + "passed": 7339, + "skipped": 80 + }, + "agentTypecheck": "passed" + }, + "conflictResolution": { + "files": 10, + "extraRegression": [ + "Responses and Chat Provider capacity", + "independent Custom endpoints and canonicalization", + "frontend no-auth endpoint identity", + "native helper bounded cleanup and late events", + "Embedding shared errors and input isolation", + "observation recovery with real retry backoff" + ] + }, + "environmentRecovery": [ + "Git uses existing macOS HTTP proxy per command; no global configuration changes", + "isolated Agent browser runtime copied from local cached binaries; no real browser profile data copied" + ], + "artifactVerification": "passed", + "package": { + "file": "Memmy-1.2.4-darwin-arm64-cn-unsigned.dmg", + "sha256": "4de797265b1790dea8ac6a0427cb5ffb17c9f7cfa09541113a8b3646923a38e2", + "integrity": "VALID", + "packagingSeconds": 212, + "electronCacheVerification": { + "archive": "electron-v38.4.0-darwin-arm64.zip", + "sha256": "0895d02b29b26f33fc984aa0fc06005093f6128343ecce42da89855c14c22aac", + "expected": "0895d02b29b26f33fc984aa0fc06005093f6128343ecce42da89855c14c22aac", + "matchesOfficialElectronPackage": true + }, + "identity": { + "memoryJavaScriptModulesCompared": 232, + "mismatches": [], + "criticalModuleSha256": { + "service/worker/worker-concurrency.js": "9814a75d7b9afa2c3c5214fda25e83f88f7d33c678d3478b0d9ba47708a06950", + "service/worker/worker-observability.js": "3bc3006c8cb386dbbaa9ae61cf1f61883903ddc4f7fa6f24c2d665422c80f3eb", + "service/worker/worker-runner.js": "54117b8f99f5b585fa43744af402c2f89bcd5144f9e41f76276f003762b3393b", + "service/worker/worker-work-requirements.js": "c551ae8a14b77d84b9d29c10b7dfe3be8f8fafaaba11bbbeeff4ee37953aeafe" + }, + "manifest": { + "version": "2.1.7", + "buildId": "3e82006f0450c9abc60058fd852d3425f08752e4f3e86291c4885924e07bc9d0", + "schemaVersion": 13, + "protocolVersion": 1, + "target": "darwin-arm64", + "entrypoint": "dist/src/server/index.js", + "viewer": "dist/viewer/index.html" + } + } + }, + "installedQueueProbe": { + "success": true, + "backlog": 10000, + "backgroundLeased": 3, + "untouchedQueued": 9997, + "interactiveStartWaitMs": { + "native": 7.00329099999999, + "onboarding": 6.665915999999925 + }, + "basis": "installed compiled queue and real SQLite; business/model work replaced by controlled gates" + }, + "installedAppSmoke": { + "success": true, + "verification": "automated installed app startup/render/health/normal quit/relaunch; no manual UI operation", + "launches": [ + { + "launch": 1, + "healthyMs": 5008.2397089999995, + "quitMs": 214.74608400000034, + "rendered": true, + "exit": { + "code": 0, + "signal": null + } + }, + { + "launch": 2, + "healthyMs": 1452.8067499999997, + "quitMs": 215.62770799999998, + "rendered": true, + "exit": { + "code": 0, + "signal": null + } + } + ] + }, + "postPackageTestsPassed": 70, + "developmentNativeRestored": true, + "manualAuthenticatedBusinessUiTest": false, + "liveCloudProviderLoadTest": false, + "evidenceDirectory": "/Users/lijia/Desktop/memmy-agent-main/queue-pr-sync-20261024" +} diff --git a/docs/qa/memory-queue-pr-sync-v126-20261024.md b/docs/qa/memory-queue-pr-sync-v126-20261024.md new file mode 100644 index 000000000..24ddda386 --- /dev/null +++ b/docs/qa/memory-queue-pr-sync-v126-20261024.md @@ -0,0 +1,53 @@ +# 队列改造提交前同步、回归与新包验证 + +日期：2026-10-24。分支 `test-memory-queue-v1.2.6`，PR 目标 `MemTensor/memmy-agent:v1.2.6`。 +源码提交 `fdc58fd9d4e64679464f9c9f06307dbd03aad0ed`，已通过 merge 合入上游 `68697053ab4e1d0a47f1f38431a7b810c20a94f1`，未 rebase、强推或合并其他个人任务分支。 + +## 同步与交叉修复 + +提交前发现上游新增154个提交，包含协议、模型配置、扫描和退出流程。10个冲突文件逐项合并，保留两边功能；以下交叉边界有回归覆盖： + +- 独立记忆连接迁到 Custom 时不继承 Provider 的凭据、headers、body；相同有效值的独立与继承连接不错误合并。 +- 前端保留新协议、region、模型能力及凭据身份校验；无 Key 独立端点改变地址或协议后不能沿用旧身份绕过验证。 +- 保留全安装 native helper 枚举和身份验证，保留有界退出、迟到事件防护、分阶段耗时记录和 runtime/native 关闭顺序。 +- Embedding 只有输入错误可拆批；认证、配额、限流和服务故障不按条扇出；保留租约 fencing、续租、失败退避及配置阻塞不消耗失败次数的语义。 +- 新上游 Responses 默认端点导致与 Chat 的容量池分裂，先新增真实协议调用回归复现请求峰值4，再修正为同服务共享上限2。不同服务及代理前缀仍独立，Bedrock保留显式endpoint规则。 +- 上游测试中依赖旧整批领取或立即重试的准备方式已适配。批错误测试明确准备完整候选批并注入专项容量，保留严格的1次共享错误请求和5次坏项隔离请求断言；恢复测试仅将明确指定的退避任务置为到期。 + +默认生产参数保持总并发4、后台最多3、实时预留1，本地Provider1、云Provider2、Embedding请求1、批大小上限16。未采用5／4配置。 + +## 最终自动检查 + +| 检查 | 结果 | +| --- | --- | +| 根目录 lint、typecheck | 通过 | +| 根目录 npm test | 完整流程退出码0 | +| Memory | 两轮均2279通过、1跳过 | +| Backend | 1030通过 | +| Frontend | 2717通过 | +| Desktop | 1308通过、126跳过 | +| Agent全量 | 7339通过、80跳过 | +| Agent类型检查 | 通过 | +| 打包后槽位、扫描、指标与Responses专项 | 70通过 | +| 差异与冲突标记检查 | 通过 | + +Agent首轮仅3项因隔离HOME缺少Chromium测试运行时失败；复制本机缓存二进制后完整重跑通过，没有复制真实浏览器用户数据或登录状态。Git首次直连失败，改用系统既有代理完成同步；未改全局代理配置。npm产生的32个锁文件dev标记已逐项确认并恢复，未夹带依赖升级。 + +## 最终安装包 + +正式脚本构建 macOS ARM64 中文未签名包。首轮在Electron网络下载处等待并中止，保留日志；使用本地同版本官方ZIP重新完整执行，SHA256与Electron包内官方checksums.json一致，未跳过打包守卫。成功构建耗时3分32秒。 + +应用版本随上游为Desktop 1.2.4、Memory 2.1.7、schema13。 +Memory buildId：`3e82006f0450c9abc60058fd852d3425f08752e4f3e86291c4885924e07bc9d0`。 +DMG SHA256：`4de797265b1790dea8ac6a0427cb5ffb17c9f7cfa09541113a8b3646923a38e2`，完整性校验VALID。 + +安装包及证据固定存放在 `/Users/lijia/Desktop/memmy-agent-main/queue-pr-sync-20261024`。只读挂载后复制到独立测试目录，没有替换 `/Applications/Memmy.app`。 + +- 包内232个Memory JavaScript模块与本次源码编译产物逐字节一致。 +- 包内Electron、原生SQLite和真实Runner运行10000条受控import_summary积压：后台领取3条持续占位，实时记忆和Onboarding首次报告依次进入第4槽；9997条仍queued、attempts0、无租约；停止后许可、Provider和scope计数清零。 +- 受控业务替身中两类实时任务开始等待约7.00／6.67毫秒。这验证槽位规则，不代表真实模型耗时或生产SLO。 +- 使用隔离HOME和测试profile两轮启动，均进入完整登录页面；已检查截图，没有将启动动画算作成功。Memory健康返回正确版本、buildId和native向量存储。 +- 两轮健康就绪约5.01／1.45秒，正常退出约215／216毫秒，退出码均0；同一profile重启正常。 +- 打包后已恢复Node22开发用SQLite原生库，内存数据库及70项专项回归通过。 + +本轮安装验证覆盖隔离profile的启动、登录页渲染、服务健康和退出重启；没有登录真实账号或进行真实云模型负载测试。共享本地模型已执行的请求仍不可抢占。此前各阶段报告保留为历史记录，本文件是同步最新上游后的提交前验收结果。 diff --git a/docs/qa/memory-queue-pr734-review-20261025-results.json b/docs/qa/memory-queue-pr734-review-20261025-results.json new file mode 100644 index 000000000..34c17f968 --- /dev/null +++ b/docs/qa/memory-queue-pr734-review-20261025-results.json @@ -0,0 +1,116 @@ +{ + "package": { + "dmgSha256": "c8ede3c1d60db3be43114d19d1dd810710b286b045c3f87644c9e64d061b30d8", + "dmgBytes": 369761935, + "manifest": { + "version": "2.1.7", + "buildId": "481c97f14693f31626bb645abb7d082bec7e70598c546bf7cdc5fc8d1be0a3db", + "schemaVersion": 13, + "protocolVersion": 1, + "target": "darwin-arm64", + "entrypoint": "dist/src/server/index.js", + "viewer": "dist/viewer/index.html" + }, + "comparedMemoryModules": 233, + "mismatches": [], + "timestamp": "2026-10-25T23:17:54.444688" + }, + "reviewChecks": [ + { + "name": "issue1-summary-vector-recall-idempotence", + "budget": 1, + "passed": true, + "completedJobs": 4 + }, + { + "name": "issue1-summary-vector-recall-idempotence", + "budget": 20, + "passed": true, + "completedJobs": 4 + }, + { + "name": "issue2-unrelated-wakeup-deep-scan", + "aging": false, + "blocked": 20000, + "releases": 312, + "passed": true + }, + { + "name": "issue2-unrelated-wakeup-deep-scan", + "aging": true, + "blocked": 20000, + "releases": 312, + "passed": true + }, + { + "name": "issue3-fixed-budget-backlog", + "backlog": 1000, + "completed": 1000, + "elapsedMs": 2327.1269580000007, + "policyMs": 44.6108679999943, + "policyCalls": 1000, + "passed": true + }, + { + "name": "issue3-fixed-budget-backlog", + "backlog": 10000, + "completed": 1000, + "elapsedMs": 2439.965833, + "policyMs": 43.595891999984815, + "policyCalls": 1000, + "passed": true + }, + { + "name": "issue3-fixed-budget-backlog", + "backlog": 50000, + "completed": 1000, + "elapsedMs": 2417.9618749999972, + "policyMs": 43.45605299999079, + "policyCalls": 1000, + "passed": true + }, + { + "name": "issue3-promotion-and-new-cycle", + "passed": true + } + ], + "reservation": { + "success": true, + "backlog": 10000, + "backgroundLeased": 3, + "untouchedQueued": 9997, + "interactiveStartWaitMs": { + "native": 2.824041999999963, + "onboarding": 4.509667000000036 + } + }, + "app": { + "success": true, + "launches": [ + { + "launch": 1, + "healthyMs": 3056.313709, + "quitMs": 616.2351250000002, + "exit": { + "code": 0, + "signal": null + }, + "rendered": true + }, + { + "launch": 2, + "healthyMs": 1472.2649999999994, + "quitMs": 159.74070800000027, + "exit": { + "code": 0, + "signal": null + }, + "rendered": true + } + ] + }, + "upgradePassed": true, + "postPackageTests": 166, + "sourceUnchangedByPackaging": true, + "prUpdated": false +} diff --git a/docs/qa/memory-queue-pr734-review-20261025.md b/docs/qa/memory-queue-pr734-review-20261025.md new file mode 100644 index 000000000..a17ce4d11 --- /dev/null +++ b/docs/qa/memory-queue-pr734-review-20261025.md @@ -0,0 +1,57 @@ +# PR #734 审核修复与打包验证 + +日期：2026-10-25。目标分支 `v1.2.6`；本轮修复基于 `77a077e215f4c9a2b1586bd197e2920cf92b0c12`。 +更新前获取上游，目标仍为 `68697053ab4e1d0a47f1f38431a7b810c20a94f1`，已包含在当前分支，无新增上游代码。 + +## 审核意见与修复 + +1. [连续补位的 smoke 断言](https://github.com/MemTensor/memmy-agent/pull/734#discussion_r4370332832)：按有限轮次汇总执行结果，不再要求向量固定在第二轮执行；覆盖预算 1/20、摘要及向量各成功一次、真实向量落库、读取、召回和空闲重复调用无副作用。 +2. [无关释放导致扫描反复归零](https://github.com/MemTensor/memmy-agent/pull/734#discussion_r4370340655)：释放通知携带具体资源，等待集合按受阻资源保存候选，保留深扫描进度；相关释放后按原 FIFO 位置重试，配置/分类/到期变化仍正确失效。覆盖 20,000 条受阻记录、持续干扰、Aging 开关、旧任务恢复及并发边界。 +3. [有效分类仍反复全量扫描](https://github.com/MemTensor/memmy-agent/pull/734#discussion_r4370348364)：以事件更新分类，缺失/旧分类进入部分索引；异步补位每片共享最多 64 条维护预算，终态历史不进入活跃维护。分类版本升为 3，升级补齐可恢复；缩小普通属性更新触发范围。保留同一未完成任务的首次报告/主动重试待遇，独立新任务不继承；去重不改变 FIFO 身份。 + +默认参数保持全局 4、后台最多 3、实时预留 1；WRR 4/3/2/2/1。 + +## 修复后自动检查 + +- Memory 全量：215 文件通过、2 跳过；2,316 项通过、2 跳过。性能基准默认跳过，另行开启后通过。 +- 根目录 lint、typecheck 通过。 +- 两类 smoke 计划回归：7 项通过。 +- 打包保护：129 通过、1 跳过；发布流程：31 通过、5 跳过。 +- 打包后：8 文件、166 项相关回归通过。 +- 源码与打包前差异完全一致，锁文件未变化；开发 Node 22 SQLite 原生模块已恢复；差异检查通过。 + +以上为本次审核修复的验证。2026-10-24 的根目录、Backend、Frontend、Desktop、Agent 全量结果属于此前提交，未作为本轮重新执行结果。 + +## 调度成本比较 + +真实 SQLite/Runner，模型与业务使用即时替身，每档固定完成前 1,000 条，断言 FIFO 和成功状态。前后均包含问题 1/2 修复，仅比较问题 3。 + +| 积压 | 修复前 | 修复后 | 分类维护前/后 | +| --- | --- | --- | --- | +| 1,000 | 2.746 s | 2.570 s | 314 / 49 ms | +| 10,000 | 6.921 s | 2.712 s | 4248 / 47 ms | +| 50,000 | 28.043 s | 2.717 s | 25168 / 47 ms | + +这是本机单轮诊断，不代表真实模型吞吐量。可选回归入口为 `Memory/tests/service/worker/worker-policy-backlog-benchmark.test.ts`，设置 `MEMMY_QUEUE_POLICY_BENCHMARK=1` 后运行。 + +## 实际安装包验证 + +macOS ARM64 中文未签名包构建通过；包内版本沿用 Desktop 1.2.4、Memory 2.1.7、schema 13。构建成功轮约 3 分 33 秒。 + +- DMG SHA-256：`c8ede3c1d60db3be43114d19d1dd810710b286b045c3f87644c9e64d061b30d8`。 +- Memory buildId：`481c97f14693f31626bb645abb7d082bec7e70598c546bf7cdc5fc8d1be0a3db`。 +- DMG 完整性通过，包内 233 个 Memory 编译模块与源码编译结果一致，实际健康接口 buildId 匹配。 +- 安装包运行时中，三项修复的 8 个场景全部通过。20,000 条受阻记录下持续无关释放时，独立任务能够执行；Aging 开关均通过。 +- 已有效分类的 1,000/10,000/50,000 条积压，各完成 1,000 条耗时约 2.327/2.440/2.418 秒。 +- 10,000 条导入积压下，后台最多占 3 槽；实时与首次报告可进入第 4 槽，其余 9,997 条保持未领取。 +- 隔离配置自动启动两次，完整登录页面渲染、服务健康、正常退出及重启通过。健康就绪约 3.056/1.472 秒，退出约 0.616/0.160 秒，退出码均 0。 +- 上一版安装包生成的合成数据库经新包启动/重启，记忆内容、用户、创建时间、待执行任务顺序、首次入队时间及延后执行约束保留。新增列存在，验证脚本调用有界维护后分类版本为 3。 + +## 限制 + +- 模型与业务负载使用受控替身；未验证真实云模型、LM Studio 长时负载或真实账号登录后的完整 UI 操作。 +- 本轮仅 macOS ARM64，未验证 Windows/Intel Mac。 +- 完整 `npm run smoke:memory-layer` 运行时脚本此前出现 `Memory processing did not become ready` 超时，本轮未重跑该脚本；不得将计划回归和包内核心链路通过等同于完整脚本通过。 +- 实时预留不突破 Provider 上限，不抢占已执行模型请求。 + +结构化安装包结果见同目录 `memory-queue-pr734-review-20261025-results.json`。安装包、截图和原始日志保存在本机 `queue-review-package-20261025`，未将安装包或用户数据提交到仓库。 diff --git a/docs/qa/memory-queue-stage2-20261022.md b/docs/qa/memory-queue-stage2-20261022.md new file mode 100644 index 000000000..012c2b024 --- /dev/null +++ b/docs/qa/memory-queue-stage2-20261022.md @@ -0,0 +1,76 @@ +# Memory 队列改造：阶段 2 租约安全与失败退避 + +日期：2026-10-22（Asia/Shanghai）。PR 目标仍为 `upstream/v1.2.6`，个人分支为 `test-memory-queue-v1.2.6`；上游基线 `3a5f5c44bd5cc557a529c2c715ce3f08cbb9fa93`。阶段 1 复现记录已保存在本地提交 `726468f4`。 + +## 本阶段结果 + +| 阶段 1 问题 | 阶段 2 行为 | +| --- | --- | +| A 的迟到失败可以覆盖 B 已成功的任务。 | 每次领取获得 owner 与唯一 token；完成、失败、延期和释放须匹配当前身份、leased 状态与有效截止时间。旧结果返回不计成功，也不计业务失败。 | +| 任务状态受到保护，但旧模型结果可能先写入记忆、向量和派生任务。 | 执行期间携带异步租约上下文，在实际 SQLite 写入前检查；过期或被接管后，迟到的业务写入被拒绝，事务中的修改回滚。 | +| 长任务或串行等待任务超过 60 秒可被重复领取。 | 主任务保留 60 秒租约，每 20 秒续租；过渡期对本轮尚在等待的已领取任务也续租。Embedding retry 的 5 分钟租约同样定期续租。 | +| 启动恢复无条件清除仍有效的租约。 | 保留有效租约，仅恢复已过期或没有截止时间的旧租约；永久失败 retry 不会因重启而复活。 | +| 普通临时失败连续三轮即可耗尽尝试。 | 失败记录 available_at，首次约 5 秒，随后指数增长，带 0.8–1.2 倍随机抖动，上限 5 分钟；未到时间不能重排，调度器按到期时间唤醒。 | +| 配置错误反复争抢任务，或终止后无法恢复。 | 主任务和 retry 均记录 configuration 阻塞，不消耗业务失败次数。有效模型配置变化后恢复原任务；同一错误配置不会每轮重新尝试。显式手工重试仍可恢复。 | +| 领取与失败次数混用，租约恢复也耗尽业务重试。 | 增加 failure_count、lease_count、lease_recovery_count；保留 attempts 的兼容用途。租约恢复不增加业务失败次数。 | + +永久输入错误进入 dead letter；未知 job 类型立即终止，避免当作临时错误反复执行。既有配额错误 `40309` 的终止与可操作错误信息保留。 + +主任务仍使用旧的批量领取结构。本阶段通过续租保护暂时等待执行的任务；取消提前领取、统一入口并发、连续补位与 WRR/FIFO 按后续阶段实施。因此一万条积压中的调度饥饿问题仍由后续阶段解决，不能把本阶段验收等同于整个方案完成。 + +## 实现边界与衔接 + +- 新增 `job-lease-context.ts`；真实业务执行受保护，普通前台操作不需要任务租约。保护依赖工作方案既定的单 Memory 服务进程与同步 SQLite 写入边界，不持有跨模型等待的数据库事务。 +- 任务完成后记账、清理日志等仍要继续执行。用量记账、outbox 交付、日志保留、历史快照压缩与新 worker 入口独立于业务租约上下文，避免旧租约终止这些基础设施操作。真实模型消耗即使发生于失效任务，也仍会记账。 +- 模型配置指纹是摘要、演化、Embedding 和角色路由的 SHA-256；读取与真实模型执行相同的有效配置来源，包括 configLoader 变化，未把 API key 原文写入新字段。 +- 配置错误仍以现有可操作失败状态呈现；配置变更后处理状态随原任务恢复。启动不会无条件解开配置阻塞。 +- 同输入重复入队 retry 保留已有有效租约和退避；输入变更则明确变回 pending 并使旧 claim 失效。只修正租约状态一致性，未引入已取消的指纹去重/生产背压方案。 + +## 数据升级 + +Schema 从 v9 升到 v10，增加租约身份、到期时间、阻塞原因与独立计数；新增回收和可运行时间索引。沿用 MemoryDb 的升级前备份机制。 + +迁移测试实际恢复旧表形状后重新打开数据库，验证任务保留、旧计数回填、预迁移备份与再次打开的幂等性。历史 attempts 原本混合了领取和失败，旧业务失败次数按状态作兼容估算；新事件开始独立记录。 + +未对当前安装版数据库执行迁移，未操作真实用户数据。 + +## 验证 + +| 检查 | 结果 | +| --- | --- | +| `npm run memory:test` | 1390 通过，1 项按条件跳过；139 个文件通过，1 个文件跳过。 | +| `npm run memory:lint` | 通过，包含共享核心编译、Memory 与 Viewer 类型检查。 | +| `npm run test:release-workflow` | 30 通过，5 项按条件跳过。 | +| `npm run test:packaging-guards` | 122 通过，1 项按条件跳过。 | +| 差异空白检查 | 通过。 | + +关键新增回归覆盖：旧 token、无 claim 绕过、未被重新领取的过期执行者、真实迟到摘要与向量写入、长任务续租、心跳清理、并行调用身份隔离、退避到期、独立计数、两类任务的配置恢复、配置 loader 变化、旧异步上下文下的新 worker、用量交付与维护清理、v9 升级。 + +生命周期测试中需耗尽多次失败的旧用例，明确推进目标任务的可运行时间；专门的退避测试仍验证到期前无法执行，没有通过全局清除退避来规避行为变化。阶段 1 的 R7–R10 对应断言已经转换为修复后的回归，原始阶段 1 JSON 证据保持不变。 + +首次完整测试因执行沙箱不允许本机 HTTP 监听而出现 EPERM，已在允许临时本地监听后重跑通过。打包保护测试最初缺少独立工作区的 Electron 路径，并受 shell /dev/fd 限制；使用本机已有 Electron 与允许执行的测试环境后已全部通过。未安装或升级依赖，未改锁文件。 + +原始日志： + +- `/private/tmp/memmy-stage2-memory-tests-final.log` +- `/private/tmp/memmy-stage2-memory-lint.log` +- `/private/tmp/memmy-stage2-release-guards.log` +- `/private/tmp/memmy-stage2-packaging-guards.log` + +以上均在 macOS arm64 上验证。打包保护测试不等于实际 DMG 构建与安装；统一打包安装验证仍在整体改造完成后执行。本阶段未推送、未创建 PR。 + +## 下一步 + +阶段 2 完成后等待用户确认，进入阶段 3：策略注册表、调度字段与 shadow 对照。后续保持一个整体改造 PR，提交前再次与用户确认，标题使用 fix: 中文说明。 + +## 阶段 3 前的审查补修 + +阶段 1–2 审查确认了三个遗漏，已在独立补修提交中修正： + +1. 启动占位摘要对账保留明确的失败处理状态，避免覆盖配置阻塞提示或为永久输入错误重新建任务。真实数据库关闭后重开验证：配置错误仍显示失败且可手动重试，永久 HTTP 400 不会被启动对账复活。 +2. 每次实际模型执行记录当时的配置指纹。旧配置的在途请求晚到失败时，如果配置已经变化，仅重新排入该任务；当前配置本身的失败继续等待修复，避免无限重试。覆盖摘要和 Embedding retry，并验证不会误恢复另一个在新配置下确实失败的任务。 +3. Embedding retry 每次领取生成独立唯一 token，检查、续租及全部结算均校验 token。即使同一 worker 在同一毫秒重领同一任务，旧凭据也无法写入新租约。数据库升级到 v11，保留尚有效的 v10 租约，过期后领取新 token。 + +同时修正阶段 1 重跑说明：原始证据应在历史提交 `726468f4` 的独立工作区复核；当前导出器记录实际 commit 和工作区状态，调用方标签仅供参考，不再被当作运行代码版本。 + +本轮验证：5 个测试文件、56 项通过，Memory TypeScript 检查通过。日志 `/private/tmp/memmy-stage2-followup-tests.log`。本轮仍未进行 DMG 打包、安装测试或提交 PR。 diff --git a/docs/qa/memory-queue-stage3-20261022.md b/docs/qa/memory-queue-stage3-20261022.md new file mode 100644 index 000000000..d86a29a1b --- /dev/null +++ b/docs/qa/memory-queue-stage3-20261022.md @@ -0,0 +1,63 @@ +# Memory 队列改造：阶段 3 策略注册表、持久化字段与影子调度 + +日期：2026-10-22。工作分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`。本阶段承接阶段 2 安全补修提交 `3836f391`，按工作方案仅建立和观察新策略，**尚未切换实际任务调度**。 + +## 本阶段完成内容 + +1. **集中策略注册表**：全部当前 `JobType` 加上重试适配类型 `embedding_retry` 都有明确策略，新增类型未登记会导致类型检查或测试失败。根据任务类型、来源、目标记忆元信息和操作模式确定 Lane、资源、基础紧急度、租约与并发策略描述。当前执行仍使用旧并发和租约入口，并发许可在后续阶段实现。 +2. **区分真实生产来源**：首次报告保持 interactive；显式手工重试进入 repair_retry；普通历史摘要和向量进入 import；反思等进入 evolution；启动占位摘要、启动处理修复、检索重建及 observation 补扫进入 maintenance。同时识别实际 producer 使用的 `payload.source` 和 `payload.reason`，避免只覆盖测试中的虚构来源。 +3. **调度字段**：主任务表与 Embedding retry 表均增加 `queue_lane`、`resource_class`、`enqueue_seq`、`first_queued_at`、`base_priority`、`policy_version`。已有 `available_at`、`blocked_reason` 和租约字段继续沿用。 +4. **稳定顺序**：两个队列共用一个持久化计数器，插入触发器分配递增编号，覆盖直接 SQL 入队路径。计数器仅保存序号，不是第二张任务事实表。重复入队、退避、续租、配置恢复与过期恢复不重置编号；删除最大编号后也不复用。 +5. **影子对比**：主任务和重试通过共同候选接口进入只读 WRR＋可运行 FIFO 计算。各 Lane、各来源表分别限量读取，避免先截取一批导入任务后看不到其他 Lane。旧领取和影子预览复用同一份可运行条件，保留现有时间、依赖、业务范围、预算、模型配置及目标过滤。 +6. **隔离异常**：未知任务明确告警并在投影中标记 `unregistered`，不静默套默认 Lane。策略元数据计算失败不阻止真实入队；影子计算或观察回调失败不阻止旧 worker 执行。租约丢失异常仍按原安全边界处理，不被吞掉。 + +## 升级与索引 + +数据库版本升级至 **v12**；保留 v2–v11 升级路径，升级前由现有机制建立 `.pre-v12.bak`。测试验证 v11 文件真实关闭、升级、重开后任务状态和编号保持稳定。 + +历史行缺少可靠的原始入队时间，因此首次回填按已有 `created_at` 排序，时间相同使用稳定的表名及行顺序；这是一种确定性近似，不能声称恢复了历史跨表的精确入队先后。新插入任务的 `first_queued_at` 使用实际数据库插入时间，避免导入 producer 回填数月前的记忆时间而夸大等待时长。 + +增加 Lane/状态/可用时间/入队顺序索引，以及 FIFO 和策略版本索引。目标记忆元信息变化会使相应策略投影失效。重试表另加 `target_id` 单列索引，查询计划回归验证目标失效更新走索引查找；现有 `(target_kind, target_id)` 索引不能直接服务该单列查询。 + +## 如何看影子结果 + +默认启用影子观察，设置 `MEMMY_WORKER_SCHEDULER_SHADOW=0` 后可关闭，真实执行继续沿用旧路径。每个 runner 最多每 **5 秒**采样一次，每次比较最多 **32 个选择**；每次补齐最多 **500 行**尚未计算的策略元数据。 + +worker 的 `scheduler.shadow` 日志记录： + +- `legacySelection`：同一时点旧规则预计选择的主任务及剩余名额重试；保留旧 cohort 行为。 +- `shadowSelection`、`shadowLanes`：新规则的选择及 Lane。 +- `sameSelection`：本轮两种选择是否相同。 +- `coverageComplete`、`pendingPolicyCount`、`unregisteredJobCount`：投影是否完整、是否存在未登记类型。投影未完成时，不能把局部候选结果当作完整比较。 +- `candidatePoolSize`、`sampleLimit`、`weights`、`nextCursor`：候选量、采样大小和 WRR 参数。 + +日志不包含记忆正文、会话内容、向量或模型凭据。影子只推进自己的内存游标，不修改任务状态、尝试次数、租约或业务写入；新字段的投影刷新属于元数据写入。重试表仍保留原事实，阶段 3 没有提前实施队列统一领取。 + +用户已确认采用工作方案正文 6.1 的权重 **4/3/2/2/1**（interactive / repair_retry / import / evolution / maintenance），后续实施以此为准；原实施参数表中的 4/2/2/2/1 不再作为实施依据。阶段 3 代码已与该决定一致，当前权重只用于观察。基础紧急度保留供策略审计，同 Lane 的选择仍以入队编号为准，不用静态紧急度破坏 FIFO。 + +## 复现对照与验收 + +真实 SQLite＋真实 WorkerRunner，业务处理器和模型请求使用受控替身，测试数据库位于临时目录。未访问真实用户数据或付费模型，未在当前 Memmy 窗口进行人工安装验证。 + +- **四条导入摘要＋反思＋重试**：旧执行仍完成四条导入，反思和重试继续等待；同一轮影子选择包含重试和反思。证明观测已接入且实际旧行为未被偷偷切换。 +- **10,000 条导入积压＋反思＋重试＋维护**：影子候选仍包含各个可运行 Lane；纯选择器完整轮次和跨小批次游标测试验证权重机会。 +- **业务事实不变**：直接采样前后，任务状态、尝试次数、租约、记忆、处理状态和 change cursor 全部相同，仅允许新投影字段刷新。 +- **异常隔离**：策略投影、候选查询、观察回调异常时，真实入队和旧执行仍成功；关闭开关后不再查询影子候选。 +- **过滤一致**：覆盖未来 runAfter、退避、配置阻塞、奖励依赖、Scope 前置任务、有效/过期租约、目标限制、预算暂停及摘要模型未配置。 +- **安全升级和顺序**：覆盖双表递增、直接 SQL、删除后不复用、去重及恢复不改序、历史回填、新导入真实等待时间、目标标签变更及晚创建目标。 + +影子结果只是同一时点的选择对比，不是“假如过去所有任务都按新规则执行”的完整模拟。它不能证明实际吞吐提升或 UI 问题已修复；资源/Provider 许可、连续补位及实际公平调度仍需后续阶段验收。 + +## 本地检查 + +- `npm run memory:test`：**1440 项通过，1 项跳过**。 +- `npm run memory:lint`：共享类型构建、Memory 与 Viewer 类型检查通过。 +- `npm run test:release-workflow`：30 项通过，5 项跳过。 +- `npm run test:packaging-guards`：122 项通过，1 项跳过。 +- `git diff --check`：通过。 + +主要测试：`job-policy-registry.test.ts`、`shadow-queue-selector.test.ts`、`queue-scheduling-store.test.ts`、`worker-scheduling-shadow.test.ts`；原调度复现及阶段 2 回归继续保留。 + +日志：`/private/tmp/memmy-stage3-memory-tests-final.log`、`/private/tmp/memmy-stage3-memory-lint.log`、`/private/tmp/memmy-stage3-release-guards.log`、`/private/tmp/memmy-stage3-packaging-guards.log`。 + +按整体计划，本阶段不单独打包、不推送、不提交 PR。统一 Mac 打包与安装验收在完整改造后进行；提交一个 PR 前再次征得用户确认。阶段 3 完成后等待用户确认，再进入阶段 4 统一执行入口。 diff --git a/docs/qa/memory-queue-stage4-20261022.md b/docs/qa/memory-queue-stage4-20261022.md new file mode 100644 index 000000000..733d9d98b --- /dev/null +++ b/docs/qa/memory-queue-stage4-20261022.md @@ -0,0 +1,76 @@ +# Memory 队列改造：阶段 4 统一执行入口 + +日期：2026-10-22。工作分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`；承接阶段 3 及权重确认提交 `e50d6023`。本阶段仅统一入口，真实任务选择仍使用旧规则，影子调度继续使用用户确认的 **4/3/2/2/1**。 + +## 问题复现与范围 + +阶段 1 已复现：同一个 runner 同时接收两个 `runWorkerOnce(4)` 请求，实际有 **8 个摘要处理器同时执行**。自动 drain 的 `running` 标记只约束自身，不能约束 HTTP 手工调用、历史导入或首次报告发出的独立请求。 + +本阶段另外通过真实 HTTP＋SQLite 复现：摘要任务已存在时，导入摘要入队接口返回 `enqueued=0`，原接口不唤醒自动 worker；在禁用短期启动兜底的测试中，目标持续等待，修前验证超时。日志 `/private/tmp/memmy-stage4-http-wake-before.log`。 + +工作文档要求：历史导入、Onboarding 只负责入队、唤醒和查询目标状态；worker/run 统一进入同一个调度器。本阶段不提前切换 WRR，不实施四层资源许可，不重新增加已取消的生产背压方案。 + +## 实现 + +### 1. 同一个执行边界 + +每个 `WorkerRunner` 持有一个 `WorkerScheduler`，自动、手工、CLI、直接 MemoryService 调用和公开 retry 排空都通过它。当前按请求 FIFO 进入执行，上一请求退出后才允许下一请求领取任务。 + +该 FIFO 是**执行请求的顺序**；业务任务的实际选择仍沿用旧 SQL 排序及 cohort 规则。内存中等待的请求不包含第二份持久化业务任务事实，任务仍保存在原数据库表。 + +- 等待请求不提前领取任务，不持有租约，不增加 attempts。 +- 每个请求保留自己的目标、limit、cohort 及结果，避免请求之间混用。 +- 同步异常、异步异常或领取失败后释放入口，后续请求仍能执行。 +- 主排空内部直接调用私有 retry 实现，避免拿着执行入口再等待自己。 +- 当前任务若直接等待同一调度器再次执行或等待自己的关闭，明确拒绝，避免死锁。 +- 自动唤醒是独立请求：自动 drain 清除继承的调度上下文后再入门，因此任务异步后代发出的正常唤醒可以排队；这不增加第二个活跃执行流程。 + +### 2. 历史导入与首次报告只入队、唤醒、等待 + +复用已有 `enqueueImportSummaries` 接口，不新增 HTTP API。它在鉴权和入队操作成功后无条件唤醒自动处理，包括 `enqueued=0` 的已有任务。 + +- 历史导入：保留已有 100 条目标分组，入队后每 250ms 查询处理状态、更新进度；保留取消、失败收集及 40 分钟无进展超时。取消停止调用方等待，不删除已持久化任务。 +- 首次报告：入队后每 100ms 查询目标状态，保留 180 秒总时限；只有摘要与索引达到 ready 才进行后续检索，失败、缺失状态和 ready_text_only 不能被当成建立索引成功。 +- 两个 producer 不再调用 `runWorker`；其依赖类型也不再要求执行能力。 +- 手工重试仍通过现有路由更新状态并唤醒后台调度。 + +### 3. 生命周期也纳入边界 + +预算准备及排空后的自动历史 Skill 对账一并放在入口内。否则仅等待 runner 的模型执行结束，MemoryService 仍可能在数据库关闭后做收尾写入。 + +`MemoryService.stop()` 先停止接收新执行请求、拒绝未开始请求，允许当前有租约保护的请求及其收尾完成后再结束；外部重复关闭仍等待同一结束状态。每次关闭都先检查自等待，防止活跃任务等待一个已经由外部发起、但又必须等它结束的关闭 Promise。 + +HTTP 关闭继续等待已接受的请求与自动 drain；测试验证关闭 sockets 不会跳过服务器已接受的工作。 + +## 验收与证据边界 + +使用真实 MemoryService、WorkerRunner、HTTP 和 SQLite，模型与业务慢请求使用受控本地替身。数据位于临时测试目录，未接入当前 Memmy 窗口、未触发真实模型请求、未改用户数据库。 + +- 两个并发 `runWorkerOnce(4)` 摘要请求：峰值由原始复现的 **8 降为 4**；第二个请求的任务保持 queued、attempts=0、没有 lease token，之后正常完成。 +- 真实 HTTP 自动处理与手工请求重叠：同样只有当前请求领取四条摘要，手工请求等待；释放受控慢请求后全部目标完成。 +- `enqueued=0`：现有摘要被成功唤醒并最终 ready。 +- 导入、首次报告：模拟后台独立推进目标状态，验证完成、失败、取消及超时路径，同时断言没有调用 `runWorker`。 +- 关闭、同步/异步错误、同门重入、旧异步上下文、独立唤醒、直接 retry、请求过滤和默认 limit 均有回归覆盖。 +- 原配置隔离与 observation 修订用例改为在新边界允许的场景下验证：并发可以发生在同一已接纳批次中，后续请求必须等待；继续验证旧配置失败不能恢复新配置下真实失败的其他任务，旧 generation 不能覆盖新 generation 的处理状态。 + +## 检查结果 + +- Memory 全量：**1465 项通过，1 项跳过**。 +- Backend 全量：**1073 项通过**。 +- `npm run memory:lint`：共享类型构建、Memory 与 Viewer 类型检查通过。 +- Backend 源码 TypeScript 检查及全模块 ESLint：通过。 +- 发布流程检查：30 项通过，5 项跳过。 +- 打包守卫检查：122 项通过，1 项跳过。 +- `git diff --check`：通过。 + +Backend 完整回归前按仓库脚本生成本地 lifecycle sidecar，并将独立工作区的 `@memmy/local-api-contracts` 链接到本分支编译结果；原链接引用另一工作区的旧产物，使两项语言测试错误返回 en-US。修正测试环境后全量通过，未修改语言功能源码，也未提交生成资源、node_modules 或锁文件。Memory 与 Backend 全量独立运行，避免一万条诊断等固定时间预算用例因并行争用资源而误超时。 + +日志：`/private/tmp/memmy-stage4-memory-tests-final.log`、`/private/tmp/memmy-stage4-backend-tests-final.log`、`/private/tmp/memmy-stage4-memory-lint-final.log`、`/private/tmp/memmy-stage4-backend-typecheck-final.log`、`/private/tmp/memmy-stage4-backend-lint-final.log`、`/private/tmp/memmy-stage4-release-guards.log`、`/private/tmp/memmy-stage4-packaging-guards.log`。 + +## 本阶段尚未改变的能力 + +统一入口不等于已经完成全部并发控制。默认 `limit=100` 保留原工作预算和返回语义；摘要在单次执行中仍每组最多四个，Embedding 仍可能整批预取和准备。**不能把阶段 4 的摘要峰值 4 当成所有任务都已具备全局逻辑活跃数 4 的证明。** 去除提前租约、四层许可、Embedding 微批次及连续补位属于阶段 6。 + +旧严格优先级、批次屏障和 retry 仅用剩余名额的规则尚在；入口统一后，首次报告也不能抢占一个已经运行中的长批次。阶段 5 将切换真实 WRR＋可运行 FIFO，阶段 6 再落实受控并发。 + +本阶段不单独打包、不推送、不提交 PR；按整体计划在全部改造后统一 Mac 打包与安装验收。完成本地提交后等待用户确认，再进入阶段 5。 diff --git a/docs/qa/memory-queue-stage5-20261022.md b/docs/qa/memory-queue-stage5-20261022.md new file mode 100644 index 000000000..1957ce12d --- /dev/null +++ b/docs/qa/memory-queue-stage5-20261022.md @@ -0,0 +1,111 @@ +# Memory 队列改造：阶段 5 真实 WRR 与可运行 FIFO + +日期：2026-10-22。工作分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`。承接阶段 4 提交 `e426aed0`。用户确认的五个 Lane 权重为 **interactive / repair_retry / import / evolution / maintenance = 4 / 3 / 2 / 2 / 1**。 + +## 复现与范围 + +阶段 1 的旧算法诊断已证明：主任务占满领取名额时，Embedding retry 只能获得零个剩余名额；严格优先级和同优先级 cohort 会让其他任务长期等待；同档内按记忆创建时间倒序会使旧任务反复被新记录插队。 + +本阶段先用阶段 4 实现运行新增的 13 项实际 runner 调度用例，结果 **9 项失败、4 项通过**。失败覆盖跨 Lane 机会、主任务与 retry 共同领取、FIFO 和连续小 limit 调用等，日志 `/private/tmp/memmy-stage5-fairness-before.log`。这些用例使用真实 SQLite 和 WorkerRunner，业务处理器使用受控本地替身，不访问用户数据库或真实模型。 + +本阶段将影子算法切换为实际任务选择规则，同时保留旧算法回退。继续沿用阶段 2 的租约保护、退避与阶段 4 的统一执行入口，不实施阶段 6 的四层资源许可和连续补位，不恢复已取消的生产背压方案。 + +## 实现与行为 + +### 1. 按 Lane 分配机会，Lane 内可运行 FIFO + +- 默认启用 WRR。所有 Lane 持续有可运行任务时，每个完整 12 次领取周期按 4/3/2/2/1 分配机会；空 Lane 直接跳过，不浪费名额。权重衡量领取次数，不是运行秒数或模型成本。 +- 同一 Lane 中按稳定的 `enqueueSeq` 从小到大选择，与记忆创建时间无关。暂时不可运行的退避、配置、依赖或业务范围外任务不会挡住后面的可运行任务，恢复后保留原入队顺序。 +- WRR 不再使用旧的“同优先级独占本轮”cohort 限制。Onboarding 首次报告保留 interactive 高权重定位，同时遵守 interactive 内 FIFO，不抢占正在运行的任务。 +- 主任务与 Embedding retry 来自原有两张持久化表，但进入同一次选择，共用总领取预算；retry 不再只使用主任务的剩余名额。直接调用公开 retry 排空接口时，也使用相同的 FIFO 选择与统一执行入口。 +- 领取与执行之间不再按 Embedding role 重新排序：主 Embedding 和 retry 都保留选中顺序，仅将连续同 role 的条目组成批次，仍保留相邻条目的批处理能力。FIFO 约束选择与启动顺序，不要求并发任务按同一顺序完成。 + +### 2. 公平性跨请求、重启持续生效 + +生产选择器与影子选择器共用纯算法实现。无过滤的实际 WRR 游标保存在数据库 KV `worker.scheduler.wrr_cursor.v1`，连同权重签名保存。受目标、类型或 retry 范围限制的请求使用各自的持久化游标，不推进无过滤全局游标；同一筛选范围内的连续小请求也延续自己的轮次。连续 `limit=1` 调用或服务重启不会反复从 interactive 起点开始。权重改变时重新开始对应的新轮次。 + +候选投影刷新、选择、按 ID 领取和游标更新处于同一事务。每次领取重新验证可运行条件，并生成租约 token；领取失败不消耗轮转机会，异常回滚同时撤销本次领取和游标推进。 + +WRR 将退避已结束、仍有失败重试次数且未被配置或依赖阻塞的 failed 主任务直接纳入候选，与 queued 任务共同按 Lane 与原入队序号竞争，再按 ID 原子领取。不再先按旧优先级限量恢复，否则尚未被恢复的旧任务会被排除在公平调度之外。legacy 回退继续保留原恢复方式。 + +生产路径完整刷新当前符合条件的待分类记录，避免影子采样的 500 条刷新上限使旧任务或某个 Lane 永久不可见。终态、未来执行时间和当前不符合筛选条件的记录不纳入本次刷新。未知任务显式进入受控清理路径并标记 dead letter，不调用未知业务处理器，也不持续触发空转。 + +### 3. 保留租约与预算安全 + +WRR 选择顺序可能包含“主任务、retry、主任务”。所有已领取任务从领取后立即开始续租，等待前面任务时也受保护。测试将前面的任务保持 310 秒，超过 retry 原始 5 分钟租期，验证续租有效且其他 worker 无法重新领取。 + +如果执行期间预算暂停，尚未启动的 retry 使用最新续租状态安全释放回 pending；不增加失败次数，不重置 FIFO 顺序和原重试时间。主 Embedding 在每个新 role 批次及输入过长后的逐项降级请求前重新检查预算；未发送请求的条目归还 queued，并将仍属于本任务的 processing 状态恢复为 embedding_pending。retry 的逐项降级同样检查预算，归还尚未发送的条目。恢复后可继续处理，不覆盖已被新租约或新任务接管的状态。 + +`limit` 仍是一次执行请求的总工作预算，WRR 中包含主任务和 retry。返回结构继续分别统计 `leased`（主任务）和 `embeddingRetries.leased`，两者之和不超过该请求预算。内部每次事务最多领取 200 条；大请求分段处理。**200 是领取分段上限，不是并发上限。** + +## 配置与回退 + +无需配置时使用 WRR 与已确认权重。进程启动环境变量： + +```sh +MEMMY_WORKER_SCHEDULER_ALGORITHM=wrr +MEMMY_WORKER_LANE_WEIGHTS='{"interactive":4,"repair_retry":3,"import":2,"evolution":2,"maintenance":1}' +``` + +需要回退时，将 `MEMMY_WORKER_SCHEDULER_ALGORITHM` 改为 `legacy` 并重启对应服务。旧严格优先级、cohort、记忆时间倒序及剩余 retry 名额规则恢复；阶段 2 的租约保护和阶段 4 的统一入口保留。回退本身也意味着旧算法的饥饿风险重新存在。 + +非法算法、非法 JSON、缺少 Lane、未知 Lane、零值或不合法整数权重会明确报错，不静默换用另一组配置。依赖注入配置按字段优先于环境配置。 + +影子观测开关 `MEMMY_WORKER_SCHEDULER_SHADOW=0` 仅关闭对比采样，不关闭实际 WRR。影子选择使用独立游标和有限采样，不能把它当成实际游标的逐条回放。实际任务日志增加 `queueLane`、`enqueueSeq`，排空日志标注 `algorithm`，WRR 日志同时记录权重和两类任务计数。 + +## 验证覆盖 + +首次实现新增实际 runner 调度测试 **26 项通过**，主要覆盖： + +- 一万条导入积压下，interactive、repair_retry、evolution、maintenance 仍获得机会；持续有任务时连续 12 次 `limit=1` 符合权重。 +- 主表与 retry 表共同选择、跨表 FIFO、记忆创建时间不插队、空 Lane 借用、旧 cohort 解除。 +- 退避、配置、依赖、目标筛选跳过不可运行任务，Onboarding 高权重，临时失败不吞掉其他 Lane 的机会。 +- 数据库关闭重开后游标延续；第二次领取异常时第一条租约和游标一起回滚；自定义权重实际生效。 +- 超过旧投影采样边界的脏记录仍按完整 FIFO 可见；未知任务不会阻断有效任务或造成持续唤醒。 +- 等待中的 retry 续租、续租后预算暂停释放、恢复处理；错误配置拒绝启动；显式 legacy 回退。 +- 主 Embedding 的 query/document 顺序真实保留，相邻同 role 继续成批；legacy 维持原 role 顺序。 + +另外增加按 ID 原子领取、重复领取失败、过期恢复、token 不匹配、释放后顺序保留、完整投影和元数据形状容错等存储测试。 + +原业务测试中依赖“新记忆优先”和“同优先级独占一轮”的断言改为验证新规则，保留摘要、向量、模型提示词和演化结果约束。历史基线复现明确指定 legacy，不将旧缺陷误写为当前默认行为。两项旧夹具同步修正：Agent 导入公平性使用真实 import 入口；L3/Skill 提示词测试先等待其前置 L2 更新完成再手动晋升，避免后续合法更新改变测试前提。 + +## 首次实现检查结果与证据 + +- Memory 全量：**1523 项通过，1 项跳过**；`npm run memory:lint` 通过。 +- Backend 全量：**1073 项通过**；Backend TypeScript 检查通过。 +- 发布流程保护：**30 项通过，5 项跳过**。 +- 打包保护：**122 项通过，1 项跳过**。 +- `git diff --check`：通过。 +- 分工实现后独立检查了实际选择、存储领取和公平性用例；发现的 Embedding 执行时 role 重排已修复并补充回归。 + +首次实现日志：`/private/tmp/memmy-stage5-memory-tests-final.log`、`/private/tmp/memmy-stage5-memory-lint-final.log`、`/private/tmp/memmy-stage5-backend-tests.log`、`/private/tmp/memmy-stage5-backend-typecheck.log`、`/private/tmp/memmy-stage5-release-guards.log`、`/private/tmp/memmy-stage5-packaging-guards.log`。这些通过结果未覆盖后续审查发现的四处边界，不能单独作为阶段 5 已完整验收的依据。 + +额外 SQLite 微基准使用 5 万条已分类终态记录加 1 万条待分类可运行记录：首次刷新 1 万条约 **324ms**，之后重复候选查询约 **4.9–5.8ms**。日志 `/private/tmp/memmy-stage5-storage-bench.log`。这只是本机小记录存储基准，不能替代长对话、真实模型吞吐量或桌面 UI 压测，也不能推断所有数据规模的耗时。 + +## 审查后的四项补修与重新验收 + +用户要求检查首次实现后，发现并复现四处遗漏；首次提交 `a074e8b9` 不能作为阶段 5 完整验收终点。本轮修复仅针对这四处问题，未进入阶段 6。 + +1. **隔离筛选请求的轮转进度。** 原来所有请求共享一个游标，定向请求看不到其他 Lane，却跳过它们并改变全局轮次。补修前自动请求与定向 maintenance 请求交错 10 轮，自动执行的 40 条全是 interactive。现在目标、允许类型、排除类型规范化去重排序，连同是否包含 retry 使用完整 SHA-256 生成独立 scope key；局部与全局轮次分别持久化，不截断大目标集合。真实 runner 回归交错 12 轮，自动领取 48 条严格符合 **16/12/8/8/4**，定向请求正常完成；公开 retry-only 请求也不消耗全局轮次。 +2. **到期失败任务直接参加 WRR。** 原来 `requeueFailedJobs(limit)` 先按旧排序筛选，较早到期的失败任务可能连候选资格都得不到。现在 WRR 候选投影、查询与按 ID 领取共同启用 `includeDueFailed`，保留时间、配置、失败次数、依赖和 scope 限制；legacy 才保留旧的限量恢复步骤。新增回归验证同 Lane 的旧任务先执行、跨 Lane 的到期失败都有机会，旧执行者不能完成或归还新租约，事务回滚同时恢复失败状态和游标。 +3. **Embedding 每次新请求前检查预算。** 主任务和 retry 均在角色批及输入过长后的逐项降级请求前检查，涵盖 WRR 和 legacy 回退。暂停时仅对归还成功的任务更新状态；主任务的租约归还与 processing 恢复处于同一事务，校验 activeJobId 和记忆内容代际，不覆盖替换租约或其他任务。回归验证暂停后不再发送后续请求、归还后能恢复、FIFO 和失败历史不变，以及替换租约隔离。 +4. **验证真实日志输出。** 补齐格式化器字段白名单，使主任务和 retry 的 `queueLane`、`enqueueSeq` 及排空的 `algorithm`、`laneWeights` 真正出现在消息中。保留原有 timestamp/level/message 外层格式与脱敏机制。测试经过真实 logger 到 stdout/stderr 写入路径，覆盖事件、级别路由、legacy 缺省权重及凭据脱敏，避免仅验证调用参数。 + +新增 **29 项**回归：runner 公平性 5 项、存储范围与失败恢复 8 项、Embedding 预算边界 8 项、日志 8 项。先运行修前核心用例并确认失败：公平性 5 项、存储 7 项、预算 6 项、日志 8 项；再修实现，并补齐租约和 legacy 边界验证。修前证据包括 `/private/tmp/memmy-stage5-review-fixes-fairness-before.log`、`/private/tmp/memmy-stage5-review-storage-before.log`、`/private/tmp/memmy-stage5-budget-boundary-before.log`。 + +重新验收结果： + +- Memory 全量：**1552 项通过，1 项跳过**。 +- Backend 全量：**1073 项通过**。 +- `npm run memory:lint`：共享类型构建、Memory 与 Viewer 类型检查通过。 +- 发布流程保护：**30 项通过，5 项跳过**。 +- 打包保护：**122 项通过，1 项跳过**。 +- 差异检查通过。独立只读复查未发现新的阻塞项；额外 SQLite 验证全局与局部游标均跨数据库重开续轮，权重改变时按新轮次开始。此前复查提出的 legacy retry 跨角色预算边界已补齐并重跑 8 项预算测试通过。 + +本轮最终日志：`/private/tmp/memmy-stage5-review-fixes-memory-full.log`、`/private/tmp/memmy-stage5-review-fixes-backend-full.log`、`/private/tmp/memmy-stage5-review-fixes-memory-lint.log`、`/private/tmp/memmy-stage5-review-fixes-release-guards.log`、`/private/tmp/memmy-stage5-review-fixes-packaging-guards.log`。新增的测试继续使用独立临时 SQLite 数据及受控处理器，没有接入当前 Memmy 窗口、用户数据或真实模型请求。 + +## 阶段边界 + +WRR 保证持续可运行 Lane 获得领取机会，但前面的慢请求、既有执行批次屏障和 Provider 延迟仍可能拉长等待；没有承诺固定时间内完成，也没有抢占已运行任务。当前仍会预先领取一段任务并续租，不应声称已完成“拿齐全部资源许可后才领取”。四层许可、去除提前领取和任务完成后立即补位属于阶段 6。 + +本阶段未变更数据库 schema 版本，未升级依赖或包版本，未使用用户数据和真实模型。本阶段未生成 DMG、未进行安装验收、未推送、未提交 PR；按整体计划在后续阶段统一 Mac 打包验收。保存本地提交后等待用户确认，再进入阶段 6。 diff --git a/docs/qa/memory-queue-stage6-20261022.md b/docs/qa/memory-queue-stage6-20261022.md new file mode 100644 index 000000000..44b4a1049 --- /dev/null +++ b/docs/qa/memory-queue-stage6-20261022.md @@ -0,0 +1,135 @@ +# Memory 队列改造：阶段 6 分层并发与连续补位 + +日期：2026-10-22。工作分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`，承接阶段 5 补修提交 `597de81c`。本阶段不单独生成 DMG、不推送、不提交 PR；整体安装验收按后续统一验证阶段执行。 + +复查更新：初版提交 `6646adce` 的全量测试虽通过，独立复查仍发现 5 项遗漏。以下验证表保留初版记录，后续修复与最终验收以 [阶段 6 复查补修记录](memory-queue-stage6-review-fixes-20261022.md) 为准。 + +## 复现与范围 + +阶段 5 已统一执行入口并修复 WRR/FIFO，但 `limit=100` 仍会预先领取整段任务；摘要一组内的快任务结束后，还要等待该组最慢任务。其他类型即使不能并行，也会提前获得租约；Embedding 的逻辑条数、批处理大小与真实请求并发尚未分开控制。 + +本阶段先新增真实 SQLite＋WorkerRunner 的受控慢任务验收。修前最初 7 个场景均失败；扩展到含异常和 Provider 场景的 16 个用例时，11 项失败、5 项通过。证据：`/private/tmp/memmy-stage6-admission-before.log`、`/private/tmp/memmy-stage6-admission-before-full.log`。新存储准入的最初 9 项修前全部失败；同源扫描最初 10 项中 8 项失败、2 项通过。所有模型请求使用本地受控替身，没有接入当前 Memmy 窗口或用户数据库。 + +范围是工作文档第七部分的四层并发限制及连续补位。继续使用一个 Memory 进程、原有持久化任务表、WRR 4/3/2/2/1 和可运行 FIFO；不恢复已取消的生产背压方案，不引入多个 SQLite 写进程。 + +## 运行方式 + +手动入口的 `limit` 表示一次执行请求的总工作预算，不再代表预领取数量或并发上限。复查补修后，自动入口通过内部 continuous 模式持续执行，直到当前无可运行工作、取消或需要交还入口给手动请求；不再受 4 条总预算的整批屏障限制。执行流程： + +1. 根据 WRR/FIFO 查找可运行候选。 +2. 同步检查全局、资源、Provider 根及模型、业务范围所需的全部许可。 +3. 全部可用时一次性取得许可，再在同一数据库事务内按 ID 领取。 +4. 立即进入执行，只有实际运行的任务维持续租。 +5. 任一任务完成即释放许可并唤醒补位，不等待其他慢任务。 + +资源不足时不领取任务、不生成 token、不增加 attempts，也不先占住部分许可再等待其他许可。因此不存在任务 A 占一半资源等任务 B、任务 B 又反向等待 A 的部分获取死锁。模型实际执行期间的内部串行请求，使用该任务已取得的许可，不重复获取自己的 Provider 槽位。 + +同一 Lane 的资源阻塞项保持入队位置，但允许寻找后续可运行任务。候选按 Lane、任务表和入队序号分页，不能因为前 200 或 500 条都被同一资源挡住，就永久看不到后面的其他资源任务。 + +并发任务可以按不同顺序完成；FIFO 约束候选选择和启动机会，不要求结果完成顺序相同。返回的主任务结果仍按领取顺序排列；同一个任务若在长 drain 内退避到期后再次领取，使用不同 lease token 分别统计每次结果，不能用后来的成功覆盖先前失败。 + +## 初始并发参数 + +| 限制 | 默认值 | 含义 | +| --- | --- | --- | +| Memory 活跃逻辑任务 | 4 | 所有 worker/run 与 retry 入口共用 | +| summary 资源 | 4 | 仍受 Provider 限额制约 | +| evolution_llm 资源 | 4 | 仍受 Provider、业务范围限额制约 | +| embedding 逻辑资源 | 4 | 每条记忆占一个逻辑槽位 | +| database 资源 | 1 | 维护类数据库任务保守串行 | +| scan 资源 | 1 | Memory 策略资源；Agent source 实际扫描另在后端同源入口互斥 | +| 本地 Provider 根及模型 | 1 | 同一实际服务共享，不因角色名不同重复计算 | +| 云 Provider 根及模型 | 2 | 模型还要服从其 Provider 根上限 | +| Embedding 请求 | 1 | 与批量条数独立 | +| Embedding 微批大小 | 最多 16 条 | 同时服从可用逻辑槽位与 embedding 资源余量 | + +因此，默认全局上限 4 时，一个微批最多 4 条；若另有两个逻辑任务活跃，最多只领取 2 条用于微批。只有显式提高全局及 embedding 逻辑资源上限后，才可能形成 16 条微批，不能为了凑批次而提前领取等待。 + +可通过启动环境变量设置： + +```sh +MEMMY_WORKER_MAX_ACTIVE_JOBS=4 +MEMMY_WORKER_RESOURCE_LIMITS='{"summary":4,"embedding":4,"evolution_llm":4,"database":1,"scan":1}' +MEMMY_WORKER_LOCAL_MODEL_CONCURRENCY=1 +MEMMY_WORKER_CLOUD_MODEL_CONCURRENCY=2 +MEMMY_WORKER_EMBEDDING_REQUEST_CONCURRENCY=1 +MEMMY_WORKER_EMBEDDING_BATCH_SIZE=16 +``` + +配置接受正整数并进行范围校验，未知资源类或非法配置明确报错。测试可以注入控制器以隔离某一层约束；显式提高测试上限的用例不作为生产默认并发的证明。 + +## Provider 与模型调用 + +在准入前固定本次任务使用的模型上下文，实际执行使用同一组客户端；运行中更新配置不会让已准入任务转而调用未取得许可的模型。Provider 身份基于真实服务地址及模型，去除凭据和查询参数；本地回环地址别名、相同地址的不同 source 标签应共享同一根限制。配置未填写地址时按模型实现的实际默认地址归一化。 + +真实模型调用也经过同一控制器。任务内部对同一个 Provider 的并行调用串行使用自身许可；前台模型请求遵守同一 Provider 和 Embedding 请求限额，但不占 Memory 后台逻辑任务名额。任务提前结束时，已经开始的物理请求仍保留相应 Provider 限额至请求真正结束，不能因逻辑任务释放而放大物理请求并发。 + +摘要任务可能回退到演化模型，反馈经验任务也可能调用向量模型；策略注册表必须声明这些次级依赖，不能只声明主资源。审查已用真实业务链识别并补齐导入摘要回退和 feedback_experience 的依赖，配合实际路由回归验证。 + +相邻且同类、同 role、同 Provider 和同模型上下文的 Embedding 项才共享一个微批请求许可。每个成员分别占逻辑、资源、业务范围许可；一个成员 CAS 失败时只归还自己的份额，不影响其他已经领取的成员。成员组全部结束后才释放共享 Provider 与请求许可。仅凭地址和模型相同还不足以混合配置，凭据、headers 或其他真实模型配置变化也会使微批上下文区分。 + +## 业务范围与扫描 + +- 同一 target 的主任务与 retry 共享排他范围，不因为来自不同表或不同 jobType 而重复执行。 +- 同 episode 的相关任务共享 episode 范围。 +- L3 与 Work Memory 按用户、项目共享业务范围。生产 L3 payload 只有 batchId/targetField，Work Memory payload 只有轨迹数据，因此从真实 batch/session 读取项目，不能把两种不同的 scope 哈希当成同一个项目范围。 +- Skill 聚类分配与演化也按用户、项目共享范围，分别从真实 episode/cluster 解析项目，避免演化的旧快照覆盖并发加入的新成员。 +- 后端对同一 repository 实例、同一个 Agent source 的实际扫描/收集及持久化 staging 进行互斥，覆盖 collectOne/All 与 scanOne/All。不同 source 可并行；等待可以取消，失败后释放。锁不覆盖摘要等待，不整体锁住 scanAll。 + +扫描限制是相同 repository 实例共享的进程内限制，不宣称跨进程，也不宣称整个历史导入生命周期都串行。后端扫描不是额外插入 Memory 主任务表的第二套任务事实。 + +## 唤醒、异常与关闭 + +补位同时响应任务完成、许可释放和新任务唤醒。HTTP 自动调度通知当前活跃 drain，即使某个慢请求未结束，只要还存在其他空闲资源，新任务也可以进入。同步 CAS 失败归还许可产生的通知不会形成持续空转。 + +许可先取得、ID CAS 后执行。CAS 失败立即归还；整个领取事务异常时，连同本事务前面已经取得的许可一起归还，游标和数据库状态同步回滚。运行中业务异常仍通过原有失败退避与 fencing 路径处理。 + +停止时不再领取新任务，允许已经准入的任务及其收尾完成，再退出统一入口。关闭流程同时处理模型许可的等待者与已开始的完整模型业务任务，避免 Provider 释放后唤醒一个本应停止的请求，或在请求之后的数据库收尾尚未完成时关闭存储。 + +## 验证记录 + +初版提交 `6646adce` 于 2026-10-22 在 macOS arm64、Node.js 22.23.3、npm 10.9.9 下完成以下检查；补修后的最终数字见本文开头链接的复查记录： + +| 检查范围 | 结果 | 证据 | +| --- | --- | --- | +| Memory 全量 | 1627 通过、1 跳过；161 个测试文件通过、1 跳过 | `/private/tmp/memmy-stage6-memory-final-clean.log` | +| 后端全量 | 1085 通过，132 个测试文件通过 | `/private/tmp/memmy-stage6-backend-final.log` | +| Memory 服务与 viewer 类型检查 | 通过 | `/private/tmp/memmy-stage6-memory-typecheck-final.log` | +| 后端类型检查 | 通过 | `/private/tmp/memmy-stage6-backend-typecheck-final.log` | +| 后端源码 ESLint（排除忽略的生成 bundle） | 通过 | `/private/tmp/memmy-stage6-backend-lint-source-final.log` | +| release workflow 保护 | 30 通过、5 跳过 | `/private/tmp/memmy-stage6-release-final.log` | +| packaging guards | 122 通过、1 跳过 | `/private/tmp/memmy-stage6-packaging-final.log` | +| Git 差异空白和冲突标记检查 | 通过 | `git diff --check` | + +首次最终全量运行中，原有 `agent-source-runtime` 自动扫描测试出现一次 `automatic scan did not complete`，当时其他 1626 项通过。未更改该测试及生产代码，随后单文件 16 项通过，完整重跑 1627 项通过。保留失败日志 `/private/tmp/memmy-stage6-memory-final-verified.log` 和单独复核日志 `/private/tmp/memmy-stage6-auto-scan-recheck.log`，不隐去该偶发等待失败。 + +后端首次全目录 lint 将测试准备生成且被 `.gitignore` 忽略的 `memmy-workspace-bridge.mjs` 也纳入检查，报两处该 bundle 内禁用规则缺失。排除这一生成文件后，全部后端源码检查通过；没有为通过检查而修改规则或提交生成物。此阶段没有执行整个前端和 Agent 工作区的全量检查，留在统一回归阶段。 + +独立审查覆盖存储事务与许可回收、真实执行补位及测试语义，并完成交叉复核。审查过程中发现并补充了下列回归： + +- 导入摘要回退模型与反馈经验向量调用需要在准入前声明完整依赖。反馈真实链路的两个用例修前均失败，修后及全量通过。 +- reflection 与 reward 必须按真实 episode 串行，同时允许另一 episode 执行。两个专用回归修前均失败，修后及全量通过。 +- 同一任务在一个长 drain 内失败并再次到期时，结果必须按 lease token 区分。修前错误汇总为 3 成功、0 失败；修后正确保留 2 成功、1 失败。 +- 批次中间 CAS 失败只释放对应成员；连续 CAS 失败不能因同步释放通知形成空转；完整领取事务失败需归还前面已取得的许可。 +- 停止服务需要等待真实请求及随后数据库收尾；已完成任务遗留的异步上下文不能被误认为仍在执行，也不能绕过已关闭路由。 + +现有测试中，依赖整批预领取、批次屏障或旧 document-first 分组的断言已改为阶段 6 的实际运行规则；原有 fencing、预算暂停、模型配置切换、业务输出和 SQLite 状态检查仍保留。测试中的显式并发覆盖仅用于隔离原有业务语义，生产默认值由独立准入与真实模型请求计数测试验证。 + +### 一万条长正文调度基准 + +使用真实 MemoryService 与 SQLite，插入 10000 条导入摘要任务，每条记忆正文 8192 字节；受控本地模型请求保持未完成。两种实现对比都只领取 1 条，其余 9999 条仍为 queued，控制器记录活跃逻辑任务为 1；不存在为了填满 `limit` 而提前领取。 + +| 同步补位中的配置解析方式 | 首个 handler 开始 | 配置加载器调用 | 候选准备次数 | +| --- | --- | --- | --- | +| 每个候选分别建立上下文 | 501.3ms | 10005 | 10000 | +| 本轮共用配置快照（最终实现） | 264.9ms | 6 | 10000 | + +证据为 `/private/tmp/memmy-stage6-provider-bench.log`。这是本机一次受控对比，配置加载器返回内存配置，不能换算为真实磁盘配置读取成本或真实模型吞吐；仍会扫描被资源挡住的候选，因此不宣称消除了所有大队列调度成本。 + +上述验证使用真实 SQLite 和 WorkerRunner、受控慢模型、实际路由与后端扫描替身；没有接入用户数据库，不替代 DMG 安装、真实 Provider 压测或 Windows 验证。打包保护测试通过不等于已生成或安装 DMG。 + +## 交付边界 + +legacy 开关保留旧的选择顺序与 cohort 策略，但同样服从新的准入、Provider 限额和连续补位；它不恢复不安全的整批预领取或旧批次屏障。对新旧选择算法的对比，应区分选择策略与执行安全边界。 + +本阶段完成后停下等待用户确认。整体回归、同步最新目标分支、真实模型参数收敛、统一 macOS 打包安装与最终 PR 确认仍在后续统一验证阶段；不把自动化保护测试描述成安装测试已通过。 diff --git a/docs/qa/memory-queue-stage6-boundary-fixes-20261022.md b/docs/qa/memory-queue-stage6-boundary-fixes-20261022.md new file mode 100644 index 000000000..12f8e34d4 --- /dev/null +++ b/docs/qa/memory-queue-stage6-boundary-fixes-20261022.md @@ -0,0 +1,64 @@ +# Memory 队列阶段 6：开关、事件循环与 Provider 边界补修 + +日期：2026-10-22。基线 `23efb8bd`，分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`。本轮仅修复已确认的 3 项，补回归并重新审查；不进入下一阶段，不生成 DMG、不推送、不提交 PR。 + +## 复现与修复 + +### 运行中关闭记忆写入 + +真实 MemoryService、SQLite、配置 reload 与受控摘要模型，首条请求挂起后关闭 `algorithm.enableMemoryAdd`。修前 continuous 仍处理完 8 条；原有限批次完成 4 条后，下个入口才停止。证明持续运行绕过了原先每个入口的开关检查。 + +现在每轮准入前、预算准备步骤返回后及当前领取波次启动后都检查开关，关闭后不再取得新任务租约，已经准入的任务可以收尾。retry-only 入口关闭时直接返回空执行结果。配置 reload 无论是否请求恢复失败任务都会唤醒当前 pump，避免它等待无关 Provider 释放后才看到开关变化。 + +真实配置回归使用单逻辑槽位：关闭后仅首条已准入摘要完成，其余 7 条仍 queued，attempts、leaseCount 为 0，token 为空；重新开启后，新一轮正常处理这些记录。另覆盖等待 `beforeRun` 时关闭，以及短任务链中由定时回调关闭。 + +### 快速任务连续补位阻塞事件循环 + +修前真实 SQLite 与 WorkerRunner 的 200 条短数据库任务全部完成后，预先安排的 0ms 取消回调才运行，单次记录约 531ms；纯 Promise 微任务链没有把机会交给定时器与 HTTP。外层 HTTP 的让步位于整次 continuous 返回之后，不能保护执行中的长任务链。 + +现在每累计领取 32 条逻辑工作，在下一轮准入前通过原生 `node:timers/promises.setImmediate()` 让出一个事件循环机会；已经领取的工作立即启动，不为了让步预领取后等待。让步结束后回到循环顶部，重新检查写入开关、取消、停止和等待的手动请求。 + +32 是让步频率，不是并发上限、任务总预算或毫秒响应承诺。任务自身的同步耗时不在此机制中被抢占。默认全局 4、本地 1、云 2、Embedding 请求 1 不变；有限手动请求仍保留其预算与目标过滤。测试允许 timers/check 阶段顺序差异，不要求取消恰好发生在第 32 条。 + +### Mistral 跨协议服务限额 + +真实聊天客户端、原生 Mistral Embedder、Router、Controller，只有网络响应使用受控替身。修前 2 个 `/v1/chat/completions` 与 1 个 `/v1/embeddings` 同时进入同一 Mistral API 服务，峰值 3 超过云限额 2。 + +现在原生 Mistral 的末尾 `/embeddings` 按该服务的 API base 归一化，与兼容聊天共享 Provider 根；保留主机及前面的代理路径，不把不同主机或不同租户 base path 合并。覆盖默认、显式端点和代理路径隔离；之前的 Anthropic 及前台/worker 上下文回归仍保留。 + +## 修前证据与验证范围 + +- 写入开关原复现：`/private/tmp/memmy-stage6-continuous-capability-review.log`；正式开关回归修前 4 项均失败：`/private/tmp/memmy-stage6-capability-reload-before.log`。 +- 事件循环原复现：`/private/tmp/memmy-stage6-continuous-abort-yield.repro.log`。 +- 正式事件循环回归的独立基线副本：`/private/tmp/memmy-stage6-eventloop-baseline.log`，6 项修前全部失败；当前新增 6 项和原连续调度 33 项共 39 项通过：`/private/tmp/memmy-stage6-eventloop-after.log`。 +- 关闭时唤醒 Provider 等待的单独对照：`/private/tmp/memmy-stage6-capability-notify-before.log`。仅在临时副本删除 reload 的通知行，该场景失败；当前正式开关文件 5 项全部通过。 +- Mistral 原复现：`/private/tmp/memmy-provider-audit.log`；正式回归修前 3 项均失败：`/private/tmp/memmy-stage6-mistral-before.log`。 + +修前对照及新测试使用隔离的临时数据库、当前源码与基线的临时加载/副本，没有回滚工作树。未接入当前桌面用户数据，没有外部模型调用或 DMG 安装测试。 + +## 最终验证与复查 + +本轮新增 14 项正式回归：写入开关与重载 5 项、事件循环机会及清理 6 项、Mistral 归一化 3 项。事件循环测试同时检查： + +- 0ms 取消、停止、手动请求能在 200 条短任务清空前生效；未领取记录保持 attempts 为 0、token 为空。 +- 跨宏任务补位后，全局峰值仍为 4、同一个 Provider 峰值仍为 2。 +- 异常与取消前确实创建了相关 deadline/heartbeat 定时器；完成后定时器、许可全部归零。 +- 配置关闭后重启一轮可以继续处理原排队记录，没有人为消耗重试次数。 + +代码冻结后，在 macOS arm64、Node.js 22.23.3、npm 10.9.9 下检查： + +| 检查 | 结果 | 日志 | +| --- | --- | --- | +| Memory 全量 | 1664 通过、1 跳过；167 个文件通过、1 跳过 | `/private/tmp/memmy-stage6-boundary3-memory-full.log` | +| Memory 服务与 viewer 类型检查 | 通过 | `/private/tmp/memmy-stage6-boundary3-typecheck-final.log` | +| 后端全量 | 1085 通过，132 个文件通过 | `/private/tmp/memmy-stage6-boundary3-backend-full.log` | +| 后端类型检查 | 通过 | `/private/tmp/memmy-stage6-boundary3-backend-typecheck.log` | +| release workflow 保护 | 30 通过、5 跳过 | `/private/tmp/memmy-stage6-boundary3-release.log` | +| packaging guards 最终重跑 | 122 通过、1 跳过 | `/private/tmp/memmy-stage6-boundary3-packaging-final.log` | +| 差异检查 | 通过 | `git diff --check` | + +首次 packaging guards 有 2 项 `packaged-runtime-config.test.mjs` 失败（Windows History recorder 正常夹具的退出状态、Windows 分隔符夹具的 ASAR JSON 读取），其余 120 项通过、1 跳过。未更改源码、测试或检查规则，随后该文件 29 项单独通过，完整保护重跑 122 项通过、1 跳过。原因尚未确认，不把首次失败隐去，也不将其归因为已经确定的 CPU 或环境问题。首次日志 `/private/tmp/memmy-stage6-boundary3-packaging.log`、单独复核 `/private/tmp/memmy-stage6-boundary3-packaging-recheck.log` 均保留；未因该次失败扩大本轮修改范围。 + +独立交叉复查已检查 Provider 服务根隔离、动态开关与重载唤醒、宏任务让步位置、准入前重新检查状态、已开始任务收尾以及异常与取消的许可/定时器清理。本轮 3 项没有遗留未处理的复现场景；既有补位、FIFO/WRR、公平性、预算与租约语义由相应回归继续验证。 + +完整前端与 Agent 工作区回归、真实模型负载和 macOS 安装测试仍留在后续统一验证阶段。此处没有执行整个 monorepo 的 `npm test`，没有生成或安装 DMG，也没有 Windows 实机验证。 diff --git a/docs/qa/memory-queue-stage6-review-fixes-20261022.md b/docs/qa/memory-queue-stage6-review-fixes-20261022.md new file mode 100644 index 000000000..ce368f885 --- /dev/null +++ b/docs/qa/memory-queue-stage6-review-fixes-20261022.md @@ -0,0 +1,63 @@ +# Memory 队列阶段 6：复查补修 + +日期：2026-10-22。基线 `6646adce`，工作分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`。本轮仅补修已确认的 5 项及其持续执行兼容边界，不进入后续阶段，不生成 DMG、不推送、不提交 PR。 + +后续复查更新：本记录对应补修提交 `23efb8bd`。该提交的既有回归通过后，额外边界复现又发现写入开关、事件循环让步与 Mistral 跨协议服务根 3 项遗漏；最新修复与验证见 [阶段 6 边界补修记录](memory-queue-stage6-boundary-fixes-20261022.md)。 + +## 复查发现与修复 + +| 问题 | 修前实际复现 | 修复与回归 | +| --- | --- | --- | +| 自动入口仍按 4 条等待整批结束 | 真实 HTTP、MemoryService、SQLite、默认云并发 2；首轮 3 条摘要与 1 条向量任务后，只剩首条慢摘要运行，3 个全局空槽、1 个云模型空槽仍无法继续领取 | 仅内部自动入口启用 continuous 模式，持续补位直至无可执行工作、取消或需要交还入口；真实 HTTP 验证第 5 条摘要在首条完成前已执行，全局及 Provider 上限不变 | +| 到期任务不能自行唤醒当前运行 | 慢任务运行时，5 秒后到期的独立任务在 10 秒仍 queued；失败退避、Embedding 重试及其他执行者的租约到期同样缺乏唤醒 | 运行中查询最近未来截止时间并设置单一可清理定时器，与任务完成、许可释放、新工作通知共同唤醒；已经到期但资源阻塞的记录不能遮住下一未来截止时间或产生零延迟定时循环 | +| Skill assign/evolve 缺少共享业务范围 | evolve 等待模型时 assign 加入新成员，memberCount 从 1 变 2；evolve 用旧快照写回后又变成 1，成员表仍 2 条 | 两类任务均按用户、项目互斥；assign 从真实 episode、evolve 从真实 cluster 读取项目，无 session/target 也正确；另一项目可继续执行，新成员保留并触发下一轮演化 | +| Anthropic 等价地址分成两个 Provider 根 | 默认地址与显式基础地址最终都是 `/v1/messages`；真实客户端、受控 fetch 测得相同 URL 的峰值 4，大于限额 2 | 按真实客户端的地址构造规则归一化，再计算许可根；相同实际 URL 共享上限，原复现峰值降为 2 | +| 已释放前台许可误拒新业务 | 首次模型调用结束后计数均归零，遗留定时回调开启新的模型业务仍报未准入，第二次 delegate 未执行 | 只在路由真正开启新前台业务时清理已释放的前台许可上下文；新业务重新准入；已释放 worker 的延迟调用仍拒绝，不放松旧任务 fencing | + +这些属于原定连续补位、并发限制的实现补全，以及扩大并发后必要的业务保护，不增加新的 Skill 功能或恢复已取消的生产背压方案。 + +## 自动与手动执行边界 + +`runWorkerOnce(limit, request)` 手动请求仍严格服从总预算和 target 过滤。自动 HTTP 入口通过内部第三参数启用 continuous，HTTP 手动请求体不暴露该控制字段。自动入口原有的 `4` 不再是整次执行的总量限制，并发仍统一由控制器限制为全局 4、本地 1、云 2、Embedding 请求 1。 + +自动执行发现有等待的手动请求时停止继续领取，待已准入任务收尾后交还同一个 scheduler；手动 retry-only 入口也通过共同的通知路径唤醒自动执行。HTTP dispose 会取消自动继续准入，并等待已经开始的任务完成；不为凑满旧预算继续领取。手动请求仍是有限、独立且串行的执行请求，不允许 reentry。 + +长时间 continuous 不再每 4 条重新进入 `beforeRun`，因此预算对账截止时需要按原规则再次执行已有准备步骤。每个截止时间只尝试一次，未变化的 in-flight 时间不能导致反复等待；异步准备返回后重新检查取消、停止和待处理手动请求。预算对账完成时也会通知当前 pump 重新判断可运行任务。 + +legacy 仍保留其选择顺序；定时唤醒后的失败重试需完成原有的重新排队步骤，不能只唤醒而让 failed 项继续滞留。WRR 保持直接领取到期 failed 项的路径。定时器在被其他通知提前唤醒、执行结束或取消时清理。 + +## 修前证据 + +- 自动 HTTP：`/private/tmp/memmy-stage6-audit-http-cloud.log`。 +- 当前运行缺少截止时间唤醒：`/private/tmp/memmy-stage6-audit-pump-boundaries.log`。 +- 正式到期回归：`/private/tmp/memmy-stage6-deadline-formal-before.log`，4 项修前失败。 +- 正式 HTTP 回归：`/private/tmp/memmy-stage6-http-formal-before.log`，新增 3 项中补位与关闭 2 项失败，手动预算对照通过。 +- Skill：`/private/tmp/memmy-stage6-skill-scope-before.log`，2 项修前失败。 +- Provider 和旧前台上下文：`/private/tmp/memmy-stage6-anthropic-root.repro.log`、`/private/tmp/memmy-stage6-stale-foreground.repro.log`；正式回归也先确认失败。 +- 共同手动 retry 唤醒：`/private/tmp/memmy-stage6-manual-retry-yield-before.log`。 +- 持续执行预算对账：`/private/tmp/memmy-stage6-reconcile-formal-before.log`。 +- legacy 自然失败退避：`/private/tmp/memmy-stage6-legacy-natural-backoff-before.log`。 + +正式测试使用隔离临时 SQLite、真实服务/路由/业务 pipeline 和受控模型；未接入用户数据或外部付费模型。修前对照使用基线源码的临时加载或隔离副本，未回滚工作树、未影响并行修改。 + +## 最终验证 + +代码冻结后，在 macOS arm64、Node.js 22.23.3、npm 10.9.9 下验证： + +| 检查 | 结果 | 日志 | +| --- | --- | --- | +| Memory 全量 | 1650 通过、1 跳过；165 个文件通过、1 跳过 | `/private/tmp/memmy-stage6-review-fixes-memory-full.log` | +| 后端全量 | 1085 通过，132 个文件通过 | `/private/tmp/memmy-stage6-review-fixes-backend-full.log` | +| Memory 服务与 viewer 类型检查 | 通过 | `/private/tmp/memmy-stage6-review-fixes-typecheck.log` | +| 后端类型检查 | 通过 | `/private/tmp/memmy-stage6-review-fixes-backend-typecheck.log` | +| release workflow 保护 | 30 通过、5 跳过 | `/private/tmp/memmy-stage6-review-fixes-release.log` | +| packaging guards | 122 通过、1 跳过 | `/private/tmp/memmy-stage6-review-fixes-packaging.log` | +| 差异检查 | 通过 | `git diff --check` | + +本轮新增 23 项正式回归：模型许可及上下文 4 项、Skill 2 项、HTTP 3 项、deadline 查询 6 项、共同手动 retry 唤醒 1 项、运行中到期与取消等 7 项。原有 HTTP 关闭测试按“停止继续准入、保留有限手动预算”的新明确行为调整，仍检查真实数据库状态、attempts、token 和已领取项全部收尾，不仅修改预期数量。 + +独立交叉审查覆盖准入与存储查询、业务范围、真实模型路由、持续执行交还入口、关闭与重入边界。复查发现的共同 retry 唤醒、持续执行预算对账及 legacy 退避兼容问题均已补修和测试；本轮 5 项没有遗留未处理的复现。定时唤醒测试不依赖手动 `notify`，取消测试检查定时器和 AbortSignal 监听器清理。 + +完整前端、Agent 工作区回归和真实桌面安装仍按后续统一验证计划执行。此处的后端全量、Memory 全量及打包保护不能表述为整个 monorepo 的 `npm test` 已通过。 + +本轮检查不替代真实 Provider 性能收敛、macOS DMG 安装及后续整体回归。 diff --git a/docs/qa/memory-queue-stage8-20261022.md b/docs/qa/memory-queue-stage8-20261022.md new file mode 100644 index 000000000..9f822edaf --- /dev/null +++ b/docs/qa/memory-queue-stage8-20261022.md @@ -0,0 +1,211 @@ +# Memory 队列改造：阶段 8 整体回归与 macOS 安装验证 + +后续复查及最终退出补修包的最新结果见 [队列方案复查与打包验证](memory-queue-plan-recheck-v126-20261023.md)。以下保留此前候选包的历史结果。 + +日期：2026-10-22。工作分支 `test-memory-queue-v1.2.6`，整体 PR 目标 `upstream/v1.2.6`。按用户确认，工作文档从阶段 6 直接进入阶段 8；不恢复已取消的生产背压部分。本轮只验证 macOS arm64，不提交 PR。 + +## 最新结论（2026-10-23，299a4a14） + +交付准备已再次合入最新上游并重新打包；补充的慢反思场景出现约 126 秒实时等待。最新结果和保留项见 [v1.2.6 交付准备复核](memory-queue-delivery-v126-20261023.md)，以下保留 299a4a14 的原轮次证据，不把约 35 秒写成固定保证。 + +队列相关统一回归、新包内负载/公平性、真实本地模型、同环境升级后 12,000 保留 + 1,000 新完整轮次扫描，以及退出重启的数据核对已通过。GUI 第 51 页加载成功，最终共 651 页；新普通记忆在 11,590 条历史摘要 queued 时约 35 秒达到 ready。 + +仍有独立的 Desktop 退出清理超时：旧包和新包均触发 5 秒保底退出，不能写成“整体无异常通过”。以下保留各轮失败与补修过程；末节及同目录 `memory-queue-stage8-20261023-results.json` 是最新包的汇总。未恢复已取消内容，未推送、未提交 PR，也未进入下一阶段。 + +## 基线与范围 + +阶段 8 开始前已通过 merge 同步 `upstream/v1.2.6` 的 `f19bd810`，合并提交为 `53d312b0`。保留阶段 1–6 的持久化单 worker、WRR 4/3/2/2/1、可运行 FIFO、租约 fencing、失败退避、四层并发和连续补位设计。 + +测试环境为 Node 22.23.3、npm 10.9.9、macOS arm64；安装包使用 Electron 自带 Node 22.20.0。项目当前桌面版本仍为 1.2.3，Memory 版本为 2.1.6；目标 Git 分支 v1.2.6 不等同于包内版本，本轮没有单独改版本号。 + +本轮还纳入用户授权的旧模型配置兼容补修：迁移把残留平台标记当成真实平台身份，可能误删仍在使用的自定义 Memory 连接。保留连接后，同一 Provider 的默认密钥、headers、body 又可能串入其请求，因此补齐独立 endpoint 的可选 `inheritProviderDefaults:false`。缺省/true 保持原继承规则。未选择向量模型仍使用默认本地模型，Memory loader 原规则及对应既有测试未改。 + +## 迁移兼容：先复现、再补修 + +- 错误保留方式先由实际 Memory 客户端请求暴露：只保留目录但不保留必要 raw fixed/custom 字段，会使摘要不可配置或 embedding 回退 local；只复制连接也会继承无关 Provider 的凭据及 `body.model`。正式修前记录见 `/private/tmp/memmy-stage8-endpoint-isolation-before.log`、`/private/tmp/memmy-stage8-endpoint-inheritance-before.log`。 +- 独立端点使用正常 canonical Provider 和有效 assignment，不创建虚假 preset、私有 Provider 或假配置密钥；端点自有参数与 Provider 默认值分离。SDK 内原有的初始化占位值仅在内部使用，明确无鉴权时不进入 HTTP Authorization。 +- 配置读取、迁移、实际模型客户端、Backend 投影、设置页保存、冲突重试、连接测试、Agent 主/备用模型及快照签名均保留独立语义。测试连接依据可信已保存端点判定，无鉴权权限不由客户端自报标记决定。 +- 新增 `v1.2.6/0001-preserve-independent-memory-connections`，覆盖已执行旧迁移的安装；不重跑整个 normalize。只处理仍留有配置信息、实际独立且激活的目标连接。不改无关标签、Agent 配置和 Provider 默认值；已丢失且无备份的连接信息无法凭空恢复。 +- 历史 normalize/repair 和后续修复均尊重已有 null/缺省/builtin-local 向量选择。对原标记范围保留 raw custom 但不自动激活；真正尚无目录的旧自定义连接仍可首次导入。 +- 交叉审查补出了 Anthropic 无环境密钥硬拦截、OpenAI 自有 key 时环境组织/项目串入，以及历史 normalize 覆盖本地 embedding 选择的边界，均先新增失败回归再修复。 + +迁移全量 279 通过、1 跳过，新增迁移回归 106 项；共享契约显式测试 60 通过；Agent 全量 7,142 通过、80 跳过。最终仓库级检查和安装包结果在下文补记，不用中间包代替最终交付。 + +## 受控负载与真实模型证据 + +使用真实 MemoryDb、Repositories、WorkerRunner、路由和并发控制器，以 10,000 条导入积压加 interactive、reflection、maintenance、embedding retry 混合工作验证调度。此组 handler 是受控替身，结果不代表 10,000 条已全部完成摘要。 + +| 场景 | 完成窗口 | 启动计数（interactive/repair_retry/import/evolution/maintenance） | 真实请求峰值 | +| --- | --- | --- | --- | +| 串行，8 KiB | 24 | 8/6/4/4/2 | 1 | +| 本地并发，8 KiB | 36 | 8/23/2/2/1 | 1 | +| 云限制，1 KiB | 36 | 12/9/6/6/3 | 2 | +| 云限制，32 KiB | 36 | 12/9/6/6/3 | 2 | + +所有 Lane 都有进展，可运行 FIFO 成立；没有尚未启动就增加 attempts/token 的任务，窗口结束无遗留许可/租约。Embedding 微批按成员条数计量，因此本地微批窗口不要求每条任务计数恰好等于 WRR 权重。云场景下一条导入在前一条慢任务完成前启动，证明连续补位。 + +证据在 `/private/tmp/memmy-stage8-load/`。仍有性能观察项：阻塞候选多时存在大量重复 SQLite candidate 查询，受控云窗口首次执行约 0.46–0.50 秒、36 条窗口约 8.9–11 秒；这些结果没有已定义的产品延迟阈值，不能据此宣称任意负载下零等待或吞吐已达标。 + +本机 LM Studio 另执行过两条短合成导入的真实摘要和向量请求，2 summary + 2 embedding 成功、两条 ready、向量 768 维，真实请求峰值 1。该次源代码证据位于 `/private/tmp/memmy-stage8-real-local/`，运行基线为 `53d312b0`；不能当作最终 DMG 或真实云 Provider 的测试。模型不支持 json_object 时，既有 text 降级成功。 + +## 自动检查中的非产品失败记录 + +1. 依赖最初缺少 Monaco，补齐本机已有的同版本 0.57.0 后检查通过，未升级依赖或改锁文件。 +2. 沙箱环境不允许开启部分本地测试端口，导致 Memory HTTP 用例超时；在允许本地端口的环境重跑后 1,694 通过、1 跳过。后续新增 1 项保存/密钥轮换边界纳入最终全量。 +3. 并行重建 contracts 的 dist 曾使 Agent 单文件导入失败；最终检查采用先构建后测试，避免共享产物竞态。 +4. Agent Session DAG 原有一项用例曾在全量中出现 30 秒 processed=false，相关源码及用例未改，也不经过本轮 Provider 路径。整文件 16/16、原用例连续 8 次以及补修后的全量均通过；最初失败原因未确认，保留原失败和复核日志，不把原因直接归于 CPU 或环境。 +5. 用系统 Node 加载 Electron ABI 的 SQLite 原生模块曾报不匹配，属于验证器运行时选错；包内验证改用该安装包自带 Electron Node。工作区与日常仓库的依赖已隔离，不修改日常安装的原生模块。 + +## 安装包真实扫描补查：native 历史任务分类 + +第一次候选包 `ca5ca477` 的自动检查全部通过，但实际 full scan 暴露新入口遗漏：当前历史扫描也走 `completeSourceTurn → completeTurn → trace_summary/capture.v7`，没有旧 `agent-source` 标签。现场导入 4,535 条时，4,405 个待处理历史摘要与 129 个待处理向量均进入 interactive，后者排在历史摘要后。该包不能作为最终队列验收通过的证据，扫描已通过正式接口取消并保留资料于 `/private/tmp/memmy-queue-stage8-profile-native-classification-repro`。 + +修复只增加内部调度来源：合法 `agent_source_scan + captureLegacyHistory=true` 的 full/initial_subset 历史捕获，将 `internal_info.capture_origin` 保存为 `agent_source_scan`。保留 `trace_summary`、`capture.v7`、capture decision 和原标签，后续 embedding 从目标记忆继承 lane。普通 Hook 与无该历史许可的 incremental 扫描保留原分类。首次报告、人工重试和启动修复规则优先；embedding retry 保留 repair_retry。 + +策略版本从 1 升到 2，存储的三条刷新路径都读取该来源。刷新不改变入队顺序、首次排队时间、尝试数或已有租约。无来源信息的旧 native 记录不猜测其生产渠道，也不把所有外部 Agent 的实时对话改成 import。 + +正式真实服务回归修前 12 项中 7 项失败；扩展后新增 18 项全部通过，连原 source-turn-capture 共 68 项。覆盖时间早于/晚于 activation、detached history、实际摘要→embedding、48 条历史积压中新任务的实际完成、同 Lane FIFO、首次报告、人工重试、启动修复、双方向 Hook/scan 去重及重启；实际 executor 的 full/initial_subset/incremental 也覆盖。注册表和存储直接新增 9 项，连选择器与连续准入共 77 项通过。 + +安装压力资料保持 12,000 个完整轮次和 1/8/32 KiB 三档原始消息。最终资料拆为 300 段对话，每段 40 轮，以覆盖多会话扫描；第一次 3 段各 4,000 轮的资料及失败现场另行保留。不能把换资料理解为旧超长会话性能问题已经修复。 + +补修后的全量检查有一次时间敏感失败：扫描响应测试 1 项未在 waitFor 默认时间内到达检查点，10,000 条基线测试 2 项超过 5 秒；原失败留在 `/private/tmp/memmy-stage8-native-root-final.log`。两个文件独立重跑 23/23 通过（`/private/tmp/memmy-stage8-native-timeout-recheck.log`），没有放宽测试超时或改行为断言。最终全量结果以下方记录为准。 + +## 最终检查与安装包验收 + +模型兼容补修版本 `ca5ca477` 的 `npm run lint`、`npm run typecheck`、Agent typecheck、`npm test`、显式 contracts 测试均通过。`npm test` 含 release 30 通过/5 跳过、packaging guards 122/1、Memory 1,695/1、Migrations 279/1、Backend 1,109、Frontend 2,452、Electron 1,259/126；跳过项沿用各套件原有条件，不能记为已覆盖平台能力。Agent 独立全量 7,142/80。 + +最终日志：`/private/tmp/memmy-stage8-{lint,typecheck,root,agent,contracts}-accepted.log`；Agent 单独类型日志为 `/private/tmp/memmy-stage8-agent-typecheck-final-compat.log`。接口和类型构建完成后再执行测试，未与全量 Agent 测试并行刷新 contracts。 + +历史扫描分类补修的 lint/typecheck 已通过，全量复核与重新打包验收进行中；最终 DMG 结论将在安装、扫描及重启验证后补充。 + + +## 2026-10-23 续验记录 + +夜间限定测试并行度的那轮在 2026-10-22 22:40:46 遭遇系统低电量休眠。`pmset` 明确记录 Low Power Sleep、电量 1%、持续 35,795 秒，与用例记录约 35,797 秒吻合。因此这轮不能作为持续运行耗时或死锁证据。`queue-admission-claim.test.ts` 续验独立重跑 11/11，通过时间 0.896 秒。更早的短时等待失败仍保留，未据此将全部超时统一归为休眠。 + +本次全量采用 `VITEST_MAX_WORKERS=2` 限定测试文件并行度，`caffeinate -i` 仅在命令运行期间防止空闲休眠。没有修改产品并发、测试断言或超时阈值。低电量或合盖仍可触发系统休眠，不能认为该临时机制能覆盖所有系统中断。 + + +续验又补出多会话标题来源的遗漏：把真实服务公平性场景从同一历史会话扩为 48 段独立历史会话时，新任务仍在 24 轮窗口内无法完成，因历史 `episode_title` 留在 interactive。修前日志 `/private/tmp/memmy-stage8-20261023-multiconversation-before.log`。标题没有 target memory，因此统一入队处根据真实 episode 的完整 L1 集合生成内部来源：只有全部明确属于历史扫描、且不含首次报告时才标记历史；混合、当前、空集合或缺失记忆保持原分类。临时/最终标题均使用同一入队逻辑，传入的来源提示不能覆盖实际判定。没有改标题生成或 capture.v7 业务逻辑。 + +多会话公平性与历史/当前/混合会话标题回归补齐后，新 source-turn 文件共 22 项；连 registry 共 40 项通过，Memory 类型检查通过。前一轮完整仓库在测试文件并行度 2 时全通过（Memory 1,722/1、其余组件计数同前），标题补修后重新执行 Memory 全量，最终结果另补。 + + +最终源码补修的 Memory 全量：1,726 通过、1 跳过，169 文件通过、1 文件跳过，55.65 秒，日志 `/private/tmp/memmy-stage8-20261023-memory-accepted.log`。标题补修前的本日仓库全量全部通过，日志 `/private/tmp/memmy-stage8-20261023-root-final.log`；没有以先前失败轮次当作通过。此次后续源码改动限于 Memory，追加了 Memory 全量与类型检查，未重复未受影响的 Agent 全量。 + + +## 前一轮打包及初步安装证据（73725434） + +官方 `package:mac:arm64:cn:unsigned` 于 2026-10-23 成功生成包，耗时 3 分 14 秒。桌面版本 1.2.3、Memory 2.1.6，runtime buildId `9db0f80665606a86d658e023f87dcfca4fd99e04b9fab19c255dbac0f7d3e2b8`，schema 12，darwin-arm64。DMG SHA-256 为 `39d34d04663be054fa82f5fc396f320ae374b7a87eed1682d41d2feb6c939ffb`，磁盘映像校验 VALID。脚本仍有安装窗口背景图替换失败的外观警告，未修改打包脚本。 + +应用从只读 DMG 复制到 `/private/tmp/memmy-queue-stage8-accepted-installed/Memmy.app`。四个关键队列文件与源码编译产物的 SHA-256 均相同（`accepted-code-manifest.json`）。启动使用独立 HOME 和测试 profile，不覆盖日常应用及数据。旧 schema 9 升级至 12 后，完整旧记忆和向量指纹仍为 `df625239a98a67faf03de2cef9e84bdf3946913d11ddabddd117a35ef54e1f37`。三种独立 Memory 连接和不相干 Provider 默认值均保留，实际受控请求未携带后者凭据或错误模型。 + +最终包混合云队列探针（独立 SQLite、受控无网络 handler）在 10,000 导入积压中完成 36 个逻辑成员，五 Lane 启动数 12/9/6/6/3；峰值逻辑任务 4、物理请求 2、embedding 请求 1，无未开始就增加 attempts 的 queued 项，FIFO/连续补位断言通过，窗口耗时 8.71 秒。日志 `/private/tmp/memmy-stage8-20261023-packaged-cloud.log` 与 `/private/tmp/memmy-stage8-load/accepted-packaged-cloud-32768.json`。 + +最终包另以实际 LM Studio 完成两条合成导入：2 summary + 2 embedding 均成功、2 条 ready、768 维向量、请求峰值 1、结束无许可泄漏，总耗时约 70.28 秒。既有 json_object→text 降级成功，未新增模型协议能力。证据 `/private/tmp/memmy-stage8-real-local/final-packaged-result.json`；这部分确实使用已安装包内的 Memory 代码，而不是源码模块。 + +12,000 条真实 full scan 验收仍在进行，最终条数、现场新任务进展与重启验证不能从以上小样本测试推断。 + + +实际 full scan 中途（约 5,300 条）新建独立 native 任务 `trace_a4b69831d9e5850172f1`，2026-10-23 12:08:39 创建，12:09:25 达到 ready。其 trace_summary/embedding 均 interactive，摘要 1 次成功；连续 3 次合成 429 耗尽 embedding 客户端内部 2 次重试后，主 embedding 任务第 2 次执行成功，processing 无错误。完成核对时仍有 5,076 条历史摘要 queued。此处是主任务重试，不是独立 embedding_retry_queue；独立重试表的公平性证据来自最终包混合探针。单次 429 的前一轮注入只触发客户端 HTTP 重试，未错误归为后台租约重试。 + +扫描期间 GUI 列表成功加载、翻到第 51 / 278 页并显示对应历史记录；这证明翻页已超过 50 页，但最终条数仍以 SQLite 为准。健康接口曾有约 3.56 秒响应，不能据服务仍响应推断界面延迟已达到未定义的产品目标。受控 Provider 首段每请求 1 秒，后续切为 5 秒慢服务场景，embedding 响应延迟 100ms；延迟变更与请求统计保存在测试证据目录中。 + +## 大负载响应补查（进行中，不能据此判阶段通过） + +扫描继续至约 6,400 条后，健康接口记录过 18,531ms 响应；约 7,900 条时仍可推进并显示第 51 / 395 页，近期健康响应多为 0.5–1 秒。服务没有崩溃不等于延迟验收通过，保留现场继续排查。 + +受阻候选遍历的独立复现：持有同一模型 Provider 唯一许可，在 10,000 条摘要后放同 Lane 的纯数据库任务；原同步领取正确找到了第 10,001 条，但检查 10,001 次、连续占用事件循环约 270ms，零延迟定时回调同样等待约 270ms。真实扫描数据库的一致副本上，7,937 次候选检查中有 7,936 次被资源拒绝，领取一条 episode_idle_close；同步占用约 315ms。大量立即完成的数据库任务可触发重复补位，原有按成功领取 32 条才让步不能约束“检查很多、领取很少”的轮次。证据 `/private/tmp/memmy-stage8-admission-scan-profile.*`、`/private/tmp/memmy-stage8-real-queue-profile.*`。这证明一个阻塞源，不把现场 18.5 秒全部归因于该单次测量。 + +标题来源判定还存在本轮引入的额外读取：每次检查 episode 的全部 L1 会反序列化正文并加载向量。独立长会话 1,200 轮捕获探针中，该 getMany 累计约 6.26 秒；正在用只返回来源及首次报告判定的投影替代，保持全部历史/混合/当前/缺失/软删及旧字段标签语义。 + +标题轻投影修复保存为 `a74b5ec2`。新增 19 项回归，连相关文件共 111 项通过，独立复核 source-turn + 新文件共 41 项通过，Memory 类型检查通过。旧 tags_json、info_json.tags、properties_json.tags 合并识别均保留；重复 ID、软删、缺失、空集合和伪造来源也覆盖。同一 1,200 轮长会话探针由 18.646→16.284 秒（约 12.7%），末 400 轮均值 23.96→20.30ms。完整集合分类仍随会话长度增加工作量，未改成缓存推测，也未宣称常量时间。日志 `/private/tmp/memmy-stage8-title-projection-final.log`、`/private/tmp/memmy-stage8-title-root-review.log`、`/private/tmp/memmy-stage8-capture-profile/run-1200-long-projection-final.log`。 + +另发现既有 FTS 写入放大：新建记忆的 reindexFts 也执行 `DELETE FROM memories_fts WHERE id=?`，id 为 UNINDEXED，执行计划是虚表扫描。独立 3,200 轮探针中删除累计约 9.61 秒；隔离诊断仅跳过新 ID 的无效删除后总耗时 30.04→18.88 秒。该项属于原有写入性能问题，未修改生产 FTS 或扩大本轮队列范围，也不能独立解释 18 秒健康延迟。证据 `/private/tmp/memmy-stage8-capture-profile/`。 + +分片补修采用每片最多 64 次候选检查或策略投影，事务结束后先启动已领取任务，再通过原生 immediate 让出事件循环。不会只看前 64/200/500 条，也不提前领取一大批任务等许可。旧策略全部投影完成后才允许 FIFO 选择；资源释放、模型配置、旧任务到期或已有任务重分类使续扫失效。新导入任务的首次分类不导致深扫反复重置。同一 SQLite 连接的多个 Repositories 共享内存修订号；不同连接/进程直接改元数据不共享该编号，本轮未扩展多进程协调。 + +正式红→绿及交叉审查覆盖：定时取消/停止、后排纯数据库任务、先启动片内已领任务、手动请求接管、WRR/legacy 续扫、CAS 失败、片间删除/暂不可运行、available_at/runAfter/过期租约旧序号、模型快照变更、legacy 重试扫描中新主任务、恰好 64/128 个受阻任务后的空片，以及真实 dedupe enqueue 使已缓存/已检查任务换 Lane。最后两类由独立审查补出，已修复重复候选报错和遗漏旧序号的问题。核心定向 75 项、Worker/相关存储 353 项、独立审查 82 项通过；Store 直接 40 项包含 20 项新增，Memory 类型检查通过。证据 `/private/tmp/memmy-stage8-slice-worker-regression-final.log`、`/private/tmp/memmy-stage8-selection-revision-after.log`。 + +隔离实测显示响应与总工作时间需要区分：7,936 条策略投影从单事务约 1.743 秒拆为 125 片，最长同步片约 60.91ms，定时回调约 61.16ms，总耗时增至 2.814 秒。真实扫描副本 7,938 候选分片后最长约 19.04ms、定时回调约 32.97ms，仍领到深部 episode_idle_close；总扫描耗时约 1.412 秒，高于原同步约 315ms。反复复查策略会增加总开销，不能称其本身提高吞吐。 + +冻结源码的 10,000 导入混合探针则因已领取任务及时开始执行，36 条窗口耗时约 582.74ms，prepare 次数 3,917，五 Lane 12/9/6/6/3；逻辑峰值 4、Provider 请求峰值 2、embedding 请求峰值 1，FIFO、连续补位及无未启动 attempts/残留许可均通过。该结果尚须最终包复测，不与旧包直接当作同运行时的最终性能结论。证据 `/private/tmp/memmy-stage8-load/sliced-source-cloud-32768.json`。 + +12,000 条扫描在 2026-10-23 12:45:33 合盖休眠时，监控请求触发 30 秒超时；12:57:40 唤醒后健康 HTTP 200、约 1.48 秒。`pmset` 证据 `/private/tmp/memmy-stage8-clamshell-20261023.log`。监控保留全部样本和该中断记录后恢复，不把该次超时当作产品卡死；更早 18.531 秒样本仍单独保留。 + +历史记录达到 10,014 条后，另建 native 任务 `trace_e783a65ec75de0309b5c`，12:59:11→12:59:34 达到 ready，摘要与 embedding 各一次成功、均 interactive；完成核对时仍有 9,205 条历史摘要 queued。监控 JSON 的旧字段名 initialBacklog 在此表示历史记录总数，不能误当成全部尚未处理的摘要任务数。reflection 仍处于 evolution queued，不将 ready 错写为整条演化链全部完成。 + +## 分片补修后的统一检查（b827f7c0） + +源码提交 `b827f7c0a5405874d88daef65c88d7cd27d12d79`。`npm run lint`、`npm run typecheck`、`VITEST_MAX_WORKERS=2 caffeinate -i npm test` 均通过。全量包含 release 30/5 跳过、packaging guards 122/1、contracts 8、Memory 1,790/1（两次）、Migrations 279/1、AgentSource 62、Knowledge 48、Backend 1,109、local contracts 32、Frontend 2,452、Electron 1,259/126、TUI 5。没有放宽测试超时或新增跳过；两个原有计时器清理用例改为一个逻辑槽位的测试布置，以确保走到等待计时器路径，仍保留原清理及取消断言。 + +日志 `/private/tmp/memmy-stage8-responsive-{lint,typecheck,root-test}.log`。Agent 全量 7,142/80 的先前证据仍有效，之后代码变化仅在 Memory，未据此重新运行未受影响的 Agent 套件。最终包正在重建；旧包的扫描数据和耗时不会冒充本提交的安装验证。 + +## 分片补修候选包（b827f7c0，尚未通过完整公平性验收） + +官方打包于 2026-10-23 13:12:52 成功，3 分 40 秒；DMG SHA-256 `d6baaa9af04f4eaedf70522b65f9b0ec7c8658ebd0c0da338d93d82166d1f650`，校验 VALID。由只读映像复制安装至 `/private/tmp/memmy-queue-stage8-responsive-installed/Memmy.app`，Memory buildId `2bd01b8203842d5d645f5ef2ebd5908d4cea0a188fc63bfa7e5ead833fe6a321`，schema 12。七个关键模块 SHA-256 与源码编译产物逐个一致，见 `responsive-code-manifest.json`。桌面仍为 1.2.3，Memory 2.1.6，未擅自改版本。 + +同一 Electron Node 22.20.0、相同 10,000 条/32 KiB 混合探针及 36 条窗口：旧包 8,706.50ms、225,283 次 prepare、首次定时回调 436.87ms；新包 656.58ms、4,109 次 prepare、首次定时回调 111.01ms。两次均五 Lane 12/9/6/6/3、逻辑峰值 4、Provider 请求峰值 2、embedding 请求峰值 1，FIFO/连续补位/无许可泄漏/未启动任务 attempts 不增加均通过。它是同机受控处理器的观察值，不是模型业务吞吐或任意负载的延迟保证。证据 `/private/tmp/memmy-stage8-load/responsive-packaged-cloud-32768.json`。 + +本包实际 LM Studio 再验：2 import_summary + 2 embedding 均成功，两条 ready、各有 768 维真实向量；请求峰值 1，结束全部许可释放，总约 73.47 秒。保留既有 json_object→text 降级及响应处理路径，不新增模型协议能力。见 `/private/tmp/memmy-stage8-real-local/responsive-packaged-result.json`。 + +打包重建原生依赖后，工作区系统 Node 的 SQLite 绑定已原子恢复并实际打开内存数据库确认（Node 22.23.3、ABI 127）。安装包内绑定保留 Electron ABI，未修改日常仓库的依赖。 + +## 持续数据库任务下的共享模型公平性：新增阻断项 + +12,000 条旧包扫描于 13:24:21 完成，SQLite 精确计数 12,000、0 Skill；原记忆及向量完整指纹仍相同，独立模型连接核对通过。扫描完成后暂停后台写入，保留现场，后续新包测试继续使用该数据。它只证明本次扫描和普通任务可进展，不代表全部历史摘要/反思已经处理完。 + +补充检查在 `b827f7c0` 又复现核心公平性问题：真实 WorkerRunner continuous、import 与 evolution 各 200 条、同一 Provider cap 1、模型 10ms；每完成一条 episode_idle_close 就补一条，模拟扫描持续的快速数据库任务。独立两次测试连续完成 100 次模型请求，全部 import，evolution 为 0、200 条仍 queued/attempts 0；maintenance 分别完成 61/58 次。关闭 maintenance 补流的对照是 50/50。5ms 变体虽非整程恒为 0，仍出现连续窗口内反思完全不执行，不能把偶然进展当作公平保证。 + +原因是全局环形游标在 Provider 繁忙时跳过反思，快速 maintenance 成功又将游标推进回环首，下一次模型空闲便再次优先 import；尚未兑现的反思轮次没有保留。修复分片只保证事件循环响应，尚不足以解决这个调度缺陷。因此上述候选包不能作为阶段 8 最终验收通过包。 + +正式新回归 `Memory/tests/service/worker/worker-provider-lane-fairness.test.ts` 已先红：无补流对照通过，5/10ms、受控本地/cloud、长期占用另一 Provider 加持续新任务的 5 项均因反思窗口为 0 失败；其余完成数、并发、停机和许可/租约清理断言先通过。证据 `/private/tmp/memmy-shared-provider-fairness-expanded-before.log`、`/private/tmp/review-stage8-provider-fairness-100-0.json`。正在补修保留有界未兑现 WRR 机会，保持 4/3/2/2/1、FIFO、原并发和已取消背压部分的范围约束；暂不将方案或纯模拟结果当生产验收结论。 + +正式修补采用有序、每 Lane 唯一的剩余轮次：只有成功租约 CAS 才扣一票，耗尽后该 Lane 重填自身权重并移到队尾；暂时受阻的 Lane 保留位置与剩余票额，其他 Lane 可跳过它继续执行。空 Lane 的跳过只在另一个任务真正领取成功时确认，避免片间新 interactive 被过早移到队尾。每 Lane 最多自身权重，不按被拒候选数量堆积信用。原 KV key/weightsKey/cursor 保留，仅附可选 turns version 1；旧值、非法值及权重变化均有回退。无需新增数据库迁移或对外接口。 + +纯算法、真实存储及连续 Worker 回归合计 28 文件 375 项通过，独立组合审查 103 项通过；原真实探针模型从 100/0 变为 50/50，maintenance 仍完成 75 次。最终新文件共 7 项，覆盖额外业务 scope 被占用，不能因受阻 Lane 再次堵住共享模型。修补中曾由审查捕获新 interactive 入队顺序回归，修复后保留原断言通过，没有删测试。日志 `/private/tmp/memmy-stage8-lane-turns-worker-final.log`、`/private/tmp/memmy-shared-provider-fairness-final.log`。统一全量和新包尚待完成。 + +## 退出链观察（不归因于队列） + +旧包 73725434 在全量扫描结束并暂停写入约 8 分钟后，通过 Cmd-Q 退出出现 `quit cleanup timed out; forcing app exit`。Memory 数据库无遗留打开句柄；现场克隆备份 `/private/tmp/memmy-queue-stage8-profile-after-12000-20261023` 的 quick_check 为 ok，原记忆与向量指纹仍相同。 + +只读核查：最后 Worker drain 为 13:24:22.840，quit cleanup-start 为 13:32:17.324，无退出时队列未完成请求证据。Desktop 总清理保底 5 秒；测试 profile 的进程生命周期会停止 Memory 子进程，此外还依次等待 gateway、原生 Computer Use agent、Backend、renderer server 和退出统计。原生 helper 的 4 秒定时器只发终止信号，并不保证 Promise 到时结束；日志没有每步完成埋点，无法确定卡在哪个 await。main/runtime-services 自 737 至本轮未改，不能据此认定是 Worker 停止引入。本轮未擅自扩大非队列 Desktop 退出流程修复；最终退出重启仍须明确记录该现象，不能宣称完全无警告。 + + +## 保留 WRR 轮次补修的统一检查（299a4a14） + +源码提交 `299a4a14144b26995db4709c808167d58b861f1b`，只包含新 admission selector、接入逻辑和三组回归文件。`npm run lint`、`npm run typecheck`、限定文件并行度 2 的根目录 `npm test` 均通过，退出码 0。Memory 1,812 通过/1 原有跳过（完整仓库流程中两次）；其余 release 30/5、packaging guards 122/1、contracts 8、Migrations 279/1、AgentSource 62、Knowledge 48、Backend 1,109、local contracts 32、Frontend 2,452、Electron 1,259/126、TUI 5。跳过均未新增。日志 `/private/tmp/memmy-stage8-turns-{lint,typecheck,root-test}.log`。Agent 全量沿用前述未受影响部分的证据，不将本次根目录流程描述为重复执行了 Agent 全套。 + +本提交的源码 10,000 条、32 KiB、cloud cap2 混合探针完成 36 个逻辑成员，Lane 12/9/6/6/3，峰值逻辑任务 4、Provider 请求 2、embedding 请求 1，FIFO 和无未开始 attempts/许可泄漏通过。耗时 775.14ms，首次定时回调 57.61ms，prepare 3,534 次；受控 handler 无外部 HTTP，仍不代表真实模型业务吞吐。证据 `/private/tmp/memmy-stage8-load/turns-source-cloud-32768.json`。最终包验收如下。 + +## 最新包实际验收（299a4a14） + +官方打包 2026-10-23 13:50:30 完成，3 分 25 秒、退出码 0。DMG `App/shell/desktop/release/Memmy-1.2.3-darwin-arm64-cn-unsigned.dmg`，SHA-256 `2260abf6ad2640cf5b52a4d243d2f4ce1256f2e9af07a87da4e61ced13ad9646`，映像校验 VALID。只读映像复制到 `/private/tmp/memmy-queue-stage8-turns-installed/Memmy.app`；Memory buildId `f06ace64abaa318259c5f1263f575c249c756aca894910ed83fe116dbaaaa3ce`、schema 12。含新 admission selector 的八个关键模块与源码编译产物 SHA-256 一致。桌面 1.2.3/Memory 2.1.6 未改，背景图替换外观警告仍在。打包后隔离源码区恢复系统 Node ABI127，安装包保留 Electron ABI139。 + +包内 10,000 条/32 KiB 混合队列完成 36 个逻辑成员，五 Lane 12/9/6/6/3，逻辑峰值 4、Provider 请求峰值 2、embedding 请求峰值 1；FIFO、连续补位、未开始 attempts 和许可/租约清理通过。窗口 572.97ms、首次定时回调 24.99ms、prepare 4,173 次。受控 handler 无外部网络，不代表生产模型吞吐。见 `/private/tmp/memmy-stage8-load/turns-packaged-cloud-32768.json`。 + +包内复测原 100/0 共享模型问题：import/evolution 各 200、Provider cap1、10ms 模型、maintenance 持续补流，100 次模型完成为 50/50，maintenance 同时完成 79 次，失败 0；剩余 queued attempts 最大 0，每 10 次模型完成窗口两 Lane 均推进。见 `/private/tmp/memmy-stage8-turns-packaged-fairness.log`。 + +实际 LM Studio Gemma/Nomic：2 条合成导入、2 import_summary + 2 embedding 成功，2 ready、各 768 维真实向量、请求峰值 1、无许可泄漏，总约 38.06 秒；既有 json_object→text 降级成功。见 `/private/tmp/memmy-stage8-real-local/turns-packaged-result.json`。 + +### 同环境升级、完整扫描和新任务 + +本包沿用旧包已完成的 12,000 条现场及独立 HOME，没有清空重建。启动后仍精确为 12,000，旧记忆完整 record + vector 指纹不变，三种独立模型连接及无关 Provider 默认值保留。随后实际 full scan 重读已有来源，并追加 25 段×40 完整轮次、1/8/32 KiB 的 1,000 条合成对话。不是声称最新二进制从空库重新导入了全部 13,000 条。 + +13:53:06→13:54:56 扫描完成：解析消息 26,000、导入 1,000 memory/0 Skill；SQLite 精确 13,000 个 Claude L1，原 12,000 未重复。达到 12,220 条时创建 native 记忆 `trace_32b734a530bc0fe32a62`，13:53:51.923→13:54:26.549 ready（34.63 秒），核对时仍有 11,590 条历史摘要 queued。其 trace_summary/embedding 均 interactive、attempts1、succeeded；reflection 仍为 evolution/queued/attempts0，不将 ready 描述为全部演化已完成。 + +扫描使用本机受控 HTTP Provider，摘要/演化延迟 5 秒、embedding 100ms。48 个健康样本中位 45.99ms、P95 308.49ms、最大 673.75ms，只是该负载观察，不能与旧包不同导入规模直接当严格基准比较或作延迟保证。实际本地模型证据另列，未测试真实云 Provider。见 `/private/tmp/memmy-stage8-package/turns-scan-acceptance.json`。 + +GUI 加载列表和第 51 页的对应历史记录，最终共 651 页；完整搜索新标识返回该记忆，详情显示正确的用户/助手正文。截图 `turns-page51.png`、`turns-live-search.png`、`turns-live-detail.png` 在同证据目录。 + +### 退出重启与保留项 + +本包 Cmd-Q 于 13:56:13.519 开始清理，仍报 `quit cleanup timed out; forcing app exit`。应用返回码 0、Memory/Backend 端口关闭，随后同包重启成功。此警告旧包也存在，尚无具体 await 的定位证据；不归因于队列，也不以返回码 0 宣称完全正常退出。未扩大 Desktop 生命周期修改范围。 + +退出后及重启后核对：13,000 总数保留；四条关键记忆的正文/内容 hash、完整向量记录 hash 和 ready 状态一致，SQLite quick_check 为 ok。旧 schema9 种子的完整指纹始终为 `df625239a98a67faf03de2cef9e84bdf3946913d11ddabddd117a35ef54e1f37`。独立模型连接再次通过，重启后 GUI 仍能检索新记忆。见 `turns-final-restart-check.json`、`turns-final-seed-fingerprint.json`、`turns-after-restart-model-audit.json`。 + +测试结束留在独立应用环境，并重新施加进程内暂停以保留积压现场。直接 Memory PATCH 的测试覆盖在重启后会被持久化配置重新加载，不断言其跨进程保存；未修改产品配置持久化机制。 + +临时 QA 脚本三处错误已保留失败并修正，未据其修改产品：新增文件首次复制到待导入目录（只重读原 12,000，新增 0）；SQLite null-prototype 对象与 JSON baseline 的原型比较失败（字段/hash 一致）；误认为运行时暂停跨重启保存。分别修正实际扫描路径、规范化对象后比较全部字段、重启后重新施加测试暂停。没有清空数据、重建 baseline 或放宽生产回归断言。 + +队列目标已有上述通过证据，Desktop 退出清理超时、背景图外观警告及原有 FTS/超长单会话写入性能仍保留。没有覆盖 Windows/macOS x64/真实云服务，没有宣称所有历史摘要/反思清空。无 PR、无推送，后续操作等待用户确认。 diff --git a/docs/qa/memory-queue-stage8-20261023-results.json b/docs/qa/memory-queue-stage8-20261023-results.json new file mode 100644 index 000000000..cb1785b78 --- /dev/null +++ b/docs/qa/memory-queue-stage8-20261023-results.json @@ -0,0 +1,272 @@ +{ + "date": "2026-10-23", + "revision": "299a4a14144b26995db4709c808167d58b861f1b", + "targetBranch": "upstream/v1.2.6", + "platform": "darwin-arm64", + "status": "Queue checks passed; existing Desktop quit-cleanup timeout unresolved; no PR/push", + "package": { + "desktopVersion": "1.2.3", + "memoryRuntime": { + "version": "2.1.6", + "buildId": "f06ace64abaa318259c5f1263f575c249c756aca894910ed83fe116dbaaaa3ce", + "schemaVersion": 12, + "protocolVersion": 1, + "target": "darwin-arm64", + "entrypoint": "dist/src/server/index.js", + "viewer": "dist/viewer/index.html" + }, + "dmgSha256": "2260abf6ad2640cf5b52a4d243d2f4ce1256f2e9af07a87da4e61ced13ad9646", + "diskImageVerified": true, + "matchedCompiledModules": [ + { + "file": "src/service/session/session-turn-service.js", + "sourceBuild": "5d3d4de66d1f402e9fc2d2a2fcd26b4c5d0e59a4cb5dd7034167bc91d4ecd5e1", + "installed": "5d3d4de66d1f402e9fc2d2a2fcd26b4c5d0e59a4cb5dd7034167bc91d4ecd5e1" + }, + { + "file": "src/service/worker/job-handlers.js", + "sourceBuild": "62c9a0b440c0f1f9519458d049723f2fe2132146f7b7b5a25aec60fe1471235c", + "installed": "62c9a0b440c0f1f9519458d049723f2fe2132146f7b7b5a25aec60fe1471235c" + }, + { + "file": "src/service/worker/job-policy-registry.js", + "sourceBuild": "c622e38df1c596cf9094397186b53aad3a533dda7a3e8e08e07cd5c91ee6226c", + "installed": "c622e38df1c596cf9094397186b53aad3a533dda7a3e8e08e07cd5c91ee6226c" + }, + { + "file": "src/service/worker/fair-worker-queue.js", + "sourceBuild": "c8bf14f1a995191df49d50f60771af156f512856fa90d4d4f8dddd1a7c6a01b9", + "installed": "c8bf14f1a995191df49d50f60771af156f512856fa90d4d4f8dddd1a7c6a01b9" + }, + { + "file": "src/service/worker/admission-lane-queue-selector.js", + "sourceBuild": "6afa4f7bc31b1af2d7298c908d2c0ae57417623345ba07288e3b5ad427a9c4b2", + "installed": "6afa4f7bc31b1af2d7298c908d2c0ae57417623345ba07288e3b5ad427a9c4b2" + }, + { + "file": "src/service/worker/worker-runner.js", + "sourceBuild": "a827b43938a960ef70f0183d53390c97a4550f9f0544dccc60580650f3b5ad6e", + "installed": "a827b43938a960ef70f0183d53390c97a4550f9f0544dccc60580650f3b5ad6e" + }, + { + "file": "src/storage/queue-scheduling-store.js", + "sourceBuild": "febe2752b960f2b3225119d4e6b1ad46bb5490848c5629abaa05fc9b644ab006", + "installed": "febe2752b960f2b3225119d4e6b1ad46bb5490848c5629abaa05fc9b644ab006" + }, + { + "file": "src/storage/repositories.js", + "sourceBuild": "11ceda0a9e06aad8d4ae12b340743c7fcb37cb3f1c692b97329e4d55099691fe", + "installed": "11ceda0a9e06aad8d4ae12b340743c7fcb37cb3f1c692b97329e4d55099691fe" + } + ] + }, + "automated": { + "lint": "passed", + "typecheck": "passed", + "rootTest": "passed", + "memory": { + "passed": 1812, + "skipped": 1 + }, + "migrations": { + "passed": 279, + "skipped": 1 + }, + "backend": 1109, + "frontend": 2452, + "electron": { + "passed": 1259, + "skipped": 126 + }, + "agentNote": "Prior unaffected Agent full suite 7142 passed/80 skipped; not repeated for Memory-only follow-up" + }, + "packagedControlledLoad": { + "initial": { + "main": 10193, + "retry": 64 + }, + "bodyBytes": 32768, + "window": 36, + "elapsedMs": 572.97, + "timerMs": 24.99, + "counts": { + "interactive": 12, + "repair_retry": 9, + "import": 6, + "evolution": 6, + "maintenance": 3 + }, + "peakJobs": 4, + "peakRequests": 2, + "peakEmbeddingRequests": 1, + "maxEmbeddingBatch": 3, + "prepareCalls": 4173, + "badQueued": 0 + }, + "packagedSharedProviderFairness": { + "backlog": 200, + "delay": 10, + "withMaintenance": true, + "elapsedMs": 1495, + "done": { + "import": 50, + "evolution": 50 + }, + "maintenance": 79, + "completed": 100, + "milestones": [ + { + "completed": 10, + "import": 6, + "evolution": 4, + "maintenance": 5 + }, + { + "completed": 20, + "import": 10, + "evolution": 10, + "maintenance": 10 + }, + { + "completed": 30, + "import": 16, + "evolution": 14, + "maintenance": 15 + }, + { + "completed": 40, + "import": 20, + "evolution": 20, + "maintenance": 24 + }, + { + "completed": 50, + "import": 26, + "evolution": 24, + "maintenance": 34 + }, + { + "completed": 60, + "import": 30, + "evolution": 30, + "maintenance": 40 + }, + { + "completed": 70, + "import": 36, + "evolution": 34, + "maintenance": 49 + }, + { + "completed": 80, + "import": 40, + "evolution": 40, + "maintenance": 59 + }, + { + "completed": 90, + "import": 46, + "evolution": 44, + "maintenance": 69 + }, + { + "completed": 100, + "import": 50, + "evolution": 50, + "maintenance": 79 + } + ], + "queued": [ + { + "queue_lane": "evolution", + "n": 150, + "maxAttempts": 0 + }, + { + "queue_lane": "import", + "n": 150, + "maxAttempts": 0 + }, + { + "queue_lane": "maintenance", + "n": 1, + "maxAttempts": 0 + } + ] + }, + "packagedActualLocalModel": { + "passed": true, + "syntheticImports": 2, + "summarySucceeded": 2, + "embeddingSucceeded": 2, + "embeddingDimensions": 768, + "actualRequestPeak": 1, + "elapsedMs": 38064.19, + "note": "Actual LM Studio Gemma/Nomic; existing json_object-to-text fallback used" + }, + "installedFullScan": { + "initialRetained": 12000, + "newlyCaptured": 1000, + "historyTotal": 13000, + "source": "claude_code", + "mode": "full", + "complete": true, + "startedAt": "2026-10-23T05:53:06.593Z", + "completedAt": "2026-10-23T05:54:56.201Z", + "liveMemory": { + "createdAt": "2026-10-23T05:53:51.923Z", + "id": "trace_32b734a530bc0fe32a62", + "initialHistoryCount": 12220, + "readyAt": "2026-10-23T05:54:26.549Z", + "remainingHistoricalSummaries": 11590, + "jobs": [ + { + "job_type": "trace_summary", + "queue_lane": "interactive", + "status": "succeeded", + "attempts": 1 + }, + { + "job_type": "embedding", + "queue_lane": "interactive", + "status": "succeeded", + "attempts": 1 + }, + { + "job_type": "reflection", + "queue_lane": "evolution", + "status": "queued", + "attempts": 0 + } + ] + }, + "healthLatencyMs": { + "samples": 48, + "min": 5.61, + "median": 45.99, + "p95": 308.49, + "max": 673.75 + }, + "guiPage51Loaded": true, + "guiTotalPages": 651, + "guiLiveSearchAndDetail": true, + "note": "Same-profile upgrade rereads existing 12000 plus 1000 new complete turns. Controlled local HTTP Provider, 5s model and100ms embedding. Not all history summaries completed; not real cloud." + }, + "restart": { + "count": 13000, + "quickCheck": "ok", + "identicalBodyAndVectorFingerprints": true, + "verifiedMemories": 4, + "oldCompleteMemoryFingerprint": "df625239a98a67faf03de2cef9e84bdf3946913d11ddabddd117a35ef54e1f37", + "independentModelConnectionsPreserved": true, + "guiLiveSearch": true, + "runtimeOnlyPauseReapplied": true + }, + "limitations": [ + "Cmd-Q still triggers existing 5s Desktop cleanup fallback; attribution to queue not established; Desktop lifecycle not changed.", + "DMG background cosmetic warning remains.", + "Existing FTS write amplification and very long single-conversation capture costs not fixed.", + "No Windows, macOS x64 or real cloud Provider test; sampled times are not latency guarantees.", + "Temporary QA driver errors corrected without product changes: inactive fixture directory, SQLite result prototype comparison, runtime-only pause persistence assumption. Original failure evidence retained." + ] +} diff --git a/docs/qa/memory-worker-interactive-reservation-v126-20261024-results.json b/docs/qa/memory-worker-interactive-reservation-v126-20261024-results.json new file mode 100644 index 000000000..1a9dc5603 --- /dev/null +++ b/docs/qa/memory-worker-interactive-reservation-v126-20261024-results.json @@ -0,0 +1,92 @@ +{ + "date": "2026-10-24", + "branch": "test-memory-queue-v1.2.6", + "target": "upstream/v1.2.6", + "sourceCommit": "f0c8070796b496d79feb410b251035efa02b5ce9", + "desktopVersion": "1.2.3", + "memoryVersion": "2.1.6", + "memoryBuildId": "3142a05c95b56fdd39a81b07ca2f8ac6a08cb8198f239a1229b0ed8c08de7547", + "scope": "reserve one global logical worker slot for interactive tasks; keep other admission layers", + "globalLimit": 4, + "backgroundLimit": 3, + "interactiveReservedSlots": 1, + "singleSlotFallback": "global=1 implies background=1, reserved=0 to preserve progress", + "secondFullQueueCensusGapChanged": false, + "rootTests": "passed", + "lint": "passed", + "typecheck": "passed", + "memoryFirstPass": 2171, + "memorySecondPass": 2172, + "memorySkippedEach": 1, + "backendPassed": 992, + "frontendPassed": 2505, + "desktopPassed": 1287, + "desktopSkipped": 126, + "agentTuiPassed": 5, + "additionalFocusedTestsPassed": 50, + "pushed": false, + "prCreated": false, + "packageSha256": "48f29997b55dd0ae226030eaf9d425dd97081b27fb015cefa823c37ac19c04f8", + "packagingSeconds": 255, + "packageVerified": true, + "packageIdentity": { + "memoryJavaScriptModulesCompared": 230, + "mismatches": [], + "criticalModuleSha256": { + "service/worker/worker-concurrency.js": "a3ac16417242aa45ef566e6cf8bedaec5ac0a64ffd42db192dd0adf4cbdea0cc", + "service/worker/worker-observability.js": "3bc3006c8cb386dbbaa9ae61cf1f61883903ddc4f7fa6f24c2d665422c80f3eb", + "service/worker/worker-runner.js": "107087ad0ee970436585636158ab1e30b88124e1d4cca0d04fa08ee365cdab7e", + "service/worker/worker-work-requirements.js": "c551ae8a14b77d84b9d29c10b7dfe3be8f8fafaaba11bbbeeff4ee37953aeafe" + }, + "manifest": { + "version": "2.1.6", + "buildId": "3142a05c95b56fdd39a81b07ca2f8ac6a08cb8198f239a1229b0ed8c08de7547", + "schemaVersion": 13, + "protocolVersion": 1, + "target": "darwin-arm64", + "entrypoint": "dist/src/server/index.js", + "viewer": "dist/viewer/index.html" + } + }, + "installedQueueProbe": { + "backlog": 10000, + "backgroundLeased": 3, + "untouchedQueued": 9997, + "interactiveStartWaitMs": { + "native": 6.95920799999999, + "onboarding": 6.6293329999998605 + }, + "basis": "installed compiled queue and real SQLite; business/model work replaced by controlled gates" + }, + "installedAppSmoke": { + "success": true, + "profile": "isolated temporary HOME copied to persistent evidence", + "verification": "automated installed app startup/render/health/normal quit/relaunch; no manual UI operation", + "launches": [ + { + "launch": 1, + "healthyMs": 1200.63925, + "quitMs": 140.1231670000002, + "rendered": true, + "exit": { + "code": 0, + "signal": null + } + }, + { + "launch": 2, + "healthyMs": 1192.0469999999996, + "quitMs": 116.80191700000069, + "rendered": true, + "exit": { + "code": 0, + "signal": null + } + } + ] + }, + "postPackageFocusedTestsPassed": 65, + "developmentNodeNativeRestored": true, + "manualAuthenticatedUIBusinessTestPerformed": false, + "realCloudOrLMStudioLoadTestPerformed": false +} diff --git a/docs/qa/memory-worker-interactive-reservation-v126-20261024.md b/docs/qa/memory-worker-interactive-reservation-v126-20261024.md new file mode 100644 index 000000000..50a85768e --- /dev/null +++ b/docs/qa/memory-worker-interactive-reservation-v126-20261024.md @@ -0,0 +1,77 @@ +# 实时任务预留全局槽位：修复与打包测试报告 + +日期：2026-10-24。开发分支：`test-memory-queue-v1.2.6`，目标基线：`upstream/v1.2.6`。 +本轮源码提交：`f0c8070796b496d79feb410b251035efa02b5ce9`（fix: 为实时任务预留一个全局槽位）。 + +## 结论与使用方式 + +已实现默认总并发 4、后台最多 3，为 interactive 实时任务专门保留 1 个全局逻辑槽位。后台空闲时也不能借用预留槽位；实时任务可以使用全部尚可用的全局容量。Onboarding 首次报告沿用已持久化的 interactive 分类。 + +安装包：`/Users/lijia/Desktop/memmy-agent-main/queue-reservation-20261024/Memmy-1.2.3-darwin-arm64-cn-unsigned.dmg`。 +这是 macOS Apple Silicon 中文未签名测试包，应用版本沿用仓库的 Desktop 1.2.3、Memory 2.1.6；开发分支基线为 v1.2.6，未擅自改版本号。 + +固定工作目录：`/Users/lijia/Desktop/memmy-agent-main/memory-queue-v126-work`。 +测试证据目录：`/Users/lijia/Desktop/memmy-agent-main/queue-reservation-20261024`。 + +## 复现与调整 + +先在旧实现上运行新增回归，确认第 4 个后台任务可占满全局容量，不存在实时专用余量。随后才修改实现。 + +1. 在统一并发控制器中增加后台活跃任务计数，许可领取前同时检查全局上限和后台上限，失败不持有部分资源，也不先领取数据库租约。 +2. 从持久化 queueLane 传递分类，仅 interactive 可用预留容量。历史 trace_summary 即便基础类型或 payload 看似实时，仍按其持久化 import 分类；启动摘要修复按 maintenance；Onboarding 首次报告保持 interactive。 +3. 每个 Embedding 批次成员独立计入逻辑任务数，共享 Provider 许可不会共享或继承实时特权。成员独立释放，重复释放无副作用。 +4. 扫描任务计入后台上限；前台模型请求继续只受 Provider／请求限制，不额外占用逻辑 worker 槽位。 +5. 指标增加 backgroundActiveJobs、maxBackgroundJobs、interactiveReservedSlots、后台峰值和 background 拒绝计数，用于验证本次限制。 +6. 总并发配置为 1 时，预留数退为 0、后台上限仍为 1，避免后台永远无法执行；总并发为 2／4／8 时，后台上限分别为 1／3／7。 + +本轮只解决第一项全局槽位预留，没有实现第二项“全队列精确 runnable／等待原因统计”。没有增加生产背压或批量修复取消。 + +## 回归覆盖与结果 + +- 总并发和后台上限共同生效；许可拒绝不泄漏 Provider、scope 或全局槽位。 +- 3 个后台任务长期占位时，新实时记忆、Onboarding 首次报告可使用第 4 槽；被拒绝的历史／启动修复任务保持 queued、attempts=0、leaseToken=null。 +- 扫描 300 条后台积压的真实事件循环切片期间插入实时任务，实时任务仍可进入；130 条积压全部检查后进入等待，再插入实时任务并通知也可启动，等待期间没有持续轮询。 +- 共享 Embedding 批次、未知 lane、单槽配置、扫描等待／取消、租约安全、配置变化、失败／停止释放和指标窗口均有对应回归。 +- WRR 游标专项测试使用非饱和容量隔离“配额游标”和“许可拒绝”两个因素；饱和场景由新增预留槽位测试独立验证。原模拟实时占位显式标记 interactive，不把真实扫描任务视为实时。 + +| 检查 | 最终结果 | +| --- | --- | +| npm run lint | 通过 | +| npm run typecheck | 通过 | +| npm test | 退出码 0 | +| Memory 首轮 | 2171 通过，1 跳过 | +| Memory workspace 第二轮 | 2172 通过，1 跳过 | +| Backend | 992 通过 | +| Frontend | 2505 通过 | +| Desktop | 1287 通过，126 跳过 | +| Agent TUI | 5 通过 | +| 打包后专项复核 | 65 通过 | +| 提交差异检查及两轮独立复查 | 通过，未发现新增问题 | + +首轮恢复环境时出现过本地测试端口被沙箱禁止、Electron 测试运行时缺失，以及旧断言仍期待后台可占满 4 个槽位等失败。已分别允许测试所需的本地监听、恢复运行时、按新规则补充和更新断言后重新执行。上表只报告最终通过结果。Memory 两轮数字不同，是因为新增指标回归在第二轮开始前已加入，最终提交与打包后专项检查均包含该用例。 + +## 安装包验证 + +使用正式 `npm run package:mac:arm64:cn:unsigned` 构建，耗时 4 分 15 秒；原生组件、兼容 macOS 12 的 SQLCipher／OpenSSL、生产 renderer、运行时裁剪和打包守卫全部通过。复用本机已缓存的模型和原生依赖，没有依赖升级或锁文件提交。 + +DMG 完整性校验通过；只读挂载后复制到独立安装目录测试，未替换 /Applications 中的正式应用。 + +SHA256：`48f29997b55dd0ae226030eaf9d425dd97081b27fb015cefa823c37ac19c04f8`。 +Memory buildId：`3142a05c95b56fdd39a81b07ca2f8ac6a08cb8198f239a1229b0ed8c08de7547`，schema 13。 +包内 230 个 Memory JavaScript 模块与本次源码编译结果逐字节相同，关键控制器、requirements、runner、指标模块均已核对。 + +安装包中的 Electron、原生 SQLite 和编译后的真实 Runner 验证了 10000 条 import_summary 积压：后台仅领取 3 条并持续占位；新实时记忆、首次报告依次进入第 4 个槽位；9997 条后台任务仍 queued、attempts=0、无租约。任务结束和停止后所有逻辑槽位、后台计数、Provider 及 scope 清零。 + +该受控队列验证中，实时任务开始等待约 6.96 ms、首次报告约 6.63 ms。这里用可控等待替代业务处理和模型调用，数值只说明全局槽位不会被后台占满，不能作为真实模型耗时或生产 SLO 承诺。 + +实际启动独立安装的应用，使用隔离 HOME 和测试 profile：两轮均进入完整登录页面（已检查截图，未把启动动画算成功）；Memory /health 正常，返回正确 buildId、schema 13 和 native vector 存储。两轮健康就绪约 1.20／1.19 秒，正常退出约 140／117 ms，退出码均为 0；同一 profile 重启正常。 + +打包会将开发目录的 SQLite 原生库切换为 Electron ABI。结束后已恢复 Node 22 ABI，并用内存数据库及 65 项专项回归验证开发环境仍可使用。 + +## 实际边界 + +本轮 UI 测试是隔离新 profile 的自动启动、完整登录页面渲染、健康、退出和重启，没有登录真实账号或人工操作完整记忆业务流程。真实队列持久化与领取行为已通过包内 1 万条受控积压及源码回归验证；没有把它宣称为云模型或 LM Studio 满载性能测试。 + +预留全局槽位不会突破资源、Provider 或业务 scope 限制。共享本地模型正在执行不可抢占请求时，实时任务仍需等该请求结束。纯后台负载下，默认并发最多 3，这是专门预留容量的预期代价。 + +本轮只有本地提交，没有推送或创建 PR。没有修改用户的 Pages 方案文件。 diff --git a/tests/smoke/async-capture-http.test.ts b/tests/smoke/async-capture-http.test.ts index 8455d4ff6..c127c021a 100644 --- a/tests/smoke/async-capture-http.test.ts +++ b/tests/smoke/async-capture-http.test.ts @@ -39,6 +39,12 @@ describe("async capture through the real Backend HTTP client", () => { expect(result.jobs.map(job => job.jobType)).toEqual(expect.arrayContaining(["episode_assignment", "episode_finalize"])); await client.runWorker({ limit: 30 }); const failedJobType = failedOperation === "episode.select.v1" ? "episode_assignment" : "trace_summary"; + // Advance the persisted retry deadline rather than waiting for real backoff timers. + for (let attempt = 0; attempt < 3; attempt++) { + db.db.prepare("UPDATE evolution_jobs SET available_at=? WHERE job_type=? AND status='failed'") + .run(new Date(Date.now() - 1).toISOString(), failedJobType); + await client.runWorker({ limit: 30 }); + } await expect.poll(() => db.db.prepare("SELECT status FROM evolution_jobs WHERE job_type=?").get(failedJobType)) .toEqual({ status: "dead_letter" }); const completedIndependentCalls = successfulIndependentCalls; diff --git a/tests/smoke/memory-layer-smoke-plan.test.ts b/tests/smoke/memory-layer-smoke-plan.test.ts index 7c34c83d4..9d05b3acb 100644 --- a/tests/smoke/memory-layer-smoke-plan.test.ts +++ b/tests/smoke/memory-layer-smoke-plan.test.ts @@ -1,7 +1,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { runCommand } from "../../Memory/src/cli/commands.js"; import { PROJECT_VERSION } from "../../Memory/src/cli/project-version.js"; import { @@ -10,12 +10,17 @@ import { MemoryDb, MemoryService, type Embedder, - type LlmClient + type JobRef, + type LlmClient, + type LlmMessage, + type LlmCompletionOptions } from "../../Memory/src/index.js"; +import { Repositories } from "../../Memory/src/storage/repositories.js"; const tempRoots: string[] = []; afterEach(() => { + vi.restoreAllMocks(); for (const root of tempRoots.splice(0)) { rmSync(root, { recursive: true, force: true }); } @@ -40,17 +45,26 @@ describe("memory layer smoke plan", () => { .toContain("tsx tests/smoke/memory-layer-smoke.ts"); }); - it.each([false, true])("stores, processes, reads, and recalls a verified turn through the real Memory service (turnJudge=%s)", async turnJudge => { + it.each([[1, false], [1, true], [20, false], [20, true]] as const)("stores, processes, reads, and recalls a verified turn with budget %i, turnJudge=%s", async (workerBudget, turnJudge) => { const root = mkdtempSync(join(tmpdir(), "memmy-memory-smoke-")); tempRoots.push(root); const db = new MemoryDb({ path: join(root, "memory.sqlite") }); + const repos = new Repositories(db.db); let judgeCalls = 0; + const llm = createSmokeSummaryLlm(() => { judgeCalls++; }); + const embedder = createSmokeEmbedder(); + const modelCalls = [ + vi.spyOn(llm, "complete"), + vi.spyOn(llm, "completeJson"), + vi.spyOn(embedder, "embed"), + vi.spyOn(embedder, "embedOne") + ]; const service = new MemoryService({ db, mode: "dev", config: { ...DEFAULT_MEMMY_CONFIG, algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, session: { ...DEFAULT_MEMMY_CONFIG.algorithm.session, turnJudge } } }, - llm: createSmokeSummaryLlm(() => { judgeCalls++; }), - embedder: createSmokeEmbedder(), + llm, + embedder, fetchAppMemoryBudget: async () => null }); const namespace = { @@ -101,13 +115,42 @@ describe("memory layer smoke plan", () => { .toMatchObject({ user_text: "How should the v1.0.2 release workflow be verified?", assistant_text: "Run the release contracts and publish only after every attachment succeeds." }); expect(completed.jobs.map((job) => job.jobType)).toContain("trace_summary"); - const summaryWorker = await service.runWorkerOnce(20, { namespace }); - expect(summaryWorker.failed).toBe(0); - expect(summaryWorker.jobs.map((job) => job.jobType)).toContain("trace_summary"); + // Continuous refill may process the summary and its embedding in one call. + // Collect bounded rounds so this smoke contract also holds with a small budget. + const processedJobs: JobRef[] = []; + let drained = false; + for (let round = 0; round < 20; round += 1) { + const worker = await service.runWorkerOnce(workerBudget, { namespace }); + expect(worker.failed).toBe(0); + expect(worker.embeddingRetries.failed).toBe(0); + expect(worker.succeeded).toBe(worker.leased); + processedJobs.push(...worker.jobs); + if (worker.leased === 0 && worker.embeddingRetries.leased === 0) { + drained = true; + break; + } + } + expect(drained, "Smoke jobs must finish within the bounded worker rounds").toBe(true); + for (const jobType of ["trace_summary", "embedding"] as const) { + const jobs = processedJobs.filter((job) => + job.jobType === jobType && job.targetMemoryId === completed.l1MemoryId + ); + expect(jobs).toHaveLength(1); + expect(jobs[0]?.status).toBe("succeeded"); + } + expect(new Set(processedJobs.map((job) => job.jobId)).size).toBe(processedJobs.length); + expect(repos.memories.hasVector(completed.l1MemoryId, "vec_summary")).toBe(true); + expect(repos.processing.get(completed.l1MemoryId)).toMatchObject({ state: "ready" }); - const embeddingWorker = await service.runWorkerOnce(20, { namespace }); - expect(embeddingWorker.failed).toBe(0); - expect(embeddingWorker.jobs.map((job) => job.jobType)).toContain("embedding"); + const callsAfterProcessing = modelCalls.map((spy) => spy.mock.calls.length); + for (let round = 0; round < 2; round += 1) { + const idleWorker = await service.runWorkerOnce(workerBudget, { namespace }); + expect(idleWorker).toMatchObject({ + leased: 0, succeeded: 0, failed: 0, jobs: [], + embeddingRetries: { leased: 0, succeeded: 0, failed: 0, items: [] } + }); + } + expect(modelCalls.map((spy) => spy.mock.calls.length)).toEqual(callsAfterProcessing); for (let attempt = 0; attempt < 8; attempt++) { const raw = db.db.prepare("SELECT episode_id FROM raw_turns WHERE id=?").get(completed.rawTurnId) as { episode_id: string | null }; @@ -232,7 +275,7 @@ function createSmokeSummaryLlm(onJudge: () => void = () => {}): LlmClient { async complete() { return JSON.stringify({ title: "Smoke turn", summary: "Verified smoke summary" }); }, - async completeJson>(messages, options) { + async completeJson>(messages: LlmMessage[], options: LlmCompletionOptions) { if (options.operation === "episode.select.v1") return { episodeId: null, title: "Release verification", summary: "Verify the release workflow" } as unknown as T; const system = messages .filter((message) => message.role === "system")