Skip to content

Commit 30b4e54

Browse files
committed
Add automatic context compaction for long conversations
Once a chat's unfolded history estimates past ~6000 tokens (keeping the most recent 12 messages exact), older turns are summarized via a plain non-streaming completion on the same model and replaced with a system-message summary before the next request is sent. The summary and fold point persist on the session, so a second compaction only summarizes what's newly eligible instead of redoing the whole thing, and a stale fold point from editing/forking is clamped rather than silently dropping recent context. Falls back to sending the full history unchanged if summarization itself fails.
1 parent 744b69c commit 30b4e54

6 files changed

Lines changed: 267 additions & 2 deletions

File tree

‎app/src/sessions-store.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ export interface ChatSession {
1919
// The agent's last set_plan checklist for this conversation, persisted so
2020
// reopening a chat mid-task shows where the plan stood.
2121
planSteps?: { text: string; done: boolean }[];
22+
// Running summary produced by context compaction, replacing everything
23+
// in `messages` before contextSummaryThroughIndex when this session's
24+
// history is sent to a provider.
25+
contextSummary?: string;
26+
contextSummaryThroughIndex?: number;
2227
tags?: string[];
2328
createdAt: string;
2429
updatedAt: string;
@@ -66,7 +71,7 @@ export function updateSession(
6671
partial: Partial<
6772
Pick<
6873
ChatSession,
69-
"title" | "model" | "messages" | "params" | "projectId" | "systemPrompt" | "agentMode" | "agentWorkspace" | "planSteps" | "tags"
74+
"title" | "model" | "messages" | "params" | "projectId" | "systemPrompt" | "agentMode" | "agentWorkspace" | "planSteps" | "contextSummary" | "contextSummaryThroughIndex" | "tags"
7075
>
7176
>
7277
): ChatSession | null {
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import { describe, it, expect } from "vitest";
2+
import {
3+
estimateTokens,
4+
estimateMessagesTokens,
5+
planCompaction,
6+
shouldCompact,
7+
buildSummarizationPrompt,
8+
} from "./context-compaction";
9+
import type { ChatMessage } from "@/types/electron";
10+
11+
function msg(role: ChatMessage["role"], content: string): ChatMessage {
12+
return { role, content };
13+
}
14+
15+
describe("estimateTokens", () => {
16+
it("scales roughly with character count", () => {
17+
expect(estimateTokens("")).toBe(0);
18+
expect(estimateTokens("a".repeat(400))).toBe(100);
19+
});
20+
});
21+
22+
describe("estimateMessagesTokens", () => {
23+
it("sums estimated tokens across messages", () => {
24+
const messages = [msg("user", "a".repeat(40)), msg("assistant", "b".repeat(80))];
25+
expect(estimateMessagesTokens(messages)).toBe(10 + 20);
26+
});
27+
});
28+
29+
describe("planCompaction", () => {
30+
it("keeps everything when there aren't more messages than keepRecent", () => {
31+
const messages = Array.from({ length: 5 }, (_, i) => msg("user", `msg ${i}`));
32+
const plan = planCompaction(messages, 0, 12);
33+
expect(plan.toFold).toEqual([]);
34+
expect(plan.kept).toEqual(messages);
35+
});
36+
37+
it("folds everything except the most recent keepRecent messages", () => {
38+
const messages = Array.from({ length: 20 }, (_, i) => msg("user", `msg ${i}`));
39+
const plan = planCompaction(messages, 0, 12);
40+
expect(plan.toFold).toHaveLength(8);
41+
expect(plan.kept).toHaveLength(12);
42+
expect(plan.kept[0].content).toBe("msg 8");
43+
expect(plan.foldEndIndex).toBe(8);
44+
});
45+
46+
it("never re-folds messages already covered by a previous compaction", () => {
47+
const messages = Array.from({ length: 20 }, (_, i) => msg("user", `msg ${i}`));
48+
const plan = planCompaction(messages, 8, 12);
49+
expect(plan.toFold).toEqual([]);
50+
expect(plan.kept).toHaveLength(12);
51+
});
52+
53+
it("folds only the newly-eligible slice on a second compaction pass", () => {
54+
const messages = Array.from({ length: 30 }, (_, i) => msg("user", `msg ${i}`));
55+
const plan = planCompaction(messages, 8, 12);
56+
expect(plan.toFold).toHaveLength(10);
57+
expect(plan.toFold[0].content).toBe("msg 8");
58+
expect(plan.kept).toHaveLength(12);
59+
});
60+
});
61+
62+
describe("shouldCompact", () => {
63+
const keepRecent = 12;
64+
const budget = 100;
65+
66+
it("returns false when there's nothing beyond the recent window", () => {
67+
const messages = Array.from({ length: 12 }, () => msg("user", "x".repeat(1000)));
68+
expect(shouldCompact(messages, 0, keepRecent, budget)).toBe(false);
69+
});
70+
71+
it("returns false when the unfolded portion is under budget", () => {
72+
const messages = Array.from({ length: 20 }, () => msg("user", "hi"));
73+
expect(shouldCompact(messages, 0, keepRecent, budget)).toBe(false);
74+
});
75+
76+
it("returns true when the unfolded portion exceeds the token budget", () => {
77+
const messages = Array.from({ length: 20 }, () => msg("user", "x".repeat(1000)));
78+
expect(shouldCompact(messages, 0, keepRecent, budget)).toBe(true);
79+
});
80+
81+
it("re-evaluates against only what's beyond the already-folded count", () => {
82+
const messages = Array.from({ length: 20 }, () => msg("user", "x".repeat(1000)));
83+
// Everything but the recent window is already folded, so nothing new to weigh.
84+
expect(shouldCompact(messages, 8, keepRecent, budget)).toBe(false);
85+
});
86+
});
87+
88+
describe("buildSummarizationPrompt", () => {
89+
it("includes the transcript with speaker labels", () => {
90+
const prompt = buildSummarizationPrompt(null, [msg("user", "hello"), msg("assistant", "hi there")]);
91+
expect(prompt).toContain("User: hello");
92+
expect(prompt).toContain("Assistant: hi there");
93+
});
94+
95+
it("carries forward an existing summary for incremental compaction", () => {
96+
const prompt = buildSummarizationPrompt("earlier context here", [msg("user", "more")]);
97+
expect(prompt).toContain("Existing summary of earlier context:\nearlier context here");
98+
});
99+
100+
it("omits tool-role messages from the excerpt", () => {
101+
const prompt = buildSummarizationPrompt(null, [
102+
msg("user", "run the tests"),
103+
{ role: "tool", content: "42 passed", toolName: "run_command" },
104+
msg("assistant", "all green"),
105+
]);
106+
expect(prompt).not.toContain("42 passed");
107+
expect(prompt).toContain("run the tests");
108+
expect(prompt).toContain("all green");
109+
});
110+
});
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import type { ChatMessage } from "@/types/electron";
2+
3+
// No tokenizer is wired up per-provider, so this trades precision for a
4+
// heuristic that works everywhere without pulling in a vendor-specific
5+
// tokenizer just to decide "is this conversation getting long".
6+
const CHARS_PER_TOKEN = 4;
7+
8+
// Never folded away — keeps immediate context exact so the model doesn't
9+
// lose track of what was just said.
10+
export const COMPACTION_KEEP_RECENT = 12;
11+
// Conservative across providers; well under even the smallest context
12+
// windows in this app's model catalog once system prompts and a response
13+
// budget are accounted for.
14+
export const COMPACTION_BUDGET_TOKENS = 6000;
15+
16+
export function estimateTokens(text: string): number {
17+
return Math.ceil(text.length / CHARS_PER_TOKEN);
18+
}
19+
20+
export function estimateMessagesTokens(messages: ChatMessage[]): number {
21+
return messages.reduce((sum, m) => sum + estimateTokens(m.content), 0);
22+
}
23+
24+
export interface CompactionPlan {
25+
toFold: ChatMessage[];
26+
kept: ChatMessage[];
27+
foldEndIndex: number;
28+
}
29+
30+
// Decides which messages get folded into a running summary vs. kept
31+
// verbatim. Never re-folds anything already covered by `alreadyFoldedCount`,
32+
// and never folds into the last `keepRecent` messages.
33+
export function planCompaction(
34+
messages: ChatMessage[],
35+
alreadyFoldedCount: number,
36+
keepRecent: number
37+
): CompactionPlan {
38+
const foldEndIndex = Math.max(alreadyFoldedCount, messages.length - keepRecent);
39+
return {
40+
toFold: messages.slice(alreadyFoldedCount, foldEndIndex),
41+
kept: messages.slice(foldEndIndex),
42+
foldEndIndex,
43+
};
44+
}
45+
46+
export function shouldCompact(
47+
messages: ChatMessage[],
48+
alreadyFoldedCount: number,
49+
keepRecent: number,
50+
budgetTokens: number
51+
): boolean {
52+
if (messages.length - alreadyFoldedCount <= keepRecent) return false;
53+
return estimateMessagesTokens(messages.slice(alreadyFoldedCount)) > budgetTokens;
54+
}
55+
56+
// Tool call/result messages are dropped from the excerpt handed to the
57+
// summarizer — they're agent-mode bookkeeping the model can act on later
58+
// via read_notes if it mattered, not context a summary needs to preserve.
59+
export function buildSummarizationPrompt(previousSummary: string | null, toFold: ChatMessage[]): string {
60+
const transcript = toFold
61+
.filter((m) => (m.role === "user" || m.role === "assistant") && m.content)
62+
.map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content}`)
63+
.join("\n\n");
64+
const previous = previousSummary ? `Existing summary of earlier context:\n${previousSummary}\n\n` : "";
65+
return (
66+
`${previous}Summarize the following conversation excerpt concisely, preserving names, decisions, ` +
67+
`numbers, and anything a continuation of this conversation would need to know. Write it as a compact ` +
68+
`paragraph or short bullet list, not a transcript — this replaces the original messages in the model's ` +
69+
`context, so it needs to stand on its own.\n\n${transcript}`
70+
);
71+
}

‎frontend/src/lib/translations.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ export interface Dictionary {
2626
toolResult: string;
2727
toolFailed: string;
2828
transientErrorRetrying: string;
29+
contextCompacted: string;
2930
alwaysAllowThisSession: string;
3031
agentStep: string;
3132
agentStepTooltip: string;
@@ -333,6 +334,7 @@ export const en: Dictionary = {
333334
toolResult: "result",
334335
toolFailed: "failed",
335336
transientErrorRetrying: "Temporary provider error — retrying once...",
337+
contextCompacted: "Older messages summarized to stay within context — nothing is lost from the saved chat.",
336338
alwaysAllowThisSession: "Always allow this session",
337339
agentStep: "Agent step",
338340
agentStepTooltip: "How many automatic tool-result → model-continuation round trips have happened for this turn.",
@@ -650,6 +652,7 @@ export const tr: Dictionary = {
650652
toolResult: "sonucu",
651653
toolFailed: "başarısız oldu",
652654
transientErrorRetrying: "Geçici sağlayıcı hatası — bir kez yeniden deneniyor...",
655+
contextCompacted: "Bağlamda kalmak için eski mesajlar özetlendi — kaydedilen sohbetten hiçbir şey kaybolmadı.",
653656
alwaysAllowThisSession: "Bu oturumda her zaman izin ver",
654657
agentStep: "Ajan adımı",
655658
agentStepTooltip: "Bu tur için kaç otomatik araç sonucu → model devamı gidiş-dönüşü gerçekleşti.",

‎frontend/src/pages/Chat.tsx‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,13 @@ import { speakText, stopSpeaking } from "@/lib/tts";
7070
import { computeLineDiff } from "@/lib/diff";
7171
import { useToast } from "@/components/toast";
7272
import { isTransientError } from "@/lib/transient-errors";
73+
import {
74+
COMPACTION_BUDGET_TOKENS,
75+
COMPACTION_KEEP_RECENT,
76+
buildSummarizationPrompt,
77+
planCompaction,
78+
shouldCompact,
79+
} from "@/lib/context-compaction";
7380
import type {
7481
ChatMessage,
7582
OllamaModel,
@@ -430,6 +437,8 @@ export default function Chat() {
430437
const [agentStepCount, setAgentStepCount] = useState(0);
431438
const [autoApprovedTools, setAutoApprovedTools] = useState<Set<string>>(new Set());
432439
const [planSteps, setPlanSteps] = useState<PlanStep[]>([]);
440+
const [contextSummary, setContextSummary] = useState<string | null>(null);
441+
const [contextSummaryThroughIndex, setContextSummaryThroughIndex] = useState(0);
433442
const [projectScripts, setProjectScripts] = useState<ProjectScripts>({});
434443
const [quickActionRunning, setQuickActionRunning] = useState(false);
435444
const [undoMessage, setUndoMessage] = useState<string | null>(null);
@@ -511,6 +520,8 @@ export default function Chat() {
511520
setPendingToolCalls([]);
512521
setAgentStepCount(0);
513522
setPlanSteps(session.planSteps ?? []);
523+
setContextSummary(session.contextSummary ?? null);
524+
setContextSummaryThroughIndex(session.contextSummaryThroughIndex ?? 0);
514525
setAutoApprovedTools(new Set());
515526
setWriteDiffPreviews({});
516527
setUndoMessage(null);
@@ -847,13 +858,74 @@ export default function Chat() {
847858
}
848859
}
849860

861+
// A plain, non-streaming-to-the-UI completion used only to produce a
862+
// compaction summary — same provider/model/agentMode:false, but its
863+
// tokens are accumulated locally instead of touching message state.
864+
async function summarizeText(prompt: string): Promise<string | null> {
865+
const parsed = parseModelRef(model);
866+
if (!parsed) return null;
867+
let text = "";
868+
const { promise } = window.api.chat.send(
869+
parsed.provider,
870+
parsed.modelId,
871+
[{ role: "user", content: prompt }],
872+
{},
873+
(chunk) => {
874+
text += chunk.message?.content ?? "";
875+
},
876+
false
877+
);
878+
const result = await promise;
879+
return result.error ? null : text.trim() || null;
880+
}
881+
882+
// Folds everything but the most recent messages into a running summary
883+
// once the unfolded portion gets too large to keep sending verbatim —
884+
// otherwise a long conversation eventually fails outright (or gets
885+
// silently truncated by the provider) once it outgrows the model's
886+
// context window. Returns null when no compaction was needed, in which
887+
// case the caller sends its already-built history unchanged.
888+
async function maybeCompactHistory(baseMessages: ChatMessage[]): Promise<ChatMessage[] | null> {
889+
// Editing/forking/regenerating can shorten the message list after a
890+
// fold point was recorded — clamp so a stale index never causes the
891+
// "kept" slice to come up empty and silently drop recent context.
892+
const alreadyFolded = Math.min(contextSummaryThroughIndex, baseMessages.length);
893+
if (!shouldCompact(baseMessages, alreadyFolded, COMPACTION_KEEP_RECENT, COMPACTION_BUDGET_TOKENS)) {
894+
return null;
895+
}
896+
const { toFold, kept, foldEndIndex } = planCompaction(baseMessages, alreadyFolded, COMPACTION_KEEP_RECENT);
897+
if (toFold.length === 0) return null;
898+
899+
const summary = await summarizeText(buildSummarizationPrompt(contextSummary, toFold));
900+
// Summarization itself failed (network blip, provider error) — fall
901+
// back to sending the full uncompacted history rather than losing
902+
// context or blocking the user's message.
903+
if (!summary) return null;
904+
905+
setContextSummary(summary);
906+
setContextSummaryThroughIndex(foldEndIndex);
907+
if (sessionId) {
908+
window.api.sessions.update(sessionId, { contextSummary: summary, contextSummaryThroughIndex: foldEndIndex });
909+
}
910+
toast.info(t.contextCompacted);
911+
return [
912+
...buildSystemMessages(),
913+
{ role: "system", content: `Summary of earlier conversation:\n${summary}` },
914+
...kept,
915+
];
916+
}
917+
850918
async function runCompletion(
851919
history: ChatMessage[],
852920
baseMessages: ChatMessage[],
853921
opts: { isFirstMessage: boolean; titleSource: string; attempt?: number }
854922
) {
855923
const parsed = parseModelRef(model);
856924
if (!parsed || !sessionId) return;
925+
926+
const compactedHistory = await maybeCompactHistory(baseMessages);
927+
const requestHistory = compactedHistory ?? history;
928+
857929
// Timing an in-flight network stream, not computing render output —
858930
// the react-hooks purity rule can't tell this closure only runs once
859931
// per user-initiated send, not during render.
@@ -910,7 +982,7 @@ export default function Chat() {
910982
const { requestId, promise } = window.api.chat.send(
911983
parsed.provider,
912984
parsed.modelId,
913-
history,
985+
requestHistory,
914986
effectiveOptions(),
915987
(chunk) => {
916988
const piece = chunk.message?.content ?? "";

‎frontend/src/types/electron.d.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,8 @@ export interface ChatSession {
275275
agentMode?: boolean;
276276
agentWorkspace?: string | null;
277277
planSteps?: { text: string; done: boolean }[];
278+
contextSummary?: string;
279+
contextSummaryThroughIndex?: number;
278280
tags?: string[];
279281
createdAt: string;
280282
updatedAt: string;
@@ -424,6 +426,8 @@ export interface ElectronApi {
424426
| "agentMode"
425427
| "agentWorkspace"
426428
| "planSteps"
429+
| "contextSummary"
430+
| "contextSummaryThroughIndex"
427431
| "tags"
428432
>
429433
>

0 commit comments

Comments
 (0)