From 9d1a294f730992917c38ee0d16a1d2c0fc233931 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 04:05:01 +0000 Subject: [PATCH] feat(hooks): ask an optional Laya model about session recall The session-recall keyword list caught 3 of 30 natural requests for earlier context in a held-out set. When TRACEDECAY_HINT_RECALL_MODEL_URL names a running laya-serve, the Codex prompt hook asks it one calibrated yes/no question for prompts no keyword rule classified; at p >= 0.6 it gives the session-recall hint. Unset, unreachable, slow, or malformed answers keep the keyword result, and delivery budget and dedupe are unchanged. Held-out result: 22/30 caught, 0/48 false hints. Drop the bare "memory curator" and "transcript files" recall phrases, which hinted on ordinary docs and cleanup tasks; the eval cases that used them still match through more specific phrases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HB4wWE2tsLyyxJNkBCqRpD --- .../tracedecay-agent-hosts/src/hooks/codex.rs | 4 +- .../src/hooks/tool_hints.rs | 45 ++ .../src/hooks/tool_hints/classifiers.rs | 2 - .../tool_hints/evals/cases_real_world.rs | 12 + .../src/hooks/tool_hints/recall_model.rs | 258 ++++++++++ .../src/hooks/tool_hints/tests.rs | 59 +++ evals/README.md | 2 + evals/hint_recall_model/README.md | 72 +++ evals/hint_recall_model/heldout.json | 470 ++++++++++++++++++ evals/hint_recall_model/run.py | 110 ++++ 10 files changed, 1030 insertions(+), 4 deletions(-) create mode 100644 crates/tracedecay-agent-hosts/src/hooks/tool_hints/recall_model.rs create mode 100644 evals/hint_recall_model/README.md create mode 100644 evals/hint_recall_model/heldout.json create mode 100644 evals/hint_recall_model/run.py diff --git a/crates/tracedecay-agent-hosts/src/hooks/codex.rs b/crates/tracedecay-agent-hosts/src/hooks/codex.rs index 11c4c09a36..b632c4ee09 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/codex.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/codex.rs @@ -12,7 +12,7 @@ use crate::ports::hook_runtime::HookRuntimeV1; use tracedecay_runtime_core::config::ProfileRoot; use super::steering::{HookWorkspaceStatus, UNINDEXED_STATUS_LINE}; -use super::tool_hints::{ToolHint, ToolHintInput, decide_hint}; +use super::tool_hints::{ToolHint, ToolHintInput, decide_hint_with_recall_model}; use super::{ additional_context_json, append_tool_hint, deduped_project_hint_with_id, event_cwd_from_parsed, event_project_root, event_project_root_from_json, event_project_root_with_identity, @@ -349,7 +349,7 @@ fn deduped_codex_hint( } fn codex_prompt_hint(profile: &ProfileRoot, parsed: &Value) -> Option { - let hint = decide_hint(&ToolHintInput { + let hint = decide_hint_with_recall_model(&ToolHintInput { agent: HostIntegrationIdV1::Codex, session_id: event_session_id(parsed), tool_name: None, diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs index 0b62ec185f..6bba64ef09 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints.rs @@ -704,6 +704,48 @@ pub fn decide_hint(input: &ToolHintInput) -> Option { classify_hint(input).map(hint_for_category) } +/// [`decide_hint`], then, for a prompt no keyword rule classified, the +/// optional session-recall model configured by +/// `TRACEDECAY_HINT_RECALL_MODEL_URL`. +#[tracing::instrument( + name = "hosts.hooks.tool_hints.decide_with_recall_model", + level = "trace", + skip_all +)] +pub fn decide_hint_with_recall_model(input: &ToolHintInput) -> Option { + decide_hint_consulting(input, RecallModel::from_env) +} + +fn decide_hint_consulting( + input: &ToolHintInput, + recall_model: impl FnOnce() -> Option, +) -> Option { + if !input.hints_enabled { + return None; + } + classify_hint(input) + .or_else(|| { + let prompt = recall_model_prompt(input)?; + recall_model()? + .asks_for_recall(prompt)? + .then_some(HintCategory::SessionRecall) + }) + .map(hint_for_category) +} + +/// The model reads only a bare user prompt; tool events keep their +/// deterministic shape rules. +fn recall_model_prompt(input: &ToolHintInput) -> Option<&str> { + if input.tool_name.is_some() || input.command.is_some() { + return None; + } + input + .prompt + .as_deref() + .map(str::trim) + .filter(|prompt| !prompt.is_empty()) +} + #[tracing::instrument(name = "hosts.hooks.tool_hints.classify", level = "trace", skip_all)] fn classify_hint(input: &ToolHintInput) -> Option { let facts = HintRequestFacts::new(input); @@ -915,6 +957,9 @@ use classifiers::{ looks_like_pasted_diagnostic, matches_normalized, signals_unexpected_change, }; +mod recall_model; +use recall_model::RecallModel; + #[cfg(test)] #[allow(clippy::unwrap_used)] mod evals; diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/classifiers.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/classifiers.rs index b6137b3b12..1a3c0f3328 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/classifiers.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/classifiers.rs @@ -689,7 +689,6 @@ pub(super) fn asks_for_session_recall(text: &str) -> bool { "prior run", "previous run", "automation run", - "memory curator", "self-improvement run", "past session", "prior conversation", @@ -711,7 +710,6 @@ pub(super) fn asks_for_session_recall(text: &str) -> bool { &[ "raw codex jsonl transcript", "raw codex jsonl transcripts", - "transcript files", "hook input", "hook usage", "hint displayed", diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_real_world.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_real_world.rs index 42a4cf247f..be7d902970 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_real_world.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/evals/cases_real_world.rs @@ -27,6 +27,18 @@ pub(super) fn real_world_prompt_cases() -> Vec { None, &[], ), + prompt_eval( + "memory-curator-docs-task-is-not-recall", + "document how the memory curator works in the README", + None, + &[], + ), + prompt_eval( + "transcript-cleanup-task-is-not-recall", + "delete transcript files older than 30 days in the cleanup job", + None, + &[], + ), prompt_eval( "what-did-we-decide-before", "where did we decide how memory curation should work before?", diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/recall_model.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/recall_model.rs new file mode 100644 index 0000000000..b5111d6bd9 --- /dev/null +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/recall_model.rs @@ -0,0 +1,258 @@ +//! Optional learned second opinion for session-recall hints. +//! +//! [`super::classifiers::asks_for_session_recall`] matches only the phrasings +//! it lists, so most ways of asking for earlier context ("pull up our earlier +//! discussion", "have we hit this panic before?") stay silent. When +//! `TRACEDECAY_HINT_RECALL_MODEL_URL` names a running Laya decision server +//! (`laya-serve`), a prompt that no keyword rule classified is asked one +//! calibrated yes/no question. Unset, unreachable, slow, or malformed answers +//! leave the keyword result standing; the model never overrides a keyword +//! category and never touches delivery budget or dedupe. +//! +//! The question, the threshold, and the measured effect live with the +//! held-out set in `evals/hint_recall_model/`. + +use std::time::Duration; + +use serde_json::{Value, json}; +use tracedecay_application::http_agent::http_agent; + +/// Base URL of a `laya-serve` instance, e.g. `http://127.0.0.1:8000`. +pub(super) const RECALL_MODEL_URL_ENV: &str = "TRACEDECAY_HINT_RECALL_MODEL_URL"; +/// Whole-request budget in milliseconds; the hook answers without the model +/// once it elapses. +pub(super) const RECALL_MODEL_TIMEOUT_MS_ENV: &str = "TRACEDECAY_HINT_RECALL_MODEL_TIMEOUT_MS"; + +const DEFAULT_TIMEOUT: Duration = Duration::from_millis(250); + +/// Calibrated probability at or above which the model's "yes" yields a +/// session-recall hint. Chosen on `evals/hint_recall_model/heldout.json`. +pub(super) const RECALL_PROBABILITY_THRESHOLD: f64 = 0.6; + +const RECALL_QUESTION_KEY: &str = "recall"; +const RECALL_QUESTION: &str = "Is the user asking to look up or reuse something from an earlier \ + conversation, a previous session, or another agent's transcript?"; + +pub(super) struct RecallModel { + endpoint: String, + agent: ureq::Agent, +} + +impl RecallModel { + pub(super) fn from_env() -> Option { + Self::from_values( + std::env::var(RECALL_MODEL_URL_ENV).ok().as_deref(), + std::env::var(RECALL_MODEL_TIMEOUT_MS_ENV).ok().as_deref(), + ) + } + + pub(super) fn from_values(url: Option<&str>, timeout_ms: Option<&str>) -> Option { + let base = url.map(str::trim).filter(|url| !url.is_empty())?; + let timeout = timeout_ms + .and_then(|raw| raw.trim().parse::().ok()) + .filter(|ms| *ms > 0) + .map_or(DEFAULT_TIMEOUT, Duration::from_millis); + // A loopback sidecar: environment proxies would only add a hop. + let agent = http_agent( + ureq::Agent::config_builder() + .timeout_global(Some(timeout)) + .http_status_as_error(false) + .proxy(None) + .max_redirects(0) + .build(), + ); + Some(Self { + endpoint: format!("{}/v1/systemone", base.trim_end_matches('/')), + agent, + }) + } + + /// `Some(true)` when the model is at least [`RECALL_PROBABILITY_THRESHOLD`] + /// sure the prompt asks for earlier context, `Some(false)` below it, and + /// `None` when no usable answer arrived. + pub(super) fn asks_for_recall(&self, prompt: &str) -> Option { + let body = json!({ + "state": prompt, + "questions": { + RECALL_QUESTION_KEY: { "type": "noul", "instructions": RECALL_QUESTION }, + }, + }); + let mut response = match self.agent.post(&self.endpoint).send_json(&body) { + Ok(response) => response, + Err(error) => { + tracing::debug!(%error, "hint recall model unreachable"); + return None; + } + }; + if response.status().as_u16() != 200 { + tracing::debug!( + status = response.status().as_u16(), + "hint recall model refused" + ); + return None; + } + let answer = response.body_mut().read_json::().ok()?; + recall_probability(&answer).map(|probability| probability >= RECALL_PROBABILITY_THRESHOLD) + } +} + +fn recall_probability(answer: &Value) -> Option { + answer + .pointer(&format!("/answers/{RECALL_QUESTION_KEY}/noul")) + .and_then(Value::as_f64) + .filter(|probability| (0.0..=1.0).contains(probability)) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +pub(super) mod test_server { + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::{Ipv4Addr, TcpListener}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + /// A loopback server answering every request with `status` and `body` + /// after `delay`, counting the requests it saw. + pub(in super::super) struct FakeLaya { + pub(in super::super) url: String, + pub(in super::super) requests: Arc, + pub(in super::super) last_body: Arc>, + } + + pub(in super::super) fn fake_laya(status: u16, body: &str, delay: Duration) -> FakeLaya { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(AtomicUsize::new(0)); + let last_body = Arc::new(std::sync::Mutex::new(String::new())); + let body = body.to_string(); + let (seen, captured) = (Arc::clone(&requests), Arc::clone(&last_body)); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut content_length = 0; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + content_length = value.trim().parse().unwrap_or(0); + } + } + let mut request_body = vec![0; content_length]; + let _ = reader.read_exact(&mut request_body); + *captured.lock().unwrap() = String::from_utf8_lossy(&request_body).into_owned(); + seen.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(delay); + let _ = write!( + stream, + "HTTP/1.1 {status} X\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } + }); + FakeLaya { + url, + requests, + last_body, + } + } + + pub(in super::super) fn recall_answer(probability: f64) -> String { + format!(r#"{{"answers":{{"recall":{{"noul":{probability}}}}}}}"#) + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use std::time::{Duration, Instant}; + + use super::test_server::{fake_laya, recall_answer}; + use super::*; + + fn model(url: &str, timeout_ms: &str) -> RecallModel { + RecallModel::from_values(Some(url), Some(timeout_ms)).unwrap() + } + + #[test] + fn an_unset_or_blank_url_disables_the_model() { + assert!(RecallModel::from_values(None, None).is_none()); + assert!(RecallModel::from_values(Some(" "), Some("100")).is_none()); + } + + #[test] + fn the_endpoint_joins_the_base_url_once() { + let model = RecallModel::from_values(Some("http://127.0.0.1:8000/"), Some("x")).unwrap(); + assert_eq!(model.endpoint, "http://127.0.0.1:8000/v1/systemone"); + } + + #[test] + fn the_threshold_splits_yes_from_no() { + let yes = fake_laya( + 200, + &recall_answer(RECALL_PROBABILITY_THRESHOLD), + Duration::ZERO, + ); + assert_eq!( + model(&yes.url, "2000").asks_for_recall("have we hit this before?"), + Some(true) + ); + let no = fake_laya(200, &recall_answer(0.59), Duration::ZERO); + assert_eq!( + model(&no.url, "2000").asks_for_recall("add a session table"), + Some(false) + ); + } + + #[test] + fn the_request_asks_one_noul_question_about_the_prompt() { + let server = fake_laya(200, &recall_answer(0.9), Duration::ZERO); + model(&server.url, "2000").asks_for_recall("recap our earlier decisions"); + let sent: Value = serde_json::from_str(&server.last_body.lock().unwrap()).unwrap(); + assert_eq!(sent["state"], "recap our earlier decisions"); + assert_eq!(sent["questions"]["recall"]["type"], "noul"); + assert_eq!(sent["questions"].as_object().unwrap().len(), 1); + } + + #[test] + fn unusable_answers_yield_no_opinion() { + for (status, body) in [ + (500, r#"{"detail":"inference failed"}"#.to_string()), + (200, "not json".to_string()), + (200, r#"{"answers":{}}"#.to_string()), + (200, recall_answer(1.5)), + ] { + let server = fake_laya(status, &body, Duration::ZERO); + assert_eq!( + model(&server.url, "2000").asks_for_recall("anything"), + None, + "{body}" + ); + } + } + + #[test] + fn a_slow_model_is_abandoned_at_the_timeout() { + let server = fake_laya(200, &recall_answer(0.99), Duration::from_secs(5)); + let started = Instant::now(); + assert_eq!(model(&server.url, "100").asks_for_recall("anything"), None); + assert!(started.elapsed() < Duration::from_secs(2)); + } + + #[test] + fn an_unreachable_model_yields_no_opinion() { + let port = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + assert_eq!( + model(&format!("http://127.0.0.1:{port}"), "500").asks_for_recall("anything"), + None + ); + } +} diff --git a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs index 049120fced..9c28df6cf6 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/tool_hints/tests.rs @@ -373,3 +373,62 @@ fn every_tool_a_hint_names_is_cataloged() { } } } + +fn prompt_input(prompt: &str) -> ToolHintInput { + ToolHintInput { + prompt: Some(prompt.to_string()), + session_id: Some("session-1".to_string()), + ..ToolHintInput::default() + } +} + +fn recall_model_at(url: &str) -> impl FnOnce() -> Option + '_ { + move || RecallModel::from_values(Some(url), Some("2000")) +} + +#[test] +fn the_recall_model_classifies_a_prompt_the_keywords_missed() { + use super::recall_model::test_server::{fake_laya, recall_answer}; + let server = fake_laya(200, &recall_answer(0.9), std::time::Duration::ZERO); + let input = prompt_input("pull up our earlier discussion on the hook budget"); + assert_eq!(classify_hint(&input), None); + let hint = decide_hint_consulting(&input, recall_model_at(&server.url)).unwrap(); + assert_eq!(hint.category, HintCategory::SessionRecall); + assert!(hint.context.contains("tracedecay_message_search")); +} + +#[test] +fn the_recall_model_is_not_consulted_when_it_cannot_change_the_answer() { + use super::recall_model::test_server::{fake_laya, recall_answer}; + use std::sync::atomic::Ordering; + let server = fake_laya(200, &recall_answer(0.9), std::time::Duration::ZERO); + + let keyword = prompt_input("who calls classify_hint?"); + let hint = decide_hint_consulting(&keyword, recall_model_at(&server.url)).unwrap(); + assert_eq!(hint.category, HintCategory::CallGraph); + + let tool_event = ToolHintInput { + tool_name: Some("Bash".to_string()), + command: Some("cargo check".to_string()), + ..prompt_input("see whether this builds") + }; + assert!(decide_hint_consulting(&tool_event, recall_model_at(&server.url)).is_none()); + + let disabled = ToolHintInput { + hints_enabled: false, + ..prompt_input("pull up our earlier discussion") + }; + assert!(decide_hint_consulting(&disabled, recall_model_at(&server.url)).is_none()); + assert!(decide_hint_consulting(&prompt_input(" "), recall_model_at(&server.url)).is_none()); + + assert_eq!(server.requests.load(Ordering::SeqCst), 0); +} + +#[test] +fn a_recall_model_no_or_absence_keeps_the_prompt_silent() { + use super::recall_model::test_server::{fake_laya, recall_answer}; + let input = prompt_input("add a session timeout setting to the config"); + let server = fake_laya(200, &recall_answer(0.1), std::time::Duration::ZERO); + assert!(decide_hint_consulting(&input, recall_model_at(&server.url)).is_none()); + assert!(decide_hint_consulting(&input, || None).is_none()); +} diff --git a/evals/README.md b/evals/README.md index cfd1c1d2a4..98188dce57 100644 --- a/evals/README.md +++ b/evals/README.md @@ -9,5 +9,7 @@ preserving the distinct runtimes: fixtures, scorers, and smoke commands. - `agent_adoption/` contains the neutral-prompt adoption and ablation matrix, including its dry-run and self-test tooling. +- `hint_recall_model/` scores the optional Laya second opinion for + session-recall hook hints against a held-out prompt set. Each subdirectory documents its own prerequisites and runnable commands. diff --git a/evals/hint_recall_model/README.md b/evals/hint_recall_model/README.md new file mode 100644 index 0000000000..99fe776310 --- /dev/null +++ b/evals/hint_recall_model/README.md @@ -0,0 +1,72 @@ +# Session-recall hint model eval + +The Codex prompt hook suggests `tracedecay_message_search` / `tracedecay_lcm_grep` +when a prompt asks for earlier context. The keyword predicate +(`asks_for_session_recall` in +`crates/tracedecay-agent-hosts/src/hooks/tool_hints/classifiers.rs`) catches +every phrasing in the Rust eval matrix but few others. This eval measures an +optional second opinion from [Laya](https://pypi.org/project/laya/), a local +non-autoregressive decision model that answers typed questions with +calibrated probabilities. + +## How the hook uses the model + +Opt-in through the environment of the process that runs the Codex hooks: + +| Variable | Meaning | Default | +| --- | --- | --- | +| `TRACEDECAY_HINT_RECALL_MODEL_URL` | base URL of a running `laya-serve` | unset: keyword rules only | +| `TRACEDECAY_HINT_RECALL_MODEL_TIMEOUT_MS` | whole-request budget | `250` | + +The model is consulted only for a bare user prompt that no keyword rule +classified. It asks a single yes/no (`noul`) question. A probability of 0.6 or +more yields the session-recall hint. An unset URL, an unreachable server, a +timeout, a non-200 status or a malformed answer all leave the keyword result +standing. Delivery budget, dedupe and escalation +(`tracedecay-policy::hint_delivery`) are unchanged. + +```bash +pip install 'laya[serve]' +LAYA_HOST=127.0.0.1 LAYA_PORT=8000 LAYA_MODELS=english LAYA_DEFAULT_MODEL=english laya-serve +export TRACEDECAY_HINT_RECALL_MODEL_URL=http://127.0.0.1:8000 +``` + +## Running the eval + +```bash +python3 evals/hint_recall_model/run.py --url http://127.0.0.1:8000 --verbose +``` + +`heldout.json` contains 78 prompts that were written apart from the keyword +lists: + +- 30 natural requests for earlier context; +- 25 hard negatives that mention memory, sessions, history or transcripts + without asking for recall; +- 23 prompts for other categories, plus off-topic chat. + +## Results + +Measured with `laya` 0.4.8, English checkpoint at revision `7b928d8`, CPU, +over `laya-serve`. The keyword column already includes the two phrases this +change removed. + +| Held-out set (78 prompts) | recall requests caught | false hints | +| --- | --- | --- | +| keyword rules | 3 / 30 | 0 / 48 | +| model alone (p ≥ 0.6) | 21 / 30 | 0 / 48 | +| shipped policy (keywords, then model) | **22 / 30** | **0 / 48** | + +On the 60 prompt-only cases of the Rust eval matrix, the policy keeps all +8/8 recall cases. It adds one false hint: "change the button text to Save +and run the narrow test" (p = 0.73). + +Before this change the keyword list also fired on "document how the memory +curator works in the README" and on "delete transcript files older than 30 +days in the cleanup job". The bare phrases `memory curator` and +`transcript files` were removed. Both prompts are now silent regression +cases in `cases_real_world.rs`. + +Latency over HTTP on a shared 4-core cloud CPU was about 200 ms median, which +is close to the default 250 ms budget, so slow requests fall back to keywords. +Expect a few milliseconds on a GPU or Apple silicon. diff --git a/evals/hint_recall_model/heldout.json b/evals/hint_recall_model/heldout.json new file mode 100644 index 0000000000..20a787ad42 --- /dev/null +++ b/evals/hint_recall_model/heldout.json @@ -0,0 +1,470 @@ +[ + { + "name": "ho-recall-0", + "prompt": "didn't we already figure out why the indexer was stalling? what was the fix", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-1", + "prompt": "what was the conclusion yesterday about switching to the bazel remote cache", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-2", + "prompt": "pull up our earlier discussion on the hook budget", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-3", + "prompt": "I explained the deploy steps to you on Monday, use those", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-4", + "prompt": "go back to what I said about naming conventions a couple of chats ago", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-5", + "prompt": "we went through this exact error a week ago, find how we solved it", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-6", + "prompt": "what approach did the other agent take on the scheduler refactor", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-7", + "prompt": "summarize what happened in yesterday's session", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-8", + "prompt": "what did I tell you about not touching the migrations", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-9", + "prompt": "find the conversation where we picked sqlite over postgres", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-10", + "prompt": "you suggested a fix for this flaky test before, what was it?", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-11", + "prompt": "recover the plan we agreed on before the context got compacted", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-12", + "prompt": "which options did we rule out previously for the cache key?", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-13", + "prompt": "the codex session from this morning was working on this, see what it found", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-14", + "prompt": "we had a long thread about retry semantics, what did we land on", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-15", + "prompt": "check our chat history for the API key rotation steps", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-16", + "prompt": "I already gave you the repro steps in another session", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-17", + "prompt": "what were the open questions we left unresolved yesterday", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-18", + "prompt": "look through previous transcripts for when the dashboard build first broke", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-19", + "prompt": "earlier today you found the root cause of the leak, what was it again?", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-20", + "prompt": "restore context from the session before the crash", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-21", + "prompt": "what did the reviewer agent say in its last pass", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-22", + "prompt": "have we hit this panic before?", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-23", + "prompt": "recap our previous decisions on error handling", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-24", + "prompt": "find where I asked for dark mode support in a past chat", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-25", + "prompt": "what was my preferred commit message style that I mentioned before", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-26", + "prompt": "what did we try last week that didn't work for the deadlock", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-27", + "prompt": "search old conversations for 'lease expired'", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-28", + "prompt": "is there a session where someone already investigated the OOM?", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-recall-29", + "prompt": "continue from where the previous agent left off on the migration", + "expected": "SessionRecall", + "split": "recall" + }, + { + "name": "ho-neg-0", + "prompt": "fix the memory leak in the session store", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-1", + "prompt": "add a session timeout setting to the config", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-2", + "prompt": "why is memory usage so high in the daemon", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-3", + "prompt": "write a unit test for the fact store search scoring", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-4", + "prompt": "rename SessionId to ConversationId across the crate", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-5", + "prompt": "document how the memory curator works in the README", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-6", + "prompt": "the session cookie is not being set on login", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-7", + "prompt": "implement a last-used timestamp on sessions", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-8", + "prompt": "add a git history view to the dashboard", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-9", + "prompt": "update the CHANGELOG with the last release notes", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-10", + "prompt": "optimize the history buffer to avoid reallocations", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-11", + "prompt": "explain how LRU caches work", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-12", + "prompt": "delete transcript files older than 30 days in the cleanup job", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-13", + "prompt": "the previous button in the pager is misaligned", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-14", + "prompt": "make the conversation list in the UI sortable by date", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-15", + "prompt": "what's the difference between stack and heap memory", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-16", + "prompt": "bump the version since the last release", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-17", + "prompt": "add a remember-me checkbox to the login form", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-18", + "prompt": "set up memory limits for the docker container", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-19", + "prompt": "write a migration that adds a sessions table", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-20", + "prompt": "run the tests again", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-21", + "prompt": "refactor the transcript parser to stream lines", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-22", + "prompt": "earlier versions of clippy didn't flag this, add an allow", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-23", + "prompt": "make the dashboard remember the selected tab", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-neg-24", + "prompt": "profile memory allocations in the indexer", + "expected": null, + "split": "hard_negative" + }, + { + "name": "ho-other-0", + "prompt": "trace everything that ends up invoking flush_ledger", + "expected": "CallGraph", + "split": "other" + }, + { + "name": "ho-other-1", + "prompt": "what functions does parse_envelope call into", + "expected": "CallGraph", + "split": "other" + }, + { + "name": "ho-other-2", + "prompt": "if I delete the legacy adapter, what tests break?", + "expected": "Impact", + "split": "other" + }, + { + "name": "ho-other-3", + "prompt": "which modules depend on SchedulerConfig and would be affected by changing it", + "expected": "Impact", + "split": "other" + }, + { + "name": "ho-other-4", + "prompt": "look over my staged changes before I open the PR", + "expected": "ReviewChanges", + "split": "other" + }, + { + "name": "ho-other-5", + "prompt": "review this pull request's diff for bugs", + "expected": "ReviewChanges", + "split": "other" + }, + { + "name": "ho-other-6", + "prompt": "where is struct HintCategorySpec declared", + "expected": "SymbolLookup", + "split": "other" + }, + { + "name": "ho-other-7", + "prompt": "jump to the definition of decide_hint_delivery", + "expected": "SymbolLookup", + "split": "other" + }, + { + "name": "ho-other-8", + "prompt": "list all types implementing the Store trait", + "expected": "TypeOrientation", + "split": "other" + }, + { + "name": "ho-other-9", + "prompt": "where is the scheduler config struct constructed", + "expected": "TypeOrientation", + "split": "other" + }, + { + "name": "ho-other-10", + "prompt": "check how the sibling api-gateway repo handles auth", + "expected": "ProjectContext", + "split": "other" + }, + { + "name": "ho-other-11", + "prompt": "compare this with our other repo tracedecay-cloud", + "expected": "ProjectContext", + "split": "other" + }, + { + "name": "ho-other-12", + "prompt": "there are commits on my branch I don't recognize, who made them", + "expected": "UnexpectedChanges", + "split": "other" + }, + { + "name": "ho-other-13", + "prompt": "someone force pushed to my branch, what changed", + "expected": "UnexpectedChanges", + "split": "other" + }, + { + "name": "ho-other-14", + "prompt": "which files contain the codex adapter", + "expected": "FileLookup", + "split": "other" + }, + { + "name": "ho-other-15", + "prompt": "find the config files for the scheduler", + "expected": "FileLookup", + "split": "other" + }, + { + "name": "ho-other-16", + "prompt": "read through the entire repo and give me an architecture overview", + "expected": "BroadRead", + "split": "other" + }, + { + "name": "ho-other-17", + "prompt": "the build is failing with type errors, help me fix them", + "expected": "BuildDiagnostics", + "split": "other" + }, + { + "name": "ho-other-18", + "prompt": "write a haiku about rust", + "expected": null, + "split": "other" + }, + { + "name": "ho-other-19", + "prompt": "what's 15% of 80", + "expected": null, + "split": "other" + }, + { + "name": "ho-other-20", + "prompt": "commit and push", + "expected": null, + "split": "other" + }, + { + "name": "ho-other-21", + "prompt": "translate this paragraph to Spanish", + "expected": null, + "split": "other" + }, + { + "name": "ho-other-22", + "prompt": "open a PR for this branch", + "expected": null, + "split": "other" + } +] \ No newline at end of file diff --git a/evals/hint_recall_model/run.py b/evals/hint_recall_model/run.py new file mode 100644 index 0000000000..1fa534281a --- /dev/null +++ b/evals/hint_recall_model/run.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Score the session-recall hint policy against a running Laya server. + +Sends each prompt in a case file to `laya-serve` with the same question the +Codex prompt hook sends (`recall_model.rs`), then reports how many recall +requests the keyword predicate, the model, and the shipped keyword-then-model +policy catch, and how many non-recall prompts each wrongly hints. + + laya-serve & # pip install 'laya[serve]' + python3 evals/hint_recall_model/run.py --url http://127.0.0.1:8000 +""" + +import argparse +import json +import statistics +import sys +import time +import urllib.request +from pathlib import Path + +# Keep in sync with RECALL_QUESTION and RECALL_PROBABILITY_THRESHOLD in +# crates/tracedecay-agent-hosts/src/hooks/tool_hints/recall_model.rs. +RECALL_QUESTION = ( + "Is the user asking to look up or reuse something from an earlier conversation, " + "a previous session, or another agent's transcript?" +) +DEFAULT_THRESHOLD = 0.6 + +# Baseline only: the phrase lists of asks_for_session_recall +# (tool_hints/classifiers.rs) as of this eval. The Rust eval matrix, not this +# copy, is the authority on keyword behavior. +PRIOR_CONTEXT = [ + "where did we", "what did we", "when did we", "did we talk", "remind me", + "remember when", "last time", "earlier we", "talk about", "discuss before", + "mentioned before", "previously discussed", "last run", "prior run", + "previous run", "automation run", "self-improvement run", "past session", + "prior conversation", "previous conversation", "earlier conversation", + "context length exceeded", "cannot compress further", "compacted context", + "compaction", "context was compacted", "session search", "session recall", + "conversation history", +] +RAW_TRANSCRIPT = [ + "raw codex jsonl transcript", "raw codex jsonl transcripts", "hook input", + "hook usage", "hint displayed", "hints displayed", "model gets", "user submitted", +] + + +def keyword_recall(prompt): + text = prompt.lower() + has = lambda needles: any(needle in text for needle in needles) # noqa: E731 + return has(PRIOR_CONTEXT) or has(RAW_TRANSCRIPT) or ( + has(["lcm sessions", "past sessions", "prior sessions"]) + and has(["check", "search", "find", "look", "review"]) + ) + + +def recall_probability(url, prompt): + body = json.dumps({ + "state": prompt, + "questions": {"recall": {"type": "noul", "instructions": RECALL_QUESTION}}, + }).encode() + request = urllib.request.Request( + f"{url.rstrip('/')}/v1/systemone", data=body, + headers={"Content-Type": "application/json"}, + ) + started = time.perf_counter() + with urllib.request.urlopen(request, timeout=30) as response: + answer = json.load(response) + return answer["answers"]["recall"]["noul"], (time.perf_counter() - started) * 1000 + + +def main(): + here = Path(__file__).resolve().parent + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--url", default="http://127.0.0.1:8000") + parser.add_argument("--cases", type=Path, default=here / "heldout.json") + parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + + cases = json.loads(args.cases.read_text()) + tally = {name: {"caught": 0, "false_hints": 0} for name in ("keyword", "model", "policy")} + recall_total = sum(case["expected"] == "SessionRecall" for case in cases) + latencies = [] + for case in cases: + probability, ms = recall_probability(args.url, case["prompt"]) + latencies.append(ms) + keyword = keyword_recall(case["prompt"]) + model = probability >= args.threshold + verdicts = {"keyword": keyword, "model": model, "policy": keyword or model} + is_recall = case["expected"] == "SessionRecall" + for name, hinted in verdicts.items(): + if hinted and is_recall: + tally[name]["caught"] += 1 + elif hinted: + tally[name]["false_hints"] += 1 + if args.verbose and verdicts["policy"] != is_recall: + print(f" miss p={probability:.2f} expected={case['expected']}: {case['prompt']}") + + print(f"{len(cases)} prompts, {recall_total} recall requests, threshold {args.threshold}") + for name, counts in tally.items(): + print(f" {name:8} caught {counts['caught']}/{recall_total}, " + f"false hints {counts['false_hints']}/{len(cases) - recall_total}") + print(f" model latency median {statistics.median(latencies):.0f} ms " + f"(includes HTTP; hardware-dependent)") + return 0 + + +if __name__ == "__main__": + sys.exit(main())