From fc7efe2565b3fe9eaf0d4874b66ae2b263a69cc7 Mon Sep 17 00:00:00 2001 From: H2Rhan <2373723338@qq.com> Date: Wed, 9 Sep 2026 19:11:26 +0800 Subject: [PATCH] =?UTF-8?q?data:=2034=20=E7=BB=84=E5=AE=9E=E9=AA=8C?= =?UTF-8?q?=E5=8E=9F=E5=A7=8B=E6=95=B0=E6=8D=AE=E5=85=A5=E5=BA=93=EF=BC=88?= =?UTF-8?q?=E9=80=90=E8=BD=AE=20jsonl=20+=20=E6=B1=87=E6=80=BB=20csv=20+?= =?UTF-8?q?=20=E6=BA=AF=E6=BA=90=E8=A1=A8=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - data/experiments/ 19 个文件：exp1-19 三组对照 3 批次、补充实验（块数证伪/token 对照）、batch3-5（协议横向/断点/20轮长对话）、TTL 双档、E20-E29 覆盖、E30-E34 进阶 - data/README.md：文件 → 实验编号溯源表 + 字段口径 - experiment-results.md 头部加原始数据指针；README 目录补 data/ - 评审可溯源性闭环：文档每个数字可回查到具体文件具体行 --- .gitignore | 4 +- README.md | 1 + data/README.md | 31 ++++++++ data/experiments/batch3_20260908_221208.jsonl | 45 ++++++++++++ data/experiments/batch4_20260908_223443.jsonl | 15 ++++ data/experiments/batch5_20260908_223934.jsonl | 25 +++++++ .../experiment_20260908_213818.jsonl | 18 +++++ .../experiment_20260908_214354.jsonl | 27 +++++++ .../experiment_20260908_214628.jsonl | 72 +++++++++++++++++++ data/experiments/rich_20260908_232421.jsonl | 6 ++ data/experiments/rich_20260908_232832.jsonl | 38 ++++++++++ data/experiments/rich_20260908_233922.jsonl | 4 ++ data/experiments/summary_20260908_213818.csv | 7 ++ data/experiments/summary_20260908_214354.csv | 10 +++ data/experiments/summary_20260908_214628.csv | 10 +++ .../supplement_20260908_220106.jsonl | 17 +++++ .../supplement_20260908_220353.jsonl | 12 ++++ data/experiments/ttl_20260908_224911.jsonl | 6 ++ data/experiments/ttl_20260908_230014.jsonl | 6 ++ ...41\245\345\205\2052_20260909_142421.jsonl" | 24 +++++++ ...41\245\345\205\2053_20260909_142829.jsonl" | 19 +++++ ...41\245\345\205\2054_20260909_143614.jsonl" | 6 ++ docs/experiment-results.md | 1 + 23 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 data/README.md create mode 100644 data/experiments/batch3_20260908_221208.jsonl create mode 100644 data/experiments/batch4_20260908_223443.jsonl create mode 100644 data/experiments/batch5_20260908_223934.jsonl create mode 100644 data/experiments/experiment_20260908_213818.jsonl create mode 100644 data/experiments/experiment_20260908_214354.jsonl create mode 100644 data/experiments/experiment_20260908_214628.jsonl create mode 100644 data/experiments/rich_20260908_232421.jsonl create mode 100644 data/experiments/rich_20260908_232832.jsonl create mode 100644 data/experiments/rich_20260908_233922.jsonl create mode 100644 data/experiments/summary_20260908_213818.csv create mode 100644 data/experiments/summary_20260908_214354.csv create mode 100644 data/experiments/summary_20260908_214628.csv create mode 100644 data/experiments/supplement_20260908_220106.jsonl create mode 100644 data/experiments/supplement_20260908_220353.jsonl create mode 100644 data/experiments/ttl_20260908_224911.jsonl create mode 100644 data/experiments/ttl_20260908_230014.jsonl create mode 100644 "data/experiments/\350\241\245\345\205\2052_20260909_142421.jsonl" create mode 100644 "data/experiments/\350\241\245\345\205\2053_20260909_142829.jsonl" create mode 100644 "data/experiments/\350\241\245\345\205\2054_20260909_143614.jsonl" diff --git a/.gitignore b/.gitignore index 50f7a79..1b314b1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,9 @@ __pycache__/ *.pyo # 运行时产物（不入库） -data/ +data/* +!data/experiments/ +!data/README.md *.db metrics.jsonl experiments/report.json diff --git a/README.md b/README.md index fe2ef58..a2fec8e 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ config/ session.json（Session 边界可切换配置） experiments/ 三组对照 + rich_experiments.py（E20–E28）+ rich_experiments2.py（E30–E34） tests/ 离线单测（录制样例 + 自查回归 + 多轮链 E2E） tools/ mock_backend + verify_cache + smoke_e2e + recordings +data/ 34 组实验原始数据（逐轮 jsonl + 汇总 csv，溯源表见 data/README.md） docs/ ir-schema.md / capability-matrix.md / experiment-results.md / LIMITATIONS.md / REVIEW.md / TRACK04_签字确认稿.md CHANGELOG.md 按日期演进记录（v0 契约冻结 → v1.2 → 实验交付 → 覆盖补全） ``` diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..c5695c0 --- /dev/null +++ b/data/README.md @@ -0,0 +1,31 @@ +# 实验原始数据（34 组实测 · 逐轮记录） + +> 本目录是 `docs/experiment-results.md` 全部数字的**原始出处**：每一次真实 API 调用的逐轮 jsonl 记录 + 汇总 csv。 +> 数据日期 2026-09-08 / 09-09，Anthropic Messages API 兼容端点真实调用。 +> 口径与防污染方法（唯一 run_id 前缀隔离等）见实验文档第七节「方法学与踩坑」。 + +## 文件 → 实验编号 溯源表 + +| 文件 | 内容 | 对应实验文档结论 | +| --- | --- | --- | +| `experiment_20260908_213818.jsonl` / `summary_*.csv` | 三组对照主实验批次 1：A_baseline / B_locked / G1_300 / G2_4x512 / G3_2048 / G4_30frag | exp1–exp5 | +| `experiment_20260908_214354.jsonl` / `summary_*.csv` | 批次 2：增加 C_dynamic / D1_head / D2_tail | exp1–exp8 | +| `experiment_20260908_214628.jsonl` / `summary_*.csv` | 批次 3（8 轮 × 9 配置全量） | exp1–exp19 主数据 | +| `supplement_20260908_220106.jsonl` | 补充实验一：E1/E2 协议横向 token 对照（99.3% 少算）+ H_10/30/60blk 块数证伪 | 20-block 回看窗口证伪 | +| `supplement_20260908_220353.jsonl` | 补充实验一续：H_5/10/30/60blk | 同上 | +| `batch3_20260908_221208.jsonl` | 三协议横向（R1_anthropic / R2_chat / R3_responses）+ 逐模型阈值探测（T_haiku-4.5 / T_sonnet-4.6 / T_opus-5）+ N1–N5 更新频率公式 | 05B 悬案 / 阈值表 / (N−1)/N | +| `batch4_20260908_223443.jsonl` | 断点数量（P1–P4，第 5 个 400）+ 边际效应（M_500–8000）+ 顺序对照（O1_fixed / O2_shuffled）+ 长前缀（OP_10000/20000） | 4 断点上限 / 阈值边际 / 确定性序列化 | +| `batch5_20260908_223934.jsonl` | LONG 20 轮长对话 + WARMUP 预热 + GAIN_LOCKED/DYNAMIC 成本收益 | 7.3× 主数字 / 首轮 11.6× | +| `ttl_20260908_224911.jsonl` / `ttl_20260908_230014.jsonl` | TTL 双档（T1_5min / T2_1h），两次运行互相印证 | 5min 档 400s 失效 | +| `rich_20260908_232421.jsonl` / `rich_20260908_232832.jsonl` / `rich_20260908_233922.jsonl` | 覆盖实验 E20–E29（跨模型隔离 / 流式无损 / schema 变更 / 阈值复测 / 断点位置 / 05B 救援 E27 / E29 埋点对账） | E20–E29 全部结论 | +| `补充2_20260909_142421.jsonl` | E30 并发冷启动 / E31 前缀字节边界 | E30–E31 | +| `补充3_20260909_142829.jsonl` | E32 single-flight / E33 命中延迟差 ~1.65s | E32–E33 | +| `补充4_20260909_143614.jsonl` | E34 opus-5 四档复测（端点异常定性）+ haiku 对照 | E34 / LIMITATIONS #9 | + +> 说明：采集当日另有两个 0 字节空文件（`补充2_20260909_141208/141913.jsonl`，为前两次未完成运行），无数据，未入库。 + +## 字段口径 + +- `cr` / `cache_read` = `cache_read_input_tokens`；`cc` / `cache_creation` = `cache_creation_input_tokens`；`ti` = `input_tokens`（均为上游真实返回）。 +- `hit_rate` = cache_read / (cache_read + input_tokens)，与实验文档定义一致。 +- `batch5` 的 `cost` 为按当时价目计算的美元成本；实验文档中的成本列统一为**相对倍数**，绝对金额以本目录原始记录为准。 diff --git a/data/experiments/batch3_20260908_221208.jsonl b/data/experiments/batch3_20260908_221208.jsonl new file mode 100644 index 0000000..3d01aca --- /dev/null +++ b/data/experiments/batch3_20260908_221208.jsonl @@ -0,0 +1,45 @@ +{"exp": "R1_anthropic", "path": "/messages", "round": 1, "total_in": 2981, "cached": 0, "wrote": 2972, "hit_rate": 0.0} +{"exp": "R1_anthropic", "path": "/messages", "round": 2, "total_in": 2981, "cached": 2972, "wrote": 0, "hit_rate": 0.997} +{"exp": "R1_anthropic", "path": "/messages", "round": 3, "total_in": 2982, "cached": 2972, "wrote": 0, "hit_rate": 0.9966} +{"exp": "R2_chat", "path": "/chat/completions", "round": 1, "total_in": 2981, "cached": 2972, "wrote": null, "hit_rate": 0.997} +{"exp": "R2_chat", "path": "/chat/completions", "round": 2, "total_in": 2981, "cached": 2972, "wrote": null, "hit_rate": 0.997} +{"exp": "R2_chat", "path": "/chat/completions", "round": 3, "total_in": 2982, "cached": 2972, "wrote": null, "hit_rate": 0.9966} +{"exp": "R3_responses", "path": "/responses", "round": 1, "total_in": 2983, "cached": 0, "wrote": null, "hit_rate": 0.0} +{"exp": "R3_responses", "path": "/responses", "round": 2, "total_in": 2983, "cached": 0, "wrote": null, "hit_rate": 0.0} +{"exp": "R3_responses", "path": "/responses", "round": 3, "total_in": 2984, "cached": 0, "wrote": null, "hit_rate": 0.0} +{"exp": "T_haiku-4.5", "model": "anthropic/claude-haiku-4.5", "size": 600, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_sonnet-4.6", "model": "anthropic/claude-sonnet-4.6", "size": 600, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_opus-5", "model": "anthropic/claude-opus-5", "size": 600, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_haiku-4.5", "model": "anthropic/claude-haiku-4.5", "size": 1200, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_sonnet-4.6", "model": "anthropic/claude-sonnet-4.6", "size": 1200, "cached": 1436, "wrote": 1436, "cached_ok": true} +{"exp": "T_opus-5", "model": "anthropic/claude-opus-5", "size": 1200, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_haiku-4.5", "model": "anthropic/claude-haiku-4.5", "size": 2500, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_sonnet-4.6", "model": "anthropic/claude-sonnet-4.6", "size": 2500, "cached": 2972, "wrote": 2972, "cached_ok": true} +{"exp": "T_opus-5", "model": "anthropic/claude-opus-5", "size": 2500, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "T_haiku-4.5", "model": "anthropic/claude-haiku-4.5", "size": 5000, "cached": 5936, "wrote": 5936, "cached_ok": true} +{"exp": "T_sonnet-4.6", "model": "anthropic/claude-sonnet-4.6", "size": 5000, "cached": 5937, "wrote": 5937, "cached_ok": true} +{"exp": "T_opus-5", "model": "anthropic/claude-opus-5", "size": 5000, "cached": 0, "wrote": 0, "cached_ok": false} +{"exp": "N1", "every_n": 1, "round": 1, "version": 0, "cache_read": 0, "cache_creation": 2974, "hit_rate": 0.0} +{"exp": "N1", "every_n": 1, "round": 2, "version": 1, "cache_read": 0, "cache_creation": 2974, "hit_rate": 0.0} +{"exp": "N1", "every_n": 1, "round": 3, "version": 2, "cache_read": 0, "cache_creation": 2974, "hit_rate": 0.0} +{"exp": "N1", "every_n": 1, "round": 4, "version": 3, "cache_read": 0, "cache_creation": 2974, "hit_rate": 0.0} +{"exp": "N1", "every_n": 1, "round": 5, "version": 4, "cache_read": 0, "cache_creation": 2974, "hit_rate": 0.0} +{"exp": "N1", "every_n": 1, "round": 6, "version": 5, "cache_read": 0, "cache_creation": 2974, "hit_rate": 0.0} +{"exp": "N2", "every_n": 2, "round": 1, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N2", "every_n": 2, "round": 2, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N2", "every_n": 2, "round": 3, "version": 1, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.9966} +{"exp": "N2", "every_n": 2, "round": 4, "version": 1, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N2", "every_n": 2, "round": 5, "version": 2, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.9966} +{"exp": "N2", "every_n": 2, "round": 6, "version": 2, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N3", "every_n": 3, "round": 1, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N3", "every_n": 3, "round": 2, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N3", "every_n": 3, "round": 3, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.9966} +{"exp": "N3", "every_n": 3, "round": 4, "version": 1, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N3", "every_n": 3, "round": 5, "version": 1, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.9966} +{"exp": "N3", "every_n": 3, "round": 6, "version": 1, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N5", "every_n": 5, "round": 1, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N5", "every_n": 5, "round": 2, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N5", "every_n": 5, "round": 3, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.9966} +{"exp": "N5", "every_n": 5, "round": 4, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} +{"exp": "N5", "every_n": 5, "round": 5, "version": 0, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.9966} +{"exp": "N5", "every_n": 5, "round": 6, "version": 1, "cache_read": 2974, "cache_creation": 0, "hit_rate": 0.997} diff --git a/data/experiments/batch4_20260908_223443.jsonl b/data/experiments/batch4_20260908_223443.jsonl new file mode 100644 index 0000000..12f95ce --- /dev/null +++ b/data/experiments/batch4_20260908_223443.jsonl @@ -0,0 +1,15 @@ +{"exp": "P1_1bp", "breakpoints": 1, "hit_rate": 0.0, "err": null} +{"exp": "P2_2bp", "breakpoints": 2, "hit_rate": 0.6635, "err": null} +{"exp": "P3_4bp", "breakpoints": 4, "hit_rate": 0.8316, "err": null} +{"exp": "P4_5bp", "breakpoints": 5, "hit_rate": null, "err": "HTTP 400: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"A maximum of 4 blocks with cache_control may be provided. Found 5.\",\"error_type\":\"invalid_req"} +{"exp": "M_500", "size": 500, "cached": 0, "hit_rate": 0.0, "cost": 0.00568} +{"exp": "M_1000", "size": 1000, "cached": 1174, "hit_rate": 0.6616, "cost": 0.00541} +{"exp": "M_2000", "size": 2000, "cached": 2323, "hit_rate": 0.6641, "cost": 0.01041} +{"exp": "M_4000", "size": 4000, "cached": 4628, "hit_rate": 0.6654, "cost": 0.02044} +{"exp": "M_8000", "size": 8000, "cached": 9240, "hit_rate": 0.666, "cost": 0.0405} +{"exp": "O1_fixed", "hit_rate": 0.7476} +{"exp": "O2_shuffled", "hit_rate": 0.2492} +{"exp": "OP_10000", "size": 10000, "round": 1, "cr": 0, "cc": 0, "ti": 0} +{"exp": "OP_10000", "size": 10000, "round": 2, "cr": 0, "cc": 0, "ti": 0} +{"exp": "OP_20000", "size": 20000, "round": 1, "cr": 0, "cc": 0, "ti": 0} +{"exp": "OP_20000", "size": 20000, "round": 2, "cr": 0, "cc": 0, "ti": 0} diff --git a/data/experiments/batch5_20260908_223934.jsonl b/data/experiments/batch5_20260908_223934.jsonl new file mode 100644 index 0000000..2a87803 --- /dev/null +++ b/data/experiments/batch5_20260908_223934.jsonl @@ -0,0 +1,25 @@ +{"exp": "LONG", "round": 1, "cr": 0, "cc": 9241, "ti": 9, "hit_rate": 0.0, "cost": 0.011585} +{"exp": "LONG", "round": 2, "cr": 9241, "cc": 0, "ti": 20, "hit_rate": 0.9978, "cost": 0.000969} +{"exp": "LONG", "round": 3, "cr": 9241, "cc": 0, "ti": 32, "hit_rate": 0.9965, "cost": 0.000986} +{"exp": "LONG", "round": 4, "cr": 9241, "cc": 0, "ti": 44, "hit_rate": 0.9953, "cost": 0.000993} +{"exp": "LONG", "round": 5, "cr": 9241, "cc": 0, "ti": 56, "hit_rate": 0.994, "cost": 0.00101} +{"exp": "LONG", "round": 6, "cr": 9241, "cc": 0, "ti": 68, "hit_rate": 0.9927, "cost": 0.001017} +{"exp": "LONG", "round": 7, "cr": 9241, "cc": 0, "ti": 80, "hit_rate": 0.9914, "cost": 0.001034} +{"exp": "LONG", "round": 8, "cr": 9241, "cc": 0, "ti": 92, "hit_rate": 0.9901, "cost": 0.001041} +{"exp": "LONG", "round": 9, "cr": 9241, "cc": 0, "ti": 104, "hit_rate": 0.9889, "cost": 0.001058} +{"exp": "LONG", "round": 10, "cr": 9241, "cc": 0, "ti": 116, "hit_rate": 0.9876, "cost": 0.001065} +{"exp": "LONG", "round": 11, "cr": 9241, "cc": 0, "ti": 127, "hit_rate": 0.9864, "cost": 0.001076} +{"exp": "LONG", "round": 12, "cr": 9241, "cc": 0, "ti": 138, "hit_rate": 0.9853, "cost": 0.001087} +{"exp": "LONG", "round": 13, "cr": 9241, "cc": 0, "ti": 150, "hit_rate": 0.984, "cost": 0.001104} +{"exp": "LONG", "round": 14, "cr": 9241, "cc": 0, "ti": 162, "hit_rate": 0.9828, "cost": 0.001111} +{"exp": "LONG", "round": 15, "cr": 9241, "cc": 0, "ti": 174, "hit_rate": 0.9815, "cost": 0.001128} +{"exp": "LONG", "round": 16, "cr": 9241, "cc": 0, "ti": 186, "hit_rate": 0.9803, "cost": 0.001135} +{"exp": "LONG", "round": 17, "cr": 9241, "cc": 0, "ti": 198, "hit_rate": 0.979, "cost": 0.001152} +{"exp": "LONG", "round": 18, "cr": 9241, "cc": 0, "ti": 210, "hit_rate": 0.9778, "cost": 0.001159} +{"exp": "LONG", "round": 19, "cr": 9241, "cc": 0, "ti": 222, "hit_rate": 0.9765, "cost": 0.001176} +{"exp": "LONG", "round": 20, "cr": 9241, "cc": 0, "ti": 234, "hit_rate": 0.9753, "cost": 0.001183} +{"exp": "WARMUP", "phase": "warmup", "cr": 0, "cc": 9240, "ti": 8} +{"exp": "WARMUP", "phase": "正式1", "cr": 9240, "cc": 0, "ti": 9, "hit_rate": 0.999} +{"exp": "WARMUP", "phase": "正式2", "cr": 9240, "cc": 0, "ti": 9, "hit_rate": 0.999} +{"exp": "GAIN_LOCKED", "mode": "LOCKED", "rounds": 20, "hit_rate": 0.9372, "total_cost": 0.03207} +{"exp": "GAIN_DYNAMIC", "mode": "DYNAMIC", "rounds": 20, "hit_rate": 0.0, "total_cost": 0.23406} diff --git a/data/experiments/experiment_20260908_213818.jsonl b/data/experiments/experiment_20260908_213818.jsonl new file mode 100644 index 0000000..9afe7c4 --- /dev/null +++ b/data/experiments/experiment_20260908_213818.jsonl @@ -0,0 +1,18 @@ +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_213818", "input_tokens": 17, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000126} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_213818", "input_tokens": 28, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000159} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_213818", "input_tokens": 40, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.00021} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_213818", "input_tokens": 954, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.002937} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_213818", "input_tokens": 965, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.00297} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_213818", "input_tokens": 977, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.003021} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_213818", "input_tokens": 255, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.00084} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_213818", "input_tokens": 266, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000873} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_213818", "input_tokens": 278, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.000924} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_213818", "input_tokens": 9, "cache_creation": 1627, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.006203} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_213818", "input_tokens": 20, "cache_creation": 0, "cache_read": 1627, "output_tokens": 5, "hit_rate": 0.9879, "cost_usd": 0.000623} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_213818", "input_tokens": 32, "cache_creation": 0, "cache_read": 1627, "output_tokens": 6, "hit_rate": 0.9807, "cost_usd": 0.000674} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_213818", "input_tokens": 9, "cache_creation": 1604, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.006117} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_213818", "input_tokens": 20, "cache_creation": 0, "cache_read": 1604, "output_tokens": 5, "hit_rate": 0.9877, "cost_usd": 0.000616} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_213818", "input_tokens": 32, "cache_creation": 0, "cache_read": 1604, "output_tokens": 6, "hit_rate": 0.9804, "cost_usd": 0.000667} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_213818", "input_tokens": 9, "cache_creation": 2948, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011157} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_213818", "input_tokens": 20, "cache_creation": 0, "cache_read": 2948, "output_tokens": 5, "hit_rate": 0.9933, "cost_usd": 0.001019} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_213818", "input_tokens": 32, "cache_creation": 0, "cache_read": 2948, "output_tokens": 6, "hit_rate": 0.9893, "cost_usd": 0.00107} diff --git a/data/experiments/experiment_20260908_214354.jsonl b/data/experiments/experiment_20260908_214354.jsonl new file mode 100644 index 0000000..2e76406 --- /dev/null +++ b/data/experiments/experiment_20260908_214354.jsonl @@ -0,0 +1,27 @@ +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214354", "input_tokens": 17, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000126} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214354", "input_tokens": 28, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000159} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214354", "input_tokens": 40, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.00021} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214354", "input_tokens": 9, "cache_creation": 2925, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011071} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214354", "input_tokens": 20, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9932, "cost_usd": 0.001012} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214354", "input_tokens": 32, "cache_creation": 0, "cache_read": 2925, "output_tokens": 6, "hit_rate": 0.9892, "cost_usd": 0.001063} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214354", "input_tokens": 9, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.01109} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214354", "input_tokens": 20, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011123} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214354", "input_tokens": 32, "cache_creation": 2930, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.011174} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214354", "input_tokens": 9, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9969, "cost_usd": 0.00098} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214354", "input_tokens": 20, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9932, "cost_usd": 0.001012} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214354", "input_tokens": 32, "cache_creation": 0, "cache_read": 2925, "output_tokens": 6, "hit_rate": 0.9892, "cost_usd": 0.001063} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214354", "input_tokens": 3, "cache_creation": 2931, "cache_read": 0, "output_tokens": 4, "hit_rate": 0.0, "cost_usd": 0.01106} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214354", "input_tokens": 3, "cache_creation": 2941, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011113} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214354", "input_tokens": 3, "cache_creation": 2953, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.011173} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214354", "input_tokens": 372, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001191} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214354", "input_tokens": 383, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001224} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214354", "input_tokens": 395, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.001275} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214354", "input_tokens": 9, "cache_creation": 2427, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.009203} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214354", "input_tokens": 20, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9918, "cost_usd": 0.000863} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214354", "input_tokens": 32, "cache_creation": 0, "cache_read": 2427, "output_tokens": 6, "hit_rate": 0.987, "cost_usd": 0.000914} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214354", "input_tokens": 9, "cache_creation": 2397, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.009091} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214354", "input_tokens": 20, "cache_creation": 0, "cache_read": 2397, "output_tokens": 5, "hit_rate": 0.9917, "cost_usd": 0.000854} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214354", "input_tokens": 32, "cache_creation": 0, "cache_read": 2397, "output_tokens": 6, "hit_rate": 0.9868, "cost_usd": 0.000905} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214354", "input_tokens": 9, "cache_creation": 4357, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.016441} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214354", "input_tokens": 20, "cache_creation": 0, "cache_read": 4357, "output_tokens": 5, "hit_rate": 0.9954, "cost_usd": 0.001442} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214354", "input_tokens": 32, "cache_creation": 0, "cache_read": 4357, "output_tokens": 6, "hit_rate": 0.9927, "cost_usd": 0.001493} diff --git a/data/experiments/experiment_20260908_214628.jsonl b/data/experiments/experiment_20260908_214628.jsonl new file mode 100644 index 0000000..22e11a4 --- /dev/null +++ b/data/experiments/experiment_20260908_214628.jsonl @@ -0,0 +1,72 @@ +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 17, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000126} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 28, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000159} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 40, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.00021} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 52, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000231} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 63, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000264} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 75, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.000315} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 87, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000336} +{"group": "1-更新频率", "config": "A_baseline", "desc": "不注入记忆（基线）", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "A_baseline-20260908_214628", "input_tokens": 98, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.000369} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 9, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9969, "cost_usd": 0.00098} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 20, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9932, "cost_usd": 0.001012} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 32, "cache_creation": 0, "cache_read": 2925, "output_tokens": 6, "hit_rate": 0.9892, "cost_usd": 0.001063} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 44, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9852, "cost_usd": 0.001084} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 55, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9815, "cost_usd": 0.001118} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 67, "cache_creation": 0, "cache_read": 2925, "output_tokens": 6, "hit_rate": 0.9776, "cost_usd": 0.001169} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 79, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9737, "cost_usd": 0.00119} +{"group": "1-更新频率", "config": "B_locked", "desc": "会话开始注入一次并锁死", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "B_locked-20260908_214628", "input_tokens": 90, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9701, "cost_usd": 0.001223} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 9, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.01109} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 20, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011123} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 32, "cache_creation": 2930, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.011174} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 44, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011195} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 55, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011228} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 67, "cache_creation": 2930, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.011279} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 79, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.0113} +{"group": "1-更新频率", "config": "C_dynamic", "desc": "每轮动态注入（前缀每轮变）", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "C_dynamic-20260908_214628", "input_tokens": 90, "cache_creation": 2930, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.011333} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 9, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9969, "cost_usd": 0.00098} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 20, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9932, "cost_usd": 0.001012} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 32, "cache_creation": 0, "cache_read": 2925, "output_tokens": 6, "hit_rate": 0.9892, "cost_usd": 0.001063} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 44, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9852, "cost_usd": 0.001084} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 55, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9815, "cost_usd": 0.001118} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 67, "cache_creation": 0, "cache_read": 2925, "output_tokens": 6, "hit_rate": 0.9776, "cost_usd": 0.001169} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 79, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9737, "cost_usd": 0.00119} +{"group": "2-位置", "config": "D1_head", "desc": "记忆放前缀头部(system)", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "D1_head-20260908_214628", "input_tokens": 90, "cache_creation": 0, "cache_read": 2925, "output_tokens": 5, "hit_rate": 0.9701, "cost_usd": 0.001223} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 0, "cache_read": 2931, "output_tokens": 4, "hit_rate": 0.999, "cost_usd": 0.000948} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 0, "cache_read": 2941, "output_tokens": 4, "hit_rate": 0.999, "cost_usd": 0.000951} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 2952, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.011169} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 2964, "cache_read": 0, "output_tokens": 4, "hit_rate": 0.0, "cost_usd": 0.011184} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 2974, "cache_read": 0, "output_tokens": 4, "hit_rate": 0.0, "cost_usd": 0.011221} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 2985, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.011293} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 2997, "cache_read": 0, "output_tokens": 4, "hit_rate": 0.0, "cost_usd": 0.011308} +{"group": "2-位置", "config": "D2_tail", "desc": "记忆放消息流尾部(append)", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "D2_tail-20260908_214628", "input_tokens": 3, "cache_creation": 3007, "cache_read": 0, "output_tokens": 4, "hit_rate": 0.0, "cost_usd": 0.011345} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 372, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001191} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 383, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001224} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 395, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.001275} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 407, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001296} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 418, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001329} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 430, "cache_creation": 0, "cache_read": 0, "output_tokens": 6, "hit_rate": 0.0, "cost_usd": 0.00138} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 442, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001401} +{"group": "3-粒度", "config": "G1_300", "desc": "单块~300token(低于阈值)", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "G1_300-20260908_214628", "input_tokens": 453, "cache_creation": 0, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.001434} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 9, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9963, "cost_usd": 0.00083} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 20, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9918, "cost_usd": 0.000863} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 32, "cache_creation": 0, "cache_read": 2427, "output_tokens": 6, "hit_rate": 0.987, "cost_usd": 0.000914} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 44, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9822, "cost_usd": 0.000935} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 55, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9778, "cost_usd": 0.000968} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 67, "cache_creation": 0, "cache_read": 2427, "output_tokens": 6, "hit_rate": 0.9731, "cost_usd": 0.001019} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 79, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9685, "cost_usd": 0.00104} +{"group": "3-粒度", "config": "G2_4x512", "desc": "4条×512token 多断点", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "G2_4x512-20260908_214628", "input_tokens": 90, "cache_creation": 0, "cache_read": 2427, "output_tokens": 5, "hit_rate": 0.9642, "cost_usd": 0.001073} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 9, "cache_creation": 2397, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.009091} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 20, "cache_creation": 0, "cache_read": 2397, "output_tokens": 5, "hit_rate": 0.9917, "cost_usd": 0.000854} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 32, "cache_creation": 0, "cache_read": 2397, "output_tokens": 6, "hit_rate": 0.9868, "cost_usd": 0.000905} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 44, "cache_creation": 0, "cache_read": 2397, "output_tokens": 5, "hit_rate": 0.982, "cost_usd": 0.000926} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 55, "cache_creation": 0, "cache_read": 2397, "output_tokens": 5, "hit_rate": 0.9776, "cost_usd": 0.000959} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 67, "cache_creation": 0, "cache_read": 2397, "output_tokens": 6, "hit_rate": 0.9728, "cost_usd": 0.00101} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 79, "cache_creation": 0, "cache_read": 2397, "output_tokens": 5, "hit_rate": 0.9681, "cost_usd": 0.001031} +{"group": "3-粒度", "config": "G3_2048", "desc": "单块2048token 大块", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "G3_2048-20260908_214628", "input_tokens": 90, "cache_creation": 0, "cache_read": 2397, "output_tokens": 5, "hit_rate": 0.9638, "cost_usd": 0.001064} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 1, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 9, "cache_creation": 4357, "cache_read": 0, "output_tokens": 5, "hit_rate": 0.0, "cost_usd": 0.016441} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 2, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 20, "cache_creation": 0, "cache_read": 4357, "output_tokens": 5, "hit_rate": 0.9954, "cost_usd": 0.001442} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 3, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 32, "cache_creation": 0, "cache_read": 4357, "output_tokens": 6, "hit_rate": 0.9927, "cost_usd": 0.001493} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 4, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 44, "cache_creation": 0, "cache_read": 4357, "output_tokens": 5, "hit_rate": 0.99, "cost_usd": 0.001514} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 5, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 55, "cache_creation": 0, "cache_read": 4357, "output_tokens": 5, "hit_rate": 0.9875, "cost_usd": 0.001547} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 6, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 67, "cache_creation": 0, "cache_read": 4357, "output_tokens": 6, "hit_rate": 0.9849, "cost_usd": 0.001598} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 7, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 79, "cache_creation": 0, "cache_read": 4357, "output_tokens": 5, "hit_rate": 0.9822, "cost_usd": 0.001619} +{"group": "3-粒度", "config": "G4_30frag", "desc": "30条细粒度(20-block窗口)", "round": 8, "model": "anthropic/claude-sonnet-4.6", "session_id": "G4_30frag-20260908_214628", "input_tokens": 90, "cache_creation": 0, "cache_read": 4357, "output_tokens": 5, "hit_rate": 0.9798, "cost_usd": 0.001652} diff --git a/data/experiments/rich_20260908_232421.jsonl b/data/experiments/rich_20260908_232421.jsonl new file mode 100644 index 0000000..6c8b2e5 --- /dev/null +++ b/data/experiments/rich_20260908_232421.jsonl @@ -0,0 +1,6 @@ +{"exp": "E20", "case": "haiku_build", "cr": 0, "cc": 5943, "ti": 9} +{"exp": "E20", "case": "sonnet_read", "cr": 0, "cc": 5944, "ti": 9} +{"exp": "E20", "case": "sonnet_build", "cr": 0, "cc": 5941, "ti": 9} +{"exp": "E20", "case": "haiku_read", "cr": 0, "cc": 5940, "ti": 9} +{"exp": "E25", "case": "build", "cr": 0, "cc": 5942, "ti": 9} +{"exp": "E29", "case": "method", "status": "待网关联调"} diff --git a/data/experiments/rich_20260908_232832.jsonl b/data/experiments/rich_20260908_232832.jsonl new file mode 100644 index 0000000..d07430b --- /dev/null +++ b/data/experiments/rich_20260908_232832.jsonl @@ -0,0 +1,38 @@ +{"exp": "E20", "case": "haiku_build", "cr": 0, "cc": 5942, "ti": 9} +{"exp": "E20", "case": "sonnet_read", "cr": 0, "cc": 5943, "ti": 9} +{"exp": "E20", "case": "sonnet_build", "cr": 0, "cc": 5941, "ti": 9} +{"exp": "E20", "case": "haiku_read", "cr": 0, "cc": 5940, "ti": 9} +{"exp": "E22", "case": "build", "cr": 0, "cc": 4993, "ti": 325} +{"exp": "E22", "case": "same_tool_read", "cr": 4993, "cc": 0, "ti": 325, "hit": 0.9389} +{"exp": "E22", "case": "renamed_tool", "cr": 0, "cc": 4995, "ti": 325, "hit": 0.0} +{"exp": "E23", "case": "tok1024", "cr": 0, "cc": 0, "ti": 1237, "hit": 0.0} +{"exp": "E23", "case": "tok2048", "cr": 0, "cc": 0, "ti": 2453, "hit": 0.0} +{"exp": "E23", "case": "tok3000", "cr": 0, "cc": 0, "ti": 3581, "hit": 0.0} +{"exp": "E23", "case": "tok3500", "cr": 4162, "cc": 0, "ti": 9, "hit": 0.9978} +{"exp": "E23", "case": "tok4096", "cr": 4869, "cc": 0, "ti": 9, "hit": 0.9982} +{"exp": "E23", "case": "tok4500", "cr": 5348, "cc": 0, "ti": 9, "hit": 0.9983} +{"exp": "E23", "case": "tok5000", "cr": 5942, "cc": 0, "ti": 9, "hit": 0.9985} +{"exp": "E23", "case": "tok6000", "cr": 7127, "cc": 0, "ti": 9, "hit": 0.9987} +{"exp": "E24", "case": "sys_bp", "cr": 0, "cc": 5941, "ti": 9, "hit": 0.0} +{"exp": "E24", "case": "sys_bp", "cr": 5941, "cc": 0, "ti": 9, "hit": 0.9985} +{"exp": "E24", "case": "user_bp", "cr": 0, "cc": 5946, "ti": 3, "hit": 0.0} +{"exp": "E24", "case": "user_bp", "cr": 5946, "cc": 0, "ti": 3, "hit": 0.9995} +{"exp": "E25", "case": "build", "cr": 0, "cc": 5941, "ti": 9} +{"exp": "E25", "case": "concurrent0", "cr": 5941, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E25", "case": "concurrent1", "cr": 5941, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E25", "case": "concurrent2", "cr": 5941, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E26", "case": "round1", "cr": 0, "cc": 5940, "ti": 82, "hit": 0.0} +{"exp": "E26", "case": "round2", "cr": 5940, "cc": 0, "ti": 82, "hit": 0.9864} +{"exp": "E26", "case": "round3", "cr": 5940, "cc": 0, "ti": 78, "hit": 0.987} +{"exp": "E26", "case": "round4", "cr": 5940, "cc": 0, "ti": 81, "hit": 0.9865} +{"exp": "E26", "case": "round5", "cr": 5940, "cc": 0, "ti": 80, "hit": 0.9867} +{"exp": "E26", "case": "round6", "cr": 5940, "cc": 0, "ti": 80, "hit": 0.9867} +{"exp": "E27", "case": "responses_backend", "cr": 0, "cc": 0, "ti": 0} +{"exp": "E27", "case": "messages_backend_build", "cr": 0, "cc": 5942, "ti": 9} +{"exp": "E27", "case": "messages_backend_read", "cr": 5942, "cc": 0, "ti": 9, "hit": 0.9985} +{"exp": "E28", "case": "LOCKED_r1", "cr": 0, "cc": 5941, "ti": 8, "hit": 0.0} +{"exp": "E28", "case": "LOCKED_r20", "cr": 5941, "cc": 0, "ti": 8, "hit": 0.9987} +{"exp": "E28", "case": "DYNAMIC_r1", "cr": 0, "cc": 5947, "ti": 8, "hit": 0.0} +{"exp": "E28", "case": "DYNAMIC_r20", "cr": 0, "cc": 5950, "ti": 8, "hit": 0.0} +{"exp": "E28", "case": "summary", "locked_hit": 0.9488, "dynamic_hit": 0.0, "locked_cost": 0.020474, "dynamic_cost": 0.150488, "multiplier": 7.35} +{"exp": "E29", "case": "method", "status": "待网关联调"} diff --git a/data/experiments/rich_20260908_233922.jsonl b/data/experiments/rich_20260908_233922.jsonl new file mode 100644 index 0000000..ee4dfc1 --- /dev/null +++ b/data/experiments/rich_20260908_233922.jsonl @@ -0,0 +1,4 @@ +{"exp": "E21", "case": "nonstream_build", "cr": 0, "cc": 5942, "ti": 9} +{"exp": "E21", "case": "nonstream_read", "cr": 5942, "cc": 0, "ti": 9, "hit": 0.9985} +{"exp": "E21", "case": "stream_build", "cr": 5942, "cc": 0, "ti": 9} +{"exp": "E21", "case": "stream_read", "cr": 5942, "cc": 0, "ti": 9, "hit": 0.9985} diff --git a/data/experiments/summary_20260908_213818.csv b/data/experiments/summary_20260908_213818.csv new file mode 100644 index 0000000..6c0f3b8 --- /dev/null +++ b/data/experiments/summary_20260908_213818.csv @@ -0,0 +1,7 @@ +group,config,desc,rounds,steady_hit_rate,cache_created_r1,cache_read_last,avg_input,cost_usd +1-更新频率,A_baseline,不注入记忆（基线）,3,0.0,0,0,28,0.00049 +1-更新频率,B_locked,会话开始注入一次并锁死,3,0.0,0,0,965,0.00893 +3-粒度,G1_300,单块~300token(低于阈值),3,0.0,0,0,266,0.00264 +3-粒度,G2_4x512,4条×512token 多断点,3,0.9843,1627,1627,20,0.0075 +3-粒度,G3_2048,单块2048token 大块,3,0.984,1604,1604,20,0.0074 +3-粒度,G4_30frag,30条细粒度(20-block窗口),3,0.9913,2948,2948,20,0.01325 diff --git a/data/experiments/summary_20260908_214354.csv b/data/experiments/summary_20260908_214354.csv new file mode 100644 index 0000000..81f74cd --- /dev/null +++ b/data/experiments/summary_20260908_214354.csv @@ -0,0 +1,10 @@ +group,config,desc,rounds,steady_hit_rate,cache_created_r1,cache_read_last,avg_input,cost_usd +1-更新频率,A_baseline,不注入记忆（基线）,3,0.0,0,0,28,0.00049 +1-更新频率,B_locked,会话开始注入一次并锁死,3,0.9912,2925,2925,20,0.01315 +1-更新频率,C_dynamic,每轮动态注入（前缀每轮变）,3,0.0,2930,0,20,0.03339 +2-位置,D1_head,记忆放前缀头部(system),3,0.9912,0,2925,20,0.00305 +2-位置,D2_tail,记忆放消息流尾部(append),3,0.0,2931,0,3,0.03335 +3-粒度,G1_300,单块~300token(低于阈值),3,0.0,0,0,383,0.00369 +3-粒度,G2_4x512,4条×512token 多断点,3,0.9894,2427,2427,20,0.01098 +3-粒度,G3_2048,单块2048token 大块,3,0.9893,2397,2397,20,0.01085 +3-粒度,G4_30frag,30条细粒度(20-block窗口),3,0.9941,4357,4357,20,0.01938 diff --git a/data/experiments/summary_20260908_214628.csv b/data/experiments/summary_20260908_214628.csv new file mode 100644 index 0000000..788fd69 --- /dev/null +++ b/data/experiments/summary_20260908_214628.csv @@ -0,0 +1,10 @@ +group,config,desc,rounds,steady_hit_rate,cache_created_r1,cache_read_last,avg_input,cost_usd +1-更新频率,A_baseline,不注入记忆（基线）,8,0.0,0,0,58,0.00201 +1-更新频率,B_locked,会话开始注入一次并锁死,8,0.9814,0,2925,50,0.00884 +1-更新频率,C_dynamic,每轮动态注入（前缀每轮变）,8,0.0,2930,0,50,0.08972 +2-位置,D1_head,记忆放前缀头部(system),8,0.9814,0,2925,50,0.00884 +2-位置,D2_tail,记忆放消息流尾部(append),8,0.1411,0,0,3,0.06942 +3-粒度,G1_300,单块~300token(低于阈值),8,0.0,0,0,412,0.01053 +3-粒度,G2_4x512,4条×512token 多断点,8,0.9777,0,2427,50,0.00764 +3-粒度,G3_2048,单块2048token 大块,8,0.9775,2397,2397,50,0.01584 +3-粒度,G4_30frag,30条细粒度(20-block窗口),8,0.9875,4357,4357,50,0.02731 diff --git a/data/experiments/supplement_20260908_220106.jsonl b/data/experiments/supplement_20260908_220106.jsonl new file mode 100644 index 0000000..5f34559 --- /dev/null +++ b/data/experiments/supplement_20260908_220106.jsonl @@ -0,0 +1,17 @@ +{"exp": "E1_anthropic", "path": "/v1/messages", "round": 1, "input_tokens": 9, "cache_read": 0, "cache_creation": 2972, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "E1_anthropic", "path": "/v1/messages", "round": 2, "input_tokens": 20, "cache_read": 2972, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9933} +{"exp": "E1_anthropic", "path": "/v1/messages", "round": 3, "input_tokens": 32, "cache_read": 2972, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9893} +{"exp": "E1_anthropic", "path": "/v1/messages", "round": 4, "input_tokens": 44, "cache_read": 2972, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9854} +{"exp": "E2_openai", "path": "/v1/chat/completions", "round": 1, "input_tokens": null, "cache_read": null, "cache_creation": null, "prompt_tokens": 2981, "cached_tokens": 2972, "hit_rate": 0.997} +{"exp": "E2_openai", "path": "/v1/chat/completions", "round": 2, "input_tokens": null, "cache_read": null, "cache_creation": null, "prompt_tokens": 2992, "cached_tokens": 2972, "hit_rate": 0.9933} +{"exp": "E2_openai", "path": "/v1/chat/completions", "round": 3, "input_tokens": null, "cache_read": null, "cache_creation": null, "prompt_tokens": 3004, "cached_tokens": 2972, "hit_rate": 0.9893} +{"exp": "E2_openai", "path": "/v1/chat/completions", "round": 4, "input_tokens": null, "cache_read": null, "cache_creation": null, "prompt_tokens": 3016, "cached_tokens": 2972, "hit_rate": 0.9854} +{"exp": "H_10blk", "path": "/v1/messages", "round": 1, "n_blocks": 10, "input_tokens": 3, "cache_read": 0, "cache_creation": 1822, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_10blk", "path": "/v1/messages", "round": 2, "n_blocks": 10, "input_tokens": 3, "cache_read": 0, "cache_creation": 1852, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_10blk", "path": "/v1/messages", "round": 3, "n_blocks": 10, "input_tokens": 3, "cache_read": 0, "cache_creation": 1882, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_30blk", "path": "/v1/messages", "round": 1, "n_blocks": 30, "input_tokens": 3, "cache_read": 1821, "cache_creation": 3621, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.3344} +{"exp": "H_30blk", "path": "/v1/messages", "round": 2, "n_blocks": 30, "input_tokens": 3, "cache_read": 0, "cache_creation": 5472, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_30blk", "path": "/v1/messages", "round": 3, "n_blocks": 30, "input_tokens": 3, "cache_read": 0, "cache_creation": 5502, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_60blk", "path": "/v1/messages", "round": 1, "n_blocks": 60, "input_tokens": 3, "cache_read": 0, "cache_creation": 10872, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_60blk", "path": "/v1/messages", "round": 2, "n_blocks": 60, "input_tokens": 3, "cache_read": 0, "cache_creation": 10902, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_60blk", "path": "/v1/messages", "round": 3, "n_blocks": 60, "input_tokens": 3, "cache_read": 0, "cache_creation": 10932, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} diff --git a/data/experiments/supplement_20260908_220353.jsonl b/data/experiments/supplement_20260908_220353.jsonl new file mode 100644 index 0000000..b1a27b2 --- /dev/null +++ b/data/experiments/supplement_20260908_220353.jsonl @@ -0,0 +1,12 @@ +{"exp": "H_5blk", "path": "/v1/messages", "round": 1, "n_blocks": 5, "input_tokens": 920, "cache_read": 0, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_5blk", "path": "/v1/messages", "round": 2, "n_blocks": 5, "input_tokens": 920, "cache_read": 0, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_5blk", "path": "/v1/messages", "round": 3, "n_blocks": 5, "input_tokens": 920, "cache_read": 0, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.0} +{"exp": "H_10blk", "path": "/v1/messages", "round": 1, "n_blocks": 10, "input_tokens": 3, "cache_read": 1822, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9984} +{"exp": "H_10blk", "path": "/v1/messages", "round": 2, "n_blocks": 10, "input_tokens": 3, "cache_read": 1822, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9984} +{"exp": "H_10blk", "path": "/v1/messages", "round": 3, "n_blocks": 10, "input_tokens": 3, "cache_read": 1822, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9984} +{"exp": "H_30blk", "path": "/v1/messages", "round": 1, "n_blocks": 30, "input_tokens": 3, "cache_read": 5442, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9994} +{"exp": "H_30blk", "path": "/v1/messages", "round": 2, "n_blocks": 30, "input_tokens": 3, "cache_read": 5442, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9994} +{"exp": "H_30blk", "path": "/v1/messages", "round": 3, "n_blocks": 30, "input_tokens": 3, "cache_read": 5442, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9994} +{"exp": "H_60blk", "path": "/v1/messages", "round": 1, "n_blocks": 60, "input_tokens": 3, "cache_read": 10872, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9997} +{"exp": "H_60blk", "path": "/v1/messages", "round": 2, "n_blocks": 60, "input_tokens": 3, "cache_read": 10872, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9997} +{"exp": "H_60blk", "path": "/v1/messages", "round": 3, "n_blocks": 60, "input_tokens": 3, "cache_read": 10872, "cache_creation": 0, "prompt_tokens": null, "cached_tokens": null, "hit_rate": 0.9997} diff --git a/data/experiments/ttl_20260908_224911.jsonl b/data/experiments/ttl_20260908_224911.jsonl new file mode 100644 index 0000000..246034c --- /dev/null +++ b/data/experiments/ttl_20260908_224911.jsonl @@ -0,0 +1,6 @@ +{"exp": "T1_5min", "phase": "建立", "cr": 0, "cc": 9495, "ti": 9} +{"exp": "T1_5min", "phase": "立即重发", "cr": 9495, "cc": 0, "ti": 9} +{"exp": "T2_1h", "phase": "建立", "cr": 9495, "cc": 0, "ti": 9} +{"exp": "T2_1h", "phase": "立即重发", "cr": 9495, "cc": 0, "ti": 9} +{"exp": "T1_5min", "phase": "等待400秒后", "cr": 9495, "cc": 0, "ti": 9, "verdict": "仍命中 ✅（1h 档生效）"} +{"exp": "T2_1h", "phase": "等待400秒后", "cr": 9495, "cc": 0, "ti": 9, "verdict": "仍命中 ✅（1h 档生效）"} diff --git a/data/experiments/ttl_20260908_230014.jsonl b/data/experiments/ttl_20260908_230014.jsonl new file mode 100644 index 0000000..5ad9be2 --- /dev/null +++ b/data/experiments/ttl_20260908_230014.jsonl @@ -0,0 +1,6 @@ +{"exp": "T1_5min", "phase": "建立", "cr": 0, "cc": 9493, "ti": 9} +{"exp": "T1_5min", "phase": "立即重发", "cr": 9493, "cc": 0, "ti": 9} +{"exp": "T2_1h", "phase": "建立", "cr": 0, "cc": 9495, "ti": 9} +{"exp": "T2_1h", "phase": "立即重发", "cr": 9495, "cc": 0, "ti": 9} +{"exp": "T1_5min", "phase": "等待400秒后", "cr": 0, "cc": 9493, "ti": 9, "verdict": "已失效，重新写入 ❌（5min 档过期）"} +{"exp": "T2_1h", "phase": "等待400秒后", "cr": 9495, "cc": 0, "ti": 9, "verdict": "仍命中 ✅（1h 档生效）"} diff --git "a/data/experiments/\350\241\245\345\205\2052_20260909_142421.jsonl" "b/data/experiments/\350\241\245\345\205\2052_20260909_142421.jsonl" new file mode 100644 index 0000000..cb68af5 --- /dev/null +++ "b/data/experiments/\350\241\245\345\205\2052_20260909_142421.jsonl" @@ -0,0 +1,24 @@ +{"exp": "E30", "case": "N1_req0", "cr": 0, "cc": 5941, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N3_req0", "cr": 5944, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N3_req1", "cr": 0, "cc": 5944, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N3_req2", "cr": 0, "cc": 5944, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N6_req0", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req1", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req2", "cr": 0, "cc": 5942, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N6_req3", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req4", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req5", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N1_req0", "cr": 0, "cc": 5941, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N3_req0", "cr": 5944, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N3_req1", "cr": 0, "cc": 5944, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N3_req2", "cr": 0, "cc": 5944, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N6_req0", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req1", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req2", "cr": 0, "cc": 5942, "ti": 10, "hit": 0.0} +{"exp": "E30", "case": "N6_req3", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req4", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E30", "case": "N6_req5", "cr": 5942, "cc": 0, "ti": 10, "hit": 0.9983} +{"exp": "E31", "case": "A_build", "cr": 0, "cc": 5944, "ti": 7} +{"exp": "E31", "case": "A_same", "cr": 5944, "cc": 0, "ti": 7, "hit": 0.9988} +{"exp": "E31", "case": "B_mutate1", "cr": 0, "cc": 5944, "ti": 7, "hit": 0.0} +{"exp": "E31", "case": "C_tail_change", "cr": 5944, "cc": 0, "ti": 9, "hit": 0.9985} diff --git "a/data/experiments/\350\241\245\345\205\2053_20260909_142829.jsonl" "b/data/experiments/\350\241\245\345\205\2053_20260909_142829.jsonl" new file mode 100644 index 0000000..3772357 --- /dev/null +++ "b/data/experiments/\350\241\245\345\205\2053_20260909_142829.jsonl" @@ -0,0 +1,19 @@ +{"exp": "E32", "case": "N3_r1", "cc": 3, "cr": 0} +{"exp": "E32", "case": "N3_r2", "cc": 3, "cr": 0} +{"exp": "E32", "case": "N3_r3", "cc": 1, "cr": 2} +{"exp": "E32", "case": "N3_r4", "cc": 2, "cr": 1} +{"exp": "E32", "case": "N6_r1", "cc": 1, "cr": 5} +{"exp": "E32", "case": "N6_r2", "cc": 5, "cr": 1} +{"exp": "E32", "case": "N6_r3", "cc": 4, "cr": 2} +{"exp": "E32", "case": "N6_r4", "cc": 2, "cr": 4} +{"exp": "E32", "case": "N3_r1", "cc": 3, "cr": 0} +{"exp": "E32", "case": "N3_r2", "cc": 3, "cr": 0} +{"exp": "E32", "case": "N3_r3", "cc": 1, "cr": 2} +{"exp": "E32", "case": "N3_r4", "cc": 2, "cr": 1} +{"exp": "E32", "case": "N6_r1", "cc": 1, "cr": 5} +{"exp": "E32", "case": "N6_r2", "cc": 5, "cr": 1} +{"exp": "E32", "case": "N6_r3", "cc": 4, "cr": 2} +{"exp": "E32", "case": "N6_r4", "cc": 2, "cr": 4} +{"exp": "E33", "case": "hit_times", "values": [2.98, 3.096, 6.642, 2.769, 3.152]} +{"exp": "E33", "case": "cold_times", "values": [5.246, 3.193, 5.711, 2.62, 4.747]} +{"exp": "E33", "case": "summary", "hit_median": 3.096, "cold_median": 4.747, "hit_mean": 3.728, "cold_mean": 4.303} diff --git "a/data/experiments/\350\241\245\345\205\2054_20260909_143614.jsonl" "b/data/experiments/\350\241\245\345\205\2054_20260909_143614.jsonl" new file mode 100644 index 0000000..91a3b87 --- /dev/null +++ "b/data/experiments/\350\241\245\345\205\2054_20260909_143614.jsonl" @@ -0,0 +1,6 @@ +{"exp": "E34_opus", "case": "tok8000", "model": "anthropic/claude-opus-5", "actual_tokens": 0, "build_cc": 0, "reread_cr": 0} +{"exp": "E34_opus", "case": "tok12000", "model": "anthropic/claude-opus-5", "actual_tokens": 0, "build_cc": 0, "reread_cr": 0} +{"exp": "E34_opus", "case": "tok16000", "model": "anthropic/claude-opus-5", "actual_tokens": 0, "build_cc": 0, "reread_cr": 0} +{"exp": "E34_opus", "case": "tok20000", "model": "anthropic/claude-opus-5", "actual_tokens": 0, "build_cc": 0, "reread_cr": 0} +{"exp": "E34_haiku", "case": "tok10000", "model": "anthropic/claude-haiku-4.5", "actual_tokens": 7, "build_cc": 11863, "reread_cr": 11863} +{"exp": "E34_haiku", "case": "tok20000", "model": "anthropic/claude-haiku-4.5", "actual_tokens": 7, "build_cc": 23711, "reread_cr": 23711} diff --git a/docs/experiment-results.md b/docs/experiment-results.md index 5b82739..cec9452 100644 --- a/docs/experiment-results.md +++ b/docs/experiment-results.md @@ -3,6 +3,7 @@ > TRACK 05A/05B 协议转换组 · 实验实测报告（34 组） > 数据日期 2026-09-08 / 09-09 · 配套脚本见 `experiments/`（三组对照 + `rich_experiments.py` 覆盖实验 E20–E28 + 补充实验 E30–E34） > 实验在 **Anthropic Messages API 兼容端点** 上真实调用完成，命中率字段（`cache_creation_input_tokens` / `cache_read_input_tokens`）为真实返回。 +> **原始数据已入库**：逐轮 jsonl + 汇总 csv 见 `data/experiments/`，文件 → 实验编号溯源表见 `data/README.md`——本文档每个数字均可回查到具体文件的具体行。 ---