From d05b3df24047e9a465c3b4faa926e629f33ed4ee Mon Sep 17 00:00:00 2001 From: Jeremy Liang Date: Sat, 17 Oct 2026 10:04:43 +0800 Subject: [PATCH 1/2] feat: optionally stagger periodic usage probes after 5h reset --- CHANGELOG.md | 1 + docs/API.md | 20 +++ src/lib/oauth/usage-probe-monitor.mjs | 38 +++++ test/unit/usage-probe-reset-delay.test.mjs | 183 +++++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 test/unit/usage-probe-reset-delay.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c115d28b..c6617e7a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - 远程 Claude Docker 槽改用镜像烘焙 kernel 入口（`/opt/kin/kin-kernel`，`REMOTE_KERNEL_ENTRY`），不改 GPT 与本机/KVM 命令；新建或重建容器后生效。（#489、#496） - 修复 Codex 返回配额耗尽（如 `usage_limit_reached`）且无利用率响应头时仍展示过期低用量：作废该槽未达上限窗口的过期利用率缓存，面板显示为未测量（`—`），保留新鲜测量与已封顶窗口；通用限流与成功无头响应行为不变。（#474、#487） - `docker/kin-kvm/fetch-cloud.py` 在调用 `qemu-img` 前校验源路径为常规文件且非符号链接，收紧路径处理（CWE-78 防御加深）。（#471） +- 用量探测可选 `usage_probe.reset_delay_sec` / `reset_jitter_sec`（各 0–3600，默认 0）：在已知 Extra 5h reset 后，将已到期的周期探测推迟到 `reset + delay + 按账号稳定偏移`；不改官方退避、手动/事件探测与 Messages 健康监视；快照暴露待探测的 `next_probe_at`。（#488、#497） ## 1.3.181 — 2026-10-17 diff --git a/docs/API.md b/docs/API.md index 1ae9b5c32..f2a364e62 100644 --- a/docs/API.md +++ b/docs/API.md @@ -199,6 +199,26 @@ curl -sS http://127.0.0.1:8787/v1/usage -H "Authorization: Bearer $KEY" +### 周期额度查询的重置后错峰 + +`PUT /api/panel/routing` 可设置（沿用面板鉴权；不是手动 `/probe` 请求体）： + +```json +{ + "usage_probe": { + "enabled": true, + "reset_delay_sec": 120, + "reset_jitter_sec": 600 + } +} +``` + +两项均默认 `0`、范围 `0`–`3600` 秒，省略时保留旧行为。对已到期且带有效重置时间的 Extra `5h` 窗口，周期查询的最早时间为 `reset + reset_delay_sec + 每账号稳定偏移`；偏移在 `0`–`reset_jitter_sec` 内，由账号身份和该轮 reset 确定，重复 tick、进程重启及列表重排不会重新抽签。上例为重置后 120–720 秒；实际执行仍受 `interval_sec` 的 tick 粒度、并发、单飞及退避限制，不保证每个账号不同秒执行。等待项在 `GET /api/panel/usage-probe` 的快照中显示 `reason: "reset_probe_delay"` 和 `next_probe_at`（最早时间，不是预约保证）。 + +这只调整**周期额度探测链路的执行时间**，不新增探测：现有链路以官方 `GET /api/oauth/usage` 查询为主，满足套餐复核条件时仍可能执行现有 Fable Messages 探测；本改动不改变这些条件。未采样账号、仅 7d 到期或无有效 5h reset 的账号保留原逻辑；已成功完成本轮 reset 后查询的账号不再被旧 reset 延迟。官方 usage 退避优先，主动/过期样本和套餐复核路径不会绕过等待。主动调用 `POST /api/panel/usage-probe` 同样运行周期规则；手动 `POST /api/panel/probe`、事件 `probeNow`、真实用户流量及独立 Messages 健康检查不受此延迟影响。 + +因此，这些设置**不保证上游 5h 计费/额度窗口错开**：官方 usage GET 是否会开启窗口没有得到验证，既有 Fable Messages、真实流量及独立 Messages 健康检查也可能触发新窗口。延迟会让面板/额度恢复信息更晚刷新；需要即时查询时使用手动 `/probe`。当前仅提供 routing/API 配置，不新增面板控件。 + ## 健康 ```bash diff --git a/src/lib/oauth/usage-probe-monitor.mjs b/src/lib/oauth/usage-probe-monitor.mjs index 6077dfa0b..d78a056fc 100644 --- a/src/lib/oauth/usage-probe-monitor.mjs +++ b/src/lib/oauth/usage-probe-monitor.mjs @@ -9,6 +9,7 @@ * wipe would otherwise pin the panel at 0 while usage_logs keep growing. * Those slots get one /usage hop, then back off. */ +import { createHash } from 'node:crypto' import { shouldProbeFable, usageProbeBackoffRemainingMs } from './crs-usage-probe.mjs' import { hasRefreshPresence } from './oauth-credentials.mjs' import { vmHasProxyPath } from './credential-refresh-monitor.mjs' @@ -17,6 +18,9 @@ export const DEFAULT_USAGE_PROBE = Object.freeze({ enabled: false, interval_sec: 60, stale_sec: 300, + // Opt-in: defer the first periodic hop after an Extra 5h reset. + reset_delay_sec: 0, + reset_jitter_sec: 0, run_on_start: true, concurrency: 2, timeout_ms: 60_000, @@ -45,6 +49,8 @@ export function normalizeUsageProbeConfig(raw = {}) { enabled: asBool(src.enabled, DEFAULT_USAGE_PROBE.enabled), interval_sec: clampInt(src.interval_sec, 30, 3600, DEFAULT_USAGE_PROBE.interval_sec), stale_sec: clampInt(src.stale_sec, 60, 86_400, DEFAULT_USAGE_PROBE.stale_sec), + reset_delay_sec: clampInt(src.reset_delay_sec, 0, 3600, DEFAULT_USAGE_PROBE.reset_delay_sec), + reset_jitter_sec: clampInt(src.reset_jitter_sec, 0, 3600, DEFAULT_USAGE_PROBE.reset_jitter_sec), run_on_start: asBool(src.run_on_start, DEFAULT_USAGE_PROBE.run_on_start), concurrency: clampInt(src.concurrency, 1, 8, DEFAULT_USAGE_PROBE.concurrency), timeout_ms: clampInt(src.timeout_ms, 10_000, 180_000, DEFAULT_USAGE_PROBE.timeout_ms), @@ -148,6 +154,27 @@ export function isUsageProbeDue(account = {}, opts = {}) { return { due: true, reason: 'extra_window_elapsed' } } +/** A stable deadline, not a sleep or a fresh random draw on each timer tick. + * Only gates an already-due periodic hop; never creates extra requests or delays + * real traffic, manual probeNow, or the separate Messages health monitor. + */ +function resetProbeAt(account, vm, config, now) { + if (!config.reset_delay_sec && !config.reset_jitter_sec) return null + const unified = account?.unified || {} + const reset = parseResetMs(unified.headers?.['5h']?.reset) + if (!Number.isFinite(reset) || reset > now) return null + const probe = account?.last_probe || unified.last_probe + const probed = Date.parse(probe?.at || probe?.probed_at || '') + if (probe?.ok === true && Number.isFinite(probed) && probed >= reset) return null + const id = String(account?.account_id || vm.claude?.account_uuid || vm.account_uuid || vm.id) + const hash = createHash('sha256') + .update(JSON.stringify([id, reset])) + .digest() + .readUInt32BE(0) + const jitter = hash % (config.reset_jitter_sec + 1) + return reset + (config.reset_delay_sec + jitter) * 1000 +} + /** Queued probes beyond the running `concurrency` workers; more are dropped, never blocked on. */ const MAX_QUEUED_PROBES = 64 const PROBE_BACKOFF_BASE_MS = 60_000 @@ -275,6 +302,17 @@ export function createUsageProbeMonitor(opts = {}) { }) continue } + const nextProbeAt = resetProbeAt(account, vm, config, nowFn()) + if (nextProbeAt != null && nextProbeAt > nowFn()) { + items.push({ + vm_id: vm.id, + ok: true, + reason: 'reset_probe_delay', + source: null, + next_probe_at: new Date(nextProbeAt).toISOString(), + }) + continue + } due.push({ vm, reason: decision.reason }) } items.push(...(await Promise.all(due.map(({ vm, reason }) => probeVm(vm, reason))))) diff --git a/test/unit/usage-probe-reset-delay.test.mjs b/test/unit/usage-probe-reset-delay.test.mjs new file mode 100644 index 000000000..2a486344b --- /dev/null +++ b/test/unit/usage-probe-reset-delay.test.mjs @@ -0,0 +1,183 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { createUsageProbeMonitor, normalizeUsageProbeConfig } from '../../src/lib/oauth/usage-probe-monitor.mjs' + +const reset = Date.parse('2026-10-17T01:00:00.000Z') +const vm = { id: 'vm-01', status: 'running', has_refresh: true, proxy: { host: '127.0.0.1', port: 1080 } } +function account(id = 'account-a', extra = {}) { + return { + account_id: id, + unified: { + headers: { '5h': { utilization: 1, reset: new Date(reset).toISOString(), stale_reason: 'reset_elapsed' } }, + ...extra, + }, + } +} +function fixture({ config = {}, accounts = [account()], active = false, result = { ok: true } } = {}) { + let now = reset + const calls = [] + const targets = accounts.map((_, i) => ({ ...vm, id: `vm-${String(i + 1).padStart(2, '0')}` })) + const monitor = createUsageProbeMonitor({ + config: { reset_delay_sec: 120, reset_jitter_sec: 600, ...config }, + now: () => now, + listTargets: () => targets, + accountForVm: (target) => accounts[targets.indexOf(target)], + isActive: () => active, + probeOne: async (target) => { + calls.push({ id: target.id, at: now }) + return result + }, + }) + return { + monitor, + calls, + setNow: (n) => { + now = n + }, + } +} + +test('reset delay is opt-in and bounded; normalization preserves zero', () => { + const defaults = normalizeUsageProbeConfig() + assert.equal(defaults.reset_delay_sec, 0) + assert.equal(defaults.reset_jitter_sec, 0) + const cfg = normalizeUsageProbeConfig({ reset_delay_sec: 9000, reset_jitter_sec: -1 }) + assert.equal(cfg.reset_delay_sec, 3600) + assert.equal(cfg.reset_jitter_sec, 0) + assert.equal(normalizeUsageProbeConfig({ reset_delay_sec: '120', reset_jitter_sec: '600' }).reset_jitter_sec, 600) +}) + +test('default settings retain immediate elapsed-window probing', async () => { + const f = fixture({ config: { reset_delay_sec: 0, reset_jitter_sec: 0 } }) + assert.equal((await f.monitor.runOnce()).due, 1) + assert.equal(f.calls.length, 1) +}) + +test('elapsed 5h accounts have stable per-account deadlines, not one fleet deadline', async () => { + const accounts = [account('account-a'), account('account-b'), account('account-c')] + const f = fixture({ accounts }) + const run = await f.monitor.runOnce() + assert.equal(run.due, 0) + assert.equal(f.calls.length, 0) + const times = run.items.map((item) => { + assert.equal(item.reason, 'reset_probe_delay') + const at = Date.parse(item.next_probe_at) + assert.ok(at >= reset + 120_000 && at <= reset + 720_000) + return at + }) + assert.ok(new Set(times).size > 1) + // Repeated ticks cannot redraw jitter and postpone a deadline indefinitely. + f.setNow(reset + 1000) + assert.deepEqual( + (await f.monitor.runOnce()).items.map((x) => x.next_probe_at), + run.items.map((x) => x.next_probe_at), + ) + // Restart and VM-list ordering changes do not move an account's deadline. + const reversed = fixture({ accounts: [...accounts].reverse() }) + assert.deepEqual( + (await reversed.monitor.runOnce()).items.map((x) => x.next_probe_at).reverse(), + run.items.map((x) => x.next_probe_at), + ) + f.setNow(reset + 720_000) + assert.equal((await f.monitor.runOnce()).due, 3) + assert.equal(f.calls.length, 3) +}) + +test('fixed post-reset delay releases exactly at the boundary', async () => { + const f = fixture({ config: { reset_jitter_sec: 0 } }) + f.setNow(reset + 119_999) + assert.equal((await f.monitor.runOnce()).due, 0) + f.setNow(reset + 120_000) + assert.equal((await f.monitor.runOnce()).due, 1) + assert.equal(f.calls[0].at, reset + 120_000) +}) + +for (const extra of [{}, { account_tier: 'pro', fable: { plan_denied: true } }]) { + test(`active/stale and tier-recheck paths cannot bypass a pending reset delay (${extra.account_tier || 'unknown'})`, async () => { + const f = fixture({ accounts: [account('account-a', extra)], active: true }) + assert.equal((await f.monitor.runOnce()).due, 0) + assert.equal(f.calls.length, 0) + }) +} + +test('manual/event probe remains immediate and shares existing backoff', async () => { + const f = fixture({ result: { ok: false } }) + assert.equal((await f.monitor.runOnce()).due, 0) + assert.equal((await f.monitor.probeNow('vm-01')).ok, false) + assert.equal(f.calls.length, 1) + assert.equal((await f.monitor.probeNow('vm-01')).reason, 'probe_backoff') + assert.equal(f.calls.length, 1) +}) + +test('a successful post-reset sample is not held again by the old reset', async () => { + const a = account() + const f = fixture({ accounts: [a], active: true, config: { reset_delay_sec: 600, reset_jitter_sec: 0 } }) + assert.equal((await f.monitor.runOnce()).due, 0) + a.unified.last_probe = { at: new Date(reset + 1000).toISOString(), ok: true } + f.setNow(reset + 400_000) + // Active freshness is overdue; a prior post-reset hop already consumed the delay. + assert.equal((await f.monitor.runOnce()).due, 1) +}) + +test('official usage backoff still wins and no wait item erases it', async () => { + const f = fixture({ + accounts: [account('account-a', { usage_rate_limited_until: new Date(reset + 900_000).toISOString() })], + }) + const run = await f.monitor.runOnce() + assert.equal(run.due, 0) + assert.equal(f.calls.length, 0) + assert.equal(run.items[0].reason, 'reconcile_extra') +}) + +test('seven-day-only elapsed and never-sampled accounts are not delayed', async () => { + const f = fixture({ + accounts: [ + account('weekly', { headers: { '7d': { reset: new Date(reset).toISOString() } } }), + { account_id: 'new' }, + ], + }) + assert.equal((await f.monitor.runOnce()).due, 2) +}) + +test('missing/invalid 5h reset cannot invent a deadline', async () => { + const f = fixture({ + accounts: [account('unknown', { headers: { '5h': { stale_reason: 'reset_elapsed', reset: 'invalid' } } })], + }) + assert.equal((await f.monitor.runOnce()).due, 1) +}) + +test('future 5h reset does not delay active freshness sampling', async () => { + const f = fixture({ + accounts: [ + account('future', { headers: { '5h': { reset: new Date(reset + 3600_000).toISOString(), utilization: 0.2 } } }), + ], + active: true, + }) + assert.equal((await f.monitor.runOnce()).due, 1) +}) + +test('routing updates preserve delay config and propagate it to the monitor', async () => { + const { createRoutingRuntime } = await import('../../src/lib/admin/routing-runtime.mjs') + const f = fixture({ config: { reset_delay_sec: 0, reset_jitter_sec: 0 } }) + const ctx = { + routingConfig: {}, + usageProbeMonitor: { setConfig: (cfg) => f.monitor.setConfig(cfg, { restart: false }) }, + } + const runtime = createRoutingRuntime(ctx) + runtime.applyUsageProbeRoutingBody({ usage_probe: { reset_delay_sec: 120, reset_jitter_sec: 600 } }) + assert.equal(ctx.routingConfig.usage_probe.reset_delay_sec, 120) + assert.equal(f.monitor.getConfig().reset_jitter_sec, 600) + assert.equal((await f.monitor.runOnce()).due, 0) + runtime.applyUsageProbeRoutingBody({ usage_probe: { concurrency: 1 } }, ctx.routingConfig.usage_probe) + assert.equal(f.monitor.getConfig().reset_delay_sec, 120) + assert.equal(f.monitor.getConfig().reset_jitter_sec, 600) + runtime.applyUsageProbeRoutingBody({ usage_probe: { reset_delay_sec: 0, reset_jitter_sec: 0 } }) + assert.equal((await f.monitor.runOnce()).due, 1) +}) + +test('config changes apply without retaining stale delay state', async () => { + const f = fixture() + assert.equal((await f.monitor.runOnce()).due, 0) + f.monitor.setConfig({ reset_delay_sec: 0, reset_jitter_sec: 0 }, { restart: false }) + assert.equal((await f.monitor.runOnce()).due, 1) +}) From 83e703785f2b09697dc9276c7b4b86a44b07f781 Mon Sep 17 00:00:00 2001 From: Jeremy Liang Date: Sat, 17 Oct 2026 10:10:07 +0800 Subject: [PATCH 2/2] docs: clarify post-reset probe deferral exemptions --- docs/API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API.md b/docs/API.md index f2a364e62..db343a40c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -215,7 +215,7 @@ curl -sS http://127.0.0.1:8787/v1/usage -H "Authorization: Bearer $KEY" 两项均默认 `0`、范围 `0`–`3600` 秒，省略时保留旧行为。对已到期且带有效重置时间的 Extra `5h` 窗口，周期查询的最早时间为 `reset + reset_delay_sec + 每账号稳定偏移`；偏移在 `0`–`reset_jitter_sec` 内，由账号身份和该轮 reset 确定，重复 tick、进程重启及列表重排不会重新抽签。上例为重置后 120–720 秒；实际执行仍受 `interval_sec` 的 tick 粒度、并发、单飞及退避限制，不保证每个账号不同秒执行。等待项在 `GET /api/panel/usage-probe` 的快照中显示 `reason: "reset_probe_delay"` 和 `next_probe_at`（最早时间，不是预约保证）。 -这只调整**周期额度探测链路的执行时间**，不新增探测：现有链路以官方 `GET /api/oauth/usage` 查询为主，满足套餐复核条件时仍可能执行现有 Fable Messages 探测；本改动不改变这些条件。未采样账号、仅 7d 到期或无有效 5h reset 的账号保留原逻辑；已成功完成本轮 reset 后查询的账号不再被旧 reset 延迟。官方 usage 退避优先，主动/过期样本和套餐复核路径不会绕过等待。主动调用 `POST /api/panel/usage-probe` 同样运行周期规则；手动 `POST /api/panel/probe`、事件 `probeNow`、真实用户流量及独立 Messages 健康检查不受此延迟影响。 +这只调整**周期额度探测链路的执行时间**，不新增探测：现有链路以官方 `GET /api/oauth/usage` 查询为主，满足套餐复核条件时仍可能执行现有 Fable Messages 探测；本改动不改变这些条件。没有有效已到期 5h reset 的未采样账号、仅 7d 到期或无有效 5h reset 的账号保留原逻辑；已有 `last_probe.ok === true` 且探测时间不早于本轮 reset 的账号不再被旧 reset 延迟，仅由真实流量更新的 `headers.sampled_at` 不触发该豁免。官方 usage 退避优先，主动/过期样本和套餐复核路径不会绕过等待。主动调用 `POST /api/panel/usage-probe` 同样运行周期规则；手动 `POST /api/panel/probe`、事件 `probeNow`、真实用户流量及独立 Messages 健康检查不受此延迟影响。 因此，这些设置**不保证上游 5h 计费/额度窗口错开**：官方 usage GET 是否会开启窗口没有得到验证，既有 Fable Messages、真实流量及独立 Messages 健康检查也可能触发新窗口。延迟会让面板/额度恢复信息更晚刷新；需要即时查询时使用手动 `/probe`。当前仅提供 routing/API 配置，不新增面板控件。