Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6,674 changes: 6,674 additions & 0 deletions package-lock.json

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -80,5 +80,8 @@
},
"engines": {
"node": ">=18.0.0"
},
"dependencies": {
"bun-types": "^1.4.3"
}
}
2 changes: 1 addition & 1 deletion src/config/agent-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { homedir } from "os";
import { basename, extname, isAbsolute, join, relative, resolve } from "path";
import type { AgentConfig } from "../types.js";

const MODEL_KEY_RE = /^[a-zA-Z0-9_-]{1,100}\/[a-zA-Z0-9._-]{1,100}$/;
const MODEL_KEY_RE = /^[a-zA-Z0-9_-]{1,100}\/[a-zA-Z0-9._~\/:-]{1,200}$/;

function isPathInside(baseDir: string, targetPath: string): boolean {
const rel = relative(baseDir, targetPath);
Expand Down
2 changes: 2 additions & 0 deletions src/config/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ export const DEFAULT_CONFIG: PluginConfig = {
cooldownMs: 300_000, // 5 minutes
retryOriginalAfterMs: 900_000, // 15 minutes
maxFallbackDepth: 3,
retriesBeforeFallback: 0,
quarantineAfterFallbackMs: 0,
},
agents: {
"*": {
Expand Down
50 changes: 43 additions & 7 deletions src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { isAbsolute, relative, resolve } from "path";
import { z } from "zod";
import { DEFAULT_CONFIG } from "./defaults.js";

const MODEL_KEY_RE = /^[a-zA-Z0-9_-]{1,100}\/[a-zA-Z0-9._-]{1,100}$/;
const MODEL_KEY_RE = /^[a-zA-Z0-9_-]{1,100}\/[a-zA-Z0-9._~\/:-]{1,200}$/;
const home = resolve(homedir());

function formatPath(path: Array<PropertyKey>): string {
Expand Down Expand Up @@ -35,6 +35,8 @@ const fallbackDefaults = z.object({
cooldownMs: z.number().min(10_000).optional(),
retryOriginalAfterMs: z.number().min(10_000).optional(),
maxFallbackDepth: z.number().int().min(1).max(10).optional(),
retriesBeforeFallback: z.number().int().min(0).max(10).optional(),
quarantineAfterFallbackMs: z.number().min(0).optional(),
});

const logPathSchema = z
Expand Down Expand Up @@ -149,7 +151,7 @@ export function parseConfig(raw: unknown): {
}
}

const depthResult = fallbackDefaults.shape.maxFallbackDepth.safeParse(
const depthResult = fallbackDefaults.shape.maxFallbackDepth.safeParse(
defaultsObj.maxFallbackDepth
);
if (defaultsObj.maxFallbackDepth !== undefined) {
Expand All @@ -165,6 +167,38 @@ export function parseConfig(raw: unknown): {
}
}

const retriesResult = fallbackDefaults.shape.retriesBeforeFallback.safeParse(
defaultsObj.retriesBeforeFallback
);
if (defaultsObj.retriesBeforeFallback !== undefined) {
if (retriesResult.success && retriesResult.data !== undefined) {
parsedDefaults.retriesBeforeFallback = retriesResult.data;
} else if (!retriesResult.success) {
for (const issue of retriesResult.error.issues) {
const suffix = issue.path.length > 0 ? `.${formatPath(issue.path)}` : "";
warnings.push(
`Config warning at defaults.retriesBeforeFallback${suffix}: ${issue.message} — using default`
);
}
}
}

const quarantineResult = fallbackDefaults.shape.quarantineAfterFallbackMs.safeParse(
defaultsObj.quarantineAfterFallbackMs
);
if (defaultsObj.quarantineAfterFallbackMs !== undefined) {
if (quarantineResult.success && quarantineResult.data !== undefined) {
parsedDefaults.quarantineAfterFallbackMs = quarantineResult.data;
} else if (!quarantineResult.success) {
for (const issue of quarantineResult.error.issues) {
const suffix = issue.path.length > 0 ? `.${formatPath(issue.path)}` : "";
warnings.push(
`Config warning at defaults.quarantineAfterFallbackMs${suffix}: ${issue.message} — using default`
);
}
}
}

for (const key of Object.keys(defaultsObj)) {
if (!Object.hasOwn(fallbackDefaults.shape, key)) {
warnings.push(`Config warning at defaults.${key}: unknown field — using default`);
Expand Down Expand Up @@ -266,13 +300,15 @@ export function mergeWithDefaults(raw: RawConfig): import("../types.js").PluginC
const def = DEFAULT_CONFIG;
const logPath = raw.logPath ? normalizeLogPath(raw.logPath) : def.logPath;

return {
return {
enabled: raw.enabled ?? def.enabled,
defaults: {
fallbackOn: raw.defaults?.fallbackOn ?? def.defaults.fallbackOn,
cooldownMs: raw.defaults?.cooldownMs ?? def.defaults.cooldownMs,
retryOriginalAfterMs: raw.defaults?.retryOriginalAfterMs ?? def.defaults.retryOriginalAfterMs,
maxFallbackDepth: raw.defaults?.maxFallbackDepth ?? def.defaults.maxFallbackDepth,
fallbackOn: raw.defaults?.fallbackOn ?? def.defaults.fallbackOn,
cooldownMs: raw.defaults?.cooldownMs ?? def.defaults.cooldownMs,
retryOriginalAfterMs: raw.defaults?.retryOriginalAfterMs ?? def.defaults.retryOriginalAfterMs,
maxFallbackDepth: raw.defaults?.maxFallbackDepth ?? def.defaults.maxFallbackDepth,
retriesBeforeFallback: raw.defaults?.retriesBeforeFallback ?? def.defaults.retriesBeforeFallback,
quarantineAfterFallbackMs: raw.defaults?.quarantineAfterFallbackMs ?? def.defaults.quarantineAfterFallbackMs,
},
agents: raw.agents ?? def.agents,
patterns: raw.patterns ?? def.patterns,
Expand Down
29 changes: 25 additions & 4 deletions src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ import { FallbackStore } from "./state/store.js";
import { createFallbackStatusTool } from "./tools/fallback-status.js";
import type { ModelKey } from "./types.js";

export function shouldDeferFallback(
attempt: number | undefined,
retriesBeforeFallback: number,
): boolean {
return typeof attempt === "number" && attempt <= retriesBeforeFallback;
}

function resolveFallbackStatusCommandPath(): string {
return join(homedir(), ".config", "opencode", "commands", "fallback-status.md");
}
Expand All @@ -32,7 +39,7 @@ export function ensureFallbackStatusCommand(logger: Logger, cmdPath: string): vo
}
}

export const createPlugin: Plugin = async ({ client, directory }) => {
export function createPlugin({ client, directory }: Parameters<Plugin>[0]): Hooks {
const { config, path: configPath, warnings, migrated } = loadConfig(directory);

const logger = new Logger(client, config.logPath, config.logging, config.logLevel);
Expand Down Expand Up @@ -125,7 +132,7 @@ export const createPlugin: Plugin = async ({ client, directory }) => {
};

return hooks;
};
}

export async function handleEvent(
event: Event,
Expand All @@ -141,7 +148,7 @@ export async function handleEvent(
const { sessionID, status } = event.properties;

if (status.type === "retry") {
await handleRetry(sessionID, status.message, client, store, config, logger, directory);
await handleRetry(sessionID, status.message, status.attempt, client, store, config, logger, directory);
} else if (status.type === "idle") {
await handleIdle(sessionID, client, store, config, logger);
}
Expand Down Expand Up @@ -195,6 +202,7 @@ export async function handleEvent(
async function handleRetry(
sessionId: string,
message: string,
attempt: number | undefined,
client: Parameters<Plugin>[0]["client"],
store: FallbackStore,
config: ReturnType<typeof loadConfig>["config"],
Expand All @@ -217,6 +225,19 @@ async function handleRetry(
return;
}

// Gate: allow N retries before falling back. When the attempt counter has
// not yet crossed the configured threshold, defer the fallback so opencode's
// built-in retry loop gets a chance to succeed on the same model.
if (shouldDeferFallback(attempt, config.defaults.retriesBeforeFallback)) {
logger.info("retry.deferred", {
sessionId,
attempt,
retriesBeforeFallback: config.defaults.retriesBeforeFallback,
category,
});
return;
}

// Seed session state with current model if unknown
const sessionState = store.sessions.get(sessionId);
if (!sessionState.currentModel) {
Expand Down Expand Up @@ -343,4 +364,4 @@ function getLastUserModelAndAgent(data: unknown): {
}

return null;
}
}
4 changes: 2 additions & 2 deletions src/preemptive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ export function tryPreemptiveRedirect(
}
}

// Only redirect if the target model is actively rate-limited
// Redirect when the target model is actively rate-limited or quarantined
const health = store.health.get(modelKey);
if (health.state !== "rate_limited") {
if (health.state !== "rate_limited" && !store.health.isQuarantined(modelKey)) {
return { redirected: false };
}

Expand Down
13 changes: 13 additions & 0 deletions src/replay/orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,14 @@ export async function attemptFallback(
);
}

// Quarantine the model we are leaving behind regardless of reason
if (currentModel) {
store.health.quarantine(
currentModel,
config.defaults.quarantineAfterFallbackMs,
);
}

// Set lastFallbackAt optimistically NOW — before the async sequence —
// so any stale retry events that fire during abort/revert/prompt are
// caught by the dedup window. (recordFallback sets it again on success.)
Expand Down Expand Up @@ -282,6 +290,11 @@ export async function attemptFallback(
agentName,
);

// Quarantine the previously used model to prevent cycling
if (currentModel && config.defaults.quarantineAfterFallbackMs > 0) {
store.health.quarantine(currentModel, config.defaults.quarantineAfterFallbackMs);
}

logger.info("fallback.success", {
sessionId,
agentName,
Expand Down
7 changes: 4 additions & 3 deletions src/resolution/fallback-resolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,17 @@ import type { ModelKey } from "../types.js";
* Strategy:
* 1. First model that is "healthy"
* 2. First model that is "cooldown" (better than nothing)
* 3. null if all are "rate_limited"
* 3. null if all are "rate_limited" or quarantined
*
* Always skips the current model.
* Always skips the current model and any model that is not usable
* (rate_limited or quarantined).
*/
export function resolveFallbackModel(
chain: ModelKey[],
currentModel: ModelKey | null,
health: ModelHealthStore
): ModelKey | null {
const candidates = chain.filter((m) => m !== currentModel);
const candidates = chain.filter((m) => m !== currentModel && health.isUsable(m));

// Prefer healthy, then cooldown
const healthy = candidates.find((m) => health.get(m).state === "healthy");
Expand Down
44 changes: 33 additions & 11 deletions src/state/model-health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ export class ModelHealthStore {
private onTransition?: (modelKey: ModelKey, from: HealthState, to: HealthState) => void;

constructor(opts?: {
onTransition?: (modelKey: ModelKey, from: HealthState, to: HealthState) => void;
onTransition?: (modelKey: ModelKey, from: HealthState, to: HealthState) => void
}) {
this.onTransition = opts?.onTransition;
// Check state transitions every 30 seconds.
Expand All @@ -32,13 +32,28 @@ export class ModelHealthStore {
this.store.set(modelKey, health);
}

quarantine(modelKey: ModelKey, ms: number): void {
if (ms <= 0) return;
const existing = this.get(modelKey);
this.store.set(modelKey, {
...existing,
quarantinedUntil: Date.now() + ms,
});
}

isQuarantined(modelKey: ModelKey): boolean {
const until = this.get(modelKey).quarantinedUntil;
return until !== null && Date.now() < until;
}

isUsable(modelKey: ModelKey): boolean {
if (this.isQuarantined(modelKey)) return false;
const h = this.get(modelKey);
return h.state === "healthy" || h.state === "cooldown";
}

preferScore(modelKey: ModelKey): number {
// healthy=2, cooldown=1, rate_limited=0
if (this.isQuarantined(modelKey)) return 0;
const state = this.get(modelKey).state;
if (state === "healthy") return 2;
if (state === "cooldown") return 1;
Expand All @@ -59,21 +74,27 @@ export class ModelHealthStore {
private tick(): void {
const now = Date.now();
for (const [key, health] of this.store) {
let current = health;
if (current.quarantinedUntil !== null && now >= current.quarantinedUntil) {
current = { ...current, quarantinedUntil: null };
this.store.set(key, current);
}

if (
health.state === "rate_limited" &&
health.cooldownExpiresAt &&
now >= health.cooldownExpiresAt
current.state === "rate_limited" &&
current.cooldownExpiresAt &&
now >= current.cooldownExpiresAt
) {
const next: ModelHealth = { ...health, state: "cooldown" };
const next: ModelHealth = { ...current, state: "cooldown" };
this.store.set(key, next);
this.onTransition?.(key, "rate_limited", "cooldown");
} else if (
health.state === "cooldown" &&
health.retryOriginalAt &&
now >= health.retryOriginalAt
current.state === "cooldown" &&
current.retryOriginalAt &&
now >= current.retryOriginalAt
) {
const next: ModelHealth = {
...health,
...current,
state: "healthy",
cooldownExpiresAt: null,
retryOriginalAt: null,
Expand All @@ -92,6 +113,7 @@ export class ModelHealthStore {
failureCount: 0,
cooldownExpiresAt: null,
retryOriginalAt: null,
quarantinedUntil: null,
};
}
}
}
10 changes: 9 additions & 1 deletion src/tools/fallback-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,24 @@ export function createFallbackStatusTool(
lines.push("- All models healthy (no issues detected)");
} else {
for (const h of allHealth) {
const stateEmoji = h.state === "healthy" ? "✓" : h.state === "cooldown" ? "~" : "✗";
const stateEmoji =
h.state === "healthy" ? "✓" : h.state === "cooldown" ? "~" : h.state === "quarantined" ? "⛔" : "✗";
let detail = `- \`${h.modelKey}\` — **${h.state}** ${stateEmoji}`;
if (h.state === "rate_limited" && h.cooldownExpiresAt) {
const secsLeft = Math.max(0, Math.round((h.cooldownExpiresAt - Date.now()) / 1000));
detail += ` (cooldown in ${secsLeft}s)`;
} else if (h.state === "cooldown" && h.retryOriginalAt) {
const secsLeft = Math.max(0, Math.round((h.retryOriginalAt - Date.now()) / 1000));
detail += ` (recovery in ${secsLeft}s)`;
} else if (h.state === "quarantined" && h.quarantinedUntil && h.quarantinedUntil > Date.now()) {
const secsLeft = Math.max(0, Math.round((h.quarantinedUntil - Date.now()) / 1000));
detail += ` (quarantined ${secsLeft}s)`;
}
if (h.failureCount > 0) detail += ` [${h.failureCount} failures]`;
if (h.quarantinedUntil && h.quarantinedUntil > Date.now()) {
const secsLeft = Math.max(0, Math.round((h.quarantinedUntil - Date.now()) / 1000));
detail += ` (quarantined ${secsLeft}s)`;
}
lines.push(detail);
}
}
Expand Down
5 changes: 4 additions & 1 deletion src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ export type ErrorCategory =
| "overloaded"
| "unknown";

export type HealthState = "healthy" | "rate_limited" | "cooldown";
export type HealthState = "healthy" | "rate_limited" | "cooldown" | "quarantined";

export interface ModelHealth {
modelKey: ModelKey;
Expand All @@ -17,6 +17,7 @@ export interface ModelHealth {
failureCount: number;
cooldownExpiresAt: number | null;
retryOriginalAt: number | null;
quarantinedUntil: number | null;
}

export interface FallbackEvent {
Expand Down Expand Up @@ -52,6 +53,8 @@ export interface FallbackDefaults {
cooldownMs: number;
retryOriginalAfterMs: number;
maxFallbackDepth: number;
retriesBeforeFallback: number;
quarantineAfterFallbackMs: number;
}

export interface PluginConfig {
Expand Down
Loading