Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/fix-deepseek-failure-capture.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 22 additions & 3 deletions actions/setup/js/detect_agent_errors.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,10 @@ const { detectErrors, buildOutputLines, isStepTimeout, sanitizeModelName } = err
const { parseUnknownModelAICreditsAndModelFromAuditLog, parseMaxCacheMissesExceededFromEventLog } = require("./ai_credits_context.cjs");
const { renderLogFromFile } = require("./render_detection_log.cjs");
const { getErrorMessage } = require("./error_helpers.cjs");
const { collectAgentExecution, agentErrorDiagnosticText, parseAgentExitCode } = require("./agent_execution.cjs");
const { collectAgentExecution, agentErrorDiagnosticText, parseAgentExitCode, isAgentExecutionEvent } = require("./agent_execution.cjs");
const { writeSessionArtifact } = require("./session_artifact.cjs");
const { collectAddMaskedValues } = require("./add_mask_redaction.cjs");
const { parseDeepSeekLog, reconcileDeepSeekExecution } = require("./parse_deepseek_log.cjs");

// File written by the engine execution step with the epoch milliseconds at which the
// engine CLI was started. Used to measure how long the engine ran before it was killed.
Expand Down Expand Up @@ -229,15 +230,30 @@ async function renderInternalEngineLogOnFailure() {
*/
function persistAgentExecution(logContent, results, rootDir = process.env.GH_AW_TMP_DIR || "/tmp/gh-aw") {
const exitPath = path.join(rootDir, "agent_execution_exit_code.txt");
const execution = collectAgentExecution({
const deepSeek = process.env.GH_AW_ENGINE_ID === "deepseek-harness";
const events = deepSeek ? parseDeepSeekLog(logContent).logEntries : [];
/** @type {import("./types/agent_session").SessionEvent[]} */
const supplementalErrors = [];
if (deepSeek && !events.some(event => event.type === "session.error")) {
if (results.agenticEngineTimeout) {
supplementalErrors.push({ type: "session.error", data: { message: "DeepSeek Harness execution timed out", code: "agentic_engine_timeout", errorType: "TimeoutError" } });
} else if (process.env.GH_AW_AGENTIC_EXECUTION_OUTCOME === "failure") {
supplementalErrors.push({ type: "session.error", data: { message: "DeepSeek Harness execution failed", code: "agentic_engine_execution_failed" } });
}
}
const diagnostics = [...events, ...supplementalErrors];
let execution = collectAgentExecution({
content: logContent,
...(process.env.GH_AW_ENGINE_ID === "aider" ? { rawSourceEngine: "aider" } : {}),
events: diagnostics,
observations: events.filter(isAgentExecutionEvent).map(event => event.data),
categories: buildOutputLines(results)
.filter(line => line.endsWith("=true"))
.map(line => line.slice(0, -5)),
...(fs.existsSync(exitPath) ? { exitCode: parseAgentExitCode(fs.readFileSync(exitPath, "utf8")) } : {}),
});
writeSessionArtifact(path.join(rootDir, "agent-errors.jsonl"), execution ? [execution] : [], collectAddMaskedValues(logContent));
if (deepSeek) execution = reconcileDeepSeekExecution(execution, diagnostics);
writeSessionArtifact(path.join(rootDir, "agent-errors.jsonl"), [...supplementalErrors, ...(execution ? [execution] : [])], collectAddMaskedValues(logContent));
}

async function main({ rootDir = process.env.GH_AW_TMP_DIR || "/tmp/gh-aw" } = {}) {
Expand All @@ -255,6 +271,9 @@ async function main({ rootDir = process.env.GH_AW_TMP_DIR || "/tmp/gh-aw" } = {}
}

const stdioResults = detectErrors(agentErrorDiagnosticText(logContent, process.env.GH_AW_ENGINE_ID === "aider" ? "aider" : undefined));
if (process.env.GH_AW_ENGINE_ID === "deepseek-harness" && parseDeepSeekLog(logContent).logEntries.some(event => event.type === "session.error" && event.data.code === "agentic_engine_timeout")) {
stdioResults.agenticEngineTimeout = true;
Comment on lines +274 to +275
}

// Also check the AWF firewall structured JSONL logs for the `unknown_model_ai_credits`
// event — the API proxy event log is preferred and the audit log is used as a fallback.
Expand Down
25 changes: 25 additions & 0 deletions actions/setup/js/detect_agent_errors.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,31 @@ const {
} = require("./detect_agent_errors.cjs");

describe("detect_agent_errors.cjs", () => {
it("emits the DeepSeek timeout classification from attributed AWF stdout", async () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-deepseek-timeout-"));
const stderr = vi.spyOn(process.stderr, "write").mockReturnValue(true);
try {
vi.stubEnv("GH_AW_ENGINE_ID", "deepseek-harness");
vi.stubEnv("GH_AW_AGENTIC_EXECUTION_OUTCOME", "failure");
vi.stubEnv("GITHUB_OUTPUT", path.join(rootDir, "github-output"));
vi.stubEnv("GH_AW_ENGINE_INTERNAL_LOGS_DIR", "");
fs.writeFileSync(path.join(rootDir, "agent-stdio.log"), "[deepseek-harness] configured provider=github model=auto\n[WARN] Agent command timed out after 15 minutes, stopping container...\n");
fs.writeFileSync(path.join(rootDir, "agent_execution_exit_code.txt"), "0");

await main({ rootDir });

expect(fs.readFileSync(path.join(rootDir, "github-output"), "utf8")).toContain("agentic_engine_timeout=true");
const execution = JSON.parse(fs.readFileSync(path.join(rootDir, "agent-errors.jsonl"), "utf8").trim());
expect(execution.data.categories).toContain("agentic_engine_timeout");

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] This assertion only inspects agent-errors.jsonl, so it passes even when the timeout never reaches the unified session. Add a check on the unified session session.error with code: "agentic_engine_timeout", since the canonical-session path currently drops it.

💡 Evidence and suggested test

A local Node probe with GH_AW_ENGINE_ID=deepseek-harness, GH_AW_AGENTIC_EXECUTION_OUTCOME=failure, the timedOut fixture, and the AWF timeout warning produced:

  • Canonical session present: session.init, agent.execution (no timeout error)
  • No canonical session: session.init, session.error(agentic_engine_timeout), agent.execution

Add a case that writes a canonical session from parseDeepSeekLog and asserts the unified session still contains agentic_engine_timeout.

@copilot please address this.

expect(execution.data).not.toHaveProperty("exitCode");
expect(stderr.mock.calls.flat().join("\n")).toContain("Detected agentic engine timeout signature");
} finally {
stderr.mockRestore();
vi.unstubAllEnvs();
fs.rmSync(rootDir, { recursive: true, force: true });
}
});

it.each([
{ log: "", outcome: "success", linked: false },
{ log: "Assistant: Access denied by policy settings", outcome: "success", linked: false },
Expand Down
40 changes: 37 additions & 3 deletions actions/setup/js/parse_deepseek_log.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const { stripVTControlCharacters } = require("node:util");

const main = createEngineLogParser({ parserName: "DeepSeek Harness", parseFunction: parseDeepSeekLog, supportsDirectories: false });
const HARNESS_FAILURE = /^\[deepseek-harness\] DeepSeek Harness execution failed with exit code ([1-9]\d*)(?: \(signal=[A-Z0-9]+\))?$/;
const AWF_TIMEOUT = /^\[WARN\] Agent command timed out after [1-9]\d* minutes, stopping container\.\.\.$/;
const TOKEN_CLEANUP = /^\[entrypoint\] Unset [A-Z][A-Z0-9_]* from \/proc\/1\/environ$/;
const PERMISSION_CLEANUP = /^\[entrypoint\] Relaxed \S+ group permissions for host-side post-processing$/;

Expand Down Expand Up @@ -69,16 +70,32 @@ function extractHeadlessErrors(lines, configuration) {
const shutdown = lines.findIndex((line, index) => index > configuration.index && line === "[INFO] Stopping containers...");
const body = lines.slice(configuration.index + 1, shutdown < 0 ? undefined : shutdown);
if (body.some(line => /^[ \t]*(?:(?:user|system)\s*(?:prompt|message)?\s*:|prompt\s*:)/i.test(line))) return [];
const start = body.findIndex(line => line.trim() !== "" && line !== "[entrypoint] Unsetting sensitive tokens from parent shell environment..." && !TOKEN_CLEANUP.test(line));
if (shutdown < 0 && start >= 0 && AWF_TIMEOUT.test(body[start]) && body.slice(start + 1).every(line => line.trim() === "" || PERMISSION_CLEANUP.test(line))) {
const error = createSessionEvent({}, "session.error", { message: body[start].slice("[WARN] ".length), code: "agentic_engine_timeout", errorType: "TimeoutError" });
const execution = collectAgentExecution({ events: [error], categories: ["agentic_engine_timeout"] });
return execution ? [error, execution] : [error];
}
const failureIndex = body.findIndex(line => HARNESS_FAILURE.test(line));
if (failureIndex < 0) return [];
if (failureIndex < 0) {
if (shutdown < 0) return [];
Comment on lines +80 to +81
const terminal = lines.slice(shutdown + 1);
const failure = terminal.find(line => /^\[WARN\] Command completed with exit code: [1-9]\d*$/.test(line));
const exit = terminal.find(line => /^Process exiting with code:/.test(line));
if (!failure || terminal.includes("[SUCCESS] Command completed successfully")) return [];
const exitCode = Number(failure.slice("[WARN] Command completed with exit code: ".length));
if (exitCode > 255 || exit !== `Process exiting with code: ${exitCode}`) return [];
const error = createSessionEvent({}, "session.error", { message: `DeepSeek Harness execution exited with code ${exitCode}`, exitCode });
const execution = collectAgentExecution({ events: [error], exitCode });
return execution ? [error, execution] : [error];
}
const exitCode = Number(body[failureIndex].match(HARNESS_FAILURE)?.[1]);
if (shutdown >= 0) {
const terminal = lines.slice(shutdown + 1);
const exit = terminal.find(line => /^Process exiting with code:/.test(line));
if (terminal.includes("[SUCCESS] Command completed successfully") || (exit !== undefined && exit !== `Process exiting with code: ${exitCode}`)) return [];
}
if (!body.slice(failureIndex + 1).every(line => line.trim() === "" || PERMISSION_CLEANUP.test(line))) return [];
const start = body.findIndex(line => line.trim() !== "" && line !== "[entrypoint] Unsetting sensitive tokens from parent shell environment..." && !TOKEN_CLEANUP.test(line));
const stackStart = body.findIndex(line => /^file:\/\/\/\S+:\d+(?::\d+)?$/.test(line));
const nodeVersion = body.findIndex((line, index) => index > stackStart && /^Node\.js v\d+\.\d+\.\d+$/.test(line));
const hasNodeStack = stackStart === start && stackStart >= 0 && nodeVersion > stackStart && nodeVersion < failureIndex && body.slice(nodeVersion + 1, failureIndex).every(line => line.trim() === "");
Expand All @@ -101,6 +118,23 @@ function extractHeadlessErrors(lines, configuration) {
return entries;
}

/**
* A wrapper's zero status cannot establish success against an attributed failure.
* Preserve an observed native failure code; a timeout alone has no known code.
* @param {import("./types/agent_session").AgentExecutionEvent | undefined} execution
* @param {import("./types/agent_session").SessionEvent[]} events
* @returns {import("./types/agent_session").AgentExecutionEvent | undefined}
*/
function reconcileDeepSeekExecution(execution, events) {
const errors = events.filter(event => event.type === "session.error");
if (!execution || execution.data.exitCode !== 0 || !errors.length) return execution;
const reconciled = structuredClone(execution);
const nativeExit = errors.map(event => event.data.exitCode).find(value => typeof value === "number" && Number.isSafeInteger(value) && value > 0 && value <= 255);
if (typeof nativeExit === "number") reconciled.data.exitCode = nativeExit;
else delete reconciled.data.exitCode;
return reconciled;
}

/** @param {string} content @returns {boolean} */
function isDeepSeekLog(content) {
return findConfiguration(stripVTControlCharacters(content).split(/\r?\n/)) !== undefined;
Expand Down Expand Up @@ -138,4 +172,4 @@ function parseDeepSeekLog(content) {
};
}

module.exports = { main, parseDeepSeekLog, isDeepSeekLog };
module.exports = { main, parseDeepSeekLog, isDeepSeekLog, reconcileDeepSeekExecution };
Loading
Loading