[DEV-100] Story creation fails with 'Failed to parse story from agent output' when the agent's response isn't clean JSON

docs/pm/configuration.md

@@ -267,3 +267,9 @@ Update-check state (last check time, seen version) is cached per package in `~/.
 - Check `AGENT_HARNESS` spelling (use kebab-case, e.g. `claude-code`, `grok`, `deepseek`)
 - The error lists every valid harness name; pick one from that list
 - Ensure the matching CLI is installed and on your `PATH`, or set `AGENT_CLI_PATH` / `<HARNESS>_CLI_PATH`
+
+**"Failed to parse story from agent output" / "malformed output"**
+
+- devpm automatically repairs common agent-output drift (markdown fences, narration, comments, trailing commas, unquoted keys, stray quotes) and re-runs the agent once with a strict-JSON reminder when repair isn't possible
+- If parsing still fails, the full agent output is saved to `/tmp/devpm-<step>-<timestamp>.log` — check the log to see what came back, then retry
+- Persistent failures usually mean the harness/model emits non-canonical JSON around long descriptions; switching harness (or pinning a different model) resolves it

packages/pm/agent-json.test.ts

@@ -36,3 +36,99 @@ describe("parseAgentJson", () => {
     expect(() => parseAgentJson("I could not produce the story.")).toThrow();
   });
 });
+
+describe("parseAgentJson object-literal drift repairs", () => {
+  test("strips line comments inside the object", () => {
+    const raw = [
+      "{",
+      "  // story below",
+      '  "summary": "S",',
+      '  "description": "URLs like https://x.io/a//b survive comment stripping",',
+      "",
+      "  ## note",
+      "}",
+    ].join("\n");
+    const parsed = parseAgentJson<Record<string, string>>(raw);
+    expect(parsed.summary).toBe("S");
+    expect(parsed.description).toBe("URLs like https://x.io/a//b survive comment stripping");
+  });
+
+  test("strips block comments inside the object", () => {
+    const raw = '{ /* draft */ "summary": "S", "description": "D" }';
+    expect(parseAgentJson<Record<string, string>>(raw).summary).toBe("S");
+  });
+
+  test("removes trailing commas", () => {
+    const raw = '{"summary": "S", "description": "D",}';
+    expect(parseAgentJson<Record<string, string>>(raw)).toEqual({
+      summary: "S",
+      description: "D",
+    });
+  });
+
+  test("quotes unquoted JavaScript-style keys", () => {
+    const raw = '{ summary: "S", description: "D, with punctuation: plenty" }';
+    expect(parseAgentJson<Record<string, string>>(raw)).toEqual({
+      summary: "S",
+      description: "D, with punctuation: plenty",
+    });
+  });
+
+  test("quotes single-quoted keys", () => {
+    const raw = `{ 'summary': "S", 'description': "D" }`;
+    expect(parseAgentJson<Record<string, string>>(raw)).toEqual({
+      summary: "S",
+      description: "D",
+    });
+  });
+
+  test("normalizes smart-quoted keys", () => {
+    const raw = '{"summary": "S", \u201cdescription\u201d: "D"}';
+    expect(parseAgentJson<Record<string, string>>(raw)).toEqual({
+      summary: "S",
+      description: "D",
+    });
+  });
+
+  test("salvages values whose inner prose quotes desync every heuristic", () => {
+    // `Pick "one", then "two"` defeats structural quote repair because commas
+    // follow the quoted words — the schema-key salvage rebuilds the payload.
+    const raw = [
+      "{",
+      '  "summary": "Story generation fails",',
+      '  "description": "Pick "one", then "two", then done"',
+      "}",
+    ].join("\n");
+    const parsed = parseAgentJson<Record<string, string>>(raw);
+    expect(parsed.summary).toBe("Story generation fails");
+    expect(parsed.description).toContain("then done");
+  });
+
+  test("salvages a richly formatted description mixing several failure shapes", () => {
+    // Reconstruction of the DEV-100 report class: comments, unquoted keys,
+    // literal newlines, and stray quotes around long markdown descriptions.
+    const raw = [
+      "Parsing the requirements now...",
+      "{",
+      "  // generated draft",
+      "  summary: Story creation fails when output isn't clean JSON",
+      '  "description": ## Problem',
+      "",
+      'Some users see "Failed to parse story from agent output".',
+      "- Retry the command",
+      "- Or switch harness/model",
+      "}",
+    ].join("\n");
+    const parsed = parseAgentJson<Record<string, string>>(raw);
+    expect(parsed.summary).toBe("Story creation fails when output isn't clean JSON");
+    expect(parsed.description).toContain('"Failed to parse story');
+    expect(parsed.description).toContain("- Or switch harness/model");
+  });
+
+  test("does not mistake prose for keys during salvage", () => {
+    // No known keys anywhere -> salvage yields nothing, parse still throws.
+    expect(() =>
+      parseAgentJson("I could not produce anything useful. total: 42 failure."),
+    ).toThrow();
+  });
+});

packages/pm/chat-core.test.ts

@@ -201,8 +201,14 @@ describe("messages", () => {
 
   test("renderError maps engine codes to friendly text without leaking detail", () => {
     const msg = renderError(new EngineError("parse-failed", "bad json", "RAW AGENT DUMP"));
-    expect(msg.text).toContain("couldn't parse");
+    expect(msg.text).toContain("malformed output");
+    expect(msg.text).toContain("switch harness/model");
     expect(msg.text).not.toContain("RAW AGENT DUMP");
+    expect(
+      renderError(
+        new EngineError("parse-failed", "bad json", "d", "/tmp/devpm-story-generation-parse-1.log"),
+      ).text,
+    ).toContain("/tmp/devpm-story-generation-parse-1.log");
     expect(renderError(new EngineError("agent-failed", "x")).text).toContain("devpm serve");
     expect(renderError(new Error("plain")).text).toContain("plain");
   });

packages/pm/engine.test.ts

@@ -130,6 +130,9 @@ describe("extractJsonPayload", () => {
     expect(caught).toBeInstanceOf(EngineError);
     expect((caught as EngineError).code).toBe("parse-failed");
     expect((caught as EngineError).detail).toContain("could not produce JSON");
+    // Friendly, actionable headline — the low-level parser message stays out.
+    expect((caught as EngineError).message).toContain("malformed output");
+    expect((caught as EngineError).message).toContain("Retry");
   });
 
   test("throws parse-failed with invalidMessage when fields are missing", () => {
@@ -265,6 +268,103 @@ describe("createEngine", () => {
     expect((caught as EngineError).detail).toBe("boom");
   });
 
+  test("generateStory retries once with a corrective reminder after malformed output", async () => {
+    const prompts: string[] = [];
+    let calls = 0;
+    const engine = await createEngine(
+      stubConfig(),
+      { promptsDir: PROMPTS_DIR },
+      {
+        backend: stubBackend(),
+        runAgent: async (_harness, _path, prompt) => {
+          prompts.push(prompt);
+          calls += 1;
+          return {
+            stdout:
+              calls === 1
+                ? "Utter word salad, no object here."
+                : '{"summary": "S", "description": "D"}',
+            stderr: "",
+            exitCode: 0,
+            maxTurnsReached: false,
+          };
+        },
+      },
+    );
+
+    const draft = await engine.generateStory({
+      source: { type: "prompt", content: "x" },
+      promptStyle: "pm",
+    });
+    expect(draft).toEqual({ summary: "S", description: "D" });
+    expect(calls).toBe(2);
+    expect(prompts[1]).toContain("could not be parsed as JSON");
+    expect(prompts[1]).toContain(prompts[0] ?? "");
+  });
+
+  test("generateStory reports malformed output with a dump after exhausting the retry", async () => {
+    let calls = 0;
+    const engine = await createEngine(
+      stubConfig(),
+      { promptsDir: PROMPTS_DIR },
+      {
+        backend: stubBackend(),
+        runAgent: async () => {
+          calls += 1;
+          return {
+            stdout: `Still unparsable attempt ${calls}.`,
+            stderr: "",
+            exitCode: 0,
+            maxTurnsReached: false,
+          };
+        },
+      },
+    );
+
+    let caught: unknown;
+    try {
+      await engine.generateStory({
+        source: { type: "prompt", content: "x" },
+        promptStyle: "pm",
+      });
+    } catch (error) {
+      caught = error;
+    }
+    expect(calls).toBe(2);
+    expect(caught).toBeInstanceOf(EngineError);
+    expect((caught as EngineError).code).toBe("parse-failed");
+    expect((caught as EngineError).message).toContain("malformed output");
+    expect((caught as EngineError).dumpFile).toContain("devpm-story-generation-parse-");
+    expect((caught as EngineError).detail).toContain("Still unparsable attempt 2.");
+  });
+
+  test("generateStory does not retry when the agent itself fails", async () => {
+    let calls = 0;
+    const engine = await createEngine(
+      stubConfig(),
+      { promptsDir: PROMPTS_DIR },
+      {
+        backend: stubBackend(),
+        runAgent: async () => {
+          calls += 1;
+          return { stdout: "", stderr: "boom\n", exitCode: 1, maxTurnsReached: false };
+        },
+      },
+    );
+
+    let caught: unknown;
+    try {
+      await engine.generateStory({
+        source: { type: "prompt", content: "x" },
+        promptStyle: "pm",
+      });
+    } catch (error) {
+      caught = error;
+    }
+    expect(calls).toBe(1);
+    expect((caught as EngineError).code).toBe("agent-failed");
+  });
+
   test("generateStory streams agent chunks through events", async () => {
     const chunks: Array<[string, string]> = [];
     const engine = await createEngine(

packages/pm/index.ts

@@ -558,7 +558,7 @@ async function runCreateFlow(params: CreateFlowParams): Promise<boolean> {
           return true; // continue create-another loop
         }
         console.error("\n❌ Failed to parse story requirements from Agent output");
-        console.error("Error:", error.message);
+        console.error(error.message);
         console.error("Output:", error.detail);
         if (error.dumpFile) {
           console.error(`Full agent output (incl. stderr): ${error.dumpFile}`);
@@ -718,7 +718,7 @@ async function runCreateFlow(params: CreateFlowParams): Promise<boolean> {
       }
       if (error instanceof EngineError && error.code === "parse-failed") {
         console.error("\n❌ Failed to parse subtasks from Agent output");
-        console.error("Error:", error.message);
+        console.error(error.message);
         console.error("Output:", error.detail);
         if (error.dumpFile) {
           console.error(`Full agent output (incl. stderr): ${error.dumpFile}`);