fix: Report incomplete ingestion parsing and extraction

.github/workflows/ci.yml

@@ -125,10 +125,22 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ["3.12"]
         mcp-version: [minimum, latest]
+        pydantic-version: [latest]
         include:
           - os: ubuntu-latest
             python-version: "3.10"
             mcp-version: minimum
+            pydantic-version: latest
+          # Exercise the MCP/Pydantic floor pair and the release that removed
+          # eval_type_backport. Other dependencies still use normal resolution.
+          - os: ubuntu-latest
+            python-version: "3.10"
+            mcp-version: minimum
+            pydantic-version: "2.11.0"
+          - os: ubuntu-latest
+            python-version: "3.12"
+            mcp-version: minimum
+            pydantic-version: "2.14.0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python ${{ matrix.python-version }}
@@ -149,6 +161,7 @@ jobs:
         shell: python
         env:
           MCP_VERSION: ${{ matrix.mcp-version }}
+          PYDANTIC_VERSION: ${{ matrix.pydantic-version }}
         run: |
           import os
           from pathlib import Path
@@ -161,7 +174,9 @@ jobs:
           python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
           requirements = [str(wheel.resolve())]
           if os.environ["MCP_VERSION"] == "minimum":
-              requirements.append("mcp[cli]==1.19.0")
+              requirements.append("mcp[cli]==1.21.1")
+          if os.environ["PYDANTIC_VERSION"] != "latest":
+              requirements.append("pydantic==" + os.environ["PYDANTIC_VERSION"])
           subprocess.run([str(python), "-m", "pip", "install", *requirements], check=True)
           subprocess.run([str(python), "-m", "pip", "check"], check=True)
           subprocess.run([str(python), "-I", "tests/mcp-wheel-smoke.py"], check=True)

.github/workflows/publish.yml

@@ -37,6 +37,12 @@ jobs:
       fail-fast: false
       matrix:
         mcp-version: [minimum, latest]
+        pydantic-version: [latest]
+        include:
+          - mcp-version: minimum
+            pydantic-version: "2.11.0"
+          - mcp-version: minimum
+            pydantic-version: "2.14.0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
@@ -54,6 +60,7 @@ jobs:
         shell: python
         env:
           MCP_VERSION: ${{ matrix.mcp-version }}
+          PYDANTIC_VERSION: ${{ matrix.pydantic-version }}
         run: |
           import os
           from pathlib import Path
@@ -66,7 +73,9 @@ jobs:
           python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
           requirements = [str(wheel.resolve())]
           if os.environ["MCP_VERSION"] == "minimum":
-              requirements.append("mcp[cli]==1.19.0")
+              requirements.append("mcp[cli]==1.21.1")
+          if os.environ["PYDANTIC_VERSION"] != "latest":
+              requirements.append("pydantic==" + os.environ["PYDANTIC_VERSION"])
           subprocess.run([str(python), "-m", "pip", "install", *requirements], check=True)
           subprocess.run([str(python), "-m", "pip", "check"], check=True)
           subprocess.run([str(python), "-I", "tests/mcp-wheel-smoke.py"], check=True)

docs/resource-budgets.md

@@ -477,3 +477,84 @@ same cases against real sqlite-vec tables when the extension is available. These
 checks establish paired-write and retry behavior, not native MPS OOM behavior,
 thermal improvement or model-performance measurements. Whole-rebuild streaming
 and corpus loading remain separate work.
+## Ingestion outcome evidence (issue #768, foundation only)
+
+`parse_transcript_outcome(source)` and `extract_facts_outcome(...)` expose
+coverage without changing pipeline or CLI completion. The existing
+`parse_transcript`, `extract_facts`, and `_parse_extraction_response` list
+contracts remain available. The transcript compatibility entry retains its
+existing parsing/read behavior. The LLM compatibility entries return the
+same selected facts from the shared detailed implementation; they do not
+consume its completion status. Heuristic extraction is unchanged.
+
+The detailed transcript statuses are:
+
+| Status | Meaning |
+| --- | --- |
+| `complete` | Recognized input was parsed, including valid empty files/arrays and intentional non-conversation records |
+| `partial` | Coverage is incomplete; supported messages remain available, but malformed records, unrecognized content, invalid UTF-8 or a changed capture prevent completion |
+| `salvaged` | An invalid JSON array used the legacy plain-text salvage path |
+| `malformed` | The input was not a usable recognized transcript, or capture/decoding failed without usable messages |
+| `unreadable` | A file could not be opened or read |
+
+An explicit `Path` is always a file, including when it is missing. A string
+retains the existing file-or-inline convention. The detailed file API uses
+the `.jsonl` suffix to recognize malformed first lines; the legacy parser's
+format selection remains unchanged. Source outcomes carry malformed-record
+counts and categorical diagnostics. Message bodies are excluded from their
+diagnostic representation.
+
+Legacy top-level `tool_use` records still use their existing name/input
+representation. Known `thinking` blocks are deliberately excluded. A text
+block must have a string `text` field; an explicitly empty string is valid,
+while a missing or non-string field is malformed. Images, audio, other
+nontext blocks and unknown block types are unrecognized coverage, even when
+they contain a text-looking key. No OCR or new decoder is added. Such records
+are `partial`, including when no supported text remains, and increment
+`unrecognized_records`. Mixed records retain supported text while reporting
+loss; each affected record increments each applicable coverage counter once.
+The legacy list parser's filtering and outputs remain unchanged.
+
+File versions contain captured byte count and SHA-256, plus descriptor
+device/inode/size/mtime/ctime before and after reading and the path's identity
+at the end of capture. No path or raw transcript is stored in that metadata.
+`stable` means those states agree and byte count equals captured size. It
+does not assert that the source stayed unchanged after this API returned.
+A future completion marker must bind this captured version; a later stat
+cannot acknowledge unread appended bytes. This first API still reads a full
+file, retaining raw bytes while decoding; it is not a streaming-memory claim.
+
+LLM extraction statuses are `complete`, `partial`, `failed`, and `limited`.
+Valid empty arrays and supported empty object wrappers are complete. Provider
+failures, wrong response shapes, all-invalid item lists and brace-salvaged
+responses are distinguishable. Valid facts from incomplete responses remain
+available for legacy compatibility; their presence never proves completion.
+In particular, the old array-first parser can select a list inside an unknown
+object wrapper. Detailed status rejects that envelope while preserving the
+legacy fact selection.
+
+`attempted_chunks = successful_chunks + partial_chunks + failed_chunks` and
+`total_chunks = attempted_chunks + deferred_chunks`. Successful chunks count
+valid responses, including responses limited by the fact cap. Invalid
+response counts exclude provider calls that failed before returning a
+response. Counts separately describe malformed items, all omitted response
+items, valid fact occurrences omitted by per-chunk slicing, and unique
+merged facts omitted by the final cap. These counts can overlap; they are
+not a unique-fact coverage percentage. Failure/partial status takes precedence
+over `limited`, with both limitations retained in counts and categories.
+
+Provider calls and caps are unchanged: 20,000 characters per ordinary chunk,
+20 selected chunks by default, and 50 facts per response selection and final
+merge. A single oversized message remains intact under the existing chunker.
+For 21 chunks, `21 - 20 = 1` is deferred and exactly 20 provider calls occur.
+No new retries or fallback providers are introduced. Existing malformed-JSON
+salvage can return more than the per-response cap; its facts still pass through
+the existing final merge cap. This compatibility behavior is explicitly
+reported as incomplete instead of being silently redesigned here. Provider
+failure logs contain fixed categories and chunk counts, without raw provider
+errors or tracebacks.
+
+This checkpoint does not activate strict ingestion, change exit codes, hold
+backlog jobs, add receipts, make mutation retries safe, or fix CLI success
+markers. Durable pre-mutation guards and storage receipts remain required
+before those routes can safely change. Issue #768 remains open.

pyproject.toml

@@ -29,8 +29,9 @@ dependencies = [
     "model2vec>=0.6.0,<1.0",
     "sqlite-vec>=0.1.0,<1.0",
     "numpy>=1.24.0,<3.0",
-    # Tool metadata requires 1.19; SDK 2 removes the FastMCP import path.
-    "mcp[cli]>=1.19.0,<2.0",
+    # 1.21.1 removes a private Pydantic import broken by Pydantic 2.14.
+    # SDK 2 removes the FastMCP import path.
+    "mcp[cli]>=1.21.1,<2.0",
     "httpx>=0.27.0,<1.0",
     "sentence-transformers>=3.0.0,<7.0",
     "torch>=2.0.0,<3.0",

tests/mcp-wheel-smoke.py

@@ -182,6 +182,16 @@ async def _check_protocol(root: Path, env: dict[str, str], origin: Path) -> None
 def main() -> None:
     _require(bool(sys.flags.isolated), "Run this probe with python -I")
     distribution = importlib.metadata.distribution("truememory")
+    # Report public package versions before subprocess checks, whose output is
+    # intentionally redacted. Dependency import failures then remain traceable.
+    print(json.dumps({
+        "stage": "dependency_versions",
+        "python_version": ".".join(str(part) for part in sys.version_info[:3]),
+        "truememory_version": distribution.version,
+        "mcp_version": importlib.metadata.version("mcp"),
+        "pydantic_version": importlib.metadata.version("pydantic"),
+        "pydantic_core_version": importlib.metadata.version("pydantic-core"),
+    }), flush=True)
     origin = Path(distribution.locate_file("truememory/__init__.py")).resolve()
     spec = importlib.util.find_spec("truememory")
     _require(origin.is_file() and spec is not None and spec.origin is not None,