fix: Stream rebuild source pages with publication fences

.github/workflows/ci.yml

@@ -125,10 +125,22 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ["3.12"]
         mcp-version: [minimum, latest]
+        pydantic-version: [latest]
         include:
           - os: ubuntu-latest
             python-version: "3.10"
             mcp-version: minimum
+            pydantic-version: latest
+          # Exercise the MCP/Pydantic floor pair and the release that removed
+          # eval_type_backport. Other dependencies still use normal resolution.
+          - os: ubuntu-latest
+            python-version: "3.10"
+            mcp-version: minimum
+            pydantic-version: "2.11.0"
+          - os: ubuntu-latest
+            python-version: "3.12"
+            mcp-version: minimum
+            pydantic-version: "2.14.0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python ${{ matrix.python-version }}
@@ -149,6 +161,7 @@ jobs:
         shell: python
         env:
           MCP_VERSION: ${{ matrix.mcp-version }}
+          PYDANTIC_VERSION: ${{ matrix.pydantic-version }}
         run: |
           import os
           from pathlib import Path
@@ -161,7 +174,9 @@ jobs:
           python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
           requirements = [str(wheel.resolve())]
           if os.environ["MCP_VERSION"] == "minimum":
-              requirements.append("mcp[cli]==1.19.0")
+              requirements.append("mcp[cli]==1.21.1")
+          if os.environ["PYDANTIC_VERSION"] != "latest":
+              requirements.append("pydantic==" + os.environ["PYDANTIC_VERSION"])
           subprocess.run([str(python), "-m", "pip", "install", *requirements], check=True)
           subprocess.run([str(python), "-m", "pip", "check"], check=True)
           subprocess.run([str(python), "-I", "tests/mcp-wheel-smoke.py"], check=True)

.github/workflows/publish.yml

@@ -37,6 +37,12 @@ jobs:
       fail-fast: false
       matrix:
         mcp-version: [minimum, latest]
+        pydantic-version: [latest]
+        include:
+          - mcp-version: minimum
+            pydantic-version: "2.11.0"
+          - mcp-version: minimum
+            pydantic-version: "2.14.0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
@@ -54,6 +60,7 @@ jobs:
         shell: python
         env:
           MCP_VERSION: ${{ matrix.mcp-version }}
+          PYDANTIC_VERSION: ${{ matrix.pydantic-version }}
         run: |
           import os
           from pathlib import Path
@@ -66,7 +73,9 @@ jobs:
           python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
           requirements = [str(wheel.resolve())]
           if os.environ["MCP_VERSION"] == "minimum":
-              requirements.append("mcp[cli]==1.19.0")
+              requirements.append("mcp[cli]==1.21.1")
+          if os.environ["PYDANTIC_VERSION"] != "latest":
+              requirements.append("pydantic==" + os.environ["PYDANTIC_VERSION"])
           subprocess.run([str(python), "-m", "pip", "install", *requirements], check=True)
           subprocess.run([str(python), "-m", "pip", "check"], check=True)
           subprocess.run([str(python), "-I", "tests/mcp-wheel-smoke.py"], check=True)

docs/resource-budgets.md

@@ -501,3 +501,124 @@ responsible for its serialization. The engine's existing manual path is unchange
 in this foundation checkpoint. Per-layer freshness enforcement, scheduler routing,
 vector dependency generations and measured maintenance performance remain later
 integration work; initially pending rows are not success claims.
+
+
+## Standalone rebuild source streaming (issue #756, first checkpoint)
+
+`build_vectors()` and `build_separation_vectors()` read database-backed inputs
+in ascending-ID pages, using the existing native batch size. They capture a
+fixed upper ID, row count and A10 source revision, including its epoch. Each
+page read ends before inference. Publication checks the source revision,
+model generation, target manifest and database schema version under writer
+ownership. Writer acquisition precedes the nonblocking model fence; that
+fence remains held through the terminal COMMIT or SAVEPOINT RELEASE. A source correction, deletion, lower-ID insertion, epoch change or
+schema change rejects the unfinished generation. An unrelated schema change
+is conservatively invalidating. New high-ID source rows remain outside the
+captured range.
+
+A versioned metadata manifest contains model/dimension/target identity,
+revision counters, the last **consumed input ID**, consumed count and inserted
+vector count. It contains no source text. Skipped zero/nonfinite vectors still
+advance consumed input, so restart cannot infer progress from `MAX(rowid)`.
+A saved manifest without matching identity and revision is restarted once on
+the next explicit call; the builder does not run an automatic restart loop.
+Legacy boolean markers alone are insufficient resume evidence.
+
+The native batch size and `txn_batch` durability cadence remain configurable.
+Each group is encoded and serialized before its owned writer transaction.
+Serialized payload is bounded by
+`max(1, txn_batch) * batch_size * dimension * 4` bytes per resident index,
+plus Python objects, one input page, the current native output and allocator
+memory. Defaults at 256 dimensions give:
+
+- CPU: `100 * 100 * 256 * 4 = 10,240,000` bytes (`9.765625 MiB`).
+- MPS: `100 * 16 * 256 * 4 = 1,638,400` bytes (`1.5625 MiB`).
+- Custom 384-dimensional CPU example: `100 * 100 * 384 * 4 = 15,360,000`
+  bytes (`14.6484375 MiB`).
+
+Completion and separation standalone calls stage their groups separately.
+This bound is independent of corpus rows with fixed controls, but can increase
+staging for small corpora compared with immediate insertion. It is not a
+whole-process memory cap. A single record is not truncated and can itself be
+large. Native model residency and thermal behavior are not measured by the
+synthetic regression tests.
+
+One maintenance owner covers the complete operation. Same-thread nesting in
+an existing A10 maintenance worker reuses its owner. A caller's open SQLite
+transaction is preserved with savepoints; the rebuild cannot release locks
+that the caller already owns. Such callers also retain control of durability.
+Otherwise, no rebuild-owned read or writer transaction remains open during
+inference. Short page snapshots avoid pinning a job-long WAL reader; they do
+not promise globally bounded WAL when other readers exist.
+
+Explicit `messages` lists preserve caller order and text. A streamed digest
+binds restart to that exact list without making another corpus-sized copy;
+the consumed list position preserves out-of-order IDs. The caller's allocated
+list is outside the database paging bound. Concurrent mutation of a supplied
+list is not supported. Database-backed calls reuse the canonical source
+tracker only with the exact built-in `messages` definition and verified tracking.
+Older connections, including deprecated
+`engine.open()` and the minimal five-column separation schema, instead install
+a rebuild-only bridge when canonical tracking is unavailable. Completion also
+supports a legacy table containing only `id INTEGER PRIMARY KEY` and `content`.
+
+The bridge adds one namespaced state table and three source triggers. It does
+not add columns to `messages`, migrate other storage features, or mark the
+canonical maintenance tracker ready. Installation and repair own the writer
+before model loading or clearing vectors; caller transactions use savepoints.
+The token explicitly identifies its tracker, epoch, covered source fields,
+table schema and index definitions. Page/publication checks verify that source
+schema and the exact trigger definitions; missing or altered tracking starts
+a new epoch on the next call, invalidating untracked intervals. These checks
+read schema metadata and scalar counters, not corpus text. Read-only legacy
+connections and unsupported source schemas fail before model allocation.
+Bridge updates compare each covered value's SQLite storage type and bytes,
+so custom `NOCASE`/`RTRIM` collations cannot hide changed encoder input.
+Other table definitions retain the bridge even after a full storage migration
+reports canonical tracking ready. This conservative definition whitelist avoids
+inferring equality semantics from arbitrary SQL; compatible custom definitions
+remain usable with the additional bridge counters.
+
+Same-ID replacement invalidates a bridge generation even with recursive
+triggers disabled. Custom schemas with unique indexes, including indexed primary keys that
+are not rowid aliases, use a conservative bridge: every successful insert or update invalidates the
+captured prefix. This includes expression and partial unique indexes, where
+`OR REPLACE` can silently delete another row without a delete trigger. Such
+schemas remain supported, but concurrent high-ID appends cannot preserve a
+running generation. They retain this bridge even if the canonical tracker
+reports ready, because its append certificate does not cover those hidden
+replacement deletions. There is no repeated automatic restart loop.
+
+When the trusted built-in definition and canonical tracking become suitable,
+the next rebuild changes tracker
+identity and starts a fresh generation; it atomically retires only the bridge's
+owned triggers. Its inactive state table remains. The bridge never certifies
+the readiness of other maintenance layers. Normal canonical schemas need no
+bridge table, triggers or additional writer commit during setup.
+
+Foreground engine add/update and `embed_single` bind inference to the active
+model generation. Same-model appends above an in-progress database range are
+published normally and cannot advance its consumed cursor. Initial clear and
+source capture share one writer transaction, so a committed earlier append
+joins the captured range. A correction within the range publishes its current
+vector and invalidates the rebuild's source fence. A late model mismatch is
+reported categorically: add rolls back its uncommitted source row; update's
+existing storage helper has already committed its source, so its error says
+to retry vector publication. Explicit-input or unattested in-progress targets
+reject concurrent publication rather than silently reporting success.
+
+Foreground publication acquires the SQLite writer before its nonblocking
+model fence, then validates the target manifest and retains both owners
+through the owned commit or caller savepoint release. Inference finishes
+before either is acquired. Add source insertion shares that transaction;
+rejection rolls back only the new add, preserving unrelated pending caller
+writes. Successful add and `embed_single` calls inside an existing caller
+transaction also leave durability to that caller. Update retains its existing
+source-helper commit behavior; the subsequent vector publication is fenced
+separately and cannot claim the source update was rolled back.
+
+This checkpoint does **not** remove the tier-switch manager/worker's retained
+message lists or solve tier configuration publication across SQLite, config
+file and process state. Issue #756 remains open for that next checkpoint.
+No global embedder-compatibility authority, models, precision, dimensions,
+reranker selection, retrieval depth or deployment defaults change here.

pyproject.toml

@@ -29,8 +29,9 @@ dependencies = [
     "model2vec>=0.6.0,<1.0",
     "sqlite-vec>=0.1.0,<1.0",
     "numpy>=1.24.0,<3.0",
-    # Tool metadata requires 1.19; SDK 2 removes the FastMCP import path.
-    "mcp[cli]>=1.19.0,<2.0",
+    # 1.21.1 removes a private Pydantic import broken by Pydantic 2.14.
+    # SDK 2 removes the FastMCP import path.
+    "mcp[cli]>=1.21.1,<2.0",
     "httpx>=0.27.0,<1.0",
     "sentence-transformers>=3.0.0,<7.0",
     "torch>=2.0.0,<3.0",

tests/mcp-wheel-smoke.py

@@ -182,6 +182,16 @@ async def _check_protocol(root: Path, env: dict[str, str], origin: Path) -> None
 def main() -> None:
     _require(bool(sys.flags.isolated), "Run this probe with python -I")
     distribution = importlib.metadata.distribution("truememory")
+    # Report public package versions before subprocess checks, whose output is
+    # intentionally redacted. Dependency import failures then remain traceable.
+    print(json.dumps({
+        "stage": "dependency_versions",
+        "python_version": ".".join(str(part) for part in sys.version_info[:3]),
+        "truememory_version": distribution.version,
+        "mcp_version": importlib.metadata.version("mcp"),
+        "pydantic_version": importlib.metadata.version("pydantic"),
+        "pydantic_core_version": importlib.metadata.version("pydantic-core"),
+    }), flush=True)
     origin = Path(distribution.locate_file("truememory/__init__.py")).resolve()
     spec = importlib.util.find_spec("truememory")
     _require(origin.is_file() and spec is not None and spec.origin is not None,