fix: enforce the release policy end to end and serialise overseer decisions

.github/workflows/ci.yml

@@ -6,6 +6,14 @@ on:
   pull_request:
 
 jobs:
+  conflict-markers:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for merge conflict markers
+        run: "! git grep -nE '^(<<<<<<<|>>>>>>>)( |$)|^=======$'"
+
   test:
     runs-on: ubuntu-latest
     strategy:
@@ -26,4 +34,28 @@ jobs:
         run: pip install -e ".[dev]"
 
       - name: Run fast test suite (no network, no Docker, no FLARE simulator)
-        run: python -m pytest -q -m "not slow"
+        run: python -m pytest -q -m "not slow"
+
+  frontend:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        working-directory: frontend
+        run: npm ci
+
+      - name: Run tests
+        working-directory: frontend
+        run: npm test
+
+      - name: Build
+        working-directory: frontend
+        run: npm run build

AGENTS.md

@@ -26,10 +26,8 @@ Implemented:
 | Server side | `server/aggregate.py` (associative merge), `server/disclosure_check.py`, `server/overseer_queue.py` (CLI); outputs under `server/out/` (gitignored, `SERVER_OUT` env) |
 | Wire contracts | `adapters/base.py` (`AggregateResult`, `VariableStats`, `Gram` — the TRE→server payload, strict Pydantic), `server/schemas.py` (`MergedResult`, `ReleasedResult`, HTTP API models); JSON Schema + OpenAPI in `docs/schemas/` via `scripts/export_schemas.py` (a test fails when stale) |
 | HTTP API + UI | `server/api.py` serves the researcher UI contract (`/metadata`, `/examples/{name}`, `POST /run`, `/run/{id}`, `/overseer`, `/overseer/{hash}/{approve,reject}`, `/audit`) and the convenience endpoints (`/allele-frequency`, `/linear-regression`, `/logistic-regression`); `frontend/` (Next.js) is wired to it — run API on :8500, UI on :3000 |
-| Ops scripts | `scripts/local_federation.sh` (real FLARE, no Docker), `scripts/provision.sh`, `scripts/onboard_tre.sh`, `scripts/start_server.sh`, `scripts/start_client.sh`, `scripts/run_job.py` (`--mode simulator|prod`, `--fedavg`), `scripts/run_local.py`, `scripts/verify.py`, `scripts/scale_sim.py` → `docs/scaling.png` |
-| Tests | `tests/` — 111 fast + 4 FLARE-simulator (`-m slow`) |
-
-A real (non-simulator) FLARE federation runs two verified ways: `scripts/local_federation.sh` with separate processes over mTLS, and Docker via `scripts/provision.sh && scripts/up.sh --flare`; both support the example specs, overseer flow, and partial-site coverage. The Next.js UI is implemented in `frontend/`; the Python API service exposing its documented contract remains separate. Not yet run: clients on remote hosts (Gefion / NextCloud).
+| Ops scripts | `scripts/local_federation.sh` (real FLARE, no Docker), `scripts/provision.sh`, `scripts/limit_server_jobs.py` (both provisioning scripts run it: server kit runs one job at a time, `max_jobs: 1`), `scripts/onboard_tre.sh`, `scripts/start_server.sh`, `scripts/start_client.sh`, `scripts/run_job.py` (`--mode simulator|prod`, `--fedavg`), `scripts/run_local.py`, `scripts/verify.py`, `scripts/scale_sim.py` → `docs/scaling.png` |
+| Tests | `tests/` — 149 fast + 4 FLARE-simulator (`-m slow`); `frontend/test/` (`npm test`) |
 
 `docs/architecture/flowchart.drawio` is the editable source for the implemented-flow overview. When changing that overview, regenerate both `docs/architecture/flowchart_drawio.svg` and `docs/architecture/flowchart.png` from it.
 
@@ -45,15 +43,15 @@ Confirm a path, command, module, or function exists before referencing it — th
 
 Checks that exist and were confirmed working on 2026-09-17:
 
-- `python -m pytest -q -m "not slow"` — 111 passed, ~4 s, no network.
+- `python -m pytest -q -m "not slow"` — 149 tests as of 2026-09-25, ~6 s, no network.
 - `python -m pytest -q -m slow` — 4 passed, ~75 s; starts the TREs as local processes and runs FLARE simulator jobs (allele_freq, straggler, FedAvg linreg, Newton-Raphson logreg).
 - `scripts/dev_tres.py` + `scripts/run_job.sh spec/examples/<spec>.json` — simulator end-to-end with `scripts/verify.py` against ground truth.
 - `scripts/local_federation.sh up && scripts/local_federation.sh job spec/examples/allele_freq.json` — real FLARE server + 3 clients on localhost; ~12 s per job.
 - `scripts/scale_sim.py` — N ∈ {3, 10, 50, 100} clients, exact at every N; ~2 min.
 - `scripts/onboard_tre.sh <id>` — adds a site, regenerates, provisions, packs a kit (~2 s).
 - `scripts/provision.sh && scripts/up.sh --flare` — builds 7 images (~5 min cold), starts everything, checks each TRE's `/health` from its own FLARE client container, asserts no TRE container can reach the public internet. Then `docker compose exec flare-server python scripts/run_job.py --mode prod --admin-kit "/workspace/federated_apis/prod_00/admin@ncfh.org" spec/examples/allele_freq.json` (~11 s) and `python scripts/verify.py server/out/<spec_hash>/result.json` on the host. Verified 2026-09-17 with Docker Desktop 29.8 on macOS.
 
-CI: `.github/workflows/ci.yml` runs `python -m pytest -q -m "not slow"` on every push and pull request, against Python 3.11 and 3.12. No linter or formatter is configured.
+CI: `.github/workflows/ci.yml` runs on every push and pull request: a merge-conflict-marker check over tracked files, `python -m pytest -q -m "not slow"` against Python 3.11 and 3.12, and a frontend job (Node 24: `npm ci`, `npm test`, `npm run build`). No linter or formatter is configured.
 
 For documentation changes, check source accuracy, links, and `git diff --check`.
 

README.md

@@ -11,7 +11,7 @@
   <a href="https://github.com/collaborativebioinformatics/Bifrost/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/collaborativebioinformatics/Bifrost/actions/workflows/ci.yml/badge.svg"></a>
   <img alt="Python 3.11+" src="https://img.shields.io/badge/python-3.11%20%7C%203.12-3776AB?logo=python&logoColor=white">
   <img alt="NVIDIA FLARE 2.9" src="https://img.shields.io/badge/NVIDIA%20FLARE-2.9-76B900?logo=nvidia&logoColor=white">
-  <img alt="tests" src="https://img.shields.io/badge/tests-111%20fast%20%2B%204%20simulator-1FB89A">
+  <img alt="tests" src="https://img.shields.io/badge/tests-149%20fast%20%2B%204%20simulator-1FB89A">
   <a href="LICENSE"><img alt="MIT" src="https://img.shields.io/badge/license-MIT-lightgrey"></a>
 </p>
 
@@ -57,7 +57,7 @@ cd frontend && npm install && npm run dev                           # Next.js UI
 The fast suite needs no network; the FLARE simulator suite is marked `slow`:
 
 ```sh
-python -m pytest -q -m "not slow"    # 111 tests, ~3 s
+python -m pytest -q -m "not slow"    # 149 tests, ~6 s
 python -m pytest -q -m slow          # 4 tests, ~70 s
 ```
 
@@ -70,7 +70,7 @@ python scripts/dev_tres.py
 python scripts/run_local.py spec/examples/allele_freq.json
 ```
 
-**FLARE simulator** — one process, real controllers and executors. Each run writes `server/out/<spec_hash>/result.json`; `released.json` appears only when the disclosure check passes or an overseer approves, and is what would leave the server. Directories are keyed by spec hash, so an earlier `released.json` can outlive a later flagged run of the same spec — `check.json` holds the current decision.
+**FLARE simulator** — one process, real controllers and executors. Each run writes `server/out/<spec_hash>/result.json`; `released.json` appears only when the disclosure check passes or an overseer approves, and is what would leave the server. Directories are keyed by spec hash and describe the latest run: a flagged rerun withdraws an earlier `released.json`, a released rerun clears a stale queue entry, and the release log keeps the history.
 
 ```sh
 scripts/run_job.sh spec/examples/allele_freq.json
@@ -102,9 +102,9 @@ scripts/up.sh --down
 1. **Request** — a JSON `AnalysisSpec` in canonical variable names; hashed, and that hash keys every log.
 2. **Dispatch** — the FLARE server sits outside every TRE; clients dial **out** over mTLS gRPC. No inbound ports in a secure environment.
 3. **Local execution** — the FLARE client loads the adapter for its own `tre_id` and speaks the TRE's native API (REST, DataSHIELD-style, SQL gateway). Adapters can ask only for schema metadata and five aggregate primitives: `count`, `describe`, `value_counts`, `gram`, `irls_step`.
-4. **Safe output** — inside the TRE, before the FLARE client sees anything: project allow-list, aggregate allow-list, `n < k` ⇒ nothing leaves, cell `< k` suppressed (and anything derivable from it), one audit line per request and per round.
+4. **Safe output** — in the adapter, before anything leaves it: inside the TRE on the FLARE path (on the direct-API demo path the adapter runs in the API server process, so unfiltered aggregates reach that process first). Project allow-list, aggregate allow-list, `k` = the request's `min_cell_size` raised to the project's floor, `n < k` ⇒ nothing leaves, a variable observed in fewer than `k` rows ⇒ nothing about it leaves, cell `< k` suppressed (and anything derivable from it), regression inputs must be complete (a missing value rejects the site), one audit line per request and per round, accepted or rejected.
 5. **Aggregation** — everything the server merges is a sum (counts, Gram matrices, gradients and Hessians, n-weighted β), so merging is exact, order-free and straggler-tolerant: `min_clients` + `wait_time`, never wait-for-all; a missing site is reported as `2/3 sites`.
-6. **Disclosure check and release** — on the merged table: minimum cell size, dominance, ≥ 2 sites, site-level suppressions, differencing against earlier releases. Pass ⇒ `released.json`; flag ⇒ overseer queue, human decision logged.
+6. **Disclosure check and release** — on the merged table: minimum cell size, dominance (per table cell, per variable's observed values, per regression sample and per released federated round), ≥ 2 sites (overall and in every released round), site-level suppressions, differencing against earlier releases. Pass ⇒ `released.json`; flag ⇒ overseer queue, human decision logged.
 
 [Edit the diagram](docs/architecture/flowchart.drawio) · [Original external diagram](https://drive.google.com/file/d/1j9t8W-cFVBYgHGrFrLGP5keU2vaFtE-l/view?usp=sharing)
 

adapters/base.py

@@ -115,13 +115,15 @@ def value_counts(self, col: str, filters: dict) -> dict[str, int]:
 
     @abstractmethod
     def gram(self, cols: list[str], filters: dict) -> dict:
-        """Cross-product matrix of [1, *cols]: {"n", "cols", "matrix"}; matrix[i][j] = sum(x_i * x_j)."""
+        """Cross-product matrix of [1, *cols]: {"n", "cols", "matrix", "complete"}; matrix[i][j] = sum(x_i * x_j).
+        `complete` is False when any row misses any of `cols`; the matrix is then meaningless (None)."""
 
     @abstractmethod
     def irls_step(self, outcome_col: str, feature_cols: list[str], beta: list[float], filters: dict) -> dict:
         """One Newton-Raphson step's worth of sufficient statistics for logistic
         regression on [1, *feature_cols] -> outcome_col, evaluated at beta:
-        {"n", "grad" (len p+1), "hess" ((p+1)x(p+1))}."""
+        {"n", "complete", "grad" (len p+1), "hess" ((p+1)x(p+1))}; grad/hess are
+        None when `complete` is False (some row misses the outcome or a feature)."""
 
     # ---- shared logic ----------------------------------------------------------------
     def local(self, canonical: str) -> str:
@@ -137,8 +139,8 @@ def run(self, spec: AnalysisSpec) -> AggregateResult:
         filters = self.local_filters(spec)
         n = self.count(filters)
         raw: dict[str, dict[str, Any]] = {}
-        if n < spec.min_cell_size:  # don't even compute on a cohort that can't be released
-            return safe_output.filter(self.tre_id, self.region, spec, n, raw)
+        if n < safe_output.effective_min_cell_size(spec.project_id, spec.min_cell_size):
+            return safe_output.filter(self.tre_id, self.region, spec, n, raw)  # don't compute on an unreleasable cohort
 
         if spec.analysis_type in ("allele_freq", "fed_stats"):
             for v in spec.variables:
@@ -151,6 +153,8 @@ def run(self, spec: AnalysisSpec) -> AggregateResult:
         elif spec.analysis_type == "fed_linreg":
             cols = [spec.outcome, *spec.variables]
             g = self.gram([self.local(c) for c in cols], filters)
+            if g.get("complete") is not True:  # missing values: reject rather than fit a silently different cohort
+                return safe_output.reject_all(self.tre_id, self.region, spec, reason="_linreg.gram:incomplete_inputs")
             raw["_linreg"] = {"gram": {"n": g["n"], "cols": ["intercept", *cols], "matrix": g["matrix"]}}
 
         return safe_output.filter(self.tre_id, self.region, spec, n, raw)

adapters/rest.py

@@ -29,4 +29,4 @@ def gram(self, cols: list[str], filters: dict) -> dict:
 
     def irls_step(self, outcome_col: str, feature_cols: list[str], beta: list[float], filters: dict) -> dict:
         r = self._post("/irls", {"outcome": outcome_col, "features": feature_cols, "beta": beta, "filters": filters})
-        return {"n": r["n"], "grad": r["grad"], "hess": r["hess"]}
+        return {"n": r["n"], "complete": r.get("complete"), "grad": r.get("grad"), "hess": r.get("hess")}