Feature/versioned recipes

.github/workflows/acceptance.yaml

@@ -0,0 +1,114 @@
+name: real-data acceptance
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  acceptance:
+    runs-on: ubuntu-latest
+    services:
+      arangodb:
+        image: arangodb:3.12
+        env:
+          ARANGO_NO_AUTH: "1"
+        ports:
+          - 8529:8529
+        options: >-
+          --health-cmd="wget --quiet --tries=1 --spider http://127.0.0.1:8529/_api/version || exit 1"
+          --health-interval=10s --health-timeout=5s --health-retries=20
+      clickhouse:
+        image: clickhouse/clickhouse-server:25.3
+        env:
+          CLICKHOUSE_DB: loom_acceptance
+          CLICKHOUSE_USER: loom_ci
+          CLICKHOUSE_PASSWORD: loom_ci_password
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+        ports:
+          - 8123:8123
+          - 9000:9000
+        options: >-
+          --health-cmd="wget --quiet --tries=1 --spider --user=loom_ci --password=loom_ci_password http://localhost:8123/ping || exit 1"
+          --health-interval=10s --health-timeout=5s --health-retries=20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Cache validated NCPI fixture
+        uses: actions/cache@v4
+        with:
+          path: .cache/acceptance/fixture
+          key: acceptance-fixture-${{ hashFiles('testdata/acceptance/ncpi-tcga-brca/fixture.lock.json') }}
+      - name: Run real-data acceptance
+        env:
+          LOOM_ACCEPTANCE_MODE: github
+          LOOM_ACCEPTANCE_ARTIFACTS: .artifacts/acceptance/github-${{ github.run_id }}
+          LOOM_ACCEPTANCE_FIXTURE_CACHE: .cache/acceptance/fixture
+          LOOM_ACCEPTANCE_CLICKHOUSE_USERNAME: loom_ci
+          LOOM_ACCEPTANCE_CLICKHOUSE_PASSWORD: loom_ci_password
+          GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+        run: make acceptance-performance
+      - name: Summarize acceptance evidence
+        if: always()
+        env:
+          ACCEPTANCE_ROOT: .artifacts/acceptance/github-${{ github.run_id }}
+        shell: bash
+        run: |
+          report="$ACCEPTANCE_ROOT/current/report.json"
+          cleanup="$ACCEPTANCE_ROOT/current/cleanup.json"
+          performance="$ACCEPTANCE_ROOT/performance.json"
+
+          if [[ ! -f "$report" ]]; then
+            echo "::warning::No acceptance report was produced. Inspect the acceptance step and uploaded evidence."
+            exit 0
+          fi
+
+          emit_summary() {
+            jq -r '
+              def stage($name): first(.stages[] | select(.name == $name));
+              (stage("generation_upload")) as $load |
+              (stage("execution_registry")) as $registry |
+              (stage("clickhouse_physical")) as $clickhouse |
+              (stage("graphql")) as $graphql |
+              (stage("idempotent_publish")) as $idempotent |
+              (stage("acceptance")) as $acceptance |
+              (.fixture.Counts | add) as $resources |
+              "Status: \(.status)",
+              "Fixture: \($resources) resources (\([.fixture.Counts | to_entries[] | "\(.key)=\(.value)"] | join(", ")))",
+              "ArangoDB: \($load.details.summary.vertices_inserted) vertices, \($load.details.summary.edges_inserted) edges, \($load.details.summary.generation_errors + $load.details.summary.edge_errors) load errors",
+              "Explorer: \($registry.details.state), idempotent republish reused execution=\($idempotent.details.execution_id)",
+              "ClickHouse: \($clickhouse.details.row_count) rows, \($clickhouse.details.unique_patients) unique patients",
+              "GraphQL: \($graphql.details.dataset.rowCount) rows, digest=\($graphql.details.row_digest)",
+              "Acceptance runtime: \($acceptance.seconds * 1000 | round / 1000)s"
+            ' "$report"
+
+            if [[ -f "$cleanup" ]]; then
+              jq -r '"Cleanup: " + (if .cleanup_status == 0 and .status == "0" then "PASSED" else "FAILED" end)' "$cleanup"
+            fi
+            if [[ -f "$performance" ]]; then
+              jq -r '"Performance: \(.status)" + (if .reason then " (\(.reason))" else "" end)' "$performance"
+            fi
+          }
+
+          {
+            echo "## Real-data acceptance"
+            echo
+            echo '```text'
+            emit_summary
+            echo '```'
+          } | tee -a "$GITHUB_STEP_SUMMARY"
+      - name: Upload acceptance evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: acceptance-${{ github.run_id }}
+          path: .artifacts/acceptance
+          if-no-files-found: warn

.gitignore

@@ -1,7 +1,13 @@
 .DS_Store
 .gocache/
+.cache/
+.artifacts/
 bin/
 data/
 META/
 /arango-fhir-proto
 /arango-fhir-server
+/ui/node_modules/
+/ui/**/node_modules/
+/ui/apps/*/dist/
+/ui/packages/*/dist/

Dockerfile

@@ -34,16 +34,25 @@ RUN --mount=type=cache,target=/go/pkg/mod \
       -trimpath \
       -ldflags="-s -w" \
       -o /out/arango-fhir-proto ./cmd/arango-fhir-proto
-
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    GOOS="$TARGETOS" GOARCH="$TARGETARCH" \
+    go build -mod=mod \
+      -trimpath \
+      -ldflags="-s -w" \
+      -o /out/loom-acceptance ./cmd/loom-acceptance
 FROM alpine:3.22
 RUN apk add --no-cache ca-certificates tzdata && \
     addgroup -S arango-fhir && \
-    adduser -S -G arango-fhir -h /app arango-fhir
+    adduser -S -G arango-fhir -h /app arango-fhir && \
+    mkdir -p /var/cache/loom /var/lib/loom/artifacts && \
+    chown -R arango-fhir:arango-fhir /var/cache/loom /var/lib/loom
 
 WORKDIR /app
 
 COPY --from=builder /out/arango-fhir-server /app/arango-fhir-server
 COPY --from=builder /out/arango-fhir-proto /app/arango-fhir-proto
+COPY --from=builder /out/loom-acceptance /app/loom-acceptance
 COPY --from=builder /src/schemas /app/schemas
 
 USER arango-fhir

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build build-cli build-server clean compiler-bench dataframe-demo dataframe-profile dataframe-boundaries dataframe-test conformance generate generate-openapi generate-fhir generate-graphql graphql-check gqlgen-check openapi-check test docker-build docker-run
+.PHONY: build build-cli build-server clean compiler-bench dataframe-demo dataframe-profile dataframe-boundaries dataframe-test conformance generate generate-openapi generate-fhir generate-graphql graphql-check gqlgen-check openapi-check test docker-build docker-run acceptance-real acceptance-performance demo-up demo-down demo-smoke demo-browser-smoke repository-up release-ui
 
 GO ?= go
 GO_VERSION ?= 1.26.5
@@ -18,8 +18,29 @@ DATAFRAME_VARIABLES ?= examples/meta_gdc_case_matrix.variables.json
 DATAFRAME_PROFILE_VARIABLES ?= examples/meta_gdc_case_matrix.variables.json
 DATAFRAME_PROFILE_LIMIT ?= 1000
 
+# Make treats the positional version as a second goal. Reject every ambiguous
+# form while parsing so no release recipe can start with the wrong version.
+ifneq ($(filter release-ui,$(MAKECMDGOALS)),)
+ifneq ($(firstword $(MAKECMDGOALS)),release-ui)
+$(error Usage: make release-ui X.Y.Z)
+endif
+ifneq ($(words $(MAKECMDGOALS)),2)
+$(error Usage: make release-ui X.Y.Z)
+endif
+ifeq ($(origin VERSION),command line)
+$(error Usage: do not set VERSION=; run make release-ui X.Y.Z)
+endif
+release-ui-version := $(word 2,$(MAKECMDGOALS))
+.PHONY: $(release-ui-version)
+$(release-ui-version):
+	@:
+endif
+
 build: build-cli build-server
 
+release-ui:
+	./scripts/release-loom-ui.sh release "$(release-ui-version)"
+
 generate: generate-fhir generate-graphql generate-openapi
 
 build-cli:
@@ -78,7 +99,7 @@ dataframe-boundaries:
 
 dataframe-test: dataframe-boundaries
 	mkdir -p $(GOCACHE_DIR)
-	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=$(GO_TOOLCHAIN) $(GO) test $(GOFLAGS) ./internal/dataframe/spec ./internal/dataframe/semantic ./internal/dataframe/compiler/ir ./internal/dataframe/compiler/lower ./internal/dataframe/compiler/optimize ./internal/dataframe/compiler/render/aql ./internal/dataframe/compiler ./internal/dataframe/runtime -count=1
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=$(GO_TOOLCHAIN) $(GO) test $(GOFLAGS) ./internal/dataframe/spec ./internal/dataframe/semantic ./internal/dataframe/compiler/ir ./internal/dataframe/compiler/lower ./internal/dataframe/compiler/optimize ./internal/dataframe/compiler/render/aql ./internal/dataframe/compiler ./internal/dataframe/execution -count=1
 
 conformance:
 	mkdir -p $(GOCACHE_DIR)
@@ -90,5 +111,28 @@ docker-build:
 docker-run:
 	docker run --rm -p 8080:8080 $(IMAGE)
 
+# Full real-data acceptance path. The script owns temporary Kubernetes
+# port-forwards, the current-worktree Loom process, and guarded run databases.
+acceptance-real:
+	./scripts/acceptance-real.sh
+
+acceptance-performance:
+	./scripts/acceptance-performance.sh
+
+demo-up:
+	./scripts/demo-up.sh
+
+demo-down:
+	./scripts/demo-down.sh
+
+demo-smoke:
+	./scripts/demo-smoke.sh
+
+demo-browser-smoke:
+	./scripts/demo-browser-smoke.sh
+
+repository-up:
+	./scripts/loom-repo-up.sh --repository "$(if $(REPOSITORY),$(REPOSITORY),$(CURDIR))"
+
 clean:
 	rm -rf bin

README.md

@@ -61,16 +61,16 @@ recipe, translation-version, and output selector.
 | `POST /graphql/graph` | Arango graph and recipe control plane: explicit graph traversal, typed FHIR reads, recipe validation, preview, execution, publication reads, and recipe control. |
 | `POST /graphql/dataframe` | Arango-backed FHIR dataframe compiler and executor (`runFhirDataframe`). |
 | `POST /api/v1/projects/:project/explorers/...` | REST Explorer lifecycle and V2 intent authoring used by the Builder. |
-| `PUT /api/v1/projects/:project/resources/:resourceType` | Primary multipart NDJSON resource loader. |
+| `POST /api/v1/datasets/:project/generations/:generation` | Multipart immutable-generation loader used by ETL. |
 
 `GET /graphql/graph` serves GraphQL Playground for the graph API. `GET /apollo`
 opens Apollo Sandbox pointed at `/graphql/graph`. There is intentionally no
 `/graphql` compatibility route.
 
 The canonical contract for every server route is
-[`openapi/openapi.yaml`](openapi/openapi.yaml). It includes health, ingestion,
-snapshot/release, recipe execution, GraphQL transport, Explorer lifecycle, and
-Explorer authoring operations. The `:project` path parameter is the tenancy
+[`openapi/openapi.yaml`](openapi/openapi.yaml). It includes health, generation
+ingestion/activation, recipe execution, GraphQL transport, Explorer lifecycle,
+and Explorer authoring operations. The `:project` path parameter is the tenancy
 identity used for authorization and becomes the published row `project_id`
 (for example, `HTAN_INT-BForePC`).
 
@@ -98,7 +98,22 @@ HTTP endpoint is required. The ClickHouse account needs database creation plus
 
 ## Local development
 
-The local Compose stack starts both ArangoDB and ClickHouse:
+To run the complete no-auth Explorer demo with the locked FHIR Aggregator
+sample, use:
+
+```bash
+make demo-up
+```
+
+Open [the standalone Explorer](http://127.0.0.1:3080). See
+[Run the standalone Explorer demo](docs/DEMO_QUICKSTART.md) for the seeded
+dataset, smoke checks, logs, and reset commands.
+
+To launch from a separate repository's hydrated `META` data and native Loom
+`CONFIG` workspace, including local CONFIG write-back on Publish, see
+[Launch Loom from a data repository](docs/LOCAL_REPOSITORY_DEMO.md).
+
+For backend-only development, the lightweight Compose stack starts ArangoDB and ClickHouse:
 
 ```bash
 rtk docker compose -f experimental/docker-compose.yml up -d
@@ -239,17 +254,12 @@ query ExplorerRows($input: DataframeRowsInput!) {
 }
 ```
 
-The reader requires the exact selector `(recipe, translationVersion, output)`.
-There is no server-side default recipe or `dataType` alias. Loom derives the
-authorized project set from the principal, selects each project's active
-pointer-backed publication, reconciles compatible outputs, and federates them
-into one logical dataframe. Every row exposes `project_id`,
-derived from the publication project identity (for example,
-`HTAN_INT-BForePC`), and it is filterable and sortable like other scalar
-columns. Rows remain permissive JSON; columns and capabilities are discovered
-at runtime so a new publication does not require a GraphQL regeneration or
-server restart. See [`docs/DATAFRAME_FEDERATION.md`](docs/DATAFRAME_FEDERATION.md)
-for the source-selection, authorization, schema, and operational contract.
+The reader requires an explicit `projectId` and exact selector
+`(recipe, translationVersion, output)`. There is no server-side default recipe,
+`dataType` alias, or cross-project federation. Loom authorizes the requested
+project, resolves its active pointer-backed publication, and reads that one
+physical dataframe. Columns and capabilities remain runtime-discovered, so a
+new publication does not require GraphQL regeneration or a server restart.
 
 ## Authorization and tenancy
 
@@ -310,12 +320,12 @@ without rebuilding the server image.
 | [`internal/dataframe/compiler`](internal/dataframe/compiler) | Typed plan IR, lowering, optimization, and AQL rendering. |
 | [`internal/dataframe/recipe`](internal/dataframe/recipe) | Recipe contract, validation, schema resolution, execution, and control services. |
 | [`internal/dataframe/publication`](internal/dataframe/publication) | Backend-neutral bounded streaming publication contract. |
-| [`internal/dataframe/published`](internal/dataframe/published) | Safe published-data reads and federation. |
+| [`internal/dataframe/published`](internal/dataframe/published) | Safe single-project published-data reads and aggregates. |
 | [`internal/store/arango`](internal/store/arango) | ArangoDB boundary. |
 | [`internal/store/clickhouse`](internal/store/clickhouse) | Typed ClickHouse driver boundary and DDL/DML. |
 | [`internal/api/graphql/graph`](internal/api/graphql/graph) | GraphQL HTTP transport and error presentation. |
 | [`internal/api/graphql/graph/query`](internal/api/graphql/graph/query) | Arango graph and FHIR dataframe API services. |
-| [`internal/api/graphql/graph/materialization`](internal/api/graphql/graph/materialization) | Published-dataframe transport mapping. |
+| [`internal/api/graphql/graph/dataframe`](internal/api/graphql/graph/dataframe) | Published-dataframe reads, aggregates, export, and GraphQL model mapping. |
 
 ## Build, generation, and tests