From 7fad78c36742f6bc7fcf378ded3963648a44e239 Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:20:31 -0400 Subject: [PATCH 01/12] Document DO NOT MERGE query materialization POC --- .../do-not-merge-query-materialization-poc.md | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 docs/do-not-merge-query-materialization-poc.md diff --git a/docs/do-not-merge-query-materialization-poc.md b/docs/do-not-merge-query-materialization-poc.md new file mode 100644 index 00000000000..fe82fda6813 --- /dev/null +++ b/docs/do-not-merge-query-materialization-poc.md @@ -0,0 +1,154 @@ +# DO NOT MERGE: query materialization POC + +This branch prepares an isolated experiment for Classroom Central. It is not a +production change and must remain a draft. Do not merge it, enable auto-merge, +deploy it to production, or write experimental changes into the source realm. + +## Environments and scope + +- Base: `main` at `e9a4b0a54a`, fetched September 10, 2026. +- Branch: `codex/do-not-merge-query-materialization-poc`. +- Source deployment realm: `https://app.boxel.ai/tribecaprep/nucleus-lms/`. +- Development fork: + `https://realms-staging.stack.cards/ctse/nucleus-lms-query-materialization-poc/`. +- The staging fork is private to its owner. Source-realm references are remapped + to the fork; independent external dependencies remain external references. +- Realm source/data stay outside the monorepo. Do not commit student records, + source archives, credentials, or local sync history here. + +The latest instruction selects staging for active development. It supersedes the +earlier proposed development location under the production account. Nucleus LMS +remains a deployment realm for separately approved promotions and migrations. + +Initial branch status: environment setup and implementation plan only. The +runtime feature and the realm summary definitions have not been implemented. + +Fork preparation: the source archive contains 1,411 files (about 9 MB). The 154 +definition/support files have been copied; all 29 card types used by the source +instances passed schema generation on staging. The full 1,257-file JSON import +is prepared with source-realm URL remapping and is awaiting explicit approval +for copying student data into the staging account. Until then, the target holds +its default configuration plus the copied definitions, not the source dataset. + +## Intended outcome + +Materialize the shared classroom/day lists and statistics during indexing. A +dashboard reader should consume those saved results without hydrating the input +graph to repeat the computation. Source changes should precisely identify which +saved summaries need recomputation, even when the previous query result was empty. + +Keep the current Boxel query language and `computeVia`. The first experiment is +same-realm, with deterministic predicates needed by the classroom workload. +Queries run when an affected owner is reindexed; this is incremental invalidation +followed by recomputation, not arithmetic maintenance of arbitrary aggregates. + +## Recheck against current main + +The preceding investigation used `730081f8b4`. Main has advanced substantially. +Do not implement directly from its old line references or assume its complete +client behavior still applies. + +Confirmed in the new base: + +- `packages/base/field-support.ts` still invokes `computeVia` from a computed + getter, including when deserialization supplied a value. +- `packages/host/app/routes/render/meta.ts` still serializes with + `omitQueryFields: true` and excludes query-only runtime dependencies. +- `packages/base/query-field-support.ts` now supports eager query resolution, + an `eager: false` opt-out, and newer-document seed handover with realm-specific + generation ordering. Snapshot mode must integrate with these paths. +- `packages/host/app/services/store.ts` now exposes selective search-entry + inflation and scoped card-initiated searches. Reassess compound-document reuse + and live-search caching before adding another cache or hydration mechanism. + +## Step 1: consume server materialization + +1. Introduce a narrow opt-in for clean indexed instances that records which + computed values and query memberships were successfully supplied. Include + provenance, completeness, errors and revision information. +2. Let computed getters use those supplied values without invoking `computeVia`. + Preserve successful zero, false, null and empty results. Apply the mode to + nested contained values as well. +3. Retain resolved membership separately from hydrated member instances. Avoid + rerunning the search or loading every member merely to recover known IDs. +4. Reuse returned raw resources for lazy hydration where current main does not + already do so. Preserve canonical instance identity and bounded cache lifetime. +5. Refresh snapshots on owner invalidation. Keep editing, unsaved cards, + indexing and source-file hydration on the appropriate live-computation path. +6. Ensure a local leaf write refreshes dependent summaries in that same client; + self-originated invalidation suppression must only protect the actual edit. +7. Serve a materialized owner's stored query membership without dynamically + replacing it at GET time or recursively assembling its input graph. + +Acceptance: displaying supplied results makes zero input-dependency requests and +zero computed-getter calls. Opening an actual source record may hydrate it. + +## Step 2: reverse query dependencies + +Persist a watch for each opted-in owner/field during indexing. A watch contains +the resolved Boxel query, source scope/type, owner identity and definition +revision. Register empty results too. There is no new author-facing predicate +language and no Elasticsearch dependency. + +For a document `d`, reverse matching is the set of saved queries whose filters +match `d`. For a source change, consider queries matching either the old or new +indexed document. A matching record's content change can affect a computation +without changing result membership. + +1. Extract conservative routing terms into an indexed lookup table. Initially + support mandatory equality or `in` terms, with broad fallback buckets when + safe extraction is impossible. Candidate selection may over-select but must + never miss a matching watch. +2. Verify candidates with the existing Boxel matching semantics. Reuse/factor the + query compiler as appropriate; test nulls, plural paths, type inheritance, + and the different reference-normalization rules of `eq` and `in`. +3. Capture old/new effective indexed rows for inserts, updates, deletions, + dependency-driven changes, and transitions into or out of indexing errors. +4. Retain concrete dependencies on query results and transitive inputs actually + consumed. Reverse watches supplement these edges by discovering new matches. +5. Queue affected owners after source data is visible. Coalesce bursts and + persist pending work; notifications are wakeups, not the durable state. +6. Publish watch registration, membership and computed values consistently. + Prevent registration/write races and an older worker clearing newer dirty + work. Cover restart recovery, definition changes and deleted owners. +7. Propagate changed feeder outputs to their consumers, with cycle controls and + full-rebuild convergence. Treat sorted/paginated queries conservatively: + an unseen matching row can enter the returned page. + +Use the standard PostgreSQL migration workflow and regenerate the SQLite schema. +Do not install a database extension or replace the existing indexing queue. + +## Realm adaptation + +Create a persisted classroom/day summary in the staging fork, with scoped query +fields and ordinary computed contained rows/statistics. Reuse the business rules +currently in `schema/classroom-day.gts`, including slot composition, coverage, +report versions and pipeline state. + +Keep selected child/date navigation, drafts, unsaved records, generation progress, +and clock-based presentation local. Store stable deadlines and shared facts. +Load full source cards when an interaction needs them. Do not mistake current +student location for dated historical attendance. + +Avoid introducing a projected-query API unless measurements show that the +remaining server indexing cost requires it. Compact display rows can use existing +contained fields and `computeVia`. + +## Validation and stopping condition + +- Record the fork's source fidelity and existing indexing/lint baseline before + editing its definitions. +- Exercise empty-to-first-match, deletion, classroom/date moves, content-only + updates, pagination changes, transitive inputs and query-parameter changes. +- Verify unrelated classroom writes do not refresh the summary and repeated + relevant writes coalesce. Test feeder chains and concurrent registration/write. +- Measure cold dashboard load, network requests/bytes, client computation, + source-write-to-summary freshness, and indexing work with multiple readers. +- Run focused host/realm-server tests, Glint where appropriate, and package lint. + Capture full host test output. Do not run the entire host suite locally. +- Keep the PR draft and titled DO NOT MERGE, including after successful tests. + Report results and limitations; production promotion is a separate decision. + +Working estimate, subject to revalidation against this base: 1,600–2,700 monorepo +implementation lines, 1,400–2,300 focused test lines, and about 600–1,000 changed +realm GTS lines, including relocation of existing logic. From 46bc4676d0b8a2720814cbd11c94ef80ecdfc0b1 Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:24:54 -0400 Subject: [PATCH 02/12] Record synthetic-only benchmarks and paused POC checkpoint --- .../do-not-merge-query-materialization-poc.md | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/do-not-merge-query-materialization-poc.md b/docs/do-not-merge-query-materialization-poc.md index fe82fda6813..67b33544007 100644 --- a/docs/do-not-merge-query-materialization-poc.md +++ b/docs/do-not-merge-query-materialization-poc.md @@ -11,8 +11,8 @@ deploy it to production, or write experimental changes into the source realm. - Source deployment realm: `https://app.boxel.ai/tribecaprep/nucleus-lms/`. - Development fork: `https://realms-staging.stack.cards/ctse/nucleus-lms-query-materialization-poc/`. -- The staging fork is private to its owner. Source-realm references are remapped - to the fork; independent external dependencies remain external references. +- The staging fork is private to its owner and contains definitions only, plus + its default configuration. Do not copy production records into this fork. - Realm source/data stay outside the monorepo. Do not commit student records, source archives, credentials, or local sync history here. @@ -23,12 +23,24 @@ remains a deployment realm for separately approved promotions and migrations. Initial branch status: environment setup and implementation plan only. The runtime feature and the realm summary definitions have not been implemented. -Fork preparation: the source archive contains 1,411 files (about 9 MB). The 154 -definition/support files have been copied; all 29 card types used by the source -instances passed schema generation on staging. The full 1,257-file JSON import -is prepared with source-realm URL remapping and is awaiting explicit approval -for copying student data into the staging account. Until then, the target holds -its default configuration plus the copied definitions, not the source dataset. +Paused at the user's request on September 10, 2026. No runtime implementation, +synthetic fixture generation, or benchmarks have started. Resume only when asked. + +Fork preparation: 154 definition/support files have been copied and verified +against their prepared source hashes; all 29 card types used by the source +instances passed schema generation on staging. No production records were +uploaded. The temporary production archive, 1,257 prepared JSON files, import +inventory and temporary source authentication cache have been removed. Only +aggregate counts remain for sizing synthetic fixtures. + +The benchmark data must be newly generated and deterministic, with fabricated +identities, narratives and relationships. Do not anonymize or reuse production +records. Initial planned sizes are 1,255 instances (approximately the current +production count, excluding realm/index configuration), 12,550 instances and +125,500 instances. These are planned fixture sizes, not generated datasets. +Preserve a representative mix of card types. Vary total realm size separately +from matching records per classroom/day and graph fan-out so the measurements +distinguish unrelated data growth from growth of an individual materialization. ## Intended outcome From e79e7ec00143ef1e335019a94bb766b3e8a6ccb9 Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:06:29 -0400 Subject: [PATCH 03/12] Plan Tessar query materialization POC --- .../do-not-merge-query-materialization-poc.md | 166 -------- docs/tessar-query-materialization-plan.md | 378 ++++++++++++++++++ 2 files changed, 378 insertions(+), 166 deletions(-) delete mode 100644 docs/do-not-merge-query-materialization-poc.md create mode 100644 docs/tessar-query-materialization-plan.md diff --git a/docs/do-not-merge-query-materialization-poc.md b/docs/do-not-merge-query-materialization-poc.md deleted file mode 100644 index 67b33544007..00000000000 --- a/docs/do-not-merge-query-materialization-poc.md +++ /dev/null @@ -1,166 +0,0 @@ -# DO NOT MERGE: query materialization POC - -This branch prepares an isolated experiment for Classroom Central. It is not a -production change and must remain a draft. Do not merge it, enable auto-merge, -deploy it to production, or write experimental changes into the source realm. - -## Environments and scope - -- Base: `main` at `e9a4b0a54a`, fetched September 10, 2026. -- Branch: `codex/do-not-merge-query-materialization-poc`. -- Source deployment realm: `https://app.boxel.ai/tribecaprep/nucleus-lms/`. -- Development fork: - `https://realms-staging.stack.cards/ctse/nucleus-lms-query-materialization-poc/`. -- The staging fork is private to its owner and contains definitions only, plus - its default configuration. Do not copy production records into this fork. -- Realm source/data stay outside the monorepo. Do not commit student records, - source archives, credentials, or local sync history here. - -The latest instruction selects staging for active development. It supersedes the -earlier proposed development location under the production account. Nucleus LMS -remains a deployment realm for separately approved promotions and migrations. - -Initial branch status: environment setup and implementation plan only. The -runtime feature and the realm summary definitions have not been implemented. - -Paused at the user's request on September 10, 2026. No runtime implementation, -synthetic fixture generation, or benchmarks have started. Resume only when asked. - -Fork preparation: 154 definition/support files have been copied and verified -against their prepared source hashes; all 29 card types used by the source -instances passed schema generation on staging. No production records were -uploaded. The temporary production archive, 1,257 prepared JSON files, import -inventory and temporary source authentication cache have been removed. Only -aggregate counts remain for sizing synthetic fixtures. - -The benchmark data must be newly generated and deterministic, with fabricated -identities, narratives and relationships. Do not anonymize or reuse production -records. Initial planned sizes are 1,255 instances (approximately the current -production count, excluding realm/index configuration), 12,550 instances and -125,500 instances. These are planned fixture sizes, not generated datasets. -Preserve a representative mix of card types. Vary total realm size separately -from matching records per classroom/day and graph fan-out so the measurements -distinguish unrelated data growth from growth of an individual materialization. - -## Intended outcome - -Materialize the shared classroom/day lists and statistics during indexing. A -dashboard reader should consume those saved results without hydrating the input -graph to repeat the computation. Source changes should precisely identify which -saved summaries need recomputation, even when the previous query result was empty. - -Keep the current Boxel query language and `computeVia`. The first experiment is -same-realm, with deterministic predicates needed by the classroom workload. -Queries run when an affected owner is reindexed; this is incremental invalidation -followed by recomputation, not arithmetic maintenance of arbitrary aggregates. - -## Recheck against current main - -The preceding investigation used `730081f8b4`. Main has advanced substantially. -Do not implement directly from its old line references or assume its complete -client behavior still applies. - -Confirmed in the new base: - -- `packages/base/field-support.ts` still invokes `computeVia` from a computed - getter, including when deserialization supplied a value. -- `packages/host/app/routes/render/meta.ts` still serializes with - `omitQueryFields: true` and excludes query-only runtime dependencies. -- `packages/base/query-field-support.ts` now supports eager query resolution, - an `eager: false` opt-out, and newer-document seed handover with realm-specific - generation ordering. Snapshot mode must integrate with these paths. -- `packages/host/app/services/store.ts` now exposes selective search-entry - inflation and scoped card-initiated searches. Reassess compound-document reuse - and live-search caching before adding another cache or hydration mechanism. - -## Step 1: consume server materialization - -1. Introduce a narrow opt-in for clean indexed instances that records which - computed values and query memberships were successfully supplied. Include - provenance, completeness, errors and revision information. -2. Let computed getters use those supplied values without invoking `computeVia`. - Preserve successful zero, false, null and empty results. Apply the mode to - nested contained values as well. -3. Retain resolved membership separately from hydrated member instances. Avoid - rerunning the search or loading every member merely to recover known IDs. -4. Reuse returned raw resources for lazy hydration where current main does not - already do so. Preserve canonical instance identity and bounded cache lifetime. -5. Refresh snapshots on owner invalidation. Keep editing, unsaved cards, - indexing and source-file hydration on the appropriate live-computation path. -6. Ensure a local leaf write refreshes dependent summaries in that same client; - self-originated invalidation suppression must only protect the actual edit. -7. Serve a materialized owner's stored query membership without dynamically - replacing it at GET time or recursively assembling its input graph. - -Acceptance: displaying supplied results makes zero input-dependency requests and -zero computed-getter calls. Opening an actual source record may hydrate it. - -## Step 2: reverse query dependencies - -Persist a watch for each opted-in owner/field during indexing. A watch contains -the resolved Boxel query, source scope/type, owner identity and definition -revision. Register empty results too. There is no new author-facing predicate -language and no Elasticsearch dependency. - -For a document `d`, reverse matching is the set of saved queries whose filters -match `d`. For a source change, consider queries matching either the old or new -indexed document. A matching record's content change can affect a computation -without changing result membership. - -1. Extract conservative routing terms into an indexed lookup table. Initially - support mandatory equality or `in` terms, with broad fallback buckets when - safe extraction is impossible. Candidate selection may over-select but must - never miss a matching watch. -2. Verify candidates with the existing Boxel matching semantics. Reuse/factor the - query compiler as appropriate; test nulls, plural paths, type inheritance, - and the different reference-normalization rules of `eq` and `in`. -3. Capture old/new effective indexed rows for inserts, updates, deletions, - dependency-driven changes, and transitions into or out of indexing errors. -4. Retain concrete dependencies on query results and transitive inputs actually - consumed. Reverse watches supplement these edges by discovering new matches. -5. Queue affected owners after source data is visible. Coalesce bursts and - persist pending work; notifications are wakeups, not the durable state. -6. Publish watch registration, membership and computed values consistently. - Prevent registration/write races and an older worker clearing newer dirty - work. Cover restart recovery, definition changes and deleted owners. -7. Propagate changed feeder outputs to their consumers, with cycle controls and - full-rebuild convergence. Treat sorted/paginated queries conservatively: - an unseen matching row can enter the returned page. - -Use the standard PostgreSQL migration workflow and regenerate the SQLite schema. -Do not install a database extension or replace the existing indexing queue. - -## Realm adaptation - -Create a persisted classroom/day summary in the staging fork, with scoped query -fields and ordinary computed contained rows/statistics. Reuse the business rules -currently in `schema/classroom-day.gts`, including slot composition, coverage, -report versions and pipeline state. - -Keep selected child/date navigation, drafts, unsaved records, generation progress, -and clock-based presentation local. Store stable deadlines and shared facts. -Load full source cards when an interaction needs them. Do not mistake current -student location for dated historical attendance. - -Avoid introducing a projected-query API unless measurements show that the -remaining server indexing cost requires it. Compact display rows can use existing -contained fields and `computeVia`. - -## Validation and stopping condition - -- Record the fork's source fidelity and existing indexing/lint baseline before - editing its definitions. -- Exercise empty-to-first-match, deletion, classroom/date moves, content-only - updates, pagination changes, transitive inputs and query-parameter changes. -- Verify unrelated classroom writes do not refresh the summary and repeated - relevant writes coalesce. Test feeder chains and concurrent registration/write. -- Measure cold dashboard load, network requests/bytes, client computation, - source-write-to-summary freshness, and indexing work with multiple readers. -- Run focused host/realm-server tests, Glint where appropriate, and package lint. - Capture full host test output. Do not run the entire host suite locally. -- Keep the PR draft and titled DO NOT MERGE, including after successful tests. - Report results and limitations; production promotion is a separate decision. - -Working estimate, subject to revalidation against this base: 1,600–2,700 monorepo -implementation lines, 1,400–2,300 focused test lines, and about 600–1,000 changed -realm GTS lines, including relocation of existing logic. diff --git a/docs/tessar-query-materialization-plan.md b/docs/tessar-query-materialization-plan.md new file mode 100644 index 00000000000..153a9aad404 --- /dev/null +++ b/docs/tessar-query-materialization-plan.md @@ -0,0 +1,378 @@ +# Tessar query materialization: DO NOT MERGE POC plan + +Status: planning and environment preparation only. Implementation, synthetic-data +generation and benchmarks remain paused until the user asks to resume. + +This is a draft experiment. Do not merge it, enable auto-merge or deploy its +runtime changes to production. Use **Tessar** as the codename in every new +monorepo-facing document, PR description, fixture family and benchmark artifact. +Keep real deployment names, account identifiers and realm URLs outside the +monorepo. Production records must never become fixtures or benchmark inputs. + +## Goal + +Use Boxel indexing to maintain shared dashboard lists and statistics. Readers +consume the materialized results instead of reconstructing the business-data +graph and repeating the computation. Source changes identify which saved views +need to be recomputed, including queries that previously returned no results. + +Reuse Boxel's query language, `computeVia`, indexed JSON, dependency tracking and +indexing workers. The initial implementation maintains results through +incremental invalidation followed by recomputation. It does not implement +arithmetic delta maintenance of arbitrary JavaScript aggregates. + +Success has two independent parts: + +1. A clean indexed view can display supplied computed results without calling + their getters or fetching their input graph again. +2. A relevant source change reliably refreshes that view, while unrelated writes + and additional readers do not repeatedly execute its query and computation. + +## Current checkpoint + +- Main was updated to `e9a4b0a54a` on September 10, 2026. +- Work uses the isolated branch `codex/do-not-merge-query-materialization-poc`. +- The draft POC is [PR #6085](https://github.com/cardstack/boxel/pull/6085). +- A private staging fork contains 154 copied definition/support files, verified + against their prepared hashes. All 29 card types used by the source dataset + passed schema generation there. +- No production records were uploaded. The temporary source-data archive, + prepared JSON copies, import inventory and temporary source authentication + cache were removed. Only aggregate counts were retained for sizing. +- No runtime implementation, database migration, synthetic generator or measured + performance result is included in this checkpoint. + +The staging fork runs the staging deployment's runtime. It does **not** run this +monorepo branch merely because its GTS files are copied there. Develop and measure +the runtime changes on an isolated local stack from the worktree, including its +own database and indexing workers. A branch-specific hosted environment is a +later option; changing a shared staging backend is outside this plan. + +Keep source-deployment writes out of the experiment. Before a synthetic UI trial, +replace any copied definition's deployment-specific defaults or external data +references with synthetic equivalents in the development environment. + +## Scope and constraints + +The first slice supports saved materialized summary cards, same-realm queries, +stable classroom/day parameters, and the typed equality, `in`, range and boolean +predicates needed by Tessar. Computations must derive from indexed inputs and +stable parameters. Keep draft state, current identity, date navigation, generation +progress and clock-based presentation local to each user. Store stable deadlines +as data rather than reindexing every minute. + +Use a small acyclic feeder graph. Keep concrete dependencies on consumed input +cards and their transitive inputs. Query watches supplement those dependencies. +Do not globally enable query-only dependencies for all existing cards. + +Defer cross-realm watches, full-text/ranking dependencies, arbitrary cyclic views, +field-level JavaScript read-set inference, a new aggregation engine, and a new +projected-query authoring API. Unsupported query shapes must be rejected for +materialization with a clear explanation or use a proven conservative fallback; +they must not silently lose invalidations. + +## Revalidate current main before implementing + +The earlier investigation used `730081f8b4`. The updated base has client behavior +that the implementation must preserve: + +| Area | Confirmed behavior and consequence | +| --- | --- | +| `packages/base/field-support.ts` | Computed getters still invoke `computeVia` even when deserialization supplied a value. A pass-scoped compute memo is not a persisted snapshot. | +| `packages/host/app/routes/render/meta.ts` | Index serialization still uses `omitQueryFields: true` and excludes query-only runtime dependencies. | +| `packages/base/query-field-support.ts` | Eager query resolution, `eager: false`, newer-document seed handover and generation ordering already exist. Integrate with these paths. | +| `packages/host/app/resources/search.ts` | Seed ordering uses realm-specific generation floors. Ordinary live resources still refresh on realm events. Existing floors are not an exact materialization revision contract. | +| `packages/host/app/services/store.ts` | Selective search-entry inflation and scoped card searches exist. `addResourceFromSearchData` still adds a single-resource document; inspect compound-resource reuse before adding a cache. | +| `packages/runtime-common/realm-index-query-engine.ts` | GET/search assembly can populate query fields and expand links. Client snapshot support alone does not remove this server work. | +| `packages/runtime-common/index-writer.ts` | Working-index buffers, batch promotion and generation guards already exist. Integrate watch publication and invalidation with these mechanisms. | + +Record the exact base commit, existing behavior and relevant feature flags in +each benchmark run. Recheck these paths if main is updated again. + +## Synthetic data and baseline design + +Generate new records from a fixed seed. Use fabricated names, narratives, IDs, +scores, statuses and dates. Do not anonymize production records or use them as +templates. Keep credentials and runtime deployment addresses out of output. + +| Preset | Instance count, excluding realm/index configuration | Purpose | +| --- | ---: | --- | +| Smoke | Small explicit fixture | Human-auditable counts, membership and graph behavior | +| 1x | 1,255 | Approximate the current production-sized workload | +| 10x | 12,550 | Expose scaling costs | +| 100x | 125,500 | Stress the design on an isolated runtime | + +Retain a representative mix of reference, schedule, roster, observation, report +and summary records. Match the intended topology and approximate payload sizes, +not merely the total record count. The exact production-shaped distribution can +remain local; committed manifests describe only synthetic data. + +Vary these axes separately: + +- **Realm size:** add unrelated classrooms/dates while holding one watched view's + matching inputs and displayed rows constant. +- **Query cardinality:** increase matching inputs for one classroom/day while + holding the displayed summary size constant where possible. +- **Graph cost:** vary link depth, fan-out, shared targets and record body size. +- **Readers and writes:** use 1, 10 and 50 readers, then relevant and unrelated + writes, bursts, moves, deletes and changes to transitive dependencies. +- **Watch selectivity:** compare tightly scoped queries with broad/fallback + predicates. Broad dependencies are expected to cause broader invalidation. + +Implement a deterministic generator and run manifests, not a checked-in tree of +125,500 JSON files. Materialize large datasets on demand, with bounded generator +memory and resumable seeding. Keep generated data and raw benchmark output out of +Git. Confirm referential integrity and expected summary values before measuring. + +Proposed monorepo locations: `scripts/tessar/` for generation/benchmark tooling, +small `tessar-*` GTS fixtures in the existing test fixture directories, and +`docs/tessar-*` for the plan and sanitized results. Use only locations needed by +the first experiment; avoid introducing a new package. + +Record the current implementation against the same synthetic fixture and the +same runtime/database resources used for the candidate. Separate cold indexing, +cold client load, warm reads, and post-write refresh. A staging baseline on a +different runtime or machine is contextual evidence, not a controlled speedup. + +## Contract to establish first + +Use one explicit opt-in on a materialized summary definition. Keep query authors +on existing Boxel query definitions and ordinary `computeVia`. Source card types +should not need new annotations simply because a summary consumes them. Settle +the smallest declaration syntax in the contract spike before changing APIs. + +An indexed response must identify its materialized fields and carry enough +provenance to establish: + +- Which computed values and query memberships are complete and successful. +- The definition/query revision and source realm that produced them. +- The input/index revision observed by the computation and the published owner + revision. Do not equate today's generation floor with an exact result revision. +- Whether a value is missing, unresolved, partial, errored or successfully empty. + +Use existing JSON attributes and relationship IDs for the values themselves. +Add the smallest versioned metadata necessary; do not invent a second card wire +format. Successful `0`, `false`, `null` and empty arrays must survive round trips. + +The mode applies to clean indexed views. Missing metadata follows existing +behavior. Unsaved cards, indexing and explicit live editing compute from their +inputs. Editing a snapshot-backed owner must explicitly leave snapshot mode or +use a local overlay; stale snapshot values must not masquerade as edited results. + +## Step 1: read the indexed results without rebuilding their inputs + +1. Capture supplied computed values in a snapshot structure separate from the + authored/default data bucket. Mark only values with valid server provenance. +2. In indexed-view mode, have the computed getter read that snapshot. Propagate + the mode through contained fields. Preserve ordinary computation elsewhere. +3. Keep complete query membership separately from hydrated member instances. + Bypass eager query resolution and realm-wide live searches for an authoritative + materialized field. Reuse existing query signatures and seed ordering. +4. Preserve already-returned raw resources in a bounded, revision-aware map where + current main loses them. Hydrate on demand through the existing identity map. + Passing the entire compound document into today's eager deserializer is not + sufficient: it can still instantiate the graph. +5. Do not bulk-fetch seed URLs simply to recover already-known membership. The + first UI uses serialized statistics and compact contained rows. Zero-hydration + arbitrary iteration of a `linksToMany` array is not a prerequisite. +6. Refresh snapshot values when the owner is invalidated, with generation and + definition checks. A local source write must refresh dependent summaries in + that same client; self-originated write suppression protects the actual edit. +7. Add the corresponding server read path: return stored membership and computed + values together, without rerunning query membership or recursively assembling + its input graph. Preserve ordinary explicit source-card reads. + +Initial tests may use supplied serialized fixtures to isolate client behavior. +This step is not ready for continuously refreshed views until step 2 works. + +Primary files: `packages/base/card-api.gts`, `packages/base/field-support.ts`, +`packages/base/query-field-support.ts`, `packages/host/app/services/store.ts`, +`packages/host/app/resources/search.ts`, +`packages/runtime-common/realm-index-query-engine.ts`, and the relevant handlers +in `packages/runtime-common/realm.ts`. + +Gate: rendering the supplied output executes zero materialized computed getters, +makes zero requests for its input dependencies, and does not run a query-field +search. Nested computeds, empty results, owner refresh, module replacement, +editing and legacy fallbacks work. Server GET does not repeat membership work. +Opening a source record is allowed to hydrate that record. + +## Step 2: durable reverse-query invalidation + +### Predicate semantics and candidate lookup + +Keep the resolved Boxel query AST as the authority. Resolve `$this` parameters +during indexing and include the source scope/type plus query/definition revision. +For a predicate `Pq` and changed indexed document `d`: + +```text +matches(d) = { q in saved queries | Pq(d) } +affected(old,new) = matches(old) union matches(new) +matches(d) is a subset of candidates(d) +``` + +An absent old/new version matches nothing. A content edit that matches both +versions still invalidates the summary: membership can stay the same while a +score, status or nested dependency changes a computed result. + +Build a registry keyed by owner and field, with source scope, resolved query and +revision. Derive a companion routing-term table; use ordinary indexed columns +such as realm, source type, field path, typed value and watch ID. The first +extractor can select one mandatory equality anchor, expand `in` anchors, and +fall back to a broader bucket when no safe anchor exists. Include type ancestry. + +Every boolean branch must be covered. Do not naively intersect term lists for +queries with different constrained fields, `OR`, or `NOT`. Candidate selection +may return extras but must never miss a matching query. Verify candidates against +the old/new documents using Boxel's existing semantics. Reuse or factor the +existing compiler rather than introduce an independently interpreted filter DSL. +Test nulls, plural paths, type inheritance and reference normalization: `eq` and +`in` currently have different rules. + +For sorted/paginated queries, watch the unpaginated membership predicate. A row +outside the current page can enter it after an insert or sort-key edit. Initially +invalidate conservatively; result fingerprints can be a later optimization. + +The algorithmic reference is [Elasticsearch Percolator](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-percolate-query). +Its implementation can inform extraction/fallback review, but no Elasticsearch +dependency is required. PostgreSQL indexes route candidates; they do not +automatically reverse-evaluate a JSON query AST. + +### Registration, dependencies and publication + +1. Register watches during owner indexing, independent of open clients. Capture + successful empty queries and replace obsolete watches when parameters or + definitions change. Delete watches when their owner is deleted. +2. Retain concrete dependencies on consumed result cards and their transitive + inputs for opted-in computations. Keep ordinary cards' dependency behavior. +3. Capture old/new effective indexed rows before replacement or tombstoning. + Include dependency-driven reindexes and indexing-error transitions, not just + raw file writes. Avoid invalidating solely because unrelated HTML changed. +4. Select and verify candidate watches, then mark owners dirty durably. Deduplicate + owners across a batch and coalesce bursts through the existing indexing queue. + Process owners after the changed source rows are visible to their queries. +5. Publish computed values, membership and dependency/watch registrations as one + consistent owner revision. Ordinary GETs must not mix that snapshot with a + newly executed membership query. +6. Establish an input-watermark/retry protocol compatible with current working + index buffers and batch promotion. A source change racing registration cannot + disappear, and a worker finishing older work cannot clear newer dirty work. + Do not hold a database transaction open across long asynchronous card renders. +7. Propagate changed feeder outputs to their parents, with cycle detection and + an actionable failure mode. Full rebuilds need source-first scheduling or a + convergence pass. Cover retries, restarts and definition/module replacement. +8. Keep pending work in durable storage. Notifications only wake workers. Maintain + explicit error/freshness state when recomputation fails; do not publish partial + results as a successful complete snapshot. + +Primary files: `packages/runtime-common/index-writer.ts`, +`packages/runtime-common/index-query-engine.ts`, +`packages/runtime-common/index-runner.ts`, +`packages/runtime-common/index-runner/card-indexer.ts`, +`packages/runtime-common/index-runner/relationship-dependency-extractor.ts`, +`packages/runtime-common/dependency-tracker.ts`, +`packages/host/app/routes/render/meta.ts`, `packages/base/searchable.ts`, and the +query normalization/signature helpers. Add one focused runtime module for watch +registration/matching if that keeps responsibilities clear. Reuse existing queue +integration before adding a separate worker or scheduler. + +Create migrations with `pnpm create ` in `packages/postgres`, +then regenerate the SQLite schema with `pnpm make-schema`. Cover both adapters. + +Gate: insert into an empty result, deletion, moves between groups/dates, +content-only changes, transitive changes, pagination, watch replacement and +definition changes refresh the correct owner. Unrelated scoped writes do not. +Race/restart tests show no lost invalidation. Feeder chains converge. One client's +write refreshes its own displayed dependent summary as well as other clients'. + +## Tessar realm adaptation + +Create a persisted day summary and only the feeder views that have independent +consumers. Query existing source card types and compute compact contained rows +and statistics. Reuse the existing schedule composition, coverage, report-version +and pipeline rules; move their shared deterministic parts out of component getters. + +Display rows contain the labels, IDs, statuses, ordering values and counts the UI +actually needs. Preserve source IDs for drill-down/editing without embedding every +source card object. Keep user selection, unsaved overlays, timers and AI-generation +state local. Do not infer historical attendance from a current location field. + +Provision summary instances through synthetic seeding/day setup so indexing +can precompute them before a user visits. Decide explicitly how an uncached day +is provisioned; do not write shared navigation state onto a singleton dashboard. + +Retain truly ad hoc and AI-input queries where needed. Remove the broad read-time +loads serving the shared displayed lists/statistics. Verify no hidden component +getter or default eager query path reconstructs the same input graph. + +Do this work in the isolated development realm. Commit only synthetic Tessar +fixtures and generic runtime changes to the monorepo. + +## Milestones and reviewable increments + +| Milestone | Deliverable | Exit evidence | +| --- | --- | --- | +| M0: baseline and contract | Revalidate main; small deterministic fixture/generator; define opt-in, metadata, modes and revisions | Baseline request/compute trace and known expected results | +| M1: snapshot consumption | Base/host/server read path from step 1 | Zero recomputation/input requests for supplied outputs, with legacy/editing regression checks | +| M2: reverse matcher | Registry schema and conservative routing/verification primitives | Predicate parity and no-missed-candidate cases, including empty, boolean and pagination cases | +| M3: index lifecycle | Register watches, retain dependencies, coalesce dirty owners and publish consistently | End-to-end change propagation plus race, restart and feeder convergence tests | +| M4: Tessar display views | Synthetic realm summaries, compact rows and dashboard consumption | Matching UI output; successful source edits/drill-down and local overlays | +| M5: scaling evidence | Controlled base/candidate runs across sizes and concurrency | Sanitized results with measured limits and remaining bottlenecks | + +Keep these as distinct commits or small reviewable groups on the draft POC. M1 +alone does not establish freshness. M2 alone does not establish durable view +maintenance. Do not present a primitive-only proof as the completed use case. + +## Validation and measurements + +Extend the nearest existing tests, with small synthetic GTS fixtures and explicit +expected results rather than tests that repeat the implementation: + +- Host query-field acceptance, seed/refresh, membership-status and barrier tests. +- Host index-query-engine, index-writer and runtime-dependency-tracker tests. +- Realm-server skip-query-backed-expansion, live-search-cache, indexing and queue + tests where the corresponding integration behavior is changed. +- Focused generator integrity checks: repeatable output, valid links and known + summary values. Never commit a large generated dataset as test fixtures. + +Use the pinned mise/Node/pnpm toolchain. Use Glint, never direct `tsc` or +`glint --declaration`. Run package lint before commits that change package code. +Run focused host/realm-server tests and capture complete host output to files. +Do not run the entire host suite locally. Run realm lint and render validation +for changed GTS, recording any pre-existing baseline separately. + +For each benchmark, record commit, runtime/database configuration, seed, dataset +manifest, query shape/cardinality, graph shape, reader count and cache state. +Collect: + +- Cold/warm dashboard latency and response bytes. +- Client requests, instantiated cards and computed-getter invocations. +- Server query counts, assembly work and relevant database query plans/timing. +- Initial indexing time, worker CPU/memory and queue backlog. +- Reverse candidates examined, watches matched and owners actually reindexed. +- Source-write-to-summary freshness and work performed during write bursts. + +Use enough repeated observations to report distributions meaningfully; do not +label a single run p95. Hold displayed output size fixed in the unrelated-growth +case. Measure membership-ID serialization separately from source-graph expansion; +do not claim constant payload size when the returned list itself grows. + +Hard correctness/performance-shape criteria are zero redundant computation and +input fetching on a snapshot read, no lost invalidations, and no reindex of an +unrelated scoped summary. Numerical latency, memory and throughput targets will +be set from M0 on the chosen runtime rather than invented now. Large matching +sets and broad watches can still require substantial indexing work; quantify it. + +## Scope estimate and completion + +Working estimate, subject to the M0 recheck: 1,600–2,700 monorepo implementation +lines plus 1,400–2,300 focused test lines. Realm adaptation is about 600–1,000 +changed GTS lines, including moving existing logic. Synthetic generation and +benchmark tooling are a separate, not-yet-sized work item. These are estimates, +not a promised diff size or elapsed-time commitment. + +Finish with a sanitized Tessar benchmark report, the tested contract, and a list +of remaining limitations. The PR remains draft and DO NOT MERGE even after its +tests pass. Production promotion and any shared-runtime deployment require a +separate decision. Until the user resumes implementation, this plan is the +stopping point. From 01fa8bd820955d20738673740fd1de571455635b Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:10:31 -0400 Subject: [PATCH 04/12] Require Tessar performance report and correctness freshness gates --- docs/tessar-query-materialization-plan.md | 91 ++++++++++++++++++++--- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/docs/tessar-query-materialization-plan.md b/docs/tessar-query-materialization-plan.md index 153a9aad404..482784402f6 100644 --- a/docs/tessar-query-materialization-plan.md +++ b/docs/tessar-query-materialization-plan.md @@ -28,6 +28,12 @@ Success has two independent parts: 2. A relevant source change reliably refreshes that view, while unrelated writes and additional readers do not repeatedly execute its query and computation. +Correctness and freshness are non-negotiable validity gates. Performance is the +question the POC must measure and report once those gates pass. Lower latency, +less memory or higher throughput cannot compensate for wrong, incomplete or +stale results. A negative performance result is still useful evidence; a +correctness or freshness violation is a failed case, not an accepted tradeoff. + ## Current checkpoint - Main was updated to `e9a4b0a54a` on September 10, 2026. @@ -312,12 +318,12 @@ fixtures and generic runtime changes to the monorepo. | Milestone | Deliverable | Exit evidence | | --- | --- | --- | -| M0: baseline and contract | Revalidate main; small deterministic fixture/generator; define opt-in, metadata, modes and revisions | Baseline request/compute trace and known expected results | +| M0: baseline and contract | Revalidate main; small deterministic fixture/generator; define opt-in, metadata, modes, revisions and mandatory freshness guarantees | Baseline request/compute trace, independent expected results and fixed validity gates | | M1: snapshot consumption | Base/host/server read path from step 1 | Zero recomputation/input requests for supplied outputs, with legacy/editing regression checks | | M2: reverse matcher | Registry schema and conservative routing/verification primitives | Predicate parity and no-missed-candidate cases, including empty, boolean and pagination cases | | M3: index lifecycle | Register watches, retain dependencies, coalesce dirty owners and publish consistently | End-to-end change propagation plus race, restart and feeder convergence tests | | M4: Tessar display views | Synthetic realm summaries, compact rows and dashboard consumption | Matching UI output; successful source edits/drill-down and local overlays | -| M5: scaling evidence | Controlled base/candidate runs across sizes and concurrency | Sanitized results with measured limits and remaining bottlenecks | +| M5: performance report | Controlled base/candidate runs across sizes and concurrency, enforcing correctness/freshness gates throughout | Reproducible Tessar performance report with measured gains, regressions, resource costs and limits | Keep these as distinct commits or small reviewable groups on the draft POC. M1 alone does not establish freshness. M2 alone does not establish durable view @@ -325,6 +331,35 @@ maintenance. Do not present a primitive-only proof as the completed use case. ## Validation and measurements +### Non-negotiable validity gates + +Establish and freeze the freshness contract in M0 before comparing performance. +Specify when a write is acknowledged, which source revision a view incorporates, +when that revision must be visible to existing and newly connected clients, and +the maximum permitted propagation delay. These requirements must not be relaxed +to make larger datasets or higher concurrency appear to pass. + +- Compare membership, order, displayed rows and statistics against independently + derived expected results from the synthetic inputs, including after changes. +- Check correctness and freshness during concurrent reads/writes and write bursts, + not only after the queue is empty. Include the writer's own client, another + client, newly connected clients, and a reconnect after missed notifications. +- Track every acknowledged relevant write through its materialized owner revision + to client visibility. Require no lost changes, backward revisions or mismatched + membership/statistics. Verify convergence after retries and worker restarts. +- If refresh is asynchronous, represent pending/failed freshness honestly; a + previous successful value cannot be labeled current while required writes are + missing. A source-write acknowledgement and a completed materialization are + distinct events unless the implemented contract explicitly makes them one. +- Count every wrong result, stale response, deadline breach, dropped operation + and error. A violated gate disqualifies the case from successful throughput or + speedup claims; retain the failed case in the report. + +Apply equivalent result semantics and freshness requirements to the baseline and +candidate. If an existing baseline truncates results or is stale, mark that case +invalid and use a correct reference path for the comparison. Never obtain an +apparent speedup by returning fewer results or by excluding failed requests. + Extend the nearest existing tests, with small synthetic GTS fixtures and explicit expected results rather than tests that repeat the implementation: @@ -357,11 +392,47 @@ label a single run p95. Hold displayed output size fixed in the unrelated-growth case. Measure membership-ID serialization separately from source-graph expansion; do not claim constant payload size when the returned list itself grows. -Hard correctness/performance-shape criteria are zero redundant computation and -input fetching on a snapshot read, no lost invalidations, and no reindex of an -unrelated scoped summary. Numerical latency, memory and throughput targets will -be set from M0 on the chosen runtime rather than invented now. Large matching -sets and broad watches can still require substantial indexing work; quantify it. +Structural performance criteria are zero redundant computation and input fetching +on a snapshot read, and no reindex of an unrelated scoped summary. Numerical read +latency, memory and throughput targets will be set from M0 on the chosen runtime +rather than invented now. The correctness/freshness gates above apply regardless +of those performance targets. Large matching sets and broad watches can still +require substantial indexing work; quantify it. + +### Required performance benchmark report + +Deliver `docs/tessar-performance-report.md` and a compact, sanitized results file +such as `docs/tessar-benchmark-results.json`. Keep large raw traces and generated +datasets outside Git, with reproducible commands, seeds and manifest hashes. +The report is required to complete the POC; passing correctness tests alone does +not complete the work. + +Include: + +1. **Validity:** the fixed correctness/freshness contract, evidence that each + qualifying case passed, and all excluded/failed cases with their reasons. +2. **Controlled comparison:** base and candidate commits, runtime/database + configuration, warmup, repetitions, dataset/query shapes, cache conditions, + reader counts and write rates. Report latency distributions and error rates. +3. **Scaling:** production-sized, 10x and 100x synthetic results, separating + unrelated realm growth from query cardinality and graph cost. Plot read + latency, resource use and refresh delay against these axes and concurrency. +4. **Cost placement:** initial materialization cost, incremental indexing and + reverse-matching work, write amplification, queue depth, client work, server + CPU/memory, database time and network bytes. Quantify the additional work on + writes alongside the work removed from reads. +5. **Capacity:** sustained readers and writes supported while the validity gates + hold. Report source-record capture latency and backlog under load, rather than + measuring fast reads in isolation from the ingestion/indexing workload. +6. **Conclusion:** measured improvements or regressions, the workload where the + approach pays off, limiting resources, and the next smallest justified change. + Do not promise that performance will improve before the measurements exist. + +Present base/candidate tables and standalone exportable charts. Compute totals +over declared read/write workloads so shifting work into indexing is visible in +the report. Do not infer spare AI-generation capacity from read latency alone; +support any such claim with measured resource headroom or a separate mixed-load +experiment. ## Scope estimate and completion @@ -371,8 +442,10 @@ changed GTS lines, including moving existing logic. Synthetic generation and benchmark tooling are a separate, not-yet-sized work item. These are estimates, not a promised diff size or elapsed-time commitment. -Finish with a sanitized Tessar benchmark report, the tested contract, and a list -of remaining limitations. The PR remains draft and DO NOT MERGE even after its +Finish with the required Tessar performance report, the tested contract, and a +list of remaining limitations. Only cases satisfying the non-negotiable +correctness/freshness gates qualify as successful benchmark results. The PR +remains draft and DO NOT MERGE even after its tests pass. Production promotion and any shared-runtime deployment require a separate decision. Until the user resumes implementation, this plan is the stopping point. From c2f0655e1ab91eb6f43564ffc35307bf32206e8b Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:16:46 -0400 Subject: [PATCH 05/12] Set Tessar benchmark target to 10x current size --- docs/tessar-query-materialization-plan.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/tessar-query-materialization-plan.md b/docs/tessar-query-materialization-plan.md index 482784402f6..795690423a2 100644 --- a/docs/tessar-query-materialization-plan.md +++ b/docs/tessar-query-materialization-plan.md @@ -104,9 +104,12 @@ templates. Keep credentials and runtime deployment addresses out of output. | Preset | Instance count, excluding realm/index configuration | Purpose | | --- | ---: | --- | | Smoke | Small explicit fixture | Human-auditable counts, membership and graph behavior | -| 1x | 1,255 | Approximate the current production-sized workload | -| 10x | 12,550 | Expose scaling costs | -| 100x | 125,500 | Stress the design on an isolated runtime | +| 1x | 1,255 | Reference workload approximating the current production size | +| 10x | 12,550 | Primary performance benchmark target | + +The benchmark target is 10x the recorded current instance count. Retain the 1x +reference and smoke fixtures for comparison and validation; do not generate or +run a 100x dataset. Retain a representative mix of reference, schedule, roster, observation, report and summary records. Match the intended topology and approximate payload sizes, @@ -126,7 +129,7 @@ Vary these axes separately: predicates. Broad dependencies are expected to cause broader invalidation. Implement a deterministic generator and run manifests, not a checked-in tree of -125,500 JSON files. Materialize large datasets on demand, with bounded generator +12,550 JSON files. Materialize datasets on demand, with bounded generator memory and resumable seeding. Keep generated data and raw benchmark output out of Git. Confirm referential integrity and expected summary values before measuring. @@ -414,8 +417,8 @@ Include: 2. **Controlled comparison:** base and candidate commits, runtime/database configuration, warmup, repetitions, dataset/query shapes, cache conditions, reader counts and write rates. Report latency distributions and error rates. -3. **Scaling:** production-sized, 10x and 100x synthetic results, separating - unrelated realm growth from query cardinality and graph cost. Plot read +3. **Scaling:** the 10x synthetic benchmark with a production-sized reference, + separating unrelated realm growth from query cardinality and graph cost. Plot read latency, resource use and refresh delay against these axes and concurrency. 4. **Cost placement:** initial materialization cost, incremental indexing and reverse-matching work, write amplification, queue depth, client work, server From f8daaf7734ae5c75de07a835ab6beaf474d87c8e Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:18:05 -0400 Subject: [PATCH 06/12] Add Tessar synthetic benchmark scaffold and validity gates --- docs/tessar-query-materialization-plan.md | 75 +++++--- scripts/tessar/README.md | 40 +++++ scripts/tessar/bench.mjs | 177 +++++++++++++++++++ scripts/tessar/generate.mjs | 206 ++++++++++++++++++++++ scripts/tessar/generate.test.mjs | 53 ++++++ scripts/tessar/realm/tessar.gts | 182 +++++++++++++++++++ 6 files changed, 706 insertions(+), 27 deletions(-) create mode 100644 scripts/tessar/README.md create mode 100644 scripts/tessar/bench.mjs create mode 100644 scripts/tessar/generate.mjs create mode 100644 scripts/tessar/generate.test.mjs create mode 100644 scripts/tessar/realm/tessar.gts diff --git a/docs/tessar-query-materialization-plan.md b/docs/tessar-query-materialization-plan.md index 795690423a2..87e2bc07c91 100644 --- a/docs/tessar-query-materialization-plan.md +++ b/docs/tessar-query-materialization-plan.md @@ -1,7 +1,7 @@ # Tessar query materialization: DO NOT MERGE POC plan -Status: planning and environment preparation only. Implementation, synthetic-data -generation and benchmarks remain paused until the user asks to resume. +Status: implementation resumed September 10, 2026. Run focused correctness and +freshness tests and browser/network performance checks at each working increment. This is a draft experiment. Do not merge it, enable auto-merge or deploy its runtime changes to production. Use **Tessar** as the codename in every new @@ -36,6 +36,27 @@ correctness or freshness violation is a failed case, not an accepted tradeoff. ## Current checkpoint +Implementation has started. The deterministic generator produces 1,255 / 12,550 +synthetic instances and a raw-document oracle. Its first two tests pass. The +initial 34-instance real-stack baseline returned correct live query membership +but stale zero-valued indexed statistics on all five GETs. Those reads fail the +correctness gate and are not valid speedup baselines. The internal snapshot +consumer and reverse-query registry have focused browser and Postgres tests. +Four browser tests pass 49 assertions; two Postgres tests cover SQL predicates, +transaction rollback, persistent dirty state and rejection of stale publication. +The existing computed-field regression suite passes 15 tests / 41 assertions. +The registry is not yet connected to index publication or worker scheduling, and +normal server/client reads do not yet opt into Tessar. These primitive tests do +not establish end-to-end freshness or a performance improvement. + +The initial freshness contract is a maximum 10,000 ms from an acknowledged +relevant write to its complete owner revision being visible in existing and new +clients, at every dataset size and concurrency. Reads during that interval must +explicitly identify pending work; they must never present older results as +current. Source acknowledgement, source index revision, owner publication and +client observation are separate measured events. Missing, mixed or regressing +revisions fail the gate regardless of elapsed time. + - Main was updated to `e9a4b0a54a` on September 10, 2026. - Work uses the isolated branch `codex/do-not-merge-query-materialization-poc`. - The draft POC is [PR #6085](https://github.com/cardstack/boxel/pull/6085). @@ -45,8 +66,8 @@ correctness or freshness violation is a failed case, not an accepted tradeoff. - No production records were uploaded. The temporary source-data archive, prepared JSON copies, import inventory and temporary source authentication cache were removed. Only aggregate counts were retained for sizing. -- No runtime implementation, database migration, synthetic generator or measured - performance result is included in this checkpoint. +- Runtime primitives, an additive registry migration and a synthetic generator + are in progress. No production or shared staging backend is changed. The staging fork runs the staging deployment's runtime. It does **not** run this monorepo branch merely because its GTS files are copied there. Develop and measure @@ -82,15 +103,15 @@ they must not silently lose invalidations. The earlier investigation used `730081f8b4`. The updated base has client behavior that the implementation must preserve: -| Area | Confirmed behavior and consequence | -| --- | --- | -| `packages/base/field-support.ts` | Computed getters still invoke `computeVia` even when deserialization supplied a value. A pass-scoped compute memo is not a persisted snapshot. | -| `packages/host/app/routes/render/meta.ts` | Index serialization still uses `omitQueryFields: true` and excludes query-only runtime dependencies. | -| `packages/base/query-field-support.ts` | Eager query resolution, `eager: false`, newer-document seed handover and generation ordering already exist. Integrate with these paths. | -| `packages/host/app/resources/search.ts` | Seed ordering uses realm-specific generation floors. Ordinary live resources still refresh on realm events. Existing floors are not an exact materialization revision contract. | -| `packages/host/app/services/store.ts` | Selective search-entry inflation and scoped card searches exist. `addResourceFromSearchData` still adds a single-resource document; inspect compound-resource reuse before adding a cache. | -| `packages/runtime-common/realm-index-query-engine.ts` | GET/search assembly can populate query fields and expand links. Client snapshot support alone does not remove this server work. | -| `packages/runtime-common/index-writer.ts` | Working-index buffers, batch promotion and generation guards already exist. Integrate watch publication and invalidation with these mechanisms. | +| Area | Confirmed behavior and consequence | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `packages/base/field-support.ts` | Computed getters still invoke `computeVia` even when deserialization supplied a value. A pass-scoped compute memo is not a persisted snapshot. | +| `packages/host/app/routes/render/meta.ts` | Index serialization still uses `omitQueryFields: true` and excludes query-only runtime dependencies. | +| `packages/base/query-field-support.ts` | Eager query resolution, `eager: false`, newer-document seed handover and generation ordering already exist. Integrate with these paths. | +| `packages/host/app/resources/search.ts` | Seed ordering uses realm-specific generation floors. Ordinary live resources still refresh on realm events. Existing floors are not an exact materialization revision contract. | +| `packages/host/app/services/store.ts` | Selective search-entry inflation and scoped card searches exist. `addResourceFromSearchData` still adds a single-resource document; inspect compound-resource reuse before adding a cache. | +| `packages/runtime-common/realm-index-query-engine.ts` | GET/search assembly can populate query fields and expand links. Client snapshot support alone does not remove this server work. | +| `packages/runtime-common/index-writer.ts` | Working-index buffers, batch promotion and generation guards already exist. Integrate watch publication and invalidation with these mechanisms. | Record the exact base commit, existing behavior and relevant feature flags in each benchmark run. Recheck these paths if main is updated again. @@ -101,11 +122,11 @@ Generate new records from a fixed seed. Use fabricated names, narratives, IDs, scores, statuses and dates. Do not anonymize production records or use them as templates. Keep credentials and runtime deployment addresses out of output. -| Preset | Instance count, excluding realm/index configuration | Purpose | -| --- | ---: | --- | -| Smoke | Small explicit fixture | Human-auditable counts, membership and graph behavior | -| 1x | 1,255 | Reference workload approximating the current production size | -| 10x | 12,550 | Primary performance benchmark target | +| Preset | Instance count, excluding realm/index configuration | Purpose | +| ------ | --------------------------------------------------: | ------------------------------------------------------------ | +| Smoke | Small explicit fixture | Human-auditable counts, membership and graph behavior | +| 1x | 1,255 | Reference workload approximating the current production size | +| 10x | 12,550 | Primary performance benchmark target | The benchmark target is 10x the recorded current instance count. Retain the 1x reference and smoke fixtures for comparison and validation; do not generate or @@ -319,14 +340,14 @@ fixtures and generic runtime changes to the monorepo. ## Milestones and reviewable increments -| Milestone | Deliverable | Exit evidence | -| --- | --- | --- | -| M0: baseline and contract | Revalidate main; small deterministic fixture/generator; define opt-in, metadata, modes, revisions and mandatory freshness guarantees | Baseline request/compute trace, independent expected results and fixed validity gates | -| M1: snapshot consumption | Base/host/server read path from step 1 | Zero recomputation/input requests for supplied outputs, with legacy/editing regression checks | -| M2: reverse matcher | Registry schema and conservative routing/verification primitives | Predicate parity and no-missed-candidate cases, including empty, boolean and pagination cases | -| M3: index lifecycle | Register watches, retain dependencies, coalesce dirty owners and publish consistently | End-to-end change propagation plus race, restart and feeder convergence tests | -| M4: Tessar display views | Synthetic realm summaries, compact rows and dashboard consumption | Matching UI output; successful source edits/drill-down and local overlays | -| M5: performance report | Controlled base/candidate runs across sizes and concurrency, enforcing correctness/freshness gates throughout | Reproducible Tessar performance report with measured gains, regressions, resource costs and limits | +| Milestone | Deliverable | Exit evidence | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- | +| M0: baseline and contract | Revalidate main; small deterministic fixture/generator; define opt-in, metadata, modes, revisions and mandatory freshness guarantees | Baseline request/compute trace, independent expected results and fixed validity gates | +| M1: snapshot consumption | Base/host/server read path from step 1 | Zero recomputation/input requests for supplied outputs, with legacy/editing regression checks | +| M2: reverse matcher | Registry schema and conservative routing/verification primitives | Predicate parity and no-missed-candidate cases, including empty, boolean and pagination cases | +| M3: index lifecycle | Register watches, retain dependencies, coalesce dirty owners and publish consistently | End-to-end change propagation plus race, restart and feeder convergence tests | +| M4: Tessar display views | Synthetic realm summaries, compact rows and dashboard consumption | Matching UI output; successful source edits/drill-down and local overlays | +| M5: performance report | Controlled base/candidate runs across sizes and concurrency, enforcing correctness/freshness gates throughout | Reproducible Tessar performance report with measured gains, regressions, resource costs and limits | Keep these as distinct commits or small reviewable groups on the draft POC. M1 alone does not establish freshness. M2 alone does not establish durable view @@ -450,5 +471,5 @@ list of remaining limitations. Only cases satisfying the non-negotiable correctness/freshness gates qualify as successful benchmark results. The PR remains draft and DO NOT MERGE even after its tests pass. Production promotion and any shared-runtime deployment require a -separate decision. Until the user resumes implementation, this plan is the +separate decision. This plan is the stopping point. diff --git a/scripts/tessar/README.md b/scripts/tessar/README.md new file mode 100644 index 00000000000..bf05df8c8e0 --- /dev/null +++ b/scripts/tessar/README.md @@ -0,0 +1,40 @@ +# Tessar experimental benchmark tools + +DO NOT MERGE. Use an isolated local runtime from this worktree. All generated +records are fabricated; the generator does not read deployment records. + +From the monorepo root, with the pinned mise toolchain installed: + +```sh +mise exec -- pnpm exec node --test scripts/tessar/generate.test.mjs +mise exec -- pnpm exec node scripts/tessar/generate.mjs --preset smoke --output /private/tmp/tessar-smoke +``` + +Presets are `smoke` (34 instances), `1x` (1,255) and `10x` (12,550). The output +directory must not exist. `manifest.json` records the seed, counts and data hash; +`expected.json` is the independent raw-document oracle. Production data is never +an input. A 100x preset is deliberately unsupported. + +Build the host and serve its icons using the package scripts. From +`packages/realm-server`: + +```sh +mise exec -- pnpm exec node ../../scripts/tessar/bench.mjs --dataset /private/tmp/tessar-smoke --output /private/tmp/tessar-smoke-result.json --serve +``` + +The harness starts disposable Postgres/Synapse/realm/host services and prints the +local realm URL. Runtime URLs and owned process IDs are saved beside the output +as `.runtime.json`. The source fingerprint includes dirty and untracked runtime +files so a cached fixture cannot silently stand in for the current implementation. +Stop the harness with SIGINT/SIGTERM when the browser trial finishes. + +For host QUnit tests, build with `RESOLVED_BASE_REALM_URL` pointing at the printed +local server's `/base/` and clear ambient `BOXEL_ENVIRONMENT`, `ENV_SLUG`, and +`ENV_MODE` settings. Filter on `Tessar`. Capture the complete console output. +After a base-module edit, invalidate that local realm's module cache or restart +the isolated stack before claiming that a browser test covered the edit. + +The initial ordinary-query GET case has failed correctness: live membership and +stored statistics disagree. The tool records those reads as invalid. It does +not treat their latency as a passing baseline. Full dashboard adaptation, +concurrency, write/freshness trials and the 10x performance report remain required. diff --git a/scripts/tessar/bench.mjs b/scripts/tessar/bench.mjs new file mode 100644 index 00000000000..cb689dd9843 --- /dev/null +++ b/scripts/tessar/bench.mjs @@ -0,0 +1,177 @@ +import { createHash } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { execFileSync } from 'node:child_process'; +import { validateSummary } from './generate.mjs'; + +// Import the harness only after removing ambient hosted-environment routing. +// Tessar always starts its own local servers and disposable database. +for (let name of [ + 'BOXEL_ENVIRONMENT', + 'ENV_SLUG', + 'ENV_MODE', + 'ICONS_URL', + 'REALM_BASE_URL', + 'REALM_TEST_URL', + 'HOST_URL', + 'MATRIX_URL_VAL', + 'PRERENDER_MGR_URL', + 'REALM_SERVER_TLS_CERT_FILE', + 'REALM_SERVER_TLS_KEY_FILE', +]) + delete process.env[name]; +const { startFactoryRealmServer } = + await import('../../packages/realm-test-harness/src/index.ts'); + +let { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + output: { type: 'string' }, + iterations: { type: 'string', default: '5' }, + serve: { type: 'boolean', default: false }, + }, +}); +if (!values.dataset || !values.output) + throw new Error('--dataset and --output are required'); +let dataset = resolve(values.dataset); +let output = resolve(values.output); +let iterations = Number(values.iterations); +if (!Number.isSafeInteger(iterations) || iterations < 1) + throw new Error('Invalid iterations'); +let manifest = JSON.parse( + await readFile(join(dataset, 'manifest.json'), 'utf8'), +); +if ( + manifest.synthetic !== true || + !['smoke', '1x', '10x'].includes(manifest.preset) +) + throw new Error('A Tessar synthetic manifest is required'); +let expected = JSON.parse( + await readFile(join(dataset, 'expected.json'), 'utf8'), +); +let hostDir = resolve(import.meta.dirname, '../../packages/host'); +process.env.TEST_HARNESS_HOST_DIST_PACKAGE_DIR = hostDir; +let hostHash = createHash('sha256') + .update(await readFile(join(hostDir, 'dist/index.html'))) + .digest('hex'); +let commit = execFileSync('git', ['rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +let repository = resolve(import.meta.dirname, '../..'); +let sourcePaths = execFileSync( + 'git', + [ + 'ls-files', + '-z', + '--cached', + '--others', + '--exclude-standard', + '--', + 'packages/base', + 'packages/runtime-common', + 'packages/host', + 'packages/postgres', + 'packages/realm-server', + 'scripts/tessar', + ], + { cwd: repository, encoding: 'utf8' }, +) + .split('\0') + .filter(Boolean); +let sourceHash = createHash('sha256'); +for (let path of [...new Set(sourcePaths)].sort()) { + try { + sourceHash + .update(path) + .update('\0') + .update(await readFile(join(repository, path))); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + sourceHash.update(`deleted:${path}`); + } +} +let runtimeHash = sourceHash.digest('hex'); +process.env.TEST_HARNESS_CACHE_SALT = `tessar:${commit}:${hostHash}:${runtimeHash}`; +let started = performance.now(); +let realm; +try { + realm = await startFactoryRealmServer({ + realms: [{ dir: join(dataset, 'realm'), path: 'tessar/' }], + }); + let startupMs = performance.now() - started; + await writeFile( + `${output}.runtime.json`, + JSON.stringify({ + realmURL: realm.realmURL.href, + realmServerURL: realm.realmServerURL.href, + databaseName: realm.databaseName, + childPids: realm.childPids, + }), + { mode: 0o600 }, + ); + console.log(`Tessar ready: ${realm.realmURL.href}`); + let results = []; + for (let iteration = 0; iteration < iterations; iteration++) { + let start = performance.now(); + let response = await fetch(realm.cardURL('DaySummary/00000'), { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(60_000), + }); + let bytes = await response.text(); + let elapsedMs = performance.now() - start; + let body = JSON.parse(bytes); + let valid = response.ok; + let failure; + try { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + validateSummary(body.data.attributes, expected['DaySummary/00000']); + } catch (error) { + valid = false; + failure = error.message; + } + results.push({ + iteration, + elapsedMs, + responseBytes: Buffer.byteLength(bytes), + status: response.status, + valid, + failure, + }); + if (iteration === 0) + await writeFile(`${output}.response.json`, JSON.stringify(body, null, 2)); + } + await writeFile( + output, + JSON.stringify( + { + version: 1, + commit, + hostHash, + runtimeHash, + manifest, + startupMs, + results, + }, + null, + 2, + ) + '\n', + ); + console.log( + JSON.stringify({ + output, + startupMs, + validReads: results.filter((r) => r.valid).length, + totalReads: results.length, + browserReady: values.serve, + }), + ); + if (values.serve) + await new Promise((done) => { + process.once('SIGINT', done); + process.once('SIGTERM', done); + }); + else if (results.some((r) => !r.valid)) process.exitCode = 1; +} finally { + await realm?.stop(); +} diff --git a/scripts/tessar/generate.mjs b/scripts/tessar/generate.mjs new file mode 100644 index 00000000000..236d013bcb2 --- /dev/null +++ b/scripts/tessar/generate.mjs @@ -0,0 +1,206 @@ +import { createHash } from 'node:crypto'; +import { mkdir, writeFile, copyFile } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { parseArgs, isDeepStrictEqual } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +// Synthetic workload proportions, not production records or identifiers. +export const counts = { + Classroom: 2, + Student: 13, + Staff: 26, + Slot: 173, + Observation: 164, + Report: 44, + Activity: 63, + Reference: 766, + DaySummary: 4, +}; +export const presets = { smoke: 0, '1x': 1, '10x': 10 }; +export const day = '2026-01-12'; +export const nextDay = '2026-01-13'; +const moduleRef = '../tessar'; + +export function generate({ + preset = 'smoke', + seed = 1729, + shape = 'distributed', +} = {}) { + if (!Object.hasOwn(presets, preset)) + throw new Error('Preset must be smoke, 1x or 10x'); + if (!Number.isSafeInteger(seed) || seed < 0) + throw new Error('Seed must be a nonnegative safe integer'); + if (!['distributed', 'focused'].includes(shape)) + throw new Error('Unknown workload shape'); + let scale = presets[preset]; + let sizes = Object.fromEntries( + Object.entries(counts).map(([type, count]) => [ + type, + scale ? count * scale : Math.min(count, 4), + ]), + ); + sizes.Classroom = scale ? 2 * scale : 2; + let state = seed >>> 0; + let random = () => + (state = (Math.imul(state, 1664525) + 1013904223) >>> 0) / 2 ** 32; + let id = (type, index) => `${type}/${String(index).padStart(5, '0')}`; + let records = new Map(); + for (let [type, count] of Object.entries(sizes)) { + for (let i = 0; i < count; i++) { + let classroom = i % (shape === 'focused' ? 2 : sizes.Classroom); + let attributes = { + label: `Tessar ${type} ${String(i).padStart(5, '0')}`, + classroomKey: `room-${classroom}`, + day: Math.floor(i / sizes.Classroom) % 2 ? nextDay : day, + sequence: i, + score: Math.floor(random() * 101), + status: i % 3 === 0 ? 'ready' : 'pending', + narrative: `Synthetic Tessar record ${i}. `.repeat( + type === 'Reference' ? 12 : 3, + ), + }; + let relationships; + if (['Slot', 'Observation', 'Report', 'Activity'].includes(type)) { + relationships = { + student: { + links: { self: `../${id('Student', i % sizes.Student)}` }, + }, + reference: { + links: { self: `../${id('Reference', i % sizes.Reference)}` }, + }, + staff: { links: { self: `../${id('Staff', i % sizes.Staff)}` } }, + }; + } + records.set(id(type, i), { + data: { + type: 'card', + attributes, + ...(relationships ? { relationships } : {}), + meta: { adoptsFrom: { module: moduleRef, name: type } }, + }, + }); + } + } + return { records, sizes, seed, preset, shape }; +} + +// Reference implementation deliberately operates on raw fixture documents, +// independently of CardDef getters, query fields, server results or the UI. +export function expectedSummary(records, ownerId) { + let owner = records.get(ownerId).data.attributes; + let expected = { + studentCount: 0, + slotCount: 0, + observationCount: 0, + reportCount: 0, + readyReportCount: 0, + scoreTotal: 0, + rows: [], + }; + for (let [sourceId, { data }] of records) { + let attrs = data.attributes; + if (attrs.classroomKey !== owner.classroomKey) continue; + let type = data.meta.adoptsFrom.name; + if (type === 'Student') expected.studentCount++; + if (attrs.day !== owner.day) continue; + if (type === 'Observation') { + expected.observationCount++; + expected.scoreTotal += attrs.score; + } + if (type === 'Report') { + expected.reportCount++; + if (attrs.status === 'ready') expected.readyReportCount++; + } + if (type === 'Slot') { + expected.slotCount++; + let studentId = data.relationships.student.links.self.slice(3); + let referenceId = data.relationships.reference.links.self.slice(3); + expected.rows.push({ + sourceId, + label: attrs.label, + sequence: attrs.sequence, + status: attrs.status, + studentLabel: records.get(studentId).data.attributes.label, + referenceLabel: records.get(referenceId).data.attributes.label, + }); + } + } + expected.rows.sort((a, b) => a.sequence - b.sequence); + return expected; +} + +export function validateSummary(actual, expected) { + let keys = Object.keys(expected); + let normalized = Object.fromEntries(keys.map((key) => [key, actual[key]])); + if (!isDeepStrictEqual(normalized, expected)) + throw new Error( + `Tessar result mismatch: ${JSON.stringify({ expected, actual: normalized })}`, + ); +} + +export async function writeDataset(output, options) { + let dataset = generate(options); + // Exclusive root creation avoids overwriting an existing run or real realm. + await mkdir(output); + let realmDir = join(output, 'realm'); + await mkdir(realmDir); + await copyFile( + new URL('./realm/tessar.gts', import.meta.url), + join(realmDir, 'tessar.gts'), + ); + let hash = createHash('sha256'); + for (let [id, document] of dataset.records) { + let bytes = JSON.stringify(document) + '\n'; + hash.update(id).update('\0').update(bytes); + await mkdir(join(realmDir, id.split('/')[0]), { recursive: true }); + await writeFile(join(realmDir, `${id}.json`), bytes); + } + let expected = Object.fromEntries( + [...dataset.records.keys()] + .filter((id) => id.startsWith('DaySummary/')) + .map((id) => [id, expectedSummary(dataset.records, id)]), + ); + let manifest = { + version: 1, + synthetic: true, + seed: dataset.seed, + preset: dataset.preset, + shape: dataset.shape, + instanceCount: dataset.records.size, + counts: dataset.sizes, + recordsSha256: hash.digest('hex'), + }; + await writeFile( + join(output, 'manifest.json'), + JSON.stringify(manifest, null, 2) + '\n', + ); + await writeFile( + join(output, 'expected.json'), + JSON.stringify(expected, null, 2) + '\n', + ); + return manifest; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + let { values } = parseArgs({ + options: { + output: { type: 'string' }, + preset: { type: 'string', default: 'smoke' }, + seed: { type: 'string', default: '1729' }, + shape: { type: 'string', default: 'distributed' }, + }, + }); + if (!values.output) throw new Error('--output must name a new directory'); + console.log( + JSON.stringify( + await writeDataset(resolve(values.output), { + preset: values.preset, + seed: Number(values.seed), + shape: values.shape, + }), + ), + ); +} diff --git a/scripts/tessar/generate.test.mjs b/scripts/tessar/generate.test.mjs new file mode 100644 index 00000000000..3ebd660ea5f --- /dev/null +++ b/scripts/tessar/generate.test.mjs @@ -0,0 +1,53 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; + +test('Tessar datasets have fixed sizes, deterministic fabricated records and valid links', () => { + for (let [preset, count] of [ + ['1x', 1255], + ['10x', 12550], + ]) { + let { records } = generate({ preset }); + assert.equal(records.size, count); + assert.deepEqual(records, generate({ preset }).records); + for (let { data } of records.values()) { + assert.match(data.attributes.label, /^Tessar /); + for (let relationship of Object.values(data.relationships ?? {})) { + assert.ok(records.has(relationship.links.self.slice(3))); + } + } + } + assert.throws(() => generate({ preset: '100x' }), /Preset/); +}); + +test('Tessar raw-data oracle reflects membership, content, transitive edits and deletions', () => { + let { records } = generate(); + let id = 'DaySummary/00000'; + let before = expectedSummary(records, id); + assert.equal(before.studentCount, 2); + assert.equal(before.slotCount, 1); + assert.equal(before.observationCount, 1); + assert.equal(before.readyReportCount, 1); + assert.deepEqual( + before.rows.map((r) => r.sourceId), + ['Slot/00000'], + ); + records.get('Student/00000').data.attributes.label = 'Tessar revised student'; + assert.equal( + expectedSummary(records, id).rows[0].studentLabel, + 'Tessar revised student', + ); + records.get('Observation/00000').data.attributes.score = 123; + assert.equal(expectedSummary(records, id).scoreTotal, 123); + records.get('Slot/00002').data.attributes.day = '2026-01-12'; + assert.equal(expectedSummary(records, id).slotCount, 2); + records.delete('Slot/00000'); + assert.deepEqual( + expectedSummary(records, id).rows.map((r) => r.sourceId), + ['Slot/00002'], + ); + assert.throws( + () => validateSummary(before, expectedSummary(records, id)), + /mismatch/, + ); +}); diff --git a/scripts/tessar/realm/tessar.gts b/scripts/tessar/realm/tessar.gts new file mode 100644 index 00000000000..94758026837 --- /dev/null +++ b/scripts/tessar/realm/tessar.gts @@ -0,0 +1,182 @@ +import { + CardDef, + FieldDef, + Component, + field, + contains, + containsMany, + linksTo, + linksToMany, + StringField, + NumberField, +} from '@cardstack/base/card-api'; + +export class TessarRecord extends CardDef { + @field label = contains(StringField); + @field classroomKey = contains(StringField); + @field day = contains(StringField); + @field sequence = contains(NumberField); + @field score = contains(NumberField); + @field status = contains(StringField); + @field narrative = contains(StringField); +} +export class Classroom extends TessarRecord {} +export class Student extends TessarRecord {} +export class Staff extends TessarRecord {} +export class Reference extends TessarRecord {} +export class Activity extends TessarRecord { + @field student = linksTo(Student); + @field staff = linksTo(Staff); + @field reference = linksTo(Reference); +} +export class Slot extends Activity {} +export class Observation extends Activity {} +export class Report extends Activity {} + +export class TessarRow extends FieldDef { + @field sourceId = contains(StringField); + @field label = contains(StringField); + @field sequence = contains(NumberField); + @field status = contains(StringField); + @field studentLabel = contains(StringField); + @field referenceLabel = contains(StringField); +} + +export class DaySummary extends TessarRecord { + @field students = linksToMany(Student, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey' } }, + page: { size: 2000 }, + }, + }); + @field slots = linksToMany(Slot, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, + }, + }); + @field observations = linksToMany(Observation, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, + }, + }); + @field reports = linksToMany(Report, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, + }, + }); + @field studentCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.students.length; + }, + }); + @field slotCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.slots.length; + }, + }); + @field observationCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.observations.length; + }, + }); + @field reportCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.reports.length; + }, + }); + @field readyReportCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.reports.filter((report) => report.status === 'ready').length; + }, + }); + @field scoreTotal = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.observations.reduce( + (sum, item) => sum + (item.score ?? 0), + 0, + ); + }, + }); + @field rows = containsMany(TessarRow, { + computeVia: function (this: DaySummary) { + return [...this.slots] + .sort((a, b) => (a.sequence ?? 0) - (b.sequence ?? 0)) + .map( + (slot) => + new TessarRow({ + sourceId: `Slot/${String(slot.sequence).padStart(5, '0')}`, + label: slot.label, + sequence: slot.sequence, + status: slot.status, + studentLabel: slot.student?.label, + referenceLabel: slot.reference?.label, + }), + ); + }, + }); + + static isolated = class extends Component { + + }; +} From 7ca8453144ba46ac34c06936bf468d1a5b5008b8 Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:18:41 -0400 Subject: [PATCH 07/12] Add Tessar snapshot consumption without eager query hydration --- packages/base/card-api.gts | 124 +++++++++- packages/base/card-serialization.ts | 8 + packages/base/field-support.ts | 51 +++++ .../components/tessar-snapshot-test.gts | 213 ++++++++++++++++++ 4 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 packages/host/tests/integration/components/tessar-snapshot-test.gts diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 0d06ea943cd..953224bd9b9 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -188,6 +188,9 @@ import { getFields, getRelationshipMembershipState, getter, + hasTessarQueryMembership, + leaveTessarSnapshot, + setTessarSnapshot, registerRelationshipProbe, relationshipStateForEntry, readFieldLoadingSignal, @@ -1400,7 +1403,10 @@ class LinksTo implements Field { let deserialized = getDataBucket(instance); entangleWithCardTracking(instance); - if (this.queryDefinition) { + if ( + this.queryDefinition && + !hasTessarQueryMembership(instance, this.name) + ) { let dependencyTrackingContext = runtimeQueryDependencyContext({ queryField: this.name, consumer: instance.id, @@ -1876,7 +1882,10 @@ class LinksToMany implements Field< let deserialized = getDataBucket(instance); - if (this.queryDefinition) { + if ( + this.queryDefinition && + !hasTessarQueryMembership(instance, this.name) + ) { let dependencyTrackingContext = runtimeQueryDependencyContext({ queryField: this.name, consumer: instance.id, @@ -4612,6 +4621,24 @@ registerRelationshipProbe((instance, field) => { // bump (load start / settle, or query resource creation) re-evaluates this. readFieldLoadingSignal(instance, field.name); if (field.queryDefinition) { + if (hasTessarQueryMembership(instance, field.name)) { + let value = getDataBucket(instance).get(field.name); + let entries = Array.isArray(value) + ? rawArrayValues(value) + : value == null + ? [] + : [value]; + return { + isLoading: false, + isQueryField: true, + queryMembership: entries.map((entry) => + relationshipStateForEntry(entry), + ), + queryTotalMatchCount: entries.length, + queryIsPartial: false, + queryHasUnreachableRealms: false, + }; + } let resource = peekQueryFieldSearchResource(instance, field.name); let isLoading = resource?.isLoading ?? false; let bucketEntry = getDataBucket(instance).get(field.name); @@ -4918,6 +4945,29 @@ async function getDeserializedValue({ if (!field) { throw new Error(`could not find field ${fieldName} in card ${card.name}`); } + if (opts?.tessarSnapshot) { + let snapshot = opts.tessarSnapshot; + if (field.fieldType === 'contains' || field.fieldType === 'containsMany') { + let prefix = `${fieldName}.${field.fieldType === 'containsMany' ? '*.' : ''}`; + opts = { + ...opts, + tessarSnapshot: { + ...snapshot, + computedFields: snapshot.computedFields + .filter((path) => path.startsWith(prefix)) + .map((path) => path.slice(prefix.length)), + queryFields: snapshot.queryFields + .filter((path) => path.startsWith(prefix)) + .map((path) => path.slice(prefix.length)), + }, + }; + } else { + // Carry membership IDs without inflating the included input graph. An + // explicit later link read uses the ordinary lazy/identity-map path. + if (snapshot.queryFields.includes(fieldName)) doc = { data: resource }; + opts = { ...opts, tessarSnapshot: undefined }; + } + } let result = await field.deserialize( value, doc, @@ -5102,6 +5152,49 @@ async function _updateFromSerialized({ store: CardStore; opts?: DeserializeOpts; }): Promise> { + if (opts?.tessarSnapshot) { + for (let path of opts.tessarSnapshot.computedFields) { + let name = path.split('.')[0]; + if ( + !Object.prototype.hasOwnProperty.call(resource.attributes ?? {}, name) + ) { + throw new Error(`Tessar snapshot is missing computed output '${path}'`); + } + } + for (let name of opts.tessarSnapshot.queryFields) { + if (name.includes('.')) continue; // checked by contained deserialization + let relationship = resource.relationships?.[name]; + if (Array.isArray(relationship)) { + throw new Error( + `Tessar snapshot query '${name}' needs complete umbrella membership`, + ); + } + let membership = relationship?.data; + let count = Array.isArray(membership) + ? membership.length + : membership === null + ? 0 + : membership + ? 1 + : undefined; + if ( + count === undefined || + relationship?.meta?.total !== count || + relationship?.meta?.errors?.length + ) { + throw new Error( + `Tessar snapshot query '${name}' is missing, partial or errored`, + ); + } + } + opts = { + ...opts, + tessarSnapshot: { + ...opts.tessarSnapshot, + scope: opts.tessarSnapshot.scope ?? { active: true }, + }, + }; + } // because our store uses a tracked map for its identity map all the assembly // work that we are doing to deserialize the instance below is "live". so we // add the actual instance silently in a non-tracked way and only track it at @@ -5380,6 +5473,9 @@ async function _updateFromSerialized({ } let deserialized = getDataBucket(instance); + let snapshotOptions = opts?.tessarSnapshot; + let snapshotValues = new Map(); + for (let [field, value] of values) { if (!field) { continue; @@ -5404,9 +5500,26 @@ async function _updateFromSerialized({ applySubscribersToInstanceValue(instance, field, existingValue, value); } deserialized.set(field.name as string, value); + if ( + snapshotOptions?.computedFields.includes(field.name) && + field.computeVia + ) { + snapshotValues.set(field.name, value); + } field.captureQueryFieldSeedData?.(instance, value, resource); } + setTessarSnapshot( + instance, + snapshotOptions + ? { + scope: snapshotOptions.scope!, + values: snapshotValues, + queryFields: new Set(snapshotOptions.queryFields), + } + : undefined, + ); + // assign the realm meta before we compute as computeds may be relying on this if (!isFieldInstance(instance) && resource.id != null) { (instance as any)[meta] = resource.meta; @@ -5439,7 +5552,10 @@ async function _updateFromSerialized({ for (let field of Object.values( getFields(instance, { includeComputeds: true }), )) { - if (field?.queryDefinition) { + if ( + field?.queryDefinition && + !hasTessarQueryMembership(instance, field.name) + ) { resolveQueryFieldEagerly(store, instance, field); } } @@ -5515,6 +5631,7 @@ function setField(instance: BaseDef, field: Field, value: any) { propagateRealmContext(value, instance); // TODO: refactor validate to not have a return value and accomplish this normalization another way value = field.validate(instance, value); + leaveTessarSnapshot(instance); let deserialized = getDataBucket(instance); deserialized.set(field.name, value); notifySubscribers(instance, field.name, value); @@ -5527,6 +5644,7 @@ function notifySubscribers( value: any, visited = new WeakSet(), ) { + leaveTessarSnapshot(instance); if (visited.has(instance)) { return; } diff --git a/packages/base/card-serialization.ts b/packages/base/card-serialization.ts index a4803f9ee12..0c119783f46 100644 --- a/packages/base/card-serialization.ts +++ b/packages/base/card-serialization.ts @@ -82,6 +82,14 @@ export interface SerializeOpts { export interface DeserializeOpts { ignoreBrokenLinks?: true; dependencyTrackingContext?: RuntimeDependencyTrackingContext; + // Internal Tessar read mode, supplied only after the caller verifies server + // provenance/completeness. Never inferred from authored attributes. Contained + // paths use dots and '*' for containsMany entries; links remain lazy. + tessarSnapshot?: { + computedFields: string[]; + queryFields: string[]; + scope?: { active: boolean }; + }; } // --- Serialization Symbols --- diff --git a/packages/base/field-support.ts b/packages/base/field-support.ts index ceae18b86ad..320f3acf147 100644 --- a/packages/base/field-support.ts +++ b/packages/base/field-support.ts @@ -57,6 +57,53 @@ const deserializedData = initSharedState( 'deserializedData', () => new WeakMap>(), ); + +// Tessar's reader overlay is separate from authored/default values. The caller +// must validate the indexed revision before requesting snapshot deserialization. +// A contained edit leaves snapshot mode for the entire owner graph. +export interface TessarSnapshotScope { + active: boolean; +} +const tessarSnapshots = initSharedState( + 'tessarSnapshots', + () => + new WeakMap< + BaseDef, + { + scope: TessarSnapshotScope; + values: Map; + queryFields: Set; + } + >(), +); + +export function setTessarSnapshot( + instance: BaseDef, + snapshot?: { + scope: TessarSnapshotScope; + values: Map; + queryFields: Set; + }, +): void { + let previous = tessarSnapshots.get(instance); + if (previous && previous.scope !== snapshot?.scope) + previous.scope.active = false; + if (snapshot) tessarSnapshots.set(instance, snapshot); + else tessarSnapshots.delete(instance); +} + +export function leaveTessarSnapshot(instance: BaseDef): void { + let snapshot = tessarSnapshots.get(instance); + if (snapshot) snapshot.scope.active = false; +} + +export function hasTessarQueryMembership( + instance: BaseDef, + fieldName: string, +): boolean { + let snapshot = tessarSnapshots.get(instance); + return Boolean(snapshot?.scope.active && snapshot.queryFields.has(fieldName)); +} // Cache for resolved field configurations per instance/field const fieldConfigurationCache = initSharedState( 'fieldConfigurationCache', @@ -162,6 +209,10 @@ export function getter( cardTracking.get(instance); if (field.computeVia) { + let snapshot = tessarSnapshots.get(instance); + if (snapshot?.scope.active && snapshot.values.has(field.name)) { + return snapshot.values.get(field.name) as BaseInstanceType; + } // Fast path when no pass is open: skip the counter + memo entirely // so production reads pay only one branch on the module-local null // check. JIT branch-predicts this and the original behaviour is diff --git a/packages/host/tests/integration/components/tessar-snapshot-test.gts b/packages/host/tests/integration/components/tessar-snapshot-test.gts new file mode 100644 index 00000000000..c467960b72e --- /dev/null +++ b/packages/host/tests/integration/components/tessar-snapshot-test.gts @@ -0,0 +1,213 @@ +import { getService } from '@universal-ember/test-support'; +import { module, test } from 'qunit'; + +import { rri } from '@cardstack/runtime-common'; + +import { + setupIntegrationTestRealm, + setupLocalIndexing, + testRealmURL, +} from '../../helpers'; +import { + setupBaseRealm, + CardDef, + FieldDef, + contains, + containsMany, + linksToMany, + field, + StringField, + NumberField, +} from '../../helpers/base-realm'; +import { setupMockMatrix } from '../../helpers/mock-matrix'; +import { setupRenderingTest } from '../../helpers/setup'; + +module('Integration | Tessar snapshot', function (hooks) { + setupRenderingTest(hooks); + setupBaseRealm(hooks); + setupLocalIndexing(hooks); + let mockMatrixUtils = setupMockMatrix(hooks); + + test('supplied outputs and membership do not compute, search or inflate included inputs', async function (assert) { + let calls = 0; + class TessarInput extends CardDef { + @field name = contains(StringField); + } + class TessarSummary extends CardDef { + @field inputs = linksToMany(TessarInput, { + query: { filter: { eq: { name: 'Tessar' } } }, + }); + @field count = contains(NumberField, { + computeVia: function () { + calls++; + return 999; + }, + }); + @field empty = containsMany(StringField, { + computeVia: function () { + calls++; + return ['live']; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarInput, TessarSummary } }, + }); + let api: typeof import('@cardstack/base/card-api') = await getService( + 'loader-service', + ).loader.import('@cardstack/base/card-api'); + let summary = new TessarSummary(); + let store = api.getStore(summary); + let searches = 0; + store.resolvesQueryFieldsEagerly = true; + store.getSearchResource = () => { + searches++; + throw new Error('Tessar snapshot must not search'); + }; + let inputId = rri(`${testRealmURL}TessarInput/1`); + let doc = { + data: { + id: `${testRealmURL}TessarSummary/1`, + type: 'card' as const, + attributes: { count: 0, empty: [] }, + relationships: { + inputs: { data: [{ type: 'card', id: inputId }], meta: { total: 1 } }, + }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + }, + }, + included: [ + { + id: inputId, + type: 'card' as const, + attributes: { name: 'Tessar' }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarInput', + }, + }, + }, + ], + }; + let opts = { + tessarSnapshot: { + computedFields: ['count', 'empty'], + queryFields: ['inputs'], + }, + }; + await api.updateFromSerialized(summary, doc, store, opts); + assert.strictEqual(summary.count, 0); + assert.deepEqual([...summary.empty], []); + assert.strictEqual(calls, 0, 'no computed getter was invoked'); + assert.strictEqual(searches, 0, 'no query search was started'); + assert.strictEqual( + store.getCard(inputId), + undefined, + 'included input stayed uninflated', + ); + let membership = api.getRelationshipMembershipState(summary, 'inputs'); + assert.strictEqual(membership.totalMatchCount, 1); + assert.strictEqual(membership.membership?.length, 1); + assert.false(membership.isPartial); + doc.data.attributes.count = 7; + await api.updateFromSerialized(summary, doc, store, opts); + assert.strictEqual( + summary.count, + 7, + 'new owner snapshot replaces old output', + ); + assert.strictEqual(calls, 0); + doc.data.relationships.inputs.meta.total = 2; + await assert.rejects( + api.updateFromSerialized(summary, doc, store, opts), + /missing, partial or errored/, + 'partial membership cannot be accepted as a complete materialization', + ); + }); + + test('nested values share edit invalidation and legacy documents keep computing', async function (assert) { + let calls = 0; + class TessarDetails extends FieldDef { + @field label = contains(StringField); + @field total = contains(NumberField, { + computeVia: function () { + calls++; + return 12; + }, + }); + } + class TessarSummary extends CardDef { + @field label = contains(StringField); + @field details = contains(TessarDetails); + @field rows = containsMany(TessarDetails); + @field total = contains(NumberField, { + computeVia: function () { + calls++; + return 42; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarSummary, TessarDetails } }, + }); + let api: typeof import('@cardstack/base/card-api') = await getService( + 'loader-service', + ).loader.import('@cardstack/base/card-api'); + let summary = new TessarSummary(); + let doc = { + data: { + type: 'card' as const, + attributes: { + label: 'Tessar', + total: 0, + details: { label: 'Nested', total: 3 }, + rows: [{ label: 'Row', total: 5 }], + }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + }, + }, + }; + let opts = { + tessarSnapshot: { + computedFields: ['total', 'details.total', 'rows.*.total'], + queryFields: [], + }, + }; + await api.updateFromSerialized(summary, doc, api.getStore(summary), opts); + assert.strictEqual(summary.total, 0); + assert.strictEqual(summary.details.total, 3); + assert.strictEqual(summary.rows[0].total, 5); + assert.strictEqual(calls, 0); + summary.details.label = 'Edited'; + assert.strictEqual( + summary.total, + 42, + 'contained edit clears the owner overlay', + ); + assert.strictEqual( + summary.rows[0].total, + 12, + 'sibling overlay also leaves snapshot mode', + ); + await api.updateFromSerialized(summary, doc); + assert.strictEqual( + summary.total, + 42, + 'ordinary deserialization never trusts supplied computeds', + ); + assert.strictEqual(summary.details.total, 12); + }); +}); From fead7a13ab02154a79e743877798166765959fbe Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:19:55 -0400 Subject: [PATCH 08/12] Add Tessar reverse query registry and durable revision guards --- ...49_schema.sql => 1789091944988_schema.sql} | 28 ++ .../tests/unit/index-query-engine-test.ts | 212 ++++++++++++++ .../1789091944988_tessar-query-watches.js | 39 +++ .../tests/tessar-query-registry-test.ts | 184 ++++++++++++ packages/runtime-common/index-query-engine.ts | 127 +++++++++ .../runtime-common/tessar-query-registry.ts | 266 ++++++++++++++++++ 6 files changed, 856 insertions(+) rename packages/host/config/schema/{1788216332849_schema.sql => 1789091944988_schema.sql} (88%) create mode 100644 packages/postgres/migrations/1789091944988_tessar-query-watches.js create mode 100644 packages/realm-server/tests/tessar-query-registry-test.ts create mode 100644 packages/runtime-common/tessar-query-registry.ts diff --git a/packages/host/config/schema/1788216332849_schema.sql b/packages/host/config/schema/1789091944988_schema.sql similarity index 88% rename from packages/host/config/schema/1788216332849_schema.sql rename to packages/host/config/schema/1789091944988_schema.sql index fbf74a8e90d..3f7b3c69751 100644 --- a/packages/host/config/schema/1788216332849_schema.sql +++ b/packages/host/config/schema/1789091944988_schema.sql @@ -222,6 +222,34 @@ PRIMARY KEY ( realm_url, username ) ); + CREATE TABLE IF NOT EXISTS tessar_owners ( + realm_url TEXT NOT NULL, + owner_url TEXT NOT NULL, + published_generation NOT NULL, + input_generation NOT NULL, + dirty_generation, + definition_revision TEXT NOT NULL, + retired BOOLEAN DEFAULT false NOT NULL, + PRIMARY KEY ( realm_url, owner_url ) +); + + CREATE TABLE IF NOT EXISTS tessar_query_terms ( + realm_url TEXT NOT NULL, + owner_url TEXT NOT NULL, + field_path TEXT NOT NULL, + path TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY ( realm_url, owner_url, field_path, path, value ) +); + + CREATE TABLE IF NOT EXISTS tessar_query_watches ( + realm_url TEXT NOT NULL, + owner_url TEXT NOT NULL, + field_path TEXT NOT NULL, + query BLOB NOT NULL, + PRIMARY KEY ( realm_url, owner_url, field_path ) +); + CREATE TABLE IF NOT EXISTS unlisted_realm_paths ( id DEFAULT (hex(randomblob(16))) NOT NULL, source_realm_url TEXT NOT NULL, diff --git a/packages/host/tests/unit/index-query-engine-test.ts b/packages/host/tests/unit/index-query-engine-test.ts index 478a7ea1043..f8ac6046370 100644 --- a/packages/host/tests/unit/index-query-engine-test.ts +++ b/packages/host/tests/unit/index-query-engine-test.ts @@ -14,6 +14,8 @@ import { identifyCard, getFieldDefinitions, rri, + coerceTypes, + type Filter, type RealmResourceIdentifier, type ResolvedCodeRef, type Definition, @@ -22,7 +24,11 @@ import { isFilterRefersToNonsearchableFieldError, } from '@cardstack/runtime-common'; +import { query } from '@cardstack/runtime-common/expression'; +import { TessarQueryRegistry } from '@cardstack/runtime-common/tessar-query-registry'; + import ENV from '@cardstack/host/config/environment'; + import { shimExternals } from '@cardstack/host/lib/externals'; import type SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter'; @@ -296,6 +302,212 @@ module('Unit | query', function (hooks) { ); }); + test('Tessar reverse verification agrees with forward predicates for indexed documents', async function (assert) { + let { mango, vangogh, ringo, paper } = testCards; + await setupIndex(dbAdapter, [ + { + card: mango, + data: { + search_doc: { + name: 'Mango', + age: 5, + isHairy: true, + friends: [{ name: 'Ringo' }], + }, + }, + }, + { + card: vangogh, + data: { + search_doc: { name: 'Van Gogh', age: 8, isHairy: false, friends: [] }, + }, + }, + { + card: ringo, + data: { + search_doc: { + name: 'Ringo', + age: null, + isHairy: true, + friends: [{ name: 'Mango' }], + }, + }, + }, + { card: paper, data: { search_doc: { name: 'Mango' } } }, + ]); + let documents = (await dbAdapter.execute( + 'SELECT url, search_doc, types FROM boxel_index WHERE type = $1', + { bind: ['instance'], coerceTypes }, + )) as unknown as Array< + Parameters[1] + >; + let on = { module: rri(`${testRealmURL}person`), name: 'Person' }; + let predicates: Filter[] = [ + { on, eq: { name: 'Mango' } }, + { on, in: { name: ['Mango', 'Ringo'] } }, + { on, eq: { age: null } }, + { on, eq: { 'friends.name': 'Ringo' } }, + { on, any: [{ eq: { name: 'Mango' } }, { eq: { name: 'Van Gogh' } }] }, + { on, not: { eq: { name: 'Mango' } } }, + { on, range: { age: { gte: 5, lt: 8 } } }, + { on, eq: { isHairy: true } }, + ]; + let expectedCounts = [1, 2, 1, 1, 2, 2, 1, 2]; + for (let [index, filter] of predicates.entries()) { + let forward = await indexQueryEngine.searchCards(new URL(testRealmURL), { + filter, + }); + let matched: string[] = []; + assert.strictEqual( + forward.cards.length, + expectedCounts[index], + 'forward fixture has the expected real matches', + ); + for (let document of documents) { + if (await indexQueryEngine.tessarMatchesDocument(filter, document)) + matched.push(document.url.replace(/\.json$/, '')); + } + assert.deepEqual( + matched.sort(), + forward.cards.map((card) => String(card.id)).sort(), + stringify(filter), + ); + } + assert.deepEqual( + await indexQueryEngine.tessarRoutingTerms({ on, eq: { name: 'Mango' } }), + [{ path: 'name', value: 'Mango' }], + ); + assert.strictEqual( + await indexQueryEngine.tessarRoutingTerms({ + on, + any: [{ eq: { name: 'Mango' } }, { not: { eq: { name: 'Ringo' } } }], + }), + undefined, + 'an unanchored OR branch requires the broad bucket', + ); + assert.strictEqual( + await indexQueryEngine.tessarRoutingTerms({ on, eq: { age: null } }), + undefined, + 'null routes broadly, including absent values', + ); + }); + + test('Tessar watches route conservatively and preserve durable invalidation', async function (assert) { + let registry = new TessarQueryRegistry(dbAdapter, indexQueryEngine); + let tx = (expression: Parameters[1]) => + query(dbAdapter, expression); + let on = { module: rri(`${testRealmURL}person`), name: 'Person' }; + let owner = (name: string, generation = 1) => ({ + realmURL: testRealmURL, + ownerURL: `${testRealmURL}${name}`, + generation, + inputGeneration: generation, + definitionRevision: 'tessar-v1', + }); + let publish = async (name: string, filter: Filter) => + registry.publish(tx, { + ...owner(name), + watches: await registry.prepare([ + { + fieldPath: 'people', + query: { filter, page: { size: 1, number: 0 } }, + }, + ]), + }); + await publish('mango-summary', { on, eq: { name: 'Mango' } }); + await publish('ringo-summary', { on, eq: { name: 'Ringo' } }); + await publish('empty-summary', { on, eq: { name: 'Future' } }); + await publish('broad-summary', { on, not: { eq: { name: 'Excluded' } } }); + await setupIndex(dbAdapter, [ + { card: testCards.mango, data: { search_doc: { name: 'Mango' } } }, + { card: testCards.ringo, data: { search_doc: { name: 'Ringo' } } }, + ]); + let rows = await dbAdapter.execute( + "SELECT url, search_doc, types FROM boxel_index WHERE type = 'instance'", + { coerceTypes }, + ); + let mango = rows.find( + (row) => (row.search_doc as any).name === 'Mango', + ) as unknown as Parameters[1]; + let moved = { + ...mango, + search_doc: { ...mango.search_doc, name: 'Ringo' }, + }; + let affected = await registry.affected(testRealmURL, mango, moved); + assert.deepEqual( + affected, + ['broad-summary', 'mango-summary', 'ringo-summary'] + .map((name) => `${testRealmURL}${name}`) + .sort(), + 'old and new scopes both invalidate', + ); + assert.deepEqual( + await registry.affected(testRealmURL, undefined, { + ...mango, + search_doc: { ...mango.search_doc, name: 'Future' }, + }), + ['broad-summary', 'empty-summary'] + .map((name) => `${testRealmURL}${name}`) + .sort(), + 'an initially empty watch sees an insertion', + ); + assert.deepEqual( + await registry.affected(testRealmURL, mango, undefined), + ['broad-summary', 'mango-summary'] + .map((name) => `${testRealmURL}${name}`) + .sort(), + 'deletions use the old document', + ); + assert.deepEqual( + await registry.affected('http://elsewhere/', mango, moved), + [], + 'realm isolation', + ); + await registry.markDirty(tx, testRealmURL, affected, 3); + await registry.markDirty(tx, testRealmURL, affected, 2); + assert.true( + ( + await new TessarQueryRegistry(dbAdapter, indexQueryEngine).pending( + testRealmURL, + ) + ).every((row) => row.generation === 3), + 'dirty watermark survives reconstruction and never regresses', + ); + let watches = await registry.prepare([ + { fieldPath: 'people', query: { filter: { on, eq: { name: 'Mango' } } } }, + ]); + assert.false( + await registry.publish(tx, { ...owner('mango-summary', 2), watches }), + 'a result that missed a newer change cannot publish', + ); + assert.true( + await registry.publish(tx, { ...owner('mango-summary', 3), watches }), + 'observing the required revision permits publication', + ); + assert.false( + (await registry.pending(testRealmURL)).some( + (row) => row.ownerURL === owner('mango-summary').ownerURL, + ), + 'successful publication clears dirty state', + ); + assert.true( + await registry.publish(tx, { + ...owner('mango-summary', 4), + watches: [], + retired: true, + }), + ); + assert.false( + await registry.publish(tx, { ...owner('mango-summary', 3), watches }), + 'a delayed worker cannot resurrect retired watches', + ); + assert.false( + (await registry.affected(testRealmURL, mango, undefined)).includes( + owner('mango-summary').ownerURL, + ), + ); + }); + test('can get all cards with empty filter', async function (assert) { let { mango, vangogh, paper } = testCards; await setupIndex(dbAdapter, [mango, vangogh, paper]); diff --git a/packages/postgres/migrations/1789091944988_tessar-query-watches.js b/packages/postgres/migrations/1789091944988_tessar-query-watches.js new file mode 100644 index 00000000000..e4120ddb890 --- /dev/null +++ b/packages/postgres/migrations/1789091944988_tessar-query-watches.js @@ -0,0 +1,39 @@ +exports.shorthands = undefined; + +exports.up = (pgm) => { + // Tessar is opt-in. The owner row survives retirement to prevent an older + // worker from resurrecting watches after a definition change or deletion. + pgm.createTable('tessar_owners', { + realm_url: { type: 'varchar', notNull: true, primaryKey: true }, + owner_url: { type: 'varchar', notNull: true, primaryKey: true }, + published_generation: { type: 'bigint', notNull: true }, + input_generation: { type: 'bigint', notNull: true }, + dirty_generation: 'bigint', + definition_revision: { type: 'varchar', notNull: true }, + retired: { type: 'boolean', notNull: true, default: false }, + }); + pgm.createTable('tessar_query_watches', { + realm_url: { type: 'varchar', notNull: true, primaryKey: true }, + owner_url: { type: 'varchar', notNull: true, primaryKey: true }, + field_path: { type: 'varchar', notNull: true, primaryKey: true }, + query: { type: 'jsonb', notNull: true }, + }); + // One safe disjunctive set of equality anchors per watch. The empty path is + // a broad route. These rows are candidates only; the ordinary SQL compiler + // verifies the saved predicate against both old and new indexed documents. + pgm.createTable('tessar_query_terms', { + realm_url: { type: 'varchar', notNull: true, primaryKey: true }, + owner_url: { type: 'varchar', notNull: true, primaryKey: true }, + field_path: { type: 'varchar', notNull: true, primaryKey: true }, + path: { type: 'varchar', notNull: true, primaryKey: true }, + value: { type: 'varchar', notNull: true, primaryKey: true }, + }); + pgm.createIndex('tessar_query_terms', ['realm_url', 'path', 'value']); + pgm.createIndex('tessar_owners', ['realm_url', 'dirty_generation']); +}; + +exports.down = (pgm) => { + pgm.dropTable('tessar_query_terms'); + pgm.dropTable('tessar_query_watches'); + pgm.dropTable('tessar_owners'); +}; diff --git a/packages/realm-server/tests/tessar-query-registry-test.ts b/packages/realm-server/tests/tessar-query-registry-test.ts new file mode 100644 index 00000000000..009160b1fc6 --- /dev/null +++ b/packages/realm-server/tests/tessar-query-registry-test.ts @@ -0,0 +1,184 @@ +import QUnit from 'qunit'; +import type { PgAdapter } from '@cardstack/postgres'; +import { + IndexQueryEngine, + VirtualNetwork, + baseRealmRRI, + rri, + type Definition, + type DefinitionLookup, + type Filter, +} from '@cardstack/runtime-common'; +import { + TessarQueryRegistry, + type TessarDocument, +} from '@cardstack/runtime-common/tessar-query-registry'; +import { setupDB } from './helpers/index.ts'; + +const { module, test } = QUnit; +const realmURL = 'https://tessar.example/'; +const on = { module: rri(`${realmURL}record`), name: 'TessarRecord' }; +const definition: Definition = { + type: 'card-def', + codeRef: on, + displayName: 'Tessar record', + fields: { + name: 'string', + score: 'number', + active: 'boolean', + tags: 'strings', + }, + fieldDefs: { + string: { + type: 'contains', + isPrimitive: true, + isComputed: false, + fieldOrCard: { module: rri(`${baseRealmRRI}string`), name: 'default' }, + }, + strings: { + type: 'containsMany', + isPrimitive: true, + isComputed: false, + fieldOrCard: { module: rri(`${baseRealmRRI}string`), name: 'default' }, + }, + number: { + type: 'contains', + isPrimitive: true, + isComputed: false, + serializerName: 'number', + fieldOrCard: { module: rri(`${baseRealmRRI}number`), name: 'default' }, + }, + boolean: { + type: 'contains', + isPrimitive: true, + isComputed: false, + serializerName: 'boolean', + fieldOrCard: { module: rri(`${baseRealmRRI}boolean`), name: 'default' }, + }, + }, +}; + +module('Tessar | Postgres query registry', function (hooks) { + let db: PgAdapter; + let engine: IndexQueryEngine; + let registry: TessarQueryRegistry; + setupDB(hooks, { + templateDatabase: process.env.TESSAR_TEST_TEMPLATE_DB, + beforeEach: async (adapter) => { + db = adapter; + // Persisted definition fixture: the SQL compiler receives the same + // loaderless field schema that a real module-index entry supplies. + let lookup = { + async lookupDefinition() { + return definition; + }, + } as unknown as DefinitionLookup; + let network = new VirtualNetwork(); + network.addRealmMapping(baseRealmRRI, 'https://cardstack.com/base/'); + engine = new IndexQueryEngine(db, lookup, network); + registry = new TessarQueryRegistry(db, engine); + }, + }); + let record = (name: string): TessarDocument => ({ + url: `${realmURL}record-1.json`, + types: [`${on.module}/${on.name}`], + search_doc: { name, score: 7, active: true, tags: ['red', 'blue'] }, + }); + + test('real SQL verifies typed values, null, plural paths and boolean composition', async function (assert) { + let predicates: Array<[Filter, boolean]> = [ + [{ on, eq: { name: 'A' } }, true], + [{ on, eq: { name: 'B' } }, false], + [{ on, in: { name: ['A', 'B'] } }, true], + [{ on, range: { score: { gte: 7, lt: 8 } } }, true], + [{ on, range: { score: { gt: 7 } } }, false], + [{ on, eq: { active: true } }, true], + [{ on, eq: { tags: 'blue' } }, true], + [{ on, eq: { score: null } }, false], + [ + { on, any: [{ eq: { name: 'B' } }, { not: { eq: { score: 8 } } }] }, + true, + ], + ]; + for (let [filter, expected] of predicates) { + assert.strictEqual( + await engine.tessarMatchesDocument(filter, record('A')), + expected, + JSON.stringify(filter), + ); + } + assert.true( + await engine.tessarMatchesDocument( + { on, eq: { score: null } }, + { ...record('A'), search_doc: { name: 'A' } }, + ), + 'missing field follows SQL null semantics', + ); + }); + + test('watch publication rolls back atomically and dirty state survives a new connection', async function (assert) { + let owner = { + realmURL, + ownerURL: `${realmURL}summary`, + generation: 1, + inputGeneration: 1, + definitionRevision: 'tessar-v1', + watches: await registry.prepare([ + { fieldPath: 'records', query: { filter: { on, eq: { name: 'A' } } } }, + ]), + }; + await assert.rejects( + db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + await registry.publish(tx!, owner); + throw new Error('Tessar injected rollback'); + }), + /injected rollback/, + ); + assert.deepEqual( + await registry.affected(realmURL, undefined, record('A')), + [], + 'rollback leaves no partial watch or terms', + ); + await db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + assert.true(await registry.publish(tx!, owner)); + }); + assert.deepEqual( + await registry.affected(realmURL, record('A'), record('B')), + [owner.ownerURL], + 'membership exit invalidates', + ); + await db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + await registry.markDirty(tx!, realmURL, [owner.ownerURL], 3); + }); + let { PgAdapter } = await import('@cardstack/postgres'); + let reopened = new PgAdapter(); + try { + let afterRestart = new TessarQueryRegistry(reopened, engine); + assert.deepEqual( + await afterRestart.pending(realmURL), + [{ ownerURL: owner.ownerURL, generation: 3 }], + 'dirty work is durable', + ); + await reopened.withWriteLock(`tessar:${realmURL}`, async (tx) => { + assert.false( + await afterRestart.publish(tx!, { + ...owner, + generation: 2, + inputGeneration: 2, + }), + 'racing stale publication is rejected', + ); + assert.true( + await afterRestart.publish(tx!, { + ...owner, + generation: 3, + inputGeneration: 3, + }), + ); + }); + assert.deepEqual(await afterRestart.pending(realmURL), []); + } finally { + await reopened.close(); + } + }); +}); diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index 07fb330b442..66bcc85b212 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -373,6 +373,122 @@ export class IndexQueryEngine { : this.#query(expression); } + // Tessar reverse-query verification uses the forward query compiler against + // one effective indexed document. No second JavaScript predicate evaluator: + // type ancestry, nulls, plural paths and reference normalization are shared. + // Sort/page never restrict invalidation membership. + async tessarMatchesDocument( + filter: Filter | undefined, + document: Pick & + Partial< + Pick< + BoxelIndexTable, + 'file_alias' | 'type' | 'last_modified' | 'resource_created_at' + > + >, + ): Promise { + assertTessarFilter(filter); + let json = (value: unknown): Expression => [ + dbExpression({ + pg: ['CAST(', param(value as any), 'AS JSONB)'], + sqlite: [param(value as any)], + }), + ]; + let expression: CardExpression = [ + 'SELECT 1 AS matched FROM (SELECT', + ...json(document.search_doc), + 'AS search_doc,', + ...json(document.types), + 'AS types,', + param(document.url), + 'AS url,', + param(document.file_alias ?? null), + 'AS file_alias,', + param(document.type ?? 'instance'), + 'AS type,', + param(document.last_modified ?? null), + 'AS last_modified,', + param(document.resource_created_at ?? null), + 'AS resource_created_at', + ') AS i', + tableValuedFunctionsPlaceholder, + 'WHERE', + ...(filter + ? this.filterCondition(filter, baseCardRef, 'positive') + : ['TRUE']), + 'LIMIT 1', + ]; + return (await this.#queryCards(expression)).length > 0; + } + + // A deliberately narrow routing optimization: direct string fields have an + // identical query/index encoding. Everything else remains in the broad + // bucket until an extractor can prove it will not miss a forward match. + async tessarRoutingTerms( + filter: Filter | undefined, + ): Promise | undefined> { + assertTessarFilter(filter); + let extract = async ( + part: Filter | undefined, + inheritedOn: CodeRef, + ): Promise | undefined> => { + if (!part) return undefined; + let on = 'on' in part && part.on ? part.on : inheritedOn; + if ('every' in part) { + for (let child of part.every) { + let terms = await extract(child, on); + if (terms) return terms; + } + } else if ('any' in part) { + let branches = await Promise.all( + part.any.map((child) => extract(child, on)), + ); + if (branches.every((branch) => branch !== undefined)) + return branches.flat() as Array<{ path: string; value: string }>; + } else if ('eq' in part || 'in' in part) { + let entries = + 'eq' in part + ? Object.entries(part.eq).map( + ([path, value]) => [path, [value]] as const, + ) + : Object.entries(part.in); + for (let [path, values] of entries) { + if ( + path.includes('.') || + isReferenceFilterField(path) || + !Array.isArray(values) || + values.length === 0 || + values.length > 64 || + path.length > 128 || + !values.every( + (value) => typeof value === 'string' && value.length <= 256, + ) + ) + continue; + let definition = await this.getDefinition(on); + let field = await getField(definition, path, this.#definitionLookup); + let key = internalKeyFor( + field.fieldOrCard, + undefined, + this.#virtualNetwork, + ); + if ( + field.type === 'contains' && + field.isPrimitive && + !field.serializerName && + [ + `${baseRealmRRI}string/default`, + `${baseRealmRRI}card-api/StringField`, + ].includes(key) + ) + return values.map((value) => ({ path, value: value as string })); + } + } + return undefined; + }; + return extract(filter, baseCardRef); + } + async getInstance( url: URL, opts?: GetEntryOptions, @@ -2468,6 +2584,17 @@ function assertNever(value: never) { return new Error(`should never happen ${value}`); } +function assertTessarFilter(filter: Filter | undefined): void { + if (!filter) return; + if ('matches' in filter) + throw new InvalidQueryError( + 'Tessar materialization does not support full-text predicates', + ); + if ('every' in filter) filter.every.forEach(assertTessarFilter); + if ('any' in filter) filter.any.forEach(assertTessarFilter); + if ('not' in filter) assertTessarFilter(filter.not); +} + type FilterFieldHandler = ( definition: Definition, expression: T, diff --git a/packages/runtime-common/tessar-query-registry.ts b/packages/runtime-common/tessar-query-registry.ts new file mode 100644 index 00000000000..302a84028c9 --- /dev/null +++ b/packages/runtime-common/tessar-query-registry.ts @@ -0,0 +1,266 @@ +import type { DBAdapter } from './db.ts'; +import type { IndexQueryEngine } from './index-query-engine.ts'; +import type { Query } from './query.ts'; +import { + any, + every, + param, + query, + type Expression, + type Querier, +} from './expression.ts'; + +export interface TessarWatch { + fieldPath: string; + query: Query; +} + +export interface TessarPreparedWatch extends TessarWatch { + terms: Array<{ path: string; value: string }>; +} + +export type TessarDocument = Parameters< + IndexQueryEngine['tessarMatchesDocument'] +>[1]; + +// All mutations take the caller's pinned transaction. Index publication and +// watch replacement must commit together. A notification is never the queue: +// dirty_generation survives disconnects and worker restarts. +export class TessarQueryRegistry { + private db: DBAdapter; + private engine: IndexQueryEngine; + constructor(db: DBAdapter, engine: IndexQueryEngine) { + this.db = db; + this.engine = engine; + } + + async prepare(watches: TessarWatch[]): Promise { + let paths = new Set(); + let prepared: TessarPreparedWatch[] = []; + for (let watch of watches) { + if (!watch.fieldPath || paths.has(watch.fieldPath)) { + throw new Error('Tessar watch paths must be nonempty and unique'); + } + paths.add(watch.fieldPath); + // Page and sort affect the output, but never narrow invalidation. Even a + // non-returned match can change an aggregate or displace a page member. + let terms = await this.engine.tessarRoutingTerms(watch.query.filter); + prepared.push({ + ...watch, + terms: terms?.length ? terms : [{ path: '', value: '' }], + }); + } + return prepared; + } + + async publish( + tx: Querier, + owner: { + realmURL: string; + ownerURL: string; + generation: number; + inputGeneration: number; + definitionRevision: string; + watches: TessarPreparedWatch[]; + retired?: boolean; + }, + ): Promise { + let { realmURL, ownerURL, generation, inputGeneration } = owner; + if ( + !Number.isSafeInteger(generation) || + generation < 1 || + !Number.isSafeInteger(inputGeneration) || + inputGeneration < 0 || + inputGeneration > generation + ) { + throw new Error( + 'Tessar publication needs valid input and owner generations', + ); + } + for (let watch of owner.watches) { + let realms = + watch.query.realms ?? + (watch.query.realm ? [watch.query.realm] : [realmURL]); + if (realms.length !== 1 || realms[0] !== realmURL) { + throw new Error('Tessar watches currently require the owner realm'); + } + } + let [previous] = await tx([ + 'SELECT published_generation, dirty_generation FROM tessar_owners WHERE', + ...this.ownerCondition(realmURL, ownerURL), + ]); + if ( + (previous && Number(previous.published_generation) > generation) || + (previous?.dirty_generation != null && + Number(previous.dirty_generation) > inputGeneration) + ) { + return false; + } + await tx([ + `INSERT INTO tessar_owners + (realm_url, owner_url, published_generation, input_generation, + dirty_generation, definition_revision, retired) VALUES (`, + param(realmURL), + ',', + param(ownerURL), + ',', + param(generation), + ',', + param(inputGeneration), + ', NULL,', + param(owner.definitionRevision), + ',', + param(owner.retired ?? false), + `) ON CONFLICT (realm_url, owner_url) DO UPDATE SET + published_generation = EXCLUDED.published_generation, + input_generation = EXCLUDED.input_generation, dirty_generation = NULL, + definition_revision = EXCLUDED.definition_revision, + retired = EXCLUDED.retired`, + ]); + for (let table of ['tessar_query_terms', 'tessar_query_watches']) { + await tx([ + `DELETE FROM ${table} WHERE`, + ...this.ownerCondition(realmURL, ownerURL), + ]); + } + if (owner.retired) return true; + for (let watch of owner.watches) { + await tx([ + `INSERT INTO tessar_query_watches + (realm_url, owner_url, field_path, query) VALUES (`, + param(realmURL), + ',', + param(ownerURL), + ',', + param(watch.fieldPath), + ',', + param(JSON.stringify(watch.query)), + ')', + ]); + let seen = new Set(); + for (let term of watch.terms) { + let key = JSON.stringify([term.path, term.value]); + if (seen.has(key)) continue; + seen.add(key); + await tx([ + `INSERT INTO tessar_query_terms + (realm_url, owner_url, field_path, path, value) VALUES (`, + param(realmURL), + ',', + param(ownerURL), + ',', + param(watch.fieldPath), + ',', + param(term.path), + ',', + param(term.value), + ')', + ]); + } + } + return true; + } + + async candidates(realmURL: string, document: TessarDocument) { + let routes = [ + every([ + ['t.path =', param('')], + ['t.value =', param('')], + ]), + ]; + for (let [path, value] of Object.entries(document.search_doc ?? {})) { + if (typeof value !== 'string') continue; + routes.push( + every([ + ['t.path =', param(path)], + ['t.value =', param(value)], + ]), + ); + } + let rows = await query( + this.db, + [ + `SELECT DISTINCT w.owner_url, w.field_path, w.query + FROM tessar_query_terms t JOIN tessar_query_watches w + ON w.realm_url = t.realm_url AND w.owner_url = t.owner_url + AND w.field_path = t.field_path + WHERE`, + ...(every([ + ['t.realm_url =', param(realmURL)], + any(routes), + ]) as Expression), + ], + { query: 'JSON' }, + ); + return rows.map((row) => ({ + ownerURL: row.owner_url as string, + fieldPath: row.field_path as string, + query: row.query as unknown as Query, + })); + } + + async affected( + realmURL: string, + oldDocument: TessarDocument | undefined, + newDocument: TessarDocument | undefined, + ): Promise { + let owners = new Set(); + for (let document of [oldDocument, newDocument]) { + if (!document) continue; + for (let watch of await this.candidates(realmURL, document)) { + if (owners.has(watch.ownerURL)) continue; + if ( + await this.engine.tessarMatchesDocument(watch.query.filter, document) + ) { + owners.add(watch.ownerURL); + } + } + } + return [...owners].sort(); + } + + async markDirty( + tx: Querier, + realmURL: string, + owners: string[], + generation: number, + ) { + for (let ownerURL of new Set(owners)) { + await tx([ + `UPDATE tessar_owners SET dirty_generation = CASE + WHEN dirty_generation IS NULL OR dirty_generation <`, + param(generation), + 'THEN', + param(generation), + 'ELSE dirty_generation END WHERE', + ...(every([ + this.ownerCondition(realmURL, ownerURL), + ['retired = FALSE'], + ['input_generation <', param(generation)], + ]) as Expression), + ]); + } + } + + async pending( + realmURL: string, + ): Promise> { + let rows = await query(this.db, [ + `SELECT owner_url, dirty_generation FROM tessar_owners + WHERE realm_url =`, + param(realmURL), + 'AND retired = FALSE AND dirty_generation IS NOT NULL ORDER BY owner_url', + ]); + return rows.map((row) => ({ + ownerURL: row.owner_url as string, + generation: Number(row.dirty_generation), + })); + } + + private ownerCondition(realmURL: string, ownerURL: string) { + return every([ + ['realm_url =', param(realmURL)], + ['owner_url =', param(ownerURL)], + ]) as Expression; + } +} From c1bfcc2ff1841d7d80625278456d491bf4a2ca2c Mon Sep 17 00:00:00 2001 From: Chris Tse <2302191+christse@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:35:23 -0400 Subject: [PATCH 09/12] Connect Tessar materialization publication and client freshness --- docs/tessar-benchmark-results.json | 1359 +++++++++++++++++ docs/tessar-performance-report.md | 156 ++ docs/tessar-query-materialization-plan.md | 76 +- packages/base/card-api.gts | 181 ++- packages/base/field-support.ts | 21 + packages/base/query-field-support.ts | 30 + packages/host/app/lib/gc-card-store.ts | 19 +- .../host/app/lib/prerender-fetch-headers.ts | 14 +- packages/host/app/routes/render.ts | 30 +- packages/host/app/routes/render/meta.ts | 21 +- packages/host/app/services/matrix-service.ts | 2 + packages/host/app/services/message-service.ts | 31 + packages/host/app/services/store.ts | 67 +- .../components/tessar-snapshot-test.gts | 179 ++- .../realm-server/handlers/handle-search.ts | 46 +- .../realm-server/prerender/prerender-app.ts | 19 + .../realm-server/prerender/prerenderer.ts | 3 + .../prerender/remote-prerenderer.ts | 2 + .../realm-server/prerender/render-runner.ts | 8 + packages/realm-server/server.ts | 2 +- .../tests/tessar-query-registry-test.ts | 245 ++- packages/runtime-common/definition-lookup.ts | 6 +- packages/runtime-common/document.ts | 33 +- packages/runtime-common/index-runner.ts | 110 +- .../runtime-common/index-runner/visit-file.ts | 3 + packages/runtime-common/index-writer.ts | 70 +- packages/runtime-common/index.ts | 6 + .../realm-index-query-engine.ts | 49 +- packages/runtime-common/realm.ts | 70 +- packages/runtime-common/resource-types.ts | 2 + packages/runtime-common/search-utils.ts | 1 + .../tessar-index-publication.ts | 227 +++ .../runtime-common/tessar-materialization.ts | 159 ++ .../runtime-common/tessar-query-registry.ts | 50 +- packages/runtime-common/worker.ts | 6 + scripts/tessar/README.md | 43 +- scripts/tessar/bench.mjs | 16 +- scripts/tessar/browser-check.mjs | 375 +++++ scripts/tessar/browser-oracle.mjs | 71 + scripts/tessar/generate.mjs | 20 +- scripts/tessar/generate.test.mjs | 18 + scripts/tessar/lifecycle-check.mjs | 149 ++ scripts/tessar/load-check.mjs | 288 ++++ scripts/tessar/realm/feeder-chain.gts | 60 + scripts/tessar/realm/get-cards.gts | 233 +++ scripts/tessar/realm/tessar.gts | 76 +- scripts/tessar/sample-processes.mjs | 69 + scripts/tessar/switch-variant.mjs | 113 ++ 48 files changed, 4688 insertions(+), 146 deletions(-) create mode 100644 docs/tessar-benchmark-results.json create mode 100644 docs/tessar-performance-report.md create mode 100644 packages/runtime-common/tessar-index-publication.ts create mode 100644 packages/runtime-common/tessar-materialization.ts create mode 100644 scripts/tessar/browser-check.mjs create mode 100644 scripts/tessar/browser-oracle.mjs create mode 100644 scripts/tessar/lifecycle-check.mjs create mode 100644 scripts/tessar/load-check.mjs create mode 100644 scripts/tessar/realm/feeder-chain.gts create mode 100644 scripts/tessar/realm/get-cards.gts create mode 100644 scripts/tessar/sample-processes.mjs create mode 100644 scripts/tessar/switch-variant.mjs diff --git a/docs/tessar-benchmark-results.json b/docs/tessar-benchmark-results.json new file mode 100644 index 00000000000..a1e230f9e3f --- /dev/null +++ b/docs/tessar-benchmark-results.json @@ -0,0 +1,1359 @@ +{ + "version": 1, + "status": "in-progress", + "candidate": { + "commit": "fead7a13ab02154a79e743877798166765959fbe", + "runtimeHash": "cacabe30049993e6a84d2f25b33bf5e130d27d3f3752dac760efdb8573de56cd", + "hostHash": "78492c253fcf49c641cd1022de72fa503796d1d23f7ab383116e7c161bfce09f" + }, + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "smoke", + "shape": "distributed", + "instanceCount": 34, + "counts": { + "Classroom": 2, + "Student": 4, + "Staff": 4, + "Slot": 4, + "Observation": 4, + "Report": 4, + "Activity": 4, + "Reference": 4, + "DaySummary": 4 + }, + "recordsSha256": "cbbee835e1b22fc96222fc1019fdba48380433ec8cf8c3eec1d44f2a811cc8d2" + }, + "startupMs": 83182.196375, + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 24.365875000003143, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 8.401167000003625, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 10.184207999991486, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 8.670916999995825, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 8.254000000000815, + "responseBytes": 7164, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 538.87025, + "inputRequests": [], + "fetchRequests": 101, + "metrics": { + "Nodes": 1469, + "ScriptDuration": 0.08405, + "TaskDuration": 0.378386, + "JSHeapUsedSize": 75435092 + } + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 428.66262500000016, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 2752, + "ScriptDuration": 0.119587, + "TaskDuration": 0.629178, + "JSHeapUsedSize": 97491200 + } + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 467.7433329999999, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.157086, + "TaskDuration": 0.926614, + "JSHeapUsedSize": 61492352 + } + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 443.6385829999999, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 4043, + "ScriptDuration": 0.190751, + "TaskDuration": 1.172988, + "JSHeapUsedSize": 118185404 + } + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 494.621083, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.226318, + "TaskDuration": 1.4741, + "JSHeapUsedSize": 61486640 + } + } + ], + "mutations": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 826, + "status": 200, + "ownerRevision": 4, + "ackToDisplayMs": 16, + "inputRequests": [], + "pendingNotificationMs": -662, + "transitions": [ + { + "sinceWriteMs": 118, + "state": "ready", + "visibleStats": 6 + }, + { + "sinceWriteMs": 164, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 813, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 1247, + "status": 200, + "ownerRevision": 6, + "ackToDisplayMs": 121, + "inputRequests": [], + "pendingNotificationMs": -1107, + "transitions": [ + { + "sinceWriteMs": 140, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1363, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "query exit", + "valid": true, + "ackMs": 1553, + "status": 200, + "ownerRevision": 8, + "ackToDisplayMs": 119, + "inputRequests": [], + "pendingNotificationMs": -1412, + "transitions": [ + { + "sinceWriteMs": 141, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1668, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "query entry", + "valid": true, + "ackMs": 893, + "status": 200, + "ownerRevision": 10, + "ackToDisplayMs": 115, + "inputRequests": [], + "pendingNotificationMs": -747, + "transitions": [ + { + "sinceWriteMs": 146, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1003, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 15, + "status": 204, + "ownerRevision": 12, + "ackToDisplayMs": 887, + "inputRequests": [], + "pendingNotificationMs": 124, + "transitions": [ + { + "sinceWriteMs": 139, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 894, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 950, + "status": 200, + "ownerRevision": 12, + "ackToDisplayMs": 123, + "inputRequests": [], + "pendingNotificationMs": -807, + "transitions": [ + { + "sinceWriteMs": 143, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1066, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 15, + "status": 204, + "ownerRevision": 16, + "ackToDisplayMs": 1289, + "inputRequests": [], + "pendingNotificationMs": 125, + "transitions": [ + { + "sinceWriteMs": 140, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1301, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 471, + "status": 204, + "ownerRevision": 18, + "ackToDisplayMs": 137, + "inputRequests": [], + "pendingNotificationMs": -309, + "transitions": [ + { + "sinceWriteMs": 162, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 595, + "state": "ready", + "visibleStats": 6 + } + ] + } + ], + "browserErrors": [], + "trace": { + "lcpMs": 1878, + "ttfbMs": 31, + "cls": 0, + "cpuThrottling": 1, + "networkThrottling": "none", + "note": "One DevTools trace; separate from headless readiness measurements. No base/candidate speedup established." + }, + "exploratory1x": { + "storage": "PostgreSQL 16.3 on 3.9 GiB tmpfs; excluded from disk-backed comparisons", + "candidate": { + "commit": "fead7a13ab02154a79e743877798166765959fbe", + "runtimeHash": "cacabe30049993e6a84d2f25b33bf5e130d27d3f3752dac760efdb8573de56cd", + "hostHash": "78492c253fcf49c641cd1022de72fa503796d1d23f7ab383116e7c161bfce09f" + }, + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "1x", + "shape": "distributed", + "instanceCount": 1255, + "counts": { + "Classroom": 2, + "Student": 13, + "Staff": 26, + "Slot": 173, + "Observation": 164, + "Report": 44, + "Activity": 63, + "Reference": 766, + "DaySummary": 4 + }, + "recordsSha256": "60ecddc1bd7b8203f1cd0ffda583325cd0b33ed8552734bc79bed99f9e671b14" + }, + "startupMs": 334014.01620799996, + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 28.61095900001237, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 11.054750000010245, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 9.332875000021886, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 10.277624999987893, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 9.901791999989655, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 5, + "elapsedMs": 9.857124999980442, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 6, + "elapsedMs": 11.62100000004284, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 7, + "elapsedMs": 9.131791999971028, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 8, + "elapsedMs": 8.908166000037454, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 9, + "elapsedMs": 11.87562499998603, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 10, + "elapsedMs": 8.843791999970563, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 11, + "elapsedMs": 8.613874999980908, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 12, + "elapsedMs": 8.231292000040412, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 13, + "elapsedMs": 10.982457999954931, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 14, + "elapsedMs": 8.227375000016764, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 15, + "elapsedMs": 6.772209000017028, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 16, + "elapsedMs": 8.223916999995708, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 17, + "elapsedMs": 8.872124999994412, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 18, + "elapsedMs": 9.011999999987893, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 19, + "elapsedMs": 9.746916999982204, + "responseBytes": 29698, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 582.6893750000002, + "fetchRequests": 98, + "metrics": { + "Nodes": 1471, + "ScriptDuration": 0.087699, + "TaskDuration": 0.384828, + "JSHeapUsedSize": 74252456 + }, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 464.07220900000016, + "fetchRequests": 103, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.127607, + "TaskDuration": 0.699661, + "JSHeapUsedSize": 60268616 + }, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 434.560833, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.16159, + "TaskDuration": 0.951681, + "JSHeapUsedSize": 118875236 + }, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 538.8636660000002, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.198748, + "TaskDuration": 1.274711, + "JSHeapUsedSize": 62192904 + }, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 450.24112499999956, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.232656, + "TaskDuration": 1.526551, + "JSHeapUsedSize": 120423820 + }, + "inputRequestCount": 0 + }, + { + "iteration": 5, + "cache": "warm-reload", + "valid": true, + "displayMs": 493.1064580000002, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.26902, + "TaskDuration": 1.842421, + "JSHeapUsedSize": 62418356 + }, + "inputRequestCount": 0 + }, + { + "iteration": 6, + "cache": "warm-reload", + "valid": true, + "displayMs": 453.88287500000024, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.305986, + "TaskDuration": 2.096136, + "JSHeapUsedSize": 120947520 + }, + "inputRequestCount": 0 + }, + { + "iteration": 7, + "cache": "warm-reload", + "valid": true, + "displayMs": 490.76133299999947, + "fetchRequests": 99, + "metrics": { + "Nodes": 8772, + "ScriptDuration": 0.341787, + "TaskDuration": 2.381155, + "JSHeapUsedSize": 158195248 + }, + "inputRequestCount": 0 + }, + { + "iteration": 8, + "cache": "warm-reload", + "valid": true, + "displayMs": 531.8857500000004, + "fetchRequests": 99, + "metrics": { + "Nodes": 11308, + "ScriptDuration": 0.380472, + "TaskDuration": 2.704089, + "JSHeapUsedSize": 196422944 + }, + "inputRequestCount": 0 + }, + { + "iteration": 9, + "cache": "warm-reload", + "valid": true, + "displayMs": 537.3765000000003, + "fetchRequests": 99, + "metrics": { + "Nodes": 7464, + "ScriptDuration": 0.417063, + "TaskDuration": 3.08325, + "JSHeapUsedSize": 96883632 + }, + "inputRequestCount": 0 + }, + { + "iteration": 10, + "cache": "warm-reload", + "valid": true, + "displayMs": 482.1635839999999, + "fetchRequests": 99, + "metrics": { + "Nodes": 1850, + "ScriptDuration": 0.452137, + "TaskDuration": 3.392069, + "JSHeapUsedSize": 44846444 + }, + "inputRequestCount": 0 + }, + { + "iteration": 11, + "cache": "warm-reload", + "valid": true, + "displayMs": 442.59000000000015, + "fetchRequests": 99, + "metrics": { + "Nodes": 4386, + "ScriptDuration": 0.489469, + "TaskDuration": 3.639918, + "JSHeapUsedSize": 101884296 + }, + "inputRequestCount": 0 + }, + { + "iteration": 12, + "cache": "warm-reload", + "valid": true, + "displayMs": 482.96099999999933, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.526201, + "TaskDuration": 3.938568, + "JSHeapUsedSize": 61734792 + }, + "inputRequestCount": 0 + }, + { + "iteration": 13, + "cache": "warm-reload", + "valid": true, + "displayMs": 464.44058399999994, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.560771, + "TaskDuration": 4.203367, + "JSHeapUsedSize": 120747992 + }, + "inputRequestCount": 0 + }, + { + "iteration": 14, + "cache": "warm-reload", + "valid": true, + "displayMs": 493.4795830000003, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.600257, + "TaskDuration": 4.524254, + "JSHeapUsedSize": 61717764 + }, + "inputRequestCount": 0 + }, + { + "iteration": 15, + "cache": "warm-reload", + "valid": true, + "displayMs": 462.02329199999986, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.635034, + "TaskDuration": 4.788153, + "JSHeapUsedSize": 120212712 + }, + "inputRequestCount": 0 + }, + { + "iteration": 16, + "cache": "warm-reload", + "valid": true, + "displayMs": 502.05570799999987, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.672395, + "TaskDuration": 5.111127, + "JSHeapUsedSize": 61703224 + }, + "inputRequestCount": 0 + }, + { + "iteration": 17, + "cache": "warm-reload", + "valid": true, + "displayMs": 467.5910410000015, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.707947, + "TaskDuration": 5.372618, + "JSHeapUsedSize": 120490356 + }, + "inputRequestCount": 0 + }, + { + "iteration": 18, + "cache": "warm-reload", + "valid": true, + "displayMs": 514.6342499999992, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.746014, + "TaskDuration": 5.698184, + "JSHeapUsedSize": 61710756 + }, + "inputRequestCount": 0 + }, + { + "iteration": 19, + "cache": "warm-reload", + "valid": true, + "displayMs": 476.0184999999983, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.782276, + "TaskDuration": 5.970163, + "JSHeapUsedSize": 120202736 + }, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 5401, + "status": 200, + "ownerRevision": 4, + "ackToDisplayMs": 38, + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 2771, + "status": 200, + "ownerRevision": 6, + "ackToDisplayMs": 150, + "inputRequestCount": 0 + }, + { + "name": "query exit", + "valid": true, + "ackMs": 3016, + "status": 200, + "ownerRevision": 8, + "ackToDisplayMs": 120, + "inputRequestCount": 0 + }, + { + "name": "query entry", + "valid": true, + "ackMs": 2423, + "status": 200, + "ownerRevision": 10, + "ackToDisplayMs": 119, + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 16, + "status": 204, + "ownerRevision": 12, + "ackToDisplayMs": 2002, + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 907, + "status": 200, + "ownerRevision": 12, + "ackToDisplayMs": 131, + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 15, + "status": 204, + "ownerRevision": 15, + "ackToDisplayMs": 1092, + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 1521, + "status": 204, + "ownerRevision": 17, + "ackToDisplayMs": 147, + "inputRequestCount": 0 + } + ], + "pageErrorCount": 0 + }, + "failedSetupRuns": [ + { + "preset": "10x", + "instanceCount": 12550, + "storage": "3.9 GiB tmpfs", + "valid": false, + "reason": "PostgreSQL SQLSTATE 53100: no space left on device", + "sourceFilesVisitedApproximately": 10037 + }, + { + "preset": "10x", + "instanceCount": 12550, + "storage": "disk-backed Docker volume", + "valid": false, + "reason": "Harness 600000 ms full-index startup timeout", + "sourceFilesVisitedApproximately": 11278 + } + ], + "smokeV5": { + "candidate": { + "commit": "fead7a13ab02154a79e743877798166765959fbe", + "runtimeHash": "23dd3be3f52df4414a9e29cc71fad5d2a5ce249800fd8eb83614cbce37404895", + "hostHash": "da5c453fe08e2bdfdf11cd201677aac8eeb2b8bf09f20cf5a8d273e0f2bfc26e" + }, + "storage": "disk-backed PostgreSQL 16.3 Docker volume", + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "smoke", + "shape": "distributed", + "variant": "materialized", + "instanceCount": 34, + "counts": { + "Classroom": 2, + "Student": 4, + "Staff": 4, + "Slot": 4, + "Observation": 4, + "Report": 4, + "Activity": 4, + "Reference": 4, + "DaySummary": 4 + }, + "recordsSha256": "cbbee835e1b22fc96222fc1019fdba48380433ec8cf8c3eec1d44f2a811cc8d2" + }, + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 23.093459000010625, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 10.064541999992798, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 11.75020900000527, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 8.958333000002312, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 8.91874999999709, + "responseBytes": 7164, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 550.5600419999998, + "fetchRequests": 98, + "metrics": { + "Nodes": 1204, + "ScriptDuration": 0.08798, + "TaskDuration": 0.38672700000000004, + "JSHeapUsedSize": 73473880 + }, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 446.2209999999998, + "fetchRequests": 103, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.03707700000000001, + "TaskDuration": 0.28576199999999996, + "JSHeapUsedSize": 59505292 + }, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 445.364834, + "fetchRequests": 99, + "metrics": { + "Nodes": 4043, + "ScriptDuration": 0.033658999999999994, + "TaskDuration": 0.2414559999999999, + "JSHeapUsedSize": 116162848 + }, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 476.3268330000001, + "fetchRequests": 99, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.03739200000000001, + "TaskDuration": 0.31247899999999995, + "JSHeapUsedSize": 61460216 + }, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 440.77620799999977, + "fetchRequests": 99, + "metrics": { + "Nodes": 4043, + "ScriptDuration": 0.033944, + "TaskDuration": 0.247892, + "JSHeapUsedSize": 118292476 + }, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 851, + "status": 200, + "ownerRevision": 4, + "ackToDisplayMs": 14, + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 1262, + "status": 200, + "ownerRevision": 6, + "ackToDisplayMs": 128, + "inputRequestCount": 0 + }, + { + "name": "query exit", + "valid": true, + "ackMs": 1566, + "status": 200, + "ownerRevision": 8, + "ackToDisplayMs": 121, + "inputRequestCount": 0 + }, + { + "name": "query entry", + "valid": true, + "ackMs": 887, + "status": 200, + "ownerRevision": 10, + "ackToDisplayMs": 121, + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 21, + "status": 204, + "ownerRevision": 12, + "ackToDisplayMs": 852, + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 952, + "status": 200, + "ownerRevision": 12, + "ackToDisplayMs": 120, + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 13, + "status": 204, + "ownerRevision": 16, + "ackToDisplayMs": 1309, + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 483, + "status": 204, + "ownerRevision": 18, + "ackToDisplayMs": 142, + "inputRequestCount": 0 + }, + { + "name": "reconnect after missed notification", + "valid": true, + "ackMs": 32, + "status": 204, + "ownerRevision": 20, + "ackToDisplayMs": 5159, + "inputRequestCount": 0 + } + ], + "correctedReference": { + "note": "Single read and nine mutation validity checks; not a controlled performance comparison", + "reads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 1333.212375, + "fetchRequests": 105, + "metrics": { + "Nodes": 1357, + "ScriptDuration": 0.078716, + "TaskDuration": 0.392484, + "JSHeapUsedSize": 46309572 + }, + "inputRequestCount": 5 + } + ], + "writes": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 341, + "status": 200, + "ackToDisplayMs": 254, + "inputRequestCount": 9 + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 492, + "status": 200, + "ackToDisplayMs": 264, + "inputRequestCount": 12 + }, + { + "name": "query exit", + "valid": true, + "ackMs": 878, + "status": 200, + "ackToDisplayMs": 258, + "inputRequestCount": 9 + }, + { + "name": "query entry", + "valid": true, + "ackMs": 243, + "status": 200, + "ackToDisplayMs": 296, + "inputRequestCount": 9 + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 10, + "status": 204, + "ackToDisplayMs": 496, + "inputRequestCount": 9 + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 428, + "status": 200, + "ackToDisplayMs": 1, + "inputRequestCount": 4 + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 13, + "status": 204, + "ackToDisplayMs": 1262, + "inputRequestCount": 12 + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 77, + "status": 204, + "ackToDisplayMs": 282, + "inputRequestCount": 6 + }, + { + "name": "reconnect after missed notification", + "valid": true, + "ackMs": 12, + "status": 204, + "ackToDisplayMs": 5309, + "inputRequestCount": 10 + } + ], + "pageErrorCount": 0 + }, + "feederChecks": [ + { + "name": "unrelated module epoch", + "valid": true, + "ackToReadyMs": 868 + }, + { + "name": "five downstream feeder stages", + "valid": true, + "ackMs": 14, + "ackToReadyMs": 3908, + "snapshots": [ + { + "id": "DaySummary/00000", + "revision": 70, + "scoreTotal": 88 + }, + { + "id": "TessarStage0/00000", + "revision": 71, + "scoreTotal": 88 + }, + { + "id": "TessarStage1/00000", + "revision": 72, + "scoreTotal": 88 + }, + { + "id": "TessarStage2/00000", + "revision": 73, + "scoreTotal": 88 + }, + { + "id": "TessarStage3/00000", + "revision": 74, + "scoreTotal": 88 + }, + { + "id": "TessarStage4/00000", + "revision": 75, + "scoreTotal": 88 + } + ] + } + ], + "concurrencyChecks": { + "note": "Correctness smoke only. Other regression checks overlapped part of the run. HTTP readers plus one browser, not 50 browser CPUs.", + "cases": [ + { + "readers": 1, + "valid": true, + "readyReads": 874, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "valid": true, + "ackMs": 22, + "ackToDisplayMs": 1145, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 12, + "ackToDisplayMs": 638, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 20, + "ackToDisplayMs": 664, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 13, + "ackToDisplayMs": 654, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 12, + "ackToDisplayMs": 652, + "inputRequestCount": 0 + } + ] + }, + { + "readers": 10, + "valid": true, + "readyReads": 3886, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "valid": true, + "ackMs": 33, + "ackToDisplayMs": 714, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 22, + "ackToDisplayMs": 712, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 20, + "ackToDisplayMs": 732, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 27, + "ackToDisplayMs": 788, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 23, + "ackToDisplayMs": 761, + "inputRequestCount": 0 + } + ] + }, + { + "readers": 50, + "valid": true, + "readyReads": 4192, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "valid": true, + "ackMs": 87, + "ackToDisplayMs": 863, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 119, + "ackToDisplayMs": 864, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 103, + "ackToDisplayMs": 816, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 113, + "ackToDisplayMs": 819, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 100, + "ackToDisplayMs": 782, + "inputRequestCount": 0 + } + ] + } + ] + } + } +} diff --git a/docs/tessar-performance-report.md b/docs/tessar-performance-report.md new file mode 100644 index 00000000000..39cacf1ff69 --- /dev/null +++ b/docs/tessar-performance-report.md @@ -0,0 +1,156 @@ +# Tessar performance report — in progress, DO NOT MERGE + +This is an implementation checkpoint, not a completed performance comparison. +The smoke and 1× candidate runs pass the tested browser and mutation cases. +The first 10× attempt exhausted the test database's temporary filesystem; it is +being rerun on a separate disk-backed database. Controlled comparisons, +concurrency trials and full dashboard adaptation remain outstanding. No speedup +or additional AI-generation capacity has been established. + +## Contract + +Only correct, complete snapshots qualify. An acknowledged relevant write must +be reflected in the already-open and newly connected client within 10,000 ms. +Pending or failed work must not be presented as current output. Report write +acknowledgement latency separately from acknowledgement-to-display delay. + +All fixtures are fabricated with seed 1729. The fixed sizes are smoke (34), +1× (1,255) and 10× (12,550). No production records are benchmark inputs. + +## Connected smoke checkpoint + +The candidate used the current uncommitted Tessar lifecycle implementation on +top of `fead7a13ab`. The harness records source, host-build and dataset hashes +in its raw result files. The latest smoke run includes queued-write checks and +an explicit pending state in the browser; sanitized observations are saved in +`tessar-benchmark-results.json`. + +Five headless Chromium loads compared all six displayed statistics and every +displayed row with the independent raw-document oracle. All passed. No input-card +fetches or query searches were observed. The displayed-readiness observations +were 539, 429, 468, 444 and 495 ms; the first used a new browser context and the +rest reloaded that context. These five samples are too few to characterize tails. + +| Synthetic mutation | Write acknowledgement | Acknowledgement to observed display | Result | +| ------------------------------- | --------------------: | ----------------------------------: | ------------------------ | +| Matching observation score | 826 ms | 16 ms | Correct | +| Transitive reference label | 1,247 ms | 121 ms | Correct | +| Observation leaves selected day | 1,553 ms | 119 ms | Correct | +| Observation enters selected day | 893 ms | 115 ms | Correct | +| Asynchronous source edit | 15 ms | 887 ms | Correct; pending shown | +| Unrelated reference edit | 950 ms | 123 ms | Owner revision unchanged | +| Matching insertion | 15 ms | 1,289 ms | Correct; pending shown | +| Matching deletion | 471 ms | 137 ms | Correct | + +The browser remained open through all eight writes. Each new total/row matched +its oracle without a reload or input-graph request. PATCH operations waited for +indexing before acknowledgement. Source POSTs acknowledged the queued write and +completed materialization later. The pending notification reached the open client +about 124–125 ms after those asynchronous acknowledgements; the raw results retain +that interval explicitly. This test proves bounded convergence and a pending UI, +not instantaneous notification delivery or linearizable reads of an already-open +screen. Cross-replica and reconnect cases remain to be validated. + +A separate Chrome DevTools trace measured LCP 1,878 ms, TTFB 31 ms and CLS 0, +with no CPU/network throttling. This traced headed-browser navigation is a +different measurement from the headless displayed-readiness samples. It does not +establish a base/candidate improvement. + +Focused checks: five Tessar browser tests / 68 assertions; four Postgres tests; +56 existing index-writer tests / 153 assertions; 15 existing computed tests / 41 +assertions; three generator/oracle tests. Package lint and Glint checks passed. + +## Failed cases retained + +- Ordinary query-field HTTP reads returned live membership with zero-valued + indexed statistics. They failed correctness and cannot be a speedup baseline. +- The first connected candidate response omitted the relationship's required + `links.self`, causing browser validation to fail despite correct numeric HTTP + results. The serializer was corrected and a wire-validation test was added. +- The first write trial deadlocked because index publication acquired the same + advisory lock held by the source write awaiting indexing. Publication now uses + a separate namespace, covered by a real Postgres lock regression test. +- The first 10× source-indexing pass failed at approximately 10,037 of 12,551 + files with PostgreSQL SQLSTATE 53100: its 3.9 GiB tmpfs was full. This is a + failed run, not a completed indexing-time or throughput observation. A separate + localhost-only PostgreSQL 16.3 container now uses a Docker disk-backed volume. + The synthetic schema and reusable 1× template were preserved before stopping + the disposable tmpfs container. Earlier tmpfs measurements are exploratory; + base/candidate comparisons must use the same database storage configuration. + +## Exploratory 1× checkpoint (tmpfs database) + +The 1,255-record run used the same source version as the smoke checkpoint. +All 20 HTTP responses and 20 real Chromium displays matched their raw-document +oracle. HTTP responses were 29,698 bytes: median 9.33 ms, p95 11.88 ms, range +6.77–28.61 ms. Browser display readiness was median 482.96 ms, p95 538.86 ms, +range 434.56–582.69 ms. There were zero observed input-card/search requests and +zero page errors. The first browser sample used a new context, followed by +19 warm reloads; these are a mixed exploratory distribution. + +All eight mutation cases passed in the already-open browser. Acknowledgement +to display ranged from 38 to 2,002 ms. The asynchronous edit acknowledged in +16 ms and appeared 2,002 ms later; insertion acknowledged in 15 ms and appeared +1,092 ms later. The unrelated reference edit left the selected owner revision +unchanged. These observations do not cover reconnect, restart or concurrent +write races. + +Harness startup took 334,014 ms, including base indexing and HTML prerendering. +The source indexing job reported 53,862 ms total, with 1,828 ms of Tessar +materialization for four owners in one wave. A separate HTML phase accounts for +much of the remaining startup time. Source and working index/HTML tables totalled +about 929 MiB (243 + 238 + 228 + 220 MiB); duplicated working rows are included. +This is a material storage and indexing cost, even though the dashboard read is +small. Do not interpret startup latency as Tessar computation alone. + +## Evidence still required + +1. Additional lifecycle tests: definition changes, feeder chains, races, + reconnects and worker restart recovery. +2. Valid getCards baseline and materialized display adaptation, with equivalent + output and freshness semantics. +3. Controlled 1× and 10× runs, including larger matching sets and unrelated + growth, repeated cold/warm browser reads and concurrent readers/writes. +4. CPU/memory, SQL/reverse-match work, source capture latency, backlog and + indexing amplification. Source indexing and Tessar recomputation now have + distinct timing fields in indexing job results. +5. Sanitized reproducible results, latency distributions and exportable charts. + +Run commands and fixture details are in `scripts/tessar/README.md`. Large raw +logs, traces and generated datasets stay outside Git. The draft PR remains +DO NOT MERGE throughout these experiments. + +## Latest lifecycle checkpoint (disk-backed database) + +The latest implementation serializes overlapping snapshot deserialization, +rejects backward owner revisions, and revalidates after offline/reconnect or +tab resumption. A failed indexing job returns an explicit failure rather than +serving the previous snapshot as current. Realm-wide definition epoch changes +schedule affected old stamps for recomputation, including owners outside the +changed module's concrete dependency closure. + +Five browser loads and nine mutations passed with no input-card/search requests. +The ninth mutation occurred while the browser was offline: it became visible +5,159 ms after acknowledgement, without a reload. A separate five-stage feeder +chain converged in 3,908 ms, with six successive published owner revisions. +The unrelated-module epoch case returned to ready in 868 ms. + +Correctness smoke checks passed for 1, 10 and 50 concurrent HTTP readers plus +one open browser. These are not 50 simultaneous browser processes, and other +regression checks overlapped part of the smoke run; throughput from this run is +not a controlled benchmark. The focused browser suite now passes 71 assertions. +The existing computed and sliding-sync regressions also pass. + +The first getCards reference failed the explicit-pending gate for asynchronous +writes. The corrected reference listens for realm publication/reconnect, +re-queries complete pages, and releases superseded resources. It passes one +browser read and the same nine mutation cases. Its first implementation used +an unavailable realm loader shim for `getOwner`; that attempt failed all five +loads and is excluded. The corrected reference uses the owner already supplied +by the host's getCards context. It must be identified as a corrected reference, +not the unmodified incumbent dashboard. + +The second 10× setup attempt exceeded the harness's 600,000 ms startup limit +at approximately 11,278 of 12,551 files. Storage was available and indexing +was still progressing. The next cold-build setup limit is 3,600,000 ms; the +acknowledged-write freshness gate remains 10,000 ms. diff --git a/docs/tessar-query-materialization-plan.md b/docs/tessar-query-materialization-plan.md index 87e2bc07c91..da0572bfb71 100644 --- a/docs/tessar-query-materialization-plan.md +++ b/docs/tessar-query-materialization-plan.md @@ -36,48 +36,37 @@ correctness or freshness violation is a failed case, not an accepted tradeoff. ## Current checkpoint -Implementation has started. The deterministic generator produces 1,255 / 12,550 -synthetic instances and a raw-document oracle. Its first two tests pass. The -initial 34-instance real-stack baseline returned correct live query membership -but stale zero-valued indexed statistics on all five GETs. Those reads fail the -correctness gate and are not valid speedup baselines. The internal snapshot -consumer and reverse-query registry have focused browser and Postgres tests. -Four browser tests pass 49 assertions; two Postgres tests cover SQL predicates, -transaction rollback, persistent dirty state and rejection of stale publication. -The existing computed-field regression suite passes 15 tests / 41 assertions. -The registry is not yet connected to index publication or worker scheduling, and -normal server/client reads do not yet opt into Tessar. These primitive tests do -not establish end-to-end freshness or a performance improvement. - -The initial freshness contract is a maximum 10,000 ms from an acknowledged -relevant write to its complete owner revision being visible in existing and new -clients, at every dataset size and concurrency. Reads during that interval must -explicitly identify pending work; they must never present older results as -current. Source acknowledgement, source index revision, owner publication and -client observation are separate measured events. Missing, mixed or regressing -revisions fail the gate regardless of elapsed time. - -- Main was updated to `e9a4b0a54a` on September 10, 2026. -- Work uses the isolated branch `codex/do-not-merge-query-materialization-poc`. -- The draft POC is [PR #6085](https://github.com/cardstack/boxel/pull/6085). -- A private staging fork contains 154 copied definition/support files, verified - against their prepared hashes. All 29 card types used by the source dataset - passed schema generation there. -- No production records were uploaded. The temporary source-data archive, - prepared JSON copies, import inventory and temporary source authentication - cache were removed. Only aggregate counts were retained for sizing. -- Runtime primitives, an additive registry migration and a synthetic generator - are in progress. No production or shared staging backend is changed. - -The staging fork runs the staging deployment's runtime. It does **not** run this -monorepo branch merely because its GTS files are copied there. Develop and measure -the runtime changes on an isolated local stack from the worktree, including its -own database and indexing workers. A branch-specific hosted environment is a -later option; changing a shared staging backend is outside this plan. - -Keep source-deployment writes out of the experiment. Before a synthetic UI trial, -replace any copied definition's deployment-specific defaults or external data -references with synthetic equivalents in the development environment. +Implementation is underway in the isolated local runtime. The deterministic +synthetic generator covers smoke (34), 1× (1,255), and 10× (12,550) records; +its three oracle/integrity tests pass. The first ordinary-query baseline +returned membership with stale zero-valued indexed statistics and failed. + +The registry is connected to atomic index publication, durable dirty owners, +and follow-up worker waves. Indexed responses automatically activate snapshot +consumption. The latest smoke check passed five Chromium reads and nine writes, +including entry/exit, transitive edits, insertion/deletion and offline reconnect, +with zero input-card/search requests. Five focused browser tests pass 71 +assertions; four Postgres tests cover SQL predicate parity, transaction rollback, +source/publication locking, queued and failed writes, stale publication and module +epoch changes. Existing computed tests pass 15 tests / 41 assertions, and the +sliding-sync regression passes one test / four assertions. + +A real five-stage feeder chain converged in 3.9 seconds after an asynchronous +source acknowledgement. Synthetic concurrency correctness checks passed at +1, 10 and 50 HTTP readers plus one already-open browser. These are smoke gates, +not a controlled throughput comparison or evidence for 50 browser CPUs. +The 1× exploratory candidate run passed 20 HTTP and browser samples and eight +mutations. Two 10× setup attempts failed (tmpfs capacity, then the harness's +600-second startup limit); the report retains both failures. The disk-backed +benchmark now allows a longer initial build while preserving the 10-second +write-freshness deadline. + +Three draft realm definitions in the private fork project schedule, observation +coverage and classroom-day rows using existing schedule composition. They have +no Glint errors in the new files; the pre-existing fork files report 830 errors. +They have not been deployed or validated in a full Classroom Central browser +flow. Worker restart/race coverage, the complete realm adaptation, controlled +base/candidate benchmarks and the final performance report remain outstanding. ## Scope and constraints @@ -471,5 +460,4 @@ list of remaining limitations. Only cases satisfying the non-negotiable correctness/freshness gates qualify as successful benchmark results. The PR remains draft and DO NOT MERGE even after its tests pass. Production promotion and any shared-runtime deployment require a -separate decision. This plan is the -stopping point. +separate decision. Implementation resumed on explicit user instruction. diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 953224bd9b9..f22977dbc1d 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -2,6 +2,11 @@ import type Modifier from 'ember-modifier'; import GlimmerComponent from '@glimmer/component'; import { isEqual } from 'lodash-es'; import { WatchedArray, rawArrayValues } from './watched-array'; +import { + currentTessarInputSnapshot, + tessarSnapshotFields, + type TessarMaterialization, +} from '@cardstack/runtime-common'; import { BoxelInput, BrokenLinkTemplate, @@ -119,6 +124,7 @@ import { peekQueryFieldSearchResource, queryFieldHasUnreachableRealms, resolveQueryFieldEagerly, + tessarQueryWatch, validateRelationshipQuery, } from './query-field-support'; import { isSavedInstance } from './-private'; @@ -189,8 +195,10 @@ import { getRelationshipMembershipState, getter, hasTessarQueryMembership, + hasTessarSnapshot, leaveTessarSnapshot, setTessarSnapshot, + tessarSnapshotState, registerRelationshipProbe, relationshipStateForEntry, readFieldLoadingSignal, @@ -239,6 +247,7 @@ import type { } from '@cardstack/runtime-common'; export const BULK_GENERATED_ITEM_COUNT = 3; +export { hasTessarSnapshot, markTessarPending } from './field-support'; interface CardOrFieldTypeIconSignature { Element: SVGSVGElement; @@ -3924,6 +3933,12 @@ export class CardInfoField extends FieldDef { } export class CardDef extends BaseDef { + // Tessar POC: explicit, inherited opt-in for an indexed summary definition. + // Normal views stay on their existing live-computation path. + static tessarMaterialized = false; + get tessarState() { + return tessarSnapshotState(this); + } readonly [localId]: string = uuidv4(); [isSavedInstance] = false; get [fieldsUntracked](): Record | undefined { @@ -5139,7 +5154,47 @@ async function _createFromSerialized( }); } -async function _updateFromSerialized({ +const tessarDeserializations = initSharedState( + 'tessarDeserializations', + () => new WeakMap>(), +); + +async function _updateFromSerialized(args: { + instance: BaseInstanceType; + resource: LooseCardResource; + doc: LooseSingleCardDocument | CardDocument; + store: CardStore; + opts?: DeserializeOpts; +}): Promise> { + let { instance, resource } = args; + if (!resource.meta.tessar) return _applySerialized(args); + // Serialize a snapshot's contained-field assembly too: rejecting an old + // root only after await is too late if it already mutated contained values. + let previous = tessarDeserializations.get(instance); + let update = (previous ?? Promise.resolve()) + .catch(() => undefined) + .then(async () => { + let accepted = (instance as any)[meta]?.tessar?.publishedGeneration; + let incoming = resource.meta.tessar?.publishedGeneration; + if ( + hasTessarSnapshot(instance) && + Number.isSafeInteger(accepted) && + Number.isSafeInteger(incoming) && + incoming! < accepted + ) + return instance; + return _applySerialized(args); + }); + tessarDeserializations.set(instance, update); + try { + return await update; + } finally { + if (tessarDeserializations.get(instance) === update) + tessarDeserializations.delete(instance); + } +} + +async function _applySerialized({ instance, resource, doc, @@ -5152,6 +5207,14 @@ async function _updateFromSerialized({ store: CardStore; opts?: DeserializeOpts; }): Promise> { + let inPrerender = (globalThis as any).__boxelRenderContext === true; + if ( + !opts?.tessarSnapshot && + resource.meta.tessar && + (!inPrerender || currentTessarInputSnapshot()) + ) { + opts = { ...opts, tessarSnapshot: tessarSnapshotFields(resource) }; + } if (opts?.tessarSnapshot) { for (let path of opts.tessarSnapshot.computedFields) { let name = path.split('.')[0]; @@ -5921,6 +5984,122 @@ declare module 'ember-provide-consume-context/context-registry' { } } +// Pull a materialized owner's declared query inputs before collecting its +// computed output. Kept explicit so ordinary cards retain their lazy policy. +export async function prepareTessarQueries(instance: CardDef): Promise { + if (!(instance.constructor as typeof CardDef).tessarMaterialized) return; + for (let field of Object.values( + getFields(instance, { includeComputeds: true }), + )) { + if (field.queryDefinition) { + if (field.fieldType !== 'linksToMany') { + throw new Error( + 'Tessar currently materializes complete linksToMany queries', + ); + } + tessarQueryWatch(getStore(instance), instance, field); + ensureQueryFieldSearchResource(getStore(instance), instance, field); + } + } + await getStore(instance).loaded(); +} + +// serializeCard continues to omit query fields, so it never recursively +// serializes the input graph. Add complete membership umbrellas directly. +export function tessarIndexManifest( + instance: CardDef, + doc: LooseSingleCardDocument, + inputGeneration?: number, +): TessarMaterialization | undefined { + if (!(instance.constructor as typeof CardDef).tessarMaterialized) return; + let computedFields: string[] = []; + let watches: TessarMaterialization['watches'] = []; + let collect = ( + value: BaseDef, + prefix = '', + ancestors = new Set(), + ) => { + if (ancestors.has(value)) + throw new Error('Tessar does not support cyclic contained outputs'); + let nextAncestors = new Set(ancestors).add(value); + for (let [name, field] of Object.entries( + getFields(value, { includeComputeds: true }), + )) { + if (name === 'id') continue; + let path = `${prefix}${name}`; + if (field.queryDefinition) { + if (prefix || field.fieldType !== 'linksToMany') { + throw new Error( + 'Tessar currently requires top-level linksToMany queries', + ); + } + let watch = tessarQueryWatch(getStore(instance), instance, field); + watches.push({ fieldPath: path, query: watch.query }); + if (inputGeneration === undefined) continue; + let resource = peekQueryFieldSearchResource(instance, name); + let membership = getRelationshipMembershipState(instance, name); + if ( + !resource || + resource.isLoading || + resource.errors?.length || + !membership.isLoaded || + membership.isPartial || + membership.totalMatchCount !== membership.membership?.length || + membership.membership?.some((member) => member.kind !== 'present') + ) { + throw new Error( + `Tessar query '${name}' is unresolved, partial or errored`, + ); + } + doc.data.relationships ??= {}; + doc.data.relationships[name] = { + links: { self: null, search: watch.searchURL }, + data: membership.membership!.map((member) => ({ + type: 'card', + id: member.reference!, + })), + meta: { total: membership.totalMatchCount }, + }; + continue; + } + if (field.computeVia) { + if ( + field.fieldType !== 'contains' && + field.fieldType !== 'containsMany' + ) { + // Computed links (including CardDef.cardTheme) retain their ordinary + // lazy behavior. Only contained outputs are covered by this stamp. + continue; + } + computedFields.push(path); + } + if ( + primitive in field.card || + (field.fieldType !== 'contains' && field.fieldType !== 'containsMany') + ) + continue; + let child = (value as any)[name]; + if (field.fieldType === 'containsMany') { + for (let item of child ?? []) { + if (item) collect(item, `${path}.*.`, nextAncestors); + } + } else if (child) collect(child, `${path}.`, nextAncestors); + } + }; + collect(instance); + return { + version: 1, + state: 'pending', + computedFields: [...new Set(computedFields)], + queryFields: watches.map((watch) => watch.fieldPath), + watches, + // A discovery pass registers watches but cannot publish an authoritative + // result. Only the writer, after checking the pinned input revision, may + // change this marker to ready. + inputGeneration: inputGeneration ?? 0, + }; +} + function getStore(instance: BaseDef): CardStore { return stores.get(instance as BaseDef) ?? new FallbackCardStore(); } diff --git a/packages/base/field-support.ts b/packages/base/field-support.ts index 320f3acf147..4af70ad2e32 100644 --- a/packages/base/field-support.ts +++ b/packages/base/field-support.ts @@ -63,6 +63,7 @@ const deserializedData = initSharedState( // A contained edit leaves snapshot mode for the entire owner graph. export interface TessarSnapshotScope { active: boolean; + pending?: boolean; } const tessarSnapshots = initSharedState( 'tessarSnapshots', @@ -97,6 +98,26 @@ export function leaveTessarSnapshot(instance: BaseDef): void { if (snapshot) snapshot.scope.active = false; } +export function hasTessarSnapshot(instance: BaseDef): boolean { + return Boolean(tessarSnapshots.get(instance)?.scope.active); +} + +export function tessarSnapshotState( + instance: BaseDef, +): 'live' | 'ready' | 'pending' { + entangleWithCardTracking(instance); + let scope = tessarSnapshots.get(instance)?.scope; + return scope?.active ? (scope.pending ? 'pending' : 'ready') : 'live'; +} + +export function markTessarPending(instance: BaseDef): void { + let scope = tessarSnapshots.get(instance)?.scope; + if (scope?.active && !scope.pending) { + scope.pending = true; + notifyCardTracking(instance); + } +} + export function hasTessarQueryMembership( instance: BaseDef, fieldName: string, diff --git a/packages/base/query-field-support.ts b/packages/base/query-field-support.ts index d0edfe5037d..c68cd5cdf70 100644 --- a/packages/base/query-field-support.ts +++ b/packages/base/query-field-support.ts @@ -902,6 +902,36 @@ function resolveInstancePathValue(instance: BaseDef, path: string): any { return current; } +export function tessarQueryWatch( + store: CardStore, + instance: BaseDef, + field: Field, +) { + let definition = buildFieldDefinition(field); + if (!definition) + throw new Error(`Tessar cannot resolve query field '${field.name}'`); + let normalized = resolveQueryAndRealm(store, instance, field, definition); + if (!normalized) + throw new Error( + `Tessar cannot resolve query parameters for '${field.name}'`, + ); + let realm = (instance as any)[realmURLSymbol] as URL | undefined; + if ( + !realm || + normalized.realmHrefs.length !== 1 || + normalized.realmHrefs[0] !== realm.href + ) { + throw new Error( + 'Tessar materialization currently requires same-realm queries', + ); + } + return { + fieldPath: field.name, + query: normalized.query, + searchURL: normalized.searchURL, + }; +} + function buildFieldDefinition(field: Field): FieldDefinition | undefined { let ref = identifyCard(field.card); if (!ref) { diff --git a/packages/host/app/lib/gc-card-store.ts b/packages/host/app/lib/gc-card-store.ts index 300f63fd76c..5d456e6c04f 100644 --- a/packages/host/app/lib/gc-card-store.ts +++ b/packages/host/app/lib/gc-card-store.ts @@ -25,6 +25,7 @@ import { type SingleFileMetaDocument, type VirtualNetwork, } from '@cardstack/runtime-common'; +import { currentTessarInputSnapshot } from '@cardstack/runtime-common/tessar-materialization'; import type { BaseDef, @@ -131,9 +132,14 @@ function currentRenderScope(): string | undefined { if (g.__boxelRenderContext !== true || typeof g.__boxelJobId !== 'string') { return undefined; } - return typeof g.__boxelRenderScope === 'string' - ? g.__boxelRenderScope - : g.__boxelJobId; + let scope = + typeof g.__boxelRenderScope === 'string' + ? g.__boxelRenderScope + : g.__boxelJobId; + let tessar = currentTessarInputSnapshot(); + return tessar + ? `${scope}:tessar:${tessar.realmURL}:${tessar.generation}` + : scope; } // we use this 2 way mapping between local ID and remote ID because if we end up @@ -570,7 +576,12 @@ export default class CardStoreWithGarbageCollection implements CardStore { } return await promise; } - promise = loadCardDocument(this.#fetch, url, this.#virtualNetwork); + promise = loadCardDocument( + this.#fetch, + url, + this.#virtualNetwork, + currentTessarInputSnapshot(), + ); // Held locally as well as in the map: a scope boundary clears the map // mid-flight, so reading it back in the `finally` would time this load // against a newer load's start — or find nothing and drop the entry. diff --git a/packages/host/app/lib/prerender-fetch-headers.ts b/packages/host/app/lib/prerender-fetch-headers.ts index 8e5ddc9b873..e874c8a5806 100644 --- a/packages/host/app/lib/prerender-fetch-headers.ts +++ b/packages/host/app/lib/prerender-fetch-headers.ts @@ -4,6 +4,10 @@ import { X_BOXEL_JOB_ID_HEADER, X_BOXEL_LOGGING_CORRELATION_ID_HEADER, } from '@cardstack/runtime-common'; +import { + currentTessarInputSnapshot, + TESSAR_INPUT_GENERATION_HEADER, +} from '@cardstack/runtime-common/tessar-materialization'; // Set by the prerender server's `evaluateOnNewDocument` before the // SPA boots, and also by the host's prerender-shaped routes @@ -16,7 +20,15 @@ import { export function duringPrerenderHeaders(): Record { let flag = (globalThis as unknown as { __boxelRenderContext?: boolean }) .__boxelRenderContext; - return flag === true ? { [DURING_PRERENDER_HEADER]: '1' } : {}; + let tessar = currentTessarInputSnapshot(); + return flag === true + ? { + [DURING_PRERENDER_HEADER]: '1', + ...(tessar + ? { [TESSAR_INPUT_GENERATION_HEADER]: String(tessar.generation) } + : {}), + } + : {}; } // The same marker, for a card write rather than a search. Gated on the diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index 69cf6a55e9f..25ffc7ed856 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -1,4 +1,5 @@ import type Controller from '@ember/controller'; + import { registerDestructor } from '@ember/destroyable'; import { action } from '@ember/object'; import Route from '@ember/routing/route'; @@ -38,6 +39,10 @@ import { coerceErrorMessage, serializableError, } from '@cardstack/runtime-common/error'; +import { + currentTessarInputSnapshot, + TESSAR_INPUT_GENERATION_HEADER, +} from '@cardstack/runtime-common/tessar-materialization'; import { windowErrorHandler, @@ -559,13 +564,23 @@ export default class RenderRoute extends Route { (globalThis as any).__renderModel = undefined; (globalThis as any).__boxelSetRenderStage?.('buildModel:fetching-source'); + let tessarInput = currentTessarInputSnapshot(); let response: Response; try { response = await this.#authGuard.race(() => - this.network.authedFetch(id, { + this.network.authedFetch(tessarInput ? id.replace(/\.json$/, '') : id, { method: 'GET', headers: { - Accept: SupportedMimeType.CardSource, + Accept: tessarInput + ? SupportedMimeType.CardJson + : SupportedMimeType.CardSource, + ...(tessarInput + ? { + [TESSAR_INPUT_GENERATION_HEADER]: String( + tessarInput.generation, + ), + } + : {}), }, }), ); @@ -581,6 +596,17 @@ export default class RenderRoute extends Route { let lastModified = new Date(response.headers.get('last-modified')!); let doc: LooseSingleCardDocument | CardErrorsJSONAPI = await response.json(); + if (tessarInput && 'data' in doc && doc.data.meta.tessar) { + // This is the owner being recomputed. Its previous membership is an + // output, never an input seed for the new generation's query. + for (let field of doc.data.meta.tessar.queryFields) { + for (let key of Object.keys(doc.data.relationships ?? {})) { + if (key === field || key.startsWith(`${field}.`)) + delete doc.data.relationships![key]; + } + } + delete doc.data.meta.tessar; + } let canonicalId = id.replace(/\.json$/, ''); let state = new TrackedMap(); diff --git a/packages/host/app/routes/render/meta.ts b/packages/host/app/routes/render/meta.ts index 9090bb70936..f660f5641af 100644 --- a/packages/host/app/routes/render/meta.ts +++ b/packages/host/app/routes/render/meta.ts @@ -24,6 +24,7 @@ import { type PrerenderMetaDiagnostics, type RenderError, } from '@cardstack/runtime-common'; +import { currentTessarInputSnapshot } from '@cardstack/runtime-common/tessar-materialization'; import type CardService from '@cardstack/host/services/card-service'; import type EnvironmentService from '@cardstack/host/services/environment-service'; @@ -168,6 +169,9 @@ export default class RenderMetaRoute extends Route { // The search doc comes from the searchable-driven generator in its own base // module. It derives link depth from the explicit `searchable` annotations // rather than from what the render happened to load. + let tessar = (instance.constructor as typeof CardDef).tessarMaterialized; + let tessarInput = currentTessarInputSnapshot(); + if (tessar && tessarInput) await api.prepareTessarQueries(instance); let searchable = await this.cardService.getSearchable(); // Produce the search doc by walking until the store's load state is @@ -238,7 +242,7 @@ export default class RenderMetaRoute extends Route { ...new Set([ SEARCHABLE_MODULE_URL, ...(renderModel?.capturedDeps ?? []), - ...snapshotRuntimeDependencies({ excludeQueryOnly: true }).deps, + ...snapshotRuntimeDependencies({ excludeQueryOnly: !tessar }).deps, ...searchableDeps, ]), ]; @@ -301,6 +305,11 @@ export default class RenderMetaRoute extends Route { delete relationship.data; } delete serialized.included; + let tessarManifest = tessar + ? api.tessarIndexManifest(instance, serialized, tessarInput?.generation) + : undefined; + delete serialized.data.meta.tessar; + if (tessarManifest) serialized.data.meta.tessar = tessarManifest; } finally { if (passOpen && typeof api.endComputePass === 'function') { passSnapshot = api.endComputePass(); @@ -361,6 +370,11 @@ export default class RenderMetaRoute extends Route { if (typeof api.getBrokenLinks === 'function') { let brokenLinks = api.getBrokenLinks(instance); if (brokenLinks.length > 0) { + if (tessar && tessarInput) { + throw new Error( + 'Tessar cannot publish with broken input dependencies', + ); + } diagnostics.brokenLinks = brokenLinks.map( ({ fieldName, reference, kind }) => ({ fieldName, reference, kind }), ); @@ -512,6 +526,11 @@ export default class RenderMetaRoute extends Route { continue; } if (!stable) { + if ((instance.constructor as typeof CardDef).tessarMaterialized) { + throw new Error( + 'Tessar materialization did not reach a stable input graph', + ); + } computePerfLog.warn( `render.meta searchable walk for ${instance.id} did not reach a stable load generation within ${SEARCHABLE_SETTLE_MAX_PASSES} passes; using the current store state`, ); diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts index e5cf963d493..c612c1cdcdc 100644 --- a/packages/host/app/services/matrix-service.ts +++ b/packages/host/app/services/matrix-service.ts @@ -1411,6 +1411,7 @@ export default class MatrixService extends Service { let roomIds: string[] = list?.ops?.[0]?.room_ids ?? []; switch (state) { case SlidingSyncState.Complete: + this.messageService.tessarConnectionChanged(true); if (!this.initialSyncCompleted) { Promise.allSettled([ this.drainRoomState(), @@ -1428,6 +1429,7 @@ export default class MatrixService extends Service { roomIds.forEach((id) => this.roomsWaitingForSync.get(id)?.fulfill()); break; case SlidingSyncState.RequestFinished: + if (!resp) this.messageService.tessarConnectionChanged(false); roomIds.forEach((id) => this.aiRoomIds.add(id)); break; } diff --git a/packages/host/app/services/message-service.ts b/packages/host/app/services/message-service.ts index 8ac7efbb8da..e46dc2c9d8a 100644 --- a/packages/host/app/services/message-service.ts +++ b/packages/host/app/services/message-service.ts @@ -1,3 +1,4 @@ +import { registerDestructor } from '@ember/destroyable'; import Service, { service } from '@ember/service'; import { tracked } from '@glimmer/tracking'; @@ -11,10 +12,40 @@ export default class MessageService extends Service { new Map(); @service declare private network: NetworkService; @service declare private session: SessionService; + private tessarConnected = true; + private tessarConnectionListeners = new Set<(connected: boolean) => void>(); constructor(...args: ConstructorParameters) { super(...args); this.session.register(this); + let offline = () => this.tessarConnectionChanged(false); + let online = () => this.tessarConnectionChanged(true); + let visible = () => { + if (document.visibilityState === 'visible') { + this.tessarConnectionChanged(false); + this.tessarConnectionChanged(navigator.onLine); + } + }; + window.addEventListener('offline', offline); + window.addEventListener('online', online); + document.addEventListener('visibilitychange', visible); + registerDestructor(this, () => { + window.removeEventListener('offline', offline); + window.removeEventListener('online', online); + document.removeEventListener('visibilitychange', visible); + this.tessarConnectionListeners.clear(); + }); + } + + subscribeTessarConnection(callback: (connected: boolean) => void) { + this.tessarConnectionListeners.add(callback); + return () => this.tessarConnectionListeners.delete(callback); + } + + tessarConnectionChanged(connected: boolean) { + if (this.tessarConnected === connected) return; + this.tessarConnected = connected; + for (let callback of this.tessarConnectionListeners) callback(connected); } register() { diff --git a/packages/host/app/services/store.ts b/packages/host/app/services/store.ts index c47ee0a26a3..0635383c3f9 100644 --- a/packages/host/app/services/store.ts +++ b/packages/host/app/services/store.ts @@ -96,6 +96,8 @@ import { type VirtualNetwork, } from '@cardstack/runtime-common'; +import { currentTessarInputSnapshot } from '@cardstack/runtime-common/tessar-materialization'; + import CardStore, { getDeps, type ReferenceCount } from '../lib/gc-card-store'; import { @@ -339,8 +341,21 @@ export default class StoreService extends Service implements StoreInterface { this.store = this.createCardStore(); this.session.register(this); this.ready = this.setup(); + let unsubscribeTessar = this.messageService.subscribeTessarConnection( + (connected) => { + if (this.isRenderStore || (globalThis as any).__boxelPrerenderApp) + return; + for (let instance of this.store.allCardInstances()) { + if (instance.id && this.cardApiCache?.hasTessarSnapshot?.(instance)) { + this.cardApiCache.markTessarPending(instance); + if (connected) this.reloadTask.perform(instance); + } + } + }, + ); registerDestructor(this, () => { clearInterval(this.gcInterval); + unsubscribeTessar(); }); } @@ -1518,6 +1533,11 @@ export default class StoreService extends Service implements StoreInterface { // the undefined wire default — and so must share one key; keying on the raw // `scope` would split them and defeat the dedup. let wireScope = this.resolveWireScope(query, scope); + // Tessar reads are pinned to a committed generation. Never reuse a raw + // source pass's job cache or an in-flight request from an earlier wave. + if (currentTessarInputSnapshot()) { + return this.fetchSearchDocUncoalesced(query, realms, wireScope); + } // Resolved-doc cache eligibility: prerender + jobId + same-realm. // Cross-realm reads bypass — see field comment. @@ -2148,6 +2168,20 @@ export default class StoreService extends Service implements StoreInterface { } private handleInvalidations = (event: RealmEventContent) => { + if (event.eventName === 'update') { + // Before indexing has classified a source change, conservatively mark + // this realm's loaded snapshots pending. Keep their inputs unhydrated. + for (let instance of this.store.allCardInstances()) { + if ( + instance.id && + asURL(instance.id, this.network.virtualNetwork).startsWith( + event.realmURL, + ) + ) + this.cardApiCache?.markTessarPending?.(instance); + } + return; + } if (event.eventName !== 'index') { return; } @@ -2172,6 +2206,18 @@ export default class StoreService extends Service implements StoreInterface { // can be the only word a card being held as awaiting-index ever gets // that the row it is waiting for now exists. let reloadsTriggered = this.reloadAwaitingIndexInstances(event.realmURL); + for (let instance of this.store.allCardInstances()) { + if ( + instance.id && + asURL(instance.id, this.network.virtualNetwork).startsWith( + event.realmURL, + ) && + this.cardApiCache?.hasTessarSnapshot?.(instance) + ) { + this.reloadTask.perform(instance); + reloadsTriggered++; + } + } // Report the pass as a thin realm-event so the dashboard still sees it. telemetry?.recordEvent({ event_type: 'realm-event', @@ -2190,7 +2236,21 @@ export default class StoreService extends Service implements StoreInterface { if (event.indexType !== 'incremental') { return; } - let invalidations = event.invalidations as string[]; + let invalidations = [...(event.invalidations as string[])]; + // An unrelated write can leave the owner revision unchanged. Revalidate + // the pending snapshot after the lane settles to clear its pending state. + for (let instance of this.store.allCardInstances()) { + if ( + instance.id && + asURL(instance.id, this.network.virtualNetwork).startsWith( + event.realmURL, + ) && + instance.tessarState === 'pending' + ) { + let url = asURL(instance.id, this.network.virtualNetwork); + if (!invalidations.includes(url)) invalidations.push(url); + } + } let ownWrite = event.clientRequestId ? this.cardService.clientRequestIds.has(event.clientRequestId) : false; @@ -2340,6 +2400,11 @@ export default class StoreService extends Service implements StoreInterface { ); } + // A clean materialized owner may depend on the card this client + // edited. Local edits leave snapshot mode, so this exception cannot + // overwrite a draft owner while suppressing self-originated events. + if (this.cardApiCache?.hasTessarSnapshot?.(instance)) + reloadFile = true; if (reloadFile) { this.reloadTask.perform(instance); reloadsTriggered++; diff --git a/packages/host/tests/integration/components/tessar-snapshot-test.gts b/packages/host/tests/integration/components/tessar-snapshot-test.gts index c467960b72e..99ccf7ae6ae 100644 --- a/packages/host/tests/integration/components/tessar-snapshot-test.gts +++ b/packages/host/tests/integration/components/tessar-snapshot-test.gts @@ -1,7 +1,7 @@ import { getService } from '@universal-ember/test-support'; import { module, test } from 'qunit'; -import { rri } from '@cardstack/runtime-common'; +import { rri, meta, isSingleCardDocument } from '@cardstack/runtime-common'; import { setupIntegrationTestRealm, @@ -123,6 +123,76 @@ module('Integration | Tessar snapshot', function (hooks) { 7, 'new owner snapshot replaces old output', ); + Object.assign(doc.data.meta, { + generation: 2, + tessar: { + version: 1, + state: 'ready', + inputGeneration: 1, + publishedGeneration: 2, + definitionRevision: 'tessar-test', + computedFields: ['count', 'empty'], + queryFields: ['inputs'], + watches: [], + }, + }); + await api.updateFromSerialized(summary, doc, store); + assert.strictEqual( + summary.count, + 7, + 'an indexed response activates snapshot consumption', + ); + assert.strictEqual(calls, 0); + assert.strictEqual(searches, 0); + assert.strictEqual(summary.tessarState, 'ready'); + api.markTessarPending(summary); + assert.strictEqual( + summary.tessarState, + 'pending', + 'source changes expose pending state without expanding inputs', + ); + assert.strictEqual(searches, 0); + await api.updateFromSerialized(summary, doc, store); + assert.strictEqual( + summary.tessarState, + 'ready', + 'revalidation clears pending state', + ); + let older = structuredClone(doc); + let newer = structuredClone(doc); + newer.data.attributes.count = 11; + Object.assign(newer.data.meta, { + generation: 3, + tessar: { + ...(newer.data.meta as any).tessar, + inputGeneration: 2, + publishedGeneration: 3, + }, + }); + await Promise.all([ + api.updateFromSerialized(summary, newer, store), + api.updateFromSerialized(summary, older, store), + ]); + assert.strictEqual( + summary.count, + 11, + 'overlapping refreshes cannot move output backwards', + ); + assert.strictEqual((summary as any)[meta].tessar.publishedGeneration, 3); + await api.updateFromSerialized(summary, older, store); + assert.strictEqual( + summary.count, + 11, + 'a delayed older response is ignored', + ); + Object.assign(doc.data.meta, structuredClone(newer.data.meta)); + (doc.data.meta as any).tessar.state = 'pending'; + await assert.rejects( + api.updateFromSerialized(summary, doc, store), + /pending or has invalid provenance/, + ); + delete (doc.data.meta as any).tessar; + delete (doc.data.meta as any).generation; assert.strictEqual(calls, 0); doc.data.relationships.inputs.meta.total = 2; await assert.rejects( @@ -132,6 +202,113 @@ module('Integration | Tessar snapshot', function (hooks) { ); }); + test('publication registers empty queries and refuses incomplete membership', async function (assert) { + class TessarInput extends CardDef { + @field name = contains(StringField); + } + class TessarSummary extends CardDef { + static tessarMaterialized = true; + @field name = contains(StringField); + @field inputs = linksToMany(TessarInput, { + query: { filter: { eq: { name: '$this.name' } } }, + }); + @field count = contains(NumberField, { + computeVia: function (this: TessarSummary) { + return this.inputs.length; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarInput, TessarSummary } }, + }); + let api: typeof import('@cardstack/base/card-api') = await getService( + 'loader-service', + ).loader.import('@cardstack/base/card-api'); + let summary = new TessarSummary({ name: 'Tessar' }); + summary.id = rri(`${testRealmURL}summary`); + (summary as any)[meta] = { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + realmURL: testRealmURL, + }; + let store = api.getStore(summary); + let result = { + instances: [], + instancesByRealm: [], + isLoading: false, + meta: { page: { total: 0 } }, + totalMatchCount: 0, + isPartial: false, + }; + store.getSearchResource = () => result; + await api.updateFromSerialized( + summary, + { + data: { + id: summary.id, + type: 'card', + attributes: { name: 'Tessar' }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + realmURL: testRealmURL, + }, + }, + }, + store, + ); + await api.prepareTessarQueries(summary); + let doc = api.serializeCard(summary, { + includeComputeds: true, + omitQueryFields: true, + }); + let discovery = api.tessarIndexManifest(summary, doc); + assert.strictEqual( + discovery?.state, + 'pending', + 'a discovery render cannot claim readiness', + ); + assert.strictEqual( + discovery?.watches.length, + 1, + 'the empty query is registered', + ); + assert.false( + JSON.stringify(discovery?.watches).includes('$this'), + 'watch parameters are resolved', + ); + assert.true(JSON.stringify(discovery?.watches).includes('Tessar')); + let snapshot = api.tessarIndexManifest(summary, doc, 7); + assert.true( + isSingleCardDocument(doc), + 'the published wire shape passes the same validator used by the host store', + ); + let membership = doc.data.relationships?.inputs; + if (Array.isArray(membership)) + throw new Error('Expected a Tessar membership umbrella'); + assert.deepEqual( + membership?.data, + [], + 'known empty membership is serialized explicitly', + ); + assert.strictEqual(membership?.meta?.total, 0); + assert.strictEqual(snapshot?.inputGeneration, 7); + assert.true(snapshot?.computedFields.includes('count')); + assert.notOk(doc.included, 'the input graph is never serialized'); + result.totalMatchCount = 2; + result.isPartial = true; + assert.throws( + () => api.tessarIndexManifest(summary, doc, 7), + /unresolved, partial or errored/, + ); + }); + test('nested values share edit invalidation and legacy documents keep computing', async function (assert) { let calls = 0; class TessarDetails extends FieldDef { diff --git a/packages/realm-server/handlers/handle-search.ts b/packages/realm-server/handlers/handle-search.ts index fcbd75f9fc3..71d941d86ac 100644 --- a/packages/realm-server/handlers/handle-search.ts +++ b/packages/realm-server/handlers/handle-search.ts @@ -1,6 +1,12 @@ import type Koa from 'koa'; +import { + assertTessarGeneration, + tessarRequestedGeneration, + tessarHasMaterializations, +} from '@cardstack/runtime-common/tessar-materialization'; import { applyServerSearchPageBound, + CardError, buildSearchErrorResponse, DURING_PRERENDER_HEADER, ifNoneMatchMatches, @@ -185,6 +191,7 @@ export default function handleSearch(opts: { // ride the run-time opts instead. let runSearchOpts = { ...searchOpts, + tessarInput: request.headers.has('x-boxel-tessar-input-generation'), ...(loggingCorrelationId !== null ? { loggingCorrelationId } : {}), ...(timings ? { timings } : {}), }; @@ -236,6 +243,43 @@ export default function handleSearch(opts: { let jobId = searchCache ? prerenderJobId : null; try { + let tessarGeneration = tessarRequestedGeneration(request); + if (tessarGeneration !== undefined) { + if (realmList.length !== 1) { + throw new CardError('Tessar inputs require exactly one realm', { + status: 400, + }); + } + await assertTessarGeneration(dbAdapter, realmList[0], tessarGeneration); + let body = await runSearch(); + await assertTessarGeneration(dbAdapter, realmList[0], tessarGeneration); + await setContextResponse( + ctxt, + new Response(body, { + headers: { + 'content-type': SupportedMimeType.CardJson, + 'cache-control': 'no-store', + }, + }), + ); + emitTimeline(); + return; + } + // Generation-keyed caches cannot see a queued source write until its + // first index swap. Tessar responses must revalidate pending state. + if (await tessarHasMaterializations(dbAdapter, realmList)) { + setContextResponse( + ctxt, + new Response(await runSearch(), { + headers: { + 'content-type': SupportedMimeType.CardJson, + 'cache-control': 'no-store', + }, + }), + ); + emitTimeline(); + return; + } await respondWithJobScopedSearchCache(ctxt, { searchCache, jobId, @@ -251,7 +295,7 @@ export default function handleSearch(opts: { // The per-request time budget fired inside `runSearch`. A bounded search // is never cacheable (cacheable ⟹ during-prerender ⟹ not bounded), so // this only surfaces on the fresh-compute path and leaves no cache entry. - if (e instanceof SearchBoundError) { + if (e instanceof SearchBoundError || e instanceof CardError) { await setContextResponse( ctxt, buildSearchErrorResponse(e.message, e.status), diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 2bc829cafbb..13ea97f2ed0 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -1182,6 +1182,24 @@ export function buildPrerenderApp(options: { ? rawRenderScope : undefined; + let tessarInputSnapshot = attrs.tessarInputSnapshot; + if ( + tessarInputSnapshot !== undefined && + (!tessarInputSnapshot || + typeof tessarInputSnapshot !== 'object' || + tessarInputSnapshot.realmURL !== realm || + !Number.isSafeInteger(tessarInputSnapshot.generation) || + tessarInputSnapshot.generation < 0) + ) { + ctxt.status = 400; + ctxt.body = { + errors: [ + { status: 400, message: 'Invalid Tessar indexed input snapshot' }, + ], + }; + return; + } + let start = Date.now(); // Hoisted so a re-render after a host-shell change replays the same // visit rather than an approximation of it. @@ -1201,6 +1219,7 @@ export function buildPrerenderApp(options: { ...(jobId ? { jobId } : {}), ...(screenshots ? { screenshots } : {}), ...(renderScope ? { renderScope } : {}), + ...(tessarInputSnapshot ? { tessarInputSnapshot } : {}), signal: ac.signal, }; let shellAtStart = options.getHostShellHash?.(); diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 0c26853b595..534ecb3a218 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -732,6 +732,7 @@ export class Prerenderer { jobId, screenshots, renderScope, + tessarInputSnapshot, } = this.#gateClearCache(rawArgs); let signal = (rawArgs as { signal?: AbortSignal }).signal; let testOnTabAcquired = ( @@ -794,6 +795,7 @@ export class Prerenderer { jobId, screenshots, renderScope, + tessarInputSnapshot, signal, onTabAcquired, }); @@ -831,6 +833,7 @@ export class Prerenderer { jobId, screenshots, renderScope, + tessarInputSnapshot, signal, onTabAcquired, }); diff --git a/packages/realm-server/prerender/remote-prerenderer.ts b/packages/realm-server/prerender/remote-prerenderer.ts index 8d32290c0c3..6c40ab60267 100644 --- a/packages/realm-server/prerender/remote-prerenderer.ts +++ b/packages/realm-server/prerender/remote-prerenderer.ts @@ -286,6 +286,7 @@ export function createRemotePrerenderer( jobId, screenshots, renderScope, + tessarInputSnapshot, }: PrerenderVisitArgs): Promise { return await requestWithRetry( 'prerender-visit', @@ -306,6 +307,7 @@ export function createRemotePrerenderer( ...(jobId ? { jobId } : {}), ...(screenshots ? { screenshots } : {}), ...(renderScope ? { renderScope } : {}), + ...(tessarInputSnapshot ? { tessarInputSnapshot } : {}), }, ); }, diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 522dccc03e3..14a15261283 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -887,6 +887,7 @@ export class RenderRunner { jobId, screenshots, renderScope, + tessarInputSnapshot, signal, onTabAcquired, }: PrerenderVisitArgs & { @@ -1030,6 +1031,7 @@ export class RenderRunner { id: string | undefined, jobPriority: number | undefined, scope: string | undefined, + tessar: PrerenderVisitArgs['tessarInputSnapshot'], ) => { localStorage.setItem('boxel-session', sessionAuth); (globalThis as unknown as { __boxelJobId?: string }).__boxelJobId = @@ -1044,6 +1046,11 @@ export class RenderRunner { ( globalThis as unknown as { __boxelRenderScope?: string } ).__boxelRenderScope = scope; + ( + globalThis as unknown as { + __tessarInputSnapshot?: PrerenderVisitArgs['tessarInputSnapshot']; + } + ).__tessarInputSnapshot = tessar; return ( ( globalThis as unknown as { @@ -1056,6 +1063,7 @@ export class RenderRunner { jobId, priority, renderScope, + tessarInputSnapshot, ), ); // A card-instance index visit fuses the file extract into the diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 195dd5f9f1b..bce4156a2b1 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -847,7 +847,7 @@ export class RealmServer { // this list the preflight fails and the player errors before any // bytes flow. allowHeaders: - 'Authorization, Content-Type, If-Match, If-None-Match, If-Range, Range, X-Requested-With, X-Boxel-Client-Request-Id, X-Boxel-Assume-User, X-HTTP-Method-Override, X-Boxel-Disable-Module-Cache, X-Filename, X-Boxel-During-Prerender, X-Boxel-Consuming-Realm, X-Boxel-Job-Id, X-Boxel-Job-Priority, X-Boxel-Logging-Correlation-Id, X-Grafana-Device-Id, X-Grafana-Action', + 'Authorization, Content-Type, If-Match, If-None-Match, If-Range, Range, X-Requested-With, X-Boxel-Client-Request-Id, X-Boxel-Assume-User, X-HTTP-Method-Override, X-Boxel-Disable-Module-Cache, X-Filename, X-Boxel-During-Prerender, X-Boxel-Tessar-Input-Generation, X-Boxel-Consuming-Realm, X-Boxel-Job-Id, X-Boxel-Job-Priority, X-Boxel-Logging-Correlation-Id, X-Grafana-Device-Id, X-Grafana-Action', // Without an explicit expose list, @koa/cors only emits the // CORS-safelisted response headers (cache-control, content-*, // expires, last-modified, pragma). ETag is not on that list, diff --git a/packages/realm-server/tests/tessar-query-registry-test.ts b/packages/realm-server/tests/tessar-query-registry-test.ts index 009160b1fc6..2df9ff58f7c 100644 --- a/packages/realm-server/tests/tessar-query-registry-test.ts +++ b/packages/realm-server/tests/tessar-query-registry-test.ts @@ -2,18 +2,26 @@ import QUnit from 'qunit'; import type { PgAdapter } from '@cardstack/postgres'; import { IndexQueryEngine, + IndexWriter, VirtualNetwork, baseRealmRRI, rri, type Definition, type DefinitionLookup, type Filter, + type InstanceEntry, } from '@cardstack/runtime-common'; import { TessarQueryRegistry, type TessarDocument, } from '@cardstack/runtime-common/tessar-query-registry'; import { setupDB } from './helpers/index.ts'; +import { + assertTessarGeneration, + tessarRequestedGeneration, + TESSAR_INPUT_GENERATION_HEADER, + tessarReadState, +} from '@cardstack/runtime-common/tessar-materialization'; const { module, test } = QUnit; const realmURL = 'https://tessar.example/'; @@ -62,6 +70,9 @@ module('Tessar | Postgres query registry', function (hooks) { let db: PgAdapter; let engine: IndexQueryEngine; let registry: TessarQueryRegistry; + let writer: IndexWriter; + let publication: ReturnType; + let network: VirtualNetwork; setupDB(hooks, { templateDatabase: process.env.TESSAR_TEST_TEMPLATE_DB, beforeEach: async (adapter) => { @@ -73,10 +84,12 @@ module('Tessar | Postgres query registry', function (hooks) { return definition; }, } as unknown as DefinitionLookup; - let network = new VirtualNetwork(); + network = new VirtualNetwork(); network.addRealmMapping(baseRealmRRI, 'https://cardstack.com/base/'); engine = new IndexQueryEngine(db, lookup, network); registry = new TessarQueryRegistry(db, engine); + writer = new IndexWriter(db); + publication = writer.tessarPublication(lookup, network); }, }); let record = (name: string): TessarDocument => ({ @@ -85,6 +98,236 @@ module('Tessar | Postgres query registry', function (hooks) { search_doc: { name, score: 7, active: true, tags: ['red', 'blue'] }, }); + test('Tessar source swap, watch registration, dirty marking and guarded owner publication share the index transaction', async function (assert) { + let ownerURL = `${realmURL}summary.json`; + let inputURL = `${realmURL}input.json`; + let entry = ( + name: string, + count?: number, + inputGeneration = 0, + ): InstanceEntry => ({ + type: 'instance', + lastModified: 1, + resourceCreatedAt: 1, + resource: { + id: rri( + name === 'summary' + ? ownerURL.replace('.json', '') + : inputURL.replace('.json', ''), + ), + type: 'card', + attributes: { name, ...(count !== undefined ? { count } : {}) }, + meta: { + adoptsFrom: on, + ...(count !== undefined + ? { + tessar: { + version: 1 as const, + state: 'pending' as const, + computedFields: ['count'], + queryFields: ['inputs'], + watches: [ + { + fieldPath: 'inputs', + query: { filter: { on, eq: { name: 'A' } } }, + }, + ], + inputGeneration, + }, + } + : {}), + }, + }, + searchData: { name, ...(count !== undefined ? { count } : {}) }, + types: [`${on.module}/${on.name}`], + displayNames: ['TessarRecord'], + deps: new Set(), + }); + let source = await writer.createBatch(new URL(realmURL), network); + await source.updateEntry(new URL(ownerURL), entry('summary', 0)); + await source.updateEntry(new URL(inputURL), entry('A')); + // The HTTP PATCH/POST path holds this source-write lock until indexing + // completes. Publication must use a different advisory-lock namespace. + await db.withWriteLock(realmURL, async () => { + let deadline: ReturnType | undefined; + try { + await Promise.race([ + source.done({ tessar: publication }), + new Promise((_, reject) => { + deadline = setTimeout( + () => + reject( + new Error('Tessar publication deadlocked a source write'), + ), + 5000, + ); + }), + ]); + } finally { + clearTimeout(deadline); + } + }); + assert.deepEqual(await registry.pending(realmURL), [ + { ownerURL, generation: 1 }, + ]); + let materialize = await writer.createBatch(new URL(realmURL), network); + await materialize.updateEntry(new URL(ownerURL), entry('summary', 1, 1)); + await materialize.done({ tessar: publication, tessarInputGeneration: 1 }); + let [published] = await db.execute( + 'SELECT pristine_doc FROM boxel_index WHERE url = $1 AND type = $2', + { bind: [ownerURL, 'instance'] }, + ); + let stamp = (published.pristine_doc as any).meta.tessar; + assert.strictEqual(stamp.state, 'ready'); + assert.strictEqual(stamp.publishedGeneration, 2); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp), + 'ready', + ); + let [queued] = await db.execute( + "INSERT INTO jobs (job_type, concurrency_group, timeout, priority, args) VALUES ('incremental-index', $1, 60, 10, '{}') RETURNING id", + { bind: [`indexing:${realmURL}`] }, + ); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp), + 'pending', + 'an acknowledged queued write is pending before the first source swap', + ); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp, { + tessarInput: true, + }), + 'ready', + 'a revision-pinned worker can consume its ready feeder', + ); + await db.execute("UPDATE jobs SET status = 'resolved' WHERE id = $1", { + bind: [queued.id], + }); + await db.execute("UPDATE jobs SET status = 'rejected' WHERE id = $1", { + bind: [queued.id], + }); + await assert.rejects( + tessarReadState(db, realmURL, ownerURL, stamp), + /indexing failed/, + 'a failed source pass cannot expose the previous snapshot as current', + ); + await db.execute("UPDATE jobs SET status = 'resolved' WHERE id = $1", { + bind: [queued.id], + }); + let change = await writer.createBatch(new URL(realmURL), network); + await change.updateEntry(new URL(inputURL), entry('B')); + await change.done({ tessar: publication }); + assert.deepEqual( + await registry.pending(realmURL), + [{ ownerURL, generation: 3 }], + 'a membership exit invalidates without an existing concrete dependency', + ); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp), + 'pending', + 'the old snapshot cannot be served as current', + ); + let obsolete = await writer.createBatch(new URL(realmURL), network); + await obsolete.updateEntry(new URL(ownerURL), entry('summary', 99, 1)); + await assert.rejects( + obsolete.done({ tessar: publication, tessarInputGeneration: 1 }), + /input revision/, + ); + let [unchanged] = await db.execute( + 'SELECT generation, pristine_doc FROM boxel_index WHERE url = $1 AND type = $2', + { bind: [ownerURL, 'instance'] }, + ); + assert.strictEqual( + Number(unchanged.generation), + 2, + 'failed transaction never promoted the obsolete owner', + ); + assert.strictEqual((unchanged.pristine_doc as any).attributes.count, 1); + await assertTessarGeneration(db, realmURL, 3); + let fresh = await writer.createBatch(new URL(realmURL), network); + await fresh.updateEntry(new URL(ownerURL), entry('summary', 0, 3)); + await fresh.done({ tessar: publication, tessarInputGeneration: 3 }); + assert.deepEqual( + await registry.pending(realmURL), + [], + 'a correct recomputation clears durable dirty state', + ); + assert.strictEqual(await registry.activeOwnerCount(realmURL), 1); + // Module writes mint the realm-wide loader epoch before indexing starts. + // An owner outside that module's dependency closure still needs a new + // stamp, or read-time epoch validation would leave it pending forever. + await db.execute( + 'UPDATE realm_generations SET loader_epoch = $1 WHERE realm_url = $2', + { bind: ['tessar-new-module-epoch', realmURL] }, + ); + let moduleChange = await writer.createBatch(new URL(realmURL), network); + await moduleChange.done({ tessar: publication }); + assert.deepEqual( + await registry.pending(realmURL), + [{ ownerURL, generation: 5 }], + 'an epoch change schedules owners outside the changed module closure', + ); + let rematerialize = await writer.createBatch(new URL(realmURL), network); + await rematerialize.updateEntry(new URL(ownerURL), entry('summary', 0, 5)); + await rematerialize.done({ + tessar: publication, + tessarInputGeneration: 5, + }); + let [restamped] = await db.execute( + 'SELECT pristine_doc FROM boxel_index WHERE url = $1 AND type = $2', + { bind: [ownerURL, 'instance'] }, + ); + assert.strictEqual( + await tessarReadState( + db, + realmURL, + ownerURL, + (restamped.pristine_doc as any).meta.tessar, + ), + 'ready', + 'the restamped owner is readable under the new definition epoch', + ); + }); + + test('Tessar input reads reject changed generations and malformed revision headers', async function (assert) { + await db.execute( + 'INSERT INTO realm_generations (realm_url, current_generation) VALUES ($1, 7)', + { bind: [realmURL] }, + ); + await assertTessarGeneration(db, realmURL, 7); + assert.strictEqual( + tessarRequestedGeneration( + new Request(realmURL, { + headers: { [TESSAR_INPUT_GENERATION_HEADER]: '7' }, + }), + ), + 7, + ); + for (let value of ['', '-1', '1.5', 'NaN', '9007199254740992']) { + assert.throws( + () => + tessarRequestedGeneration( + new Request(realmURL, { + headers: { [TESSAR_INPUT_GENERATION_HEADER]: value }, + }), + ), + /nonnegative input generation/, + ); + } + await db.execute( + 'UPDATE realm_generations SET current_generation = 8 WHERE realm_url = $1', + { bind: [realmURL] }, + ); + await assert.rejects( + assertTessarGeneration(db, realmURL, 7), + /input revision changed/, + ); + await db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + await assertTessarGeneration(db, realmURL, 8, tx); + assert.ok(true, 'publication can validate its pinned transaction'); + }); + }); + test('real SQL verifies typed values, null, plural paths and boolean composition', async function (assert) { let predicates: Array<[Filter, boolean]> = [ [{ on, eq: { name: 'A' } }, true], diff --git a/packages/runtime-common/definition-lookup.ts b/packages/runtime-common/definition-lookup.ts index 229b2466424..f32e537cb0b 100644 --- a/packages/runtime-common/definition-lookup.ts +++ b/packages/runtime-common/definition-lookup.ts @@ -1246,7 +1246,11 @@ export class CachingDefinitionLookup implements DefinitionLookup { } registerRealm(realm: LocalRealm): void { - this.#realms.push(realm); + // Tessar workers scope lookups for each indexing job. Replace a previous + // registration for that URL so jobs do not retain retired realm contexts. + let existing = this.#realms.findIndex((item) => item.url === realm.url); + if (existing === -1) this.#realms.push(realm); + else this.#realms[existing] = realm; } forRealm(realm: LocalRealm): DefinitionLookup { diff --git a/packages/runtime-common/document.ts b/packages/runtime-common/document.ts index df5f298a748..29ae4ee4111 100644 --- a/packages/runtime-common/document.ts +++ b/packages/runtime-common/document.ts @@ -8,12 +8,17 @@ import { } from './index.ts'; import type { RealmResourceIdentifier } from './realm-identifiers.ts'; import type { VirtualNetwork } from './virtual-network.ts'; +import { + TESSAR_INPUT_GENERATION_HEADER, + type TessarInputSnapshot, +} from './tessar-materialization.ts'; async function loadDocumentWithRequest( fetch: typeof globalThis.fetch, url: string, requestURL: URL, accept: SupportedMimeType, + extraHeaders?: Record, ) { let response: Response; requestURL.searchParams.set('noCache', 'true'); @@ -28,6 +33,7 @@ async function loadDocumentWithRequest( // documents being indexed and not finding the document yet in the index. headers: { Accept: accept, + ...extraHeaders, }, }); } catch (err: any) { @@ -74,14 +80,27 @@ export async function loadCardDocument( fetch: typeof globalThis.fetch, url: string, virtualNetwork: VirtualNetwork, + tessar?: TessarInputSnapshot, ) { - let target = !url.endsWith('.json') ? `${url}.json` : url; + let target = tessar + ? url.replace(/\.json$/, '') + : !url.endsWith('.json') + ? `${url}.json` + : url; let requestURL = virtualNetwork.toURL(target); + if (tessar && !requestURL.href.startsWith(tessar.realmURL)) { + return new CardError('Tessar indexed inputs must be in the owner realm', { + status: 400, + }); + } let json = await loadDocumentWithRequest( fetch, url, requestURL, - SupportedMimeType.CardSource, + tessar ? SupportedMimeType.CardJson : SupportedMimeType.CardSource, + tessar + ? { [TESSAR_INPUT_GENERATION_HEADER]: String(tessar.generation) } + : undefined, ); if (isCardError(json)) { return json; @@ -95,6 +114,16 @@ export async function loadCardDocument( )}`, ); } + if ( + tessar && + json.data.meta.tessar && + json.data.meta.tessar.state !== 'ready' + ) { + return new CardError( + 'Tessar feeder is pending; defer its dependent owner', + { status: 409 }, + ); + } if (!json.data.id) { // card source format is not serialized with the ID, so we add that back in. json.data.id = url as RealmResourceIdentifier; diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 5810a359ecd..84645c5a3f7 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -1,4 +1,6 @@ import { ignore, type Ignore } from './ignore.ts'; +import type { TessarIndexPublication } from './tessar-index-publication.ts'; +import type { TessarInputSnapshot } from './tessar-materialization.ts'; // Isomorphic UUID — works in both Node and the browser (host tests // instantiate IndexRunner inside a Chrome tab, so Node's built-in // `crypto.randomUUID` is not available). @@ -64,6 +66,16 @@ type VisitRenderOutcome = | { status: 'error'; error: unknown }; export class IndexRunner { + #tessar: TessarIndexPublication; + #tessarInputSnapshot: TessarInputSnapshot | undefined; + #tessarTimings = { + tessarMaterializationMs: 0, + tessarWriteMs: 0, + tessarSwapMs: 0, + tessarWaves: 0, + tessarOwnersRendered: 0, + }; + #tessarSourceWriteMs: number | undefined; #indexingInstances = new Map>(); #reader: Reader; #indexWriter: IndexWriter; @@ -169,6 +181,17 @@ export class IndexRunner { }): void; }) { this.#indexWriter = indexWriter; + this.#tessar = indexWriter.tessarPublication( + definitionLookup.forRealm({ + url: realmURL.href, + getRealmOwnerUserId: async () => realmOwnerUserId, + visibility: async () => + (await this.getModuleCacheContext()).cacheScope === 'public' + ? 'public' + : 'private', + }), + virtualNetwork, + ); this.#realmPaths = new RealmPaths(realmURL, virtualNetwork); this.#reader = reader; this.#realmURL = realmURL; @@ -342,8 +365,12 @@ export class IndexRunner { `${jobIdentity(current.#jobInfo)} completed index visit in ${Date.now() - visitStart} ms`, ); let finalizeStart = Date.now(); - let { totalIndexEntries } = await current.batch.done(); + let { totalIndexEntries } = await current.batch.done({ + tessar: current.#tessar, + }); swapMs = Date.now() - finalizeStart; + current.#tessarSourceWriteMs = current.batch.writeMs; + invalidations.push(...(await current.#drainTessar())); current.#perfLog.debug( `${jobIdentity(current.#jobInfo)} completed index finalization in ${swapMs} ms`, ); @@ -390,7 +417,8 @@ export class IndexRunner { discoverMs, orderMs, ...(visitLoopMs !== undefined ? { visitLoopMs } : {}), - writeMs: current.batch.writeMs, + writeMs: current.#tessarSourceWriteMs ?? current.batch.writeMs, + ...(current.#tessarTimings.tessarWaves ? current.#tessarTimings : {}), ...(swapMs !== undefined ? { swapMs } : {}), }, }; @@ -563,8 +591,12 @@ export class IndexRunner { visitLoopMs = Date.now() - loopStart; let finalizeStart = Date.now(); - let { totalIndexEntries } = await current.batch.done(); + let { totalIndexEntries } = await current.batch.done({ + tessar: current.#tessar, + }); swapMs = Date.now() - finalizeStart; + current.#tessarSourceWriteMs = current.batch.writeMs; + invalidations.push(...(await current.#drainTessar())); current.stats.totalIndexEntries = totalIndexEntries; } finally { current.#onProgress?.({ @@ -605,12 +637,81 @@ export class IndexRunner { discoverMs, orderMs, ...(visitLoopMs !== undefined ? { visitLoopMs } : {}), - writeMs: current.batch.writeMs, + writeMs: current.#tessarSourceWriteMs ?? current.batch.writeMs, + ...(current.#tessarTimings.tessarWaves ? current.#tessarTimings : {}), ...(swapMs !== undefined ? { swapMs } : {}), }, }; } + // Drain the durable registry inside the existing per-realm indexing job. + // A crash retries that queue job and discovers pending work again. Failed + // waves remain pending and fail the job; they never publish partial output. + async #drainTessar(): Promise { + let startedAt = Date.now(); + let published = new Set(); + let pending = await this.#tessar.registry.pending(this.realmURL.href); + // A leaf write can activate one new downstream owner per wave. Bound an + // acyclic chain by all owners, not only the initially dirty subset. + let maxWaves = pending.length + ? (await this.#tessar.registry.activeOwnerCount(this.realmURL.href)) + 1 + : 0; + try { + for (let wave = 0; pending.length && wave < maxWaves; wave++) { + this.#tessarTimings.tessarWaves++; + // No job resume seed: previous source-pass working rows have the same + // job id, but must never be re-promoted by a materialization wave. + this.#batch = await this.#indexWriter.createBatch( + this.realmURL, + this.#virtualNetwork, + ); + this.#tessarInputSnapshot = { + realmURL: this.realmURL.href, + generation: this.batch.currentGeneration - 1, + }; + this.#dependencyResolver.reset(); + this.#indexingInstances.clear(); + let succeeded = 0; + let failures: string[] = []; + for (let { ownerURL } of pending) { + let outcome = await this.#renderVisit(new URL(ownerURL)); + if ( + outcome.status !== 'rendered' || + outcome.result.card?.error || + !outcome.result.card?.serialized?.data.meta.tessar + ) { + failures.push(ownerURL); + continue; + } + await this.#finishVisit(outcome.result); + published.add(ownerURL); + succeeded++; + this.#tessarTimings.tessarOwnersRendered++; + } + if (!succeeded) + throw new Error( + `Tessar could not materialize pending owners: ${failures.join(', ')}`, + ); + let swapStartedAt = Date.now(); + await this.batch.done({ + tessar: this.#tessar, + tessarInputGeneration: this.#tessarInputSnapshot.generation, + }); + this.#tessarTimings.tessarSwapMs += Date.now() - swapStartedAt; + this.#tessarTimings.tessarWriteMs += this.batch.writeMs; + pending = await this.#tessar.registry.pending(this.realmURL.href); + } + if (pending.length) + throw new Error( + 'Tessar feeder graph did not converge; owners remain pending', + ); + return [...published].map((url) => new URL(url)); + } finally { + this.#tessarInputSnapshot = undefined; + this.#tessarTimings.tessarMaterializationMs += Date.now() - startedAt; + } + } + // Announce this pass's now-fixed invalidation set, tagged per URL: // genuine deletions (the URLs in `deletes`) as 'delete', everything else — // fan-out dependents are always re-renders — as 'update'. Only fires in @@ -659,6 +760,7 @@ export class IndexRunner { async #renderVisit(url: URL): Promise { try { let result = await renderFileForIndexing({ + tessarInputSnapshot: this.#tessarInputSnapshot, url, realmURL: this.#realmURL, ignoreMap: this.ignoreMap, diff --git a/packages/runtime-common/index-runner/visit-file.ts b/packages/runtime-common/index-runner/visit-file.ts index 4e594b528eb..e4e4df9ed3e 100644 --- a/packages/runtime-common/index-runner/visit-file.ts +++ b/packages/runtime-common/index-runner/visit-file.ts @@ -27,6 +27,7 @@ import { resolveFileDefCodeRef } from '../file-def-code-ref.ts'; import type { VirtualNetwork } from '../virtual-network.ts'; interface RenderFileForIndexingOptions { + tessarInputSnapshot?: import('../tessar-materialization.ts').TessarInputSnapshot; url: URL; realmURL: URL; ignoreMap: Map; @@ -127,6 +128,7 @@ interface RouteIndexVisitCallbacks { // render before this one's row writes land. Returns `undefined` when the // file is ignored or belongs to a different realm. export async function renderFileForIndexing({ + tessarInputSnapshot, url, realmURL, ignoreMap, @@ -235,6 +237,7 @@ export async function renderFileForIndexing({ } let visitArgs = { + ...(tessarInputSnapshot ? { tessarInputSnapshot } : {}), affinityType: 'realm' as const, affinityValue: realmURL.href, realm: realmURL.href, diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 6470da105e5..9af9784fd4c 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -1,4 +1,9 @@ import { flatten } from 'lodash-es'; +import { TessarIndexPublication } from './tessar-index-publication.ts'; +import { TessarQueryRegistry } from './tessar-query-registry.ts'; +import { IndexQueryEngine } from './index-query-engine.ts'; +import type { DefinitionLookup } from './definition-lookup.ts'; +import type { Querier } from './expression.ts'; import { flattenDeep } from 'lodash-es'; import { type CardResource, @@ -63,6 +68,16 @@ export class IndexWriter { this.#dbAdapter = dbAdapter; } + tessarPublication(definitions: DefinitionLookup, network: VirtualNetwork) { + return new TessarIndexPublication( + this.#dbAdapter, + new TessarQueryRegistry( + this.#dbAdapter, + new IndexQueryEngine(this.#dbAdapter, definitions, network), + ), + ); + } + async createBatch( realmURL: URL, virtualNetwork: VirtualNetwork, @@ -276,6 +291,7 @@ function prerenderedHtmlEntryFrom( const WRITE_BUFFER_FLUSH_THRESHOLD = 100; export class Batch { + #tessarTransaction: Querier | undefined; readonly ready: Promise; #invalidations = new Set(); #nodeResolvedInvalidations: string[] | undefined; @@ -1684,6 +1700,8 @@ export class Batch { } async done(opts?: { + tessar?: TessarIndexPublication; + tessarInputGeneration?: number; // Write the previous generation's realm_meta value at this batch's // generation instead of recomputing it from `boxel_index_working`. The // setup-phase failure recovery finalizes while the working table still @@ -1709,15 +1727,45 @@ export class Batch { await this.#query(['COMMIT']); return { totalIndexEntries: this.#invalidations.size }; } - await this.#query(['BEGIN']); - if (opts?.carryForwardRealmMeta) { - await this.carryForwardRealmMeta(); - } else { - await this.updateRealmMeta(); - } - await this.applyBatchUpdates(); - await this.pruneObsoleteEntries(); - await this.#query(['COMMIT']); + let tessarPrepared = await opts?.tessar?.prepare( + this.realmURL.href, + this.generation, + ); + // Source writes hold the realm's write lock while awaiting indexing. + // Publication needs its own namespace or that wait deadlocks the worker. + await this.#dbAdapter.withWriteLock( + `tessar:index:${this.realmURL.href}`, + async (tx) => { + // Pg pins this connection and owns BEGIN/COMMIT/ROLLBACK. SQLite's + // adapter has one connection, so explicitly bracket that fallback. + this.#tessarTransaction = tx; + if (!tx) await this.#query(['BEGIN']); + try { + if (opts?.tessar && tessarPrepared) { + await opts.tessar.commit(tx ?? ((expr) => this.#query(expr)), { + realmURL: this.realmURL.href, + generation: this.generation, + definitionRevision: this.loaderEpoch, + prepared: tessarPrepared, + inputGeneration: opts.tessarInputGeneration, + }); + } + if (opts?.carryForwardRealmMeta) { + await this.carryForwardRealmMeta(); + } else { + await this.updateRealmMeta(); + } + await this.applyBatchUpdates(); + await this.pruneObsoleteEntries(); + if (!tx) await this.#query(['COMMIT']); + } catch (error) { + if (!tx) await this.#query(['ROLLBACK']); + throw error; + } finally { + this.#tessarTransaction = undefined; + } + }, + ); let totalIndexEntries = await this.numberOfIndexEntries(); return { totalIndexEntries }; @@ -1767,7 +1815,9 @@ export class Batch { } #query(expression: Expression) { - return query(this.#dbAdapter, expression, coerceTypes); + return this.#tessarTransaction + ? this.#tessarTransaction(expression) + : query(this.#dbAdapter, expression, coerceTypes); } private async getProductionVersion( diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index cd604b448de..0c08f533951 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -932,6 +932,7 @@ export type VisitPass = (typeof VISIT_PASS_ORDER)[number]; export type PrerenderVisitType = 'index' | 'prerender-html'; export type PrerenderVisitArgs = { + tessarInputSnapshot?: import('./tessar-materialization.ts').TessarInputSnapshot; affinityType: AffinityType; affinityValue: string; realm: string; @@ -1996,3 +1997,8 @@ export { type BotCommandFilter, type BotCommandMatrixFilter, } from './bot-command.ts'; +export { + currentTessarInputSnapshot, + tessarSnapshotFields, + type TessarMaterialization, +} from './tessar-materialization.ts'; diff --git a/packages/runtime-common/realm-index-query-engine.ts b/packages/runtime-common/realm-index-query-engine.ts index d33acaac341..33f38c8bb84 100644 --- a/packages/runtime-common/realm-index-query-engine.ts +++ b/packages/runtime-common/realm-index-query-engine.ts @@ -1,4 +1,5 @@ import { isScopedCSSRequest } from './scoped-css.ts'; +import { tessarReadState } from './tessar-materialization.ts'; import { cloneDeep } from 'lodash-es'; import { SupportedMimeType, @@ -107,6 +108,7 @@ import { const RECURSING_DEPTH = 3; type Options = { + tessarInput?: boolean; loadLinks?: true; linkFields?: string[]; // When true, populateQueryFields will only use cached definitions from the @@ -257,6 +259,7 @@ export class RealmIndexQueryEngine { realm.virtualNetwork, ); this.#definitionLookup = definitionLookup; + this.#tessarDB = dbAdapter; this.#realm = realm; this.#fetch = fetch; } @@ -264,6 +267,32 @@ export class RealmIndexQueryEngine { private get realmURL() { return (this.#realmURL ??= new URL(this.#realm.url)); } + #tessarDB: DBAdapter; + + private async tessarReadResource( + resource: LooseCardResource, + realmURL: URL, + opts?: Options, + ) { + if (!resource.meta.tessar || !resource.id) return; + let url = + this.#realm.virtualNetwork + .toURL(resource.id) + .href.replace(/\.json$/, '') + '.json'; + resource.meta = { + ...resource.meta, + tessar: { + ...resource.meta.tessar, + state: await tessarReadState( + this.#tessarDB, + realmURL.href, + url, + resource.meta.tessar, + opts, + ), + }, + }; + } // The entry engine. Runs the parsed entry query — the // `item.` membership query against the SQL core, then the htmlQuery @@ -572,6 +601,9 @@ export class RealmIndexQueryEngine { id: cardUrl as RealmResourceIdentifier, links: { self: cardUrl }, }; + // This check also applies when assembly is omitted or fields are + // sparse; neither path may advertise a dirty snapshot as ready. + await this.tessarReadResource(item, this.realmURL, opts); if (fieldset.item.kind === 'sparse') { item = buildSparseItemResource(item, fieldset.item.fields); } else { @@ -768,7 +800,8 @@ export class RealmIndexQueryEngine { `bug: should never get here--search index doc is undefined`, ); } - if (opts?.loadLinks) { + await this.tessarReadResource(doc.data, this.realmURL, opts); + if (opts?.loadLinks && !doc.data.meta.tessar) { let included = await this.loadLinks( { realmURL: this.realmURL, @@ -1685,6 +1718,14 @@ export class RealmIndexQueryEngine { try { await Promise.all( layer.map(async ({ resource, applyLinkFields }) => { + if ((resource as LooseCardResource).meta.tessar) { + await this.tessarReadResource( + resource as LooseCardResource, + realmURL, + opts, + ); + return; + } let popOpts = applyLinkFields ? opts : opts?.linkFields @@ -1796,6 +1837,12 @@ export class RealmIndexQueryEngine { for (let entry of relationshipEntries(resource.relationships)) { let { relationship, key, fieldName } = entry; + if ( + (resource as LooseCardResource).meta.tessar?.queryFields.includes( + fieldName, + ) + ) + continue; if (processed.has(key)) { continue; } diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 9c2b0792721..b946e59621c 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -1,4 +1,9 @@ import { Deferred } from './deferred.ts'; +import { + assertTessarGeneration, + tessarRequestedGeneration, + TESSAR_INPUT_GENERATION_HEADER, +} from './tessar-materialization.ts'; import { resolveRangeHeader } from './http-range.ts'; import { awaitRealmIndexSettled, @@ -3480,11 +3485,43 @@ export class Realm { }); } if (this.#router.handles(request)) { + let tessarGeneration = tessarRequestedGeneration(request); + if (tessarGeneration !== undefined) { + if (!['GET', 'QUERY'].includes(request.method)) { + return badRequest({ + requestContext, + message: 'Tessar input revisions apply only to reads', + }); + } + await assertTessarGeneration( + this.#dbAdapter, + this.url, + tessarGeneration, + ); + let response = await this.#router.handle(request, requestContext); + await assertTessarGeneration( + this.#dbAdapter, + this.url, + tessarGeneration, + ); + response.headers.set('cache-control', 'no-store'); + return response; + } return this.#router.handle(request, requestContext); } else { return this.fallbackHandle(request, requestContext); } } catch (e) { + if ( + request.headers.has(TESSAR_INPUT_GENERATION_HEADER) && + e instanceof CardError + ) { + return systemError({ + requestContext, + status: e.status, + message: e.message, + }); + } if (e instanceof AuthenticationError) { return createResponse({ body: e.message, @@ -6489,7 +6526,10 @@ export class Realm { // under the old schema. Read endpoints serve the realm's canonical // indexed view; the small wait when indexing is genuinely pending is // the right tradeoff vs returning stale state. - let pending = this.incrementalIndexing(); + let tessarInput = request.headers.has(TESSAR_INPUT_GENERATION_HEADER); + // An indexer's revision-pinned input read must not wait on the job that + // issued it. Its before/after generation checks reject concurrent swaps. + let pending = tessarInput ? undefined : this.incrementalIndexing(); if (pending) { await pending; } @@ -6535,7 +6575,7 @@ export class Realm { // peek when the client sent a validator — otherwise we'd be // paying an extra DB round-trip on cache misses since // `cardDocument()` below does its own `instance()` lookup. - if (ifNoneMatch) { + if (ifNoneMatch && !tessarInput) { await this.getRealmInfo(); let realmInfoHash = this.getCachedRealmInfoHash(); let instanceEntry = await this.#realmIndexQueryEngine.instance(url, { @@ -6565,6 +6605,7 @@ export class Realm { if ( !this.hasForeignRealmDeps(instanceEntry.deps) && instanceEntry.type === 'instance' && + !instanceEntry.instance.meta.tessar && instanceEntry.indexedAt != null ) { let etag = buildCardJsonEtag( @@ -6601,8 +6642,10 @@ export class Realm { // read for the response ETag below reflects the post-assembly // realm info. let maybeError = await this.#realmIndexQueryEngine.cardDocument(url, { - loadLinks: true, - skipQueryBackedExpansion: isDuringPrerenderRequest(request), + tessarInput, + ...(tessarInput ? {} : { loadLinks: true as const }), + skipQueryBackedExpansion: + tessarInput || isDuringPrerenderRequest(request), }); if (maybeError === undefined) { if (await this.nonJsonFileExists(localPath)) { @@ -6712,19 +6755,20 @@ export class Realm { // could 304 a follow-up request whose `included[]` should have // been re-fetched from the foreign realm. let foreignDeps = this.hasForeignRealmDeps(maybeError.deps); - let responseEtag = foreignDeps - ? undefined - : buildCardJsonEtag( - maybeError.indexedAt, - this.getCachedRealmInfoHash(), - screenshotsEtagFingerprint(maybeError.screenshots), - ); + let responseEtag = + foreignDeps || card.data.meta.tessar + ? undefined + : buildCardJsonEtag( + maybeError.indexedAt, + this.getCachedRealmInfoHash(), + screenshotsEtagFingerprint(maybeError.screenshots), + ); return createResponse({ body: JSON.stringify(card, null, 2), init: { headers: { 'content-type': SupportedMimeType.CardJson, - 'cache-control': cacheControl, + 'cache-control': card.data.meta.tessar ? 'no-store' : cacheControl, ...(responseEtag ? { etag: responseEtag } : {}), ...etagSuppressedHeader(foreignDeps), ...lastModifiedHeader(card), @@ -7075,6 +7119,7 @@ export class Realm { ): Promise { let engineOpts = { loadLinks: true as const, + ...(opts?.tessarInput ? { tessarInput: true } : {}), ...(opts?.cacheOnlyDefinitions ? { cacheOnlyDefinitions: true } : {}), ...(opts?.omitIncluded ? { omitIncluded: true } : {}), // `!== undefined` so an explicit priority 0 (system-initiated) survives. @@ -7142,6 +7187,7 @@ export class Realm { } let runSearch = (signal?: AbortSignal) => this.searchEntries(searchEntryQuery, { + tessarInput: request.headers.has(TESSAR_INPUT_GENERATION_HEADER), cacheOnlyDefinitions: duringPrerender, // Inside a prerender the search skips the `loadLinks` // relationship-assembly pass entirely: the host re-resolves every diff --git a/packages/runtime-common/resource-types.ts b/packages/runtime-common/resource-types.ts index ef214538ded..c1eedbe3431 100644 --- a/packages/runtime-common/resource-types.ts +++ b/packages/runtime-common/resource-types.ts @@ -10,6 +10,7 @@ import type { import type { VirtualNetwork } from './virtual-network.ts'; import type { Query } from './query.ts'; import type { ScreenshotsMeta } from './capture-spec.ts'; +import type { TessarMaterialization } from './tessar-materialization.ts'; // Metadata for a query-based linksTo/linksToMany field on a FileDef subclass, // extracted during file prerendering so that file-meta responses can populate @@ -113,6 +114,7 @@ export type CardResourceMeta = Meta & { // data from stale. Additive — absent when the serialization did not come off // the index (e.g. a freshly-built resource that has not been persisted). generation?: number; + tessar?: TessarMaterialization; // The instance's declared-screenshot captures, joined at serve time from // the prerendered manifest (`prerendered_html.screenshots`) — never // persisted into the index or the source file, and stripped from incoming diff --git a/packages/runtime-common/search-utils.ts b/packages/runtime-common/search-utils.ts index 308831dd627..0bc7655fa29 100644 --- a/packages/runtime-common/search-utils.ts +++ b/packages/runtime-common/search-utils.ts @@ -106,6 +106,7 @@ export function resolveSearchRequestMethod(request: Request): string { // dropping a field here (e.g. priority) silently breaks the threading from // the realm-server handler down to searchCards. export type SearchOpts = { + tessarInput?: boolean; cacheOnlyDefinitions?: boolean; // Prerender searches set this so `searchCardsUncoalesced` skips the // `loadLinks` relationship-assembly pass entirely (the host re-resolves diff --git a/packages/runtime-common/tessar-index-publication.ts b/packages/runtime-common/tessar-index-publication.ts new file mode 100644 index 00000000000..52ed7485dd8 --- /dev/null +++ b/packages/runtime-common/tessar-index-publication.ts @@ -0,0 +1,227 @@ +import type { DBAdapter } from './db.ts'; +import type { BoxelIndexTable } from './index-structure.ts'; +import { coerceTypes } from './index-structure.ts'; +import { param, query, type Querier } from './expression.ts'; +import { assertTessarGeneration } from './tessar-materialization.ts'; +import type { + TessarQueryRegistry, + TessarDocument, + TessarPreparedWatch, +} from './tessar-query-registry.ts'; + +// IndexRunner prepares these outside the commit lock. The actual publication, +// watch replacement and dirty markers all use Batch.done's pinned transaction. +export class TessarIndexPublication { + private db: DBAdapter; + readonly registry: TessarQueryRegistry; + constructor(db: DBAdapter, registry: TessarQueryRegistry) { + this.db = db; + this.registry = registry; + } + + async prepare(realmURL: string, generation: number) { + let active = await query(this.db, [ + 'SELECT owner_url FROM tessar_owners WHERE realm_url =', + param(realmURL), + 'AND retired = FALSE', + ]); + let rows = (await query( + this.db, + [ + 'SELECT * FROM boxel_index_working WHERE realm_url =', + param(realmURL), + 'AND generation =', + param(generation), + ...(active.length + ? [] + : ["AND pristine_doc->'meta'->'tessar' IS NOT NULL"]), + ], + coerceTypes, + )) as unknown as BoxelIndexTable[]; + let watches = new Map(); + for (let row of rows) { + let manifest = row.pristine_doc?.meta.tessar; + if ( + row.type === 'instance' && + !row.has_error && + !row.is_deleted && + manifest + ) { + watches.set(row.url, await this.registry.prepare(manifest.watches)); + } + } + return { rows, watches, hadOwners: active.length > 0 }; + } + + async commit( + tx: Querier, + args: { + realmURL: string; + generation: number; + definitionRevision: string; + prepared: Awaited>; + inputGeneration?: number; + }, + ): Promise { + let { + realmURL, + generation, + definitionRevision, + prepared, + inputGeneration, + } = args; + if (!prepared.rows.length && !prepared.hadOwners) return; + await assertTessarGeneration(this.db, realmURL, generation - 1, tx); + if (inputGeneration !== undefined && inputGeneration !== generation - 1) { + throw new Error('Tessar publication does not follow its input revision'); + } + if (inputGeneration !== undefined) { + let [realm] = await tx([ + 'SELECT loader_epoch FROM realm_generations WHERE realm_url =', + param(realmURL), + ]); + if (realm?.loader_epoch !== definitionRevision) + throw new Error('Tessar definitions changed during computation'); + } + let changed = new Set(); + let dirty = new Set(); + if (inputGeneration === undefined) { + // The loader epoch is realm-wide. Even a module outside an owner's + // recorded dependencies can change it; every older snapshot must get + // work scheduled instead of remaining permanently pending at read time. + let outdated = await tx([ + 'SELECT owner_url FROM tessar_owners WHERE realm_url =', + param(realmURL), + 'AND retired = FALSE AND definition_revision <>', + param(definitionRevision), + ]); + for (let owner of outdated) dirty.add(owner.owner_url as string); + } + for (let row of prepared.rows) { + if (row.type !== 'instance') continue; + let [old] = await tx([ + 'SELECT * FROM boxel_index WHERE realm_url =', + param(realmURL), + 'AND url =', + param(row.url), + "AND type = 'instance'", + ]); + let previous = old as unknown as BoxelIndexTable | undefined; + // Exclude only provenance from equality: membership and output changes + // propagate to feeders; a recomputation with identical data does not. + if (tessarSemanticRow(previous) !== tessarSemanticRow(row)) { + changed.add(row.url); + for (let owner of await this.registry.affected( + realmURL, + tessarDocument(previous), + tessarDocument(row), + tx, + )) + dirty.add(owner); + } + let manifest = row.pristine_doc?.meta.tessar; + if (!row.has_error && !row.is_deleted && manifest) { + if ( + inputGeneration !== undefined && + manifest.inputGeneration !== inputGeneration + ) { + throw new Error( + 'Tessar renderer returned a different input revision', + ); + } + let ready = inputGeneration !== undefined; + if ( + !(await this.registry.publish(tx, { + realmURL, + ownerURL: row.url, + generation, + inputGeneration: inputGeneration ?? 0, + definitionRevision, + watches: prepared.watches.get(row.url)!, + pending: !ready, + })) + ) + throw new Error('Tessar rejected an obsolete owner publication'); + let resource = structuredClone(row.pristine_doc!); + resource.meta.tessar = { + ...manifest, + state: ready ? 'ready' : 'pending', + publishedGeneration: generation, + definitionRevision, + }; + await tx([ + 'UPDATE boxel_index_working SET pristine_doc =', + param(JSON.stringify(resource)), + 'WHERE realm_url =', + param(realmURL), + 'AND url =', + param(row.url), + "AND type = 'instance' AND generation =", + param(generation), + ]); + } else if (previous?.pristine_doc?.meta.tessar) { + if (row.is_deleted || !row.has_error) { + await this.registry.publish(tx, { + realmURL, + ownerURL: row.url, + generation, + inputGeneration: generation, + definitionRevision, + watches: [], + retired: true, + }); + } else dirty.add(row.url); + } + } + // Concrete dependencies supplement watches, including transitive inputs + // whose fields a getter consumed without appearing in a query predicate. + // Existing source passes already expand their dependency closure. This + // handles a newly published feeder's outputs in a Tessar follow-up wave. + if (changed.size) { + let owners = await tx([ + 'SELECT o.owner_url, i.deps FROM tessar_owners o JOIN boxel_index i ON i.realm_url = o.realm_url AND i.url = o.owner_url', + "AND i.type = 'instance' WHERE o.realm_url =", + param(realmURL), + 'AND o.retired = FALSE', + ]); + for (let owner of owners) { + let deps = ( + typeof owner.deps === 'string' ? JSON.parse(owner.deps) : owner.deps + ) as string[] | null; + if (deps?.some((dep) => changed.has(dep))) + dirty.add(owner.owner_url as string); + } + } + // A successful owner in this wave was computed from the previous revision. + // If another output it consumes changed in the wave, it must run again. + await this.registry.markDirty(tx, realmURL, [...dirty], generation); + // A retry after this commit must discover durable dirty owners, not resume + // and re-promote an already committed source pass under its old generation. + await tx([ + 'UPDATE boxel_index_working SET job_id = NULL WHERE realm_url =', + param(realmURL), + 'AND generation =', + param(generation), + ]); + } +} + +function tessarDocument( + row: BoxelIndexTable | undefined, +): TessarDocument | undefined { + return row && !row.is_deleted && row.search_doc + ? { url: row.url, types: row.types ?? [], search_doc: row.search_doc } + : undefined; +} + +function tessarSemanticRow(row: BoxelIndexTable | undefined): string { + if (!row || row.is_deleted) return 'deleted'; + let resource = row.pristine_doc ? structuredClone(row.pristine_doc) : null; + if (resource?.meta) delete resource.meta.tessar; + return JSON.stringify([ + Boolean(row.has_error), + resource, + row.search_doc, + row.types, + ]); +} diff --git a/packages/runtime-common/tessar-materialization.ts b/packages/runtime-common/tessar-materialization.ts new file mode 100644 index 00000000000..775ddc23bd8 --- /dev/null +++ b/packages/runtime-common/tessar-materialization.ts @@ -0,0 +1,159 @@ +import type { Query } from './query.ts'; +import type { DBAdapter } from './db.ts'; +import { query, param, type Querier } from './expression.ts'; +import { CardError } from './error.ts'; +import type { LooseCardResource } from './resource-types.ts'; +import { indexingConcurrencyGroup } from './jobs/indexing.ts'; + +// Versioned provenance for the ordinary attributes/relationship payload. The +// writer supplies revisions; realm source files must never supply this stamp. +export interface TessarMaterialization { + version: 1; + state: 'pending' | 'ready'; + computedFields: string[]; + queryFields: string[]; + watches: Array<{ fieldPath: string; query: Query }>; + inputGeneration: number; + publishedGeneration?: number; + definitionRevision?: string; +} + +export interface TessarInputSnapshot { + realmURL: string; + generation: number; +} + +export const TESSAR_INPUT_GENERATION_HEADER = 'x-boxel-tessar-input-generation'; + +export async function tessarHasMaterializations( + db: DBAdapter, + realms: string[], +): Promise { + for (let realm of realms) { + let rows = await query(db, [ + 'SELECT 1 FROM tessar_owners WHERE realm_url =', + param(realm), + 'AND retired = FALSE LIMIT 1', + ]); + if (rows.length) return true; + } + return false; +} + +export function tessarRequestedGeneration( + request: Request, +): number | undefined { + let value = request.headers.get(TESSAR_INPUT_GENERATION_HEADER); + if (value === null) return undefined; + let generation = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(generation)) { + throw new CardError('Tessar requires a nonnegative input generation', { + status: 400, + }); + } + return generation; +} + +export async function assertTessarGeneration( + db: DBAdapter, + realmURL: string, + expected: number, + tx?: Querier, +): Promise { + let execute: Querier = tx ?? ((expression) => query(db, expression)); + let [row] = await execute([ + 'SELECT current_generation FROM realm_generations WHERE realm_url =', + param(realmURL), + ]); + if (Number(row?.current_generation ?? 0) !== expected) { + throw new CardError( + 'Tessar input revision changed; recompute the complete owner', + { status: 409 }, + ); + } +} + +// A prerender visit sets and clears this explicitly. Ordinary readers and +// source edits never opt into the indexed-input protocol via this helper. +export function currentTessarInputSnapshot(): TessarInputSnapshot | undefined { + let globals = globalThis as unknown as { + __boxelRenderContext?: boolean; + __tessarInputSnapshot?: TessarInputSnapshot; + }; + return globals.__boxelRenderContext === true + ? globals.__tessarInputSnapshot + : undefined; +} + +export function tessarSnapshotFields(resource: LooseCardResource) { + let stamp = resource.meta.tessar; + if (!stamp) return undefined; + if ( + stamp.version !== 1 || + stamp.state !== 'ready' || + !Number.isSafeInteger(stamp.publishedGeneration) || + !Number.isSafeInteger(stamp.inputGeneration) || + stamp.inputGeneration < 0 || + stamp.publishedGeneration! <= stamp.inputGeneration || + !stamp.definitionRevision || + !Array.isArray(stamp.computedFields) || + !Array.isArray(stamp.queryFields) || + (resource.meta.generation !== undefined && + resource.meta.generation !== stamp.publishedGeneration) + ) { + throw new CardError( + 'Tessar materialization is pending or has invalid provenance', + { status: 409 }, + ); + } + return { + computedFields: stamp.computedFields, + queryFields: stamp.queryFields, + }; +} + +export async function tessarReadState( + db: DBAdapter, + realmURL: string, + ownerURL: string, + stamp: TessarMaterialization, + opts?: { tessarInput?: boolean }, +): Promise<'ready' | 'pending'> { + // Source endpoints durably enqueue before acknowledging. Across replicas, + // an unprocessed write is visible here even before its indexed old/new + // documents exist for precise reverse matching. A revision-pinned worker + // must ignore its own queue job while consuming already-ready feeders. + if (db.kind === 'pg' && !opts?.tessarInput) { + let pending = await query(db, [ + "SELECT 1 FROM jobs WHERE status = 'unfulfilled' AND concurrency_group =", + param(indexingConcurrencyGroup(realmURL)), + 'LIMIT 1', + ]); + if (pending.length) return 'pending'; + let [latest] = await query(db, [ + 'SELECT status FROM jobs WHERE concurrency_group =', + param(indexingConcurrencyGroup(realmURL)), + 'ORDER BY id DESC LIMIT 1', + ]); + if (latest?.status === 'rejected') + throw new CardError( + 'Tessar indexing failed; retry indexing before treating this view as current', + { status: 503 }, + ); + } + let [row] = await query(db, [ + 'SELECT o.published_generation, o.dirty_generation, o.retired, o.definition_revision, r.loader_epoch FROM tessar_owners o JOIN realm_generations r ON r.realm_url = o.realm_url WHERE o.realm_url =', + param(realmURL), + 'AND o.owner_url =', + param(ownerURL), + ]); + return row && + !row.retired && + row.dirty_generation == null && + Number(row.published_generation) === stamp.publishedGeneration && + row.definition_revision === stamp.definitionRevision && + row.loader_epoch === stamp.definitionRevision && + stamp.state === 'ready' + ? 'ready' + : 'pending'; +} diff --git a/packages/runtime-common/tessar-query-registry.ts b/packages/runtime-common/tessar-query-registry.ts index 302a84028c9..6ab1699a7f5 100644 --- a/packages/runtime-common/tessar-query-registry.ts +++ b/packages/runtime-common/tessar-query-registry.ts @@ -63,6 +63,7 @@ export class TessarQueryRegistry { definitionRevision: string; watches: TessarPreparedWatch[]; retired?: boolean; + pending?: boolean; }, ): Promise { let { realmURL, ownerURL, generation, inputGeneration } = owner; @@ -91,7 +92,9 @@ export class TessarQueryRegistry { ]); if ( (previous && Number(previous.published_generation) > generation) || - (previous?.dirty_generation != null && + (!owner.pending && + !owner.retired && + previous?.dirty_generation != null && Number(previous.dirty_generation) > inputGeneration) ) { return false; @@ -107,13 +110,15 @@ export class TessarQueryRegistry { param(generation), ',', param(inputGeneration), - ', NULL,', + ',', + param(owner.pending ? generation : null), + ',', param(owner.definitionRevision), ',', param(owner.retired ?? false), `) ON CONFLICT (realm_url, owner_url) DO UPDATE SET published_generation = EXCLUDED.published_generation, - input_generation = EXCLUDED.input_generation, dirty_generation = NULL, + input_generation = EXCLUDED.input_generation, dirty_generation = EXCLUDED.dirty_generation, definition_revision = EXCLUDED.definition_revision, retired = EXCLUDED.retired`, ]); @@ -161,7 +166,7 @@ export class TessarQueryRegistry { return true; } - async candidates(realmURL: string, document: TessarDocument) { + async candidates(realmURL: string, document: TessarDocument, tx?: Querier) { let routes = [ every([ ['t.path =', param('')], @@ -177,25 +182,26 @@ export class TessarQueryRegistry { ]), ); } - let rows = await query( - this.db, - [ - `SELECT DISTINCT w.owner_url, w.field_path, w.query + let expression: Expression = [ + `SELECT DISTINCT w.owner_url, w.field_path, w.query FROM tessar_query_terms t JOIN tessar_query_watches w ON w.realm_url = t.realm_url AND w.owner_url = t.owner_url AND w.field_path = t.field_path WHERE`, - ...(every([ - ['t.realm_url =', param(realmURL)], - any(routes), - ]) as Expression), - ], - { query: 'JSON' }, - ); + ...(every([ + ['t.realm_url =', param(realmURL)], + any(routes), + ]) as Expression), + ]; + let rows = tx + ? await tx(expression) + : await query(this.db, expression, { query: 'JSON' }); return rows.map((row) => ({ ownerURL: row.owner_url as string, fieldPath: row.field_path as string, - query: row.query as unknown as Query, + query: (typeof row.query === 'string' + ? JSON.parse(row.query) + : row.query) as Query, })); } @@ -203,11 +209,12 @@ export class TessarQueryRegistry { realmURL: string, oldDocument: TessarDocument | undefined, newDocument: TessarDocument | undefined, + tx?: Querier, ): Promise { let owners = new Set(); for (let document of [oldDocument, newDocument]) { if (!document) continue; - for (let watch of await this.candidates(realmURL, document)) { + for (let watch of await this.candidates(realmURL, document, tx)) { if (owners.has(watch.ownerURL)) continue; if ( await this.engine.tessarMatchesDocument(watch.query.filter, document) @@ -242,6 +249,15 @@ export class TessarQueryRegistry { } } + async activeOwnerCount(realmURL: string): Promise { + let [row] = await query(this.db, [ + 'SELECT COUNT(*) AS total FROM tessar_owners WHERE realm_url =', + param(realmURL), + 'AND retired = FALSE', + ]); + return Number(row.total); + } + async pending( realmURL: string, ): Promise> { diff --git a/packages/runtime-common/worker.ts b/packages/runtime-common/worker.ts index 92c7b7ae6aa..ebf1828d65d 100644 --- a/packages/runtime-common/worker.ts +++ b/packages/runtime-common/worker.ts @@ -76,6 +76,12 @@ export interface IndexPhaseTimings { // The final atomic swap: `batch.done()` (realm-meta update, working → main // promotion, obsolete-row prune) in one transaction. swapMs?: number; + // Tessar follow-up waves are separate from the source visit/swap timings. + tessarMaterializationMs?: number; + tessarWriteMs?: number; + tessarSwapMs?: number; + tessarWaves?: number; + tessarOwnersRendered?: number; } export interface StreamFileRef { diff --git a/scripts/tessar/README.md b/scripts/tessar/README.md index bf05df8c8e0..3bbddc560ea 100644 --- a/scripts/tessar/README.md +++ b/scripts/tessar/README.md @@ -15,6 +15,11 @@ directory must not exist. `manifest.json` records the seed, counts and data hash `expected.json` is the independent raw-document oracle. Production data is never an input. A 100x preset is deliberately unsupported. +`--variant materialized` is the default. `--variant get-cards` changes only the +dashboard definition; source inputs and expected displayed output stay the same. +The reference pages broad type queries in batches of 100 without a fixed number +of pages. It must pass the browser oracle before being used as a baseline. + Build the host and serve its icons using the package scripts. From `packages/realm-server`: @@ -28,13 +33,43 @@ as `.runtime.json`. The source fingerprint includes dirty and untracked runtime files so a cached fixture cannot silently stand in for the current implementation. Stop the harness with SIGINT/SIGTERM when the browser trial finishes. +Use a fully migrated template database containing the Tessar migration. If the +default harness template predates the migration, set +`TEST_HARNESS_MIGRATED_TEMPLATE_DB` to a separately prepared disposable template; +do not reset a database that an active harness is using. `--realm-server-url` +can pin a localhost URL so the host is built against the same base realm. +Set `TEST_HARNESS_PGPORT=55437` for the separate disk-backed Tessar Postgres +container. The 10× cold build requires +`TEST_HARNESS_FULL_INDEX_REALM_STARTUP_TIMEOUT_MS=3600000`; this is a setup +allowance, not a change to the fixed 10,000 ms write-freshness deadline. + +Once the harness is ready, run the real Chromium check from the monorepo root: + +```sh +mise exec -- pnpm exec node scripts/tessar/browser-check.mjs --dataset /private/tmp/tessar-smoke --runtime /private/tmp/tessar-smoke-result.json.runtime.json --matrix-url http://localhost:/ --output /private/tmp/tessar-smoke-browser.json --iterations 20 --writes +``` + +The script authenticates only the synthetic harness account. It compares all +statistics and displayed rows with the raw-document oracle, records input-card +and search requests, and exercises content, transitive, entry/exit, asynchronous, +unrelated, insertion, deletion and offline/reconnect mutations in the already-open browser. Each +write series needs a fresh generated realm: it deliberately changes the fixture. +The `get-cards` variant's write/freshness behavior is still being validated; +passing initial reads alone does not qualify a baseline for a freshness claim. + +`lifecycle-check.mjs --dataset ... --runtime ... --output ...` adds a synthetic +five-stage feeder chain after the size benchmark. It verifies an unrelated +module epoch change and leaf-to-dashboard convergence. Its additional records +must not be included in the fixed-size performance samples. + For host QUnit tests, build with `RESOLVED_BASE_REALM_URL` pointing at the printed local server's `/base/` and clear ambient `BOXEL_ENVIRONMENT`, `ENV_SLUG`, and `ENV_MODE` settings. Filter on `Tessar`. Capture the complete console output. After a base-module edit, invalidate that local realm's module cache or restart the isolated stack before claiming that a browser test covered the edit. -The initial ordinary-query GET case has failed correctness: live membership and -stored statistics disagree. The tool records those reads as invalid. It does -not treat their latency as a passing baseline. Full dashboard adaptation, -concurrency, write/freshness trials and the 10x performance report remain required. +The initial ordinary-query GET case failed correctness: live membership and +stored statistics disagreed. The tool records those reads as invalid. It does +not treat their latency as a passing baseline. Initial connected materialized +smoke reads and eight mutations now pass. Full dashboard adaptation, feeder and +restart tests, concurrency, and the 10x performance report remain required. diff --git a/scripts/tessar/bench.mjs b/scripts/tessar/bench.mjs index cb689dd9843..f41b85fa0dd 100644 --- a/scripts/tessar/bench.mjs +++ b/scripts/tessar/bench.mjs @@ -30,6 +30,7 @@ let { values } = parseArgs({ output: { type: 'string' }, iterations: { type: 'string', default: '5' }, serve: { type: 'boolean', default: false }, + 'realm-server-url': { type: 'string' }, }, }); if (!values.dataset || !values.output) @@ -73,7 +74,6 @@ let sourcePaths = execFileSync( 'packages/host', 'packages/postgres', 'packages/realm-server', - 'scripts/tessar', ], { cwd: repository, encoding: 'utf8' }, ) @@ -92,11 +92,23 @@ for (let path of [...new Set(sourcePaths)].sort()) { } } let runtimeHash = sourceHash.digest('hex'); -process.env.TEST_HARNESS_CACHE_SALT = `tessar:${commit}:${hostHash}:${runtimeHash}`; +// A tooling-only edit or commit must not force a fresh 12,550-record index. +// Runtime source and built-host contents still invalidate the template; the +// harness separately hashes every fixture module and synthetic source record. +process.env.TEST_HARNESS_CACHE_SALT = `tessar:${hostHash}:${runtimeHash}`; let started = performance.now(); let realm; +let realmServerURL = values['realm-server-url'] + ? new URL(values['realm-server-url']) + : undefined; +if ( + realmServerURL && + !['localhost', '127.0.0.1'].includes(realmServerURL.hostname) +) + throw new Error('Tessar benchmarks require a local realm server'); try { realm = await startFactoryRealmServer({ + realmServerURL, realms: [{ dir: join(dataset, 'realm'), path: 'tessar/' }], }); let startupMs = performance.now() - started; diff --git a/scripts/tessar/browser-check.mjs b/scripts/tessar/browser-check.mjs new file mode 100644 index 00000000000..8d16eb642dd --- /dev/null +++ b/scripts/tessar/browser-check.mjs @@ -0,0 +1,375 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { chromium } from '@playwright/test'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; +import { + readDisplay, + validateDisplay, + waitForDisplay, +} from './browser-oracle.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + 'matrix-url': { type: 'string' }, + output: { type: 'string' }, + iterations: { type: 'string', default: '5' }, + writes: { type: 'boolean', default: false }, + }, +}); +if ( + !values.dataset || + !values.runtime || + !values.output || + !values['matrix-url'] +) + throw new Error( + '--dataset, --runtime, --matrix-url and --output are required', + ); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const manifest = JSON.parse( + await readFile(join(resolve(values.dataset), 'manifest.json'), 'utf8'), +); +const realm = new URL(runtime.realmURL); +const matrix = new URL(values['matrix-url']); +for (const url of [realm, matrix, new URL(runtime.realmServerURL)]) + if (!['localhost', '127.0.0.1'].includes(url.hostname)) + throw new Error('Tessar browser checks require isolated local services'); +if (manifest.synthetic !== true || realm.pathname !== '/tessar/') + throw new Error('Only a synthetic Tessar fixture realm may be tested'); +const iterations = Number(values.iterations); +if (!Number.isSafeInteger(iterations) || iterations < 1) + throw new Error('Invalid iterations'); +const { records } = generate(manifest); +const ownerId = 'DaySummary/00000'; +const ownerURL = new URL(ownerId, realm).href; +const url = new URL(ownerURL); +url.searchParams.set( + 'operatorModeState', + JSON.stringify({ + aiAssistantOpen: false, + stacks: [[{ format: 'isolated', id: ownerURL }]], + submode: 'interact', + workspaceChooserOpened: false, + }), +); +process.env.TEST_HARNESS_BROWSER_MATRIX_URL = matrix.href; +const { buildBrowserState, installBrowserState } = + await import('../../packages/software-factory/tests/helpers/browser-auth.ts'); +const state = await buildBrowserState(realm.href, runtime.realmServerURL); +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext(); +await installBrowserState(context, state); +const page = await context.newPage(); +const reads = []; +const writes = []; +const browserErrors = []; +page.on('pageerror', (error) => browserErrors.push(error.message)); +let requests = []; +page.on('request', (request) => { + if (['fetch', 'xhr'].includes(request.resourceType())) + requests.push({ url: request.url(), method: request.method() }); +}); +const cdp = await context.newCDPSession(page); +await cdp.send('Performance.enable'); + +function inputRequests() { + return requests.filter( + (request) => + (request.url.startsWith(realm.href) && + /\/(Student|Staff|Slot|Observation|Report|Reference|Activity)\//.test( + request.url, + )) || + (request.url.startsWith(runtime.realmServerURL) && + /\/_(federated-)?search(?:[?/#]|$)/.test(request.url)), + ); +} + +try { + for (let iteration = 0; iteration < iterations; iteration++) { + requests = []; + const beforeMetrics = Object.fromEntries( + (await cdp.send('Performance.getMetrics')).metrics.map((m) => [ + m.name, + m.value, + ]), + ); + const started = performance.now(); + let result = { + iteration, + cache: iteration === 0 ? 'new-browser-context' : 'warm-reload', + valid: false, + }; + try { + if (iteration === 0) + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + else await page.reload({ waitUntil: 'domcontentloaded' }); + await page + .locator('[data-tessar-stat="scoreTotal"]') + .waitFor({ timeout: 30_000 }); + validateDisplay( + await readDisplay(page), + expectedSummary(records, ownerId), + ); + result.displayMs = performance.now() - started; + // Keep the readiness timestamp separate from the observation window; + // delayed eager work after the first render still counts as input work. + await page.waitForTimeout(250); + result.valid = true; + result.inputRequests = inputRequests(); + if (manifest.variant !== 'get-cards' && result.inputRequests.length) + throw new Error( + 'Tessar materialized display loaded input cards or searched', + ); + result.fetchRequests = requests.length; + result.metrics = Object.fromEntries( + (await cdp.send('Performance.getMetrics')).metrics + .filter((m) => + [ + 'JSHeapUsedSize', + 'Nodes', + 'TaskDuration', + 'ScriptDuration', + ].includes(m.name), + ) + .map((m) => [ + m.name, + ['TaskDuration', 'ScriptDuration'].includes(m.name) + ? m.value - (beforeMetrics[m.name] ?? 0) + : m.value, + ]), + ); + } catch (error) { + result.valid = false; + result.failure = error.message; + result.visibleFailure = (await page.locator('body').innerText()).slice( + 0, + 3000, + ); + } + reads.push(result); + console.log( + JSON.stringify({ + iteration, + valid: result.valid, + displayMs: result.displayMs, + failure: result.failure, + }), + ); + } + if (values.writes && reads.every((read) => read.valid)) { + const token = buildRealmToken(realm, new URL(runtime.realmServerURL)); + const trials = [ + { + id: 'Observation/00000', + patch: { score: 80 }, + name: 'matching content edit', + }, + { + id: 'Reference/00000', + patch: { label: 'Tessar reference updated' }, + name: 'transitive row label edit', + }, + { + id: 'Observation/00000', + patch: { day: '2026-01-13' }, + name: 'query exit', + }, + { + id: 'Observation/00000', + patch: { day: '2026-01-12' }, + name: 'query entry', + }, + { + id: 'Observation/00000', + patch: { score: 81 }, + name: 'asynchronous source write', + source: true, + }, + { + id: `Reference/${String(manifest.counts.Reference - 1).padStart(5, '0')}`, + patch: { label: 'Tessar unrelated reference updated' }, + name: 'unrelated write', + unrelated: true, + }, + { + id: 'Observation/99999', + patch: { + score: 5, + sequence: 99999, + label: 'Tessar inserted observation', + }, + name: 'matching insertion', + source: true, + insert: true, + }, + { id: 'Observation/99999', name: 'matching deletion', delete: true }, + { + id: 'Observation/00000', + patch: { score: 82 }, + name: 'reconnect after missed notification', + source: true, + offline: true, + }, + ]; + for (const trial of trials) { + if (trial.insert) + records.set( + trial.id, + structuredClone(records.get('Observation/00000')), + ); + const source = records.get(trial.id); + if (trial.delete) records.delete(trial.id); + else Object.assign(source.data.attributes, trial.patch); + const expected = expectedSummary(records, ownerId); + const previousOwner = await ( + await fetch(ownerURL, { + headers: { Accept: 'application/vnd.card+json' }, + }) + ).json(); + await page.evaluate(() => { + window.__tessarTransitions = []; + window.__tessarObserver?.disconnect(); + window.__tessarObserver = new MutationObserver(() => { + const state = document + .querySelector('[data-tessar-state]') + ?.getAttribute('data-tessar-state'); + const events = window.__tessarTransitions; + if (!events.length || events.at(-1).state !== state) + events.push({ + at: Date.now(), + state, + visibleStats: + document.querySelectorAll('[data-tessar-stat]').length, + }); + }); + window.__tessarObserver.observe(document.body, { + attributes: true, + childList: true, + characterData: true, + subtree: true, + }); + }); + requests = []; + const startedAt = Date.now(); + let result = { name: trial.name, startedAt, valid: false }; + try { + if (trial.offline) await context.setOffline(true); + const mime = trial.source + ? 'application/vnd.card+source' + : 'application/vnd.card+json'; + const response = await fetch( + new URL(trial.id + (trial.source ? '.json' : ''), realm), + { + method: trial.delete ? 'DELETE' : trial.source ? 'POST' : 'PATCH', + headers: { + Accept: mime, + 'Content-Type': mime, + Authorization: `Bearer ${token}`, + }, + body: trial.delete + ? undefined + : JSON.stringify( + trial.source + ? source + : { + data: { + type: 'card', + attributes: trial.patch, + meta: source.data.meta, + }, + }, + ), + signal: AbortSignal.timeout(30_000), + }, + ); + result.ackAt = Date.now(); + result.ackMs = result.ackAt - startedAt; + result.status = response.status; + if (!response.ok) + throw new Error( + `Write HTTP ${response.status}: ${await response.text()}`, + ); + await response.arrayBuffer(); + if (manifest.variant !== 'get-cards') { + const ownerResponse = await fetch(ownerURL, { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(10_000), + }); + const document = await ownerResponse.json(); + validateSummary(document.data.attributes, expected); + if (document.data.meta.tessar?.state !== 'ready') + throw new Error('Owner is pending'); + result.ownerRevision = document.data.meta.tessar.publishedGeneration; + if ( + trial.unrelated && + result.ownerRevision !== + previousOwner.data.meta.tessar.publishedGeneration + ) + throw new Error('An unrelated write reindexed the selected owner'); + } + if (trial.offline) { + await page.waitForTimeout(250); + await context.setOffline(false); + } + // Observe the already-open client; a reload would hide lost invalidation. + await waitForDisplay( + page, + expected, + Math.max(1, 10_000 - (Date.now() - result.ackAt)), + ); + result.observedAt = Date.now(); + result.ackToDisplayMs = result.observedAt - result.ackAt; + result.inputRequests = inputRequests(); + result.transitions = await page.evaluate( + () => window.__tessarTransitions, + ); + if ( + trial.source && + !result.transitions.some( + (event) => event.state === 'pending' && event.visibleStats === 0, + ) + ) + throw new Error( + 'The asynchronous write did not expose an explicit pending view', + ); + result.valid = + result.ackToDisplayMs <= 10_000 && + (manifest.variant === 'get-cards' || + result.inputRequests.length === 0); + } catch (error) { + result.failure = error.message; + } + writes.push(result); + if (!result.valid) break; + } + } +} finally { + await writeFile( + values.output, + JSON.stringify( + { version: 1, manifest, runtime, reads, writes, browserErrors }, + null, + 2, + ) + '\n', + ); + await browser.close(); +} +console.log( + JSON.stringify({ + output: values.output, + validReads: reads.filter((r) => r.valid).length, + totalReads: reads.length, + validWrites: writes.filter((w) => w.valid).length, + totalWrites: writes.length, + }), +); +if ( + reads.some((r) => !r.valid) || + writes.some((w) => !w.valid) || + browserErrors.length +) + process.exitCode = 1; diff --git a/scripts/tessar/browser-oracle.mjs b/scripts/tessar/browser-oracle.mjs new file mode 100644 index 00000000000..9319c115b2c --- /dev/null +++ b/scripts/tessar/browser-oracle.mjs @@ -0,0 +1,71 @@ +import { validateSummary } from './generate.mjs'; + +export async function readDisplay(page) { + return page.evaluate(() => ({ + stats: Object.fromEntries( + [...document.querySelectorAll('[data-tessar-stat]')].map((el) => [ + el.dataset.tessarStat, + Number(el.textContent), + ]), + ), + rows: [...document.querySelectorAll('[data-tessar-source]')].map((el) => ({ + sourceId: el.dataset.tessarSource, + cells: [...el.querySelectorAll('td')].map((td) => td.textContent.trim()), + })), + })); +} + +export function validateDisplay(actual, expected) { + const { rows, ...stats } = expected; + validateSummary(actual.stats, stats); + const expectedRows = rows.map((row) => ({ + sourceId: row.sourceId, + cells: [row.label, row.studentLabel, row.referenceLabel, row.status], + })); + if (JSON.stringify(actual.rows) !== JSON.stringify(expectedRows)) + throw new Error( + 'Tessar displayed row IDs, order or values differ from the oracle', + ); +} + +export async function waitForDisplay(page, expected, timeout = 10000) { + await page.waitForFunction( + (expected) => { + const stats = Object.fromEntries( + [...document.querySelectorAll('[data-tessar-stat]')].map((el) => [ + el.dataset.tessarStat, + Number(el.textContent), + ]), + ); + const { rows, ...expectedStats } = expected; + const actualRows = [ + ...document.querySelectorAll('[data-tessar-source]'), + ].map((el) => ({ + sourceId: el.dataset.tessarSource, + cells: [...el.querySelectorAll('td')].map((td) => + td.textContent.trim(), + ), + })); + return ( + Object.entries(expectedStats).every( + ([key, value]) => stats[key] === value, + ) && + JSON.stringify(actualRows) === + JSON.stringify( + rows.map((row) => ({ + sourceId: row.sourceId, + cells: [ + row.label, + row.studentLabel, + row.referenceLabel, + row.status, + ], + })), + ) + ); + }, + expected, + { timeout }, + ); + validateDisplay(await readDisplay(page), expected); +} diff --git a/scripts/tessar/generate.mjs b/scripts/tessar/generate.mjs index 236d013bcb2..257661b4ac8 100644 --- a/scripts/tessar/generate.mjs +++ b/scripts/tessar/generate.mjs @@ -25,6 +25,7 @@ export function generate({ preset = 'smoke', seed = 1729, shape = 'distributed', + variant = 'materialized', } = {}) { if (!Object.hasOwn(presets, preset)) throw new Error('Preset must be smoke, 1x or 10x'); @@ -32,6 +33,8 @@ export function generate({ throw new Error('Seed must be a nonnegative safe integer'); if (!['distributed', 'focused'].includes(shape)) throw new Error('Unknown workload shape'); + if (!['materialized', 'get-cards'].includes(variant)) + throw new Error('Variant must be materialized or get-cards'); let scale = presets[preset]; let sizes = Object.fromEntries( Object.entries(counts).map(([type, count]) => [ @@ -76,12 +79,17 @@ export function generate({ type: 'card', attributes, ...(relationships ? { relationships } : {}), - meta: { adoptsFrom: { module: moduleRef, name: type } }, + meta: { + adoptsFrom: + variant === 'get-cards' && type === 'DaySummary' + ? { module: '../get-cards', name: 'TessarGetCardsPage' } + : { module: moduleRef, name: type }, + }, }, }); } } - return { records, sizes, seed, preset, shape }; + return { records, sizes, seed, preset, shape, variant }; } // Reference implementation deliberately operates on raw fixture documents, @@ -148,6 +156,11 @@ export async function writeDataset(output, options) { new URL('./realm/tessar.gts', import.meta.url), join(realmDir, 'tessar.gts'), ); + if (dataset.variant === 'get-cards') + await copyFile( + new URL('./realm/get-cards.gts', import.meta.url), + join(realmDir, 'get-cards.gts'), + ); let hash = createHash('sha256'); for (let [id, document] of dataset.records) { let bytes = JSON.stringify(document) + '\n'; @@ -166,6 +179,7 @@ export async function writeDataset(output, options) { seed: dataset.seed, preset: dataset.preset, shape: dataset.shape, + variant: dataset.variant, instanceCount: dataset.records.size, counts: dataset.sizes, recordsSha256: hash.digest('hex'), @@ -191,6 +205,7 @@ if ( preset: { type: 'string', default: 'smoke' }, seed: { type: 'string', default: '1729' }, shape: { type: 'string', default: 'distributed' }, + variant: { type: 'string', default: 'materialized' }, }, }); if (!values.output) throw new Error('--output must name a new directory'); @@ -200,6 +215,7 @@ if ( preset: values.preset, seed: Number(values.seed), shape: values.shape, + variant: values.variant, }), ), ); diff --git a/scripts/tessar/generate.test.mjs b/scripts/tessar/generate.test.mjs index 3ebd660ea5f..ff8f076d5e3 100644 --- a/scripts/tessar/generate.test.mjs +++ b/scripts/tessar/generate.test.mjs @@ -51,3 +51,21 @@ test('Tessar raw-data oracle reflects membership, content, transitive edits and /mismatch/, ); }); + +test('Tessar getCards and materialized variants have identical source inputs and expected outputs', () => { + let candidate = generate({ preset: '10x' }).records; + let baseline = generate({ preset: '10x', variant: 'get-cards' }).records; + assert.equal(baseline.size, 12550); + for (let [id, document] of candidate) { + if (id.startsWith('DaySummary/')) { + assert.deepEqual( + expectedSummary(baseline, id), + expectedSummary(candidate, id), + ); + assert.equal( + baseline.get(id).data.meta.adoptsFrom.name, + 'TessarGetCardsPage', + ); + } else assert.deepEqual(baseline.get(id), document); + } +}); diff --git a/scripts/tessar/lifecycle-check.mjs b/scripts/tessar/lifecycle-check.mjs new file mode 100644 index 00000000000..54e6618fbb9 --- /dev/null +++ b/scripts/tessar/lifecycle-check.mjs @@ -0,0 +1,149 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary } from './generate.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + output: { type: 'string' }, + }, +}); +if (!values.dataset || !values.runtime || !values.output) + throw new Error('--dataset, --runtime and --output are required'); +const manifest = JSON.parse( + await readFile(join(values.dataset, 'manifest.json'), 'utf8'), +); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const realm = new URL(runtime.realmURL), + server = new URL(runtime.realmServerURL); +if ( + manifest.synthetic !== true || + realm.pathname !== '/tessar/' || + ![realm, server].every((url) => + ['localhost', '127.0.0.1'].includes(url.hostname), + ) +) + throw new Error( + 'Tessar lifecycle checks require isolated synthetic services', + ); +const token = buildRealmToken(realm, server); +const { records } = generate(manifest); +const results = []; +async function post(path, bytes) { + const response = await fetch(new URL(path, realm), { + method: 'POST', + headers: { + Accept: 'application/vnd.card+source', + 'Content-Type': 'application/vnd.card+source', + Authorization: `Bearer ${token}`, + }, + body: bytes, + signal: AbortSignal.timeout(60000), + }); + await response.arrayBuffer(); + if (!response.ok) throw new Error(`${path}: HTTP ${response.status}`); +} +async function awaitScore(id, expected, deadline) { + let last; + while (Date.now() < deadline) { + try { + const response = await fetch(new URL(id, realm), { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())), + }); + last = await response.json(); + if ( + response.ok && + last.data?.meta?.tessar?.state === 'ready' && + last.data.attributes.scoreTotal === expected + ) + return { + id, + revision: last.data.meta.tessar.publishedGeneration, + scoreTotal: expected, + }; + } catch (error) { + last = { error: error.message }; + } + await delay(100); + } + throw new Error( + `Tessar lifecycle deadline exceeded for ${id}: ${JSON.stringify(last)}`, + ); +} + +try { + const source = structuredClone(records.get('Observation/00000')); + await post('Observation/00000.json', JSON.stringify(source)); + const expected = expectedSummary(records, 'DaySummary/00000').scoreTotal; + await awaitScore('DaySummary/00000', expected, Date.now() + 60000); + await post( + 'feeder-chain.gts', + await readFile( + new URL('./realm/feeder-chain.gts', import.meta.url), + 'utf8', + ), + ); + // A new unrelated module changes the loader epoch. Existing views must + // return to ready even though none depended on this new module beforehand. + const epochAck = Date.now(); + await awaitScore('DaySummary/00000', expected, epochAck + 10000); + results.push({ + name: 'unrelated module epoch', + valid: true, + ackToReadyMs: Date.now() - epochAck, + }); + const owner = records.get('DaySummary/00000').data.attributes; + for (let stage = 0; stage < 5; stage++) + await post( + `TessarStage${stage}/00000.json`, + JSON.stringify({ + data: { + type: 'card', + attributes: { classroomKey: owner.classroomKey, day: owner.day }, + meta: { + adoptsFrom: { + module: '../feeder-chain', + name: `TessarStage${stage}`, + }, + }, + }, + }), + ); + const setupDeadline = Date.now() + 60000; + await awaitScore('TessarStage4/00000', expected, setupDeadline); + source.data.attributes.score += 9; + const start = Date.now(); + await post('Observation/00000.json', JSON.stringify(source)); + const ack = Date.now(); + const snapshots = []; + for (const id of [ + 'DaySummary/00000', + ...Array.from({ length: 5 }, (_, i) => `TessarStage${i}/00000`), + ]) + snapshots.push(await awaitScore(id, expected + 9, ack + 10000)); + results.push({ + name: 'five downstream feeder stages', + valid: true, + ackMs: ack - start, + ackToReadyMs: Date.now() - ack, + snapshots, + }); +} catch (error) { + results.push({ + name: 'lifecycle failure', + valid: false, + error: error.message, + }); + process.exitCode = 1; +} finally { + await writeFile( + values.output, + JSON.stringify({ version: 1, manifest, results }, null, 2) + '\n', + ); + console.log(JSON.stringify(results)); +} diff --git a/scripts/tessar/load-check.mjs b/scripts/tessar/load-check.mjs new file mode 100644 index 00000000000..9c3171ab265 --- /dev/null +++ b/scripts/tessar/load-check.mjs @@ -0,0 +1,288 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { chromium } from '@playwright/test'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; +import { waitForDisplay } from './browser-oracle.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + output: { type: 'string' }, + 'matrix-url': { type: 'string' }, + readers: { type: 'string', default: '1,10,50' }, + 'duration-ms': { type: 'string', default: '10000' }, + writes: { type: 'boolean', default: false }, + }, +}); +if ( + !values.dataset || + !values.runtime || + !values.output || + !values['matrix-url'] +) + throw new Error( + '--dataset, --runtime, --output and --matrix-url are required', + ); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const manifest = JSON.parse( + await readFile(join(values.dataset, 'manifest.json'), 'utf8'), +); +const realm = new URL(runtime.realmURL), + server = new URL(runtime.realmServerURL), + matrix = new URL(values['matrix-url']); +for (const url of [realm, server, matrix]) + if (!['localhost', '127.0.0.1'].includes(url.hostname)) + throw new Error('Tessar load checks require local services'); +if ( + realm.pathname !== '/tessar/' || + manifest.synthetic !== true || + manifest.variant === 'get-cards' +) + throw new Error('A synthetic materialized Tessar dataset is required'); +const readers = values.readers.split(',').map(Number), + durationMs = Number(values['duration-ms']); +if ( + readers.some((n) => !Number.isSafeInteger(n) || n < 1 || n > 50) || + !Number.isSafeInteger(durationMs) || + durationMs < 1000 || + durationMs > 60000 +) + throw new Error('Readers must be 1–50 and duration 1000–60000 ms'); +const { records } = generate(manifest), + ownerId = 'DaySummary/00000', + ownerURL = new URL(ownerId, realm).href; +const token = buildRealmToken(realm, server); +const cardHeaders = { Accept: 'application/vnd.card+json' }; +const cases = []; + +async function sourceWrite(id, data) { + const response = await fetch(new URL(`${id}.json`, realm), { + method: 'POST', + headers: { + Accept: 'application/vnd.card+source', + 'Content-Type': 'application/vnd.card+source', + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify(data), + signal: AbortSignal.timeout(30000), + }); + if (!response.ok) + throw new Error(`Tessar source write failed: ${response.status}`); + await response.arrayBuffer(); +} + +// Browser-check mutations are known synthetic records. Restore them before +// each load case; setup costs are separate from the measured workload. +async function reset() { + for (const id of [ + 'Observation/00000', + 'Reference/00000', + `Reference/${String(manifest.counts.Reference - 1).padStart(5, '0')}`, + ]) + await sourceWrite(id, records.get(id)); + const response = await fetch(ownerURL, { + headers: cardHeaders, + signal: AbortSignal.timeout(60000), + }); + const doc = await response.json(); + validateSummary(doc.data.attributes, expectedSummary(records, ownerId)); + if (doc.data.meta.tessar?.state !== 'ready') + throw new Error('Tessar setup did not settle'); +} + +process.env.TEST_HARNESS_BROWSER_MATRIX_URL = matrix.href; +const { buildBrowserState, installBrowserState } = + await import('../../packages/software-factory/tests/helpers/browser-auth.ts'); +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext(); +await installBrowserState( + context, + await buildBrowserState(realm.href, server.href), +); +const page = await context.newPage(); +const url = new URL(ownerURL); +url.searchParams.set( + 'operatorModeState', + JSON.stringify({ + aiAssistantOpen: false, + stacks: [[{ format: 'isolated', id: ownerURL }]], + submode: 'interact', + workspaceChooserOpened: false, + }), +); + +function distribution(values) { + values.sort((a, b) => a - b); + const quantile = (q) => values[Math.max(0, Math.ceil(values.length * q) - 1)]; + return { + n: values.length, + min: values[0], + median: quantile(0.5), + p95: quantile(0.95), + max: values.at(-1), + }; +} + +try { + for (const concurrency of readers) { + await reset(); + const expected = expectedSummary(records, ownerId), + originalScore = records.get('Observation/00000').data.attributes.score; + const otherScores = expected.scoreTotal - originalScore; + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + await waitForDisplay(page, expected, 30000); + const startedAt = Date.now(), + until = startedAt + durationMs; + const result = { + readers: concurrency, + durationMs, + startedAt, + readyReads: 0, + pendingReads: 0, + failures: [], + writes: [], + latenciesMs: [], + responseBytes: 0, + browserInputRequests: 0, + }; + const onRequest = (request) => { + if ( + (request.url().startsWith(realm.href) && + /\/(Student|Staff|Slot|Observation|Report|Reference|Activity)\//.test( + request.url(), + )) || + /\/_(federated-)?search(?:[?/#]|$)/.test(request.url()) + ) + result.browserInputRequests++; + }; + page.on('request', onRequest); + let publishedVersion = 0; + const lastAcknowledgedVersion = (at) => + result.writes.filter((w) => w.ackAt !== undefined && w.ackAt <= at).at(-1) + ?.version ?? 0; + async function reader() { + while (Date.now() < until) { + const readAt = Date.now(), + start = performance.now(), + minimumVersion = lastAcknowledgedVersion(readAt); + try { + const response = await fetch(ownerURL, { + headers: cardHeaders, + signal: AbortSignal.timeout(10000), + }); + const text = await response.text(); + result.responseBytes += Buffer.byteLength(text); + if (!response.ok) throw new Error(`Read HTTP ${response.status}`); + const doc = JSON.parse(text); + if (doc.data.meta.tessar?.state === 'pending') { + result.pendingReads++; + continue; + } + if (doc.data.meta.tessar?.state !== 'ready') + throw new Error('Missing materialization'); + const score = doc.data.attributes.scoreTotal - otherScores; + const version = score === originalScore ? 0 : score - 500; + if ( + !Number.isSafeInteger(version) || + version < minimumVersion || + version > publishedVersion + ) + throw new Error( + `Stale or unknown score version ${version}; minimum ${minimumVersion}`, + ); + validateSummary(doc.data.attributes, { + ...expected, + scoreTotal: otherScores + score, + }); + result.readyReads++; + result.latenciesMs.push(performance.now() - start); + } catch (error) { + result.failures.push({ + at: Date.now(), + kind: 'read', + message: error.message, + }); + } + } + } + async function writer() { + if (!values.writes) return; + while (Date.now() + 1500 < until) { + await delay(1000); + const version = ++publishedVersion, + startedAt = Date.now(); + const write = { version, startedAt, valid: false }; + result.writes.push(write); + try { + const source = structuredClone(records.get('Observation/00000')); + source.data.attributes.score = 500 + version; + await sourceWrite('Observation/00000', source); + write.ackAt = Date.now(); + write.ackMs = write.ackAt - startedAt; + await waitForDisplay(page, { + ...expected, + scoreTotal: otherScores + 500 + version, + }); + write.observedAt = Date.now(); + write.ackToDisplayMs = write.observedAt - write.ackAt; + write.valid = write.ackToDisplayMs <= 10000; + } catch (error) { + result.failures.push({ + at: Date.now(), + kind: 'write', + message: error.message, + }); + } + } + } + await Promise.all([ + ...Array.from({ length: concurrency }, reader), + writer(), + ]); + page.off('request', onRequest); + result.elapsedMs = Date.now() - startedAt; + result.latency = distribution([...result.latenciesMs]); + result.readyReadsPerSecond = result.readyReads / (result.elapsedMs / 1000); + result.valid = + result.failures.length === 0 && + result.writes.every((w) => w.valid) && + result.browserInputRequests === 0; + cases.push(result); + console.log( + JSON.stringify({ + readers: concurrency, + valid: result.valid, + readyReads: result.readyReads, + pendingReads: result.pendingReads, + throughput: result.readyReadsPerSecond, + latency: result.latency, + writes: result.writes.map(({ ackMs, ackToDisplayMs, valid }) => ({ + ackMs, + ackToDisplayMs, + valid, + })), + failures: result.failures.slice(0, 3), + }), + ); + } +} finally { + await writeFile( + values.output, + JSON.stringify( + { + version: 1, + manifest, + note: 'Concurrent HTTP readers plus one already-open Chromium dashboard; this does not simulate the client CPU of 50 browsers.', + cases, + }, + null, + 2, + ) + '\n', + ); + await browser.close(); +} +if (cases.some((result) => !result.valid)) process.exitCode = 1; diff --git a/scripts/tessar/realm/feeder-chain.gts b/scripts/tessar/realm/feeder-chain.gts new file mode 100644 index 00000000000..d7db13afaad --- /dev/null +++ b/scripts/tessar/realm/feeder-chain.gts @@ -0,0 +1,60 @@ +import { + Component, + field, + contains, + linksToMany, + NumberField, +} from '@cardstack/base/card-api'; +import { TessarRecord, DaySummary } from './tessar'; + +const tessarPending = (state: string) => state === 'pending'; + +// A deliberately long acyclic chain: one initially dirty leaf is insufficient +// to bound the number of waves needed to refresh all downstream consumers. +class TessarFeederOutput extends TessarRecord { + static tessarMaterialized = true; + + @field scoreTotal = contains(NumberField, { + computeVia: function (this: TessarFeederOutput) { + return ((this as any).inputs ?? []).reduce( + (sum: number, input: any) => sum + input.scoreTotal, + 0, + ); + }, + }); + + static isolated = class extends Component { + + }; +} + +const query = { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, +}; + +export class TessarStage0 extends TessarFeederOutput { + @field inputs = linksToMany(DaySummary, { query }); +} +export class TessarStage1 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage0, { query }); +} +export class TessarStage2 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage1, { query }); +} +export class TessarStage3 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage2, { query }); +} +export class TessarStage4 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage3, { query }); +} diff --git a/scripts/tessar/realm/get-cards.gts b/scripts/tessar/realm/get-cards.gts new file mode 100644 index 00000000000..96f2c7054a8 --- /dev/null +++ b/scripts/tessar/realm/get-cards.gts @@ -0,0 +1,233 @@ +import { Component } from '@cardstack/base/card-api'; +import { tracked } from '@glimmer/tracking'; +import { associateDestroyableChild, destroy } from '@ember/destroyable'; +import { TessarRecord } from './tessar'; + +// Correct, explicitly paged read-time reference for the same synthetic output. +// It intentionally follows the dashboard's broad type-search/getCards pattern. +// There is no fixed page-count cutoff: the entire result must be available. +export class TessarGetCardsPage extends TessarRecord { + static isolated = class extends Component { + @tracked pages: Record = {}; + @tracked loading = true; + @tracked error = ''; + @tracked freshnessPending = false; + private resourceOwner?: object; + private loadRevision = 0; + private unsubscribe?: () => void; + private offline = () => { + this.freshnessPending = true; + }; + private online = () => { + this.load(); + }; + + constructor(owner: unknown, args: any) { + super(owner, args); + if (!(globalThis as any).__boxelRenderContext) { + // The incumbent's resource loading flag does not expose the interval + // between an async source acknowledgement and index notification. + // The corrected reference uses the host event bus for that interval, + // and re-queries complete pages on publication/reconnect. + this.unsubscribe = ( + globalThis as any + )._CARDSTACK_REALM_SUBSCRIBE?.subscribe( + new URL('./', import.meta.url).href, + (event: any) => { + if (event.eventName === 'update') this.freshnessPending = true; + if (event.eventName === 'index') this.load(); + }, + ); + window.addEventListener('offline', this.offline); + window.addEventListener('online', this.online); + this.load(); + } + } + + willDestroy() { + this.loadRevision++; + this.unsubscribe?.(); + window.removeEventListener('offline', this.offline); + window.removeEventListener('online', this.online); + super.willDestroy(); + } + + async load() { + const revision = ++this.loadRevision; + this.loading = true; + this.error = ''; + const resourceOwner = {}; + associateDestroyableChild(this, resourceOwner); + try { + let context = (this.args as any).context; + let getCards = context?.getCards ?? context?.actions?.getCards; + if (!getCards) throw new Error('Tessar getCards context is missing'); + let realm = new URL('./', import.meta.url).href; + let module = new URL('./tessar', import.meta.url).href; + let pages: Record = {}; + for (let name of ['Student', 'Slot', 'Observation', 'Report']) { + pages[name] = []; + for (let number = 0; ; number++) { + let resource = getCards( + resourceOwner, + () => ({ + filter: { type: { module, name } }, + page: { size: 100, number }, + }), + () => [realm], + ); + let deadline = Date.now() + 30_000; + // Resources begin their asynchronous task on the following turn. + await new Promise((resolve) => setTimeout(resolve, 0)); + while (resource.isLoading) { + if (revision !== this.loadRevision) return; + if (Date.now() >= deadline) + throw new Error('Tessar getCards page timed out'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + if (resource.errors?.length || resource.queryErrors?.length) + throw new Error('Tessar getCards page failed'); + if (!Array.isArray(resource.instances)) + throw new Error('Tessar getCards page has no instances'); + pages[name].push(resource); + if (resource.instances.length < 100) break; + } + } + if (revision !== this.loadRevision) return; + const previousOwner = this.resourceOwner; + this.resourceOwner = resourceOwner; + this.pages = pages; + this.freshnessPending = false; + if (previousOwner) destroy(previousOwner); + } catch (error: any) { + if (revision === this.loadRevision) this.error = error.message; + } finally { + if (resourceOwner !== this.resourceOwner) destroy(resourceOwner); + if (revision === this.loadRevision) this.loading = false; + } + } + + get pending() { + return ( + this.loading || + this.freshnessPending || + Object.values(this.pages).some((pages) => + pages.some((resource) => resource.isLoading), + ) + ); + } + + get tessarState() { + return this.error ? 'error' : this.pending ? 'pending' : 'ready'; + } + + matching(type: string, day = true): any[] { + let owner = this.args.model; + return (this.pages[type] ?? []) + .flatMap((resource) => resource.instances) + .filter( + (item) => + item.classroomKey === owner.classroomKey && + (!day || item.day === owner.day), + ); + } + + get stats() { + let observations = this.matching('Observation'); + let reports = this.matching('Report'); + return [ + { + key: 'studentCount', + label: 'Students', + value: this.matching('Student', false).length, + }, + { + key: 'slotCount', + label: 'Sessions', + value: this.matching('Slot').length, + }, + { + key: 'observationCount', + label: 'Observations', + value: observations.length, + }, + { key: 'reportCount', label: 'Reports', value: reports.length }, + { + key: 'readyReportCount', + label: 'Ready reports', + value: reports.filter((item) => item.status === 'ready').length, + }, + { + key: 'scoreTotal', + label: 'Score total', + value: observations.reduce((sum, item) => sum + item.score, 0), + }, + ]; + } + + get rows() { + return this.matching('Slot') + .sort((a, b) => a.sequence - b.sequence) + .map((slot) => ({ + sourceId: `Slot/${String(slot.sequence).padStart(5, '0')}`, + label: slot.label, + studentLabel: slot.student?.label, + referenceLabel: slot.reference?.label, + status: slot.status, + })); + } + + + }; +} diff --git a/scripts/tessar/realm/tessar.gts b/scripts/tessar/realm/tessar.gts index 94758026837..1409f811747 100644 --- a/scripts/tessar/realm/tessar.gts +++ b/scripts/tessar/realm/tessar.gts @@ -11,6 +11,8 @@ import { NumberField, } from '@cardstack/base/card-api'; +const tessarPending = (state: string) => state === 'pending'; + export class TessarRecord extends CardDef { @field label = contains(StringField); @field classroomKey = contains(StringField); @@ -43,6 +45,7 @@ export class TessarRow extends FieldDef { } export class DaySummary extends TessarRecord { + static tessarMaterialized = true; @field students = linksToMany(Student, { query: { filter: { eq: { classroomKey: '$this.classroomKey' } }, @@ -120,41 +123,48 @@ export class DaySummary extends TessarRecord { static isolated = class extends Component {