From 8285dace3b294edcbda1a92cc6bbf004f1f3de25 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:48:28 -0400 Subject: [PATCH 01/78] llm-benchmark: add normalized table evals --- .../answers/csharp.cs | 20 ++ .../answers/rust.rs | 22 ++ .../answers/typescript.ts | 24 ++ .../t_047_normalized_collection/spec.rs | 8 + .../tasks/csharp.txt | 13 + .../tasks/rust.txt | 13 + .../tasks/typescript.txt | 13 + .../answers/csharp.cs | 22 ++ .../t_048_heartbeat_isolation/answers/rust.rs | 22 ++ .../answers/typescript.ts | 26 ++ .../tables/t_048_heartbeat_isolation/spec.rs | 8 + .../tasks/csharp.txt | 15 ++ .../t_048_heartbeat_isolation/tasks/rust.txt | 15 ++ .../tasks/typescript.txt | 15 ++ .../t_050_normalized_schema/answers/csharp.cs | 29 +++ .../t_050_normalized_schema/answers/rust.rs | 35 +++ .../answers/typescript.ts | 34 +++ .../tables/t_050_normalized_schema/spec.rs | 8 + .../t_050_normalized_schema/tasks/csharp.txt | 18 ++ .../t_050_normalized_schema/tasks/rust.txt | 18 ++ .../tasks/typescript.txt | 18 ++ tools/xtask-llm-benchmark/src/eval/scorers.rs | 234 +++++++++++++++++- .../src/generated/registry.rs | 21 ++ 23 files changed, 648 insertions(+), 3 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs new file mode 100644 index 00000000000..2a89d17f3ce --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "CollectionOwner", Public = true)] + public partial struct CollectionOwner + { + [PrimaryKey, AutoInc] public ulong Id; + public string Name; + } + + [Table(Accessor = "ChildItem", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_owner", Columns = new[] { nameof(OwnerId) })] + public partial struct ChildItem + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong OwnerId; + public string Value; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs new file mode 100644 index 00000000000..97a2b33446c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs @@ -0,0 +1,22 @@ +use spacetimedb::table; + +#[table(accessor = collection_owner, public)] +pub struct CollectionOwner { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table( + accessor = child_item, + public, + index(accessor = by_owner, btree(columns = [owner_id])) +)] +pub struct ChildItem { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner_id: u64, + pub value: String, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts new file mode 100644 index 00000000000..c3933734b99 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts @@ -0,0 +1,24 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const collectionOwner = table( + { name: 'collection_owner', public: true }, + { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + } +); + +const childItem = table( + { + name: 'child_item', + public: true, + indexes: [{ accessor: 'byOwner', algorithm: 'btree', columns: ['ownerId'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + ownerId: t.u64(), + value: t.string(), + } +); + +export default schema({ collectionOwner, childItem }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt new file mode 100644 index 00000000000..4d2038b695a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in C# that models an unbounded child collection with normalized tables. + +TABLES +- CollectionOwner (public, accessor CollectionOwner) + - Id: ulong (primary key, auto increment) + - Name: string +- ChildItem (public, accessor ChildItem) + - Id: ulong (primary key, auto increment) + - OwnerId: ulong + - Value: string + - index by_owner: btree(OwnerId) + +Do not store child items in an array on CollectionOwner. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt new file mode 100644 index 00000000000..91a4b60a0b2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in Rust that models an unbounded child collection with normalized tables. + +TABLES +- collection_owner (public), struct CollectionOwner + - id: u64 (primary key, auto_inc) + - name: String +- child_item (public), struct ChildItem + - id: u64 (primary key, auto_inc) + - owner_id: u64 + - value: String + - index by_owner: btree(owner_id) + +Do not store child items in an array on CollectionOwner. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt new file mode 100644 index 00000000000..2ef2b0935a8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in TypeScript that models an unbounded child collection with normalized tables. + +TABLES +- collection_owner (public) + - id: u64 (primary key, auto increment) + - name: string +- child_item (public) + - id: u64 (primary key, auto increment) + - ownerId: u64 + - value: string + - index byOwner: btree(ownerId) + +Do not store child items in an array on collection_owner. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs new file mode 100644 index 00000000000..98978397b1f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs @@ -0,0 +1,22 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "UserProfile", Public = true)] + public partial struct UserProfile + { + [PrimaryKey] public Identity Identity; + public string DisplayName; + public Timestamp CreatedAt; + } + + [Table(Accessor = "ConnectionPresence", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_user", Columns = new[] { nameof(UserIdentity) })] + public partial struct ConnectionPresence + { + [PrimaryKey] public ConnectionId ConnectionId; + public Identity UserIdentity; + public Timestamp LastHeartbeat; + public bool Online; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs new file mode 100644 index 00000000000..0bf35e7ff3a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs @@ -0,0 +1,22 @@ +use spacetimedb::{table, ConnectionId, Identity, Timestamp}; + +#[table(accessor = user_profile, public)] +pub struct UserProfile { + #[primary_key] + pub identity: Identity, + pub display_name: String, + pub created_at: Timestamp, +} + +#[table( + accessor = connection_presence, + public, + index(accessor = by_user, btree(columns = [user_identity])) +)] +pub struct ConnectionPresence { + #[primary_key] + pub connection_id: ConnectionId, + pub user_identity: Identity, + pub last_heartbeat: Timestamp, + pub online: bool, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts new file mode 100644 index 00000000000..babde8b0f53 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts @@ -0,0 +1,26 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const userProfile = table( + { name: 'user_profile', public: true }, + { + identity: t.identity().primaryKey(), + displayName: t.string(), + createdAt: t.timestamp(), + } +); + +const connectionPresence = table( + { + name: 'connection_presence', + public: true, + indexes: [{ accessor: 'byUser', algorithm: 'btree', columns: ['userIdentity'] }], + }, + { + connectionId: t.connectionId().primaryKey(), + userIdentity: t.identity(), + lastHeartbeat: t.timestamp(), + online: t.bool(), + } +); + +export default schema({ userProfile, connectionPresence }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt new file mode 100644 index 00000000000..cea83835fb2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that separates stable profile data from frequently updated connection presence. + +TABLES +- UserProfile (public, accessor UserProfile) + - Identity: Identity (primary key) + - DisplayName: string + - CreatedAt: Timestamp +- ConnectionPresence (public, accessor ConnectionPresence) + - ConnectionId: ConnectionId (primary key) + - UserIdentity: Identity + - LastHeartbeat: Timestamp + - Online: bool + - index by_user: btree(UserIdentity) + +Do not put LastHeartbeat or Online on UserProfile. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt new file mode 100644 index 00000000000..51ffd285a05 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust that separates stable profile data from frequently updated connection presence. + +TABLES +- user_profile (public), struct UserProfile + - identity: Identity (primary key) + - display_name: String + - created_at: Timestamp +- connection_presence (public), struct ConnectionPresence + - connection_id: ConnectionId (primary key) + - user_identity: Identity + - last_heartbeat: Timestamp + - online: bool + - index by_user: btree(user_identity) + +Do not put last_heartbeat or online on UserProfile. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt new file mode 100644 index 00000000000..2d21d61606d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in TypeScript that separates stable profile data from frequently updated connection presence. + +TABLES +- user_profile (public) + - identity: Identity (primary key) + - displayName: string + - createdAt: Timestamp +- connection_presence (public) + - connectionId: ConnectionId (primary key) + - userIdentity: Identity + - lastHeartbeat: Timestamp + - online: boolean + - index byUser: btree(userIdentity) + +Do not put lastHeartbeat or online on user_profile. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs new file mode 100644 index 00000000000..368f0499e92 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs @@ -0,0 +1,29 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Organization", Public = true)] + public partial struct Organization + { + [PrimaryKey, AutoInc] public ulong Id; + public string Name; + } + + [Table(Accessor = "Department", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_organization", Columns = new[] { nameof(OrganizationId) })] + public partial struct Department + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong OrganizationId; + public string Name; + } + + [Table(Accessor = "Employee", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_department", Columns = new[] { nameof(DepartmentId) })] + public partial struct Employee + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong DepartmentId; + public string Name; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs new file mode 100644 index 00000000000..d072ad87eac --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs @@ -0,0 +1,35 @@ +use spacetimedb::table; + +#[table(accessor = organization, public)] +pub struct Organization { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table( + accessor = department, + public, + index(accessor = by_organization, btree(columns = [organization_id])) +)] +pub struct Department { + #[primary_key] + #[auto_inc] + pub id: u64, + pub organization_id: u64, + pub name: String, +} + +#[table( + accessor = employee, + public, + index(accessor = by_department, btree(columns = [department_id])) +)] +pub struct Employee { + #[primary_key] + #[auto_inc] + pub id: u64, + pub department_id: u64, + pub name: String, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts new file mode 100644 index 00000000000..ea21bdeb647 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts @@ -0,0 +1,34 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const organization = table( + { name: 'organization', public: true }, + { id: t.u64().primaryKey().autoInc(), name: t.string() } +); + +const department = table( + { + name: 'department', + public: true, + indexes: [{ accessor: 'byOrganization', algorithm: 'btree', columns: ['organizationId'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + organizationId: t.u64(), + name: t.string(), + } +); + +const employee = table( + { + name: 'employee', + public: true, + indexes: [{ accessor: 'byDepartment', algorithm: 'btree', columns: ['departmentId'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + departmentId: t.u64(), + name: t.string(), + } +); + +export default schema({ organization, department, employee }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt new file mode 100644 index 00000000000..0c1d3a28f1f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in C# that normalizes organization data into three related tables. + +TABLES +- Organization (public, accessor Organization) + - Id: ulong (primary key, auto increment) + - Name: string +- Department (public, accessor Department) + - Id: ulong (primary key, auto increment) + - OrganizationId: ulong + - Name: string + - index by_organization: btree(OrganizationId) +- Employee (public, accessor Employee) + - Id: ulong (primary key, auto increment) + - DepartmentId: ulong + - Name: string + - index by_department: btree(DepartmentId) + +Do not use nested arrays. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt new file mode 100644 index 00000000000..88d85ac39d5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in Rust that normalizes organization data into three related tables. + +TABLES +- organization (public), struct Organization + - id: u64 (primary key, auto_inc) + - name: String +- department (public), struct Department + - id: u64 (primary key, auto_inc) + - organization_id: u64 + - name: String + - index by_organization: btree(organization_id) +- employee (public), struct Employee + - id: u64 (primary key, auto_inc) + - department_id: u64 + - name: String + - index by_department: btree(department_id) + +Do not use nested arrays. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt new file mode 100644 index 00000000000..6258b64389b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in TypeScript that normalizes organization data into three related tables. + +TABLES +- organization (public) + - id: u64 (primary key, auto increment) + - name: string +- department (public) + - id: u64 (primary key, auto increment) + - organizationId: u64 + - name: string + - index byOrganization: btree(organizationId) +- employee (public) + - id: u64 (primary key, auto increment) + - departmentId: u64 + - name: string + - index byDepartment: btree(departmentId) + +Do not use nested arrays. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 2b1c850a562..716ba0e90c0 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -112,18 +112,70 @@ fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { fn extract_schema(v: &Value) -> (BTreeMap>, BTreeSet) { let mut tables: BTreeMap> = BTreeMap::new(); let mut reducers: BTreeSet = BTreeSet::new(); + let types = v + .pointer("/typespace/types") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); if let Some(ts) = v.get("tables").and_then(|x| x.as_array()) { for t in ts { let name = t.get("name").and_then(|x| x.as_str()).unwrap_or("").to_string(); let mut cols = BTreeMap::new(); - if let Some(cs) = t.get("columns").and_then(|x| x.as_array()) { + + // Older CLI descriptions put columns directly on the table. Keep + // accepting that shape while also reading the current typespace + // representation. + let legacy_columns = t.get("columns").and_then(Value::as_array); + let current_columns = t + .get("product_type_ref") + .and_then(Value::as_u64) + .and_then(|idx| types.get(idx as usize)) + .and_then(|ty| ty.pointer("/Product/elements")) + .and_then(Value::as_array); + + if let Some(cs) = legacy_columns.or(current_columns) { for c in cs { - let cname = c.get("name").and_then(|x| x.as_str()).unwrap_or("").to_string(); - let cty = c.get("type").and_then(|x| x.as_str()).unwrap_or("").to_string(); + let cname = schema_name(c.get("name")); + let cty = c + .get("type") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| { + c.get("algebraic_type") + .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) + }) + .unwrap_or_default(); cols.insert(cname, cty); } } + + let column_names: Vec = current_columns + .or(legacy_columns) + .into_iter() + .flatten() + .map(|column| schema_name(column.get("name"))) + .collect(); + + insert_schema_property( + &mut cols, + "primary_key", + column_list(t.get("primary_key"), &column_names), + ); + insert_schema_property(&mut cols, "indexes", normalize_indexes(t.get("indexes"), &column_names)); + insert_schema_property( + &mut cols, + "constraints", + normalize_constraints(t.get("constraints"), &column_names), + ); + insert_schema_property( + &mut cols, + "sequences", + normalize_sequences(t.get("sequences"), &column_names), + ); + insert_schema_property(&mut cols, "schedule", canonical_value(t.get("schedule"))); + insert_schema_property(&mut cols, "table_type", canonical_value(t.get("table_type"))); + insert_schema_property(&mut cols, "table_access", canonical_value(t.get("table_access"))); tables.insert(name, cols); } } @@ -147,6 +199,123 @@ fn extract_schema(v: &Value) -> (BTreeMap>, BTr (tables, reducers) } +fn schema_name(value: Option<&Value>) -> String { + value + .and_then(Value::as_str) + .or_else(|| value.and_then(|value| value.get("some")).and_then(Value::as_str)) + .unwrap_or("") + .to_owned() +} + +fn canonical_type(value: &Value, types: &[Value], visiting: &mut BTreeSet) -> Value { + if let Some(idx) = value.get("Ref").and_then(Value::as_u64).map(|idx| idx as usize) { + if !visiting.insert(idx) { + return json!({ "recursive_ref": idx }); + } + let resolved = types + .get(idx) + .map(|value| canonical_type(value, types, visiting)) + .unwrap_or_else(|| json!({ "missing_ref": idx })); + visiting.remove(&idx); + return resolved; + } + + match value { + Value::Array(values) => Value::Array( + values + .iter() + .map(|value| canonical_type(value, types, visiting)) + .collect(), + ), + Value::Object(values) => Value::Object( + values + .iter() + .map(|(key, value)| (key.clone(), canonical_type(value, types, visiting))) + .collect(), + ), + _ => value.clone(), + } +} + +fn column_name(value: &Value, columns: &[String]) -> String { + value + .as_u64() + .and_then(|idx| columns.get(idx as usize)) + .cloned() + .unwrap_or_else(|| value.to_string()) +} + +fn column_list(value: Option<&Value>, columns: &[String]) -> String { + value + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .map(|value| column_name(value, columns)) + .collect::>() + .join(",") + }) + .unwrap_or_default() +} + +fn normalize_indexes(value: Option<&Value>, columns: &[String]) -> String { + let mut indexes = BTreeSet::new(); + for index in value.and_then(Value::as_array).into_iter().flatten() { + let Some(algorithm) = index.get("algorithm").and_then(Value::as_object) else { + continue; + }; + for (kind, indexed_columns) in algorithm { + let normalized_columns = match indexed_columns { + Value::Array(values) => values + .iter() + .map(|value| column_name(value, columns)) + .collect::>() + .join(","), + value => column_name(value, columns), + }; + indexes.insert(format!("{kind}({normalized_columns})")); + } + } + indexes.into_iter().collect::>().join(";") +} + +fn normalize_constraints(value: Option<&Value>, columns: &[String]) -> String { + let mut constraints = BTreeSet::new(); + for constraint in value.and_then(Value::as_array).into_iter().flatten() { + let Some(data) = constraint.get("data").and_then(Value::as_object) else { + continue; + }; + for (kind, detail) in data { + let normalized_columns = column_list(detail.get("columns"), columns); + constraints.insert(format!("{kind}({normalized_columns})")); + } + } + constraints.into_iter().collect::>().join(";") +} + +fn normalize_sequences(value: Option<&Value>, columns: &[String]) -> String { + let mut sequences = BTreeSet::new(); + for sequence in value.and_then(Value::as_array).into_iter().flatten() { + let column = sequence + .get("column") + .map(|value| column_name(value, columns)) + .unwrap_or_default(); + let increment = sequence.get("increment").and_then(Value::as_i64).unwrap_or_default(); + sequences.insert(format!("{column}:{increment}")); + } + sequences.into_iter().collect::>().join(";") +} + +fn canonical_value(value: Option<&Value>) -> String { + value.map(Value::to_string).unwrap_or_default() +} + +fn insert_schema_property(columns: &mut BTreeMap, name: &str, value: String) { + if !value.is_empty() { + columns.insert(format!("@{name}"), value); + } +} + fn diff_maps(a: &BTreeMap>, b: &BTreeMap>) -> Value { let mut only_a = BTreeMap::new(); let mut only_b = BTreeMap::new(); @@ -646,3 +815,62 @@ impl Scorer for ReducerCallBothScorer { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn current_schema(include_owner_index: bool) -> Value { + let mut indexes = vec![json!({ "algorithm": { "BTree": [0] } })]; + if include_owner_index { + indexes.push(json!({ "algorithm": { "BTree": [1] } })); + } + json!({ + "typespace": { + "types": [{ + "Product": { + "elements": [ + { "name": { "some": "id" }, "algebraic_type": { "U64": [] } }, + { "name": { "some": "owner_id" }, "algebraic_type": { "U64": [] } } + ] + } + }] + }, + "tables": [{ + "name": "child_item", + "product_type_ref": 0, + "primary_key": [0], + "indexes": indexes, + "constraints": [{ "data": { "Unique": { "columns": [0] } } }], + "sequences": [{ "column": 0, "increment": 1 }], + "schedule": { "none": [] }, + "table_type": { "User": [] }, + "table_access": { "Public": [] } + }], + "reducers": [] + }) + } + + #[test] + fn current_schema_extracts_columns_and_table_properties() { + let (tables, reducers) = extract_schema(¤t_schema(true)); + let child_item = &tables["child_item"]; + + assert_eq!(child_item["id"], r#"{"U64":[]}"#); + assert_eq!(child_item["owner_id"], r#"{"U64":[]}"#); + assert_eq!(child_item["@primary_key"], "id"); + assert_eq!(child_item["@indexes"], "BTree(id);BTree(owner_id)"); + assert_eq!(child_item["@constraints"], "Unique(id)"); + assert_eq!(child_item["@sequences"], "id:1"); + assert_eq!(child_item["@table_access"], r#"{"Public":[]}"#); + assert!(reducers.is_empty()); + } + + #[test] + fn missing_index_produces_a_schema_diff() { + let (golden, _) = extract_schema(¤t_schema(true)); + let (candidate, _) = extract_schema(¤t_schema(false)); + + assert!(!diff_maps(&golden, &candidate).is_null()); + } +} diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index ef74e3aac20..412aab230b6 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -284,6 +284,24 @@ mod schema_t_021_multi_column_index { include!("../benchmarks/schema/t_021_multi_column_index/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_047_normalized_collection { + include!("../benchmarks/tables/t_047_normalized_collection/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_048_heartbeat_isolation { + include!("../benchmarks/tables/t_048_heartbeat_isolation/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_050_normalized_schema { + include!("../benchmarks/tables/t_050_normalized_schema/spec.rs"); +} + pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { let task = task_root .file_name() @@ -342,6 +360,9 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("schema", "t_019_many_to_many") => schema_t_019_many_to_many::spec, ("schema", "t_020_ecs") => schema_t_020_ecs::spec, ("schema", "t_021_multi_column_index") => schema_t_021_multi_column_index::spec, + ("tables", "t_047_normalized_collection") => tables_t_047_normalized_collection::spec, + ("tables", "t_048_heartbeat_isolation") => tables_t_048_heartbeat_isolation::spec, + ("tables", "t_050_normalized_schema") => tables_t_050_normalized_schema::spec, _ => return Err(anyhow!("no spec registered for {}/{} (need spec.rs)", category, task)), }; From e51e2874f9df861bc17693e8b6bdab795a7be8af Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:59:34 -0400 Subject: [PATCH 02/78] llm-benchmark: add behavioral table evals --- .../t_049_binary_storage/answers/csharp.cs | 30 +++++++++ .../t_049_binary_storage/answers/rust.rs | 29 +++++++++ .../answers/typescript.ts | 34 ++++++++++ .../tables/t_049_binary_storage/spec.rs | 34 ++++++++++ .../t_049_binary_storage/tasks/csharp.txt | 5 ++ .../t_049_binary_storage/tasks/rust.txt | 5 ++ .../t_049_binary_storage/tasks/typescript.txt | 5 ++ .../answers/csharp.cs | 53 ++++++++++++++++ .../t_051_denormalized_index/answers/rust.rs | 62 +++++++++++++++++++ .../answers/typescript.ts | 52 ++++++++++++++++ .../tables/t_051_denormalized_index/spec.rs | 47 ++++++++++++++ .../t_051_denormalized_index/tasks/csharp.txt | 5 ++ .../t_051_denormalized_index/tasks/rust.txt | 5 ++ .../tasks/typescript.txt | 5 ++ .../t_052_autoinc_reference/answers/csharp.cs | 30 +++++++++ .../t_052_autoinc_reference/answers/rust.rs | 37 +++++++++++ .../answers/typescript.ts | 28 +++++++++ .../tables/t_052_autoinc_reference/spec.rs | 34 ++++++++++ .../t_052_autoinc_reference/tasks/csharp.txt | 5 ++ .../t_052_autoinc_reference/tasks/rust.txt | 5 ++ .../tasks/typescript.txt | 5 ++ .../t_054_special_types/answers/csharp.cs | 17 +++++ .../t_054_special_types/answers/rust.rs | 17 +++++ .../t_054_special_types/answers/typescript.ts | 17 +++++ .../tables/t_054_special_types/spec.rs | 8 +++ .../t_054_special_types/tasks/csharp.txt | 3 + .../tables/t_054_special_types/tasks/rust.txt | 3 + .../t_054_special_types/tasks/typescript.txt | 3 + .../src/generated/registry.rs | 28 +++++++++ 29 files changed, 611 insertions(+) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs new file mode 100644 index 00000000000..36bb8b1fc67 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs @@ -0,0 +1,30 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "BlobRecord", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_owner", Columns = new[] { nameof(Owner) })] + public partial struct BlobRecord + { + [PrimaryKey, AutoInc] public ulong Id; + public Identity Owner; + public string Filename; + public string MimeType; + public ulong Size; + public List Data; + } + + [Reducer] + public static void StoreBlob(ReducerContext ctx, string filename, string mimeType, List data) + { + ctx.Db.BlobRecord.Insert(new BlobRecord + { + Id = 0, + Owner = ctx.Sender, + Filename = filename, + MimeType = mimeType, + Size = (ulong)data.Count, + Data = data, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs new file mode 100644 index 00000000000..031b3645072 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs @@ -0,0 +1,29 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table( + accessor = blob_record, + public, + index(accessor = by_owner, btree(columns = [owner])) +)] +pub struct BlobRecord { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner: Identity, + pub filename: String, + pub mime_type: String, + pub size: u64, + pub data: Vec, +} + +#[reducer] +pub fn store_blob(ctx: &ReducerContext, filename: String, mime_type: String, data: Vec) { + ctx.db.blob_record().insert(BlobRecord { + id: 0, + owner: ctx.sender(), + filename, + mime_type, + size: data.len() as u64, + data, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts new file mode 100644 index 00000000000..e6982c6745e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts @@ -0,0 +1,34 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const blobRecord = table( + { + name: 'blob_record', + public: true, + indexes: [{ accessor: 'byOwner', algorithm: 'btree', columns: ['owner'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + owner: t.identity(), + filename: t.string(), + mimeType: t.string(), + size: t.u64(), + data: t.array(t.u8()), + } +); + +const spacetimedb = schema({ blobRecord }); +export default spacetimedb; + +export const store_blob = spacetimedb.reducer( + { filename: t.string(), mimeType: t.string(), data: t.array(t.u8()) }, + (ctx, { filename, mimeType, data }) => { + ctx.db.blobRecord.insert({ + id: 0n, + owner: ctx.sender, + filename, + mimeType, + size: BigInt(data.length), + data, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs new file mode 100644 index 00000000000..3161c7ce1df --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs @@ -0,0 +1,34 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("blob_record", lang); + let columns = sql.cols(&["filename", "mime_type", "size", "data"]).join(", "); + let filename = ident("filename", casing_for_lang(lang)); + let select_query = format!("SELECT {columns} FROM {table} WHERE {filename}='eval.bin'"); + + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "store_blob".into(), + args: vec![ + json!("eval.bin"), + json!("application/octet-stream"), + json!([0, 1, 2, 127, 128, 255]), + ], + select_query, + id_str: "binary_round_trip", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt new file mode 100644 index 00000000000..9e1f3878efa --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in C# that stores binary content and its metadata. + +Define a public BlobRecord table with an auto-incremented ulong primary key named Id, Owner: Identity, Filename: string, MimeType: string, Size: ulong, and Data: List. Add a B-tree index named by_owner on Owner. + +Define a StoreBlob reducer taking filename, mimeType, and data. Store the sender as Owner, derive Size from the byte count, and insert the bytes unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt new file mode 100644 index 00000000000..dd87130f42f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in Rust that stores binary content and its metadata. + +Define a public blob_record table with an auto-incremented u64 primary key named id, owner: Identity, filename: String, mime_type: String, size: u64, and data: Vec. Add a B-tree index named by_owner on owner. + +Define a store_blob reducer taking filename, mime_type, and data. Store the sender as owner, derive size from the byte length, and insert the bytes unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt new file mode 100644 index 00000000000..ecbc109f031 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in TypeScript that stores binary content and its metadata. + +Define a public blob_record table with an auto-incremented u64 primary key named id, owner: Identity, filename: string, mimeType: string, size: u64, and data: u8 array. Add a B-tree index named byOwner on owner. + +Define a store_blob reducer taking filename, mimeType, and data. Store ctx.sender as owner, derive size from the byte length, and insert the bytes unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs new file mode 100644 index 00000000000..0fe6e1675eb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs @@ -0,0 +1,53 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Category", Public = true)] + public partial struct Category + { + [PrimaryKey] public ulong Id; + public string Slug; + } + + [Table(Accessor = "Product", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_category", Columns = new[] { nameof(CategoryId) })] + [SpacetimeDB.Index.BTree(Accessor = "by_category_slug", Columns = new[] { nameof(CategorySlug) })] + public partial struct Product + { + [PrimaryKey] public ulong Id; + public ulong CategoryId; + public string CategorySlug; + public string Name; + } + + [Reducer] + public static void CreateCategory(ReducerContext ctx, ulong id, string slug) + { + ctx.Db.Category.Insert(new Category { Id = id, Slug = slug }); + } + + [Reducer] + public static void CreateProduct(ReducerContext ctx, ulong id, ulong categoryId, string name) + { + var category = ctx.Db.Category.Id.Find(categoryId) ?? throw new Exception("category not found"); + ctx.Db.Product.Insert(new Product + { + Id = id, + CategoryId = categoryId, + CategorySlug = category.Slug, + Name = name, + }); + } + + [Reducer] + public static void RenameCategory(ReducerContext ctx, ulong id, string newSlug) + { + var category = ctx.Db.Category.Id.Find(id) ?? throw new Exception("category not found"); + ctx.Db.Category.Id.Update(category with { Slug = newSlug }); + + foreach (var product in ctx.Db.Product.by_category.Filter(id)) + { + ctx.Db.Product.Id.Update(product with { CategorySlug = newSlug }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs new file mode 100644 index 00000000000..8122437ebc9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs @@ -0,0 +1,62 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = category, public)] +pub struct Category { + #[primary_key] + pub id: u64, + pub slug: String, +} + +#[table( + accessor = product, + public, + index(accessor = by_category, btree(columns = [category_id])), + index(accessor = by_category_slug, btree(columns = [category_slug])) +)] +pub struct Product { + #[primary_key] + pub id: u64, + pub category_id: u64, + pub category_slug: String, + pub name: String, +} + +#[reducer] +pub fn create_category(ctx: &ReducerContext, id: u64, slug: String) { + ctx.db.category().insert(Category { id, slug }); +} + +#[reducer] +pub fn create_product(ctx: &ReducerContext, id: u64, category_id: u64, name: String) -> Result<(), String> { + let category = ctx + .db + .category() + .id() + .find(category_id) + .ok_or_else(|| "category not found".to_string())?; + ctx.db.product().insert(Product { + id, + category_id, + category_slug: category.slug, + name, + }); + Ok(()) +} + +#[reducer] +pub fn rename_category(ctx: &ReducerContext, id: u64, new_slug: String) -> Result<(), String> { + let mut category = ctx + .db + .category() + .id() + .find(id) + .ok_or_else(|| "category not found".to_string())?; + category.slug = new_slug.clone(); + ctx.db.category().id().update(category); + + for mut product in ctx.db.product().by_category().filter(id) { + product.category_slug = new_slug.clone(); + ctx.db.product().id().update(product); + } + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts new file mode 100644 index 00000000000..85d8088d52a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts @@ -0,0 +1,52 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const category = table( + { name: 'category', public: true }, + { id: t.u64().primaryKey(), slug: t.string() } +); + +const product = table( + { + name: 'product', + public: true, + indexes: [ + { accessor: 'byCategory', algorithm: 'btree', columns: ['categoryId'] }, + { accessor: 'byCategorySlug', algorithm: 'btree', columns: ['categorySlug'] }, + ], + }, + { + id: t.u64().primaryKey(), + categoryId: t.u64(), + categorySlug: t.string(), + name: t.string(), + } +); + +const spacetimedb = schema({ category, product }); +export default spacetimedb; + +export const create_category = spacetimedb.reducer( + { id: t.u64(), slug: t.string() }, + (ctx, { id, slug }) => ctx.db.category.insert({ id, slug }) +); + +export const create_product = spacetimedb.reducer( + { id: t.u64(), categoryId: t.u64(), name: t.string() }, + (ctx, { id, categoryId, name }) => { + const found = ctx.db.category.id.find(categoryId); + if (!found) throw new Error('category not found'); + ctx.db.product.insert({ id, categoryId, categorySlug: found.slug, name }); + } +); + +export const rename_category = spacetimedb.reducer( + { id: t.u64(), newSlug: t.string() }, + (ctx, { id, newSlug }) => { + const found = ctx.db.category.id.find(id); + if (!found) throw new Error('category not found'); + ctx.db.category.id.update({ ...found, slug: newSlug }); + for (const existing of ctx.db.product.byCategory.filter(id)) { + ctx.db.product.id.update({ ...existing, categorySlug: newSlug }); + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs new file mode 100644 index 00000000000..710e155e5e0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs @@ -0,0 +1,47 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_category", + vec![json!(1), json!("old-slug")], + "create_category", + )); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_product", + vec![json!(10), json!(1), json!("Widget")], + "create_product", + )); + + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("product", lang); + let columns = sql.cols(&["id", "category_id", "category_slug", "name"]).join(", "); + let id = ident("id", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "rename_category".into(), + args: vec![json!(1), json!("new-slug")], + select_query: format!("SELECT {columns} FROM {table} WHERE {id}=10"), + id_str: "denormalized_value_stays_synchronized", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt new file mode 100644 index 00000000000..8ba59dd8537 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in C# that maintains an indexed denormalized category slug on products. + +Define public Category and Product tables. Category has Id: ulong primary key and Slug: string. Product has Id: ulong primary key, CategoryId: ulong with a B-tree index named by_category, CategorySlug: string with a B-tree index named by_category_slug, and Name: string. + +Add CreateCategory(id, slug), CreateProduct(id, categoryId, name), and RenameCategory(id, newSlug) reducers. CreateProduct must copy the current category slug. RenameCategory must update both the category and every related product so CategorySlug stays synchronized. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt new file mode 100644 index 00000000000..8586fe2926f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in Rust that maintains an indexed denormalized category slug on products. + +Define public category and product tables. Category has id: u64 primary key and slug: String. Product has id: u64 primary key, category_id: u64 with a B-tree index named by_category, category_slug: String with a B-tree index named by_category_slug, and name: String. + +Add create_category(id, slug), create_product(id, category_id, name), and rename_category(id, new_slug) reducers. create_product must copy the current category slug. rename_category must update both the category and every related product so category_slug stays synchronized. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt new file mode 100644 index 00000000000..e5277f700b6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in TypeScript that maintains an indexed denormalized category slug on products. + +Define public category and product tables. Category has id: u64 primary key and slug: string. Product has id: u64 primary key, categoryId: u64 with a B-tree index named byCategory, categorySlug: string with a B-tree index named byCategorySlug, and name: string. + +Add create_category(id, slug), create_product(id, categoryId, name), and rename_category(id, newSlug) reducers. create_product must copy the current category slug. rename_category must update both the category and every related product so categorySlug stays synchronized. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs new file mode 100644 index 00000000000..7f761292915 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs @@ -0,0 +1,30 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Parent", Public = true)] + public partial struct Parent + { + [PrimaryKey, AutoInc] public ulong Id; + public string Name; + } + + [Table(Accessor = "Child", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_parent", Columns = new[] { nameof(ParentId) })] + public partial struct Child + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong ParentId; + public string Name; + } + + [Reducer] + public static void CreateFamily(ReducerContext ctx, string parentName, List childNames) + { + var parent = ctx.Db.Parent.Insert(new Parent { Id = 0, Name = parentName }); + foreach (var name in childNames) + { + ctx.Db.Child.Insert(new Child { Id = 0, ParentId = parent.Id, Name = name }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs new file mode 100644 index 00000000000..298ee10457a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs @@ -0,0 +1,37 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = parent, public)] +pub struct Parent { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table( + accessor = child, + public, + index(accessor = by_parent, btree(columns = [parent_id])) +)] +pub struct Child { + #[primary_key] + #[auto_inc] + pub id: u64, + pub parent_id: u64, + pub name: String, +} + +#[reducer] +pub fn create_family(ctx: &ReducerContext, parent_name: String, child_names: Vec) { + let parent = ctx.db.parent().insert(Parent { + id: 0, + name: parent_name, + }); + for name in child_names { + ctx.db.child().insert(Child { + id: 0, + parent_id: parent.id, + name, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts new file mode 100644 index 00000000000..33603dcbd56 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts @@ -0,0 +1,28 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const parent = table( + { name: 'parent', public: true }, + { id: t.u64().primaryKey().autoInc(), name: t.string() } +); + +const child = table( + { + name: 'child', + public: true, + indexes: [{ accessor: 'byParent', algorithm: 'btree', columns: ['parentId'] }], + }, + { id: t.u64().primaryKey().autoInc(), parentId: t.u64(), name: t.string() } +); + +const spacetimedb = schema({ parent, child }); +export default spacetimedb; + +export const create_family = spacetimedb.reducer( + { parentName: t.string(), childNames: t.array(t.string()) }, + (ctx, { parentName, childNames }) => { + const insertedParent = ctx.db.parent.insert({ id: 0n, name: parentName }); + for (const name of childNames) { + ctx.db.child.insert({ id: 0n, parentId: insertedParent.id, name }); + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs new file mode 100644 index 00000000000..0bc0798dd09 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs @@ -0,0 +1,34 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let casing = casing_for_lang(lang); + let parent = table_name("parent", lang); + let child = table_name("child", lang); + let parent_id = ident("parent_id", casing); + let id = ident("id", casing); + let name = ident("name", casing); + let sql = format!( + "SELECT COUNT(*) AS n FROM {child} JOIN {parent} ON {child}.{parent_id}={parent}.{id} WHERE {parent}.{name}='Taylor'" + ); + + scorers.push(make_reducer_sql_count_scorer( + host_url, + ReducerSqlCountConfig { + src_file: file!(), + route_tag, + reducer: "create_family".into(), + args: vec![json!("Taylor"), json!(["Avery", "Jordan"])], + sql_count_query: sql, + expected_count: 2, + id_str: "children_reference_inserted_parent", + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt new file mode 100644 index 00000000000..5717ee4754d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in C# that safely creates related rows with an auto-incremented parent ID. + +Define public Parent and Child tables. Both have auto-incremented ulong primary keys named Id. Parent also has Name: string. Child has ParentId: ulong with a B-tree index named by_parent and Name: string. + +Define CreateFamily(parentName, childNames). Insert the parent, capture the row returned by Insert, and use that returned Id for every child. Never calculate or assume the next auto-increment value. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt new file mode 100644 index 00000000000..3ea61625ffa --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in Rust that safely creates related rows with an auto-incremented parent ID. + +Define public parent and child tables. Both have auto-incremented u64 primary keys named id. Parent also has name: String. Child has parent_id: u64 with a B-tree index named by_parent and name: String. + +Define create_family(parent_name, child_names). Insert the parent, capture the row returned by insert, and use that returned id for every child. Never calculate or assume the next auto-increment value. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt new file mode 100644 index 00000000000..16f0578ca4b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt @@ -0,0 +1,5 @@ +Write a SpacetimeDB backend module in TypeScript that safely creates related rows with an auto-incremented parent ID. + +Define public parent and child tables. Both have auto-incremented u64 primary keys named id. Parent also has name: string. Child has parentId: u64 with a B-tree index named byParent and name: string. + +Define create_family(parentName, childNames). Insert the parent, capture the row returned by insert, and use that returned id for every child. Never calculate or assume the next auto-increment value. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs new file mode 100644 index 00000000000..34b49f90ae5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs @@ -0,0 +1,17 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "SpecialValue", Public = true)] + public partial struct SpecialValue + { + [PrimaryKey] public ulong Id; + public Uuid Uuid; + public ConnectionId ConnectionId; + public TimeDuration Duration; + public U128 Unsigned128; + public I128 Signed128; + public U256 Unsigned256; + public I256 Signed256; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs new file mode 100644 index 00000000000..f247846178c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs @@ -0,0 +1,17 @@ +use spacetimedb::{ + sats::{i256, u256}, + table, ConnectionId, TimeDuration, Uuid, +}; + +#[table(accessor = special_value, public)] +pub struct SpecialValue { + #[primary_key] + pub id: u64, + pub uuid: Uuid, + pub connection_id: ConnectionId, + pub duration: TimeDuration, + pub unsigned_128: u128, + pub signed_128: i128, + pub unsigned_256: u256, + pub signed_256: i256, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts new file mode 100644 index 00000000000..6ff811f7303 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts @@ -0,0 +1,17 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const specialValue = table( + { name: 'special_value', public: true }, + { + id: t.u64().primaryKey(), + uuid: t.uuid(), + connectionId: t.connectionId(), + duration: t.timeDuration(), + unsigned128: t.u128(), + signed128: t.i128(), + unsigned256: t.u256(), + signed256: t.i256(), + } +); + +export default schema({ specialValue }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt new file mode 100644 index 00000000000..18509d2e760 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt @@ -0,0 +1,3 @@ +Write a SpacetimeDB backend module in C# defining one public SpecialValue table. + +The table must contain Id: ulong primary key, Uuid: Uuid, ConnectionId: ConnectionId, Duration: TimeDuration, Unsigned128: U128, Signed128: I128, Unsigned256: U256, and Signed256: I256. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt new file mode 100644 index 00000000000..29d9000dd2b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt @@ -0,0 +1,3 @@ +Write a SpacetimeDB backend module in Rust defining one public special_value table. + +The table must contain id: u64 primary key, uuid: Uuid, connection_id: ConnectionId, duration: TimeDuration, unsigned_128: u128, signed_128: i128, unsigned_256: u256, and signed_256: i256. Use the SpacetimeDB integer types for 256-bit values. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt new file mode 100644 index 00000000000..81a70148e5f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt @@ -0,0 +1,3 @@ +Write a SpacetimeDB backend module in TypeScript defining one public special_value table. + +The table must contain id: u64 primary key, uuid: Uuid, connectionId: ConnectionId, duration: TimeDuration, unsigned128: u128, signed128: i128, unsigned256: u256, and signed256: i256. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 412aab230b6..001197631c6 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -296,12 +296,36 @@ mod tables_t_048_heartbeat_isolation { include!("../benchmarks/tables/t_048_heartbeat_isolation/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_049_binary_storage { + include!("../benchmarks/tables/t_049_binary_storage/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod tables_t_050_normalized_schema { include!("../benchmarks/tables/t_050_normalized_schema/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_051_denormalized_index { + include!("../benchmarks/tables/t_051_denormalized_index/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_052_autoinc_reference { + include!("../benchmarks/tables/t_052_autoinc_reference/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_054_special_types { + include!("../benchmarks/tables/t_054_special_types/spec.rs"); +} + pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { let task = task_root .file_name() @@ -362,7 +386,11 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("schema", "t_021_multi_column_index") => schema_t_021_multi_column_index::spec, ("tables", "t_047_normalized_collection") => tables_t_047_normalized_collection::spec, ("tables", "t_048_heartbeat_isolation") => tables_t_048_heartbeat_isolation::spec, + ("tables", "t_049_binary_storage") => tables_t_049_binary_storage::spec, ("tables", "t_050_normalized_schema") => tables_t_050_normalized_schema::spec, + ("tables", "t_051_denormalized_index") => tables_t_051_denormalized_index::spec, + ("tables", "t_052_autoinc_reference") => tables_t_052_autoinc_reference::spec, + ("tables", "t_054_special_types") => tables_t_054_special_types::spec, _ => return Err(anyhow!("no spec registered for {}/{} (need spec.rs)", category, task)), }; From ba223b31e3cc0f112f4893c9fae2736c3e285e09 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:09:25 -0400 Subject: [PATCH 03/78] llm-benchmark: add reducer behavior evals --- .../answers/csharp.cs | 33 +++++++++++++ .../answers/rust.rs | 48 +++++++++++++++++++ .../answers/typescript.ts | 29 +++++++++++ .../t_055_atomic_idempotent_transfer/spec.rs | 44 +++++++++++++++++ .../tasks/csharp.txt | 3 ++ .../tasks/rust.txt | 3 ++ .../tasks/typescript.txt | 3 ++ .../t_056_nested_update/answers/csharp.cs | 32 +++++++++++++ .../t_056_nested_update/answers/rust.rs | 35 ++++++++++++++ .../t_056_nested_update/answers/typescript.ts | 32 +++++++++++++ .../reducers/t_056_nested_update/spec.rs | 38 +++++++++++++++ .../t_056_nested_update/tasks/csharp.txt | 3 ++ .../t_056_nested_update/tasks/rust.txt | 3 ++ .../t_056_nested_update/tasks/typescript.txt | 3 ++ .../answers/csharp.cs | 36 ++++++++++++++ .../answers/rust.rs | 35 ++++++++++++++ .../answers/typescript.ts | 28 +++++++++++ .../t_057_nested_cascade_delete/spec.rs | 45 +++++++++++++++++ .../tasks/csharp.txt | 3 ++ .../tasks/rust.txt | 3 ++ .../tasks/typescript.txt | 3 ++ .../answers/csharp.cs | 20 ++++++++ .../answers/rust.rs | 19 ++++++++ .../answers/typescript.ts | 16 +++++++ .../t_059_deterministic_context/spec.rs | 23 +++++++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 16 +++++++ .../answers/rust.rs | 18 +++++++ .../answers/typescript.ts | 17 +++++++ .../t_060_reducer_result_table/spec.rs | 28 +++++++++++ .../tasks/csharp.txt | 1 + .../t_060_reducer_result_table/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../src/generated/registry.rs | 35 ++++++++++++++ 36 files changed, 660 insertions(+) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs new file mode 100644 index 00000000000..a873dcc0bdc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs @@ -0,0 +1,33 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Account", Public = true)] + public partial struct Account { [PrimaryKey] public ulong Id; public long Balance; } + + [Table(Accessor = "TransferRequest", Public = true)] + public partial struct TransferRequest + { + [PrimaryKey] public string RequestId; + public ulong FromId; + public ulong ToId; + public long Amount; + } + + [Reducer] + public static void CreateAccount(ReducerContext ctx, ulong id, long balance) => + ctx.Db.Account.Insert(new Account { Id = id, Balance = balance }); + + [Reducer] + public static void Transfer(ReducerContext ctx, string requestId, ulong fromId, ulong toId, long amount) + { + if (ctx.Db.TransferRequest.RequestId.Find(requestId) is not null) return; + if (amount <= 0 || fromId == toId) throw new Exception("invalid transfer"); + var source = ctx.Db.Account.Id.Find(fromId) ?? throw new Exception("source account not found"); + var to = ctx.Db.Account.Id.Find(toId) ?? throw new Exception("destination account not found"); + if (source.Balance < amount) throw new Exception("insufficient balance"); + ctx.Db.Account.Id.Update(source with { Balance = source.Balance - amount }); + ctx.Db.Account.Id.Update(to with { Balance = to.Balance + amount }); + ctx.Db.TransferRequest.Insert(new TransferRequest { RequestId = requestId, FromId = fromId, ToId = toId, Amount = amount }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs new file mode 100644 index 00000000000..8054a55f80c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs @@ -0,0 +1,48 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = account, public)] +pub struct Account { + #[primary_key] + pub id: u64, + pub balance: i64, +} + +#[table(accessor = transfer_request, public)] +pub struct TransferRequest { + #[primary_key] + pub request_id: String, + pub from_id: u64, + pub to_id: u64, + pub amount: i64, +} + +#[reducer] +pub fn create_account(ctx: &ReducerContext, id: u64, balance: i64) { + ctx.db.account().insert(Account { id, balance }); +} + +#[reducer] +pub fn transfer(ctx: &ReducerContext, request_id: String, from_id: u64, to_id: u64, amount: i64) -> Result<(), String> { + if ctx.db.transfer_request().request_id().find(&request_id).is_some() { + return Ok(()); + } + if amount <= 0 || from_id == to_id { + return Err("invalid transfer".into()); + } + let mut from = ctx.db.account().id().find(from_id).ok_or("source account not found")?; + let mut to = ctx.db.account().id().find(to_id).ok_or("destination account not found")?; + if from.balance < amount { + return Err("insufficient balance".into()); + } + from.balance -= amount; + to.balance += amount; + ctx.db.account().id().update(from); + ctx.db.account().id().update(to); + ctx.db.transfer_request().insert(TransferRequest { + request_id, + from_id, + to_id, + amount, + }); + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts new file mode 100644 index 00000000000..092c4fc6c3f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts @@ -0,0 +1,29 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const account = table({ name: 'account', public: true }, { id: t.u64().primaryKey(), balance: t.i64() }); +const transferRequest = table( + { name: 'transfer_request', public: true }, + { requestId: t.string().primaryKey(), fromId: t.u64(), toId: t.u64(), amount: t.i64() } +); +const spacetimedb = schema({ account, transferRequest }); +export default spacetimedb; + +export const create_account = spacetimedb.reducer( + { id: t.u64(), balance: t.i64() }, + (ctx, { id, balance }) => ctx.db.account.insert({ id, balance }) +); + +export const transfer = spacetimedb.reducer( + { requestId: t.string(), fromId: t.u64(), toId: t.u64(), amount: t.i64() }, + (ctx, { requestId, fromId, toId, amount }) => { + if (ctx.db.transferRequest.requestId.find(requestId)) return; + if (amount <= 0n || fromId === toId) throw new Error('invalid transfer'); + const from = ctx.db.account.id.find(fromId); + const to = ctx.db.account.id.find(toId); + if (!from || !to) throw new Error('account not found'); + if (from.balance < amount) throw new Error('insufficient balance'); + ctx.db.account.id.update({ ...from, balance: from.balance - amount }); + ctx.db.account.id.update({ ...to, balance: to.balance + amount }); + ctx.db.transferRequest.insert({ requestId, fromId, toId, amount }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs new file mode 100644 index 00000000000..409784e7db5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs @@ -0,0 +1,44 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + for (id, balance) in [(1, 100), (2, 25)] { + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_account", + vec![json!(id), json!(balance)], + "create_account", + )); + } + let sql = SqlBuilder::new(casing_for_lang(lang)); + let account = table_name("account", lang); + let columns = sql.cols(&["id", "balance"]).join(", "); + let id = ident("id", casing_for_lang(lang)); + let query = format!("SELECT {columns} FROM {account} WHERE {id}=1 OR {id}=2"); + + for scorer_id in ["transfer_once", "duplicate_transfer_is_noop"] { + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "transfer".into(), + args: vec![json!("request-1"), json!(1), json!(2), json!(40)], + select_query: query.clone(), + id_str: scorer_id, + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + } + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt new file mode 100644 index 00000000000..44419cf8444 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt @@ -0,0 +1,3 @@ +Write a C# SpacetimeDB module with public Account and TransferRequest tables. Account has Id: ulong primary key and Balance: long. TransferRequest has RequestId: string primary key, FromId: ulong, ToId: ulong, and Amount: long. + +Add CreateAccount(id, balance) and Transfer(requestId, fromId, toId, amount). Transfer must reject invalid or insufficient transfers, update both balances atomically, and record RequestId. Repeating a recorded RequestId must be a no-op. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt new file mode 100644 index 00000000000..4f19303b89c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt @@ -0,0 +1,3 @@ +Write a Rust SpacetimeDB module with public account and transfer_request tables. Account has id: u64 primary key and balance: i64. TransferRequest has request_id: String primary key, from_id: u64, to_id: u64, and amount: i64. + +Add create_account(id, balance) and transfer(request_id, from_id, to_id, amount). Transfer must reject invalid or insufficient transfers, update both balances atomically, and record request_id. Repeating a recorded request_id must be a no-op. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt new file mode 100644 index 00000000000..54949f1ddeb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt @@ -0,0 +1,3 @@ +Write a TypeScript SpacetimeDB module with public account and transfer_request tables. Account has id: u64 primary key and balance: i64. TransferRequest has requestId: string primary key, fromId: u64, toId: u64, and amount: i64. + +Add create_account(id, balance) and transfer(requestId, fromId, toId, amount). Transfer must reject invalid or insufficient transfers, update both balances atomically, and record requestId. Repeating a recorded requestId must be a no-op. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs new file mode 100644 index 00000000000..154f8749e4c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs @@ -0,0 +1,32 @@ +using SpacetimeDB; + +public static partial class Module +{ + [SpacetimeDB.Type] + public partial struct Preferences + { + public string Theme; + public bool EmailNotifications; + public string Timezone; + } + + [Table(Accessor = "Profile", Public = true)] + public partial struct Profile { [PrimaryKey] public ulong Id; public Preferences Preferences; } + + [Reducer] + public static void CreateProfile(ReducerContext ctx, ulong id, string theme, bool emailNotifications, string timezone) => + ctx.Db.Profile.Insert(new Profile + { + Id = id, + Preferences = new Preferences { Theme = theme, EmailNotifications = emailNotifications, Timezone = timezone } + }); + + [Reducer] + public static void UpdateTheme(ReducerContext ctx, ulong id, string theme) + { + var profile = ctx.Db.Profile.Id.Find(id) ?? throw new Exception("profile not found"); + var preferences = profile.Preferences; + preferences.Theme = theme; + ctx.Db.Profile.Id.Update(profile with { Preferences = preferences }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs new file mode 100644 index 00000000000..0a59cae7261 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs @@ -0,0 +1,35 @@ +use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + +#[derive(SpacetimeType)] +pub struct Preferences { + pub theme: String, + pub email_notifications: bool, + pub timezone: String, +} + +#[table(accessor = profile, public)] +pub struct Profile { + #[primary_key] + pub id: u64, + pub preferences: Preferences, +} + +#[reducer] +pub fn create_profile(ctx: &ReducerContext, id: u64, theme: String, email_notifications: bool, timezone: String) { + ctx.db.profile().insert(Profile { + id, + preferences: Preferences { + theme, + email_notifications, + timezone, + }, + }); +} + +#[reducer] +pub fn update_theme(ctx: &ReducerContext, id: u64, theme: String) -> Result<(), String> { + let mut profile = ctx.db.profile().id().find(id).ok_or("profile not found")?; + profile.preferences.theme = theme; + ctx.db.profile().id().update(profile); + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts new file mode 100644 index 00000000000..545e3b89e3b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts @@ -0,0 +1,32 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const Preferences = t.object('Preferences', { + theme: t.string(), + emailNotifications: t.bool(), + timezone: t.string(), +}); +const profile = table( + { name: 'profile', public: true }, + { id: t.u64().primaryKey(), preferences: Preferences } +); +const spacetimedb = schema({ profile }); +export default spacetimedb; + +export const create_profile = spacetimedb.reducer( + { id: t.u64(), theme: t.string(), emailNotifications: t.bool(), timezone: t.string() }, + (ctx, args) => ctx.db.profile.insert({ id: args.id, preferences: { + theme: args.theme, emailNotifications: args.emailNotifications, timezone: args.timezone, + } }) +); + +export const update_theme = spacetimedb.reducer( + { id: t.u64(), theme: t.string() }, + (ctx, { id, theme }) => { + const found = ctx.db.profile.id.find(id); + if (!found) throw new Error('profile not found'); + ctx.db.profile.id.update({ + ...found, + preferences: { ...found.preferences, theme }, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs new file mode 100644 index 00000000000..eb1023b0c7a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs @@ -0,0 +1,38 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_profile", + vec![json!(1), json!("light"), json!(true), json!("UTC")], + "create_profile", + )); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("profile", lang); + let columns = sql.cols(&["id", "preferences"]).join(", "); + let id = ident("id", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "update_theme".into(), + args: vec![json!(1), json!("dark")], + select_query: format!("SELECT {columns} FROM {table} WHERE {id}=1"), + id_str: "nested_siblings_preserved", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt new file mode 100644 index 00000000000..1d8a4de1051 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt @@ -0,0 +1,3 @@ +Write a C# SpacetimeDB module with a Preferences product type containing Theme: string, EmailNotifications: bool, and Timezone: string. Store it in a public Profile table with Id: ulong primary key. + +Add CreateProfile(id, theme, emailNotifications, timezone) and UpdateTheme(id, theme). UpdateTheme must change only the nested Theme and preserve both sibling fields. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt new file mode 100644 index 00000000000..1f299c856bd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt @@ -0,0 +1,3 @@ +Write a Rust SpacetimeDB module with a Preferences product type containing theme: String, email_notifications: bool, and timezone: String. Store it in a public profile table with id: u64 primary key. + +Add create_profile(id, theme, email_notifications, timezone) and update_theme(id, theme). update_theme must change only the nested theme and preserve both sibling fields. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt new file mode 100644 index 00000000000..5a877900a09 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt @@ -0,0 +1,3 @@ +Write a TypeScript SpacetimeDB module with a Preferences object type containing theme: string, emailNotifications: boolean, and timezone: string. Store it in a public profile table with id: u64 primary key. + +Add create_profile(id, theme, emailNotifications, timezone) and update_theme(id, theme). update_theme must change only the nested theme and preserve both sibling fields. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs new file mode 100644 index 00000000000..48de17c6c05 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs @@ -0,0 +1,36 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Workspace", Public = true)] public partial struct Workspace { [PrimaryKey] public ulong Id; } + [Table(Accessor = "Project", Public = true)] public partial struct Project { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong WorkspaceId; } + [Table(Accessor = "TaskItem", Public = true)] public partial struct TaskItem { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong ProjectId; } + [Table(Accessor = "TaskNote", Public = true)] public partial struct TaskNote { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong TaskId; } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + foreach (ulong id in new ulong[] { 1, 2 }) + { + ctx.Db.Workspace.Insert(new Workspace { Id = id }); + ctx.Db.Project.Insert(new Project { Id = id, WorkspaceId = id }); + ctx.Db.TaskItem.Insert(new TaskItem { Id = id, ProjectId = id }); + ctx.Db.TaskNote.Insert(new TaskNote { Id = id, TaskId = id }); + } + } + + [Reducer] + public static void DeleteWorkspace(ReducerContext ctx, ulong id) + { + foreach (var project in ctx.Db.Project.WorkspaceId.Filter(id).ToList()) + { + foreach (var task in ctx.Db.TaskItem.ProjectId.Filter(project.Id).ToList()) + { + foreach (var note in ctx.Db.TaskNote.TaskId.Filter(task.Id).ToList()) ctx.Db.TaskNote.Id.Delete(note.Id); + ctx.Db.TaskItem.Id.Delete(task.Id); + } + ctx.Db.Project.Id.Delete(project.Id); + } + ctx.Db.Workspace.Id.Delete(id); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs new file mode 100644 index 00000000000..685d18baecc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs @@ -0,0 +1,35 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = workspace, public)] +pub struct Workspace { #[primary_key] pub id: u64 } +#[table(accessor = project, public)] +pub struct Project { #[primary_key] pub id: u64, #[index(btree)] pub workspace_id: u64 } +#[table(accessor = task_item, public)] +pub struct TaskItem { #[primary_key] pub id: u64, #[index(btree)] pub project_id: u64 } +#[table(accessor = task_note, public)] +pub struct TaskNote { #[primary_key] pub id: u64, #[index(btree)] pub task_id: u64 } + +#[reducer] +pub fn seed(ctx: &ReducerContext) { + for id in [1, 2] { + ctx.db.workspace().insert(Workspace { id }); + ctx.db.project().insert(Project { id, workspace_id: id }); + ctx.db.task_item().insert(TaskItem { id, project_id: id }); + ctx.db.task_note().insert(TaskNote { id, task_id: id }); + } +} + +#[reducer] +pub fn delete_workspace(ctx: &ReducerContext, id: u64) { + let projects: Vec<_> = ctx.db.project().workspace_id().filter(id).collect(); + for project in projects { + let tasks: Vec<_> = ctx.db.task_item().project_id().filter(project.id).collect(); + for task in tasks { + let notes: Vec<_> = ctx.db.task_note().task_id().filter(task.id).collect(); + for note in notes { ctx.db.task_note().id().delete(note.id); } + ctx.db.task_item().id().delete(task.id); + } + ctx.db.project().id().delete(project.id); + } + ctx.db.workspace().id().delete(id); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts new file mode 100644 index 00000000000..db060c4a5e9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts @@ -0,0 +1,28 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const workspace = table({ name: 'workspace', public: true }, { id: t.u64().primaryKey() }); +const project = table({ name: 'project', public: true }, { id: t.u64().primaryKey(), workspaceId: t.u64().index('btree') }); +const taskItem = table({ name: 'task_item', public: true }, { id: t.u64().primaryKey(), projectId: t.u64().index('btree') }); +const taskNote = table({ name: 'task_note', public: true }, { id: t.u64().primaryKey(), taskId: t.u64().index('btree') }); +const spacetimedb = schema({ workspace, project, taskItem, taskNote }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + for (const id of [1n, 2n]) { + ctx.db.workspace.insert({ id }); + ctx.db.project.insert({ id, workspaceId: id }); + ctx.db.taskItem.insert({ id, projectId: id }); + ctx.db.taskNote.insert({ id, taskId: id }); + } +}); + +export const delete_workspace = spacetimedb.reducer({ id: t.u64() }, (ctx, { id }) => { + for (const project of [...ctx.db.project.workspaceId.filter(id)]) { + for (const task of [...ctx.db.taskItem.projectId.filter(project.id)]) { + for (const note of [...ctx.db.taskNote.taskId.filter(task.id)]) ctx.db.taskNote.id.delete(note.id); + ctx.db.taskItem.id.delete(task.id); + } + ctx.db.project.id.delete(project.id); + } + ctx.db.workspace.id.delete(id); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs new file mode 100644 index 00000000000..a51402e4f3f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs @@ -0,0 +1,45 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer, +}; +use crate::eval::{table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "seed", + vec![], + "seed_nested_rows", + )); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "delete_workspace", + vec![json!(1)], + "delete_workspace", + )); + for (table, scorer_id) in [ + ("workspace", "workspace_count"), + ("project", "project_count"), + ("task_item", "task_count"), + ("task_note", "note_count"), + ] { + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {}", table_name(table, lang)), + 1, + scorer_id, + Duration::from_secs(10), + )); + } + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt new file mode 100644 index 00000000000..53179698207 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt @@ -0,0 +1,3 @@ +Write a C# SpacetimeDB module with public Workspace, Project, TaskItem, and TaskNote tables. Each has an ulong primary key. Project has indexed WorkspaceId, TaskItem has indexed ProjectId, and TaskNote has indexed TaskId. + +Add Seed() creating two independent workspace→project→task→note chains using IDs 1 and 2. Add DeleteWorkspace(id) that deletes the selected workspace and every level of its descendants while preserving the unrelated chain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt new file mode 100644 index 00000000000..8dbbd333444 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt @@ -0,0 +1,3 @@ +Write a Rust SpacetimeDB module with public workspace, project, task_item, and task_note tables. Each has a u64 primary key. Project has indexed workspace_id, TaskItem has indexed project_id, and TaskNote has indexed task_id. + +Add seed() creating two independent workspace→project→task→note chains using IDs 1 and 2. Add delete_workspace(id) that deletes the selected workspace and every level of its descendants while preserving the unrelated chain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt new file mode 100644 index 00000000000..e8d30be14e6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt @@ -0,0 +1,3 @@ +Write a TypeScript SpacetimeDB module with public workspace, project, task_item, and task_note tables. Each has a u64 primary key. Project has indexed workspaceId, TaskItem has indexed projectId, and TaskNote has indexed taskId. + +Add seed() creating two independent workspace→project→task→note chains using IDs 1 and 2. Add delete_workspace(id) that deletes the selected workspace and every level of its descendants while preserving the unrelated chain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs new file mode 100644 index 00000000000..8bd8c979635 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "GeneratedValue", Public = true)] + public partial struct GeneratedValue + { + [PrimaryKey, AutoInc] public ulong Id; + public Timestamp CreatedAt; + public long RandomValue; + } + + [Reducer] + public static void Generate(ReducerContext ctx) => ctx.Db.GeneratedValue.Insert(new GeneratedValue + { + Id = 0, + CreatedAt = ctx.Timestamp, + RandomValue = ctx.Rng.NextInt64(), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs new file mode 100644 index 00000000000..c1c16b4eca3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs @@ -0,0 +1,19 @@ +use spacetimedb::{reducer, table, ReducerContext, Table, Timestamp}; + +#[table(accessor = generated_value, public)] +pub struct GeneratedValue { + #[primary_key] + #[auto_inc] + pub id: u64, + pub created_at: Timestamp, + pub random_value: u64, +} + +#[reducer] +pub fn generate(ctx: &ReducerContext) { + ctx.db.generated_value().insert(GeneratedValue { + id: 0, + created_at: ctx.timestamp, + random_value: ctx.random(), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts new file mode 100644 index 00000000000..eeae887f791 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts @@ -0,0 +1,16 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const generatedValue = table( + { name: 'generated_value', public: true }, + { id: t.u64().primaryKey().autoInc(), createdAt: t.timestamp(), randomValue: t.i64() } +); +const spacetimedb = schema({ generatedValue }); +export default spacetimedb; + +export const generate = spacetimedb.reducer(ctx => { + ctx.db.generatedValue.insert({ + id: 0n, + createdAt: ctx.timestamp, + randomValue: BigInt(ctx.random.integerInRange(0, Number.MAX_SAFE_INTEGER)), + }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs new file mode 100644 index 00000000000..df8acdce33d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs @@ -0,0 +1,23 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer( + host_url, + ReducerSqlCountConfig { + src_file: file!(), + route_tag, + reducer: "generate".into(), + args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("generated_value", lang)), + expected_count: 1, + id_str: "context_value_written", + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt new file mode 100644 index 00000000000..3067cabc484 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public GeneratedValue table containing Id: ulong auto-increment primary key, CreatedAt: Timestamp, and RandomValue: long. Add Generate() that stores ctx.Timestamp and a random long from ctx.Rng. Do not use a host clock or separately seeded Random. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt new file mode 100644 index 00000000000..1353d94fc59 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public generated_value table containing id: u64 auto-increment primary key, created_at: Timestamp, and random_value: u64. Add generate() that stores the reducer context timestamp and a u64 from the reducer context RNG. Do not use a host clock or separately seeded RNG. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt new file mode 100644 index 00000000000..f8311899670 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public generated_value table containing id: u64 auto-increment primary key, createdAt: Timestamp, and randomValue: i64. Add generate() that stores ctx.timestamp and a random integer from ctx.random. Do not use Date, Math.random, or a separately seeded RNG. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs new file mode 100644 index 00000000000..709ad50df32 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs @@ -0,0 +1,16 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "CommandResult", Public = true)] + public partial struct CommandResult + { + [PrimaryKey] public string RequestId; + public bool Success; + public string Message; + } + + [Reducer] + public static void RunCommand(ReducerContext ctx, string requestId, int value) => + ctx.Db.CommandResult.Insert(new CommandResult { RequestId = requestId, Success = true, Message = $"value={value}" }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs new file mode 100644 index 00000000000..146e5eefbd3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs @@ -0,0 +1,18 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = command_result, public)] +pub struct CommandResult { + #[primary_key] + pub request_id: String, + pub success: bool, + pub message: String, +} + +#[reducer] +pub fn run_command(ctx: &ReducerContext, request_id: String, value: i32) { + ctx.db.command_result().insert(CommandResult { + request_id, + success: true, + message: format!("value={value}"), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts new file mode 100644 index 00000000000..784561fc821 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts @@ -0,0 +1,17 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const commandResult = table( + { name: 'command_result', public: true }, + { requestId: t.string().primaryKey(), success: t.bool(), message: t.string() } +); +const spacetimedb = schema({ commandResult }); +export default spacetimedb; + +export const run_command = spacetimedb.reducer( + { requestId: t.string(), value: t.i32() }, + (ctx, { requestId, value }) => ctx.db.commandResult.insert({ + requestId, + success: true, + message: `value=${value}`, + }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs new file mode 100644 index 00000000000..5b5bf42e027 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("command_result", lang); + let columns = sql.cols(&["request_id", "success", "message"]).join(", "); + let request_id = ident("request_id", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "run_command".into(), + args: vec![json!("req-1"), json!(7)], + select_query: format!("SELECT {columns} FROM {table} WHERE {request_id}='req-1'"), + id_str: "result_row_written", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt new file mode 100644 index 00000000000..ea20c847c94 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public CommandResult table containing RequestId: string primary key, Success: bool, and Message: string. Add RunCommand(requestId, value). Reducers do not return application data; write a result row whose Message is "value=" and Success is true. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt new file mode 100644 index 00000000000..5096ccc8d82 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public command_result table containing request_id: String primary key, success: bool, and message: String. Add run_command(request_id, value). Reducers do not return application data; write a result row whose message is "value=" and success is true. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt new file mode 100644 index 00000000000..85e54e92c02 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public command_result table containing requestId: string primary key, success: boolean, and message: string. Add run_command(requestId, value). Reducers do not return application data; write a result row whose message is "value=" and success is true. diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 001197631c6..42acde35eee 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -224,6 +224,36 @@ mod queries_t_037_multi_column_filter { include!("../benchmarks/queries/t_037_multi_column_filter/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_055_atomic_idempotent_transfer { + include!("../benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_056_nested_update { + include!("../benchmarks/reducers/t_056_nested_update/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_057_nested_cascade_delete { + include!("../benchmarks/reducers/t_057_nested_cascade_delete/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_059_deterministic_context { + include!("../benchmarks/reducers/t_059_deterministic_context/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_060_reducer_result_table { + include!("../benchmarks/reducers/t_060_reducer_result_table/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod schema_t_012_spacetime_product_type { @@ -374,6 +404,11 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("queries", "t_035_select_distinct") => queries_t_035_select_distinct::spec, ("queries", "t_036_count_without_collect") => queries_t_036_count_without_collect::spec, ("queries", "t_037_multi_column_filter") => queries_t_037_multi_column_filter::spec, + ("reducers", "t_055_atomic_idempotent_transfer") => reducers_t_055_atomic_idempotent_transfer::spec, + ("reducers", "t_056_nested_update") => reducers_t_056_nested_update::spec, + ("reducers", "t_057_nested_cascade_delete") => reducers_t_057_nested_cascade_delete::spec, + ("reducers", "t_059_deterministic_context") => reducers_t_059_deterministic_context::spec, + ("reducers", "t_060_reducer_result_table") => reducers_t_060_reducer_result_table::spec, ("schema", "t_012_spacetime_product_type") => schema_t_012_spacetime_product_type::spec, ("schema", "t_013_spacetime_sum_type") => schema_t_013_spacetime_sum_type::spec, ("schema", "t_014_elementary_columns") => schema_t_014_elementary_columns::spec, From 0d3f74c1706e4d0f7f22e5f599f5c035c1a667f4 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:14:28 -0400 Subject: [PATCH 04/78] llm-benchmark: add scheduled batch deletion eval --- .../t_058_batched_delete/answers/csharp.cs | 37 ++++++++++++++++ .../t_058_batched_delete/answers/rust.rs | 44 +++++++++++++++++++ .../answers/typescript.ts | 35 +++++++++++++++ .../reducers/t_058_batched_delete/spec.rs | 40 +++++++++++++++++ .../t_058_batched_delete/tasks/csharp.txt | 3 ++ .../t_058_batched_delete/tasks/rust.txt | 3 ++ .../t_058_batched_delete/tasks/typescript.txt | 3 ++ .../xtask-llm-benchmark/src/eval/defaults.rs | 25 ++++++++++- tools/xtask-llm-benchmark/src/eval/scorers.rs | 40 +++++++++++++++++ .../src/generated/registry.rs | 7 +++ 10 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs new file mode 100644 index 00000000000..042dbd0ca33 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs @@ -0,0 +1,37 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "WorkItem", Public = true)] + public partial struct WorkItem { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong GroupId; } + + [Table(Accessor = "DeleteJob", Scheduled = nameof(RunDeleteBatch), ScheduledAt = nameof(DeleteJob.ScheduledAt))] + public partial struct DeleteJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong GroupId; + } + + private static void Enqueue(ReducerContext ctx, ulong groupId) => ctx.Db.DeleteJob.Insert(new DeleteJob + { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), + GroupId = groupId, + }); + + [Reducer] + public static void SeedGroup(ReducerContext ctx, ulong groupId, uint count) + { + for (uint offset = 0; offset < count; offset++) ctx.Db.WorkItem.Insert(new WorkItem { Id = groupId * 100 + offset, GroupId = groupId }); + } + + [Reducer] + public static void RequestDelete(ReducerContext ctx, ulong groupId) => Enqueue(ctx, groupId); + + [Reducer] + public static void RunDeleteBatch(ReducerContext ctx, DeleteJob job) + { + foreach (var row in ctx.Db.WorkItem.GroupId.Filter(job.GroupId).Take(2).ToList()) ctx.Db.WorkItem.Id.Delete(row.Id); + if (ctx.Db.WorkItem.GroupId.Filter(job.GroupId).Any()) Enqueue(ctx, job.GroupId); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs new file mode 100644 index 00000000000..40f21e21d96 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs @@ -0,0 +1,44 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = work_item, public)] +pub struct WorkItem { + #[primary_key] + pub id: u64, + #[index(btree)] + pub group_id: u64, +} + +#[table(accessor = delete_job, scheduled(run_delete_batch))] +pub struct DeleteJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub group_id: u64, +} + +fn enqueue(ctx: &ReducerContext, group_id: u64) { + ctx.db.delete_job().insert(DeleteJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + group_id, + }); +} + +#[reducer] +pub fn seed_group(ctx: &ReducerContext, group_id: u64, count: u32) { + for offset in 0..count { + ctx.db.work_item().insert(WorkItem { id: group_id * 100 + offset as u64, group_id }); + } +} + +#[reducer] +pub fn request_delete(ctx: &ReducerContext, group_id: u64) { enqueue(ctx, group_id); } + +#[reducer] +pub fn run_delete_batch(ctx: &ReducerContext, job: DeleteJob) { + let rows: Vec<_> = ctx.db.work_item().group_id().filter(job.group_id).take(2).collect(); + for row in rows { ctx.db.work_item().id().delete(row.id); } + if ctx.db.work_item().group_id().filter(job.group_id).next().is_some() { enqueue(ctx, job.group_id); } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts new file mode 100644 index 00000000000..4832038dc30 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts @@ -0,0 +1,35 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const workItem = table( + { name: 'work_item', public: true }, + { id: t.u64().primaryKey(), groupId: t.u64().index('btree') } +); +const deleteJob = table( + { name: 'delete_job', scheduled: (): any => runDeleteBatch }, + { scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), groupId: t.u64() } +); +const spacetimedb = schema({ workItem, deleteJob }); +export default spacetimedb; + +function enqueue(ctx: any, groupId: bigint) { + ctx.db.deleteJob.insert({ + scheduledId: 0n, + scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), + groupId, + }); +} + +export const seed_group = spacetimedb.reducer({ groupId: t.u64(), count: t.u32() }, (ctx, { groupId, count }) => { + for (let offset = 0; offset < count; offset++) ctx.db.workItem.insert({ id: groupId * 100n + BigInt(offset), groupId }); +}); +export const request_delete = spacetimedb.reducer({ groupId: t.u64() }, (ctx, { groupId }) => enqueue(ctx, groupId)); +export const runDeleteBatch = spacetimedb.reducer({ timer: deleteJob.rowType }, (ctx, { timer }) => { + let deleted = 0; + for (const row of ctx.db.workItem.groupId.filter(timer.groupId)) { + if (deleted === 2) break; + ctx.db.workItem.id.delete(row.id); + deleted++; + } + if (ctx.db.workItem.groupId.filter(timer.groupId)[Symbol.iterator]().next().done === false) enqueue(ctx, timer.groupId); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs new file mode 100644 index 00000000000..1bde4f8755c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs @@ -0,0 +1,40 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "seed_group", + vec![json!(7), json!(5)], + "seed_group", + )); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "request_delete", + vec![json!(7)], + "request_delete", + )); + let table = table_name("work_item", lang); + let group_id = ident("group_id", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {group_id}=7"), + 0, + "scheduled_batches_finish", + Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt new file mode 100644 index 00000000000..564e00ecfc4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt @@ -0,0 +1,3 @@ +Write a C# SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. + +Define a public WorkItem table with Id: ulong primary key and indexed GroupId: ulong. Define a private DeleteJob scheduled table with ScheduledId, ScheduledAt, and GroupId. Add SeedGroup(groupId, count), RequestDelete(groupId), and the scheduled reducer. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt new file mode 100644 index 00000000000..18328c64d6a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt @@ -0,0 +1,3 @@ +Write a Rust SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. + +Define a public work_item table with id: u64 primary key and indexed group_id: u64. Define a private delete_job scheduled table with scheduled_id, scheduled_at, and group_id. Add seed_group(group_id, count), request_delete(group_id), and the scheduled reducer. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt new file mode 100644 index 00000000000..4905cfa7394 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt @@ -0,0 +1,3 @@ +Write a TypeScript SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. + +Define a public work_item table with id: u64 primary key and indexed groupId: u64. Define a private delete_job scheduled table with scheduledId, scheduledAt, and groupId. Add seed_group(groupId, count), request_delete(groupId), and the scheduled reducer. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index ea40f7a8b93..dd34ee24fd3 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -1,7 +1,7 @@ use crate::bench::utils::sanitize_db_name; use crate::eval::scorers::{ - ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, - SqlCountOnlyScorer, SqlExecBothScorer, + EventuallySqlCountScorer, ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, + SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, }; use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSqlCountConfig}; use std::time::Duration; @@ -58,6 +58,27 @@ pub fn make_sql_count_only_scorer( }) } +pub fn make_eventually_sql_count_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + sql: impl Into, + expected: i64, + id_str: &'static str, + timeout: Duration, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(EventuallySqlCountScorer { + server: host_url.to_string(), + db: llm_db, + sql: sql.into(), + expected, + timeout, + id_str, + }) +} + pub fn make_reducer_data_parity_scorer(host_url: &str, cfg: ReducerDataParityConfig<'_>) -> Box { let (cat, task) = derive_cat_task_from_file(cfg.src_file); let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 716ba0e90c0..36f874f00da 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -701,6 +701,46 @@ pub struct SqlCountOnlyScorer { pub id_str: &'static str, } +pub struct EventuallySqlCountScorer { + pub server: String, + pub db: String, + pub sql: String, + pub expected: i64, + pub timeout: Duration, + pub id_str: &'static str, +} + +impl Scorer for EventuallySqlCountScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let started = Instant::now(); + loop { + let last = match sql_count(&self.db, &self.sql, Some(&self.server)) { + Ok(actual) if actual == self.expected => { + return ScoreDetails { + pass: true, + partial: 1.0, + notes: json!({ "sql": self.sql, "expected": self.expected, "actual": actual }), + }; + } + Ok(actual) => json!({ "actual": actual }), + Err(error) => json!({ "error": error }), + }; + if started.elapsed() >= self.timeout { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "sql": self.sql, "expected": self.expected, "last": last }), + }; + } + thread::sleep(Duration::from_millis(50)); + } + } +} + impl Scorer for SqlCountOnlyScorer { fn id(&self) -> &'static str { self.id_str diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 42acde35eee..a528dfca067 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -242,6 +242,12 @@ mod reducers_t_057_nested_cascade_delete { include!("../benchmarks/reducers/t_057_nested_cascade_delete/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_058_batched_delete { + include!("../benchmarks/reducers/t_058_batched_delete/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod reducers_t_059_deterministic_context { @@ -407,6 +413,7 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("reducers", "t_055_atomic_idempotent_transfer") => reducers_t_055_atomic_idempotent_transfer::spec, ("reducers", "t_056_nested_update") => reducers_t_056_nested_update::spec, ("reducers", "t_057_nested_cascade_delete") => reducers_t_057_nested_cascade_delete::spec, + ("reducers", "t_058_batched_delete") => reducers_t_058_batched_delete::spec, ("reducers", "t_059_deterministic_context") => reducers_t_059_deterministic_context::spec, ("reducers", "t_060_reducer_result_table") => reducers_t_060_reducer_result_table::spec, ("schema", "t_012_spacetime_product_type") => schema_t_012_spacetime_product_type::spec, From fb2f938838e1ac35a1196205745d5178761e88a6 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:22:18 -0400 Subject: [PATCH 05/78] llm-benchmark: add query builder view evals --- .../answers/csharp.cs | 17 +++++++++++++++ .../answers/rust.rs | 18 ++++++++++++++++ .../answers/typescript.ts | 14 +++++++++++++ .../views/t_062_semijoin_intersection/spec.rs | 15 +++++++++++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 8 +++++++ .../t_066_query_builder_view/answers/rust.rs | 10 +++++++++ .../answers/typescript.ts | 5 +++++ .../views/t_066_query_builder_view/spec.rs | 15 +++++++++++++ .../t_066_query_builder_view/tasks/csharp.txt | 1 + .../t_066_query_builder_view/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../t_067_view_primary_key/answers/csharp.cs | 7 +++++++ .../t_067_view_primary_key/answers/rust.rs | 5 +++++ .../answers/typescript.ts | 5 +++++ .../views/t_067_view_primary_key/spec.rs | 3 +++ .../t_067_view_primary_key/tasks/csharp.txt | 1 + .../t_067_view_primary_key/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../src/generated/registry.rs | 21 +++++++++++++++++++ 22 files changed, 152 insertions(+) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs new file mode 100644 index 00000000000..64ea861edfb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs @@ -0,0 +1,17 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Member", Public = true)] public partial struct Member { [PrimaryKey] public ulong Id; public string Name; } + [Table(Accessor = "Eligibility", Public = true)] public partial struct Eligibility { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong MemberId; } + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Member.Insert(new Member { Id = 1, Name = "Ada" }); + ctx.Db.Member.Insert(new Member { Id = 2, Name = "Grace" }); + ctx.Db.Eligibility.Insert(new Eligibility { Id = 10, MemberId = 1 }); + } + [SpacetimeDB.View(Accessor = "EligibleMember", Public = true)] + public static IQuery EligibleMember(ViewContext ctx) => + ctx.From.Eligibility().RightSemijoin(ctx.From.Member(), (eligibility, member) => eligibility.MemberId.Eq(member.Id)); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs new file mode 100644 index 00000000000..601a25f1070 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs @@ -0,0 +1,18 @@ +use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; + +#[table(accessor = member, public)] +pub struct Member { #[primary_key] pub id: u64, pub name: String } +#[table(accessor = eligibility, public)] +pub struct Eligibility { #[primary_key] pub id: u64, #[index(btree)] pub member_id: u64 } + +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.member().insert(Member { id: 1, name: "Ada".into() }); + ctx.db.member().insert(Member { id: 2, name: "Grace".into() }); + ctx.db.eligibility().insert(Eligibility { id: 10, member_id: 1 }); +} + +#[view(accessor = eligible_member, public)] +pub fn eligible_member(ctx: &ViewContext) -> impl Query { + ctx.from.eligibility().right_semijoin(ctx.from.member(), |eligibility, member| eligibility.member_id.eq(member.id)) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts new file mode 100644 index 00000000000..aa887f103e8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts @@ -0,0 +1,14 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const member = table({ name: 'member', public: true }, { id: t.u64().primaryKey(), name: t.string() }); +const eligibility = table({ name: 'eligibility', public: true }, { id: t.u64().primaryKey(), memberId: t.u64().index('btree') }); +const spacetimedb = schema({ member, eligibility }); +export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { + ctx.db.member.insert({ id: 1n, name: 'Ada' }); ctx.db.member.insert({ id: 2n, name: 'Grace' }); + ctx.db.eligibility.insert({ id: 10n, memberId: 1n }); +}); +export const eligible_member = spacetimedb.view( + { name: 'eligible_member', public: true }, t.array(member.rowType), + ctx => ctx.from.eligibility.rightSemijoin(ctx.from.member, (eligibility, member) => eligibility.memberId.eq(member.id)) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs new file mode 100644 index 00000000000..28d83f6b87e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs @@ -0,0 +1,15 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { + src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("eligible_member", lang)), + expected_count: 1, id_str: "semijoin_intersection", timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt new file mode 100644 index 00000000000..fde1d658389 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with public Member(Id primary key, Name) and Eligibility(Id primary key, indexed MemberId) tables. Add Seed() with two members but eligibility for only member 1. Expose a public EligibleMember view using a query-builder semijoin that returns members only when a matching eligibility row exists. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt new file mode 100644 index 00000000000..90a63f186ec --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with public member(id primary key, name) and eligibility(id primary key, indexed member_id) tables. Add seed() with two members but eligibility for only member 1. Expose a public eligible_member view using a query-builder semijoin that returns members only when a matching eligibility row exists. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt new file mode 100644 index 00000000000..b8a6d1b3cd6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with public member(id primary key, name) and eligibility(id primary key, indexed memberId) tables. Add seed() with two members but eligibility for only member 1. Expose a public eligible_member view using a query-builder semijoin that returns members only when a matching eligibility row exists. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs new file mode 100644 index 00000000000..b6694a6ac78 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs @@ -0,0 +1,8 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "Ticket", Public = true)] public partial struct Ticket { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public string Status; public string Title; } + [Reducer] public static void Seed(ReducerContext ctx) { ctx.Db.Ticket.Insert(new Ticket { Id = 1, Status = "open", Title = "A" }); ctx.Db.Ticket.Insert(new Ticket { Id = 2, Status = "closed", Title = "B" }); } + [SpacetimeDB.View(Accessor = "OpenTicket", Public = true)] + public static IQuery OpenTicket(ViewContext ctx) => ctx.From.Ticket().Where(ticket => ticket.Status.Eq("open")); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs new file mode 100644 index 00000000000..030479f11af --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs @@ -0,0 +1,10 @@ +use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; +#[table(accessor = ticket, public)] +pub struct Ticket { #[primary_key] pub id: u64, #[index(btree)] pub status: String, pub title: String } +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.ticket().insert(Ticket { id: 1, status: "open".into(), title: "A".into() }); + ctx.db.ticket().insert(Ticket { id: 2, status: "closed".into(), title: "B".into() }); +} +#[view(accessor = open_ticket, public)] +pub fn open_ticket(ctx: &ViewContext) -> impl Query { ctx.from.ticket().filter(|ticket| ticket.status.eq("open".to_string())) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts new file mode 100644 index 00000000000..1ed6045b4c6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts @@ -0,0 +1,5 @@ +import { schema, table, t } from 'spacetimedb/server'; +const ticket = table({ name: 'ticket', public: true }, { id: t.u64().primaryKey(), status: t.string().index('btree'), title: t.string() }); +const spacetimedb = schema({ ticket }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { ctx.db.ticket.insert({ id: 1n, status: 'open', title: 'A' }); ctx.db.ticket.insert({ id: 2n, status: 'closed', title: 'B' }); }); +export const open_ticket = spacetimedb.view({ name: 'open_ticket', public: true }, t.array(ticket.rowType), ctx => ctx.from.ticket.where(ticket => ticket.status.eq('open'))); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs new file mode 100644 index 00000000000..ae71428264b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs @@ -0,0 +1,15 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { + src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("open_ticket", lang)), + expected_count: 1, id_str: "query_builder_filter", timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt new file mode 100644 index 00000000000..a2918536e9b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public Ticket table containing Id: ulong primary key, indexed Status: string, and Title: string. Add Seed() with one open and one closed ticket. Define a public OpenTicket view that returns the query-builder filter for Status "open" rather than materializing rows manually. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt new file mode 100644 index 00000000000..a0cb8b67b3b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: String, and title: String. Add seed() with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt new file mode 100644 index 00000000000..afab7a2aec9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: string, and title: string. Add seed() with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs new file mode 100644 index 00000000000..df432aa7bf9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs @@ -0,0 +1,7 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "SourceRow", Public = true)] public partial struct SourceRow { [PrimaryKey] public ulong Id; public string Value; [SpacetimeDB.Index.BTree] public bool Visible; } + [SpacetimeDB.View(Accessor = "SourceView", Public = true, PrimaryKey = nameof(SourceRow.Id))] + public static IEnumerable SourceView(AnonymousViewContext ctx) => ctx.Db.SourceRow.Visible.Filter(true); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs new file mode 100644 index 00000000000..4604879a3e6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs @@ -0,0 +1,5 @@ +use spacetimedb::{table, view, AnonymousViewContext}; +#[table(accessor = source_row, public)] +pub struct SourceRow { #[primary_key] pub id: u64, pub value: String, #[index(btree)] pub visible: bool } +#[view(accessor = source_view, public, primary_key = id)] +pub fn source_view(ctx: &AnonymousViewContext) -> Vec { ctx.db.source_row().visible().filter(true).collect() } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts new file mode 100644 index 00000000000..b1f3b99b417 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts @@ -0,0 +1,5 @@ +import { schema, table, t } from 'spacetimedb/server'; +const SourceViewRow = t.row('SourceViewRow', { id: t.u64().primaryKey(), value: t.string(), visible: t.bool().index('btree') }); +const sourceRow = table({ name: 'source_row', public: true }, SourceViewRow); +const spacetimedb = schema({ sourceRow }); export default spacetimedb; +export const source_view = spacetimedb.anonymousView({ name: 'source_view', public: true }, t.array(SourceViewRow), ctx => Array.from(ctx.db.sourceRow.visible.filter(true))); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs new file mode 100644 index 00000000000..8fc7c4dda44 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs @@ -0,0 +1,3 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; +pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| default_schema_parity_scorers(host_url, file!(), route_tag)) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt new file mode 100644 index 00000000000..3512de8cd80 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public SourceRow table containing Id: ulong primary key, Value: string, and indexed Visible: bool. Expose visible rows through a public SourceView procedural view and explicitly declare Id as the view primary key. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt new file mode 100644 index 00000000000..98a8a4aa8eb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public source_row table containing id: u64 primary key, value: String, and indexed visible: bool. Expose visible rows through a public source_view procedural view and explicitly declare id as the view primary key. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt new file mode 100644 index 00000000000..ebc15400c35 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public source_row table containing id: u64 primary key, value: string, and indexed visible: boolean. Expose visible rows through a public source_view procedural view whose returned row type declares id as its primary key. diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index a528dfca067..e5bf693bb57 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -362,6 +362,24 @@ mod tables_t_054_special_types { include!("../benchmarks/tables/t_054_special_types/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_062_semijoin_intersection { + include!("../benchmarks/views/t_062_semijoin_intersection/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_066_query_builder_view { + include!("../benchmarks/views/t_066_query_builder_view/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_067_view_primary_key { + include!("../benchmarks/views/t_067_view_primary_key/spec.rs"); +} + pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { let task = task_root .file_name() @@ -433,6 +451,9 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("tables", "t_051_denormalized_index") => tables_t_051_denormalized_index::spec, ("tables", "t_052_autoinc_reference") => tables_t_052_autoinc_reference::spec, ("tables", "t_054_special_types") => tables_t_054_special_types::spec, + ("views", "t_062_semijoin_intersection") => views_t_062_semijoin_intersection::spec, + ("views", "t_066_query_builder_view") => views_t_066_query_builder_view::spec, + ("views", "t_067_view_primary_key") => views_t_067_view_primary_key::spec, _ => return Err(anyhow!("no spec registered for {}/{} (need spec.rs)", category, task)), }; From 644340ef89b30badcc3963b57265fd2cc6029459 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:31:03 -0400 Subject: [PATCH 06/78] llm-benchmark: add procedural view evals --- .../t_061_three_table_join/answers/csharp.cs | 18 ++++++++++++++++++ .../t_061_three_table_join/answers/rust.rs | 14 ++++++++++++++ .../answers/typescript.ts | 10 ++++++++++ .../views/t_061_three_table_join/spec.rs | 13 +++++++++++++ .../t_061_three_table_join/tasks/csharp.txt | 1 + .../t_061_three_table_join/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../t_064_index_and_filter/answers/csharp.cs | 15 +++++++++++++++ .../t_064_index_and_filter/answers/rust.rs | 16 ++++++++++++++++ .../answers/typescript.ts | 13 +++++++++++++ .../views/t_064_index_and_filter/spec.rs | 15 +++++++++++++++ .../t_064_index_and_filter/tasks/csharp.txt | 1 + .../t_064_index_and_filter/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../src/generated/registry.rs | 14 ++++++++++++++ 15 files changed, 134 insertions(+) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs new file mode 100644 index 00000000000..af579b9000d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs @@ -0,0 +1,18 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "Customer", Public = true)] public partial struct Customer { [PrimaryKey] public ulong Id; public string Name; } + [Table(Accessor = "Purchase", Public = true)] public partial struct Purchase { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong CustomerId; } + [Table(Accessor = "LineItem", Public = true)] public partial struct LineItem { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong PurchaseId; public string Sku; [SpacetimeDB.Index.BTree] public bool Visible; } + [SpacetimeDB.Type] public partial struct OrderLineDetail { public ulong LineId; public string CustomerName; public string Sku; } + [Reducer] public static void Seed(ReducerContext ctx) { ctx.Db.Customer.Insert(new Customer { Id = 1, Name = "Ada" }); ctx.Db.Purchase.Insert(new Purchase { Id = 10, CustomerId = 1 }); ctx.Db.LineItem.Insert(new LineItem { Id = 100, PurchaseId = 10, Sku = "SKU-1", Visible = true }); } + [SpacetimeDB.View(Accessor = "OrderLineDetail", Public = true)] + public static IEnumerable OrderLineDetails(AnonymousViewContext ctx) + { + foreach (var line in ctx.Db.LineItem.Visible.Filter(true)) { + var purchase = ctx.Db.Purchase.Id.Find(line.PurchaseId); if (purchase is null) continue; + var customer = ctx.Db.Customer.Id.Find(purchase.Value.CustomerId); if (customer is null) continue; + yield return new OrderLineDetail { LineId = line.Id, CustomerName = customer.Value.Name, Sku = line.Sku }; + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs new file mode 100644 index 00000000000..dc0ee315cd2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs @@ -0,0 +1,14 @@ +use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, SpacetimeType, Table}; +#[table(accessor = customer, public)] pub struct Customer { #[primary_key] pub id: u64, pub name: String } +#[table(accessor = purchase, public)] pub struct Purchase { #[primary_key] pub id: u64, #[index(btree)] pub customer_id: u64 } +#[table(accessor = line_item, public)] pub struct LineItem { #[primary_key] pub id: u64, #[index(btree)] pub purchase_id: u64, pub sku: String, #[index(btree)] pub visible: bool } +#[derive(SpacetimeType)] pub struct OrderLineDetail { pub line_id: u64, pub customer_name: String, pub sku: String } +#[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.customer().insert(Customer { id: 1, name: "Ada".into() }); ctx.db.purchase().insert(Purchase { id: 10, customer_id: 1 }); ctx.db.line_item().insert(LineItem { id: 100, purchase_id: 10, sku: "SKU-1".into(), visible: true }); } +#[view(accessor = order_line_detail, public)] +pub fn order_line_detail(ctx: &AnonymousViewContext) -> Vec { + ctx.db.line_item().visible().filter(true).filter_map(|line| { + let purchase = ctx.db.purchase().id().find(line.purchase_id)?; + let customer = ctx.db.customer().id().find(purchase.customer_id)?; + Some(OrderLineDetail { line_id: line.id, customer_name: customer.name, sku: line.sku }) + }).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts new file mode 100644 index 00000000000..bdcf65523ef --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts @@ -0,0 +1,10 @@ +import { schema, table, t } from 'spacetimedb/server'; +const customer = table({ name: 'customer', public: true }, { id: t.u64().primaryKey(), name: t.string() }); +const purchase = table({ name: 'purchase', public: true }, { id: t.u64().primaryKey(), customerId: t.u64().index('btree') }); +const lineItem = table({ name: 'line_item', public: true }, { id: t.u64().primaryKey(), purchaseId: t.u64().index('btree'), sku: t.string(), visible: t.bool().index('btree') }); +const OrderLineDetail = t.row('ThreeTableJoinRow', { lineId: t.u64(), customerName: t.string(), sku: t.string() }); +const spacetimedb = schema({ customer, purchase, lineItem }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { ctx.db.customer.insert({ id: 1n, name: 'Ada' }); ctx.db.purchase.insert({ id: 10n, customerId: 1n }); ctx.db.lineItem.insert({ id: 100n, purchaseId: 10n, sku: 'SKU-1', visible: true }); }); +export const order_line_detail = spacetimedb.anonymousView({ name: 'order_line_detail', public: true }, t.array(OrderLineDetail), ctx => { + const rows: Array<{ lineId: bigint; customerName: string; sku: string }> = []; for (const line of ctx.db.lineItem.visible.filter(true)) { const purchase = ctx.db.purchase.id.find(line.purchaseId); if (!purchase) continue; const customer = ctx.db.customer.id.find(purchase.customerId); if (customer) rows.push({ lineId: line.id, customerName: customer.name, sku: line.sku }); } return rows; +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs new file mode 100644 index 00000000000..777e7059caa --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs @@ -0,0 +1,13 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { + src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("order_line_detail", lang)), + expected_count: 1, id_str: "three_table_join", timeout: Duration::from_secs(10), + })); scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt new file mode 100644 index 00000000000..99b497746f4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with public Customer(Id, Name), Purchase(Id, CustomerId), and LineItem(Id, PurchaseId, Sku, indexed Visible) tables. IDs are ulong primary keys and relationship IDs are indexed. Add Seed() with one complete visible chain. Expose a public OrderLineDetail view returning LineId, CustomerName, and Sku by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt new file mode 100644 index 00000000000..4c88a932189 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with public customer(id, name), purchase(id, customer_id), and line_item(id, purchase_id, sku, indexed visible) tables. IDs are u64 primary keys and relationship IDs are indexed. Add seed() with one complete visible chain. Expose a public order_line_detail view returning line_id, customer_name, and sku by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt new file mode 100644 index 00000000000..99fda7ea833 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with public customer(id, name), purchase(id, customerId), and line_item(id, purchaseId, sku, indexed visible) tables. IDs are u64 primary keys and relationship IDs are indexed. Add seed() with one complete visible chain. Expose a public order_line_detail view returning lineId, customerName, and sku by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs new file mode 100644 index 00000000000..60aeb5c3d82 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs @@ -0,0 +1,15 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "Content", Public = true)] public partial struct Content { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public string Category; public bool Active; public int Score; } + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Content.Insert(new Content { Id = 1, Category = "news", Active = true, Score = 20 }); + ctx.Db.Content.Insert(new Content { Id = 2, Category = "news", Active = false, Score = 20 }); + ctx.Db.Content.Insert(new Content { Id = 3, Category = "news", Active = true, Score = 5 }); + ctx.Db.Content.Insert(new Content { Id = 4, Category = "sports", Active = true, Score = 20 }); + } + [SpacetimeDB.View(Accessor = "FeaturedContent", Public = true)] + public static IEnumerable FeaturedContent(AnonymousViewContext ctx) => ctx.Db.Content.Category.Filter("news").Where(row => row.Active && row.Score >= 10); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs new file mode 100644 index 00000000000..853331e9fb9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs @@ -0,0 +1,16 @@ +use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, Table}; +#[table(accessor = content, public)] +pub struct Content { #[primary_key] pub id: u64, #[index(btree)] pub category: String, pub active: bool, pub score: i32 } +#[reducer] +pub fn seed(ctx: &ReducerContext) { + for row in [ + Content { id: 1, category: "news".into(), active: true, score: 20 }, + Content { id: 2, category: "news".into(), active: false, score: 20 }, + Content { id: 3, category: "news".into(), active: true, score: 5 }, + Content { id: 4, category: "sports".into(), active: true, score: 20 }, + ] { ctx.db.content().insert(row); } +} +#[view(accessor = featured_content, public)] +pub fn featured_content(ctx: &AnonymousViewContext) -> Vec { + ctx.db.content().category().filter(&"news".to_string()).filter(|row| row.active && row.score >= 10).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts new file mode 100644 index 00000000000..a7d8093aa81 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts @@ -0,0 +1,13 @@ +import { schema, table, t } from 'spacetimedb/server'; +const content = table({ name: 'content', public: true }, { id: t.u64().primaryKey(), category: t.string().index('btree'), active: t.bool(), score: t.i32() }); +const spacetimedb = schema({ content }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { + ctx.db.content.insert({ id: 1n, category: 'news', active: true, score: 20 }); + ctx.db.content.insert({ id: 2n, category: 'news', active: false, score: 20 }); + ctx.db.content.insert({ id: 3n, category: 'news', active: true, score: 5 }); + ctx.db.content.insert({ id: 4n, category: 'sports', active: true, score: 20 }); +}); +export const featured_content = spacetimedb.anonymousView( + { name: 'featured_content', public: true }, t.array(content.rowType), + ctx => Array.from(ctx.db.content.category.filter('news')).filter(row => row.active && row.score >= 10) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs new file mode 100644 index 00000000000..92ef8e979bf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs @@ -0,0 +1,15 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { + src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("featured_content", lang)), + expected_count: 1, id_str: "indexed_candidates_are_filtered", timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt new file mode 100644 index 00000000000..97dfb07ebaf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public Content table containing Id: ulong primary key, indexed Category: string, Active: bool, and Score: int. Add Seed() with rows that separately fail category, active, and score checks plus one passing row. Expose a public FeaturedContent procedural view that first uses the Category index for "news", then filters for active rows with Score at least 10. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt new file mode 100644 index 00000000000..44edce017a8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public content table containing id: u64 primary key, indexed category: String, active: bool, and score: i32. Add seed() with rows that separately fail category, active, and score checks plus one passing row. Expose a public featured_content procedural view that first uses the category index for "news", then filters for active rows with score at least 10. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt new file mode 100644 index 00000000000..27ae0bdc4e8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public content table containing id: u64 primary key, indexed category: string, active: boolean, and score: i32. Add seed() with rows that separately fail category, active, and score checks plus one passing row. Expose a public featured_content procedural view that first uses the category index for "news", then filters for active rows with score at least 10. diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index e5bf693bb57..90a332c7e56 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -362,12 +362,24 @@ mod tables_t_054_special_types { include!("../benchmarks/tables/t_054_special_types/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_061_three_table_join { + include!("../benchmarks/views/t_061_three_table_join/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod views_t_062_semijoin_intersection { include!("../benchmarks/views/t_062_semijoin_intersection/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_064_index_and_filter { + include!("../benchmarks/views/t_064_index_and_filter/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod views_t_066_query_builder_view { @@ -451,7 +463,9 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("tables", "t_051_denormalized_index") => tables_t_051_denormalized_index::spec, ("tables", "t_052_autoinc_reference") => tables_t_052_autoinc_reference::spec, ("tables", "t_054_special_types") => tables_t_054_special_types::spec, + ("views", "t_061_three_table_join") => views_t_061_three_table_join::spec, ("views", "t_062_semijoin_intersection") => views_t_062_semijoin_intersection::spec, + ("views", "t_064_index_and_filter") => views_t_064_index_and_filter::spec, ("views", "t_066_query_builder_view") => views_t_066_query_builder_view::spec, ("views", "t_067_view_primary_key") => views_t_067_view_primary_key::spec, _ => return Err(anyhow!("no spec registered for {}/{} (need spec.rs)", category, task)), From 2f647d671f5d95bec114b4036e484e16356ae1d2 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:41:13 -0400 Subject: [PATCH 07/78] llm-benchmark: complete view eval coverage --- .../t_063_timestamp_window/answers/csharp.cs | 27 ++++++++ .../t_063_timestamp_window/answers/rust.rs | 28 +++++++++ .../answers/typescript.ts | 26 ++++++++ .../views/t_063_timestamp_window/spec.rs | 15 +++++ .../t_063_timestamp_window/tasks/csharp.txt | 1 + .../t_063_timestamp_window/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 54 ++++++++++++++++ .../answers/rust.rs | 62 +++++++++++++++++++ .../answers/typescript.ts | 53 ++++++++++++++++ .../t_065_materialized_aggregate/spec.rs | 17 +++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../src/generated/registry.rs | 14 +++++ 15 files changed, 302 insertions(+) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs new file mode 100644 index 00000000000..0a051a85ca4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs @@ -0,0 +1,27 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Event", Public = true)] + public partial struct Event + { + [PrimaryKey] public ulong Id; + [SpacetimeDB.Index.BTree] public Timestamp OccurredAt; + public string Label; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + long[] times = [100, 200, 300, 400, 500]; + for (var index = 0; index < times.Length; index++) + { + var micros = times[index]; + ctx.Db.Event.Insert(new Event { Id = (ulong)index + 1, OccurredAt = new Timestamp(micros), Label = $"event-{micros}" }); + } + } + + [SpacetimeDB.View(Accessor = "WindowEvent", Public = true)] + public static IEnumerable WindowEvent(AnonymousViewContext ctx) => + ctx.Db.Event.OccurredAt.Filter((new Timestamp(200), new Timestamp(400))); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs new file mode 100644 index 00000000000..86a8ce431b9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs @@ -0,0 +1,28 @@ +use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, Table, Timestamp}; + +#[table(accessor = event, public)] +pub struct Event { + #[primary_key] + pub id: u64, + #[index(btree)] + pub occurred_at: Timestamp, + pub label: String, +} + +#[reducer] +pub fn seed(ctx: &ReducerContext) { + for (id, micros) in [100, 200, 300, 400, 500].into_iter().enumerate() { + ctx.db.event().insert(Event { + id: id as u64 + 1, + occurred_at: Timestamp::from_micros_since_unix_epoch(micros), + label: format!("event-{micros}"), + }); + } +} + +#[view(accessor = window_event, public)] +pub fn window_event(ctx: &AnonymousViewContext) -> Vec { + let start = Timestamp::from_micros_since_unix_epoch(200); + let end = Timestamp::from_micros_since_unix_epoch(400); + ctx.db.event().occurred_at().filter(start..=end).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts new file mode 100644 index 00000000000..251fcbb0597 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts @@ -0,0 +1,26 @@ +import { Timestamp } from 'spacetimedb'; +import { Range, schema, table, t } from 'spacetimedb/server'; + +const event = table({ name: 'event', public: true }, { + id: t.u64().primaryKey(), + occurredAt: t.timestamp().index('btree'), + label: t.string(), +}); + +const spacetimedb = schema({ event }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + [100n, 200n, 300n, 400n, 500n].forEach((micros, index) => { + ctx.db.event.insert({ id: BigInt(index + 1), occurredAt: new Timestamp(micros), label: `event-${micros}` }); + }); +}); + +export const window_event = spacetimedb.anonymousView( + { name: 'window_event', public: true }, + t.array(event.rowType), + ctx => Array.from(ctx.db.event.occurredAt.filter(new Range( + { tag: 'included', value: new Timestamp(200n) }, + { tag: 'included', value: new Timestamp(400n) }, + ))) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs new file mode 100644 index 00000000000..441010dcad0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs @@ -0,0 +1,15 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { + src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("window_event", lang)), + expected_count: 3, id_str: "inclusive_timestamp_window", timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt new file mode 100644 index 00000000000..22a53eab57c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public Event table containing Id: ulong primary key, indexed OccurredAt: Timestamp, and Label: string. Add Seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public WindowEvent procedural view that uses the OccurredAt index to return the inclusive range from 200 through 400 microseconds. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt new file mode 100644 index 00000000000..3f089e547c2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public event table containing id: u64 primary key, indexed occurred_at: Timestamp, and label: String. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurred_at index to return the inclusive range from 200 through 400 microseconds. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt new file mode 100644 index 00000000000..0f7f8d1035f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public event table containing id: u64 primary key, indexed occurredAt: Timestamp, and label: string. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurredAt index to return the inclusive range from 200 through 400 microseconds. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs new file mode 100644 index 00000000000..989a4fc7abe --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs @@ -0,0 +1,54 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Sale", Public = true)] + public partial struct Sale { [PrimaryKey] public ulong Id; public string Category; public long Amount; } + + [Table(Accessor = "CategoryTotal", Public = true)] + public partial struct CategoryTotal { [PrimaryKey] public string Category; public long TotalAmount; public ulong SaleCount; } + + private static void AddToTotal(ReducerContext ctx, string category, long amount) + { + var total = ctx.Db.CategoryTotal.Category.Find(category); + if (total is null) ctx.Db.CategoryTotal.Insert(new CategoryTotal { Category = category, TotalAmount = amount, SaleCount = 1 }); + else { var row = total.Value; row.TotalAmount += amount; row.SaleCount += 1; ctx.Db.CategoryTotal.Category.Update(row); } + } + + private static void RemoveFromTotal(ReducerContext ctx, string category, long amount) + { + var row = ctx.Db.CategoryTotal.Category.Find(category) ?? throw new InvalidOperationException("missing category total"); + if (row.SaleCount == 1) ctx.Db.CategoryTotal.Category.Delete(category); + else { row.TotalAmount -= amount; row.SaleCount -= 1; ctx.Db.CategoryTotal.Category.Update(row); } + } + + private static void UpsertSale(ReducerContext ctx, Sale sale) + { + var old = ctx.Db.Sale.Id.Find(sale.Id); + if (old is null) ctx.Db.Sale.Insert(sale); + else { RemoveFromTotal(ctx, old.Value.Category, old.Value.Amount); ctx.Db.Sale.Id.Update(sale); } + AddToTotal(ctx, sale.Category, sale.Amount); + } + + private static void DeleteSale(ReducerContext ctx, ulong id) + { + var old = ctx.Db.Sale.Id.Find(id); + if (old is null) return; + ctx.Db.Sale.Id.Delete(id); + RemoveFromTotal(ctx, old.Value.Category, old.Value.Amount); + } + + [Reducer] + public static void Exercise(ReducerContext ctx) + { + UpsertSale(ctx, new Sale { Id = 1, Category = "books", Amount = 10 }); + UpsertSale(ctx, new Sale { Id = 2, Category = "books", Amount = 20 }); + UpsertSale(ctx, new Sale { Id = 2, Category = "books", Amount = 25 }); + UpsertSale(ctx, new Sale { Id = 3, Category = "games", Amount = 40 }); + DeleteSale(ctx, 3); + DeleteSale(ctx, 1); + } + + [SpacetimeDB.View(Accessor = "CategorySummary", Public = true)] + public static IQuery CategorySummary(ViewContext ctx) => ctx.From.CategoryTotal(); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs new file mode 100644 index 00000000000..e01c0e461d7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs @@ -0,0 +1,62 @@ +use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; + +#[table(accessor = sale, public)] +pub struct Sale { #[primary_key] pub id: u64, pub category: String, pub amount: i64 } + +#[table(accessor = category_total, public)] +pub struct CategoryTotal { #[primary_key] pub category: String, pub total_amount: i64, pub sale_count: u64 } + +fn add_to_total(ctx: &ReducerContext, category: &String, amount: i64) { + if let Some(mut total) = ctx.db.category_total().category().find(category) { + total.total_amount += amount; + total.sale_count += 1; + ctx.db.category_total().category().update(total); + } else { + ctx.db.category_total().insert(CategoryTotal { category: category.clone(), total_amount: amount, sale_count: 1 }); + } +} + +fn remove_from_total(ctx: &ReducerContext, category: &String, amount: i64) { + let mut total = ctx.db.category_total().category().find(category).expect("missing category total"); + if total.sale_count == 1 { + ctx.db.category_total().category().delete(category); + } else { + total.total_amount -= amount; + total.sale_count -= 1; + ctx.db.category_total().category().update(total); + } +} + +fn upsert_sale(ctx: &ReducerContext, sale: Sale) { + let id = sale.id; + let category = sale.category.clone(); + let amount = sale.amount; + if let Some(old) = ctx.db.sale().id().find(sale.id) { + remove_from_total(ctx, &old.category, old.amount); + ctx.db.sale().id().update(sale); + } else { + ctx.db.sale().insert(sale); + } + debug_assert!(ctx.db.sale().id().find(id).is_some()); + add_to_total(ctx, &category, amount); +} + +fn delete_sale(ctx: &ReducerContext, id: u64) { + if let Some(old) = ctx.db.sale().id().find(id) { + ctx.db.sale().id().delete(id); + remove_from_total(ctx, &old.category, old.amount); + } +} + +#[reducer] +pub fn exercise(ctx: &ReducerContext) { + upsert_sale(ctx, Sale { id: 1, category: "books".into(), amount: 10 }); + upsert_sale(ctx, Sale { id: 2, category: "books".into(), amount: 20 }); + upsert_sale(ctx, Sale { id: 2, category: "books".into(), amount: 25 }); + upsert_sale(ctx, Sale { id: 3, category: "games".into(), amount: 40 }); + delete_sale(ctx, 3); + delete_sale(ctx, 1); +} + +#[view(accessor = category_summary, public)] +pub fn category_summary(ctx: &ViewContext) -> impl Query { ctx.from.category_total() } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts new file mode 100644 index 00000000000..d3542e12993 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts @@ -0,0 +1,53 @@ +import { schema, table, t } from 'spacetimedb/server'; +import type { InferSchema, ReducerCtx } from 'spacetimedb/server'; + +const sale = table({ name: 'sale', public: true }, { + id: t.u64().primaryKey(), category: t.string(), amount: t.i64(), +}); +const categoryTotal = table({ name: 'category_total', public: true }, { + category: t.string().primaryKey(), totalAmount: t.i64(), saleCount: t.u64(), +}); +const spacetimedb = schema({ sale, categoryTotal }); +export default spacetimedb; + +type Ctx = ReducerCtx>; + +function addToTotal(ctx: Ctx, category: string, amount: bigint) { + const total = ctx.db.categoryTotal.category.find(category); + if (total) ctx.db.categoryTotal.category.update({ ...total, totalAmount: total.totalAmount + amount, saleCount: total.saleCount + 1n }); + else ctx.db.categoryTotal.insert({ category, totalAmount: amount, saleCount: 1n }); +} + +function removeFromTotal(ctx: Ctx, category: string, amount: bigint) { + const total = ctx.db.categoryTotal.category.find(category); + if (!total) throw new Error('missing category total'); + if (total.saleCount === 1n) ctx.db.categoryTotal.category.delete(category); + else ctx.db.categoryTotal.category.update({ ...total, totalAmount: total.totalAmount - amount, saleCount: total.saleCount - 1n }); +} + +function upsertSale(ctx: Ctx, row: { id: bigint; category: string; amount: bigint }) { + const old = ctx.db.sale.id.find(row.id); + if (old) { removeFromTotal(ctx, old.category, old.amount); ctx.db.sale.id.update(row); } + else ctx.db.sale.insert(row); + addToTotal(ctx, row.category, row.amount); +} + +function deleteSale(ctx: Ctx, id: bigint) { + const old = ctx.db.sale.id.find(id); + if (!old) return; + ctx.db.sale.id.delete(id); + removeFromTotal(ctx, old.category, old.amount); +} + +export const exercise = spacetimedb.reducer(ctx => { + upsertSale(ctx, { id: 1n, category: 'books', amount: 10n }); + upsertSale(ctx, { id: 2n, category: 'books', amount: 20n }); + upsertSale(ctx, { id: 2n, category: 'books', amount: 25n }); + upsertSale(ctx, { id: 3n, category: 'games', amount: 40n }); + deleteSale(ctx, 3n); + deleteSale(ctx, 1n); +}); + +export const category_summary = spacetimedb.view( + { name: 'category_summary', public: true }, t.array(categoryTotal.rowType), ctx => ctx.from.categoryTotal +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs new file mode 100644 index 00000000000..8202ff0bc15 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs @@ -0,0 +1,17 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let columns = sql.cols(&["category", "total_amount", "sale_count"]).join(", "); + scorers.push(make_reducer_data_parity_scorer(host_url, ReducerDataParityConfig { + src_file: file!(), route_tag, reducer: "exercise".into(), args: vec![], + select_query: format!("SELECT {columns} FROM {}", table_name("category_summary", lang)), + id_str: "aggregate_is_synchronized", collapse_ws: true, timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt new file mode 100644 index 00000000000..fd4514eab70 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with public Sale(Id: ulong primary key, Category: string, Amount: long) and CategoryTotal(Category: string primary key, TotalAmount: long, SaleCount: ulong) tables. Implement shared upsert and delete logic that keeps CategoryTotal transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Add Exercise() that inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose CategoryTotal through a public CategorySummary query-builder view. The final view must contain only books with TotalAmount 25 and SaleCount 1. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt new file mode 100644 index 00000000000..31ccfc0dbf1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with public sale(id: u64 primary key, category: String, amount: i64) and category_total(category: String primary key, total_amount: i64, sale_count: u64) tables. Implement shared upsert and delete logic that keeps category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Add exercise() that inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with total_amount 25 and sale_count 1. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt new file mode 100644 index 00000000000..f9cc937da92 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with public sale(id: u64 primary key, category: string, amount: i64) and category_total(category: string primary key, totalAmount: i64, saleCount: u64) tables. Implement shared upsert and delete logic that keeps category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Add exercise() that inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with totalAmount 25 and saleCount 1. diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 90a332c7e56..b17c885de2a 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -374,12 +374,24 @@ mod views_t_062_semijoin_intersection { include!("../benchmarks/views/t_062_semijoin_intersection/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_063_timestamp_window { + include!("../benchmarks/views/t_063_timestamp_window/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod views_t_064_index_and_filter { include!("../benchmarks/views/t_064_index_and_filter/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_065_materialized_aggregate { + include!("../benchmarks/views/t_065_materialized_aggregate/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod views_t_066_query_builder_view { @@ -465,7 +477,9 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("tables", "t_054_special_types") => tables_t_054_special_types::spec, ("views", "t_061_three_table_join") => views_t_061_three_table_join::spec, ("views", "t_062_semijoin_intersection") => views_t_062_semijoin_intersection::spec, + ("views", "t_063_timestamp_window") => views_t_063_timestamp_window::spec, ("views", "t_064_index_and_filter") => views_t_064_index_and_filter::spec, + ("views", "t_065_materialized_aggregate") => views_t_065_materialized_aggregate::spec, ("views", "t_066_query_builder_view") => views_t_066_query_builder_view::spec, ("views", "t_067_view_primary_key") => views_t_067_view_primary_key::spec, _ => return Err(anyhow!("no spec registered for {}/{} (need spec.rs)", category, task)), From ba06e21adce8bc716cd45be291e8577005b1dfb7 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:56:10 -0400 Subject: [PATCH 08/78] llm-benchmark: add auth and lifecycle evals --- .../t_068_secure_projection/answers/csharp.cs | 26 ++++++++++++ .../t_068_secure_projection/answers/rust.rs | 29 +++++++++++++ .../answers/typescript.ts | 20 +++++++++ .../auth/t_068_secure_projection/spec.rs | 17 ++++++++ .../t_068_secure_projection/tasks/csharp.txt | 1 + .../t_068_secure_projection/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 42 +++++++++++++++++++ .../answers/rust.rs | 41 ++++++++++++++++++ .../answers/typescript.ts | 26 ++++++++++++ .../t_069_scheduled_materialization/spec.rs | 19 +++++++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 36 ++++++++++++++++ .../answers/rust.rs | 35 ++++++++++++++++ .../answers/typescript.ts | 31 ++++++++++++++ .../t_070_connection_scoped_presence/spec.rs | 15 +++++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../t_071_scheduled_private/answers/csharp.cs | 32 ++++++++++++++ .../t_071_scheduled_private/answers/rust.rs | 31 ++++++++++++++ .../answers/typescript.ts | 24 +++++++++++ .../lifecycle/t_071_scheduled_private/spec.rs | 22 ++++++++++ .../t_071_scheduled_private/tasks/csharp.txt | 1 + .../t_071_scheduled_private/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/typescript.ts | 4 +- .../src/generated/registry.rs | 28 +++++++++++++ 30 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs new file mode 100644 index 00000000000..c936d35d44d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs @@ -0,0 +1,26 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "SecretNote")] + public partial struct SecretNote + { + [PrimaryKey] public ulong Id; + [SpacetimeDB.Index.BTree] public Identity Owner; + public string Title; + public string SecretBody; + } + + [SpacetimeDB.Type] + public partial struct SafeNote { public ulong Id; public string Title; } + + [Reducer] + public static void SeedPrivateNote(ReducerContext ctx) => ctx.Db.SecretNote.Insert(new SecretNote + { + Id = 1, Owner = ctx.Sender, Title = "Visible title", SecretBody = "never expose this", + }); + + [SpacetimeDB.View(Accessor = "MySafeNote", Public = true)] + public static IEnumerable MySafeNote(ViewContext ctx) => + ctx.Db.SecretNote.Owner.Filter(ctx.Sender).Select(note => new SafeNote { Id = note.Id, Title = note.Title }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs new file mode 100644 index 00000000000..b7c75d6b7a2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs @@ -0,0 +1,29 @@ +use spacetimedb::{reducer, table, view, Identity, ReducerContext, SpacetimeType, Table, ViewContext}; + +#[table(accessor = secret_note)] +pub struct SecretNote { + #[primary_key] + pub id: u64, + #[index(btree)] + pub owner: Identity, + pub title: String, + pub secret_body: String, +} + +#[derive(SpacetimeType)] +pub struct SafeNote { pub id: u64, pub title: String } + +#[reducer] +pub fn seed_private_note(ctx: &ReducerContext) { + ctx.db.secret_note().insert(SecretNote { + id: 1, + owner: ctx.sender(), + title: "Visible title".into(), + secret_body: "never expose this".into(), + }); +} + +#[view(accessor = my_safe_note, public)] +pub fn my_safe_note(ctx: &ViewContext) -> Vec { + ctx.db.secret_note().owner().filter(ctx.sender()).map(|note| SafeNote { id: note.id, title: note.title }).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts new file mode 100644 index 00000000000..89ba18d21a7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts @@ -0,0 +1,20 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const secretNote = table({ name: 'secret_note' }, { + id: t.u64().primaryKey(), + owner: t.identity().index('btree'), + title: t.string(), + secretBody: t.string(), +}); +const SafeNote = t.row('SafeNoteRow', { id: t.u64(), title: t.string() }); +const spacetimedb = schema({ secretNote }); +export default spacetimedb; + +export const seed_private_note = spacetimedb.reducer(ctx => { + ctx.db.secretNote.insert({ id: 1n, owner: ctx.sender, title: 'Visible title', secretBody: 'never expose this' }); +}); + +export const my_safe_note = spacetimedb.view( + { name: 'my_safe_note', public: true }, t.array(SafeNote), + ctx => Array.from(ctx.db.secretNote.owner.filter(ctx.sender)).map(note => ({ id: note.id, title: note.title })) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs new file mode 100644 index 00000000000..6e9c69dcc1b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs @@ -0,0 +1,17 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let columns = sql.cols(&["id", "title"]).join(", "); + scorers.push(make_reducer_data_parity_scorer(host_url, ReducerDataParityConfig { + src_file: file!(), route_tag, reducer: "seed_private_note".into(), args: vec![], + select_query: format!("SELECT {columns} FROM {}", table_name("my_safe_note", lang)), + id_str: "caller_safe_projection", collapse_ws: true, timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt new file mode 100644 index 00000000000..bad513cf11b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a private SecretNote table containing Id: ulong primary key, indexed Owner: Identity, Title: string, and SecretBody: string. Add SeedPrivateNote() to insert one note for the caller. Expose a public per-user MySafeNote view that returns only the caller's Id and Title, never Owner or SecretBody. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt new file mode 100644 index 00000000000..b49962f8658 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a private secret_note table containing id: u64 primary key, indexed owner: Identity, title: String, and secret_body: String. Add seed_private_note() to insert one note for the caller. Expose a public per-user my_safe_note view that returns only the caller's id and title, never owner or secret_body. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt new file mode 100644 index 00000000000..a894ba6b864 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a private secret_note table containing id: u64 primary key, indexed owner: Identity, title: string, and secretBody: string. Add seed_private_note() to insert one note for the caller. Expose a public per-user my_safe_note view that returns only the caller's id and title, never owner or secretBody. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs new file mode 100644 index 00000000000..cf7fd14353b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs @@ -0,0 +1,42 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "MaterializedState", Public = true)] + public partial struct MaterializedState + { + [PrimaryKey] public ulong Id; + public string Status; + public ulong Version; + public Timestamp RefreshedAt; + } + + [Table(Accessor = "RefreshJob", Scheduled = nameof(RefreshMaterialized), ScheduledAt = nameof(RefreshJob.ScheduledAt))] + public partial struct RefreshJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong StateId; + } + + [Reducer] + public static void StartRefresh(ReducerContext ctx) + { + var pending = new MaterializedState { Id = 1, Status = "pending", Version = 0, RefreshedAt = ctx.Timestamp }; + if (ctx.Db.MaterializedState.Id.Find(1) is null) ctx.Db.MaterializedState.Insert(pending); + else ctx.Db.MaterializedState.Id.Update(pending); + ctx.Db.RefreshJob.Insert(new RefreshJob { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), StateId = 1, + }); + } + + [Reducer] + public static void RefreshMaterialized(ReducerContext ctx, RefreshJob job) + { + var state = ctx.Db.MaterializedState.Id.Find(job.StateId) ?? throw new InvalidOperationException("materialized state missing"); + state.Status = "ready"; + state.Version = 1; + state.RefreshedAt = ctx.Timestamp; + ctx.Db.MaterializedState.Id.Update(state); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs new file mode 100644 index 00000000000..f8e517e1843 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs @@ -0,0 +1,41 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table, Timestamp}; +use std::time::Duration; + +#[table(accessor = materialized_state, public)] +pub struct MaterializedState { + #[primary_key] + pub id: u64, + pub status: String, + pub version: u64, + pub refreshed_at: Timestamp, +} + +#[table(accessor = refresh_job, scheduled(refresh_materialized))] +pub struct RefreshJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub state_id: u64, +} + +#[reducer] +pub fn start_refresh(ctx: &ReducerContext) { + let pending = MaterializedState { id: 1, status: "pending".into(), version: 0, refreshed_at: ctx.timestamp }; + if ctx.db.materialized_state().id().find(1).is_some() { ctx.db.materialized_state().id().update(pending); } + else { ctx.db.materialized_state().insert(pending); } + ctx.db.refresh_job().insert(RefreshJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + state_id: 1, + }); +} + +#[reducer] +pub fn refresh_materialized(ctx: &ReducerContext, job: RefreshJob) { + let mut state = ctx.db.materialized_state().id().find(job.state_id).expect("materialized state missing"); + state.status = "ready".into(); + state.version = 1; + state.refreshed_at = ctx.timestamp; + ctx.db.materialized_state().id().update(state); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts new file mode 100644 index 00000000000..789712f83ef --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts @@ -0,0 +1,26 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const materializedState = table({ name: 'materialized_state', public: true }, { + id: t.u64().primaryKey(), status: t.string(), version: t.u64(), refreshedAt: t.timestamp(), +}); +const refreshJob = table({ name: 'refresh_job', scheduled: (): any => refreshMaterialized }, { + scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), stateId: t.u64(), +}); +const spacetimedb = schema({ materializedState, refreshJob }); +export default spacetimedb; + +export const start_refresh = spacetimedb.reducer(ctx => { + const pending = { id: 1n, status: 'pending', version: 0n, refreshedAt: ctx.timestamp }; + if (ctx.db.materializedState.id.find(1n)) ctx.db.materializedState.id.update(pending); + else ctx.db.materializedState.insert(pending); + ctx.db.refreshJob.insert({ + scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), stateId: 1n, + }); +}); + +export const refreshMaterialized = spacetimedb.reducer({ job: refreshJob.rowType }, (ctx, { job }) => { + const state = ctx.db.materializedState.id.find(job.stateId); + if (!state) throw new Error('materialized state missing'); + ctx.db.materializedState.id.update({ ...state, status: 'ready', version: 1n, refreshedAt: ctx.timestamp }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs new file mode 100644 index 00000000000..c7dd0d2103c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs @@ -0,0 +1,19 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer(host_url, file!(), route_tag, "start_refresh", vec![], "start_refresh")); + let table = table_name("materialized_state", lang); + let status = ident("status", casing_for_lang(lang)); + let version = ident("version", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, file!(), route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}='ready' AND {version}=1"), + 1, "scheduled_refresh_completes", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt new file mode 100644 index 00000000000..69d25eb3161 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public MaterializedState table containing Id: ulong primary key, Status: string, Version: ulong, and RefreshedAt: Timestamp. Add StartRefresh() to write a pending row and schedule a refresh job one millisecond later. The scheduled reducer must update the row to Status "ready", Version 1, and the scheduled reducer timestamp without any further client call. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt new file mode 100644 index 00000000000..aa8e5d75c2b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public materialized_state table containing id: u64 primary key, status: String, version: u64, and refreshed_at: Timestamp. Add start_refresh() to write a pending row and schedule a refresh job one millisecond later. The scheduled reducer must update the row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt new file mode 100644 index 00000000000..1e82356be39 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public materialized_state table containing id: u64 primary key, status: string, version: u64, and refreshedAt: Timestamp. Add start_refresh() to write a pending row and schedule a refresh job one millisecond later. The scheduled reducer must update the row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs new file mode 100644 index 00000000000..fa43d7ed84f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs @@ -0,0 +1,36 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "PresenceSession", Public = true)] + public partial struct PresenceSession + { + [PrimaryKey] public ConnectionId ConnectionId; + [SpacetimeDB.Index.BTree] public Identity Identity; + public Timestamp ConnectedAt; + } + + private static void AddSession(ReducerContext ctx, ConnectionId connectionId) => + ctx.Db.PresenceSession.Insert(new PresenceSession { ConnectionId = connectionId, Identity = ctx.Sender, ConnectedAt = ctx.Timestamp }); + + private static void RemoveSession(ReducerContext ctx, ConnectionId connectionId) => + ctx.Db.PresenceSession.ConnectionId.Delete(connectionId); + + [Reducer(ReducerKind.ClientConnected)] + public static void ClientConnected(ReducerContext ctx) => AddSession(ctx, ctx.ConnectionId ?? throw new InvalidOperationException("connection id missing")); + + [Reducer(ReducerKind.ClientDisconnected)] + public static void ClientDisconnected(ReducerContext ctx) => RemoveSession(ctx, ctx.ConnectionId ?? throw new InvalidOperationException("connection id missing")); + + [Reducer] + public static void ExercisePresence(ReducerContext ctx) + { + var firstBytes = new byte[16]; firstBytes[0] = 1; + var secondBytes = new byte[16]; secondBytes[0] = 2; + var first = ConnectionId.From(firstBytes) ?? throw new InvalidOperationException(); + var second = ConnectionId.From(secondBytes) ?? throw new InvalidOperationException(); + AddSession(ctx, first); + AddSession(ctx, second); + RemoveSession(ctx, first); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs new file mode 100644 index 00000000000..0bc4637029e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs @@ -0,0 +1,35 @@ +use spacetimedb::{reducer, table, ConnectionId, Identity, ReducerContext, Table, Timestamp}; + +#[table(accessor = presence_session, public)] +pub struct PresenceSession { + #[primary_key] + pub connection_id: ConnectionId, + #[index(btree)] + pub identity: Identity, + pub connected_at: Timestamp, +} + +fn add_session(ctx: &ReducerContext, connection_id: ConnectionId) { + ctx.db.presence_session().insert(PresenceSession { + connection_id, identity: ctx.sender(), connected_at: ctx.timestamp, + }); +} + +fn remove_session(ctx: &ReducerContext, connection_id: ConnectionId) { + ctx.db.presence_session().connection_id().delete(connection_id); +} + +#[reducer(client_connected)] +pub fn client_connected(ctx: &ReducerContext) { add_session(ctx, ctx.connection_id().expect("connection id missing")); } + +#[reducer(client_disconnected)] +pub fn client_disconnected(ctx: &ReducerContext) { remove_session(ctx, ctx.connection_id().expect("connection id missing")); } + +#[reducer] +pub fn exercise_presence(ctx: &ReducerContext) { + let first = ConnectionId::from_u128(1); + let second = ConnectionId::from_u128(2); + add_session(ctx, first); + add_session(ctx, second); + remove_session(ctx, first); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts new file mode 100644 index 00000000000..c833e4d48f6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts @@ -0,0 +1,31 @@ +import { ConnectionId } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; +import type { InferSchema, ReducerCtx } from 'spacetimedb/server'; + +const presenceSession = table({ name: 'presence_session', public: true }, { + connectionId: t.connectionId().primaryKey(), identity: t.identity().index('btree'), connectedAt: t.timestamp(), +}); +const spacetimedb = schema({ presenceSession }); +export default spacetimedb; +type Ctx = ReducerCtx>; + +function addSession(ctx: Ctx, connectionId: ConnectionId) { + ctx.db.presenceSession.insert({ connectionId, identity: ctx.sender, connectedAt: ctx.timestamp }); +} +function removeSession(ctx: Ctx, connectionId: ConnectionId) { ctx.db.presenceSession.connectionId.delete(connectionId); } + +export const clientConnected = spacetimedb.clientConnected(ctx => { + if (!ctx.connectionId) throw new Error('connection id missing'); + addSession(ctx, ctx.connectionId); +}); +export const clientDisconnected = spacetimedb.clientDisconnected(ctx => { + if (!ctx.connectionId) throw new Error('connection id missing'); + removeSession(ctx, ctx.connectionId); +}); +export const exercise_presence = spacetimedb.reducer(ctx => { + const first = new ConnectionId(1n); + const second = new ConnectionId(2n); + addSession(ctx, first); + addSession(ctx, second); + removeSession(ctx, first); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs new file mode 100644 index 00000000000..1f8a0154950 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs @@ -0,0 +1,15 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { + src_file: file!(), route_tag, reducer: "exercise_presence".into(), args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("presence_session", lang)), + expected_count: 1, id_str: "disconnect_removes_one_connection", timeout: Duration::from_secs(10), + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt new file mode 100644 index 00000000000..c817cdee964 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public PresenceSession table keyed by ConnectionId: ConnectionId and containing indexed Identity: Identity plus ConnectedAt: Timestamp. ClientConnected must insert one row per connection, and ClientDisconnected must delete only that connection's row. Share that logic with ExercisePresence(), which inserts two synthetic connections for the caller and removes only the first so one session remains. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt new file mode 100644 index 00000000000..b0ab6391bbf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public presence_session table keyed by connection_id: ConnectionId and containing indexed identity: Identity plus connected_at: Timestamp. Client-connected must insert one row per connection, and client-disconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first so one session remains. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt new file mode 100644 index 00000000000..3625e9999c5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public presence_session table keyed by connectionId: ConnectionId and containing indexed identity: Identity plus connectedAt: Timestamp. clientConnected must insert one row per connection, and clientDisconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first so one session remains. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs new file mode 100644 index 00000000000..b786613fae7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs @@ -0,0 +1,32 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "JobResult", Public = true)] + public partial struct JobResult { [PrimaryKey] public ulong Id; public string Status; } + + [Table(Accessor = "PrivateJob", Scheduled = nameof(RunPrivateJob), ScheduledAt = nameof(PrivateJob.ScheduledAt))] + public partial struct PrivateJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong ResultId; + } + + [Reducer] + public static void EnqueuePrivate(ReducerContext ctx, ulong id) + { + ctx.Db.JobResult.Insert(new JobResult { Id = id, Status = "queued" }); + ctx.Db.PrivateJob.Insert(new PrivateJob { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), ResultId = id, + }); + } + + [Reducer] + public static void RunPrivateJob(ReducerContext ctx, PrivateJob job) + { + var result = ctx.Db.JobResult.Id.Find(job.ResultId) ?? throw new InvalidOperationException("job result missing"); + result.Status = "complete"; + ctx.Db.JobResult.Id.Update(result); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs new file mode 100644 index 00000000000..12b086f0b75 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs @@ -0,0 +1,31 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = job_result, public)] +pub struct JobResult { #[primary_key] pub id: u64, pub status: String } + +#[table(accessor = private_job, scheduled(run_private_job))] +pub struct PrivateJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub result_id: u64, +} + +#[reducer] +pub fn enqueue_private(ctx: &ReducerContext, id: u64) { + ctx.db.job_result().insert(JobResult { id, status: "queued".into() }); + ctx.db.private_job().insert(PrivateJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + result_id: id, + }); +} + +#[reducer] +pub fn run_private_job(ctx: &ReducerContext, job: PrivateJob) { + let mut result = ctx.db.job_result().id().find(job.result_id).expect("job result missing"); + result.status = "complete".into(); + ctx.db.job_result().id().update(result); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts new file mode 100644 index 00000000000..29535b8a82a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts @@ -0,0 +1,24 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const jobResult = table({ name: 'job_result', public: true }, { + id: t.u64().primaryKey(), status: t.string(), +}); +const privateJob = table({ name: 'private_job', scheduled: (): any => runPrivateJob }, { + scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), resultId: t.u64(), +}); +const spacetimedb = schema({ jobResult, privateJob }); +export default spacetimedb; + +export const enqueue_private = spacetimedb.reducer({ id: t.u64() }, (ctx, { id }) => { + ctx.db.jobResult.insert({ id, status: 'queued' }); + ctx.db.privateJob.insert({ + scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), resultId: id, + }); +}); + +export const runPrivateJob = spacetimedb.reducer({ job: privateJob.rowType }, (ctx, { job }) => { + const result = ctx.db.jobResult.id.find(job.resultId); + if (!result) throw new Error('job result missing'); + ctx.db.jobResult.id.update({ ...result, status: 'complete' }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs new file mode 100644 index 00000000000..c6693feff71 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs @@ -0,0 +1,22 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, file!(), route_tag, "enqueue_private", vec![json!(7)], "enqueue_private", + )); + let table = table_name("job_result", lang); + let id = ident("id", casing_for_lang(lang)); + let status = ident("status", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, file!(), route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=7 AND {status}='complete'"), + 1, "private_scheduled_job_completes", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt new file mode 100644 index 00000000000..3e90a5df20a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public JobResult table containing Id: ulong primary key and Status: string. Add public EnqueuePrivate(Id) to write Status "queued" and insert a private schedule-table row for one millisecond later. Its scheduled RunPrivateJob reducer must be scheduler-only, not client-callable, and update the result to "complete". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt new file mode 100644 index 00000000000..07efb7cc04d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public job_result table containing id: u64 primary key and status: String. Add public enqueue_private(id) to write status "queued" and insert a private schedule-table row for one millisecond later. Its scheduled run_private_job reducer must be scheduler-only, not client-callable, and update the result to "complete". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt new file mode 100644 index 00000000000..80b9fdfb3fc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public job_result table containing id: u64 primary key and status: string. Add public enqueue_private(id) to write status "queued" and insert a private schedule-table row for one millisecond later. Its scheduled runPrivateJob reducer must be scheduler-only, not client-callable, and update the result to "complete". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts index 4832038dc30..ab06c4599a4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts @@ -1,5 +1,6 @@ import { ScheduleAt } from 'spacetimedb'; import { schema, table, t } from 'spacetimedb/server'; +import type { InferSchema, ReducerCtx } from 'spacetimedb/server'; const workItem = table( { name: 'work_item', public: true }, @@ -11,8 +12,9 @@ const deleteJob = table( ); const spacetimedb = schema({ workItem, deleteJob }); export default spacetimedb; +type Ctx = ReducerCtx>; -function enqueue(ctx: any, groupId: bigint) { +function enqueue(ctx: Ctx, groupId: bigint) { ctx.db.deleteJob.insert({ scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index b17c885de2a..3c474782ebe 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -50,6 +50,12 @@ mod auth_t_046_shared_document { include!("../benchmarks/auth/t_046_shared_document/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_068_secure_projection { + include!("../benchmarks/auth/t_068_secure_projection/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod basics_t_000_empty_reducers { @@ -176,6 +182,24 @@ mod data_modeling_t_031_unique_constraint { include!("../benchmarks/data_modeling/t_031_unique_constraint/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod lifecycle_t_069_scheduled_materialization { + include!("../benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod lifecycle_t_070_connection_scoped_presence { + include!("../benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod lifecycle_t_071_scheduled_private { + include!("../benchmarks/lifecycle/t_071_scheduled_private/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod queries_t_022_view_basic { @@ -423,6 +447,7 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("auth", "t_044_ban_list") => auth_t_044_ban_list::spec, ("auth", "t_045_rate_limit") => auth_t_045_rate_limit::spec, ("auth", "t_046_shared_document") => auth_t_046_shared_document::spec, + ("auth", "t_068_secure_projection") => auth_t_068_secure_projection::spec, ("basics", "t_000_empty_reducers") => basics_t_000_empty_reducers::spec, ("basics", "t_001_basic_tables") => basics_t_001_basic_tables::spec, ("basics", "t_002_scheduled_table") => basics_t_002_scheduled_table::spec, @@ -444,6 +469,9 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("data_modeling", "t_029_filter_and_aggregate") => data_modeling_t_029_filter_and_aggregate::spec, ("data_modeling", "t_030_two_table_join") => data_modeling_t_030_two_table_join::spec, ("data_modeling", "t_031_unique_constraint") => data_modeling_t_031_unique_constraint::spec, + ("lifecycle", "t_069_scheduled_materialization") => lifecycle_t_069_scheduled_materialization::spec, + ("lifecycle", "t_070_connection_scoped_presence") => lifecycle_t_070_connection_scoped_presence::spec, + ("lifecycle", "t_071_scheduled_private") => lifecycle_t_071_scheduled_private::spec, ("queries", "t_022_view_basic") => queries_t_022_view_basic::spec, ("queries", "t_023_view_per_user") => queries_t_023_view_per_user::spec, ("queries", "t_032_range_query") => queries_t_032_range_query::spec, From 8c510ff8d4fa07f0fd1981c22cb47ed1da8b8403 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:16:44 -0400 Subject: [PATCH 09/78] llm-benchmark: add procedure and http evals --- .../src/bench/templates.rs | 4 +- .../t_072_procedure_return/answers/csharp.cs | 12 ++ .../t_072_procedure_return/answers/rust.rs | 9 ++ .../answers/typescript.ts | 10 ++ .../procedures/t_072_procedure_return/spec.rs | 13 ++ .../t_072_procedure_return/tasks/csharp.txt | 1 + .../t_072_procedure_return/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../t_073_http_fetch/answers/csharp.cs | 26 ++++ .../t_073_http_fetch/answers/rust.rs | 15 ++ .../t_073_http_fetch/answers/typescript.ts | 19 +++ .../procedures/t_073_http_fetch/spec.rs | 13 ++ .../t_073_http_fetch/tasks/csharp.txt | 1 + .../t_073_http_fetch/tasks/rust.txt | 1 + .../t_073_http_fetch/tasks/typescript.txt | 1 + .../t_074_fetch_and_store/answers/csharp.cs | 20 +++ .../t_074_fetch_and_store/answers/rust.rs | 13 ++ .../answers/typescript.ts | 18 +++ .../procedures/t_074_fetch_and_store/spec.rs | 22 +++ .../t_074_fetch_and_store/tasks/csharp.txt | 1 + .../t_074_fetch_and_store/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 28 ++++ .../t_075_scheduled_procedure/answers/rust.rs | 28 ++++ .../answers/typescript.ts | 27 ++++ .../t_075_scheduled_procedure/spec.rs | 21 +++ .../tasks/csharp.txt | 1 + .../t_075_scheduled_procedure/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../t_076_http_handler/answers/csharp.cs | 15 ++ .../t_076_http_handler/answers/rust.rs | 11 ++ .../t_076_http_handler/answers/typescript.ts | 9 ++ .../procedures/t_076_http_handler/spec.rs | 12 ++ .../t_076_http_handler/tasks/csharp.txt | 1 + .../t_076_http_handler/tasks/rust.txt | 1 + .../t_076_http_handler/tasks/typescript.txt | 1 + .../t_077_http_router/answers/csharp.cs | 20 +++ .../t_077_http_router/answers/rust.rs | 13 ++ .../t_077_http_router/answers/typescript.ts | 12 ++ .../procedures/t_077_http_router/spec.rs | 19 +++ .../t_077_http_router/tasks/csharp.txt | 1 + .../t_077_http_router/tasks/rust.txt | 1 + .../t_077_http_router/tasks/typescript.txt | 1 + .../answers/csharp.cs | 33 +++++ .../t_078_idempotent_webhook/answers/rust.rs | 34 +++++ .../answers/typescript.ts | 27 ++++ .../t_078_idempotent_webhook/spec.rs | 28 ++++ .../t_078_idempotent_webhook/tasks/csharp.txt | 1 + .../t_078_idempotent_webhook/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 32 +++++ .../answers/rust.rs | 25 ++++ .../answers/typescript.ts | 25 ++++ .../t_079_external_upload_flow/spec.rs | 23 +++ .../tasks/csharp.txt | 1 + .../t_079_external_upload_flow/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../xtask-llm-benchmark/src/eval/defaults.rs | 52 ++++++- tools/xtask-llm-benchmark/src/eval/scorers.rs | 131 ++++++++++++++++++ .../src/generated/registry.rs | 56 ++++++++ .../src/templates/rust/server/Cargo.toml | 2 +- 61 files changed, 896 insertions(+), 5 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/bench/templates.rs b/tools/xtask-llm-benchmark/src/bench/templates.rs index 35176de8200..c54458a367f 100644 --- a/tools/xtask-llm-benchmark/src/bench/templates.rs +++ b/tools/xtask-llm-benchmark/src/bench/templates.rs @@ -131,11 +131,11 @@ fn inject_rust(root: &Path, llm_code: &str) -> anyhow::Result<()> { if !sdk_path.is_dir() { bail!("local Rust SDK not found at {}", sdk_path.display()); } - let replacement = format!(r#"spacetimedb = {{ path = "{}" }}"#, relative); + let replacement = format!(r#"spacetimedb = {{ path = "{}", features = ["unstable"] }}"#, relative); let cargo_toml = root.join("Cargo.toml"); let mut toml = fs::read_to_string(&cargo_toml).with_context(|| format!("read {}", cargo_toml.display()))?; toml = toml.replace( - "spacetimedb = { path = \"../../../../../../sdks/rust/\" }", + "spacetimedb = { path = \"../../../../../../sdks/rust/\", features = [\"unstable\"] }", &replacement, ); fs::write(&cargo_toml, toml).with_context(|| format!("write {}", cargo_toml.display()))?; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs new file mode 100644 index 00000000000..d8ab28724ea --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs @@ -0,0 +1,12 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +[SpacetimeDB.Type] +public partial struct Summary { public uint Total; public string Label; } + +public static partial class Module +{ + [SpacetimeDB.Procedure] + public static Summary CalculateSummary(ProcedureContext ctx, uint lhs, uint rhs) => + new() { Total = lhs + rhs, Label = "calculated" }; +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs new file mode 100644 index 00000000000..7942c99720c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs @@ -0,0 +1,9 @@ +use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; + +#[derive(SpacetimeType)] +pub struct Summary { pub total: u32, pub label: String } + +#[procedure] +pub fn calculate_summary(_ctx: &mut ProcedureContext, lhs: u32, rhs: u32) -> Summary { + Summary { total: lhs + rhs, label: "calculated".into() } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts new file mode 100644 index 00000000000..1a66b711d85 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts @@ -0,0 +1,10 @@ +import { schema, t } from 'spacetimedb/server'; + +const Summary = t.object('Summary', { total: t.u32(), label: t.string() }); +const spacetimedb = schema({}); +export default spacetimedb; + +export const calculate_summary = spacetimedb.procedure( + { lhs: t.u32(), rhs: t.u32() }, Summary, + (_ctx, { lhs, rhs }) => ({ total: lhs + rhs, label: 'calculated' }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs new file mode 100644 index 00000000000..87f2cdefd74 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs @@ -0,0 +1,13 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer}; +use crate::eval::BenchmarkSpec; +use serde_json::json; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_call_output_parity_scorer( + host_url, file!(), route_tag, "calculate_summary", vec![json!(7), json!(5)], "typed_procedure_return", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt new file mode 100644 index 00000000000..9452907e348 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a typed Summary result containing Total: uint and Label: string. Add a CalculateSummary(lhs: uint, rhs: uint) procedure that returns Total lhs + rhs and Label "calculated". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt new file mode 100644 index 00000000000..a94a4d672c5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a typed Summary result containing total: u32 and label: String. Add a calculate_summary(lhs: u32, rhs: u32) procedure that returns total lhs + rhs and label "calculated". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt new file mode 100644 index 00000000000..07438a3503b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a typed Summary result containing total: u32 and label: string. Add a calculate_summary(lhs: u32, rhs: u32) procedure that returns total lhs + rhs and label "calculated". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs new file mode 100644 index 00000000000..88134eda8b8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs @@ -0,0 +1,26 @@ +using SpacetimeDB; +using System.Text; +#pragma warning disable STDB_UNSTABLE + +[SpacetimeDB.Type] +public partial struct FetchSummary { public ushort Status; public bool JsonContentType; public bool HasTables; } + +public static partial class Module +{ + [SpacetimeDB.Procedure] + public static FetchSummary FetchSchemaSummary(ProcedureContext ctx, string serverUrl) + { + var url = $"{serverUrl.TrimEnd('/')}/v1/database/{ProcedureContextBase.Identity}/schema?version=9"; + var result = ctx.Http.Get(url); + return result.Match(response => { + var contentType = response.Headers.FirstOrDefault(h => h.Name.Equals("content-type", StringComparison.OrdinalIgnoreCase)); + var contentTypeValue = contentType.Value is null ? "" : Encoding.UTF8.GetString(contentType.Value); + var body = response.Body.ToStringUtf8Lossy(); + return new FetchSummary { + Status = response.StatusCode, + JsonContentType = contentTypeValue.Contains("application/json"), + HasTables = body.Contains("\"tables\""), + }; + }, error => throw new Exception(error.Message)); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs new file mode 100644 index 00000000000..d61da9cf8e9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs @@ -0,0 +1,15 @@ +use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; + +#[derive(SpacetimeType)] +pub struct FetchSummary { pub status: u16, pub json_content_type: bool, pub has_tables: bool } + +#[procedure] +pub fn fetch_schema_summary(ctx: &mut ProcedureContext, server_url: String) -> FetchSummary { + let url = format!("{}/v1/database/{}/schema?version=9", server_url.trim_end_matches('/'), ctx.database_identity()); + let response = ctx.http.get(url).expect("schema request failed"); + let status = response.status().as_u16(); + let json_content_type = response.headers().get("content-type") + .and_then(|value| value.to_str().ok()).is_some_and(|value| value.contains("application/json")); + let body = response.into_body().into_string_lossy(); + FetchSummary { status, json_content_type, has_tables: body.contains("\"tables\"") } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts new file mode 100644 index 00000000000..6edb0b73b16 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts @@ -0,0 +1,19 @@ +import { schema, t } from 'spacetimedb/server'; + +const FetchSummary = t.object('FetchSummary', { + status: t.u16(), jsonContentType: t.bool(), hasTables: t.bool(), +}); +const spacetimedb = schema({}); +export default spacetimedb; + +export const fetch_schema_summary = spacetimedb.procedure( + { serverUrl: t.string() }, FetchSummary, + (ctx, { serverUrl }) => { + const response = ctx.http.fetch(`${serverUrl.replace(/\/+$/, '')}/v1/database/${ctx.databaseIdentity}/schema?version=9`); + return { + status: response.status, + jsonContentType: (response.headers.get('content-type') ?? '').includes('application/json'), + hasTables: response.text().includes('"tables"'), + }; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs new file mode 100644 index 00000000000..35758fb4da1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs @@ -0,0 +1,13 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer}; +use crate::eval::BenchmarkSpec; +use serde_json::json; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_call_output_parity_scorer( + host_url, file!(), route_tag, "fetch_schema_summary", vec![json!(host_url)], "http_response_summary", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt new file mode 100644 index 00000000000..fc5f8ddd3cf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB procedure FetchSchemaSummary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a typed FetchSummary with the HTTP status, whether Content-Type contains "application/json", and whether the body contains the tables field. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt new file mode 100644 index 00000000000..9d432fb69d0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB procedure fetch_schema_summary(server_url: String) that performs an outbound GET to this database's schema endpoint. Return a typed FetchSummary with the HTTP status, whether Content-Type contains "application/json", and whether the body contains the tables field. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt new file mode 100644 index 00000000000..b927a98a4d2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB procedure fetch_schema_summary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a typed FetchSummary with the HTTP status, whether Content-Type contains "application/json", and whether the body contains the tables field. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs new file mode 100644 index 00000000000..d9b11eed48f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "FetchedRecord", Public = true)] + public partial struct FetchedRecord { [PrimaryKey] public ulong Id; public ushort Status; public bool ValidSchema; } + + [SpacetimeDB.Procedure] + public static void FetchAndStore(ProcedureContext ctx, string serverUrl) + { + var url = $"{serverUrl.TrimEnd('/')}/v1/database/{ProcedureContextBase.Identity}/schema?version=9"; + var result = ctx.Http.Get(url); + result.Match(response => { + var validSchema = response.Body.ToStringUtf8Lossy().Contains("\"tables\""); + ctx.WithTx(tx => { tx.Db.FetchedRecord.Insert(new FetchedRecord { Id = 1, Status = response.StatusCode, ValidSchema = validSchema }); return 0; }); + return 0; + }, error => throw new Exception(error.Message)); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs new file mode 100644 index 00000000000..e744709d8f3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs @@ -0,0 +1,13 @@ +use spacetimedb::{procedure, table, ProcedureContext, Table}; + +#[table(accessor = fetched_record, public)] +pub struct FetchedRecord { #[primary_key] pub id: u64, pub status: u16, pub valid_schema: bool } + +#[procedure] +pub fn fetch_and_store(ctx: &mut ProcedureContext, server_url: String) { + let url = format!("{}/v1/database/{}/schema?version=9", server_url.trim_end_matches('/'), ctx.database_identity()); + let response = ctx.http.get(url).expect("schema request failed"); + let status = response.status().as_u16(); + let valid_schema = response.into_body().into_string_lossy().contains("\"tables\""); + ctx.with_tx(|tx| { tx.db.fetched_record().insert(FetchedRecord { id: 1, status, valid_schema }); }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts new file mode 100644 index 00000000000..2a3c89d86f6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts @@ -0,0 +1,18 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const fetchedRecord = table({ name: 'fetched_record', public: true }, { + id: t.u64().primaryKey(), status: t.u16(), validSchema: t.bool(), +}); +const spacetimedb = schema({ fetchedRecord }); +export default spacetimedb; + +export const fetch_and_store = spacetimedb.procedure( + { serverUrl: t.string() }, t.unit(), + (ctx, { serverUrl }) => { + const response = ctx.http.fetch(`${serverUrl.replace(/\/+$/, '')}/v1/database/${ctx.databaseIdentity}/schema?version=9`); + const status = response.status; + const validSchema = response.text().includes('"tables"'); + ctx.withTx(tx => tx.db.fetchedRecord.insert({ id: 1n, status, validSchema })); + return {}; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs new file mode 100644 index 00000000000..b44bd1d7341 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs @@ -0,0 +1,22 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, file!(), route_tag, "fetch_and_store", vec![json!(host_url)], "fetch_and_store", + )); + let table = table_name("fetched_record", lang); + let status = ident("status", casing_for_lang(lang)); + let valid = ident("valid_schema", casing_for_lang(lang)); + scorers.push(make_sql_count_only_scorer( + host_url, file!(), route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}=200 AND {valid}=true"), + 1, "fetched_row_stored", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt new file mode 100644 index 00000000000..cfd6d5c7a72 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public FetchedRecord table containing Id: ulong primary key, Status: ushort, and ValidSchema: bool. Add FetchAndStore(serverUrl: string), which performs an outbound GET to this database's schema endpoint before opening a transaction, then stores Id 1, the response status, and whether the body contains the tables field inside a short transaction. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt new file mode 100644 index 00000000000..e40850893c9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and valid_schema: bool. Add fetch_and_store(server_url: String), which performs an outbound GET to this database's schema endpoint before opening a transaction, then stores id 1, the response status, and whether the body contains the tables field inside a short transaction. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt new file mode 100644 index 00000000000..57fe73cdf80 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and validSchema: bool. Add fetch_and_store(serverUrl: string), which performs an outbound GET to this database's schema endpoint before opening a transaction, then stores id 1, the response status, and whether the body contains the tables field inside a short transaction. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs new file mode 100644 index 00000000000..a54ba551864 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs @@ -0,0 +1,28 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "ProcedureResult", Public = true)] + public partial struct ProcedureResult { [PrimaryKey] public ulong Id; public uint Value; } + + [Table(Accessor = "ProcedureJob", Scheduled = nameof(RunScheduledProcedure), ScheduledAt = nameof(ProcedureJob.ScheduledAt))] + public partial struct ProcedureJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong Id; + public uint Lhs; + public uint Rhs; + } + + [Reducer] + public static void ScheduleProcedure(ReducerContext ctx, ulong id, uint lhs, uint rhs) => + ctx.Db.ProcedureJob.Insert(new ProcedureJob { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), Id = id, Lhs = lhs, Rhs = rhs, + }); + + [SpacetimeDB.Procedure] + public static void RunScheduledProcedure(ProcedureContext ctx, ProcedureJob job) => + ctx.WithTx(tx => { tx.Db.ProcedureResult.Insert(new ProcedureResult { Id = job.Id, Value = job.Lhs + job.Rhs }); return 0; }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs new file mode 100644 index 00000000000..7c9bff5e95a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs @@ -0,0 +1,28 @@ +use spacetimedb::{procedure, reducer, table, ProcedureContext, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = procedure_result, public)] +pub struct ProcedureResult { #[primary_key] pub id: u64, pub value: u32 } + +#[table(accessor = procedure_job, scheduled(run_scheduled_procedure))] +pub struct ProcedureJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub id: u64, + pub lhs: u32, + pub rhs: u32, +} + +#[reducer] +pub fn schedule_procedure(ctx: &ReducerContext, id: u64, lhs: u32, rhs: u32) { + ctx.db.procedure_job().insert(ProcedureJob { + scheduled_id: 0, scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), id, lhs, rhs, + }); +} + +#[procedure] +pub fn run_scheduled_procedure(ctx: &mut ProcedureContext, job: ProcedureJob) { + ctx.with_tx(|tx| { tx.db.procedure_result().insert(ProcedureResult { id: job.id, value: job.lhs + job.rhs }); }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts new file mode 100644 index 00000000000..28d1108af48 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts @@ -0,0 +1,27 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const procedureResult = table({ name: 'procedure_result', public: true }, { + id: t.u64().primaryKey(), value: t.u32(), +}); +const procedureJob = table({ name: 'procedure_job', scheduled: (): any => runScheduledProcedure }, { + scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), + id: t.u64(), lhs: t.u32(), rhs: t.u32(), +}); +const spacetimedb = schema({ procedureResult, procedureJob }); +export default spacetimedb; + +export const schedule_procedure = spacetimedb.reducer( + { id: t.u64(), lhs: t.u32(), rhs: t.u32() }, + (ctx, { id, lhs, rhs }) => ctx.db.procedureJob.insert({ + scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), id, lhs, rhs, + }) +); + +export const runScheduledProcedure = spacetimedb.procedure( + { job: procedureJob.rowType }, t.unit(), + (ctx, { job }) => { + ctx.withTx(tx => tx.db.procedureResult.insert({ id: job.id, value: job.lhs + job.rhs })); + return {}; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs new file mode 100644 index 00000000000..62ba0f29ca8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs @@ -0,0 +1,21 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, file!(), route_tag, "schedule_procedure", vec![json!(9), json!(7), json!(5)], "schedule_procedure", + )); + let table = table_name("procedure_result", lang); + let id = ident("id", casing_for_lang(lang)); + let value = ident("value", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, file!(), route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=9 AND {value}=12"), + 1, "scheduled_procedure_completes", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt new file mode 100644 index 00000000000..72885d1f502 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public ProcedureResult table containing Id: ulong primary key and Value: uint. Add ScheduleProcedure(id, lhs, rhs) to enqueue a procedure for one millisecond later. The scheduled procedure must open a transaction and insert Value lhs + rhs into ProcedureResult. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt new file mode 100644 index 00000000000..5bfcb9417da --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public procedure_result table containing id: u64 primary key and value: u32. Add schedule_procedure(id, lhs, rhs) to enqueue a procedure for one millisecond later. The scheduled procedure must open a transaction and insert value lhs + rhs into procedure_result. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt new file mode 100644 index 00000000000..92f0f8ee73c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public procedure_result table containing id: u64 primary key and value: u32. Add schedule_procedure(id, lhs, rhs) to enqueue a procedure for one millisecond later. The scheduled procedure must open a transaction and insert value lhs + rhs into procedure_result. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs new file mode 100644 index 00000000000..d3ce2fc9239 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs @@ -0,0 +1,15 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [SpacetimeDB.HttpHandler] + public static HttpResponse Echo(HandlerContext ctx, HttpRequest request) => new( + 201, HttpVersion.Http11, + new List { new("content-type", System.Text.Encoding.UTF8.GetBytes("text/plain")) }, + HttpBody.FromString($"echo:{request.Body.ToStringUtf8Lossy()}") + ); + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New().Post("/echo", Handlers.Echo); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs new file mode 100644 index 00000000000..5d0f0f1952a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs @@ -0,0 +1,11 @@ +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[handler] +fn echo(_ctx: &mut HandlerContext, request: Request) -> Response { + let body = request.into_body().into_string_lossy(); + Response::builder().status(201).header("content-type", "text/plain") + .body(Body::from_bytes(format!("echo:{body}"))).unwrap() +} + +#[router] +fn routes() -> Router { Router::new().post("/echo", echo) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts new file mode 100644 index 00000000000..4cd17257190 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts @@ -0,0 +1,9 @@ +import { Router, SyncResponse, schema } from 'spacetimedb/server'; + +const spacetimedb = schema({}); +export default spacetimedb; + +export const echo = spacetimedb.httpHandler((_ctx, request) => + new SyncResponse(`echo:${request.text()}`, { status: 201, headers: { 'content-type': 'text/plain' } }) +); +export const routes = spacetimedb.httpRouter(new Router().post('/echo', echo)); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs new file mode 100644 index 00000000000..8e4596af499 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs @@ -0,0 +1,12 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parity_scorer}; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_http_route_parity_scorer( + host_url, file!(), route_tag, vec![("POST", "/echo", Some("hello"))], "http_handler_response", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt new file mode 100644 index 00000000000..54974663f7b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with an inbound POST /echo HTTP handler. It must read the request body and return status 201, Content-Type text/plain, and body "echo:" followed by the input body. Register the handler through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt new file mode 100644 index 00000000000..0418b842212 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with an inbound POST /echo HTTP handler. It must read the request body and return status 201, Content-Type text/plain, and body "echo:" followed by the input body. Register the handler through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt new file mode 100644 index 00000000000..08a77d4a8e1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with an inbound POST /echo HTTP handler. It must read the request body and return status 201, Content-Type text/plain, and body "echo:" followed by the input body. Register the handler through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs new file mode 100644 index 00000000000..b6c460e20a4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [SpacetimeDB.HttpHandler] + public static HttpResponse ListItems(HandlerContext ctx, HttpRequest request) => new( + 200, HttpVersion.Http11, new List(), HttpBody.FromString("list") + ); + + [SpacetimeDB.HttpHandler] + public static HttpResponse CreateItem(HandlerContext ctx, HttpRequest request) => new( + 201, HttpVersion.Http11, new List(), HttpBody.FromString($"created:{request.Body.ToStringUtf8Lossy()}") + ); + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New() + .Get("/items", Handlers.ListItems) + .Post("/items", Handlers.CreateItem); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs new file mode 100644 index 00000000000..50441c35fe5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs @@ -0,0 +1,13 @@ +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[handler] +fn list_items(_ctx: &mut HandlerContext, _request: Request) -> Response { Response::new(Body::from_bytes("list")) } + +#[handler] +fn create_item(_ctx: &mut HandlerContext, request: Request) -> Response { + let body = request.into_body().into_string_lossy(); + Response::builder().status(201).body(Body::from_bytes(format!("created:{body}"))).unwrap() +} + +#[router] +fn routes() -> Router { Router::new().get("/items", list_items).post("/items", create_item) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts new file mode 100644 index 00000000000..1ab18ab0276 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts @@ -0,0 +1,12 @@ +import { Router, SyncResponse, schema } from 'spacetimedb/server'; + +const spacetimedb = schema({}); +export default spacetimedb; + +export const listItems = spacetimedb.httpHandler((_ctx, _request) => new SyncResponse('list')); +export const createItem = spacetimedb.httpHandler((_ctx, request) => + new SyncResponse(`created:${request.text()}`, { status: 201 }) +); +export const routes = spacetimedb.httpRouter( + new Router().get('/items', listItems).post('/items', createItem) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs new file mode 100644 index 00000000000..c6c67c098e3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs @@ -0,0 +1,19 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parity_scorer}; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_http_route_parity_scorer( + host_url, file!(), route_tag, + vec![ + ("GET", "/items", None), + ("POST", "/items", Some("book")), + ("PUT", "/items", Some("no")), + ("GET", "/missing", None), + ], + "router_method_matrix", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt new file mode 100644 index 00000000000..9b964a87e8e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB HTTP router with GET /items returning status 200 and body "list", and POST /items returning status 201 and body "created:" followed by the request body. Unsupported methods on /items and unknown paths must remain rejected by the router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt new file mode 100644 index 00000000000..ff0f9920ca7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB HTTP router with GET /items returning status 200 and body "list", and POST /items returning status 201 and body "created:" followed by the request body. Unsupported methods on /items and unknown paths must remain rejected by the router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt new file mode 100644 index 00000000000..5fddf222270 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB HTTP router with GET /items returning status 200 and body "list", and POST /items returning status 201 and body "created:" followed by the request body. Unsupported methods on /items and unknown paths must remain rejected by the router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs new file mode 100644 index 00000000000..3e84f4089e0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs @@ -0,0 +1,33 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "ProcessedEvent")] + public partial struct ProcessedEvent { [PrimaryKey] public string EventId; } + + [Table(Accessor = "WebhookState", Public = true)] + public partial struct WebhookState { [PrimaryKey] public string Key; public ulong LastSequence; public string Value; } + + [SpacetimeDB.HttpHandler] + public static HttpResponse Webhook(HandlerContext ctx, HttpRequest request) + { + var parts = request.Body.ToStringUtf8Lossy().Split('|', 3); + if (parts.Length != 3) return new(400, HttpVersion.Http11, new(), HttpBody.FromString("invalid")); + var eventId = parts[0]; var sequence = ulong.Parse(parts[1]); var value = parts[2]; + var outcome = ctx.WithTx(tx => { + if (tx.Db.ProcessedEvent.EventId.Find(eventId) is not null) return "duplicate"; + tx.Db.ProcessedEvent.Insert(new ProcessedEvent { EventId = eventId }); + var state = tx.Db.WebhookState.Key.Find("account"); + if (state is not null) { + if (sequence <= state.Value.LastSequence) return "stale"; + var row = state.Value; row.LastSequence = sequence; row.Value = value; tx.Db.WebhookState.Key.Update(row); + } else tx.Db.WebhookState.Insert(new WebhookState { Key = "account", LastSequence = sequence, Value = value }); + return "applied"; + }); + return new(200, HttpVersion.Http11, new(), HttpBody.FromString(outcome)); + } + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New().Post("/webhook", Handlers.Webhook); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs new file mode 100644 index 00000000000..0d4921ba5d2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs @@ -0,0 +1,34 @@ +use spacetimedb::{table, Table}; +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[table(accessor = processed_event)] +pub struct ProcessedEvent { #[primary_key] pub event_id: String } + +#[table(accessor = webhook_state, public)] +pub struct WebhookState { #[primary_key] pub key: String, pub last_sequence: u64, pub value: String } + +#[handler] +fn webhook(ctx: &mut HandlerContext, request: Request) -> Response { + let body = request.into_body().into_string_lossy(); + let parts: Vec<_> = body.splitn(3, '|').collect(); + if parts.len() != 3 { return Response::builder().status(400).body(Body::from_bytes("invalid")).unwrap(); } + let event_id = parts[0].to_string(); + let sequence: u64 = parts[1].parse().expect("invalid sequence"); + let value = parts[2].to_string(); + let outcome = ctx.with_tx(|tx| { + if tx.db.processed_event().event_id().find(&event_id).is_some() { return "duplicate"; } + tx.db.processed_event().insert(ProcessedEvent { event_id: event_id.clone() }); + let key = "account".to_string(); + if let Some(mut state) = tx.db.webhook_state().key().find(&key) { + if sequence <= state.last_sequence { return "stale"; } + state.last_sequence = sequence; state.value = value.clone(); tx.db.webhook_state().key().update(state); + } else { + tx.db.webhook_state().insert(WebhookState { key, last_sequence: sequence, value: value.clone() }); + } + "applied" + }); + Response::new(Body::from_bytes(outcome)) +} + +#[router] +fn routes() -> Router { Router::new().post("/webhook", webhook) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts new file mode 100644 index 00000000000..5c748904ba5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts @@ -0,0 +1,27 @@ +import { Router, SyncResponse, schema, table, t } from 'spacetimedb/server'; + +const processedEvent = table({ name: 'processed_event' }, { eventId: t.string().primaryKey() }); +const webhookState = table({ name: 'webhook_state', public: true }, { + key: t.string().primaryKey(), lastSequence: t.u64(), value: t.string(), +}); +const spacetimedb = schema({ processedEvent, webhookState }); +export default spacetimedb; + +export const webhook = spacetimedb.httpHandler((ctx, request) => { + const parts = request.text().split('|', 3); + if (parts.length !== 3) return new SyncResponse('invalid', { status: 400 }); + const [eventId, sequenceText, value] = parts; + const sequence = BigInt(sequenceText); + const outcome = ctx.withTx(tx => { + if (tx.db.processedEvent.eventId.find(eventId)) return 'duplicate'; + tx.db.processedEvent.insert({ eventId }); + const state = tx.db.webhookState.key.find('account'); + if (state) { + if (sequence <= state.lastSequence) return 'stale'; + tx.db.webhookState.key.update({ ...state, lastSequence: sequence, value }); + } else tx.db.webhookState.insert({ key: 'account', lastSequence: sequence, value }); + return 'applied'; + }); + return new SyncResponse(outcome); +}); +export const routes = spacetimedb.httpRouter(new Router().post('/webhook', webhook)); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs new file mode 100644 index 00000000000..c0913cd8579 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parity_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_http_route_parity_scorer( + host_url, file!(), route_tag, + vec![ + ("POST", "/webhook", Some("evt-1|2|new")), + ("POST", "/webhook", Some("evt-1|2|new")), + ("POST", "/webhook", Some("evt-2|1|old")), + ], + "webhook_idempotency", + )); + let table = table_name("webhook_state", lang); + let key = ident("key", casing_for_lang(lang)); + let sequence = ident("last_sequence", casing_for_lang(lang)); + let value = ident("value", casing_for_lang(lang)); + scorers.push(make_sql_count_only_scorer( + host_url, file!(), route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {key}='account' AND {sequence}=2 AND {value}='new'"), + 1, "webhook_state_is_current", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt new file mode 100644 index 00000000000..9fc233393b7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Persist processed event IDs and one public WebhookState row for key "account". Duplicate event IDs must return "duplicate" without applying twice, and events whose sequence is not newer must return "stale" without replacing state. Register the route through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt new file mode 100644 index 00000000000..6a36c9747d2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Persist processed event IDs and one public webhook_state row for key "account". Duplicate event IDs must return "duplicate" without applying twice, and events whose sequence is not newer must return "stale" without replacing state. Register the route through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt new file mode 100644 index 00000000000..0903cfa02b2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Persist processed event IDs and one public webhook_state row for key "account". Duplicate event IDs must return "duplicate" without applying twice, and events whose sequence is not newer must return "stale" without replacing state. Register the route through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs new file mode 100644 index 00000000000..423b653d114 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs @@ -0,0 +1,32 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "UploadedAsset", Public = true)] + public partial struct UploadedAsset { [PrimaryKey] public ulong Id; public string Url; public ulong Size; } + + [SpacetimeDB.HttpHandler] + public static HttpResponse Upload(HandlerContext ctx, HttpRequest request) => new( + 201, HttpVersion.Http11, new(), HttpBody.FromString("https://files.local/object-1") + ); + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New().Post("/upload", Handlers.Upload); + + [SpacetimeDB.Procedure] + public static string UploadAndRegister(ProcedureContext ctx, string serverUrl, byte[] data) + { + var request = new HttpRequest { + Uri = $"{serverUrl.TrimEnd('/')}/v1/database/{ProcedureContextBase.Identity}/route/upload", + Method = SpacetimeDB.HttpMethod.Post, + Headers = new() { new HttpHeader("content-type", "application/octet-stream") }, + Body = new HttpBody(data), + }; + return ctx.Http.Send(request).Match(response => { + var assetUrl = response.Body.ToStringUtf8Lossy(); + ctx.WithTx(tx => { tx.Db.UploadedAsset.Insert(new UploadedAsset { Id = 1, Url = assetUrl, Size = (ulong)data.Length }); return 0; }); + return assetUrl; + }, error => throw new Exception(error.Message)); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs new file mode 100644 index 00000000000..b8637a1e208 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs @@ -0,0 +1,25 @@ +use spacetimedb::{procedure, table, ProcedureContext, Table}; +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[table(accessor = uploaded_asset, public)] +pub struct UploadedAsset { #[primary_key] pub id: u64, pub url: String, pub size: u64 } + +#[handler] +fn upload(_ctx: &mut HandlerContext, _request: Request) -> Response { + Response::builder().status(201).body(Body::from_bytes("https://files.local/object-1")).unwrap() +} + +#[router] +fn routes() -> Router { Router::new().post("/upload", upload) } + +#[procedure] +pub fn upload_and_register(ctx: &mut ProcedureContext, server_url: String, data: Vec) -> String { + let url = format!("{}/v1/database/{}/route/upload", server_url.trim_end_matches('/'), ctx.database_identity()); + let request = Request::builder().method("POST").uri(url).header("content-type", "application/octet-stream") + .body(Body::from_bytes(data.clone())).unwrap(); + let response = ctx.http.send(request).expect("upload failed"); + let asset_url = response.into_body().into_string_lossy(); + let row_url = asset_url.clone(); + ctx.with_tx(|tx| { tx.db.uploaded_asset().insert(UploadedAsset { id: 1, url: row_url.clone(), size: data.len() as u64 }); }); + asset_url +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts new file mode 100644 index 00000000000..17873876a0b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts @@ -0,0 +1,25 @@ +import { Router, SyncResponse, schema, table, t } from 'spacetimedb/server'; + +const uploadedAsset = table({ name: 'uploaded_asset', public: true }, { + id: t.u64().primaryKey(), url: t.string(), size: t.u64(), +}); +const spacetimedb = schema({ uploadedAsset }); +export default spacetimedb; + +export const upload = spacetimedb.httpHandler((_ctx, _request) => + new SyncResponse('https://files.local/object-1', { status: 201 }) +); +export const routes = spacetimedb.httpRouter(new Router().post('/upload', upload)); + +export const upload_and_register = spacetimedb.procedure( + { serverUrl: t.string(), data: t.array(t.u8()) }, t.string(), + (ctx, { serverUrl, data }) => { + const response = ctx.http.fetch( + `${serverUrl.replace(/\/+$/, '')}/v1/database/${ctx.databaseIdentity}/route/upload`, + { method: 'POST', headers: { 'content-type': 'application/octet-stream' }, body: new Uint8Array(data) } + ); + const assetUrl = response.text(); + ctx.withTx(tx => tx.db.uploadedAsset.insert({ id: 1n, url: assetUrl, size: BigInt(data.length) })); + return assetUrl; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs new file mode 100644 index 00000000000..fb7c107e15e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs @@ -0,0 +1,23 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_call_output_parity_scorer( + host_url, file!(), route_tag, "upload_and_register", + vec![json!(host_url), json!([1, 2, 3, 4])], "upload_return_url", + )); + let table = table_name("uploaded_asset", lang); + let url = ident("url", casing_for_lang(lang)); + let size = ident("size", casing_for_lang(lang)); + scorers.push(make_sql_count_only_scorer( + host_url, file!(), route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {url}='https://files.local/object-1' AND {size}=4"), + 1, "upload_metadata_stored", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt new file mode 100644 index 00000000000..97f8170bd59 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public UploadedAsset table containing Id: ulong primary key, Url: string, and Size: ulong. Register a POST /upload handler that returns the URL "https://files.local/object-1". Add UploadAndRegister(serverUrl: string, data: byte[]) to POST the binary data to this database's upload route, then open a transaction to store Id 1, the returned URL, and the byte length, and return the URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt new file mode 100644 index 00000000000..dca9d03bb50 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: String, and size: u64. Register a POST /upload handler that returns the URL "https://files.local/object-1". Add upload_and_register(server_url: String, data: Vec) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt new file mode 100644 index 00000000000..3aa4bc5ac00 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: string, and size: u64. Register a POST /upload handler that returns the URL "https://files.local/object-1". Add upload_and_register(serverUrl: string, data: u8[]) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index dd34ee24fd3..b2ee77a3370 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -1,7 +1,7 @@ use crate::bench::utils::sanitize_db_name; use crate::eval::scorers::{ - EventuallySqlCountScorer, ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, - SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, + CallOutputParityScorer, EventuallySqlCountScorer, HttpRouteCase, HttpRouteParityScorer, ReducerCallBothScorer, + ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, }; use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSqlCountConfig}; use std::time::Duration; @@ -140,3 +140,51 @@ pub fn make_reducer_call_both_scorer( id_str, }) as Box } + +pub fn make_call_output_parity_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + function: &str, + args: Vec, + id_str: &'static str, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(CallOutputParityScorer { + server: host_url.to_string(), + golden_db, + llm_db, + function: function.to_string(), + args, + collapse_ws: true, + id_str, + }) +} + +pub fn make_http_route_parity_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + cases: Vec<(&str, &str, Option<&str>)>, + id_str: &'static str, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(HttpRouteParityScorer { + server: host_url.to_string(), + golden_db, + llm_db, + id_str, + cases: cases + .into_iter() + .map(|(method, path, body)| HttpRouteCase { + method: method.to_string(), + path: path.to_string(), + body: body.map(str::to_string), + }) + .collect(), + }) +} diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 36f874f00da..a1b35068b85 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -819,6 +819,137 @@ pub struct ReducerCallBothScorer { pub id_str: &'static str, } +pub struct CallOutputParityScorer { + pub server: String, + pub golden_db: String, + pub llm_db: String, + pub function: String, + pub args: Vec, + pub collapse_ws: bool, + pub id_str: &'static str, +} + +pub struct HttpRouteCase { + pub method: String, + pub path: String, + pub body: Option, +} + +pub struct HttpRouteParityScorer { + pub server: String, + pub golden_db: String, + pub llm_db: String, + pub cases: Vec, + pub id_str: &'static str, +} + +fn call_http_route(server: &str, db: &str, case: &HttpRouteCase) -> Result<(u16, String, String), String> { + let server = server.trim_end_matches('/').to_string(); + let db = db.to_string(); + let method = case.method.clone(); + let path = case.path.clone(); + let body = case.body.clone(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; + runtime.block_on(async move { + let method = reqwest::Method::from_bytes(method.as_bytes()).map_err(|error| error.to_string())?; + let url = format!("{server}/v1/database/{db}/route{path}"); + let mut request = reqwest::Client::new().request(method, url); + if let Some(body) = body { + request = request.header("content-type", "text/plain").body(body); + } + let response = request.send().await.map_err(|error| error.to_string())?; + let status = response.status().as_u16(); + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body = response.text().await.map_err(|error| error.to_string())?; + Ok((status, content_type, body)) + }) + }) + .join() + .map_err(|_| "HTTP route worker panicked".to_string())? +} + +impl Scorer for HttpRouteParityScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let mut golden_results = Vec::new(); + let mut llm_results = Vec::new(); + for case in &self.cases { + match call_http_route(&self.server, &self.golden_db, case) { + Ok(result) => golden_results.push(result), + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "http_golden", "error": error }), + } + } + } + match call_http_route(&self.server, &self.llm_db, case) { + Ok(result) => llm_results.push(result), + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "http_llm", "error": error }), + } + } + } + } + let pass = golden_results == llm_results; + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "golden": golden_results, "llm": llm_results }), + } + } +} + +impl Scorer for CallOutputParityScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let golden = match call_reducer_json_out(&self.golden_db, &self.function, &self.args, Some(&self.server)) { + Ok(output) => output, + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "call_golden", "function": self.function, "error": error }), + } + } + }; + let llm = match call_reducer_json_out(&self.llm_db, &self.function, &self.args, Some(&self.server)) { + Ok(output) => output, + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "call_llm", "function": self.function, "error": error }), + } + } + }; + let golden = normalize(&golden, self.collapse_ws); + let llm = normalize(&llm, self.collapse_ws); + let pass = golden == llm; + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "function": self.function, "golden": golden, "llm": llm }), + } + } +} + impl Scorer for ReducerCallBothScorer { fn id(&self) -> &'static str { self.id_str diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 3c474782ebe..363294b87b5 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -200,6 +200,54 @@ mod lifecycle_t_071_scheduled_private { include!("../benchmarks/lifecycle/t_071_scheduled_private/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_072_procedure_return { + include!("../benchmarks/procedures/t_072_procedure_return/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_073_http_fetch { + include!("../benchmarks/procedures/t_073_http_fetch/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_074_fetch_and_store { + include!("../benchmarks/procedures/t_074_fetch_and_store/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_075_scheduled_procedure { + include!("../benchmarks/procedures/t_075_scheduled_procedure/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_076_http_handler { + include!("../benchmarks/procedures/t_076_http_handler/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_077_http_router { + include!("../benchmarks/procedures/t_077_http_router/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_078_idempotent_webhook { + include!("../benchmarks/procedures/t_078_idempotent_webhook/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_079_external_upload_flow { + include!("../benchmarks/procedures/t_079_external_upload_flow/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod queries_t_022_view_basic { @@ -472,6 +520,14 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("lifecycle", "t_069_scheduled_materialization") => lifecycle_t_069_scheduled_materialization::spec, ("lifecycle", "t_070_connection_scoped_presence") => lifecycle_t_070_connection_scoped_presence::spec, ("lifecycle", "t_071_scheduled_private") => lifecycle_t_071_scheduled_private::spec, + ("procedures", "t_072_procedure_return") => procedures_t_072_procedure_return::spec, + ("procedures", "t_073_http_fetch") => procedures_t_073_http_fetch::spec, + ("procedures", "t_074_fetch_and_store") => procedures_t_074_fetch_and_store::spec, + ("procedures", "t_075_scheduled_procedure") => procedures_t_075_scheduled_procedure::spec, + ("procedures", "t_076_http_handler") => procedures_t_076_http_handler::spec, + ("procedures", "t_077_http_router") => procedures_t_077_http_router::spec, + ("procedures", "t_078_idempotent_webhook") => procedures_t_078_idempotent_webhook::spec, + ("procedures", "t_079_external_upload_flow") => procedures_t_079_external_upload_flow::spec, ("queries", "t_022_view_basic") => queries_t_022_view_basic::spec, ("queries", "t_023_view_per_user") => queries_t_023_view_per_user::spec, ("queries", "t_032_range_query") => queries_t_032_range_query::spec, diff --git a/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml b/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml index d925f98168b..a0b4af5cf50 100644 --- a/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml +++ b/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml @@ -9,5 +9,5 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -spacetimedb = { path = "../../../../../../sdks/rust/" } +spacetimedb = { path = "../../../../../../sdks/rust/", features = ["unstable"] } log = "0.4" From ce8757596b36ed9686ce87199f7b5cb93622b353 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:27:58 -0400 Subject: [PATCH 10/78] llm-benchmark: add migration-aware default eval --- .../src/bench/publishers.rs | 88 ++++++------ tools/xtask-llm-benchmark/src/bench/runner.rs | 132 ++++++++++++++++-- tools/xtask-llm-benchmark/src/bench/types.rs | 1 + .../t_053_default_values/answers/csharp.cs | 21 +++ .../t_053_default_values/answers/rust.rs | 18 +++ .../answers/typescript.ts | 15 ++ .../t_053_default_values/setup/csharp.cs | 20 +++ .../tables/t_053_default_values/setup/rust.rs | 16 +++ .../t_053_default_values/setup/typescript.ts | 15 ++ .../tables/t_053_default_values/spec.rs | 28 ++++ .../t_053_default_values/tasks/csharp.txt | 1 + .../t_053_default_values/tasks/rust.txt | 1 + .../t_053_default_values/tasks/typescript.txt | 1 + .../src/generated/registry.rs | 7 + 14 files changed, 303 insertions(+), 61 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index b7fb74c6936..9b20f0badd2 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -6,11 +6,7 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - LazyLock, -}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::sync::LazyLock; fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -126,6 +122,7 @@ fn resolve_node_exe(nodejs_dir: Option<&Path>) -> Option { struct CliRootDir { path: PathBuf, + remove_on_drop: bool, } impl CliRootDir { @@ -136,28 +133,10 @@ impl CliRootDir { impl Drop for CliRootDir { fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.path); - } -} - -fn isolated_cli_root() -> Result { - static COUNTER: AtomicU64 = AtomicU64::new(0); - - for _ in 0..16 { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let id = COUNTER.fetch_add(1, Ordering::Relaxed); - let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{nanos}-{id}", std::process::id())); - match fs::create_dir(&path) { - Ok(()) => return Ok(CliRootDir { path }), - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error.into()), + if self.remove_on_drop { + let _ = fs::remove_dir_all(&self.path); } } - - bail!("failed to create isolated SpacetimeDB CLI root directory"); } fn spacetime_cmd(cli_root: &CliRootDir) -> Command { @@ -206,7 +185,17 @@ fn strip_ansi_codes(s: &str) -> Cow<'_, str> { /* -------------------------------------------------------------------------- */ pub trait Publisher: Send + Sync { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()>; + fn publish(&self, host_url: &str, source: &Path, module_name: &str, clear_database: bool) -> Result<()>; +} + +fn database_cli_root(module_name: &str) -> Result { + let db = sanitize_db_name(module_name); + let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{db}", std::process::id())); + fs::create_dir_all(&path)?; + Ok(CliRootDir { + path, + remove_on_drop: false, + }) } /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) @@ -342,7 +331,7 @@ impl DotnetPublisher { } impl Publisher for DotnetPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str, clear_database: bool) -> Result<()> { if !source.exists() { bail!("no source: {}", source.display()); } @@ -354,12 +343,14 @@ impl Publisher for DotnetPublisher { let source = source .canonicalize() .with_context(|| format!("failed to resolve C# source path {}", source.display()))?; - let cli_root = isolated_cli_root()?; + let cli_root = database_cli_root(module_name)?; let mut pubcmd = spacetime_cmd(&cli_root); + pubcmd.arg("publish"); + if clear_database { + pubcmd.arg("-c"); + } pubcmd - .arg("publish") - .arg("-c") .arg("-y") .arg("--server") .arg(host_url) @@ -390,7 +381,7 @@ impl SpacetimeRustPublisher { } impl Publisher for SpacetimeRustPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str, clear_database: bool) -> Result<()> { if !source.exists() { bail!("no source: {}", source.display()); } @@ -401,20 +392,21 @@ impl Publisher for SpacetimeRustPublisher { // sanitize db + server let db = sanitize_db_name(module_name); - let cli_root = isolated_cli_root()?; + let cli_root = database_cli_root(module_name)?; // 2) Publish - run( - spacetime_cmd(&cli_root) - .arg("publish") - .arg("-c") - .arg("-y") - .arg("--server") - .arg(host_url) - .arg(&db) - .current_dir(source), - "spacetime publish", - )?; + let mut pubcmd = spacetime_cmd(&cli_root); + pubcmd.arg("publish"); + if clear_database { + pubcmd.arg("-c"); + } + pubcmd + .arg("-y") + .arg("--server") + .arg(host_url) + .arg(&db) + .current_dir(source); + run(&mut pubcmd, "spacetime publish")?; Ok(()) } @@ -437,7 +429,7 @@ impl TypeScriptPublisher { } impl Publisher for TypeScriptPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str, clear_database: bool) -> Result<()> { if !source.exists() { bail!("no source: {}", source.display()); } @@ -445,7 +437,7 @@ impl Publisher for TypeScriptPublisher { Self::ensure_package_json(source)?; let db = sanitize_db_name(module_name); - let cli_root = isolated_cli_root()?; + let cli_root = database_cli_root(module_name)?; // Install dependencies (--ignore-workspace to avoid parent workspace interference). let nodejs_dir = configured_nodejs_dir(); @@ -520,9 +512,11 @@ impl Publisher for TypeScriptPublisher { // Publish (spacetime CLI handles TypeScript compilation internally) let mut publish_cmd = spacetime_cmd(&cli_root); + publish_cmd.arg("publish"); + if clear_database { + publish_cmd.arg("-c"); + } publish_cmd - .arg("publish") - .arg("-c") .arg("-y") .arg("--server") .arg(host_url) diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 2536b5e5fe1..2e729c383ce 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -19,6 +19,7 @@ use crate::bench::utils::{ print_llm_output, sanitize_db_name, task_slug, work_server_dir_scoped, }; use crate::bench::Publisher; +use crate::eval::scorers::call_reducer_json_out; use crate::eval::{Lang, ScoreDetails}; use crate::generated::resolve_by_path; use crate::llm::model_routes::ModelRoute; @@ -87,16 +88,29 @@ async fn publish_rust_async( host_url: String, wdir: PathBuf, db: String, + clear_database: bool, ) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db, clear_database)).await??; Ok(()) } -async fn publish_cs_async(publisher: DotnetPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; +async fn publish_cs_async( + publisher: DotnetPublisher, + host_url: String, + wdir: PathBuf, + db: String, + clear_database: bool, +) -> Result<()> { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db, clear_database)).await??; Ok(()) } -async fn publish_ts_async(publisher: TypeScriptPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; +async fn publish_ts_async( + publisher: TypeScriptPublisher, + host_url: String, + wdir: PathBuf, + db: String, + clear_database: bool, +) -> Result<()> { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db, clear_database)).await??; Ok(()) } @@ -123,6 +137,7 @@ impl TaskRunner { golden_src_text: &str, golden_db: String, host: Option, + clear_database: bool, ) -> Result<()> { self.publish( PublishParams { @@ -133,6 +148,7 @@ impl TaskRunner { source_text: golden_src_text, db_name: golden_db, host, + clear_database, }, "golden", ) @@ -165,9 +181,22 @@ impl TaskRunner { let host_url = params.host.unwrap_or_else(|| "local".to_owned()); match params.lang { - Lang::Rust => publish_rust_async(self.rust_publisher, host_url, wdir, params.db_name).await?, - Lang::CSharp => publish_cs_async(self.cs_publisher, host_url, wdir, params.db_name).await?, - Lang::TypeScript => publish_ts_async(self.ts_publisher, host_url, wdir, params.db_name).await?, + Lang::Rust => { + publish_rust_async( + self.rust_publisher, + host_url, + wdir, + params.db_name, + params.clear_database, + ) + .await? + } + Lang::CSharp => { + publish_cs_async(self.cs_publisher, host_url, wdir, params.db_name, params.clear_database).await? + } + Lang::TypeScript => { + publish_ts_async(self.ts_publisher, host_url, wdir, params.db_name, params.clear_database).await? + } } Ok(()) @@ -263,12 +292,42 @@ impl TaskRunner { let output_tokens = llm_result.output_tokens; let llm_output = llm_result.text; + let migration_setup = load_migration_setup_source(task, cfg.lang)?; + if debug_llm() { print_llm_output(&cfg.route.display_name, &task_id, &llm_output); } - let publish_error: Option = self - .publish_llm(PublishParams { + let setup_error = if let Some(setup_source) = migration_setup.as_deref() { + let setup_result = self + .publish( + PublishParams { + lang: cfg.lang, + category: &category, + task_id: &task_id, + route_tag: &route_tag, + source_text: setup_source, + db_name: llm_db.clone(), + host: cfg.host.clone(), + clear_database: true, + }, + "llm-setup", + ) + .await; + match setup_result { + Ok(()) => call_reducer_json_out(&llm_db, "seed", &[], Some(cfg.host.as_deref().unwrap_or("local"))) + .map(|_| ()) + .map_err(anyhow::Error::msg), + Err(error) => Err(error), + } + } else { + Ok(()) + }; + + let publish_error: Option = if let Err(error) = setup_error { + Some(format!("migration setup failed: {error:#}")) + } else { + self.publish_llm(PublishParams { lang: cfg.lang, category: &category, task_id: &task_id, @@ -276,6 +335,7 @@ impl TaskRunner { source_text: &llm_output, db_name: llm_db.clone(), host: cfg.host.clone(), + clear_database: migration_setup.is_none(), }) .await .err() @@ -285,7 +345,8 @@ impl TaskRunner { category, task_id, cfg.route.display_name ); format!("{:#}", e) - }); + }) + }; let mut passed = 0usize; let mut partial_sum = 0f32; @@ -930,10 +991,38 @@ pub async fn build_goldens_only_for_lang( let task_id = task_slug(&task.root); let golden_db = sanitize_db_name(&format!("{}-{}-golden", category, task_id)); let golden_src_text = load_golden_source(&task, lang)?; + let migration_setup = load_migration_setup_source(&task, lang)?; println!("→ [{}] build golden {} {}", lang_name, category, task_id); - runner - .publish_golden_only(lang, &category, &task_id, &golden_src_text, golden_db, host_clone) - .await + if let Some(setup_source) = migration_setup { + runner + .publish_golden_only( + lang, + &category, + &task_id, + &setup_source, + golden_db.clone(), + host_clone.clone(), + true, + ) + .await?; + call_reducer_json_out(&golden_db, "seed", &[], Some(host_clone.as_deref().unwrap_or("local"))) + .map_err(anyhow::Error::msg)?; + runner + .publish_golden_only( + lang, + &category, + &task_id, + &golden_src_text, + golden_db, + host_clone, + false, + ) + .await + } else { + runner + .publish_golden_only(lang, &category, &task_id, &golden_src_text, golden_db, host_clone, true) + .await + } } })) .buffer_unordered(buf) @@ -1044,6 +1133,21 @@ fn load_golden_source(task: &TaskPaths, lang: Lang) -> Result { } } +fn load_migration_setup_source(task: &TaskPaths, lang: Lang) -> Result> { + let file = match lang { + Lang::Rust => "rust.rs", + Lang::CSharp => "csharp.cs", + Lang::TypeScript => "typescript.ts", + }; + let path = task.root.join("setup").join(file); + if !path.is_file() { + return Ok(None); + } + fs::read_to_string(&path) + .with_context(|| format!("read {}", path.display())) + .map(Some) +} + // "1" | "01" | "001" | "t_001" -> "t_001" // "t_000_empty_reducers" | "t_001_basic_tables" -> accepted as-is (full task dir name) fn normalize_task_selector(raw: &str) -> Result { diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index e54df0d4902..6a463a39554 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -47,6 +47,7 @@ pub struct PublishParams<'a> { pub source_text: &'a str, pub db_name: String, pub host: Option, + pub clear_database: bool, } #[derive(Debug, Serialize, Deserialize, Clone)] diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs new file mode 100644 index 00000000000..457349fbdf1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs @@ -0,0 +1,21 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Widget", Public = true)] + public partial struct Widget + { + [PrimaryKey] public ulong Id; + public string Name; + [Default(true)] public bool Enabled; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Widget.Insert(new Widget { Id = 1, Name = "legacy", Enabled = true }); + } + + [Reducer] + public static void Touch(ReducerContext ctx) { } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs new file mode 100644 index 00000000000..2e77e2393c0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs @@ -0,0 +1,18 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = widget, public)] +pub struct Widget { + #[primary_key] + id: u64, + name: String, + #[default(true)] + enabled: bool, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.widget().insert(Widget { id: 1, name: "legacy".into(), enabled: true }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts new file mode 100644 index 00000000000..836945ca178 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts @@ -0,0 +1,15 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const widget = table( + { name: 'widget', public: true }, + { id: t.u64().primaryKey(), name: t.string(), enabled: t.bool().default(true) } +); + +const spacetimedb = schema({ widget }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.widget.insert({ id: 1n, name: 'legacy', enabled: true }); +}); + +export const touch = spacetimedb.reducer(_ctx => {}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs new file mode 100644 index 00000000000..316091c1af2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Widget", Public = true)] + public partial struct Widget + { + [PrimaryKey] public ulong Id; + public string Name; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Widget.Insert(new Widget { Id = 1, Name = "legacy" }); + } + + [Reducer] + public static void Touch(ReducerContext ctx) { } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs new file mode 100644 index 00000000000..ff3b23c6283 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs @@ -0,0 +1,16 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = widget, public)] +pub struct Widget { + #[primary_key] + id: u64, + name: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.widget().insert(Widget { id: 1, name: "legacy".into() }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts new file mode 100644 index 00000000000..efb842f3499 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts @@ -0,0 +1,15 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const widget = table( + { name: 'widget', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); + +const spacetimedb = schema({ widget }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.widget.insert({ id: 1n, name: 'legacy' }); +}); + +export const touch = spacetimedb.reducer(_ctx => {}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs new file mode 100644 index 00000000000..3e5dfa7286a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let widget = table_name("widget", lang); + let casing = casing_for_lang(lang); + let id = ident("id", casing); + let name = ident("name", casing); + let enabled = ident("enabled", casing); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "touch".into(), + args: vec![], + select_query: format!("SELECT {id}, {name}, {enabled} FROM {widget} ORDER BY {id}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "existing_row_backfilled", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt new file mode 100644 index 00000000000..a01574d65c0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt @@ -0,0 +1 @@ +Update the C# SpacetimeDB module without removing or renaming its public Widget table, Seed reducer, or Touch reducer. Add a trailing bool field named Enabled to Widget and annotate that field with a default value of true so a compatible republish backfills existing rows. Keep Id as the ulong primary key and do not put the default annotation on a primary-key, unique, or auto-increment column. Update new Widget values to set Enabled to true. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt new file mode 100644 index 00000000000..dc7219adb21 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt @@ -0,0 +1 @@ +Update the Rust SpacetimeDB module without removing or renaming its public widget table, seed reducer, or touch reducer. Add a trailing bool field named enabled to Widget and annotate that field with a default value of true so a compatible republish backfills existing rows. Keep id as the u64 primary key and do not put the default annotation on a primary-key, unique, or auto-increment column. Update new Widget values to set enabled to true. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt new file mode 100644 index 00000000000..770e9dbd563 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt @@ -0,0 +1 @@ +Update the TypeScript SpacetimeDB module without removing or renaming its public widget table, seed reducer, or touch reducer. Add a trailing bool field named enabled to widget and give that field a default value of true so a compatible republish backfills existing rows. Keep id as the u64 primary key and do not put the default on a primary-key, unique, or auto-increment column. Update new widget values to set enabled to true. diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 363294b87b5..f269761d4bc 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -428,6 +428,12 @@ mod tables_t_052_autoinc_reference { include!("../benchmarks/tables/t_052_autoinc_reference/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_053_default_values { + include!("../benchmarks/tables/t_053_default_values/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod tables_t_054_special_types { @@ -558,6 +564,7 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("tables", "t_050_normalized_schema") => tables_t_050_normalized_schema::spec, ("tables", "t_051_denormalized_index") => tables_t_051_denormalized_index::spec, ("tables", "t_052_autoinc_reference") => tables_t_052_autoinc_reference::spec, + ("tables", "t_053_default_values") => tables_t_053_default_values::spec, ("tables", "t_054_special_types") => tables_t_054_special_types::spec, ("views", "t_061_three_table_join") => views_t_061_three_table_join::spec, ("views", "t_062_semijoin_intersection") => views_t_062_semijoin_intersection::spec, From 633f6d73509e34aaa2dd19aeb95e5bba1265114f Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:39:24 -0400 Subject: [PATCH 11/78] llm-benchmark: add migration and RLS evals --- .../src/bench/publishers.rs | 14 +---- .../answers/csharp.cs | 20 +++++++ .../t_083_row_level_security/answers/rust.rs | 17 ++++++ .../answers/typescript.ts | 17 ++++++ .../auth/t_083_row_level_security/spec.rs | 26 +++++++++ .../t_083_row_level_security/tasks/csharp.txt | 1 + .../t_083_row_level_security/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 29 ++++++++++ .../t_080_automatic_migration/answers/rust.rs | 28 +++++++++ .../answers/typescript.ts | 21 +++++++ .../t_080_automatic_migration/setup/csharp.cs | 18 ++++++ .../t_080_automatic_migration/setup/rust.rs | 16 ++++++ .../setup/typescript.ts | 13 +++++ .../t_080_automatic_migration/spec.rs | 41 +++++++++++++ .../tasks/csharp.txt | 1 + .../t_080_automatic_migration/tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 40 +++++++++++++ .../answers/rust.rs | 36 ++++++++++++ .../answers/typescript.ts | 32 +++++++++++ .../setup/csharp.cs | 15 +++++ .../t_081_incremental_migration/setup/rust.rs | 13 +++++ .../setup/typescript.ts | 12 ++++ .../t_081_incremental_migration/spec.rs | 57 +++++++++++++++++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + .../answers/csharp.cs | 33 +++++++++++ .../answers/rust.rs | 31 ++++++++++ .../answers/typescript.ts | 28 +++++++++ .../setup/csharp.cs | 23 ++++++++ .../setup/rust.rs | 20 +++++++ .../setup/typescript.ts | 20 +++++++ .../t_082_hot_swap_compatibility/spec.rs | 28 +++++++++ .../tasks/csharp.txt | 1 + .../tasks/rust.txt | 1 + .../tasks/typescript.txt | 1 + tools/xtask-llm-benchmark/src/eval/scorers.rs | 48 +++++++++++++--- .../src/generated/registry.rs | 28 +++++++++ .../templates/csharp/server/StdbModule.csproj | 1 + 41 files changed, 716 insertions(+), 21 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 9b20f0badd2..407d8695a89 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -122,7 +122,6 @@ fn resolve_node_exe(nodejs_dir: Option<&Path>) -> Option { struct CliRootDir { path: PathBuf, - remove_on_drop: bool, } impl CliRootDir { @@ -131,14 +130,6 @@ impl CliRootDir { } } -impl Drop for CliRootDir { - fn drop(&mut self) { - if self.remove_on_drop { - let _ = fs::remove_dir_all(&self.path); - } - } -} - fn spacetime_cmd(cli_root: &CliRootDir) -> Command { let mut cmd = Command::new("spacetime"); cmd.arg("--root-dir").arg(cli_root.path()); @@ -192,10 +183,7 @@ fn database_cli_root(module_name: &str) -> Result { let db = sanitize_db_name(module_name); let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{db}", std::process::id())); fs::create_dir_all(&path)?; - Ok(CliRootDir { - path, - remove_on_drop: false, - }) + Ok(CliRootDir { path }) } /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs new file mode 100644 index 00000000000..fc2b526804a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "UserRecord", Public = true)] + public partial struct UserRecord + { + [PrimaryKey] public Identity Identity; + public string Name; + } + + [ClientVisibilityFilter] + public static readonly Filter UserRecordFilter = new Filter.Sql( + "SELECT * FROM UserRecord WHERE Identity = :sender" + ); + + [Reducer] + public static void RegisterSelf(ReducerContext ctx, string name) => + ctx.Db.UserRecord.Insert(new UserRecord { Identity = ctx.Sender, Name = name }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs new file mode 100644 index 00000000000..8ebfd7e9ff4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs @@ -0,0 +1,17 @@ +use spacetimedb::{Filter, Identity, ReducerContext, Table}; + +#[spacetimedb::table(accessor = user_record, public)] +pub struct UserRecord { + #[primary_key] + identity: Identity, + name: String, +} + +#[spacetimedb::client_visibility_filter] +const USER_RECORD_FILTER: Filter = + Filter::Sql("SELECT * FROM user_record WHERE identity = :sender"); + +#[spacetimedb::reducer] +pub fn register_self(ctx: &ReducerContext, name: String) { + ctx.db.user_record().insert(UserRecord { identity: ctx.sender(), name }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts new file mode 100644 index 00000000000..aae8e7058cd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts @@ -0,0 +1,17 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const userRecord = table( + { name: 'user_record', public: true }, + { identity: t.identity().primaryKey(), name: t.string() } +); +const spacetimedb = schema({ userRecord }); +export default spacetimedb; + +export const userRecordFilter = spacetimedb.clientVisibilityFilter.sql( + 'SELECT * FROM user_record WHERE identity = :sender' +); + +export const register_self = spacetimedb.reducer( + { name: t.string() }, + (ctx, { name }) => ctx.db.userRecord.insert({ identity: ctx.sender, name }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs new file mode 100644 index 00000000000..34c450f9f7b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs @@ -0,0 +1,26 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let table = table_name("user_record", lang); + let name = ident("name", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "register_self".into(), + args: vec![json!("caller")], + select_query: format!("SELECT {name} FROM {table}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "caller_sees_own_row", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt new file mode 100644 index 00000000000..5d630c2dab5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt @@ -0,0 +1 @@ +Write a C# SpacetimeDB module with a public UserRecord table containing Identity: Identity as its primary key and Name: string. Add a client visibility filter with SQL exactly `SELECT * FROM UserRecord WHERE Identity = :sender` so each caller can only see its own row. Add RegisterSelf(name), which inserts a UserRecord using ctx.Sender rather than accepting an identity argument. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt new file mode 100644 index 00000000000..e1315395dc7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt @@ -0,0 +1 @@ +Write a Rust SpacetimeDB module with a public UserRecord table containing identity: Identity as its primary key and name: String. Add a client visibility filter with SQL exactly `SELECT * FROM user_record WHERE identity = :sender` so each caller can only see its own row. Add register_self(name), which inserts a UserRecord using ctx.sender() rather than accepting an identity argument. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt new file mode 100644 index 00000000000..e6ba69cb6a4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt @@ -0,0 +1 @@ +Write a TypeScript SpacetimeDB module with a public user_record table containing identity: Identity as its primary key and name: string. Add a client visibility filter with SQL exactly `SELECT * FROM user_record WHERE identity = :sender` so each caller can only see its own row. Add register_self(name), which inserts a user_record using ctx.sender rather than accepting an identity argument. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs new file mode 100644 index 00000000000..1471824e930 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs @@ -0,0 +1,29 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Product", Public = true)] + public partial struct Product + { + [PrimaryKey] public ulong Id; + public string Name; + } + + [Table(Accessor = "Category", Public = true)] + public partial struct Category + { + [PrimaryKey] public ulong Id; + public string Label; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Product.Insert(new Product { Id = 1, Name = "legacy" }); + + [Reducer] + public static void Touch(ReducerContext ctx) { } + + [Reducer] + public static void CreateCategory(ReducerContext ctx, ulong id, string label) => + ctx.Db.Category.Insert(new Category { Id = id, Label = label }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs new file mode 100644 index 00000000000..375aece6f7c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs @@ -0,0 +1,28 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = product, public)] +pub struct Product { + #[primary_key] + id: u64, + name: String, +} + +#[spacetimedb::table(accessor = category, public)] +pub struct Category { + #[primary_key] + id: u64, + label: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.product().insert(Product { id: 1, name: "legacy".into() }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer] +pub fn create_category(ctx: &ReducerContext, id: u64, label: String) { + ctx.db.category().insert(Category { id, label }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts new file mode 100644 index 00000000000..8f7098d07bb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts @@ -0,0 +1,21 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const product = table( + { name: 'product', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); +const category = table( + { name: 'category', public: true }, + { id: t.u64().primaryKey(), label: t.string() } +); +const spacetimedb = schema({ product, category }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.product.insert({ id: 1n, name: 'legacy' }); +}); +export const touch = spacetimedb.reducer(_ctx => {}); +export const create_category = spacetimedb.reducer( + { id: t.u64(), label: t.string() }, + (ctx, { id, label }) => ctx.db.category.insert({ id, label }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs new file mode 100644 index 00000000000..3e82962a547 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs @@ -0,0 +1,18 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Product", Public = true)] + public partial struct Product + { + [PrimaryKey] public ulong Id; + public string Name; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Product.Insert(new Product { Id = 1, Name = "legacy" }); + + [Reducer] + public static void Touch(ReducerContext ctx) { } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs new file mode 100644 index 00000000000..862af19aa02 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs @@ -0,0 +1,16 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = product, public)] +pub struct Product { + #[primary_key] + id: u64, + name: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.product().insert(Product { id: 1, name: "legacy".into() }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts new file mode 100644 index 00000000000..e0ed489261c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts @@ -0,0 +1,13 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const product = table( + { name: 'product', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); +const spacetimedb = schema({ product }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.product.insert({ id: 1n, name: 'legacy' }); +}); +export const touch = spacetimedb.reducer(_ctx => {}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs new file mode 100644 index 00000000000..2371dbb576b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs @@ -0,0 +1,41 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_data_parity_scorer, make_sql_count_only_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let casing = casing_for_lang(lang); + let product = table_name("product", lang); + let product_name = ident("name", casing); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {product} WHERE {product_name}='legacy'"), + 1, + "existing_data_preserved", + Duration::from_secs(10), + )); + let category = table_name("category", lang); + let id = ident("id", casing); + let label = ident("label", casing); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "create_category".into(), + args: vec![json!(7), json!("general")], + select_query: format!("SELECT {id}, {label} FROM {category} ORDER BY {id}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "new_schema_usable", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt new file mode 100644 index 00000000000..6e15528e3cf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt @@ -0,0 +1 @@ +Update the C# SpacetimeDB module with a compatible schema change. Preserve the public Product table and its fields, plus the Seed and Touch reducers. Add a public Category table with Id: ulong as its primary key and Label: string. Add CreateCategory(id, label) to insert a Category. The populated Product table must survive a normal republish without deleting data. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt new file mode 100644 index 00000000000..082deb4a1cf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt @@ -0,0 +1 @@ +Update the Rust SpacetimeDB module with a compatible schema change. Preserve the public Product table and its fields, plus the seed and touch reducers. Add a public Category table with id: u64 as its primary key and label: String. Add create_category(id, label) to insert a Category. The populated Product table must survive a normal republish without deleting data. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt new file mode 100644 index 00000000000..068469276ea --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt @@ -0,0 +1 @@ +Update the TypeScript SpacetimeDB module with a compatible schema change. Preserve the public product table and its fields, plus the seed and touch reducers. Add a public category table with id: u64 as its primary key and label: string. Add create_category(id, label) to insert a category. The populated product table must survive a normal republish without deleting data. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs new file mode 100644 index 00000000000..9601ba34cb6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs @@ -0,0 +1,40 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "LegacyItem", Public = true)] + public partial struct LegacyItem + { + [PrimaryKey] public ulong Id; + public string Value; + } + + [Table(Accessor = "ItemV2", Public = true)] + public partial struct ItemV2 + { + [PrimaryKey] public ulong Id; + public string Value; + public uint Version; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.LegacyItem.Insert(new LegacyItem { Id = 1, Value = "old" }); + + [Reducer] + public static void Migrate(ReducerContext ctx) + { + foreach (var row in ctx.Db.LegacyItem.Iter()) + { + if (ctx.Db.ItemV2.Id.Find(row.Id) is null) + ctx.Db.ItemV2.Insert(new ItemV2 { Id = row.Id, Value = row.Value, Version = 2 }); + } + } + + [Reducer] + public static void DualWrite(ReducerContext ctx, ulong id, string value) + { + ctx.Db.LegacyItem.Insert(new LegacyItem { Id = id, Value = value }); + ctx.Db.ItemV2.Insert(new ItemV2 { Id = id, Value = value, Version = 2 }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs new file mode 100644 index 00000000000..6a81e46129b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs @@ -0,0 +1,36 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = legacy_item, public)] +pub struct LegacyItem { + #[primary_key] + id: u64, + value: String, +} + +#[spacetimedb::table(accessor = item_v2, public)] +pub struct ItemV2 { + #[primary_key] + id: u64, + value: String, + version: u32, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.legacy_item().insert(LegacyItem { id: 1, value: "old".into() }); +} + +#[spacetimedb::reducer] +pub fn migrate(ctx: &ReducerContext) { + for row in ctx.db.legacy_item().iter() { + if ctx.db.item_v2().id().find(row.id).is_none() { + ctx.db.item_v2().insert(ItemV2 { id: row.id, value: row.value, version: 2 }); + } + } +} + +#[spacetimedb::reducer] +pub fn dual_write(ctx: &ReducerContext, id: u64, value: String) { + ctx.db.legacy_item().insert(LegacyItem { id, value: value.clone() }); + ctx.db.item_v2().insert(ItemV2 { id, value, version: 2 }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts new file mode 100644 index 00000000000..758e4d5be79 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts @@ -0,0 +1,32 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const legacyItem = table( + { name: 'legacy_item', public: true }, + { id: t.u64().primaryKey(), value: t.string() } +); +const itemV2 = table( + { name: 'item_v2', public: true }, + { id: t.u64().primaryKey(), value: t.string(), version: t.u32() } +); +const spacetimedb = schema({ legacyItem, itemV2 }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.legacyItem.insert({ id: 1n, value: 'old' }); +}); + +export const migrate = spacetimedb.reducer(ctx => { + for (const row of ctx.db.legacyItem.iter()) { + if (!ctx.db.itemV2.id.find(row.id)) { + ctx.db.itemV2.insert({ id: row.id, value: row.value, version: 2 }); + } + } +}); + +export const dual_write = spacetimedb.reducer( + { id: t.u64(), value: t.string() }, + (ctx, { id, value }) => { + ctx.db.legacyItem.insert({ id, value }); + ctx.db.itemV2.insert({ id, value, version: 2 }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs new file mode 100644 index 00000000000..2dbe7c6f2c9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs @@ -0,0 +1,15 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "LegacyItem", Public = true)] + public partial struct LegacyItem + { + [PrimaryKey] public ulong Id; + public string Value; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.LegacyItem.Insert(new LegacyItem { Id = 1, Value = "old" }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs new file mode 100644 index 00000000000..182924f4c81 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs @@ -0,0 +1,13 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = legacy_item, public)] +pub struct LegacyItem { + #[primary_key] + id: u64, + value: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.legacy_item().insert(LegacyItem { id: 1, value: "old".into() }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts new file mode 100644 index 00000000000..246be8c6bec --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts @@ -0,0 +1,12 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const legacyItem = table( + { name: 'legacy_item', public: true }, + { id: t.u64().primaryKey(), value: t.string() } +); +const spacetimedb = schema({ legacyItem }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.legacyItem.insert({ id: 1n, value: 'old' }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs new file mode 100644 index 00000000000..0618d30b249 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs @@ -0,0 +1,57 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let casing = casing_for_lang(lang); + let item_v2 = table_name("item_v2", lang); + let id = ident("id", casing); + let value = ident("value", casing); + let version = ident("version", casing); + let v2_query = format!("SELECT {id}, {value}, {version} FROM {item_v2} ORDER BY {id}"); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "migrate".into(), + args: vec![], + select_query: v2_query.clone(), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "legacy_rows_migrated", + }, + )); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "dual_write".into(), + args: vec![json!(2), json!("new")], + select_query: v2_query, + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "dual_write_keeps_v2_current", + }, + )); + let legacy = table_name("legacy_item", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "migrate".into(), + args: vec![], + select_query: format!("SELECT {id}, {value} FROM {legacy} ORDER BY {id}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "legacy_table_remains_current", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt new file mode 100644 index 00000000000..d574884d1c5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt @@ -0,0 +1 @@ +Update the C# SpacetimeDB module for an incremental migration while preserving the public LegacyItem table and Seed reducer. Add a public ItemV2 table with Id: ulong primary key, Value: string, and Version: uint. Add an idempotent Migrate reducer that copies missing legacy rows into ItemV2 with Version 2. Add DualWrite(id, value) that writes the same new item to both tables with Version 2 in ItemV2. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt new file mode 100644 index 00000000000..5729bc4fee2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt @@ -0,0 +1 @@ +Update the Rust SpacetimeDB module for an incremental migration while preserving the public LegacyItem table and seed reducer. Add a public ItemV2 table with id: u64 primary key, value: String, and version: u32. Add an idempotent migrate reducer that copies missing legacy rows into ItemV2 with version 2. Add dual_write(id, value) that writes the same new item to both tables with version 2 in ItemV2. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt new file mode 100644 index 00000000000..4c1bd596a42 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt @@ -0,0 +1 @@ +Update the TypeScript SpacetimeDB module for an incremental migration while preserving the public legacy_item table and seed reducer. Add a public item_v2 table with id: u64 primary key, value: string, and version: u32. Add an idempotent migrate reducer that copies missing legacy rows into item_v2 with version 2. Add dual_write(id, value) that writes the same new item to both tables with version 2 in item_v2. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs new file mode 100644 index 00000000000..0f358013e5b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs @@ -0,0 +1,33 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Counter", Public = true)] + public partial struct Counter + { + [PrimaryKey] public ulong Id; + public long Value; + } + + [Table(Accessor = "Release")] + public partial struct Release + { + [PrimaryKey] public uint Version; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Counter.Insert(new Counter { Id = 1, Value = 1 }); + + [Reducer] + public static void Increment(ReducerContext ctx, ulong id, long amount) + { + var row = ctx.Db.Counter.Id.Find(id) ?? throw new Exception("counter"); + row.Value += amount; + ctx.Db.Counter.Id.Update(row); + } + + [Reducer] + public static void RecordRelease(ReducerContext ctx, uint version) => + ctx.Db.Release.Insert(new Release { Version = version }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs new file mode 100644 index 00000000000..a0f1a3498d3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs @@ -0,0 +1,31 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = counter, public)] +pub struct Counter { + #[primary_key] + id: u64, + value: i64, +} + +#[spacetimedb::table(accessor = release)] +pub struct Release { + #[primary_key] + version: u32, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.counter().insert(Counter { id: 1, value: 1 }); +} + +#[spacetimedb::reducer] +pub fn increment(ctx: &ReducerContext, id: u64, amount: i64) { + let mut row = ctx.db.counter().id().find(id).expect("counter"); + row.value += amount; + ctx.db.counter().id().update(row); +} + +#[spacetimedb::reducer] +pub fn record_release(ctx: &ReducerContext, version: u32) { + ctx.db.release().insert(Release { version }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts new file mode 100644 index 00000000000..0e90ee4a465 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts @@ -0,0 +1,28 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const counter = table( + { name: 'counter', public: true }, + { id: t.u64().primaryKey(), value: t.i64() } +); +const release = table( + { name: 'release' }, + { version: t.u32().primaryKey() } +); +const spacetimedb = schema({ counter, release }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.counter.insert({ id: 1n, value: 1n }); +}); +export const increment = spacetimedb.reducer( + { id: t.u64(), amount: t.i64() }, + (ctx, { id, amount }) => { + const row = ctx.db.counter.id.find(id); + if (!row) throw new Error('counter'); + ctx.db.counter.id.update({ ...row, value: row.value + amount }); + } +); +export const record_release = spacetimedb.reducer( + { version: t.u32() }, + (ctx, { version }) => ctx.db.release.insert({ version }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs new file mode 100644 index 00000000000..315677e3901 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs @@ -0,0 +1,23 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Counter", Public = true)] + public partial struct Counter + { + [PrimaryKey] public ulong Id; + public long Value; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Counter.Insert(new Counter { Id = 1, Value = 1 }); + + [Reducer] + public static void Increment(ReducerContext ctx, ulong id, long amount) + { + var row = ctx.Db.Counter.Id.Find(id) ?? throw new Exception("counter"); + row.Value += amount; + ctx.Db.Counter.Id.Update(row); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs new file mode 100644 index 00000000000..341fa49e039 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs @@ -0,0 +1,20 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = counter, public)] +pub struct Counter { + #[primary_key] + id: u64, + value: i64, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.counter().insert(Counter { id: 1, value: 1 }); +} + +#[spacetimedb::reducer] +pub fn increment(ctx: &ReducerContext, id: u64, amount: i64) { + let mut row = ctx.db.counter().id().find(id).expect("counter"); + row.value += amount; + ctx.db.counter().id().update(row); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts new file mode 100644 index 00000000000..e2aca9da9fd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts @@ -0,0 +1,20 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const counter = table( + { name: 'counter', public: true }, + { id: t.u64().primaryKey(), value: t.i64() } +); +const spacetimedb = schema({ counter }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.counter.insert({ id: 1n, value: 1n }); +}); +export const increment = spacetimedb.reducer( + { id: t.u64(), amount: t.i64() }, + (ctx, { id, amount }) => { + const row = ctx.db.counter.id.find(id); + if (!row) throw new Error('counter'); + ctx.db.counter.id.update({ ...row, value: row.value + amount }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs new file mode 100644 index 00000000000..4ac90563db6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let counter = table_name("counter", lang); + let casing = casing_for_lang(lang); + let id = ident("id", casing); + let value = ident("value", casing); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "increment".into(), + args: vec![json!(1), json!(2)], + select_query: format!("SELECT {id}, {value} FROM {counter} ORDER BY {id}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "existing_api_survives_republish", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt new file mode 100644 index 00000000000..5dba097dd4e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt @@ -0,0 +1 @@ +Update the C# SpacetimeDB module through a compatible republish while preserving the public Counter table, its exact fields, and the Seed and Increment reducers. Add a private Release table with Version: uint as its primary key and add RecordRelease(version). Do not rename or change the signature of Increment. Existing counter data and the existing server API must remain usable after republish. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt new file mode 100644 index 00000000000..841d8319498 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt @@ -0,0 +1 @@ +Update the Rust SpacetimeDB module through a compatible republish while preserving the public Counter table, its exact fields, and the seed and increment reducers. Add a private Release table with version: u32 as its primary key and add record_release(version). Do not rename or change the signature of increment. Existing counter data and the existing server API must remain usable after republish. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt new file mode 100644 index 00000000000..a2bcda5f49f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt @@ -0,0 +1 @@ +Update the TypeScript SpacetimeDB module through a compatible republish while preserving the public counter table, its exact fields, and the seed and increment reducers. Add a private release table with version: u32 as its primary key and add record_release(version). Do not rename or change the signature of increment. Existing counter data and the existing server API must remain usable after republish. diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index a1b35068b85..53aacfb6d52 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -44,12 +44,13 @@ impl Scorer for SchemaParityScorer { } } - let (tables_a, reducers_a) = extract_schema(&golden); - let (tables_b, reducers_b) = extract_schema(&llm); + let (tables_a, reducers_a, rls_a) = extract_schema(&golden); + let (tables_b, reducers_b, rls_b) = extract_schema(&llm); let tables_diff = diff_maps(&tables_a, &tables_b); let reducers_diff = diff_sets(&reducers_a, &reducers_b); - let pass = tables_diff.is_null() && reducers_diff.is_null(); + let rls_diff = diff_sets(&rls_a, &rls_b); + let pass = tables_diff.is_null() && reducers_diff.is_null() && rls_diff.is_null(); ScoreDetails { pass, @@ -60,8 +61,10 @@ impl Scorer for SchemaParityScorer { "llm_db": self.llm_db, "tables_equal": tables_diff.is_null(), "reducers_equal": reducers_diff.is_null(), + "row_level_security_equal": rls_diff.is_null(), "tables_diff": tables_diff, "reducers_diff": reducers_diff, + "row_level_security_diff": rls_diff, }), } } @@ -109,9 +112,16 @@ fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { Ok(v) } -fn extract_schema(v: &Value) -> (BTreeMap>, BTreeSet) { +fn extract_schema( + v: &Value, +) -> ( + BTreeMap>, + BTreeSet, + BTreeSet, +) { let mut tables: BTreeMap> = BTreeMap::new(); let mut reducers: BTreeSet = BTreeSet::new(); + let mut row_level_security: BTreeSet = BTreeSet::new(); let types = v .pointer("/typespace/types") .and_then(Value::as_array) @@ -196,7 +206,17 @@ fn extract_schema(v: &Value) -> (BTreeMap>, BTr } } - (tables, reducers) + for rule in v + .get("row_level_security") + .or_else(|| v.get("rowLevelSecurity")) + .and_then(Value::as_array) + .into_iter() + .flatten() + { + row_level_security.insert(canonical_value(Some(rule))); + } + + (tables, reducers, row_level_security) } fn schema_name(value: Option<&Value>) -> String { @@ -1024,7 +1044,7 @@ mod tests { #[test] fn current_schema_extracts_columns_and_table_properties() { - let (tables, reducers) = extract_schema(¤t_schema(true)); + let (tables, reducers, row_level_security) = extract_schema(¤t_schema(true)); let child_item = &tables["child_item"]; assert_eq!(child_item["id"], r#"{"U64":[]}"#); @@ -1035,13 +1055,25 @@ mod tests { assert_eq!(child_item["@sequences"], "id:1"); assert_eq!(child_item["@table_access"], r#"{"Public":[]}"#); assert!(reducers.is_empty()); + assert!(row_level_security.is_empty()); } #[test] fn missing_index_produces_a_schema_diff() { - let (golden, _) = extract_schema(¤t_schema(true)); - let (candidate, _) = extract_schema(¤t_schema(false)); + let (golden, _, _) = extract_schema(¤t_schema(true)); + let (candidate, _, _) = extract_schema(¤t_schema(false)); assert!(!diff_maps(&golden, &candidate).is_null()); } + + #[test] + fn row_level_security_produces_a_schema_diff() { + let mut golden = current_schema(true); + golden["row_level_security"] = json!([{ "sql": "SELECT * FROM users WHERE identity = :sender" }]); + let candidate = current_schema(true); + let (_, _, golden_rls) = extract_schema(&golden); + let (_, _, candidate_rls) = extract_schema(&candidate); + + assert!(!diff_sets(&golden_rls, &candidate_rls).is_null()); + } } diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index f269761d4bc..b1c15e5e32b 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -56,6 +56,12 @@ mod auth_t_068_secure_projection { include!("../benchmarks/auth/t_068_secure_projection/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_083_row_level_security { + include!("../benchmarks/auth/t_083_row_level_security/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod basics_t_000_empty_reducers { @@ -200,6 +206,24 @@ mod lifecycle_t_071_scheduled_private { include!("../benchmarks/lifecycle/t_071_scheduled_private/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod migrations_t_080_automatic_migration { + include!("../benchmarks/migrations/t_080_automatic_migration/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod migrations_t_081_incremental_migration { + include!("../benchmarks/migrations/t_081_incremental_migration/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod migrations_t_082_hot_swap_compatibility { + include!("../benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod procedures_t_072_procedure_return { @@ -502,6 +526,7 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("auth", "t_045_rate_limit") => auth_t_045_rate_limit::spec, ("auth", "t_046_shared_document") => auth_t_046_shared_document::spec, ("auth", "t_068_secure_projection") => auth_t_068_secure_projection::spec, + ("auth", "t_083_row_level_security") => auth_t_083_row_level_security::spec, ("basics", "t_000_empty_reducers") => basics_t_000_empty_reducers::spec, ("basics", "t_001_basic_tables") => basics_t_001_basic_tables::spec, ("basics", "t_002_scheduled_table") => basics_t_002_scheduled_table::spec, @@ -526,6 +551,9 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("lifecycle", "t_069_scheduled_materialization") => lifecycle_t_069_scheduled_materialization::spec, ("lifecycle", "t_070_connection_scoped_presence") => lifecycle_t_070_connection_scoped_presence::spec, ("lifecycle", "t_071_scheduled_private") => lifecycle_t_071_scheduled_private::spec, + ("migrations", "t_080_automatic_migration") => migrations_t_080_automatic_migration::spec, + ("migrations", "t_081_incremental_migration") => migrations_t_081_incremental_migration::spec, + ("migrations", "t_082_hot_swap_compatibility") => migrations_t_082_hot_swap_compatibility::spec, ("procedures", "t_072_procedure_return") => procedures_t_072_procedure_return::spec, ("procedures", "t_073_http_fetch") => procedures_t_073_http_fetch::spec, ("procedures", "t_074_fetch_and_store") => procedures_t_074_fetch_and_store::spec, diff --git a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj index f286932badd..160dcec288f 100644 --- a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj +++ b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj @@ -5,6 +5,7 @@ wasi-wasm enable enable + $(NoWarn);STDB_UNSTABLE From 810617719fd699d410e35559c6ec183ce5fec69e Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:43:54 -0400 Subject: [PATCH 12/78] Support stable local LLM benchmark builds --- tools/xtask-llm-benchmark/src/bench/publishers.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 407d8695a89..a47785435e9 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -131,7 +131,8 @@ impl CliRootDir { } fn spacetime_cmd(cli_root: &CliRootDir) -> Command { - let mut cmd = Command::new("spacetime"); + let spacetime = env::var_os("LLM_BENCH_SPACETIME_BIN").unwrap_or_else(|| "spacetime".into()); + let mut cmd = Command::new(spacetime); cmd.arg("--root-dir").arg(cli_root.path()); cmd } @@ -394,6 +395,9 @@ impl Publisher for SpacetimeRustPublisher { .arg(host_url) .arg(&db) .current_dir(source); + if let Some(target_dir) = env::var_os("LLM_BENCH_RUST_TARGET_DIR") { + pubcmd.env("CARGO_TARGET_DIR", target_dir); + } run(&mut pubcmd, "spacetime publish")?; Ok(()) From 6c3139fc4c62a46f168406597537209cbae3370a Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:12:14 -0400 Subject: [PATCH 13/78] Retry silent LLM benchmark publish failures --- .../src/bench/publishers.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index a47785435e9..ba2e38d8f81 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -203,6 +203,7 @@ fn signal_killed_by(_status: &std::process::ExitStatus) -> Option { /// These are resource contention issues in the dotnet WASI SDK. fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let combined = format!("{stderr}{stdout}"); + let diagnostic = combined.to_ascii_lowercase(); // "Pipe is broken" errors from WASI SDK parallel builds combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") @@ -214,6 +215,33 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { || (combined.contains("MSB3073") && combined.contains("exited with code 2")) // dotnet can crash below spacetime while spacetime exits 1. || combined.contains("code Result<()> { From 104fafda406046cafda9e2c55622c0e6b6df0c30 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:28:07 -0400 Subject: [PATCH 14/78] Make local LLM benchmark logs collision proof --- tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 6ec030a49e8..f3a21f22f54 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -230,7 +230,14 @@ fn cmd_run(args: RunArgs) -> Result<()> { fn run_benchmarks(args: RunArgs) -> Result<()> { let dry_run = args.dry_run; let local_analysis = args.local_analysis; - let dry_run_id = dry_run.then(|| chrono::Utc::now().format("%Y-%m-%d_%H%M%S").to_string()); + let dry_run_id = dry_run.then(|| { + format!( + "{}-{}-{}", + chrono::Utc::now().format("%Y-%m-%d_%H%M%S_%3f"), + args.lang.as_str(), + std::process::id() + ) + }); let should_fetch_remote_routes = should_fetch_remote_routes(&args); let needs_api_client = should_fetch_remote_routes || !dry_run; From 25c8b31df1e3d66e0dced1b890983164df5d2470 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:39:25 -0400 Subject: [PATCH 15/78] Include existing modules in migration eval prompts --- tools/xtask-llm-benchmark/src/llm/prompt.rs | 33 ++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/llm/prompt.rs b/tools/xtask-llm-benchmark/src/llm/prompt.rs index 7a46d6826cd..bfb15acdc86 100644 --- a/tools/xtask-llm-benchmark/src/llm/prompt.rs +++ b/tools/xtask-llm-benchmark/src/llm/prompt.rs @@ -104,8 +104,14 @@ pub fn make_prompt_from_task(spec_file: &str, task_id: &str, lang: Lang) -> Resu let tasks_file = find_tasks_file(task_root, lang) .with_context(|| format!("missing tasks file for {} in {}", lang.as_str(), task_root.display()))?; - let instructions = + let mut instructions = std::fs::read_to_string(&tasks_file).with_context(|| format!("read {}", tasks_file.display()))?; + if let Some(setup_file) = find_setup_file(task_root, lang) { + let setup_source = + std::fs::read_to_string(&setup_file).with_context(|| format!("read {}", setup_file.display()))?; + instructions.push_str("\n\nEXISTING MODULE SOURCE TO UPDATE:\n"); + instructions.push_str(&setup_source); + } Ok(PromptBuilder { lang: lang.display_name().to_string(), @@ -134,6 +140,8 @@ fn find_tasks_file(task_root: &Path, lang: Lang) -> Option { #[cfg(test)] mod tests { + use crate::eval::Lang; + #[test] fn prompt_uses_workspace_version() { let pb = super::PromptBuilder { @@ -155,4 +163,27 @@ mod tests { assert!(parts.next().unwrap().parse::().is_ok(), "minor not numeric: {v}"); assert_eq!(parts.next(), None, "expected major.minor only: {v}"); } + + #[test] + fn update_task_prompt_includes_existing_module_source() { + let spec = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/benchmarks/tables/t_053_default_values/spec.rs"); + let prompt = super::make_prompt_from_task(spec.to_str().unwrap(), "t_053_default_values", Lang::Rust) + .expect("build migration prompt"); + + assert!(prompt.instructions.contains("EXISTING MODULE SOURCE TO UPDATE:")); + assert!(prompt.instructions.contains("pub struct Widget")); + assert!(prompt.instructions.contains("name: String")); + assert!(prompt.instructions.contains("pub fn touch")); + } +} + +fn find_setup_file(task_root: &Path, lang: Lang) -> Option { + let file = match lang { + Lang::CSharp => "csharp.cs", + Lang::Rust => "rust.rs", + Lang::TypeScript => "typescript.ts", + }; + let path = task_root.join("setup").join(file); + path.exists().then_some(path) } From 401b20626a5194123d80cede8b96d33cf4269e7a Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:52:16 -0400 Subject: [PATCH 16/78] Make advanced LLM evals self-contained --- .../benchmarks/auth/t_068_secure_projection/spec.rs | 12 +++++------- .../auth/t_068_secure_projection/tasks/csharp.txt | 2 +- .../auth/t_068_secure_projection/tasks/rust.txt | 2 +- .../t_068_secure_projection/tasks/typescript.txt | 2 +- .../t_069_scheduled_materialization/tasks/csharp.txt | 2 +- .../t_069_scheduled_materialization/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_071_scheduled_private/tasks/csharp.txt | 2 +- .../lifecycle/t_071_scheduled_private/tasks/rust.txt | 2 +- .../t_071_scheduled_private/tasks/typescript.txt | 2 +- .../migrations/t_080_automatic_migration/spec.rs | 2 +- .../migrations/t_081_incremental_migration/spec.rs | 4 ++-- .../migrations/t_082_hot_swap_compatibility/spec.rs | 2 +- .../procedures/t_073_http_fetch/tasks/csharp.txt | 2 +- .../procedures/t_073_http_fetch/tasks/rust.txt | 2 +- .../procedures/t_073_http_fetch/tasks/typescript.txt | 2 +- .../t_075_scheduled_procedure/tasks/csharp.txt | 2 +- .../t_075_scheduled_procedure/tasks/rust.txt | 2 +- .../t_075_scheduled_procedure/tasks/typescript.txt | 2 +- .../t_078_idempotent_webhook/tasks/csharp.txt | 2 +- .../t_078_idempotent_webhook/tasks/rust.txt | 2 +- .../t_078_idempotent_webhook/tasks/typescript.txt | 2 +- .../t_079_external_upload_flow/tasks/csharp.txt | 2 +- .../t_079_external_upload_flow/tasks/rust.txt | 2 +- .../t_079_external_upload_flow/tasks/typescript.txt | 2 +- .../reducers/t_058_batched_delete/tasks/csharp.txt | 2 +- .../reducers/t_058_batched_delete/tasks/rust.txt | 2 +- .../t_058_batched_delete/tasks/typescript.txt | 2 +- .../benchmarks/tables/t_053_default_values/spec.rs | 2 +- .../views/t_061_three_table_join/tasks/csharp.txt | 2 +- .../views/t_061_three_table_join/tasks/rust.txt | 2 +- .../t_061_three_table_join/tasks/typescript.txt | 2 +- 32 files changed, 37 insertions(+), 39 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs index 6e9c69dcc1b..a17f6272cb6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs @@ -1,16 +1,14 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; -use crate::eval::{casing_for_lang, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - let sql = SqlBuilder::new(casing_for_lang(lang)); - let columns = sql.cols(&["id", "title"]).join(", "); - scorers.push(make_reducer_data_parity_scorer(host_url, ReducerDataParityConfig { + scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { src_file: file!(), route_tag, reducer: "seed_private_note".into(), args: vec![], - select_query: format!("SELECT {columns} FROM {}", table_name("my_safe_note", lang)), - id_str: "caller_safe_projection", collapse_ws: true, timeout: Duration::from_secs(10), + sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("my_safe_note", lang)), + expected_count: 1, id_str: "caller_safe_projection", timeout: Duration::from_secs(10), })); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt index bad513cf11b..b061a674246 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a private SecretNote table containing Id: ulong primary key, indexed Owner: Identity, Title: string, and SecretBody: string. Add SeedPrivateNote() to insert one note for the caller. Expose a public per-user MySafeNote view that returns only the caller's Id and Title, never Owner or SecretBody. +Write a C# SpacetimeDB module with a private SecretNote table containing Id: ulong primary key, indexed Owner: Identity, Title: string, and SecretBody: string. Define a SafeNote product type containing only Id: ulong and Title: string. Add SeedPrivateNote() to insert Id 1 for the caller with Title "Visible title" and SecretBody "never expose this". Expose a public per-user MySafeNote view returning IEnumerable for only the caller, never Owner or SecretBody. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt index b49962f8658..a0707312bff 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a private secret_note table containing id: u64 primary key, indexed owner: Identity, title: String, and secret_body: String. Add seed_private_note() to insert one note for the caller. Expose a public per-user my_safe_note view that returns only the caller's id and title, never owner or secret_body. +Write a Rust SpacetimeDB module with a private secret_note table (struct SecretNote) containing id: u64 primary key, indexed owner: Identity, title: String, and secret_body: String. Define a SafeNote product type containing only id: u64 and title: String. Add seed_private_note() to insert id 1 for the caller with title "Visible title" and secret_body "never expose this". Expose a public per-user my_safe_note view returning Vec for only the caller, never owner or secret_body. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt index a894ba6b864..075ccdb8884 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a private secret_note table containing id: u64 primary key, indexed owner: Identity, title: string, and secretBody: string. Add seed_private_note() to insert one note for the caller. Expose a public per-user my_safe_note view that returns only the caller's id and title, never owner or secretBody. +Write a TypeScript SpacetimeDB module with a private secret_note table containing id: u64 primary key, indexed owner: Identity, title: string, and secretBody: string. Define a SafeNote row type containing only id: u64 and title: string. Add seed_private_note() to insert id 1 for the caller with title "Visible title" and secretBody "never expose this". Expose a public per-user my_safe_note view returning an array of SafeNote for only the caller, never owner or secretBody. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt index 69d25eb3161..14910e3c80a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public MaterializedState table containing Id: ulong primary key, Status: string, Version: ulong, and RefreshedAt: Timestamp. Add StartRefresh() to write a pending row and schedule a refresh job one millisecond later. The scheduled reducer must update the row to Status "ready", Version 1, and the scheduled reducer timestamp without any further client call. +Write a C# SpacetimeDB module with a public MaterializedState table containing Id: ulong primary key, Status: string, Version: ulong, and RefreshedAt: Timestamp. Define a private RefreshJob scheduled table for reducer RefreshMaterialized with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, and StateId: ulong in that order. Add StartRefresh() to write Id 1 with Status "pending" and Version 0, then schedule state 1 one millisecond later. RefreshMaterialized must update that row to Status "ready", Version 1, and the scheduled reducer timestamp without any further client call. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt index aa8e5d75c2b..4fcb7ff5818 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public materialized_state table containing id: u64 primary key, status: String, version: u64, and refreshed_at: Timestamp. Add start_refresh() to write a pending row and schedule a refresh job one millisecond later. The scheduled reducer must update the row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. +Write a Rust SpacetimeDB module with a public materialized_state table (struct MaterializedState) containing id: u64 primary key, status: String, version: u64, and refreshed_at: Timestamp. Define a private refresh_job scheduled table (struct RefreshJob) for reducer refresh_materialized with scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, and state_id: u64 in that order. Add start_refresh() to write id 1 with status "pending" and version 0, then schedule state 1 one millisecond later. refresh_materialized must update that row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt index 1e82356be39..483cbe447b0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public materialized_state table containing id: u64 primary key, status: string, version: u64, and refreshedAt: Timestamp. Add start_refresh() to write a pending row and schedule a refresh job one millisecond later. The scheduled reducer must update the row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. +Write a TypeScript SpacetimeDB module with a public materialized_state table containing id: u64 primary key, status: string, version: u64, and refreshedAt: Timestamp. Define a private refresh_job scheduled table for reducer refreshMaterialized with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, and stateId: u64 in that order. Add start_refresh() to write id 1 with status "pending" and version 0, then schedule state 1 one millisecond later. refreshMaterialized must update that row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt index 3e90a5df20a..044e54ebc11 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public JobResult table containing Id: ulong primary key and Status: string. Add public EnqueuePrivate(Id) to write Status "queued" and insert a private schedule-table row for one millisecond later. Its scheduled RunPrivateJob reducer must be scheduler-only, not client-callable, and update the result to "complete". +Write a C# SpacetimeDB module with a public JobResult table containing Id: ulong primary key and Status: string. Define a private PrivateJob scheduled table for reducer RunPrivateJob with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, and ResultId: ulong in that order. Add EnqueuePrivate(Id) to write Status "queued" and schedule that Id one millisecond later. RunPrivateJob must be scheduler-only, not client-callable, and update the result to "complete". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt index 07efb7cc04d..f9b657a4860 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public job_result table containing id: u64 primary key and status: String. Add public enqueue_private(id) to write status "queued" and insert a private schedule-table row for one millisecond later. Its scheduled run_private_job reducer must be scheduler-only, not client-callable, and update the result to "complete". +Write a Rust SpacetimeDB module with a public job_result table (struct JobResult) containing id: u64 primary key and status: String. Define a private private_job scheduled table (struct PrivateJob) for reducer run_private_job with scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, and result_id: u64 in that order. Add enqueue_private(id) to write status "queued" and schedule that id one millisecond later. run_private_job must be scheduler-only, not client-callable, and update the result to "complete". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt index 80b9fdfb3fc..f8c6cf379a1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public job_result table containing id: u64 primary key and status: string. Add public enqueue_private(id) to write status "queued" and insert a private schedule-table row for one millisecond later. Its scheduled runPrivateJob reducer must be scheduler-only, not client-callable, and update the result to "complete". +Write a TypeScript SpacetimeDB module with a public job_result table containing id: u64 primary key and status: string. Define a private private_job scheduled table for reducer runPrivateJob with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, and resultId: u64 in that order. Add enqueue_private(id) to write status "queued" and schedule that id one millisecond later. runPrivateJob must be scheduler-only, not client-callable, and update the result to "complete". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs index 2371dbb576b..bb3ee0fb7f4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs @@ -30,7 +30,7 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "create_category".into(), args: vec![json!(7), json!("general")], - select_query: format!("SELECT {id}, {label} FROM {category} ORDER BY {id}"), + select_query: format!("SELECT {id}, {label} FROM {category}"), collapse_ws: true, timeout: Duration::from_secs(10), id_str: "new_schema_usable", diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs index 0618d30b249..d51b23f243a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs @@ -11,7 +11,7 @@ pub fn spec() -> BenchmarkSpec { let id = ident("id", casing); let value = ident("value", casing); let version = ident("version", casing); - let v2_query = format!("SELECT {id}, {value}, {version} FROM {item_v2} ORDER BY {id}"); + let v2_query = format!("SELECT {id}, {value}, {version} FROM {item_v2}"); scorers.push(make_reducer_data_parity_scorer( host_url, ReducerDataParityConfig { @@ -46,7 +46,7 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "migrate".into(), args: vec![], - select_query: format!("SELECT {id}, {value} FROM {legacy} ORDER BY {id}"), + select_query: format!("SELECT {id}, {value} FROM {legacy}"), collapse_ws: true, timeout: Duration::from_secs(10), id_str: "legacy_table_remains_current", diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs index 4ac90563db6..7223b5d9571 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs @@ -17,7 +17,7 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "increment".into(), args: vec![json!(1), json!(2)], - select_query: format!("SELECT {id}, {value} FROM {counter} ORDER BY {id}"), + select_query: format!("SELECT {id}, {value} FROM {counter}"), collapse_ws: true, timeout: Duration::from_secs(10), id_str: "existing_api_survives_republish", diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt index fc5f8ddd3cf..1a635211cbc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB procedure FetchSchemaSummary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a typed FetchSummary with the HTTP status, whether Content-Type contains "application/json", and whether the body contains the tables field. +Write a C# SpacetimeDB procedure FetchSchemaSummary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a FetchSummary product type with Status: ushort, JsonContentType: bool indicating whether Content-Type contains "application/json", and HasTables: bool indicating whether the body contains the tables field. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt index 9d432fb69d0..80052485240 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB procedure fetch_schema_summary(server_url: String) that performs an outbound GET to this database's schema endpoint. Return a typed FetchSummary with the HTTP status, whether Content-Type contains "application/json", and whether the body contains the tables field. +Write a Rust SpacetimeDB procedure fetch_schema_summary(server_url: String) that performs an outbound GET to this database's schema endpoint. Return a FetchSummary product type with status: u16, json_content_type: bool indicating whether Content-Type contains "application/json", and has_tables: bool indicating whether the body contains the tables field. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt index b927a98a4d2..d0ac5ce66d8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB procedure fetch_schema_summary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a typed FetchSummary with the HTTP status, whether Content-Type contains "application/json", and whether the body contains the tables field. +Write a TypeScript SpacetimeDB procedure fetch_schema_summary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a FetchSummary object type with status: u16, jsonContentType: bool indicating whether Content-Type contains "application/json", and hasTables: bool indicating whether the body contains the tables field. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt index 72885d1f502..087915c44fd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public ProcedureResult table containing Id: ulong primary key and Value: uint. Add ScheduleProcedure(id, lhs, rhs) to enqueue a procedure for one millisecond later. The scheduled procedure must open a transaction and insert Value lhs + rhs into ProcedureResult. +Write a C# SpacetimeDB module with a public ProcedureResult table containing Id: ulong primary key and Value: uint. Define a private ProcedureJob scheduled table for procedure RunScheduledProcedure with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, Id: ulong, Lhs: uint, and Rhs: uint in that order. Add ScheduleProcedure(id, lhs, rhs) to enqueue that procedure for one millisecond later. RunScheduledProcedure must open a transaction and insert Value lhs + rhs into ProcedureResult. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt index 5bfcb9417da..9d569667134 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public procedure_result table containing id: u64 primary key and value: u32. Add schedule_procedure(id, lhs, rhs) to enqueue a procedure for one millisecond later. The scheduled procedure must open a transaction and insert value lhs + rhs into procedure_result. +Write a Rust SpacetimeDB module with a public procedure_result table (struct ProcedureResult) containing id: u64 primary key and value: u32. Define a private procedure_job scheduled table (struct ProcedureJob) for procedure run_scheduled_procedure with scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, id: u64, lhs: u32, and rhs: u32 in that order. Add schedule_procedure(id, lhs, rhs) to enqueue that procedure for one millisecond later. run_scheduled_procedure must open a transaction and insert value lhs + rhs into procedure_result. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt index 92f0f8ee73c..6cf21b8c579 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public procedure_result table containing id: u64 primary key and value: u32. Add schedule_procedure(id, lhs, rhs) to enqueue a procedure for one millisecond later. The scheduled procedure must open a transaction and insert value lhs + rhs into procedure_result. +Write a TypeScript SpacetimeDB module with a public procedure_result table containing id: u64 primary key and value: u32. Define a private procedure_job scheduled table for procedure runScheduledProcedure with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, id: u64, lhs: u32, and rhs: u32 in that order. Add schedule_procedure(id, lhs, rhs) to enqueue that procedure for one millisecond later. runScheduledProcedure must open a transaction and insert value lhs + rhs into procedure_result. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt index 9fc233393b7..3b453e8b631 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Persist processed event IDs and one public WebhookState row for key "account". Duplicate event IDs must return "duplicate" without applying twice, and events whose sequence is not newer must return "stale" without replacing state. Register the route through an HTTP router. +Write a C# SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Define a private ProcessedEvent table with EventId: string primary key. Define a public WebhookState table with Key: string primary key, LastSequence: ulong, and Value: string. Store state under Key "account". A newly applied event must return status 200 and body "applied". Duplicate event IDs must return status 200 and body "duplicate" without applying twice. Events whose sequence is not newer must return status 200 and body "stale" without replacing state. Register POST /webhook through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt index 6a36c9747d2..c411d7879af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Persist processed event IDs and one public webhook_state row for key "account". Duplicate event IDs must return "duplicate" without applying twice, and events whose sequence is not newer must return "stale" without replacing state. Register the route through an HTTP router. +Write a Rust SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Define a private processed_event table (struct ProcessedEvent) with event_id: String primary key. Define a public webhook_state table (struct WebhookState) with key: String primary key, last_sequence: u64, and value: String. Store state under key "account". A newly applied event must return status 200 and body "applied". Duplicate event IDs must return status 200 and body "duplicate" without applying twice. Events whose sequence is not newer must return status 200 and body "stale" without replacing state. Register POST /webhook through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt index 0903cfa02b2..134fb018f6f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Persist processed event IDs and one public webhook_state row for key "account". Duplicate event IDs must return "duplicate" without applying twice, and events whose sequence is not newer must return "stale" without replacing state. Register the route through an HTTP router. +Write a TypeScript SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Define a private processed_event table with eventId: string primary key. Define a public webhook_state table with key: string primary key, lastSequence: u64, and value: string. Store state under key "account". A newly applied event must return status 200 and body "applied". Duplicate event IDs must return status 200 and body "duplicate" without applying twice. Events whose sequence is not newer must return status 200 and body "stale" without replacing state. Register POST /webhook through an HTTP router. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt index 97f8170bd59..b42b9b0c286 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public UploadedAsset table containing Id: ulong primary key, Url: string, and Size: ulong. Register a POST /upload handler that returns the URL "https://files.local/object-1". Add UploadAndRegister(serverUrl: string, data: byte[]) to POST the binary data to this database's upload route, then open a transaction to store Id 1, the returned URL, and the byte length, and return the URL. +Write a C# SpacetimeDB module with a public UploadedAsset table containing Id: ulong primary key, Url: string, and Size: ulong. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure UploadAndRegister(serverUrl: string, data: byte[]) to POST the binary data to this database's upload route, then open a transaction to store Id 1, the returned URL, and the byte length, and return the URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt index dca9d03bb50..58218c7cc32 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: String, and size: u64. Register a POST /upload handler that returns the URL "https://files.local/object-1". Add upload_and_register(server_url: String, data: Vec) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. +Write a Rust SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: String, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(server_url: String, data: Vec) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt index 3aa4bc5ac00..382434b8ef7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: string, and size: u64. Register a POST /upload handler that returns the URL "https://files.local/object-1". Add upload_and_register(serverUrl: string, data: u8[]) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. +Write a TypeScript SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: string, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(serverUrl: string, data: u8[]) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt index 564e00ecfc4..0f57f81f8f3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt @@ -1,3 +1,3 @@ Write a C# SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. -Define a public WorkItem table with Id: ulong primary key and indexed GroupId: ulong. Define a private DeleteJob scheduled table with ScheduledId, ScheduledAt, and GroupId. Add SeedGroup(groupId, count), RequestDelete(groupId), and the scheduled reducer. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. +Define a public WorkItem table with Id: ulong primary key and indexed GroupId: ulong. Define a private DeleteJob scheduled table with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, and GroupId: ulong. Add SeedGroup(groupId, count), RequestDelete(groupId), and the scheduled reducer RunDeleteBatch. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt index 18328c64d6a..25ee2d53f41 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt @@ -1,3 +1,3 @@ Write a Rust SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. -Define a public work_item table with id: u64 primary key and indexed group_id: u64. Define a private delete_job scheduled table with scheduled_id, scheduled_at, and group_id. Add seed_group(group_id, count), request_delete(group_id), and the scheduled reducer. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. +Define a public work_item table with struct WorkItem and fields id: u64 primary key and indexed group_id: u64. Define a private delete_job scheduled table with struct DeleteJob and fields scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, and group_id: u64. Add seed_group(group_id, count), request_delete(group_id), and the scheduled reducer run_delete_batch. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt index 4905cfa7394..6ba828b1f31 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt @@ -1,3 +1,3 @@ Write a TypeScript SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. -Define a public work_item table with id: u64 primary key and indexed groupId: u64. Define a private delete_job scheduled table with scheduledId, scheduledAt, and groupId. Add seed_group(groupId, count), request_delete(groupId), and the scheduled reducer. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. +Define a public work_item table with id: u64 primary key and indexed groupId: u64. Define a private delete_job scheduled table with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, and groupId: u64. Add seed_group(groupId, count), request_delete(groupId), and the scheduled reducer runDeleteBatch. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs index 3e5dfa7286a..332980dcb89 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs @@ -17,7 +17,7 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "touch".into(), args: vec![], - select_query: format!("SELECT {id}, {name}, {enabled} FROM {widget} ORDER BY {id}"), + select_query: format!("SELECT {id}, {name}, {enabled} FROM {widget}"), collapse_ws: true, timeout: Duration::from_secs(10), id_str: "existing_row_backfilled", diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt index 99b497746f4..15232665c43 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with public Customer(Id, Name), Purchase(Id, CustomerId), and LineItem(Id, PurchaseId, Sku, indexed Visible) tables. IDs are ulong primary keys and relationship IDs are indexed. Add Seed() with one complete visible chain. Expose a public OrderLineDetail view returning LineId, CustomerName, and Sku by joining all three tables through indexed lookups. +Write a C# SpacetimeDB module with public Customer, Purchase, and LineItem tables. Customer has Id: ulong primary key and Name: string. Purchase has Id: ulong primary key and indexed CustomerId: ulong. LineItem has Id: ulong primary key, indexed PurchaseId: ulong, Sku: string, and indexed Visible: bool. Add Seed() with one complete visible chain. Define an OrderLineDetail product type with LineId: ulong, CustomerName: string, and Sku: string. Expose it through a public OrderLineDetail view by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt index 4c88a932189..c46e4d7e127 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with public customer(id, name), purchase(id, customer_id), and line_item(id, purchase_id, sku, indexed visible) tables. IDs are u64 primary keys and relationship IDs are indexed. Add seed() with one complete visible chain. Expose a public order_line_detail view returning line_id, customer_name, and sku by joining all three tables through indexed lookups. +Write a Rust SpacetimeDB module with public customer (struct Customer), purchase (struct Purchase), and line_item (struct LineItem) tables. Customer has id: u64 primary key and name: String. Purchase has id: u64 primary key and indexed customer_id: u64. LineItem has id: u64 primary key, indexed purchase_id: u64, sku: String, and indexed visible: bool. Add seed() with one complete visible chain. Define an OrderLineDetail product type with line_id: u64, customer_name: String, and sku: String. Expose it through a public order_line_detail view by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt index 99fda7ea833..009d670d893 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with public customer(id, name), purchase(id, customerId), and line_item(id, purchaseId, sku, indexed visible) tables. IDs are u64 primary keys and relationship IDs are indexed. Add seed() with one complete visible chain. Expose a public order_line_detail view returning lineId, customerName, and sku by joining all three tables through indexed lookups. +Write a TypeScript SpacetimeDB module with public customer, purchase, and line_item tables. Customer has id: u64 primary key and name: string. Purchase has id: u64 primary key and indexed customerId: u64. Line item has id: u64 primary key, indexed purchaseId: u64, sku: string, and indexed visible: bool. Add seed() with one complete visible chain. Define an OrderLineDetail row type with lineId: u64, customerName: string, and sku: string. Expose it through a public order_line_detail view by joining all three tables through indexed lookups. From f49cc64048adfb8819d50bf7b23573904f7b3bfd Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:52:33 -0400 Subject: [PATCH 17/78] Document advanced TypeScript server APIs --- skills/typescript-server/SKILL.md | 95 ++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 70d702bf6da..f108c3a6987 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -87,7 +87,11 @@ Every column is a `t` builder value: | `t.timeDuration()` | TimeDuration | | | `t.scheduleAt()` | ScheduleAt | | -Modifiers (complete set): `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')` +Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. + +Use `.default(value)` only for a newly appended migration-safe field. Preserve existing fields and reducers exactly, and do not put defaults on primary-key, unique, or auto-increment columns. + +Additional numeric builders include `t.u128()`, `t.i128()`, `t.u256()`, and `t.i256()`. Optional columns: `nickname: t.option(t.string())` @@ -171,6 +175,19 @@ new Date(Number(row.createdAt.microsSinceUnixEpoch / 1000n)); Do not construct `Identity` values from strings (e.g. `'hex' as Identity`): serialization fails and kills the module. Identities come from `ctx.sender` or `t.identity()` columns. +Synthetic connection IDs for module logic/tests can use `new ConnectionId(1n)` after importing `ConnectionId` from `spacetimedb`. + +Construct exact timestamps with `new Timestamp(micros)` after importing `Timestamp` from `spacetimedb`. Inclusive index ranges use `Range`: + +```typescript +import { Range, Timestamp } from 'spacetimedb'; + +ctx.db.event.occurredAt.filter(new Range( + { tag: 'included', value: new Timestamp(200n) }, + { tag: 'included', value: new Timestamp(400n) }, +)); +``` + ## Scheduled Tables ```typescript @@ -228,3 +245,79 @@ export const myProfile = spacetimedb.view( (ctx) => ctx.db.entity.identity.find(ctx.sender) ?? undefined ); ``` + +For a procedural view primary key, define the output with `t.row` and mark its field `.primaryKey()`: + +```typescript +const SourceViewRow = t.row('SourceViewRow', { + id: t.u64().primaryKey(), + value: t.string(), +}); +``` + +Query-builder views use `ctx.from` and return the query directly: + +```typescript +export const openTicket = spacetimedb.view( + { name: 'open_ticket', public: true }, + t.array(ticket.rowType), + ctx => ctx.from.ticket.where(ticket => ticket.status.eq('open')) +); + +export const eligibleMember = spacetimedb.view( + { name: 'eligible_member', public: true }, + t.array(member.rowType), + ctx => ctx.from.eligibility.rightSemijoin( + ctx.from.member, + (eligibility, member) => eligibility.memberId.eq(member.id) + ) +); +``` + +## Client Visibility Filters + +```typescript +export const userRecordFilter = spacetimedb.clientVisibilityFilter.sql( + 'SELECT * FROM user_record WHERE identity = :sender' +); +``` + +## Procedures and HTTP + +Procedures declare argument and return types. They can perform outbound HTTP through `ctx.http` and open short transactions with `ctx.withTx`: + +```typescript +const Summary = t.object('Summary', { total: t.u32(), label: t.string() }); + +export const calculateSummary = spacetimedb.procedure( + { lhs: t.u32(), rhs: t.u32() }, + Summary, + (_ctx, { lhs, rhs }) => ({ total: lhs + rhs, label: 'calculated' }) +); + +export const fetchAndStore = spacetimedb.procedure( + { url: t.string() }, + t.unit(), + (ctx, { url }) => { + const response = ctx.http.fetch(url); + ctx.withTx(tx => tx.db.fetchedRecord.insert({ id: 1n, status: response.status })); + return {}; + } +); +``` + +Scheduled procedures use a normal scheduled table, but reference a `spacetimedb.procedure(...)` callback instead of a reducer. + +Inbound HTTP uses `httpHandler`, `httpRouter`, `Router`, and `SyncResponse`: + +```typescript +import { Router, SyncResponse } from 'spacetimedb/server'; + +export const echo = spacetimedb.httpHandler((_ctx, request) => + new SyncResponse(`echo:${request.text()}`, { + status: 201, + headers: { 'content-type': 'text/plain' }, + }) +); +export const routes = spacetimedb.httpRouter(new Router().post('/echo', echo)); +``` From 43c03306da6a273698fe480c59bdad68039ee9e0 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:22:41 -0400 Subject: [PATCH 18/78] Validate LLM benchmark goldens behaviorally --- tools/xtask-llm-benchmark/src/bench/runner.rs | 129 ++++++++++++++++++ .../t_070_connection_scoped_presence/spec.rs | 5 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_073_http_fetch/answers/csharp.cs | 9 +- .../t_073_http_fetch/answers/rust.rs | 13 +- .../t_073_http_fetch/answers/typescript.ts | 14 +- .../procedures/t_073_http_fetch/spec.rs | 7 +- .../t_073_http_fetch/tasks/csharp.txt | 2 +- .../t_073_http_fetch/tasks/rust.txt | 2 +- .../t_073_http_fetch/tasks/typescript.txt | 2 +- .../t_074_fetch_and_store/answers/csharp.cs | 9 +- .../t_074_fetch_and_store/answers/rust.rs | 11 +- .../answers/typescript.ts | 12 +- .../procedures/t_074_fetch_and_store/spec.rs | 9 +- .../t_074_fetch_and_store/tasks/csharp.txt | 2 +- .../t_074_fetch_and_store/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../answers/csharp.cs | 10 +- .../answers/rust.rs | 13 +- .../answers/typescript.ts | 12 +- .../t_079_external_upload_flow/spec.rs | 16 ++- .../tasks/csharp.txt | 2 +- .../t_079_external_upload_flow/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../src/bin/llm_benchmark.rs | 8 +- 27 files changed, 224 insertions(+), 77 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 2e729c383ce..7a70069d705 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -1035,6 +1035,135 @@ pub async fn build_goldens_only_for_lang( Ok(()) } +/// Build each selected golden, publish the same source as a stand-in candidate, +/// and run the task's complete scorer set against the pair. This proves more +/// than compilation: the golden must also satisfy every behavior and parity +/// assertion used to grade model output. +pub async fn validate_goldens_for_lang( + host: Option, + bench_root: &Path, + lang: Lang, + selectors: Option<&[String]>, +) -> Result<()> { + build_goldens_only_for_lang(host.clone(), bench_root, lang, selectors).await?; + + let tasks = if let Some(sels) = selectors { + let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; + let filtered: Vec = discover_tasks(bench_root)? + .into_iter() + .filter(|task| { + let name = task.root.file_name().and_then(|name| name.to_str()).unwrap_or(""); + wanted.iter().any(|wanted| name.starts_with(wanted)) + }) + .collect(); + if filtered.is_empty() { + bail!("no tasks matched {:?}", wanted); + } + filtered + } else { + discover_tasks(bench_root)? + }; + + let runner = TaskRunner::new( + PathBuf::from(bench_root), + SpacetimeRustPublisher, + DotnetPublisher, + TypeScriptPublisher, + ); + let lang_name = lang.as_str(); + let route_tag = "golden-validation"; + let host_url = host.as_deref().unwrap_or("local"); + let mut failures = Vec::new(); + + // Scorers can mutate both databases, so validate one task at a time and + // preserve the scorer ordering declared by its spec. + for task in tasks { + let category = category_slug(&task.root); + let task_id = task_slug(&task.root); + let candidate_db = sanitize_db_name(&format!("{}-{}-{}-llm", category, task_id, route_tag)); + let golden_src_text = load_golden_source(&task, lang)?; + let migration_setup = load_migration_setup_source(&task, lang)?; + + println!("→ [{}] validate golden {} {}", lang_name, category, task_id); + if let Some(setup_source) = migration_setup { + runner + .publish( + PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag, + source_text: &setup_source, + db_name: candidate_db.clone(), + host: host.clone(), + clear_database: true, + }, + "golden-validation-setup", + ) + .await?; + call_reducer_json_out(&candidate_db, "seed", &[], Some(host_url)).map_err(anyhow::Error::msg)?; + runner + .publish( + PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag, + source_text: &golden_src_text, + db_name: candidate_db, + host: host.clone(), + clear_database: false, + }, + "golden-validation", + ) + .await?; + } else { + runner + .publish( + PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag, + source_text: &golden_src_text, + db_name: candidate_db, + host: host.clone(), + clear_database: true, + }, + "golden-validation", + ) + .await?; + } + + let ctor = resolve_by_path(&task.root)?; + let spec = ctor(); + for scorer in spec.scorers_for(lang, route_tag, host_url) { + let details = scorer.score(&golden_src_text); + if !details.pass { + failures.push(format!( + "{}/{} [{}]: {}", + category, + task_id, + scorer.id(), + serde_json::to_string(&details.notes).unwrap_or_else(|_| "".to_string()) + )); + } + } + } + + if !failures.is_empty() { + bail!( + "[{}] golden validation failed ({} scorer failures):\n{}", + lang_name, + failures.len(), + failures.join("\n") + ); + } + + println!("✓ [{}] goldens behavior/scorer validation: complete", lang_name); + Ok(()) +} + fn discover_tasks(benchmarks_root: &Path) -> Result> { let mut out = Vec::new(); for cat in read_dirs(benchmarks_root)? { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs index 1f8a0154950..e6c1fd0f1a8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs @@ -8,7 +8,10 @@ pub fn spec() -> BenchmarkSpec { scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { src_file: file!(), route_tag, reducer: "exercise_presence".into(), args: vec![], sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("presence_session", lang)), - expected_count: 1, id_str: "disconnect_removes_one_connection", timeout: Duration::from_secs(10), + // The CLI call itself creates one lifecycle-managed session. The + // helper adds two synthetic sessions and removes one, leaving two + // total rows while still proving connection-scoped deletion. + expected_count: 2, id_str: "disconnect_removes_one_connection", timeout: Duration::from_secs(10), })); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index c817cdee964..3b9d399c7d5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public PresenceSession table keyed by ConnectionId: ConnectionId and containing indexed Identity: Identity plus ConnectedAt: Timestamp. ClientConnected must insert one row per connection, and ClientDisconnected must delete only that connection's row. Share that logic with ExercisePresence(), which inserts two synthetic connections for the caller and removes only the first so one session remains. +Write a C# SpacetimeDB module with a public PresenceSession table keyed by ConnectionId: ConnectionId and containing indexed Identity: Identity plus ConnectedAt: Timestamp. ClientConnected must insert one row per connection, and ClientDisconnected must delete only that connection's row. Share that logic with ExercisePresence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt index b0ab6391bbf..b93ae85af3d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public presence_session table keyed by connection_id: ConnectionId and containing indexed identity: Identity plus connected_at: Timestamp. Client-connected must insert one row per connection, and client-disconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first so one session remains. +Write a Rust SpacetimeDB module with a public presence_session table keyed by connection_id: ConnectionId and containing indexed identity: Identity plus connected_at: Timestamp. Client-connected must insert one row per connection, and client-disconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt index 3625e9999c5..2814d91ba92 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public presence_session table keyed by connectionId: ConnectionId and containing indexed identity: Identity plus connectedAt: Timestamp. clientConnected must insert one row per connection, and clientDisconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first so one session remains. +Write a TypeScript SpacetimeDB module with a public presence_session table keyed by connectionId: ConnectionId and containing indexed identity: Identity plus connectedAt: Timestamp. clientConnected must insert one row per connection, and clientDisconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs index 88134eda8b8..a3295bb0f93 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs @@ -3,14 +3,13 @@ #pragma warning disable STDB_UNSTABLE [SpacetimeDB.Type] -public partial struct FetchSummary { public ushort Status; public bool JsonContentType; public bool HasTables; } +public partial struct FetchSummary { public ushort Status; public bool HtmlContentType; public bool HasExampleDomain; } public static partial class Module { [SpacetimeDB.Procedure] - public static FetchSummary FetchSchemaSummary(ProcedureContext ctx, string serverUrl) + public static FetchSummary FetchPageSummary(ProcedureContext ctx, string url) { - var url = $"{serverUrl.TrimEnd('/')}/v1/database/{ProcedureContextBase.Identity}/schema?version=9"; var result = ctx.Http.Get(url); return result.Match(response => { var contentType = response.Headers.FirstOrDefault(h => h.Name.Equals("content-type", StringComparison.OrdinalIgnoreCase)); @@ -18,8 +17,8 @@ public static FetchSummary FetchSchemaSummary(ProcedureContext ctx, string serve var body = response.Body.ToStringUtf8Lossy(); return new FetchSummary { Status = response.StatusCode, - JsonContentType = contentTypeValue.Contains("application/json"), - HasTables = body.Contains("\"tables\""), + HtmlContentType = contentTypeValue.Contains("text/html"), + HasExampleDomain = body.Contains("Example Domain"), }; }, error => throw new Exception(error.Message)); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs index d61da9cf8e9..98585213d74 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs @@ -1,15 +1,14 @@ use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; #[derive(SpacetimeType)] -pub struct FetchSummary { pub status: u16, pub json_content_type: bool, pub has_tables: bool } +pub struct FetchSummary { pub status: u16, pub html_content_type: bool, pub has_example_domain: bool } #[procedure] -pub fn fetch_schema_summary(ctx: &mut ProcedureContext, server_url: String) -> FetchSummary { - let url = format!("{}/v1/database/{}/schema?version=9", server_url.trim_end_matches('/'), ctx.database_identity()); - let response = ctx.http.get(url).expect("schema request failed"); +pub fn fetch_page_summary(ctx: &mut ProcedureContext, url: String) -> FetchSummary { + let response = ctx.http.get(url).expect("page request failed"); let status = response.status().as_u16(); - let json_content_type = response.headers().get("content-type") - .and_then(|value| value.to_str().ok()).is_some_and(|value| value.contains("application/json")); + let html_content_type = response.headers().get("content-type") + .and_then(|value| value.to_str().ok()).is_some_and(|value| value.contains("text/html")); let body = response.into_body().into_string_lossy(); - FetchSummary { status, json_content_type, has_tables: body.contains("\"tables\"") } + FetchSummary { status, html_content_type, has_example_domain: body.contains("Example Domain") } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts index 6edb0b73b16..c027095ce68 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts @@ -1,19 +1,19 @@ import { schema, t } from 'spacetimedb/server'; const FetchSummary = t.object('FetchSummary', { - status: t.u16(), jsonContentType: t.bool(), hasTables: t.bool(), + status: t.u16(), htmlContentType: t.bool(), hasExampleDomain: t.bool(), }); const spacetimedb = schema({}); export default spacetimedb; -export const fetch_schema_summary = spacetimedb.procedure( - { serverUrl: t.string() }, FetchSummary, - (ctx, { serverUrl }) => { - const response = ctx.http.fetch(`${serverUrl.replace(/\/+$/, '')}/v1/database/${ctx.databaseIdentity}/schema?version=9`); +export const fetch_page_summary = spacetimedb.procedure( + { url: t.string() }, FetchSummary, + (ctx, { url }) => { + const response = ctx.http.fetch(url); return { status: response.status, - jsonContentType: (response.headers.get('content-type') ?? '').includes('application/json'), - hasTables: response.text().includes('"tables"'), + htmlContentType: (response.headers.get('content-type') ?? '').includes('text/html'), + hasExampleDomain: response.text().includes('Example Domain'), }; } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs index 35758fb4da1..0abea3ef113 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs @@ -6,7 +6,12 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_call_output_parity_scorer( - host_url, file!(), route_tag, "fetch_schema_summary", vec![json!(host_url)], "http_response_summary", + host_url, + file!(), + route_tag, + "fetch_page_summary", + vec![json!("https://example.com")], + "http_response_summary", )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt index 1a635211cbc..fb9aec69b98 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB procedure FetchSchemaSummary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a FetchSummary product type with Status: ushort, JsonContentType: bool indicating whether Content-Type contains "application/json", and HasTables: bool indicating whether the body contains the tables field. +Write a C# SpacetimeDB procedure FetchPageSummary(url: string) that performs an outbound GET to the supplied public URL. Return a FetchSummary product type with Status: ushort, HtmlContentType: bool indicating whether Content-Type contains "text/html", and HasExampleDomain: bool indicating whether the body contains "Example Domain". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt index 80052485240..9b49b7f3c8f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB procedure fetch_schema_summary(server_url: String) that performs an outbound GET to this database's schema endpoint. Return a FetchSummary product type with status: u16, json_content_type: bool indicating whether Content-Type contains "application/json", and has_tables: bool indicating whether the body contains the tables field. +Write a Rust SpacetimeDB procedure fetch_page_summary(url: String) that performs an outbound GET to the supplied public URL. Return a FetchSummary product type with status: u16, html_content_type: bool indicating whether Content-Type contains "text/html", and has_example_domain: bool indicating whether the body contains "Example Domain". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt index d0ac5ce66d8..be65d246ef9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB procedure fetch_schema_summary(serverUrl: string) that performs an outbound GET to this database's schema endpoint. Return a FetchSummary object type with status: u16, jsonContentType: bool indicating whether Content-Type contains "application/json", and hasTables: bool indicating whether the body contains the tables field. +Write a TypeScript SpacetimeDB procedure fetch_page_summary(url: string) that performs an outbound GET to the supplied public URL. Return a FetchSummary object type with status: u16, htmlContentType: bool indicating whether Content-Type contains "text/html", and hasExampleDomain: bool indicating whether the body contains "Example Domain". diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs index d9b11eed48f..9b118401472 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs @@ -4,16 +4,15 @@ public static partial class Module { [Table(Accessor = "FetchedRecord", Public = true)] - public partial struct FetchedRecord { [PrimaryKey] public ulong Id; public ushort Status; public bool ValidSchema; } + public partial struct FetchedRecord { [PrimaryKey] public ulong Id; public ushort Status; public bool ValidBody; } [SpacetimeDB.Procedure] - public static void FetchAndStore(ProcedureContext ctx, string serverUrl) + public static void FetchAndStore(ProcedureContext ctx, string url) { - var url = $"{serverUrl.TrimEnd('/')}/v1/database/{ProcedureContextBase.Identity}/schema?version=9"; var result = ctx.Http.Get(url); result.Match(response => { - var validSchema = response.Body.ToStringUtf8Lossy().Contains("\"tables\""); - ctx.WithTx(tx => { tx.Db.FetchedRecord.Insert(new FetchedRecord { Id = 1, Status = response.StatusCode, ValidSchema = validSchema }); return 0; }); + var validBody = response.Body.ToStringUtf8Lossy().Contains("Example Domain"); + ctx.WithTx(tx => { tx.Db.FetchedRecord.Insert(new FetchedRecord { Id = 1, Status = response.StatusCode, ValidBody = validBody }); return 0; }); return 0; }, error => throw new Exception(error.Message)); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs index e744709d8f3..561c93390c0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs @@ -1,13 +1,12 @@ use spacetimedb::{procedure, table, ProcedureContext, Table}; #[table(accessor = fetched_record, public)] -pub struct FetchedRecord { #[primary_key] pub id: u64, pub status: u16, pub valid_schema: bool } +pub struct FetchedRecord { #[primary_key] pub id: u64, pub status: u16, pub valid_body: bool } #[procedure] -pub fn fetch_and_store(ctx: &mut ProcedureContext, server_url: String) { - let url = format!("{}/v1/database/{}/schema?version=9", server_url.trim_end_matches('/'), ctx.database_identity()); - let response = ctx.http.get(url).expect("schema request failed"); +pub fn fetch_and_store(ctx: &mut ProcedureContext, url: String) { + let response = ctx.http.get(url).expect("page request failed"); let status = response.status().as_u16(); - let valid_schema = response.into_body().into_string_lossy().contains("\"tables\""); - ctx.with_tx(|tx| { tx.db.fetched_record().insert(FetchedRecord { id: 1, status, valid_schema }); }); + let valid_body = response.into_body().into_string_lossy().contains("Example Domain"); + ctx.with_tx(|tx| { tx.db.fetched_record().insert(FetchedRecord { id: 1, status, valid_body }); }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts index 2a3c89d86f6..a4e23be8b00 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts @@ -1,18 +1,18 @@ import { schema, table, t } from 'spacetimedb/server'; const fetchedRecord = table({ name: 'fetched_record', public: true }, { - id: t.u64().primaryKey(), status: t.u16(), validSchema: t.bool(), + id: t.u64().primaryKey(), status: t.u16(), validBody: t.bool(), }); const spacetimedb = schema({ fetchedRecord }); export default spacetimedb; export const fetch_and_store = spacetimedb.procedure( - { serverUrl: t.string() }, t.unit(), - (ctx, { serverUrl }) => { - const response = ctx.http.fetch(`${serverUrl.replace(/\/+$/, '')}/v1/database/${ctx.databaseIdentity}/schema?version=9`); + { url: t.string() }, t.unit(), + (ctx, { url }) => { + const response = ctx.http.fetch(url); const status = response.status; - const validSchema = response.text().includes('"tables"'); - ctx.withTx(tx => tx.db.fetchedRecord.insert({ id: 1n, status, validSchema })); + const validBody = response.text().includes('Example Domain'); + ctx.withTx(tx => tx.db.fetchedRecord.insert({ id: 1n, status, validBody })); return {}; } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs index b44bd1d7341..ed226ed2319 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs @@ -7,11 +7,16 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_reducer_call_both_scorer( - host_url, file!(), route_tag, "fetch_and_store", vec![json!(host_url)], "fetch_and_store", + host_url, + file!(), + route_tag, + "fetch_and_store", + vec![json!("https://example.com")], + "fetch_and_store", )); let table = table_name("fetched_record", lang); let status = ident("status", casing_for_lang(lang)); - let valid = ident("valid_schema", casing_for_lang(lang)); + let valid = ident("valid_body", casing_for_lang(lang)); scorers.push(make_sql_count_only_scorer( host_url, file!(), route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}=200 AND {valid}=true"), diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt index cfd6d5c7a72..83b5ec2994a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public FetchedRecord table containing Id: ulong primary key, Status: ushort, and ValidSchema: bool. Add FetchAndStore(serverUrl: string), which performs an outbound GET to this database's schema endpoint before opening a transaction, then stores Id 1, the response status, and whether the body contains the tables field inside a short transaction. +Write a C# SpacetimeDB module with a public FetchedRecord table containing Id: ulong primary key, Status: ushort, and ValidBody: bool. Add FetchAndStore(url: string), which performs an outbound GET to the supplied public URL before opening a transaction, then stores Id 1, the response status, and whether the body contains "Example Domain" inside a short transaction. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt index e40850893c9..8ad28c12131 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and valid_schema: bool. Add fetch_and_store(server_url: String), which performs an outbound GET to this database's schema endpoint before opening a transaction, then stores id 1, the response status, and whether the body contains the tables field inside a short transaction. +Write a Rust SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and valid_body: bool. Add fetch_and_store(url: String), which performs an outbound GET to the supplied public URL before opening a transaction, then stores id 1, the response status, and whether the body contains "Example Domain" inside a short transaction. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt index 57fe73cdf80..2c49c4f7ad8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and validSchema: bool. Add fetch_and_store(serverUrl: string), which performs an outbound GET to this database's schema endpoint before opening a transaction, then stores id 1, the response status, and whether the body contains the tables field inside a short transaction. +Write a TypeScript SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and validBody: bool. Add fetch_and_store(url: string), which performs an outbound GET to the supplied public URL before opening a transaction, then stores id 1, the response status, and whether the body contains "Example Domain" inside a short transaction. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs index 423b653d114..453021cf307 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs @@ -15,18 +15,18 @@ public partial struct UploadedAsset { [PrimaryKey] public ulong Id; public strin public static Router Routes() => SpacetimeDB.Router.New().Post("/upload", Handlers.Upload); [SpacetimeDB.Procedure] - public static string UploadAndRegister(ProcedureContext ctx, string serverUrl, byte[] data) + public static string UploadAndRegister(ProcedureContext ctx, string uploadUrl, byte[] data) { var request = new HttpRequest { - Uri = $"{serverUrl.TrimEnd('/')}/v1/database/{ProcedureContextBase.Identity}/route/upload", + Uri = uploadUrl, Method = SpacetimeDB.HttpMethod.Post, Headers = new() { new HttpHeader("content-type", "application/octet-stream") }, Body = new HttpBody(data), }; return ctx.Http.Send(request).Match(response => { - var assetUrl = response.Body.ToStringUtf8Lossy(); - ctx.WithTx(tx => { tx.Db.UploadedAsset.Insert(new UploadedAsset { Id = 1, Url = assetUrl, Size = (ulong)data.Length }); return 0; }); - return assetUrl; + if (response.StatusCode < 200 || response.StatusCode >= 300) throw new Exception($"upload failed: {response.StatusCode}"); + ctx.WithTx(tx => { tx.Db.UploadedAsset.Insert(new UploadedAsset { Id = 1, Url = uploadUrl, Size = (ulong)data.Length }); return 0; }); + return uploadUrl; }, error => throw new Exception(error.Message)); } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs index b8637a1e208..c698fccb704 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs @@ -13,13 +13,12 @@ fn upload(_ctx: &mut HandlerContext, _request: Request) -> Response { fn routes() -> Router { Router::new().post("/upload", upload) } #[procedure] -pub fn upload_and_register(ctx: &mut ProcedureContext, server_url: String, data: Vec) -> String { - let url = format!("{}/v1/database/{}/route/upload", server_url.trim_end_matches('/'), ctx.database_identity()); - let request = Request::builder().method("POST").uri(url).header("content-type", "application/octet-stream") +pub fn upload_and_register(ctx: &mut ProcedureContext, upload_url: String, data: Vec) -> String { + let request = Request::builder().method("POST").uri(upload_url.clone()).header("content-type", "application/octet-stream") .body(Body::from_bytes(data.clone())).unwrap(); let response = ctx.http.send(request).expect("upload failed"); - let asset_url = response.into_body().into_string_lossy(); - let row_url = asset_url.clone(); - ctx.with_tx(|tx| { tx.db.uploaded_asset().insert(UploadedAsset { id: 1, url: row_url.clone(), size: data.len() as u64 }); }); - asset_url + assert!(response.status().is_success(), "upload failed: {}", response.status()); + let row_url = upload_url.clone(); + ctx.with_tx(|tx| { tx.db.uploaded_asset().insert(UploadedAsset { id: 1, url: row_url, size: data.len() as u64 }); }); + upload_url } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts index 17873876a0b..13f52a86438 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts @@ -12,14 +12,14 @@ export const upload = spacetimedb.httpHandler((_ctx, _request) => export const routes = spacetimedb.httpRouter(new Router().post('/upload', upload)); export const upload_and_register = spacetimedb.procedure( - { serverUrl: t.string(), data: t.array(t.u8()) }, t.string(), - (ctx, { serverUrl, data }) => { + { uploadUrl: t.string(), data: t.array(t.u8()) }, t.string(), + (ctx, { uploadUrl, data }) => { const response = ctx.http.fetch( - `${serverUrl.replace(/\/+$/, '')}/v1/database/${ctx.databaseIdentity}/route/upload`, + uploadUrl, { method: 'POST', headers: { 'content-type': 'application/octet-stream' }, body: new Uint8Array(data) } ); - const assetUrl = response.text(); - ctx.withTx(tx => tx.db.uploadedAsset.insert({ id: 1n, url: assetUrl, size: BigInt(data.length) })); - return assetUrl; + if (response.status < 200 || response.status >= 300) throw new Error(`upload failed: ${response.status}`); + ctx.withTx(tx => tx.db.uploadedAsset.insert({ id: 1n, url: uploadUrl, size: BigInt(data.length) })); + return uploadUrl; } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs index fb7c107e15e..f5f7dcab972 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs @@ -1,4 +1,7 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer, make_sql_count_only_scorer}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_call_output_parity_scorer, make_http_route_parity_scorer, + make_sql_count_only_scorer, +}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use serde_json::json; use std::time::Duration; @@ -8,14 +11,21 @@ pub fn spec() -> BenchmarkSpec { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_call_output_parity_scorer( host_url, file!(), route_tag, "upload_and_register", - vec![json!(host_url), json!([1, 2, 3, 4])], "upload_return_url", + vec![json!("https://postman-echo.com/post"), json!([1, 2, 3, 4])], "upload_return_url", + )); + scorers.push(make_http_route_parity_scorer( + host_url, + file!(), + route_tag, + vec![("POST", "/upload", Some("payload"))], + "upload_route", )); let table = table_name("uploaded_asset", lang); let url = ident("url", casing_for_lang(lang)); let size = ident("size", casing_for_lang(lang)); scorers.push(make_sql_count_only_scorer( host_url, file!(), route_tag, - format!("SELECT COUNT(*) AS n FROM {table} WHERE {url}='https://files.local/object-1' AND {size}=4"), + format!("SELECT COUNT(*) AS n FROM {table} WHERE {url}='https://postman-echo.com/post' AND {size}=4"), 1, "upload_metadata_stored", Duration::from_secs(10), )); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt index b42b9b0c286..44f301049c1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public UploadedAsset table containing Id: ulong primary key, Url: string, and Size: ulong. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure UploadAndRegister(serverUrl: string, data: byte[]) to POST the binary data to this database's upload route, then open a transaction to store Id 1, the returned URL, and the byte length, and return the URL. +Write a C# SpacetimeDB module with a public UploadedAsset table containing Id: ulong primary key, Url: string, and Size: ulong. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure UploadAndRegister(uploadUrl: string, data: byte[]) to POST the binary data to the supplied public URL, require a successful response, then open a transaction to store Id 1, the supplied URL, and the byte length, and return the supplied URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt index 58218c7cc32..2d62220ebe8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: String, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(server_url: String, data: Vec) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. +Write a Rust SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: String, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(upload_url: String, data: Vec) to POST the binary data to the supplied public URL, require a successful response, then open a transaction to store id 1, the supplied URL, and the byte length, and return the supplied URL. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt index 382434b8ef7..98adcc2ac06 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: string, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(serverUrl: string, data: u8[]) to POST the binary data to this database's upload route, then open a transaction to store id 1, the returned URL, and the byte length, and return the URL. +Write a TypeScript SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: string, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(uploadUrl: string, data: u8[]) to POST the binary data to the supplied public URL, require a successful response, then open a transaction to store id 1, the supplied URL, and the byte length, and return the supplied URL. diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index f3a21f22f54..9aec12cd932 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -13,7 +13,7 @@ use tokio::runtime::Runtime; use xtask_llm_benchmark::api::ApiClient; use xtask_llm_benchmark::bench::bench_route_concurrency; use xtask_llm_benchmark::bench::runner::{ - build_goldens_only_for_lang, ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, + ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, validate_goldens_for_lang, }; use xtask_llm_benchmark::bench::types::{BenchRunContext, RouteRun, RunConfig, RunOutcome}; use xtask_llm_benchmark::context::constants::ALL_MODES; @@ -100,7 +100,7 @@ struct RunArgs { #[arg(long, conflicts_with = "goldens_only")] hash_only: bool, - /// Build/publish goldens only (skip LLM calls) + /// Build/publish and self-score goldens (skip LLM calls) #[arg(long, conflicts_with = "hash_only")] goldens_only: bool, @@ -302,13 +302,13 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { if config.goldens_only { let rt = runtime.as_ref().expect("runtime required for --goldens-only"); - rt.block_on(build_goldens_only_for_lang( + rt.block_on(validate_goldens_for_lang( config.host.clone(), &bench_root, config.lang, selectors_ref, ))?; - println!("[{}] goldens-only build complete", config.lang.as_str()); + println!("[{}] goldens-only validation complete", config.lang.as_str()); return Ok(()); } From 6d06baac99d6b33af06cb85c5d4926762ce51617 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:23:00 -0400 Subject: [PATCH 19/78] Generalize advanced TypeScript server guidance --- skills/typescript-server/SKILL.md | 96 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index f108c3a6987..828f0f9e1b0 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -42,12 +42,12 @@ export const addRecord = spacetimedb.reducer( ## Imports -`spacetimedb/server` is the only import path for server modules: +Schema builders and module exports come from `spacetimedb/server`. Runtime value classes such as `ScheduleAt`, `Timestamp`, `Range`, and `ConnectionId` come from the root `spacetimedb` package: ```typescript import { schema, table, t } from 'spacetimedb/server'; import { SenderError } from 'spacetimedb/server'; -import { ScheduleAt } from 'spacetimedb'; // for scheduled tables only +import { ConnectionId, Range, ScheduleAt, Timestamp } from 'spacetimedb'; ``` ## Tables @@ -75,9 +75,11 @@ Every column is a `t` builder value: | Builder | JS type | Notes | |---------|---------|-------| +| `t.u8()` / `t.u16()` / `t.u32()` | number | | +| `t.i8()` / `t.i16()` / `t.i32()` | number | | | `t.u64()` | bigint | Use `0n` literals | | `t.i64()` | bigint | Use `0n` literals | -| `t.u32()` / `t.i32()` | number | | +| `t.u128()` / `t.i128()` / `t.u256()` / `t.i256()` | bigint | | | `t.f64()` / `t.f32()` | number | | | `t.bool()` | boolean | | | `t.string()` | string | | @@ -91,8 +93,6 @@ Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.defa Use `.default(value)` only for a newly appended migration-safe field. Preserve existing fields and reducers exactly, and do not put defaults on primary-key, unique, or auto-increment columns. -Additional numeric builders include `t.u128()`, `t.i128()`, `t.u256()`, and `t.i256()`. - Optional columns: `nickname: t.option(t.string())` ## Indexes @@ -166,7 +166,7 @@ ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); // Deterministic RNG const f: number = ctx.random(); // [0.0, 1.0) -const roll: number = ctx.random.integerInRange(1, 6); // inclusive +const roll: number = ctx.random.integerInRange(1, 6); // inclusive; wrap with BigInt(...) for an i64/u64 column const bytes: Uint8Array = ctx.random.fill(new Uint8Array(16)); // Client: Timestamp → Date @@ -182,9 +182,9 @@ Construct exact timestamps with `new Timestamp(micros)` after importing `Timesta ```typescript import { Range, Timestamp } from 'spacetimedb'; -ctx.db.event.occurredAt.filter(new Range( - { tag: 'included', value: new Timestamp(200n) }, - { tag: 'included', value: new Timestamp(400n) }, +ctx.db.shipment.deliverBy.filter(new Range( + { tag: 'included', value: new Timestamp(1_000n) }, + { tag: 'included', value: new Timestamp(2_000n) }, )); ``` @@ -230,6 +230,8 @@ const Shape = t.enum('Shape', { ## Views +`t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. + ```typescript // Anonymous view (same for all clients): export const activeUsers = spacetimedb.anonymousView( @@ -249,27 +251,27 @@ export const myProfile = spacetimedb.view( For a procedural view primary key, define the output with `t.row` and mark its field `.primaryKey()`: ```typescript -const SourceViewRow = t.row('SourceViewRow', { - id: t.u64().primaryKey(), - value: t.string(), +const CatalogKey = t.row('CatalogKey', { + sku: t.u64().primaryKey(), + label: t.string(), }); ``` Query-builder views use `ctx.from` and return the query directly: ```typescript -export const openTicket = spacetimedb.view( - { name: 'open_ticket', public: true }, - t.array(ticket.rowType), - ctx => ctx.from.ticket.where(ticket => ticket.status.eq('open')) +export const discountedProduct = spacetimedb.view( + { name: 'discounted_product', public: true }, + t.array(product.rowType), + ctx => ctx.from.product.where(product => product.discounted.eq(true)) ); -export const eligibleMember = spacetimedb.view( - { name: 'eligible_member', public: true }, - t.array(member.rowType), - ctx => ctx.from.eligibility.rightSemijoin( - ctx.from.member, - (eligibility, member) => eligibility.memberId.eq(member.id) +export const taggedProduct = spacetimedb.view( + { name: 'tagged_product', public: true }, + t.array(product.rowType), + ctx => ctx.from.productTag.rightSemijoin( + ctx.from.product, + (tag, product) => tag.productId.eq(product.id) ) ); ``` @@ -277,8 +279,8 @@ export const eligibleMember = spacetimedb.view( ## Client Visibility Filters ```typescript -export const userRecordFilter = spacetimedb.clientVisibilityFilter.sql( - 'SELECT * FROM user_record WHERE identity = :sender' +export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( + 'SELECT * FROM private_note WHERE author = :sender' ); ``` @@ -287,37 +289,61 @@ export const userRecordFilter = spacetimedb.clientVisibilityFilter.sql( Procedures declare argument and return types. They can perform outbound HTTP through `ctx.http` and open short transactions with `ctx.withTx`: ```typescript -const Summary = t.object('Summary', { total: t.u32(), label: t.string() }); +const Product = t.object('Product', { value: t.u32(), description: t.string() }); -export const calculateSummary = spacetimedb.procedure( +export const multiply = spacetimedb.procedure( { lhs: t.u32(), rhs: t.u32() }, - Summary, - (_ctx, { lhs, rhs }) => ({ total: lhs + rhs, label: 'calculated' }) + Product, + (_ctx, { lhs, rhs }) => ({ value: lhs * rhs, description: 'product' }) ); -export const fetchAndStore = spacetimedb.procedure( +export const refreshCache = spacetimedb.procedure( { url: t.string() }, t.unit(), (ctx, { url }) => { const response = ctx.http.fetch(url); - ctx.withTx(tx => tx.db.fetchedRecord.insert({ id: 1n, status: response.status })); + ctx.withTx(tx => tx.db.cacheEntry.insert({ key: url, status: response.status })); return {}; } ); ``` -Scheduled procedures use a normal scheduled table, but reference a `spacetimedb.procedure(...)` callback instead of a reducer. +Outbound responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. Perform network I/O before `withTx`; only database work belongs inside its callback. + +Scheduled procedures use a normal scheduled table, but its `scheduled` option references a `spacetimedb.procedure(...)` export instead of a reducer. The procedure receives its scheduled row as an argument and uses `withTx` for database access: + +```typescript +const cleanup_timer = table( + { name: 'cleanup_timer', scheduled: (): any => runCleanup }, + { + scheduledId: t.u64().primaryKey().autoInc(), + scheduledAt: t.scheduleAt(), + cacheKey: t.string(), + } +); + +export const runCleanup = spacetimedb.procedure( + { timer: cleanup_timer.rowType }, + t.unit(), + (ctx, { timer }) => { + ctx.withTx(tx => tx.db.cacheEntry.key.delete(timer.cacheKey)); + return {}; + } +); +``` Inbound HTTP uses `httpHandler`, `httpRouter`, `Router`, and `SyncResponse`: ```typescript import { Router, SyncResponse } from 'spacetimedb/server'; -export const echo = spacetimedb.httpHandler((_ctx, request) => - new SyncResponse(`echo:${request.text()}`, { - status: 201, +export const health = spacetimedb.httpHandler((_ctx, _request) => + new SyncResponse('ok', { + status: 200, headers: { 'content-type': 'text/plain' }, }) ); -export const routes = spacetimedb.httpRouter(new Router().post('/echo', echo)); +export const routes = spacetimedb.httpRouter(new Router().get('/health', health)); ``` + +Pass the exported `httpHandler(...)` value to the router, not its raw callback. A handler context does not expose `ctx.db`; use `ctx.withTx(tx => ...)` when a handler needs transactional database access. From 9a775e507a9b2f08e98bf5d60145270f1cb769f7 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:31:10 -0400 Subject: [PATCH 20/78] Clarify advanced view eval requirements --- skills/typescript-server/SKILL.md | 15 +++++++++------ .../views/t_061_three_table_join/tasks/csharp.txt | 2 +- .../views/t_061_three_table_join/tasks/rust.txt | 2 +- .../t_061_three_table_join/tasks/typescript.txt | 2 +- .../t_065_materialized_aggregate/tasks/csharp.txt | 2 +- .../t_065_materialized_aggregate/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_066_query_builder_view/tasks/csharp.txt | 2 +- .../views/t_066_query_builder_view/tasks/rust.txt | 2 +- .../t_066_query_builder_view/tasks/typescript.txt | 2 +- 10 files changed, 18 insertions(+), 15 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 828f0f9e1b0..dcac6d677fd 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -47,7 +47,8 @@ Schema builders and module exports come from `spacetimedb/server`. Runtime value ```typescript import { schema, table, t } from 'spacetimedb/server'; import { SenderError } from 'spacetimedb/server'; -import { ConnectionId, Range, ScheduleAt, Timestamp } from 'spacetimedb'; +import { ConnectionId, ScheduleAt, Timestamp } from 'spacetimedb'; +import { Range } from 'spacetimedb/server'; ``` ## Tables @@ -166,7 +167,8 @@ ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); // Deterministic RNG const f: number = ctx.random(); // [0.0, 1.0) -const roll: number = ctx.random.integerInRange(1, 6); // inclusive; wrap with BigInt(...) for an i64/u64 column +const roll: number = ctx.random.integerInRange(1, 6); // number bounds/result, inclusive +const storedRoll: bigint = BigInt(roll); // convert the result for an i64/u64 column const bytes: Uint8Array = ctx.random.fill(new Uint8Array(16)); // Client: Timestamp → Date @@ -177,10 +179,11 @@ Do not construct `Identity` values from strings (e.g. `'hex' as Identity`): seri Synthetic connection IDs for module logic/tests can use `new ConnectionId(1n)` after importing `ConnectionId` from `spacetimedb`. -Construct exact timestamps with `new Timestamp(micros)` after importing `Timestamp` from `spacetimedb`. Inclusive index ranges use `Range`: +Construct exact timestamps with `new Timestamp(micros)` after importing `Timestamp` from `spacetimedb`. Inclusive index ranges use `Range` from `spacetimedb/server`: ```typescript -import { Range, Timestamp } from 'spacetimedb'; +import { Timestamp } from 'spacetimedb'; +import { Range } from 'spacetimedb/server'; ctx.db.shipment.deliverBy.filter(new Range( { tag: 'included', value: new Timestamp(1_000n) }, @@ -230,7 +233,7 @@ const Shape = t.enum('Shape', { ## Views -`t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. +`t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). ```typescript // Anonymous view (same for all clients): @@ -346,4 +349,4 @@ export const health = spacetimedb.httpHandler((_ctx, _request) => export const routes = spacetimedb.httpRouter(new Router().get('/health', health)); ``` -Pass the exported `httpHandler(...)` value to the router, not its raw callback. A handler context does not expose `ctx.db`; use `ctx.withTx(tx => ...)` when a handler needs transactional database access. +Handlers are synchronous: return `SyncResponse` directly rather than marking the callback `async`. Pass the exported `httpHandler(...)` value to the router, not its raw callback. The router selects the path, while a handler reads request data with APIs such as `request.text()`; `Request` has no `path` property. A handler context does not expose `ctx.db`; use `ctx.withTx(tx => ...)` when a handler needs transactional database access. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt index 15232665c43..5335ede3449 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with public Customer, Purchase, and LineItem tables. Customer has Id: ulong primary key and Name: string. Purchase has Id: ulong primary key and indexed CustomerId: ulong. LineItem has Id: ulong primary key, indexed PurchaseId: ulong, Sku: string, and indexed Visible: bool. Add Seed() with one complete visible chain. Define an OrderLineDetail product type with LineId: ulong, CustomerName: string, and Sku: string. Expose it through a public OrderLineDetail view by joining all three tables through indexed lookups. +Write a C# SpacetimeDB module with public Customer, Purchase, and LineItem tables. Customer has Id: ulong primary key and Name: string. Purchase has Id: ulong primary key and indexed CustomerId: ulong. LineItem has Id: ulong primary key, indexed PurchaseId: ulong, Sku: string, and indexed Visible: bool. Add a Seed() reducer with one complete visible chain. Expose rows containing LineId: ulong, CustomerName: string, and Sku: string through a public OrderLineDetail view by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt index c46e4d7e127..238760763e9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with public customer (struct Customer), purchase (struct Purchase), and line_item (struct LineItem) tables. Customer has id: u64 primary key and name: String. Purchase has id: u64 primary key and indexed customer_id: u64. LineItem has id: u64 primary key, indexed purchase_id: u64, sku: String, and indexed visible: bool. Add seed() with one complete visible chain. Define an OrderLineDetail product type with line_id: u64, customer_name: String, and sku: String. Expose it through a public order_line_detail view by joining all three tables through indexed lookups. +Write a Rust SpacetimeDB module with public customer (struct Customer), purchase (struct Purchase), and line_item (struct LineItem) tables. Customer has id: u64 primary key and name: String. Purchase has id: u64 primary key and indexed customer_id: u64. LineItem has id: u64 primary key, indexed purchase_id: u64, sku: String, and indexed visible: bool. Add a seed() reducer with one complete visible chain. Expose rows containing line_id: u64, customer_name: String, and sku: String through a public order_line_detail view by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt index 009d670d893..c6fcac455d7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with public customer, purchase, and line_item tables. Customer has id: u64 primary key and name: string. Purchase has id: u64 primary key and indexed customerId: u64. Line item has id: u64 primary key, indexed purchaseId: u64, sku: string, and indexed visible: bool. Add seed() with one complete visible chain. Define an OrderLineDetail row type with lineId: u64, customerName: string, and sku: string. Expose it through a public order_line_detail view by joining all three tables through indexed lookups. +Write a TypeScript SpacetimeDB module with public customer, purchase, and line_item tables. Customer has id: u64 primary key and name: string. Purchase has id: u64 primary key and indexed customerId: u64. Line item has id: u64 primary key, indexed purchaseId: u64, sku: string, and indexed visible: bool. Add a seed() reducer with one complete visible chain. Expose rows containing lineId: u64, customerName: string, and sku: string through a public order_line_detail view by joining all three tables through indexed lookups. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt index fd4514eab70..decc35d6a14 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with public Sale(Id: ulong primary key, Category: string, Amount: long) and CategoryTotal(Category: string primary key, TotalAmount: long, SaleCount: ulong) tables. Implement shared upsert and delete logic that keeps CategoryTotal transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Add Exercise() that inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose CategoryTotal through a public CategorySummary query-builder view. The final view must contain only books with TotalAmount 25 and SaleCount 1. +Write a C# SpacetimeDB module with public Sale(Id: ulong primary key, Category: string, Amount: long) and CategoryTotal(Category: string primary key, TotalAmount: long, SaleCount: ulong) tables. Implement internal shared upsert and delete helpers that keep CategoryTotal transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Export only an Exercise() reducer; it inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose CategoryTotal through a public CategorySummary query-builder view. The final view must contain only books with TotalAmount 25 and SaleCount 1. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt index 31ccfc0dbf1..3179959c75f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with public sale(id: u64 primary key, category: String, amount: i64) and category_total(category: String primary key, total_amount: i64, sale_count: u64) tables. Implement shared upsert and delete logic that keeps category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Add exercise() that inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with total_amount 25 and sale_count 1. +Write a Rust SpacetimeDB module with public sale(id: u64 primary key, category: String, amount: i64) and category_total(category: String primary key, total_amount: i64, sale_count: u64) tables. Implement internal shared upsert and delete helpers that keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Export only an exercise() reducer; it inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with total_amount 25 and sale_count 1. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt index f9cc937da92..41153d9e415 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with public sale(id: u64 primary key, category: string, amount: i64) and category_total(category: string primary key, totalAmount: i64, saleCount: u64) tables. Implement shared upsert and delete logic that keeps category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Add exercise() that inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with totalAmount 25 and saleCount 1. +Write a TypeScript SpacetimeDB module with public sale(id: u64 primary key, category: string, amount: i64) and category_total(category: string primary key, totalAmount: i64, saleCount: u64) tables. Implement internal shared upsert and delete helpers that keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Export only an exercise() reducer; it inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with totalAmount 25 and saleCount 1. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt index a2918536e9b..de32f779b09 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public Ticket table containing Id: ulong primary key, indexed Status: string, and Title: string. Add Seed() with one open and one closed ticket. Define a public OpenTicket view that returns the query-builder filter for Status "open" rather than materializing rows manually. +Write a C# SpacetimeDB module with a public Ticket table containing Id: ulong primary key, indexed Status: string, and Title: string. Add a Seed() reducer with one open and one closed ticket. Define a public OpenTicket view that returns the query-builder filter for Status "open" rather than materializing rows manually. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt index a0cb8b67b3b..50014678baf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: String, and title: String. Add seed() with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. +Write a Rust SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: String, and title: String. Add a seed() reducer with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt index afab7a2aec9..32f27b5ce69 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: string, and title: string. Add seed() with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. +Write a TypeScript SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: string, and title: string. Add a seed() reducer with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. From 2f6bb292f2fb324afb800591c1c1399747dad920 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:34:17 -0400 Subject: [PATCH 21/78] Document TypeScript server API edge cases --- skills/typescript-server/SKILL.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index dcac6d677fd..5366163af8b 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -154,6 +154,8 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ``` +`ctx.connectionId` is `ConnectionId | null`, including in lifecycle contexts. Guard it before passing it to a helper or using it as a table key. + ## Reducer Context API `ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. In helpers, type it as `ReducerCtx>`. @@ -167,7 +169,7 @@ ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); // Deterministic RNG const f: number = ctx.random(); // [0.0, 1.0) -const roll: number = ctx.random.integerInRange(1, 6); // number bounds/result, inclusive +const roll: number = ctx.random.integerInRange(1, 6); // safe JS number bounds/result, inclusive const storedRoll: bigint = BigInt(roll); // convert the result for an i64/u64 column const bytes: Uint8Array = ctx.random.fill(new Uint8Array(16)); @@ -311,7 +313,7 @@ export const refreshCache = spacetimedb.procedure( ); ``` -Outbound responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. Perform network I/O before `withTx`; only database work belongs inside its callback. +TypeScript outbound HTTP uses `ctx.http.fetch(url, options)`, including for non-GET requests; it does not provide convenience methods such as `get()` or `post()`. Responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. Perform network I/O before `withTx`; only database work belongs inside its callback. Scheduled procedures use a normal scheduled table, but its `scheduled` option references a `spacetimedb.procedure(...)` export instead of a reducer. The procedure receives its scheduled row as an argument and uses `withTx` for database access: From b3f4bbc46746a4bafbe98fe367f2fa4dd9cebfbe Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:37:56 -0400 Subject: [PATCH 22/78] Grade only specified HTTP response headers --- .../procedures/t_076_http_handler/spec.rs | 2 +- .../procedures/t_077_http_router/spec.rs | 1 + .../t_078_idempotent_webhook/spec.rs | 1 + .../t_079_external_upload_flow/spec.rs | 1 + .../xtask-llm-benchmark/src/eval/defaults.rs | 2 ++ tools/xtask-llm-benchmark/src/eval/scorers.rs | 30 +++++++++++++++++-- 6 files changed, 34 insertions(+), 3 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs index 8e4596af499..f765e8d410a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs @@ -5,7 +5,7 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_http_route_parity_scorer( - host_url, file!(), route_tag, vec![("POST", "/echo", Some("hello"))], "http_handler_response", + host_url, file!(), route_tag, vec![("POST", "/echo", Some("hello"))], true, "http_handler_response", )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs index c6c67c098e3..3b4b74aff41 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs @@ -12,6 +12,7 @@ pub fn spec() -> BenchmarkSpec { ("PUT", "/items", Some("no")), ("GET", "/missing", None), ], + false, "router_method_matrix", )); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs index c0913cd8579..794697a54c2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs @@ -12,6 +12,7 @@ pub fn spec() -> BenchmarkSpec { ("POST", "/webhook", Some("evt-1|2|new")), ("POST", "/webhook", Some("evt-2|1|old")), ], + false, "webhook_idempotency", )); let table = table_name("webhook_state", lang); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs index f5f7dcab972..2fb34d7c179 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs @@ -18,6 +18,7 @@ pub fn spec() -> BenchmarkSpec { file!(), route_tag, vec![("POST", "/upload", Some("payload"))], + false, "upload_route", )); let table = table_name("uploaded_asset", lang); diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index b2ee77a3370..668dba07847 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -168,6 +168,7 @@ pub fn make_http_route_parity_scorer( src_file: &str, route_tag: &str, cases: Vec<(&str, &str, Option<&str>)>, + compare_content_type: bool, id_str: &'static str, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); @@ -178,6 +179,7 @@ pub fn make_http_route_parity_scorer( golden_db, llm_db, id_str, + compare_content_type, cases: cases .into_iter() .map(|(method, path, body)| HttpRouteCase { diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 53aacfb6d52..bf9168bd45a 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -860,6 +860,7 @@ pub struct HttpRouteParityScorer { pub golden_db: String, pub llm_db: String, pub cases: Vec, + pub compare_content_type: bool, pub id_str: &'static str, } @@ -894,6 +895,18 @@ fn call_http_route(server: &str, db: &str, case: &HttpRouteCase) -> Result<(u16, .map_err(|_| "HTTP route worker panicked".to_string())? } +fn http_route_results_equal( + golden_results: &[(u16, String, String)], + llm_results: &[(u16, String, String)], + compare_content_type: bool, +) -> bool { + golden_results.len() == llm_results.len() + && golden_results + .iter() + .zip(llm_results) + .all(|(golden, llm)| golden.0 == llm.0 && golden.2 == llm.2 && (!compare_content_type || golden.1 == llm.1)) +} + impl Scorer for HttpRouteParityScorer { fn id(&self) -> &'static str { self.id_str @@ -924,11 +937,15 @@ impl Scorer for HttpRouteParityScorer { } } } - let pass = golden_results == llm_results; + let pass = http_route_results_equal(&golden_results, &llm_results, self.compare_content_type); ScoreDetails { pass, partial: if pass { 1.0 } else { 0.0 }, - notes: json!({ "golden": golden_results, "llm": llm_results }), + notes: json!({ + "golden": golden_results, + "llm": llm_results, + "compared_content_type": self.compare_content_type, + }), } } } @@ -1076,4 +1093,13 @@ mod tests { assert!(!diff_sets(&golden_rls, &candidate_rls).is_null()); } + + #[test] + fn http_route_parity_ignores_unspecified_content_type() { + let golden = vec![(201, String::new(), "created".to_string())]; + let candidate = vec![(201, "text/plain".to_string(), "created".to_string())]; + + assert!(http_route_results_equal(&golden, &candidate, false)); + assert!(!http_route_results_equal(&golden, &candidate, true)); + } } From 4a084de3721e3e54c27b9536bb36a5b58ebf952e Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:46:11 -0400 Subject: [PATCH 23/78] Bound LLM benchmark scorer calls --- tools/xtask-llm-benchmark/src/eval/scorers.rs | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index bf9168bd45a..23b47e159fd 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -87,6 +87,7 @@ fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Resu } if start.elapsed() > timeout { let _ = child.kill(); + let _ = child.wait(); return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); } thread::sleep(Duration::from_millis(30)); @@ -401,24 +402,20 @@ pub fn call_reducer_json_out(db: &str, reducer: &str, args: &[Value], host: Opti eprintln!("[dbg] spacetime call: {:?}", cmd); } - let out = cmd - .output() - .map_err(|e| format!("failed to spawn spacetime call: {e}"))?; + let (code, stdout, stderr) = run_with_timeout(cmd, Path::new("."), Duration::from_secs(30)) + .map_err(|e| format!("spacetime call failed or timed out: {e}"))?; if debug_llm_verbose() { eprintln!( "[dbg] spacetime call exit={} stdout:\n{}\n-- stderr:\n{}\n", - out.status.code().unwrap_or(-1), - String::from_utf8_lossy(&out.stdout), - String::from_utf8_lossy(&out.stderr) + code, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) ); } - if !out.status.success() { - return Err(format!( - "spacetime call failed:\n{}", - String::from_utf8_lossy(&out.stderr) - )); + if code != 0 { + return Err(format!("spacetime call failed:\n{}", String::from_utf8_lossy(&stderr))); } - Ok(String::from_utf8_lossy(&out.stdout).to_string()) + Ok(String::from_utf8_lossy(&stdout).to_string()) } pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result { From 239ebbe2358783a4786374d020f2d7831ddc070b Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:54:12 -0400 Subject: [PATCH 24/78] Document advanced Rust and C# server APIs --- skills/csharp-server/SKILL.md | 69 +++++++++++++++++++++++++++ skills/rust-server/SKILL.md | 90 ++++++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index d88fd5ec0d2..074ac58b566 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -90,8 +90,11 @@ The complete set of column attributes: [AutoInc] // auto-increment (use 0 as placeholder on insert) [Unique] // unique constraint; indexes the column, enables .Find() [SpacetimeDB.Index.BTree] // btree index (enables .Filter() on this column) +[Default(true)] // migration-safe default for a newly appended field ``` +Defaults are for compatible schema upgrades. Preserve existing fields and reducers, append the defaulted field, and do not apply `[Default(...)]` to primary-key, unique, or auto-increment fields. + ## Indexes Write the index attribute fully qualified: `[SpacetimeDB.Index.BTree]`. Prefer inline for single-column; multi-column uses struct-level: @@ -173,6 +176,34 @@ public static Entity? MyProfile(ViewContext ctx) } ``` +Query-builder views use `ViewContext`, `ctx.From`, and return `IQuery` directly: + +```csharp +[SpacetimeDB.View(Accessor = "DiscountedProduct", Public = true)] +public static IQuery DiscountedProduct(ViewContext ctx) => + ctx.From.Product().Where(product => product.Discounted.Eq(true)); + +[SpacetimeDB.View(Accessor = "TaggedProduct", Public = true)] +public static IQuery TaggedProduct(ViewContext ctx) => + ctx.From.ProductTag().RightSemijoin( + ctx.From.Product(), + (tag, product) => tag.ProductId.Eq(product.Id) + ); +``` + +Declare a procedural view primary key in its attribute: `[SpacetimeDB.View(Accessor = "CatalogEntry", Public = true, PrimaryKey = nameof(CatalogRow.Sku))]`. + +Inclusive btree ranges use tuples, for example `ctx.Db.Shipment.DeliverBy.Filter((new Timestamp(1_000), new Timestamp(2_000)))`. + +## Client Visibility Filters + +```csharp +[ClientVisibilityFilter] +public static readonly Filter PrivateNoteFilter = new Filter.Sql( + "SELECT * FROM PrivateNote WHERE Author = :sender" +); +``` + ## Reducer Context API `ReducerContext` (`ctx`) is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. @@ -229,6 +260,44 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); ``` +Synthetic connection IDs can be constructed from 16 bytes with `ConnectionId.From(bytes)`; the result is nullable and must be checked. + +## Procedures and HTTP + +Procedures are unstable APIs, so modules using them should include `#pragma warning disable STDB_UNSTABLE`. They receive `ProcedureContext`, may return `[SpacetimeDB.Type]` values, perform outbound HTTP through `ctx.Http`, and open short transactions with `ctx.WithTx`: + +```csharp +[SpacetimeDB.Type] +public partial struct ProductResult { public uint Value; public string Description; } + +[SpacetimeDB.Procedure] +public static ProductResult Multiply(ProcedureContext ctx, uint lhs, uint rhs) => + new() { Value = lhs * rhs, Description = "product" }; + +[SpacetimeDB.Procedure] +public static void RefreshCache(ProcedureContext ctx, string url) +{ + ctx.Http.Get(url).Match(response => { + ctx.WithTx(tx => { tx.Db.CacheEntry.Insert(new CacheEntry { Key = url, Status = response.StatusCode }); return 0; }); + return 0; + }, error => throw new Exception(error.Message)); +} +``` + +Scheduled procedures use an ordinary scheduled table whose `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row. Database access inside the procedure goes through `ctx.WithTx`. + +Inbound HTTP uses handler attributes and one router. Handler database access also goes through `ctx.WithTx`: + +```csharp +[SpacetimeDB.HttpHandler] +public static HttpResponse Health(HandlerContext ctx, HttpRequest request) => new( + 200, HttpVersion.Http11, new(), HttpBody.FromString("ok") +); + +[SpacetimeDB.HttpRouter] +public static Router Routes() => SpacetimeDB.Router.New().Get("/health", Handlers.Health); +``` + ## Custom Types ```csharp diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index f83a4387b1c..49b618fdc54 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -17,8 +17,9 @@ metadata: ```rust use spacetimedb::{ - reducer, table, Identity, ReducerContext, SpacetimeType, Table, - ConnectionId, ScheduleAt, TimeDuration, Timestamp, Uuid, + procedure, reducer, table, Filter, Identity, ProcedureContext, Query, + ReducerContext, SpacetimeType, Table, ConnectionId, ScheduleAt, + TimeDuration, Timestamp, Uuid, }; ``` @@ -50,6 +51,7 @@ Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, |-----------|-------| | `u8` / `u16` / `u32` / `u64` / `u128` | unsigned integers | | `i8` / `i16` / `i32` / `i64` / `i128` | signed integers | +| `spacetimedb::sats::u256` / `spacetimedb::sats::i256` | 256-bit integers | | `f32` / `f64` | floats | | `bool` | boolean | | `String` | text | @@ -68,8 +70,11 @@ Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, #[auto_inc] // auto-increment (use 0 as placeholder on insert) #[unique] // unique constraint #[index(btree)] // btree index (enables .filter() on this column) +#[default(true)] // migration-safe default for a newly appended column ``` +Defaults are for compatible schema upgrades. Append the defaulted field, preserve all existing fields and reducers exactly, and do not place `#[default(...)]` on primary-key, unique, or auto-increment columns. + ## Indexes Prefer `#[index(btree)]` inline for single-column. Multi-column uses table-level: @@ -129,6 +134,8 @@ Note: `iter()` and `filter()` return iterators. Collect to Vec if you need `.sor Range queries on btree indexes: `filter(18..=65)`, `filter(18..)`, `filter(..18)`. +String index filters borrow the key: `ctx.db.product().category().filter(&"hardware".to_string())`. + ## Lifecycle Hooks ```rust @@ -162,6 +169,42 @@ fn my_profile(ctx: &ViewContext) -> Option { } ``` +Declare a procedural view primary key in the view attribute: + +```rust +#[view(accessor = catalog_entry, public, primary_key = sku)] +fn catalog_entry(ctx: &AnonymousViewContext) -> Vec { ... } +``` + +Query-builder views use `ViewContext`, `ctx.from`, and return `impl Query`: + +```rust +use spacetimedb::{view, Query, ViewContext}; + +#[view(accessor = discounted_product, public)] +fn discounted_product(ctx: &ViewContext) -> impl Query { + ctx.from.product().filter(|product| product.discounted.eq(true)) +} + +#[view(accessor = tagged_product, public)] +fn tagged_product(ctx: &ViewContext) -> impl Query { + ctx.from.product_tag().right_semijoin( + ctx.from.product(), + |tag, product| tag.product_id.eq(product.id), + ) +} +``` + +## Client Visibility Filters + +```rust +use spacetimedb::Filter; + +#[spacetimedb::client_visibility_filter] +const PRIVATE_NOTE_FILTER: Filter = + Filter::Sql("SELECT * FROM private_note WHERE author = :sender"); +``` + ## Reducer Context API `ReducerContext` (`ctx`) is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. @@ -212,6 +255,49 @@ let at = ScheduleAt::Interval(std::time::Duration::from_secs(5).into()); ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, scheduled_at: at }); ``` +Synthetic connection IDs for module logic/tests can use `ConnectionId::from_u128(1)`. + +## Procedures and HTTP + +Procedures use `&mut ProcedureContext`, may return typed values, perform outbound HTTP through `ctx.http`, and open short database transactions with `ctx.with_tx`: + +```rust +use spacetimedb::{procedure, ProcedureContext, SpacetimeType, Table}; + +#[derive(SpacetimeType)] +pub struct Product { pub value: u32, pub description: String } + +#[procedure] +pub fn multiply(_ctx: &mut ProcedureContext, lhs: u32, rhs: u32) -> Product { + Product { value: lhs * rhs, description: "product".into() } +} + +#[procedure] +pub fn refresh_cache(ctx: &mut ProcedureContext, url: String) { + let response = ctx.http.get(url).expect("request failed"); + let status = response.status().as_u16(); + ctx.with_tx(|tx| { + tx.db.cache_entry().insert(CacheEntry { key: url, status }); + }); +} +``` + +Scheduled procedures use a normal scheduled table, but the scheduled function is marked `#[procedure]` and receives `&mut ProcedureContext` plus the scheduled row. + +Inbound HTTP uses handler functions and one router: + +```rust +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[handler] +fn health(_ctx: &mut HandlerContext, _request: Request) -> Response { + Response::builder().status(200).body(Body::from_bytes("ok")).unwrap() +} + +#[router] +fn routes() -> Router { Router::new().get("/health", health) } +``` + ## Logging ```rust From fd97271cc7ecf21b6da1de3d07ddaae7ef1ee8bb Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:06:35 -0400 Subject: [PATCH 25/78] Refine TypeScript eval guidance and prompts --- skills/typescript-server/SKILL.md | 10 +++++++++- .../t_070_connection_scoped_presence/tasks/csharp.txt | 2 +- .../t_070_connection_scoped_presence/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../views/t_063_timestamp_window/tasks/csharp.txt | 2 +- .../views/t_063_timestamp_window/tasks/rust.txt | 2 +- .../views/t_063_timestamp_window/tasks/typescript.txt | 2 +- 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 5366163af8b..e268bf9aeb7 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -42,7 +42,7 @@ export const addRecord = spacetimedb.reducer( ## Imports -Schema builders and module exports come from `spacetimedb/server`. Runtime value classes such as `ScheduleAt`, `Timestamp`, `Range`, and `ConnectionId` come from the root `spacetimedb` package: +Schema builders and module exports come from `spacetimedb/server`. Runtime value classes such as `ScheduleAt`, `Timestamp`, and `ConnectionId` come from the root `spacetimedb` package; `Range` comes from `spacetimedb/server`: ```typescript import { schema, table, t } from 'spacetimedb/server'; @@ -96,6 +96,10 @@ Use `.default(value)` only for a newly appended migration-safe field. Preserve e Optional columns: `nickname: t.option(t.string())` +Schema builders describe the database's wire types; they are not TypeScript type names. For example, a `t.u16()` value is a TypeScript `number`, not a value cast to a type named `u16`. + +Treat requested table options and column modifiers as part of the schema contract. Do not add `event`, `autoInc`, `unique`, indexes, or defaults unless the module requirements call for them. + ## Indexes Prefer inline `.index('btree')` for single-column. Use named indexes only for multi-column: @@ -142,6 +146,8 @@ ctx.db.score_record.id.update({ ...existing, value: 2 }); // Update (spread + o ctx.db.score_record.id.delete(recordId); // Delete by PK ``` +Insert through the table accessor (`ctx.db.score_record.insert(...)`). Primary-key, unique, and index accessors support lookup or mutation of existing rows, but do not have `insert(...)`. + Note: `iter()` and `filter()` return iterators. Spread to Array for `.sort()`, `.filter()`, `.map()`. ## Lifecycle Hooks @@ -291,6 +297,8 @@ export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( ## Procedures and HTTP +`spacetimedb` is the local schema value returned by `schema({...})`; it is not a named export to import from `spacetimedb/server`. + Procedures declare argument and return types. They can perform outbound HTTP through `ctx.http` and open short transactions with `ctx.withTx`: ```typescript diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index 3b9d399c7d5..f069c100624 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public PresenceSession table keyed by ConnectionId: ConnectionId and containing indexed Identity: Identity plus ConnectedAt: Timestamp. ClientConnected must insert one row per connection, and ClientDisconnected must delete only that connection's row. Share that logic with ExercisePresence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. +Write a C# SpacetimeDB module with a public PresenceSession table keyed by ConnectionId: ConnectionId and containing indexed Identity: Identity plus ConnectedAt: Timestamp. Export the lifecycle reducers with the exact method names ClientConnected and ClientDisconnected. ClientConnected must insert one row per connection, and ClientDisconnected must delete only that connection's row. Share that logic with ExercisePresence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt index b93ae85af3d..e8788e68fcf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public presence_session table keyed by connection_id: ConnectionId and containing indexed identity: Identity plus connected_at: Timestamp. Client-connected must insert one row per connection, and client-disconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. +Write a Rust SpacetimeDB module with a public presence_session table keyed by connection_id: ConnectionId and containing indexed identity: Identity plus connected_at: Timestamp. Export the lifecycle reducers with the exact identifiers client_connected and client_disconnected. Client-connected must insert one row per connection, and client-disconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt index 2814d91ba92..c792905e139 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public presence_session table keyed by connectionId: ConnectionId and containing indexed identity: Identity plus connectedAt: Timestamp. clientConnected must insert one row per connection, and clientDisconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. +Write a TypeScript SpacetimeDB module with a public presence_session table keyed by connectionId: ConnectionId and containing indexed identity: Identity plus connectedAt: Timestamp. Export the lifecycle hooks with the exact identifiers clientConnected and clientDisconnected. clientConnected must insert one row per connection, and clientDisconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt index 22a53eab57c..633a39fd76e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -1 +1 @@ -Write a C# SpacetimeDB module with a public Event table containing Id: ulong primary key, indexed OccurredAt: Timestamp, and Label: string. Add Seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public WindowEvent procedural view that uses the OccurredAt index to return the inclusive range from 200 through 400 microseconds. +Write a C# SpacetimeDB module with a public table named Event containing Id: ulong primary key, indexed OccurredAt: Timestamp, and Label: string. Add Seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public WindowEvent procedural view that uses the OccurredAt index to return the inclusive range from 200 through 400 microseconds. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt index 3f089e547c2..ec65aa328e5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -1 +1 @@ -Write a Rust SpacetimeDB module with a public event table containing id: u64 primary key, indexed occurred_at: Timestamp, and label: String. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurred_at index to return the inclusive range from 200 through 400 microseconds. +Write a Rust SpacetimeDB module with a public table named event containing id: u64 primary key, indexed occurred_at: Timestamp, and label: String. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurred_at index to return the inclusive range from 200 through 400 microseconds. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt index 0f7f8d1035f..2f3ec91a5ad 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -1 +1 @@ -Write a TypeScript SpacetimeDB module with a public event table containing id: u64 primary key, indexed occurredAt: Timestamp, and label: string. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurredAt index to return the inclusive range from 200 through 400 microseconds. +Write a TypeScript SpacetimeDB module with a public table named event containing id: u64 primary key, indexed occurredAt: Timestamp, and label: string. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurredAt index to return the inclusive range from 200 through 400 microseconds. From 87ee8c8ab44e75e2157db770871c745e1c2dacfd Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:35:39 -0400 Subject: [PATCH 26/78] Align new eval prompts with benchmark format --- .../t_068_secure_projection/tasks/csharp.txt | 26 +++++++++++++++- .../t_068_secure_projection/tasks/rust.txt | 26 +++++++++++++++- .../tasks/typescript.txt | 26 +++++++++++++++- .../t_083_row_level_security/tasks/csharp.txt | 17 +++++++++- .../t_083_row_level_security/tasks/rust.txt | 17 +++++++++- .../tasks/typescript.txt | 17 +++++++++- .../tasks/csharp.txt | 24 +++++++++++++- .../tasks/rust.txt | 24 +++++++++++++- .../tasks/typescript.txt | 24 +++++++++++++- .../tasks/csharp.txt | 26 +++++++++++++++- .../tasks/rust.txt | 26 +++++++++++++++- .../tasks/typescript.txt | 26 +++++++++++++++- .../t_071_scheduled_private/tasks/csharp.txt | 21 ++++++++++++- .../t_071_scheduled_private/tasks/rust.txt | 21 ++++++++++++- .../tasks/typescript.txt | 21 ++++++++++++- .../tasks/csharp.txt | 18 ++++++++++- .../t_080_automatic_migration/tasks/rust.txt | 18 ++++++++++- .../tasks/typescript.txt | 18 ++++++++++- .../tasks/csharp.txt | 20 +++++++++++- .../tasks/rust.txt | 20 +++++++++++- .../tasks/typescript.txt | 20 +++++++++++- .../tasks/csharp.txt | 18 ++++++++++- .../tasks/rust.txt | 18 ++++++++++- .../tasks/typescript.txt | 18 ++++++++++- .../t_072_procedure_return/tasks/csharp.txt | 12 ++++++- .../t_072_procedure_return/tasks/rust.txt | 12 ++++++- .../tasks/typescript.txt | 12 ++++++- .../t_073_http_fetch/tasks/csharp.txt | 15 ++++++++- .../t_073_http_fetch/tasks/rust.txt | 15 ++++++++- .../t_073_http_fetch/tasks/typescript.txt | 15 ++++++++- .../t_074_fetch_and_store/tasks/csharp.txt | 15 ++++++++- .../t_074_fetch_and_store/tasks/rust.txt | 15 ++++++++- .../tasks/typescript.txt | 15 ++++++++- .../tasks/csharp.txt | 24 +++++++++++++- .../t_075_scheduled_procedure/tasks/rust.txt | 24 +++++++++++++- .../tasks/typescript.txt | 24 +++++++++++++- .../t_076_http_handler/tasks/csharp.txt | 12 ++++++- .../t_076_http_handler/tasks/rust.txt | 12 ++++++- .../t_076_http_handler/tasks/typescript.txt | 12 ++++++- .../t_077_http_router/tasks/csharp.txt | 16 +++++++++- .../t_077_http_router/tasks/rust.txt | 16 +++++++++- .../t_077_http_router/tasks/typescript.txt | 16 +++++++++- .../t_078_idempotent_webhook/tasks/csharp.txt | 23 +++++++++++++- .../t_078_idempotent_webhook/tasks/rust.txt | 23 +++++++++++++- .../tasks/typescript.txt | 23 +++++++++++++- .../tasks/csharp.txt | 24 +++++++++++++- .../t_079_external_upload_flow/tasks/rust.txt | 24 +++++++++++++- .../tasks/typescript.txt | 24 +++++++++++++- .../tasks/csharp.txt | 23 ++++++++++++-- .../tasks/rust.txt | 23 ++++++++++++-- .../tasks/typescript.txt | 23 ++++++++++++-- .../t_056_nested_update/tasks/csharp.txt | 21 +++++++++++-- .../t_056_nested_update/tasks/rust.txt | 21 +++++++++++-- .../t_056_nested_update/tasks/typescript.txt | 21 +++++++++++-- .../tasks/csharp.txt | 26 ++++++++++++++-- .../tasks/rust.txt | 26 ++++++++++++++-- .../tasks/typescript.txt | 26 ++++++++++++++-- .../t_058_batched_delete/tasks/csharp.txt | 20 ++++++++++-- .../t_058_batched_delete/tasks/rust.txt | 20 ++++++++++-- .../t_058_batched_delete/tasks/typescript.txt | 20 ++++++++++-- .../tasks/csharp.txt | 15 ++++++++- .../tasks/rust.txt | 15 ++++++++- .../tasks/typescript.txt | 15 ++++++++- .../tasks/csharp.txt | 15 ++++++++- .../t_060_reducer_result_table/tasks/rust.txt | 15 ++++++++- .../tasks/typescript.txt | 15 ++++++++- .../t_049_binary_storage/tasks/csharp.txt | 16 ++++++++-- .../t_049_binary_storage/tasks/rust.txt | 16 ++++++++-- .../t_049_binary_storage/tasks/typescript.txt | 16 ++++++++-- .../t_051_denormalized_index/tasks/csharp.txt | 20 ++++++++++-- .../t_051_denormalized_index/tasks/rust.txt | 20 ++++++++++-- .../tasks/typescript.txt | 20 ++++++++++-- .../t_052_autoinc_reference/tasks/csharp.txt | 17 ++++++++-- .../t_052_autoinc_reference/tasks/rust.txt | 17 ++++++++-- .../tasks/typescript.txt | 17 ++++++++-- .../t_053_default_values/tasks/csharp.txt | 17 +++++++++- .../t_053_default_values/tasks/rust.txt | 17 +++++++++- .../t_053_default_values/tasks/typescript.txt | 17 +++++++++- .../t_054_special_types/tasks/csharp.txt | 16 ++++++++-- .../tables/t_054_special_types/tasks/rust.txt | 17 ++++++++-- .../t_054_special_types/tasks/typescript.txt | 16 ++++++++-- .../t_061_three_table_join/tasks/csharp.txt | 27 +++++++++++++++- .../t_061_three_table_join/tasks/rust.txt | 27 +++++++++++++++- .../tasks/typescript.txt | 27 +++++++++++++++- .../tasks/csharp.txt | 22 ++++++++++++- .../tasks/rust.txt | 22 ++++++++++++- .../tasks/typescript.txt | 22 ++++++++++++- .../t_063_timestamp_window/tasks/csharp.txt | 18 ++++++++++- .../t_063_timestamp_window/tasks/rust.txt | 18 ++++++++++- .../tasks/typescript.txt | 18 ++++++++++- .../t_064_index_and_filter/tasks/csharp.txt | 20 +++++++++++- .../t_064_index_and_filter/tasks/rust.txt | 20 +++++++++++- .../tasks/typescript.txt | 20 +++++++++++- .../tasks/csharp.txt | 31 ++++++++++++++++++- .../tasks/rust.txt | 31 ++++++++++++++++++- .../tasks/typescript.txt | 31 ++++++++++++++++++- .../t_066_query_builder_view/tasks/csharp.txt | 18 ++++++++++- .../t_066_query_builder_view/tasks/rust.txt | 18 ++++++++++- .../tasks/typescript.txt | 18 ++++++++++- .../t_067_view_primary_key/tasks/csharp.txt | 14 ++++++++- .../t_067_view_primary_key/tasks/rust.txt | 14 ++++++++- .../tasks/typescript.txt | 14 ++++++++- 102 files changed, 1876 insertions(+), 126 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt index b061a674246..2282d2a24fc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt @@ -1 +1,25 @@ -Write a C# SpacetimeDB module with a private SecretNote table containing Id: ulong primary key, indexed Owner: Identity, Title: string, and SecretBody: string. Define a SafeNote product type containing only Id: ulong and Title: string. Add SeedPrivateNote() to insert Id 1 for the caller with Title "Visible title" and SecretBody "never expose this". Expose a public per-user MySafeNote view returning IEnumerable for only the caller, never Owner or SecretBody. +Write a SpacetimeDB backend module in C# that exposes a safe per-user projection of private rows. + +TYPES +- SafeNote + - Id: ulong + - Title: string + +TABLE +- SecretNote (private, accessor SecretNote) + - Fields: + - Id: ulong (primary key) + - Owner: Identity (btree index) + - Title: string + - SecretBody: string + +REDUCERS +- SeedPrivateNote() + - Insert Id 1 for the caller + - Set Title to "Visible title" + - Set SecretBody to "never expose this" + +VIEW +- MySafeNote (per-user, public) + - Return IEnumerable for only the caller + - Never expose Owner or SecretBody diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt index a0707312bff..34252c59cd8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt @@ -1 +1,25 @@ -Write a Rust SpacetimeDB module with a private secret_note table (struct SecretNote) containing id: u64 primary key, indexed owner: Identity, title: String, and secret_body: String. Define a SafeNote product type containing only id: u64 and title: String. Add seed_private_note() to insert id 1 for the caller with title "Visible title" and secret_body "never expose this". Expose a public per-user my_safe_note view returning Vec for only the caller, never owner or secret_body. +Write a SpacetimeDB backend module in Rust that exposes a safe per-user projection of private rows. + +TYPES +- SafeNote + - id: u64 + - title: String + +TABLE +- secret_note (private), struct SecretNote + - Fields: + - id: u64 (primary key) + - owner: Identity (btree index) + - title: String + - secret_body: String + +REDUCERS +- seed_private_note() + - Insert id 1 for the caller + - Set title to "Visible title" + - Set secret_body to "never expose this" + +VIEW +- my_safe_note (per-user, public) + - Return Vec for only the caller + - Never expose owner or secret_body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt index 075ccdb8884..ca31bd4cc7d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt @@ -1 +1,25 @@ -Write a TypeScript SpacetimeDB module with a private secret_note table containing id: u64 primary key, indexed owner: Identity, title: string, and secretBody: string. Define a SafeNote row type containing only id: u64 and title: string. Add seed_private_note() to insert id 1 for the caller with title "Visible title" and secretBody "never expose this". Expose a public per-user my_safe_note view returning an array of SafeNote for only the caller, never owner or secretBody. +Write a SpacetimeDB backend module in TypeScript that exposes a safe per-user projection of private rows. + +TYPES +- SafeNote + - id: u64 + - title: string + +TABLE +- secret_note (private) + - Fields: + - id: u64 (primary key) + - owner: Identity (btree index) + - title: string + - secretBody: string + +REDUCERS +- seed_private_note() + - Insert id 1 for the caller + - Set title to "Visible title" + - Set secretBody to "never expose this" + +VIEW +- my_safe_note (per-user, public) + - Return an array of SafeNote for only the caller + - Never expose owner or secretBody diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt index 5d630c2dab5..7bfa3c2e3d9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt @@ -1 +1,16 @@ -Write a C# SpacetimeDB module with a public UserRecord table containing Identity: Identity as its primary key and Name: string. Add a client visibility filter with SQL exactly `SELECT * FROM UserRecord WHERE Identity = :sender` so each caller can only see its own row. Add RegisterSelf(name), which inserts a UserRecord using ctx.Sender rather than accepting an identity argument. +Write a SpacetimeDB backend module in C# that applies row-level visibility to a public table. + +TABLE +- UserRecord (public, accessor UserRecord) + - Fields: + - Identity: Identity (primary key) + - Name: string + +CLIENT VISIBILITY FILTER +- SQL exactly: `SELECT * FROM UserRecord WHERE Identity = :sender` +- Each caller can see only its own row + +REDUCERS +- RegisterSelf(name: string) + - Insert a UserRecord using ctx.Sender + - Do not accept an identity argument diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt index e1315395dc7..cf8d6854729 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt @@ -1 +1,16 @@ -Write a Rust SpacetimeDB module with a public UserRecord table containing identity: Identity as its primary key and name: String. Add a client visibility filter with SQL exactly `SELECT * FROM user_record WHERE identity = :sender` so each caller can only see its own row. Add register_self(name), which inserts a UserRecord using ctx.sender() rather than accepting an identity argument. +Write a SpacetimeDB backend module in Rust that applies row-level visibility to a public table. + +TABLE +- user_record (public), struct UserRecord + - Fields: + - identity: Identity (primary key) + - name: String + +CLIENT VISIBILITY FILTER +- SQL exactly: `SELECT * FROM user_record WHERE identity = :sender` +- Each caller can see only its own row + +REDUCERS +- register_self(name: String) + - Insert a UserRecord using ctx.sender() + - Do not accept an identity argument diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt index e6ba69cb6a4..45ec9688a86 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt @@ -1 +1,16 @@ -Write a TypeScript SpacetimeDB module with a public user_record table containing identity: Identity as its primary key and name: string. Add a client visibility filter with SQL exactly `SELECT * FROM user_record WHERE identity = :sender` so each caller can only see its own row. Add register_self(name), which inserts a user_record using ctx.sender rather than accepting an identity argument. +Write a SpacetimeDB backend module in TypeScript that applies row-level visibility to a public table. + +TABLE +- user_record (public) + - Fields: + - identity: Identity (primary key) + - name: string + +CLIENT VISIBILITY FILTER +- SQL exactly: `SELECT * FROM user_record WHERE identity = :sender` +- Each caller can see only its own row + +REDUCERS +- register_self(name: string) + - Insert a user_record using ctx.sender + - Do not accept an identity argument diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt index 14910e3c80a..223d96fa9f3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt @@ -1 +1,23 @@ -Write a C# SpacetimeDB module with a public MaterializedState table containing Id: ulong primary key, Status: string, Version: ulong, and RefreshedAt: Timestamp. Define a private RefreshJob scheduled table for reducer RefreshMaterialized with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, and StateId: ulong in that order. Add StartRefresh() to write Id 1 with Status "pending" and Version 0, then schedule state 1 one millisecond later. RefreshMaterialized must update that row to Status "ready", Version 1, and the scheduled reducer timestamp without any further client call. +Write a SpacetimeDB backend module in C# that refreshes materialized state through a scheduled reducer. + +TABLES +- MaterializedState (public, accessor MaterializedState) + - Fields: + - Id: ulong (primary key) + - Status: string + - Version: ulong + - RefreshedAt: Timestamp +- RefreshJob (private, scheduled for RefreshMaterialized, accessor RefreshJob) + - Fields, in this order: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - StateId: ulong + +REDUCERS +- StartRefresh() + - Write Id 1 with Status "pending" and Version 0 + - Schedule state 1 for one millisecond later +- RefreshMaterialized(job: RefreshJob) (scheduled reducer) + - Update state 1 to Status "ready" and Version 1 + - Set RefreshedAt to the scheduled reducer timestamp + - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt index 4fcb7ff5818..da3e3c30f99 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt @@ -1 +1,23 @@ -Write a Rust SpacetimeDB module with a public materialized_state table (struct MaterializedState) containing id: u64 primary key, status: String, version: u64, and refreshed_at: Timestamp. Define a private refresh_job scheduled table (struct RefreshJob) for reducer refresh_materialized with scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, and state_id: u64 in that order. Add start_refresh() to write id 1 with status "pending" and version 0, then schedule state 1 one millisecond later. refresh_materialized must update that row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. +Write a SpacetimeDB backend module in Rust that refreshes materialized state through a scheduled reducer. + +TABLES +- materialized_state (public), struct MaterializedState + - Fields: + - id: u64 (primary key) + - status: String + - version: u64 + - refreshed_at: Timestamp +- refresh_job (private, scheduled for refresh_materialized), struct RefreshJob + - Fields, in this order: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - state_id: u64 + +REDUCERS +- start_refresh() + - Write id 1 with status "pending" and version 0 + - Schedule state 1 for one millisecond later +- refresh_materialized(job: RefreshJob) (scheduled reducer) + - Update state 1 to status "ready" and version 1 + - Set refreshed_at to the scheduled reducer timestamp + - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt index 483cbe447b0..fc5ae4f85a0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt @@ -1 +1,23 @@ -Write a TypeScript SpacetimeDB module with a public materialized_state table containing id: u64 primary key, status: string, version: u64, and refreshedAt: Timestamp. Define a private refresh_job scheduled table for reducer refreshMaterialized with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, and stateId: u64 in that order. Add start_refresh() to write id 1 with status "pending" and version 0, then schedule state 1 one millisecond later. refreshMaterialized must update that row to status "ready", version 1, and the scheduled reducer timestamp without any further client call. +Write a SpacetimeDB backend module in TypeScript that refreshes materialized state through a scheduled reducer. + +TABLES +- materialized_state (public) + - Fields: + - id: u64 (primary key) + - status: string + - version: u64 + - refreshedAt: Timestamp +- refresh_job (private, scheduled for refreshMaterialized) + - Fields, in this order: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - stateId: u64 + +REDUCERS +- start_refresh() + - Write id 1 with status "pending" and version 0 + - Schedule state 1 for one millisecond later +- refreshMaterialized(job: refresh_job row) (scheduled reducer) + - Update state 1 to status "ready" and version 1 + - Set refreshedAt to the scheduled reducer timestamp + - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index f069c100624..5db7f8420ef 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -1 +1,25 @@ -Write a C# SpacetimeDB module with a public PresenceSession table keyed by ConnectionId: ConnectionId and containing indexed Identity: Identity plus ConnectedAt: Timestamp. Export the lifecycle reducers with the exact method names ClientConnected and ClientDisconnected. ClientConnected must insert one row per connection, and ClientDisconnected must delete only that connection's row. Share that logic with ExercisePresence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. +Write a SpacetimeDB backend module in C# that tracks presence separately for each connection. + +TABLE +- PresenceSession (public, accessor PresenceSession) + - Fields: + - ConnectionId: ConnectionId (primary key) + - Identity: Identity (btree index) + - ConnectedAt: Timestamp + +HELPERS +- Share the insert and delete logic between the lifecycle reducers and ExercisePresence + +LIFECYCLE HOOKS +- ClientConnected + - Export with this exact method name + - Insert one row for the connected connection +- ClientDisconnected + - Export with this exact method name + - Delete only the disconnected connection's row + +REDUCERS +- ExercisePresence() + - Insert two synthetic connections for the caller + - Remove only the first + - Leave the second synthetic session intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt index e8788e68fcf..01af65378e9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -1 +1,25 @@ -Write a Rust SpacetimeDB module with a public presence_session table keyed by connection_id: ConnectionId and containing indexed identity: Identity plus connected_at: Timestamp. Export the lifecycle reducers with the exact identifiers client_connected and client_disconnected. Client-connected must insert one row per connection, and client-disconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. +Write a SpacetimeDB backend module in Rust that tracks presence separately for each connection. + +TABLE +- presence_session (public), struct PresenceSession + - Fields: + - connection_id: ConnectionId (primary key) + - identity: Identity (btree index) + - connected_at: Timestamp + +HELPERS +- Share the insert and delete logic between the lifecycle reducers and exercise_presence + +LIFECYCLE HOOKS +- client_connected + - Export with this exact identifier + - Insert one row for the connected connection +- client_disconnected + - Export with this exact identifier + - Delete only the disconnected connection's row + +REDUCERS +- exercise_presence() + - Insert two synthetic connections for the caller + - Remove only the first + - Leave the second synthetic session intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt index c792905e139..9090ce63274 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -1 +1,25 @@ -Write a TypeScript SpacetimeDB module with a public presence_session table keyed by connectionId: ConnectionId and containing indexed identity: Identity plus connectedAt: Timestamp. Export the lifecycle hooks with the exact identifiers clientConnected and clientDisconnected. clientConnected must insert one row per connection, and clientDisconnected must delete only that connection's row. Share that logic with exercise_presence(), which inserts two synthetic connections for the caller and removes only the first, leaving the second synthetic session intact. +Write a SpacetimeDB backend module in TypeScript that tracks presence separately for each connection. + +TABLE +- presence_session (public) + - Fields: + - connectionId: ConnectionId (primary key) + - identity: Identity (btree index) + - connectedAt: Timestamp + +HELPERS +- Share the insert and delete logic between the lifecycle hooks and exercise_presence + +LIFECYCLE HOOKS +- clientConnected + - Export with this exact identifier + - Insert one row for the connected connection +- clientDisconnected + - Export with this exact identifier + - Delete only the disconnected connection's row + +REDUCERS +- exercise_presence() + - Insert two synthetic connections for the caller + - Remove only the first + - Leave the second synthetic session intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt index 044e54ebc11..6f9254ae94e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt @@ -1 +1,20 @@ -Write a C# SpacetimeDB module with a public JobResult table containing Id: ulong primary key and Status: string. Define a private PrivateJob scheduled table for reducer RunPrivateJob with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, and ResultId: ulong in that order. Add EnqueuePrivate(Id) to write Status "queued" and schedule that Id one millisecond later. RunPrivateJob must be scheduler-only, not client-callable, and update the result to "complete". +Write a SpacetimeDB backend module in C# with a private scheduled table that is not client-callable. + +TABLES +- JobResult (public, accessor JobResult) + - Fields: + - Id: ulong (primary key) + - Status: string +- PrivateJob (private, scheduled for RunPrivateJob, accessor PrivateJob) + - Fields, in this order: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - ResultId: ulong + +REDUCERS +- EnqueuePrivate(id: ulong) + - Write JobResult Status "queued" + - Schedule that Id for one millisecond later +- RunPrivateJob(job: PrivateJob) (scheduled reducer) + - Must be scheduler-only and not client-callable + - Update the matching JobResult Status to "complete" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt index f9b657a4860..0e1f732ecfc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt @@ -1 +1,20 @@ -Write a Rust SpacetimeDB module with a public job_result table (struct JobResult) containing id: u64 primary key and status: String. Define a private private_job scheduled table (struct PrivateJob) for reducer run_private_job with scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, and result_id: u64 in that order. Add enqueue_private(id) to write status "queued" and schedule that id one millisecond later. run_private_job must be scheduler-only, not client-callable, and update the result to "complete". +Write a SpacetimeDB backend module in Rust with a private scheduled table that is not client-callable. + +TABLES +- job_result (public), struct JobResult + - Fields: + - id: u64 (primary key) + - status: String +- private_job (private, scheduled for run_private_job), struct PrivateJob + - Fields, in this order: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - result_id: u64 + +REDUCERS +- enqueue_private(id: u64) + - Write job_result status "queued" + - Schedule that id for one millisecond later +- run_private_job(job: PrivateJob) (scheduled reducer) + - Must be scheduler-only and not client-callable + - Update the matching job_result status to "complete" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt index f8c6cf379a1..0dc1884a710 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt @@ -1 +1,20 @@ -Write a TypeScript SpacetimeDB module with a public job_result table containing id: u64 primary key and status: string. Define a private private_job scheduled table for reducer runPrivateJob with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, and resultId: u64 in that order. Add enqueue_private(id) to write status "queued" and schedule that id one millisecond later. runPrivateJob must be scheduler-only, not client-callable, and update the result to "complete". +Write a SpacetimeDB backend module in TypeScript with a private scheduled table that is not client-callable. + +TABLES +- job_result (public) + - Fields: + - id: u64 (primary key) + - status: string +- private_job (private, scheduled for runPrivateJob) + - Fields, in this order: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - resultId: u64 + +REDUCERS +- enqueue_private(id: u64) + - Write job_result status "queued" + - Schedule that id for one millisecond later +- runPrivateJob(job: private_job row) (scheduled reducer) + - Must be scheduler-only and not client-callable + - Update the matching job_result status to "complete" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt index 6e15528e3cf..f2f2d6fe3c7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt @@ -1 +1,17 @@ -Update the C# SpacetimeDB module with a compatible schema change. Preserve the public Product table and its fields, plus the Seed and Touch reducers. Add a public Category table with Id: ulong as its primary key and Label: string. Add CreateCategory(id, label) to insert a Category. The populated Product table must survive a normal republish without deleting data. +Update the existing SpacetimeDB backend module in C# with a compatible schema change. + +TABLES +- Product (public, accessor Product) + - Preserve the existing table and its fields +- Category (public, accessor Category) + - Fields: + - Id: ulong (primary key) + - Label: string + +REDUCERS +- Preserve Seed and Touch +- CreateCategory(id: ulong, label: string) + - Insert a Category with the supplied values + +MIGRATION +- The populated Product table must survive a normal republish without deleting data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt index 082deb4a1cf..92134a461db 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt @@ -1 +1,17 @@ -Update the Rust SpacetimeDB module with a compatible schema change. Preserve the public Product table and its fields, plus the seed and touch reducers. Add a public Category table with id: u64 as its primary key and label: String. Add create_category(id, label) to insert a Category. The populated Product table must survive a normal republish without deleting data. +Update the existing SpacetimeDB backend module in Rust with a compatible schema change. + +TABLES +- product (public), struct Product + - Preserve the existing table and its fields +- category (public), struct Category + - Fields: + - id: u64 (primary key) + - label: String + +REDUCERS +- Preserve seed and touch +- create_category(id: u64, label: String) + - Insert a Category with the supplied values + +MIGRATION +- The populated product table must survive a normal republish without deleting data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt index 068469276ea..b9e084866f4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt @@ -1 +1,17 @@ -Update the TypeScript SpacetimeDB module with a compatible schema change. Preserve the public product table and its fields, plus the seed and touch reducers. Add a public category table with id: u64 as its primary key and label: string. Add create_category(id, label) to insert a category. The populated product table must survive a normal republish without deleting data. +Update the existing SpacetimeDB backend module in TypeScript with a compatible schema change. + +TABLES +- product (public) + - Preserve the existing table and its fields +- category (public) + - Fields: + - id: u64 (primary key) + - label: string + +REDUCERS +- Preserve seed and touch +- create_category(id: u64, label: string) + - Insert a category with the supplied values + +MIGRATION +- The populated product table must survive a normal republish without deleting data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt index d574884d1c5..78d33872c6d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt @@ -1 +1,19 @@ -Update the C# SpacetimeDB module for an incremental migration while preserving the public LegacyItem table and Seed reducer. Add a public ItemV2 table with Id: ulong primary key, Value: string, and Version: uint. Add an idempotent Migrate reducer that copies missing legacy rows into ItemV2 with Version 2. Add DualWrite(id, value) that writes the same new item to both tables with Version 2 in ItemV2. +Update the existing SpacetimeDB backend module in C# with an incremental migration. + +TABLES +- LegacyItem (public, accessor LegacyItem) + - Preserve the existing table and fields +- ItemV2 (public, accessor ItemV2) + - Fields: + - Id: ulong (primary key) + - Value: string + - Version: uint + +REDUCERS +- Preserve Seed +- Migrate() + - Be idempotent + - Copy missing LegacyItem rows into ItemV2 with Version 2 +- DualWrite(id: ulong, value: string) + - Write the same new item to both tables + - Set Version 2 in ItemV2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt index 5729bc4fee2..11bfd78b357 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt @@ -1 +1,19 @@ -Update the Rust SpacetimeDB module for an incremental migration while preserving the public LegacyItem table and seed reducer. Add a public ItemV2 table with id: u64 primary key, value: String, and version: u32. Add an idempotent migrate reducer that copies missing legacy rows into ItemV2 with version 2. Add dual_write(id, value) that writes the same new item to both tables with version 2 in ItemV2. +Update the existing SpacetimeDB backend module in Rust with an incremental migration. + +TABLES +- legacy_item (public), struct LegacyItem + - Preserve the existing table and fields +- item_v2 (public), struct ItemV2 + - Fields: + - id: u64 (primary key) + - value: String + - version: u32 + +REDUCERS +- Preserve seed +- migrate() + - Be idempotent + - Copy missing legacy_item rows into item_v2 with version 2 +- dual_write(id: u64, value: String) + - Write the same new item to both tables + - Set version 2 in item_v2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt index 4c1bd596a42..e13eacad628 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt @@ -1 +1,19 @@ -Update the TypeScript SpacetimeDB module for an incremental migration while preserving the public legacy_item table and seed reducer. Add a public item_v2 table with id: u64 primary key, value: string, and version: u32. Add an idempotent migrate reducer that copies missing legacy rows into item_v2 with version 2. Add dual_write(id, value) that writes the same new item to both tables with version 2 in item_v2. +Update the existing SpacetimeDB backend module in TypeScript with an incremental migration. + +TABLES +- legacy_item (public) + - Preserve the existing table and fields +- item_v2 (public) + - Fields: + - id: u64 (primary key) + - value: string + - version: u32 + +REDUCERS +- Preserve seed +- migrate() + - Be idempotent + - Copy missing legacy_item rows into item_v2 with version 2 +- dual_write(id: u64, value: string) + - Write the same new item to both tables + - Set version 2 in item_v2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt index 5dba097dd4e..b77b572b46f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt @@ -1 +1,17 @@ -Update the C# SpacetimeDB module through a compatible republish while preserving the public Counter table, its exact fields, and the Seed and Increment reducers. Add a private Release table with Version: uint as its primary key and add RecordRelease(version). Do not rename or change the signature of Increment. Existing counter data and the existing server API must remain usable after republish. +Update the existing SpacetimeDB backend module in C# through a compatible republish. + +TABLES +- Counter (public, accessor Counter) + - Preserve the existing table and its exact fields +- Release (private, accessor Release) + - Fields: + - Version: uint (primary key) + +REDUCERS +- Preserve Seed +- Preserve Increment without renaming it or changing its signature +- RecordRelease(version: uint) + +MIGRATION +- Existing Counter data must remain usable after republish +- The existing server API must remain usable after republish diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt index 841d8319498..e4c7cf786ac 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt @@ -1 +1,17 @@ -Update the Rust SpacetimeDB module through a compatible republish while preserving the public Counter table, its exact fields, and the seed and increment reducers. Add a private Release table with version: u32 as its primary key and add record_release(version). Do not rename or change the signature of increment. Existing counter data and the existing server API must remain usable after republish. +Update the existing SpacetimeDB backend module in Rust through a compatible republish. + +TABLES +- counter (public), struct Counter + - Preserve the existing table and its exact fields +- release (private), struct Release + - Fields: + - version: u32 (primary key) + +REDUCERS +- Preserve seed +- Preserve increment without renaming it or changing its signature +- record_release(version: u32) + +MIGRATION +- Existing counter data must remain usable after republish +- The existing server API must remain usable after republish diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt index a2bcda5f49f..2a1c3acebd9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt @@ -1 +1,17 @@ -Update the TypeScript SpacetimeDB module through a compatible republish while preserving the public counter table, its exact fields, and the seed and increment reducers. Add a private release table with version: u32 as its primary key and add record_release(version). Do not rename or change the signature of increment. Existing counter data and the existing server API must remain usable after republish. +Update the existing SpacetimeDB backend module in TypeScript through a compatible republish. + +TABLES +- counter (public) + - Preserve the existing table and its exact fields +- release (private) + - Fields: + - version: u32 (primary key) + +REDUCERS +- Preserve seed +- Preserve increment without renaming it or changing its signature +- record_release(version: u32) + +MIGRATION +- Existing counter data must remain usable after republish +- The existing server API must remain usable after republish diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt index 9452907e348..cdbb236e4e9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt @@ -1 +1,11 @@ -Write a C# SpacetimeDB module with a typed Summary result containing Total: uint and Label: string. Add a CalculateSummary(lhs: uint, rhs: uint) procedure that returns Total lhs + rhs and Label "calculated". +Write a SpacetimeDB backend module in C# that defines a procedure with a typed return value. + +TYPES +- Summary + - Total: uint + - Label: string + +PROCEDURE +- CalculateSummary(lhs: uint, rhs: uint) -> Summary + - Return Total as lhs + rhs + - Return Label as "calculated" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt index a94a4d672c5..f2df9647087 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt @@ -1 +1,11 @@ -Write a Rust SpacetimeDB module with a typed Summary result containing total: u32 and label: String. Add a calculate_summary(lhs: u32, rhs: u32) procedure that returns total lhs + rhs and label "calculated". +Write a SpacetimeDB backend module in Rust that defines a procedure with a typed return value. + +TYPES +- Summary + - total: u32 + - label: String + +PROCEDURE +- calculate_summary(lhs: u32, rhs: u32) -> Summary + - Return total as lhs + rhs + - Return label as "calculated" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt index 07438a3503b..48ee3ce58f6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt @@ -1 +1,11 @@ -Write a TypeScript SpacetimeDB module with a typed Summary result containing total: u32 and label: string. Add a calculate_summary(lhs: u32, rhs: u32) procedure that returns total lhs + rhs and label "calculated". +Write a SpacetimeDB backend module in TypeScript that defines a procedure with a typed return value. + +TYPES +- Summary + - total: u32 + - label: string + +PROCEDURE +- calculate_summary(lhs: u32, rhs: u32) -> Summary + - Return total as lhs + rhs + - Return label as "calculated" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt index fb9aec69b98..128c436ca71 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt @@ -1 +1,14 @@ -Write a C# SpacetimeDB procedure FetchPageSummary(url: string) that performs an outbound GET to the supplied public URL. Return a FetchSummary product type with Status: ushort, HtmlContentType: bool indicating whether Content-Type contains "text/html", and HasExampleDomain: bool indicating whether the body contains "Example Domain". +Write a SpacetimeDB backend module in C# that performs outbound HTTP in a procedure. + +TYPES +- FetchSummary + - Status: ushort + - HtmlContentType: bool + - HasExampleDomain: bool + +PROCEDURE +- FetchPageSummary(url: string) -> FetchSummary + - Perform an outbound GET to the supplied public URL + - Set Status from the response status + - Set HtmlContentType based on whether Content-Type contains "text/html" + - Set HasExampleDomain based on whether the body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt index 9b49b7f3c8f..059a78203ab 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt @@ -1 +1,14 @@ -Write a Rust SpacetimeDB procedure fetch_page_summary(url: String) that performs an outbound GET to the supplied public URL. Return a FetchSummary product type with status: u16, html_content_type: bool indicating whether Content-Type contains "text/html", and has_example_domain: bool indicating whether the body contains "Example Domain". +Write a SpacetimeDB backend module in Rust that performs outbound HTTP in a procedure. + +TYPES +- FetchSummary + - status: u16 + - html_content_type: bool + - has_example_domain: bool + +PROCEDURE +- fetch_page_summary(url: String) -> FetchSummary + - Perform an outbound GET to the supplied public URL + - Set status from the response status + - Set html_content_type based on whether Content-Type contains "text/html" + - Set has_example_domain based on whether the body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt index be65d246ef9..7d0e9808a5f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt @@ -1 +1,14 @@ -Write a TypeScript SpacetimeDB procedure fetch_page_summary(url: string) that performs an outbound GET to the supplied public URL. Return a FetchSummary object type with status: u16, htmlContentType: bool indicating whether Content-Type contains "text/html", and hasExampleDomain: bool indicating whether the body contains "Example Domain". +Write a SpacetimeDB backend module in TypeScript that performs outbound HTTP in a procedure. + +TYPES +- FetchSummary + - status: u16 + - htmlContentType: bool + - hasExampleDomain: bool + +PROCEDURE +- fetch_page_summary(url: string) -> FetchSummary + - Perform an outbound GET to the supplied public URL + - Set status from the response status + - Set htmlContentType based on whether Content-Type contains "text/html" + - Set hasExampleDomain based on whether the body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt index 83b5ec2994a..2e042f4e476 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -1 +1,14 @@ -Write a C# SpacetimeDB module with a public FetchedRecord table containing Id: ulong primary key, Status: ushort, and ValidBody: bool. Add FetchAndStore(url: string), which performs an outbound GET to the supplied public URL before opening a transaction, then stores Id 1, the response status, and whether the body contains "Example Domain" inside a short transaction. +Write a SpacetimeDB backend module in C# that fetches external data before storing a result in a transaction. + +TABLE +- FetchedRecord (public, accessor FetchedRecord) + - Fields: + - Id: ulong (primary key) + - Status: ushort + - ValidBody: bool + +PROCEDURE +- FetchAndStore(url: string) + - Perform an outbound GET to the supplied public URL before opening a transaction + - Inside a short transaction, store Id 1 and the response status + - Set ValidBody based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt index 8ad28c12131..5b42d9d9811 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -1 +1,14 @@ -Write a Rust SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and valid_body: bool. Add fetch_and_store(url: String), which performs an outbound GET to the supplied public URL before opening a transaction, then stores id 1, the response status, and whether the body contains "Example Domain" inside a short transaction. +Write a SpacetimeDB backend module in Rust that fetches external data before storing a result in a transaction. + +TABLE +- fetched_record (public), struct FetchedRecord + - Fields: + - id: u64 (primary key) + - status: u16 + - valid_body: bool + +PROCEDURE +- fetch_and_store(url: String) + - Perform an outbound GET to the supplied public URL before opening a transaction + - Inside a short transaction, store id 1 and the response status + - Set valid_body based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt index 2c49c4f7ad8..6ae5f60bfc8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -1 +1,14 @@ -Write a TypeScript SpacetimeDB module with a public fetched_record table containing id: u64 primary key, status: u16, and validBody: bool. Add fetch_and_store(url: string), which performs an outbound GET to the supplied public URL before opening a transaction, then stores id 1, the response status, and whether the body contains "Example Domain" inside a short transaction. +Write a SpacetimeDB backend module in TypeScript that fetches external data before storing a result in a transaction. + +TABLE +- fetched_record (public) + - Fields: + - id: u64 (primary key) + - status: u16 + - validBody: bool + +PROCEDURE +- fetch_and_store(url: string) + - Perform an outbound GET to the supplied public URL before opening a transaction + - Inside a short transaction, store id 1 and the response status + - Set validBody based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt index 087915c44fd..31d717818ff 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt @@ -1 +1,23 @@ -Write a C# SpacetimeDB module with a public ProcedureResult table containing Id: ulong primary key and Value: uint. Define a private ProcedureJob scheduled table for procedure RunScheduledProcedure with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, Id: ulong, Lhs: uint, and Rhs: uint in that order. Add ScheduleProcedure(id, lhs, rhs) to enqueue that procedure for one millisecond later. RunScheduledProcedure must open a transaction and insert Value lhs + rhs into ProcedureResult. +Write a SpacetimeDB backend module in C# that schedules a procedure which writes inside a transaction. + +TABLES +- ProcedureResult (public, accessor ProcedureResult) + - Fields: + - Id: ulong (primary key) + - Value: uint +- ProcedureJob (private, scheduled for RunScheduledProcedure, accessor ProcedureJob) + - Fields, in this order: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - Id: ulong + - Lhs: uint + - Rhs: uint + +REDUCERS +- ScheduleProcedure(id: ulong, lhs: uint, rhs: uint) + - Enqueue RunScheduledProcedure for one millisecond later + +PROCEDURE +- RunScheduledProcedure(job: ProcedureJob) -> void (scheduled procedure) + - Open a transaction + - Insert Id and Value lhs + rhs into ProcedureResult diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt index 9d569667134..f6d9ad78d8a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt @@ -1 +1,23 @@ -Write a Rust SpacetimeDB module with a public procedure_result table (struct ProcedureResult) containing id: u64 primary key and value: u32. Define a private procedure_job scheduled table (struct ProcedureJob) for procedure run_scheduled_procedure with scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, id: u64, lhs: u32, and rhs: u32 in that order. Add schedule_procedure(id, lhs, rhs) to enqueue that procedure for one millisecond later. run_scheduled_procedure must open a transaction and insert value lhs + rhs into procedure_result. +Write a SpacetimeDB backend module in Rust that schedules a procedure which writes inside a transaction. + +TABLES +- procedure_result (public), struct ProcedureResult + - Fields: + - id: u64 (primary key) + - value: u32 +- procedure_job (private, scheduled for run_scheduled_procedure), struct ProcedureJob + - Fields, in this order: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - id: u64 + - lhs: u32 + - rhs: u32 + +REDUCERS +- schedule_procedure(id: u64, lhs: u32, rhs: u32) + - Enqueue run_scheduled_procedure for one millisecond later + +PROCEDURE +- run_scheduled_procedure(job: ProcedureJob) -> () (scheduled procedure) + - Open a transaction + - Insert id and value lhs + rhs into procedure_result diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt index 6cf21b8c579..97cf9738c3d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt @@ -1 +1,23 @@ -Write a TypeScript SpacetimeDB module with a public procedure_result table containing id: u64 primary key and value: u32. Define a private procedure_job scheduled table for procedure runScheduledProcedure with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, id: u64, lhs: u32, and rhs: u32 in that order. Add schedule_procedure(id, lhs, rhs) to enqueue that procedure for one millisecond later. runScheduledProcedure must open a transaction and insert value lhs + rhs into procedure_result. +Write a SpacetimeDB backend module in TypeScript that schedules a procedure which writes inside a transaction. + +TABLES +- procedure_result (public) + - Fields: + - id: u64 (primary key) + - value: u32 +- procedure_job (private, scheduled for runScheduledProcedure) + - Fields, in this order: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - id: u64 + - lhs: u32 + - rhs: u32 + +REDUCERS +- schedule_procedure(id: u64, lhs: u32, rhs: u32) + - Enqueue runScheduledProcedure for one millisecond later + +PROCEDURE +- runScheduledProcedure(job: procedure_job row) -> unit (scheduled procedure) + - Open a transaction + - Insert id and value lhs + rhs into procedure_result diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt index 54974663f7b..519ad043afc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt @@ -1 +1,11 @@ -Write a C# SpacetimeDB module with an inbound POST /echo HTTP handler. It must read the request body and return status 201, Content-Type text/plain, and body "echo:" followed by the input body. Register the handler through an HTTP router. +Write a SpacetimeDB backend module in C# that defines and routes an inbound HTTP handler. + +HANDLER +- POST /echo + - Read the request body + - Return status 201 + - Return Content-Type text/plain + - Return body "echo:" followed by the input body + +ROUTER +- Register the POST /echo handler through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt index 0418b842212..628b58a290b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt @@ -1 +1,11 @@ -Write a Rust SpacetimeDB module with an inbound POST /echo HTTP handler. It must read the request body and return status 201, Content-Type text/plain, and body "echo:" followed by the input body. Register the handler through an HTTP router. +Write a SpacetimeDB backend module in Rust that defines and routes an inbound HTTP handler. + +HANDLER +- POST /echo + - Read the request body + - Return status 201 + - Return Content-Type text/plain + - Return body "echo:" followed by the input body + +ROUTER +- Register the POST /echo handler through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt index 08a77d4a8e1..9595ae843f7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt @@ -1 +1,11 @@ -Write a TypeScript SpacetimeDB module with an inbound POST /echo HTTP handler. It must read the request body and return status 201, Content-Type text/plain, and body "echo:" followed by the input body. Register the handler through an HTTP router. +Write a SpacetimeDB backend module in TypeScript that defines and routes an inbound HTTP handler. + +HANDLER +- POST /echo + - Read the request body + - Return status 201 + - Return Content-Type text/plain + - Return body "echo:" followed by the input body + +ROUTER +- Register the POST /echo handler through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt index 9b964a87e8e..e1dcc494794 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt @@ -1 +1,15 @@ -Write a C# SpacetimeDB HTTP router with GET /items returning status 200 and body "list", and POST /items returning status 201 and body "created:" followed by the request body. Unsupported methods on /items and unknown paths must remain rejected by the router. +Write a SpacetimeDB backend module in C# that routes two HTTP methods on the same path. + +HANDLERS +- GET /items + - Return status 200 + - Return body "list" +- POST /items + - Read the request body + - Return status 201 + - Return body "created:" followed by the request body + +ROUTER +- Register both handlers for /items +- Reject unsupported methods on /items +- Reject unknown paths diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt index ff0f9920ca7..ed040b322f9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt @@ -1 +1,15 @@ -Write a Rust SpacetimeDB HTTP router with GET /items returning status 200 and body "list", and POST /items returning status 201 and body "created:" followed by the request body. Unsupported methods on /items and unknown paths must remain rejected by the router. +Write a SpacetimeDB backend module in Rust that routes two HTTP methods on the same path. + +HANDLERS +- GET /items + - Return status 200 + - Return body "list" +- POST /items + - Read the request body + - Return status 201 + - Return body "created:" followed by the request body + +ROUTER +- Register both handlers for /items +- Reject unsupported methods on /items +- Reject unknown paths diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt index 5fddf222270..cfa7421bee6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt @@ -1 +1,15 @@ -Write a TypeScript SpacetimeDB HTTP router with GET /items returning status 200 and body "list", and POST /items returning status 201 and body "created:" followed by the request body. Unsupported methods on /items and unknown paths must remain rejected by the router. +Write a SpacetimeDB backend module in TypeScript that routes two HTTP methods on the same path. + +HANDLERS +- GET /items + - Return status 200 + - Return body "list" +- POST /items + - Read the request body + - Return status 201 + - Return body "created:" followed by the request body + +ROUTER +- Register both handlers for /items +- Reject unsupported methods on /items +- Reject unknown paths diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt index 3b453e8b631..604a8d3eb13 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt @@ -1 +1,22 @@ -Write a C# SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Define a private ProcessedEvent table with EventId: string primary key. Define a public WebhookState table with Key: string primary key, LastSequence: ulong, and Value: string. Store state under Key "account". A newly applied event must return status 200 and body "applied". Duplicate event IDs must return status 200 and body "duplicate" without applying twice. Events whose sequence is not newer must return status 200 and body "stale" without replacing state. Register POST /webhook through an HTTP router. +Write a SpacetimeDB backend module in C# that handles idempotent, ordered webhook events. + +TABLES +- ProcessedEvent (private, accessor ProcessedEvent) + - Fields: + - EventId: string (primary key) +- WebhookState (public, accessor WebhookState) + - Fields: + - Key: string (primary key) + - LastSequence: ulong + - Value: string + +HANDLER +- POST /webhook + - Accept a pipe-delimited body: event_id|sequence|value + - Store state under Key "account" + - For a newly applied event, return status 200 and body "applied" + - For a duplicate EventId, return status 200 and body "duplicate" without applying twice + - When sequence is not newer, return status 200 and body "stale" without replacing state + +ROUTER +- Register POST /webhook through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt index c411d7879af..ece1fe91867 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt @@ -1 +1,22 @@ -Write a Rust SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Define a private processed_event table (struct ProcessedEvent) with event_id: String primary key. Define a public webhook_state table (struct WebhookState) with key: String primary key, last_sequence: u64, and value: String. Store state under key "account". A newly applied event must return status 200 and body "applied". Duplicate event IDs must return status 200 and body "duplicate" without applying twice. Events whose sequence is not newer must return status 200 and body "stale" without replacing state. Register POST /webhook through an HTTP router. +Write a SpacetimeDB backend module in Rust that handles idempotent, ordered webhook events. + +TABLES +- processed_event (private), struct ProcessedEvent + - Fields: + - event_id: String (primary key) +- webhook_state (public), struct WebhookState + - Fields: + - key: String (primary key) + - last_sequence: u64 + - value: String + +HANDLER +- POST /webhook + - Accept a pipe-delimited body: event_id|sequence|value + - Store state under key "account" + - For a newly applied event, return status 200 and body "applied" + - For a duplicate event_id, return status 200 and body "duplicate" without applying twice + - When sequence is not newer, return status 200 and body "stale" without replacing state + +ROUTER +- Register POST /webhook through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt index 134fb018f6f..dc053af7d9d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt @@ -1 +1,22 @@ -Write a TypeScript SpacetimeDB module with a POST /webhook handler accepting a pipe-delimited body event_id|sequence|value. Define a private processed_event table with eventId: string primary key. Define a public webhook_state table with key: string primary key, lastSequence: u64, and value: string. Store state under key "account". A newly applied event must return status 200 and body "applied". Duplicate event IDs must return status 200 and body "duplicate" without applying twice. Events whose sequence is not newer must return status 200 and body "stale" without replacing state. Register POST /webhook through an HTTP router. +Write a SpacetimeDB backend module in TypeScript that handles idempotent, ordered webhook events. + +TABLES +- processed_event (private) + - Fields: + - eventId: string (primary key) +- webhook_state (public) + - Fields: + - key: string (primary key) + - lastSequence: u64 + - value: string + +HANDLER +- POST /webhook + - Accept a pipe-delimited body: event_id|sequence|value + - Store state under key "account" + - For a newly applied event, return status 200 and body "applied" + - For a duplicate eventId, return status 200 and body "duplicate" without applying twice + - When sequence is not newer, return status 200 and body "stale" without replacing state + +ROUTER +- Register POST /webhook through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt index 44f301049c1..5514c722006 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -1 +1,23 @@ -Write a C# SpacetimeDB module with a public UploadedAsset table containing Id: ulong primary key, Url: string, and Size: ulong. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure UploadAndRegister(uploadUrl: string, data: byte[]) to POST the binary data to the supplied public URL, require a successful response, then open a transaction to store Id 1, the supplied URL, and the byte length, and return the supplied URL. +Write a SpacetimeDB backend module in C# that exposes an upload route and an external upload procedure. + +TABLE +- UploadedAsset (public, accessor UploadedAsset) + - Fields: + - Id: ulong (primary key) + - Url: string + - Size: ulong + +HANDLER +- POST /upload + - Return status 201 + - Return body "https://files.local/object-1" + +PROCEDURE +- UploadAndRegister(uploadUrl: string, data: byte[]) -> string + - POST the binary data to the supplied public URL + - Require a successful response + - Open a transaction and store Id 1, the supplied URL, and the byte length + - Return the supplied URL + +ROUTER +- Register POST /upload through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt index 2d62220ebe8..c3bb0c861fc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -1 +1,23 @@ -Write a Rust SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: String, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(upload_url: String, data: Vec) to POST the binary data to the supplied public URL, require a successful response, then open a transaction to store id 1, the supplied URL, and the byte length, and return the supplied URL. +Write a SpacetimeDB backend module in Rust that exposes an upload route and an external upload procedure. + +TABLE +- uploaded_asset (public), struct UploadedAsset + - Fields: + - id: u64 (primary key) + - url: String + - size: u64 + +HANDLER +- POST /upload + - Return status 201 + - Return body "https://files.local/object-1" + +PROCEDURE +- upload_and_register(upload_url: String, data: Vec) -> String + - POST the binary data to the supplied public URL + - Require a successful response + - Open a transaction and store id 1, the supplied URL, and the byte length + - Return the supplied URL + +ROUTER +- Register POST /upload through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt index 98adcc2ac06..1a2e9e067d3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -1 +1,23 @@ -Write a TypeScript SpacetimeDB module with a public uploaded_asset table containing id: u64 primary key, url: string, and size: u64. Register a POST /upload handler that returns status 201 and body "https://files.local/object-1". Add the procedure upload_and_register(uploadUrl: string, data: u8[]) to POST the binary data to the supplied public URL, require a successful response, then open a transaction to store id 1, the supplied URL, and the byte length, and return the supplied URL. +Write a SpacetimeDB backend module in TypeScript that exposes an upload route and an external upload procedure. + +TABLE +- uploaded_asset (public) + - Fields: + - id: u64 (primary key) + - url: string + - size: u64 + +HANDLER +- POST /upload + - Return status 201 + - Return body "https://files.local/object-1" + +PROCEDURE +- upload_and_register(uploadUrl: string, data: u8[]) -> string + - POST the binary data to the supplied public URL + - Require a successful response + - Open a transaction and store id 1, the supplied URL, and the byte length + - Return the supplied URL + +ROUTER +- Register POST /upload through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt index 44419cf8444..2791f08bec4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt @@ -1,3 +1,22 @@ -Write a C# SpacetimeDB module with public Account and TransferRequest tables. Account has Id: ulong primary key and Balance: long. TransferRequest has RequestId: string primary key, FromId: ulong, ToId: ulong, and Amount: long. +Write a SpacetimeDB backend module in C# that performs atomic, idempotent transfers between accounts. -Add CreateAccount(id, balance) and Transfer(requestId, fromId, toId, amount). Transfer must reject invalid or insufficient transfers, update both balances atomically, and record RequestId. Repeating a recorded RequestId must be a no-op. +TABLES +- Account (public, accessor Account) + - Fields: + - Id: ulong (primary key) + - Balance: long +- TransferRequest (public, accessor TransferRequest) + - Fields: + - RequestId: string (primary key) + - FromId: ulong + - ToId: ulong + - Amount: long + +REDUCERS +- CreateAccount(id: ulong, balance: long) + - Insert an Account with the supplied values +- Transfer(requestId: string, fromId: ulong, toId: ulong, amount: long) + - Reject invalid or insufficient transfers + - Update both Account balances atomically + - Record RequestId in TransferRequest + - If RequestId is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt index 4f19303b89c..5dc491ee6be 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt @@ -1,3 +1,22 @@ -Write a Rust SpacetimeDB module with public account and transfer_request tables. Account has id: u64 primary key and balance: i64. TransferRequest has request_id: String primary key, from_id: u64, to_id: u64, and amount: i64. +Write a SpacetimeDB backend module in Rust that performs atomic, idempotent transfers between accounts. -Add create_account(id, balance) and transfer(request_id, from_id, to_id, amount). Transfer must reject invalid or insufficient transfers, update both balances atomically, and record request_id. Repeating a recorded request_id must be a no-op. +TABLES +- account (public), struct Account + - Fields: + - id: u64 (primary key) + - balance: i64 +- transfer_request (public), struct TransferRequest + - Fields: + - request_id: String (primary key) + - from_id: u64 + - to_id: u64 + - amount: i64 + +REDUCERS +- create_account(id: u64, balance: i64) + - Insert an account with the supplied values +- transfer(request_id: String, from_id: u64, to_id: u64, amount: i64) + - Reject invalid or insufficient transfers + - Update both account balances atomically + - Record request_id in transfer_request + - If request_id is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt index 54949f1ddeb..fc87b8ae076 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt @@ -1,3 +1,22 @@ -Write a TypeScript SpacetimeDB module with public account and transfer_request tables. Account has id: u64 primary key and balance: i64. TransferRequest has requestId: string primary key, fromId: u64, toId: u64, and amount: i64. +Write a SpacetimeDB backend module in TypeScript that performs atomic, idempotent transfers between accounts. -Add create_account(id, balance) and transfer(requestId, fromId, toId, amount). Transfer must reject invalid or insufficient transfers, update both balances atomically, and record requestId. Repeating a recorded requestId must be a no-op. +TABLES +- account (public) + - Fields: + - id: u64 (primary key) + - balance: i64 +- transfer_request (public) + - Fields: + - requestId: string (primary key) + - fromId: u64 + - toId: u64 + - amount: i64 + +REDUCERS +- create_account(id: u64, balance: i64) + - Insert an account with the supplied values +- transfer(requestId: string, fromId: u64, toId: u64, amount: i64) + - Reject invalid or insufficient transfers + - Update both account balances atomically + - Record requestId in transfer_request + - If requestId is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt index 1d8a4de1051..508762521fe 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt @@ -1,3 +1,20 @@ -Write a C# SpacetimeDB module with a Preferences product type containing Theme: string, EmailNotifications: bool, and Timezone: string. Store it in a public Profile table with Id: ulong primary key. +Write a SpacetimeDB backend module in C# that stores and updates nested profile preferences. -Add CreateProfile(id, theme, emailNotifications, timezone) and UpdateTheme(id, theme). UpdateTheme must change only the nested Theme and preserve both sibling fields. +TYPES +- Preferences + - Theme: string + - EmailNotifications: bool + - Timezone: string + +TABLE +- Profile (public, accessor Profile) + - Fields: + - Id: ulong (primary key) + - Preferences: Preferences + +REDUCERS +- CreateProfile(id: ulong, theme: string, emailNotifications: bool, timezone: string) + - Insert a Profile with all three preference fields +- UpdateTheme(id: ulong, theme: string) + - Change only Preferences.Theme + - Preserve Preferences.EmailNotifications and Preferences.Timezone diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt index 1f299c856bd..8c0f974aba3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt @@ -1,3 +1,20 @@ -Write a Rust SpacetimeDB module with a Preferences product type containing theme: String, email_notifications: bool, and timezone: String. Store it in a public profile table with id: u64 primary key. +Write a SpacetimeDB backend module in Rust that stores and updates nested profile preferences. -Add create_profile(id, theme, email_notifications, timezone) and update_theme(id, theme). update_theme must change only the nested theme and preserve both sibling fields. +TYPES +- Preferences + - theme: String + - email_notifications: bool + - timezone: String + +TABLE +- profile (public), struct Profile + - Fields: + - id: u64 (primary key) + - preferences: Preferences + +REDUCERS +- create_profile(id: u64, theme: String, email_notifications: bool, timezone: String) + - Insert a profile with all three preference fields +- update_theme(id: u64, theme: String) + - Change only preferences.theme + - Preserve preferences.email_notifications and preferences.timezone diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt index 5a877900a09..cec8627b326 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt @@ -1,3 +1,20 @@ -Write a TypeScript SpacetimeDB module with a Preferences object type containing theme: string, emailNotifications: boolean, and timezone: string. Store it in a public profile table with id: u64 primary key. +Write a SpacetimeDB backend module in TypeScript that stores and updates nested profile preferences. -Add create_profile(id, theme, emailNotifications, timezone) and update_theme(id, theme). update_theme must change only the nested theme and preserve both sibling fields. +TYPES +- Preferences + - theme: string + - emailNotifications: boolean + - timezone: string + +TABLE +- profile (public) + - Fields: + - id: u64 (primary key) + - preferences: Preferences + +REDUCERS +- create_profile(id: u64, theme: string, emailNotifications: boolean, timezone: string) + - Insert a profile with all three preference fields +- update_theme(id: u64, theme: string) + - Change only preferences.theme + - Preserve preferences.emailNotifications and preferences.timezone diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt index 53179698207..49a158aaae6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt @@ -1,3 +1,25 @@ -Write a C# SpacetimeDB module with public Workspace, Project, TaskItem, and TaskNote tables. Each has an ulong primary key. Project has indexed WorkspaceId, TaskItem has indexed ProjectId, and TaskNote has indexed TaskId. +Write a SpacetimeDB backend module in C# that performs a nested cascade delete across normalized tables. -Add Seed() creating two independent workspace→project→task→note chains using IDs 1 and 2. Add DeleteWorkspace(id) that deletes the selected workspace and every level of its descendants while preserving the unrelated chain. +TABLES +- Workspace (public, accessor Workspace) + - Fields: + - Id: ulong (primary key) +- Project (public, accessor Project) + - Fields: + - Id: ulong (primary key) + - WorkspaceId: ulong (btree index) +- TaskItem (public, accessor TaskItem) + - Fields: + - Id: ulong (primary key) + - ProjectId: ulong (btree index) +- TaskNote (public, accessor TaskNote) + - Fields: + - Id: ulong (primary key) + - TaskId: ulong (btree index) + +REDUCERS +- Seed() + - Create two independent Workspace -> Project -> TaskItem -> TaskNote chains using IDs 1 and 2 +- DeleteWorkspace(id: ulong) + - Delete the selected Workspace and every level of its descendants + - Preserve the unrelated chain diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt index 8dbbd333444..98897315991 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt @@ -1,3 +1,25 @@ -Write a Rust SpacetimeDB module with public workspace, project, task_item, and task_note tables. Each has a u64 primary key. Project has indexed workspace_id, TaskItem has indexed project_id, and TaskNote has indexed task_id. +Write a SpacetimeDB backend module in Rust that performs a nested cascade delete across normalized tables. -Add seed() creating two independent workspace→project→task→note chains using IDs 1 and 2. Add delete_workspace(id) that deletes the selected workspace and every level of its descendants while preserving the unrelated chain. +TABLES +- workspace (public), struct Workspace + - Fields: + - id: u64 (primary key) +- project (public), struct Project + - Fields: + - id: u64 (primary key) + - workspace_id: u64 (btree index) +- task_item (public), struct TaskItem + - Fields: + - id: u64 (primary key) + - project_id: u64 (btree index) +- task_note (public), struct TaskNote + - Fields: + - id: u64 (primary key) + - task_id: u64 (btree index) + +REDUCERS +- seed() + - Create two independent workspace -> project -> task_item -> task_note chains using IDs 1 and 2 +- delete_workspace(id: u64) + - Delete the selected workspace and every level of its descendants + - Preserve the unrelated chain diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt index e8d30be14e6..c33a2395d4b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt @@ -1,3 +1,25 @@ -Write a TypeScript SpacetimeDB module with public workspace, project, task_item, and task_note tables. Each has a u64 primary key. Project has indexed workspaceId, TaskItem has indexed projectId, and TaskNote has indexed taskId. +Write a SpacetimeDB backend module in TypeScript that performs a nested cascade delete across normalized tables. -Add seed() creating two independent workspace→project→task→note chains using IDs 1 and 2. Add delete_workspace(id) that deletes the selected workspace and every level of its descendants while preserving the unrelated chain. +TABLES +- workspace (public) + - Fields: + - id: u64 (primary key) +- project (public) + - Fields: + - id: u64 (primary key) + - workspaceId: u64 (btree index) +- task_item (public) + - Fields: + - id: u64 (primary key) + - projectId: u64 (btree index) +- task_note (public) + - Fields: + - id: u64 (primary key) + - taskId: u64 (btree index) + +REDUCERS +- seed() + - Create two independent workspace -> project -> task_item -> task_note chains using IDs 1 and 2 +- delete_workspace(id: u64) + - Delete the selected workspace and every level of its descendants + - Preserve the unrelated chain diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt index 0f57f81f8f3..cf9d8096d7e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt @@ -1,3 +1,19 @@ -Write a C# SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. +Write a SpacetimeDB backend module in C# that deletes a logical group in scheduled batches of at most two rows. -Define a public WorkItem table with Id: ulong primary key and indexed GroupId: ulong. Define a private DeleteJob scheduled table with ScheduledId: ulong auto-increment primary key, ScheduledAt: ScheduleAt, and GroupId: ulong. Add SeedGroup(groupId, count), RequestDelete(groupId), and the scheduled reducer RunDeleteBatch. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. +TABLES +- WorkItem (public, accessor WorkItem) + - Fields: + - Id: ulong (primary key) + - GroupId: ulong (btree index) +- DeleteJob (private, scheduled for RunDeleteBatch, accessor DeleteJob) + - Fields: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - GroupId: ulong + +REDUCERS +- SeedGroup(groupId: ulong, count: uint) +- RequestDelete(groupId: ulong) +- RunDeleteBatch(job: DeleteJob) (scheduled reducer) + - Delete at most two WorkItem rows matching the scheduled GroupId + - If matching rows remain, enqueue another one-shot DeleteJob diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt index 25ee2d53f41..154e2fea8d5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt @@ -1,3 +1,19 @@ -Write a Rust SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. +Write a SpacetimeDB backend module in Rust that deletes a logical group in scheduled batches of at most two rows. -Define a public work_item table with struct WorkItem and fields id: u64 primary key and indexed group_id: u64. Define a private delete_job scheduled table with struct DeleteJob and fields scheduled_id: u64 auto-increment primary key, scheduled_at: ScheduleAt, and group_id: u64. Add seed_group(group_id, count), request_delete(group_id), and the scheduled reducer run_delete_batch. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. +TABLES +- work_item (public), struct WorkItem + - Fields: + - id: u64 (primary key) + - group_id: u64 (btree index) +- delete_job (private, scheduled for run_delete_batch), struct DeleteJob + - Fields: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - group_id: u64 + +REDUCERS +- seed_group(group_id: u64, count: u32) +- request_delete(group_id: u64) +- run_delete_batch(job: DeleteJob) (scheduled reducer) + - Delete at most two work_item rows matching the scheduled group_id + - If matching rows remain, enqueue another one-shot delete_job diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt index 6ba828b1f31..f9d5dd8759c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt @@ -1,3 +1,19 @@ -Write a TypeScript SpacetimeDB module that deletes a logical group in scheduled batches of at most two rows. +Write a SpacetimeDB backend module in TypeScript that deletes a logical group in scheduled batches of at most two rows. -Define a public work_item table with id: u64 primary key and indexed groupId: u64. Define a private delete_job scheduled table with scheduledId: u64 auto-increment primary key, scheduledAt: ScheduleAt, and groupId: u64. Add seed_group(groupId, count), request_delete(groupId), and the scheduled reducer runDeleteBatch. Each scheduled invocation may delete at most two matching items and must enqueue another one-shot job when items remain. +TABLES +- work_item (public) + - Fields: + - id: u64 (primary key) + - groupId: u64 (btree index) +- delete_job (private, scheduled for runDeleteBatch) + - Fields: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - groupId: u64 + +REDUCERS +- seed_group(groupId: u64, count: u32) +- request_delete(groupId: u64) +- runDeleteBatch(timer: delete_job row) (scheduled reducer) + - Delete at most two work_item rows matching the scheduled groupId + - If matching rows remain, enqueue another one-shot delete_job diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt index 3067cabc484..e0d32d485f9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt @@ -1 +1,14 @@ -Write a C# SpacetimeDB module with a public GeneratedValue table containing Id: ulong auto-increment primary key, CreatedAt: Timestamp, and RandomValue: long. Add Generate() that stores ctx.Timestamp and a random long from ctx.Rng. Do not use a host clock or separately seeded Random. +Write a SpacetimeDB backend module in C# that records deterministic reducer time and randomness. + +TABLE +- GeneratedValue (public, accessor GeneratedValue) + - Fields: + - Id: ulong (primary key, auto-increment) + - CreatedAt: Timestamp + - RandomValue: long + +REDUCERS +- Generate() + - Store ctx.Timestamp in CreatedAt + - Store a random long from ctx.Rng in RandomValue + - Do not use a host clock or a separately seeded Random diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt index 1353d94fc59..ee479b62124 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt @@ -1 +1,14 @@ -Write a Rust SpacetimeDB module with a public generated_value table containing id: u64 auto-increment primary key, created_at: Timestamp, and random_value: u64. Add generate() that stores the reducer context timestamp and a u64 from the reducer context RNG. Do not use a host clock or separately seeded RNG. +Write a SpacetimeDB backend module in Rust that records deterministic reducer time and randomness. + +TABLE +- generated_value (public), struct GeneratedValue + - Fields: + - id: u64 (primary key, auto-increment) + - created_at: Timestamp + - random_value: u64 + +REDUCERS +- generate() + - Store the reducer context timestamp in created_at + - Store a u64 from the reducer context RNG in random_value + - Do not use a host clock or a separately seeded RNG diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt index f8311899670..6d3d105d6e2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt @@ -1 +1,14 @@ -Write a TypeScript SpacetimeDB module with a public generated_value table containing id: u64 auto-increment primary key, createdAt: Timestamp, and randomValue: i64. Add generate() that stores ctx.timestamp and a random integer from ctx.random. Do not use Date, Math.random, or a separately seeded RNG. +Write a SpacetimeDB backend module in TypeScript that records deterministic reducer time and randomness. + +TABLE +- generated_value (public) + - Fields: + - id: u64 (primary key, auto-increment) + - createdAt: Timestamp + - randomValue: i64 + +REDUCERS +- generate() + - Store ctx.timestamp in createdAt + - Store a random integer from ctx.random in randomValue + - Do not use Date, Math.random, or a separately seeded RNG diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt index ea20c847c94..b104afbfed6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt @@ -1 +1,14 @@ -Write a C# SpacetimeDB module with a public CommandResult table containing RequestId: string primary key, Success: bool, and Message: string. Add RunCommand(requestId, value). Reducers do not return application data; write a result row whose Message is "value=" and Success is true. +Write a SpacetimeDB backend module in C# that writes reducer results to a table. + +TABLE +- CommandResult (public, accessor CommandResult) + - Fields: + - RequestId: string (primary key) + - Success: bool + - Message: string + +REDUCERS +- RunCommand(requestId: string, value: int) + - Reducers do not return application data + - Insert a CommandResult row with Success set to true + - Set Message to "value=" using the supplied value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt index 5096ccc8d82..4f59c30455c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt @@ -1 +1,14 @@ -Write a Rust SpacetimeDB module with a public command_result table containing request_id: String primary key, success: bool, and message: String. Add run_command(request_id, value). Reducers do not return application data; write a result row whose message is "value=" and success is true. +Write a SpacetimeDB backend module in Rust that writes reducer results to a table. + +TABLE +- command_result (public), struct CommandResult + - Fields: + - request_id: String (primary key) + - success: bool + - message: String + +REDUCERS +- run_command(request_id: String, value: i32) + - Reducers do not return application data + - Insert a command_result row with success set to true + - Set message to "value=" using the supplied value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt index 85e54e92c02..011e5797d96 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt @@ -1 +1,14 @@ -Write a TypeScript SpacetimeDB module with a public command_result table containing requestId: string primary key, success: boolean, and message: string. Add run_command(requestId, value). Reducers do not return application data; write a result row whose message is "value=" and success is true. +Write a SpacetimeDB backend module in TypeScript that writes reducer results to a table. + +TABLE +- command_result (public) + - Fields: + - requestId: string (primary key) + - success: boolean + - message: string + +REDUCERS +- run_command(requestId: string, value: i32) + - Reducers do not return application data + - Insert a command_result row with success set to true + - Set message to "value=" using the supplied value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt index 9e1f3878efa..687052ddd10 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt @@ -1,5 +1,17 @@ Write a SpacetimeDB backend module in C# that stores binary content and its metadata. -Define a public BlobRecord table with an auto-incremented ulong primary key named Id, Owner: Identity, Filename: string, MimeType: string, Size: ulong, and Data: List. Add a B-tree index named by_owner on Owner. +TABLE +- BlobRecord (public, accessor BlobRecord) + - Fields: + - Id: ulong (primary key, auto-increment) + - Owner: Identity (btree index by_owner) + - Filename: string + - MimeType: string + - Size: ulong + - Data: List -Define a StoreBlob reducer taking filename, mimeType, and data. Store the sender as Owner, derive Size from the byte count, and insert the bytes unchanged. +REDUCERS +- StoreBlob(filename: string, mimeType: string, data: List) + - Store the sender as Owner + - Derive Size from the byte count + - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt index dd87130f42f..9126ebc5b62 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt @@ -1,5 +1,17 @@ Write a SpacetimeDB backend module in Rust that stores binary content and its metadata. -Define a public blob_record table with an auto-incremented u64 primary key named id, owner: Identity, filename: String, mime_type: String, size: u64, and data: Vec. Add a B-tree index named by_owner on owner. +TABLE +- blob_record (public), struct BlobRecord + - Fields: + - id: u64 (primary key, auto-increment) + - owner: Identity (btree index by_owner) + - filename: String + - mime_type: String + - size: u64 + - data: Vec -Define a store_blob reducer taking filename, mime_type, and data. Store the sender as owner, derive size from the byte length, and insert the bytes unchanged. +REDUCERS +- store_blob(filename: String, mime_type: String, data: Vec) + - Store the sender as owner + - Derive size from the byte length + - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt index ecbc109f031..13dd98d2208 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt @@ -1,5 +1,17 @@ Write a SpacetimeDB backend module in TypeScript that stores binary content and its metadata. -Define a public blob_record table with an auto-incremented u64 primary key named id, owner: Identity, filename: string, mimeType: string, size: u64, and data: u8 array. Add a B-tree index named byOwner on owner. +TABLE +- blob_record (public) + - Fields: + - id: u64 (primary key, auto-increment) + - owner: Identity (btree index byOwner) + - filename: string + - mimeType: string + - size: u64 + - data: u8 array -Define a store_blob reducer taking filename, mimeType, and data. Store ctx.sender as owner, derive size from the byte length, and insert the bytes unchanged. +REDUCERS +- store_blob(filename: string, mimeType: string, data: u8 array) + - Store ctx.sender as owner + - Derive size from the byte length + - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt index 8ba59dd8537..9e7dac73cd5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt @@ -1,5 +1,21 @@ Write a SpacetimeDB backend module in C# that maintains an indexed denormalized category slug on products. -Define public Category and Product tables. Category has Id: ulong primary key and Slug: string. Product has Id: ulong primary key, CategoryId: ulong with a B-tree index named by_category, CategorySlug: string with a B-tree index named by_category_slug, and Name: string. +TABLES +- Category (public, accessor Category) + - Fields: + - Id: ulong (primary key) + - Slug: string +- Product (public, accessor Product) + - Fields: + - Id: ulong (primary key) + - CategoryId: ulong (btree index by_category) + - CategorySlug: string (btree index by_category_slug) + - Name: string -Add CreateCategory(id, slug), CreateProduct(id, categoryId, name), and RenameCategory(id, newSlug) reducers. CreateProduct must copy the current category slug. RenameCategory must update both the category and every related product so CategorySlug stays synchronized. +REDUCERS +- CreateCategory(id: ulong, slug: string) +- CreateProduct(id: ulong, categoryId: ulong, name: string) + - Copy the current Category Slug into CategorySlug +- RenameCategory(id: ulong, newSlug: string) + - Update the Category Slug + - Update every related Product so CategorySlug stays synchronized diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt index 8586fe2926f..1aea0d4e82b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt @@ -1,5 +1,21 @@ Write a SpacetimeDB backend module in Rust that maintains an indexed denormalized category slug on products. -Define public category and product tables. Category has id: u64 primary key and slug: String. Product has id: u64 primary key, category_id: u64 with a B-tree index named by_category, category_slug: String with a B-tree index named by_category_slug, and name: String. +TABLES +- category (public), struct Category + - Fields: + - id: u64 (primary key) + - slug: String +- product (public), struct Product + - Fields: + - id: u64 (primary key) + - category_id: u64 (btree index by_category) + - category_slug: String (btree index by_category_slug) + - name: String -Add create_category(id, slug), create_product(id, category_id, name), and rename_category(id, new_slug) reducers. create_product must copy the current category slug. rename_category must update both the category and every related product so category_slug stays synchronized. +REDUCERS +- create_category(id: u64, slug: String) +- create_product(id: u64, category_id: u64, name: String) + - Copy the current category slug into category_slug +- rename_category(id: u64, new_slug: String) + - Update the category slug + - Update every related product so category_slug stays synchronized diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt index e5277f700b6..08113e51758 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt @@ -1,5 +1,21 @@ Write a SpacetimeDB backend module in TypeScript that maintains an indexed denormalized category slug on products. -Define public category and product tables. Category has id: u64 primary key and slug: string. Product has id: u64 primary key, categoryId: u64 with a B-tree index named byCategory, categorySlug: string with a B-tree index named byCategorySlug, and name: string. +TABLES +- category (public) + - Fields: + - id: u64 (primary key) + - slug: string +- product (public) + - Fields: + - id: u64 (primary key) + - categoryId: u64 (btree index byCategory) + - categorySlug: string (btree index byCategorySlug) + - name: string -Add create_category(id, slug), create_product(id, categoryId, name), and rename_category(id, newSlug) reducers. create_product must copy the current category slug. rename_category must update both the category and every related product so categorySlug stays synchronized. +REDUCERS +- create_category(id: u64, slug: string) +- create_product(id: u64, categoryId: u64, name: string) + - Copy the current category slug into categorySlug +- rename_category(id: u64, newSlug: string) + - Update the category slug + - Update every related product so categorySlug stays synchronized diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt index 5717ee4754d..1b800b26673 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt @@ -1,5 +1,18 @@ Write a SpacetimeDB backend module in C# that safely creates related rows with an auto-incremented parent ID. -Define public Parent and Child tables. Both have auto-incremented ulong primary keys named Id. Parent also has Name: string. Child has ParentId: ulong with a B-tree index named by_parent and Name: string. +TABLES +- Parent (public, accessor Parent) + - Fields: + - Id: ulong (primary key, auto-increment) + - Name: string +- Child (public, accessor Child) + - Fields: + - Id: ulong (primary key, auto-increment) + - ParentId: ulong (btree index by_parent) + - Name: string -Define CreateFamily(parentName, childNames). Insert the parent, capture the row returned by Insert, and use that returned Id for every child. Never calculate or assume the next auto-increment value. +REDUCERS +- CreateFamily(parentName: string, childNames: List) + - Insert the Parent and capture the row returned by Insert + - Use the returned Parent Id for every Child + - Never calculate or assume the next auto-increment value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt index 3ea61625ffa..a7ec69b64b3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt @@ -1,5 +1,18 @@ Write a SpacetimeDB backend module in Rust that safely creates related rows with an auto-incremented parent ID. -Define public parent and child tables. Both have auto-incremented u64 primary keys named id. Parent also has name: String. Child has parent_id: u64 with a B-tree index named by_parent and name: String. +TABLES +- parent (public), struct Parent + - Fields: + - id: u64 (primary key, auto-increment) + - name: String +- child (public), struct Child + - Fields: + - id: u64 (primary key, auto-increment) + - parent_id: u64 (btree index by_parent) + - name: String -Define create_family(parent_name, child_names). Insert the parent, capture the row returned by insert, and use that returned id for every child. Never calculate or assume the next auto-increment value. +REDUCERS +- create_family(parent_name: String, child_names: Vec) + - Insert the parent and capture the row returned by insert + - Use the returned parent id for every child + - Never calculate or assume the next auto-increment value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt index 16f0578ca4b..7a6b678ecee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt @@ -1,5 +1,18 @@ Write a SpacetimeDB backend module in TypeScript that safely creates related rows with an auto-incremented parent ID. -Define public parent and child tables. Both have auto-incremented u64 primary keys named id. Parent also has name: string. Child has parentId: u64 with a B-tree index named byParent and name: string. +TABLES +- parent (public) + - Fields: + - id: u64 (primary key, auto-increment) + - name: string +- child (public) + - Fields: + - id: u64 (primary key, auto-increment) + - parentId: u64 (btree index byParent) + - name: string -Define create_family(parentName, childNames). Insert the parent, capture the row returned by insert, and use that returned id for every child. Never calculate or assume the next auto-increment value. +REDUCERS +- create_family(parentName: string, childNames: string array) + - Insert the parent and capture the row returned by insert + - Use the returned parent id for every child + - Never calculate or assume the next auto-increment value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt index a01574d65c0..f00c940541f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt @@ -1 +1,16 @@ -Update the C# SpacetimeDB module without removing or renaming its public Widget table, Seed reducer, or Touch reducer. Add a trailing bool field named Enabled to Widget and annotate that field with a default value of true so a compatible republish backfills existing rows. Keep Id as the ulong primary key and do not put the default annotation on a primary-key, unique, or auto-increment column. Update new Widget values to set Enabled to true. +Update the existing SpacetimeDB backend module in C# to add a migration-safe default value. + +TABLE +- Widget (public, accessor Widget) + - Preserve the existing table and fields + - Keep Id: ulong as the primary key + - Add Enabled: bool as the trailing field + - Give Enabled a default value of true + +REDUCERS +- Preserve Seed and Touch without renaming them +- Update new Widget values to set Enabled to true + +MIGRATION +- A compatible republish must backfill existing rows +- Do not put the default annotation on a primary-key, unique, or auto-increment column diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt index dc7219adb21..20ab31dcb0b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt @@ -1 +1,16 @@ -Update the Rust SpacetimeDB module without removing or renaming its public widget table, seed reducer, or touch reducer. Add a trailing bool field named enabled to Widget and annotate that field with a default value of true so a compatible republish backfills existing rows. Keep id as the u64 primary key and do not put the default annotation on a primary-key, unique, or auto-increment column. Update new Widget values to set enabled to true. +Update the existing SpacetimeDB backend module in Rust to add a migration-safe default value. + +TABLE +- widget (public), struct Widget + - Preserve the existing table and fields + - Keep id: u64 as the primary key + - Add enabled: bool as the trailing field + - Give enabled a default value of true + +REDUCERS +- Preserve seed and touch without renaming them +- Update new Widget values to set enabled to true + +MIGRATION +- A compatible republish must backfill existing rows +- Do not put the default annotation on a primary-key, unique, or auto-increment column diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt index 770e9dbd563..3ee41ae4228 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt @@ -1 +1,16 @@ -Update the TypeScript SpacetimeDB module without removing or renaming its public widget table, seed reducer, or touch reducer. Add a trailing bool field named enabled to widget and give that field a default value of true so a compatible republish backfills existing rows. Keep id as the u64 primary key and do not put the default on a primary-key, unique, or auto-increment column. Update new widget values to set enabled to true. +Update the existing SpacetimeDB backend module in TypeScript to add a migration-safe default value. + +TABLE +- widget (public) + - Preserve the existing table and fields + - Keep id: u64 as the primary key + - Add enabled: bool as the trailing field + - Give enabled a default value of true + +REDUCERS +- Preserve seed and touch without renaming them +- Update new widget values to set enabled to true + +MIGRATION +- A compatible republish must backfill existing rows +- Do not put the default on a primary-key, unique, or auto-increment column diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt index 18509d2e760..2da04668d5e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt @@ -1,3 +1,15 @@ -Write a SpacetimeDB backend module in C# defining one public SpecialValue table. +Write a SpacetimeDB backend module in C# defining one public table with special value types. -The table must contain Id: ulong primary key, Uuid: Uuid, ConnectionId: ConnectionId, Duration: TimeDuration, Unsigned128: U128, Signed128: I128, Unsigned256: U256, and Signed256: I256. No reducers are required. +TABLE +- SpecialValue (public, accessor SpecialValue) + - Fields: + - Id: ulong (primary key) + - Uuid: Uuid + - ConnectionId: ConnectionId + - Duration: TimeDuration + - Unsigned128: U128 + - Signed128: I128 + - Unsigned256: U256 + - Signed256: I256 + +No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt index 29d9000dd2b..fd0a5b9a3ec 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt @@ -1,3 +1,16 @@ -Write a SpacetimeDB backend module in Rust defining one public special_value table. +Write a SpacetimeDB backend module in Rust defining one public table with special value types. -The table must contain id: u64 primary key, uuid: Uuid, connection_id: ConnectionId, duration: TimeDuration, unsigned_128: u128, signed_128: i128, unsigned_256: u256, and signed_256: i256. Use the SpacetimeDB integer types for 256-bit values. No reducers are required. +TABLE +- special_value (public), struct SpecialValue + - Fields: + - id: u64 (primary key) + - uuid: Uuid + - connection_id: ConnectionId + - duration: TimeDuration + - unsigned_128: u128 + - signed_128: i128 + - unsigned_256: u256 + - signed_256: i256 + - Use the SpacetimeDB integer types for 256-bit values + +No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt index 81a70148e5f..d2866eb8bcd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt @@ -1,3 +1,15 @@ -Write a SpacetimeDB backend module in TypeScript defining one public special_value table. +Write a SpacetimeDB backend module in TypeScript defining one public table with special value types. -The table must contain id: u64 primary key, uuid: Uuid, connectionId: ConnectionId, duration: TimeDuration, unsigned128: u128, signed128: i128, unsigned256: u256, and signed256: i256. No reducers are required. +TABLE +- special_value (public) + - Fields: + - id: u64 (primary key) + - uuid: Uuid + - connectionId: ConnectionId + - duration: TimeDuration + - unsigned128: u128 + - signed128: i128 + - unsigned256: u256 + - signed256: i256 + +No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt index 5335ede3449..1061f2c5872 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -1 +1,26 @@ -Write a C# SpacetimeDB module with public Customer, Purchase, and LineItem tables. Customer has Id: ulong primary key and Name: string. Purchase has Id: ulong primary key and indexed CustomerId: ulong. LineItem has Id: ulong primary key, indexed PurchaseId: ulong, Sku: string, and indexed Visible: bool. Add a Seed() reducer with one complete visible chain. Expose rows containing LineId: ulong, CustomerName: string, and Sku: string through a public OrderLineDetail view by joining all three tables through indexed lookups. +Write a SpacetimeDB backend module in C# that defines a public view joining three normalized tables. + +TABLES +- Customer (public, accessor Customer) + - Fields: + - Id: ulong (primary key) + - Name: string +- Purchase (public, accessor Purchase) + - Fields: + - Id: ulong (primary key) + - CustomerId: ulong (btree index) +- LineItem (public, accessor LineItem) + - Fields: + - Id: ulong (primary key) + - PurchaseId: ulong (btree index) + - Sku: string + - Visible: bool (btree index) + +REDUCERS +- Seed() + - Insert one complete visible Customer -> Purchase -> LineItem chain + +VIEW +- OrderLineDetail (public) + - Returns rows with LineId: ulong, CustomerName: string, and Sku: string + - Join all three tables through indexed lookups diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt index 238760763e9..1dd2ef4a609 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -1 +1,26 @@ -Write a Rust SpacetimeDB module with public customer (struct Customer), purchase (struct Purchase), and line_item (struct LineItem) tables. Customer has id: u64 primary key and name: String. Purchase has id: u64 primary key and indexed customer_id: u64. LineItem has id: u64 primary key, indexed purchase_id: u64, sku: String, and indexed visible: bool. Add a seed() reducer with one complete visible chain. Expose rows containing line_id: u64, customer_name: String, and sku: String through a public order_line_detail view by joining all three tables through indexed lookups. +Write a SpacetimeDB backend module in Rust that defines a public view joining three normalized tables. + +TABLES +- customer (public), struct Customer + - Fields: + - id: u64 (primary key) + - name: String +- purchase (public), struct Purchase + - Fields: + - id: u64 (primary key) + - customer_id: u64 (btree index) +- line_item (public), struct LineItem + - Fields: + - id: u64 (primary key) + - purchase_id: u64 (btree index) + - sku: String + - visible: bool (btree index) + +REDUCERS +- seed() + - Insert one complete visible customer -> purchase -> line_item chain + +VIEW +- order_line_detail (public) + - Returns rows with line_id: u64, customer_name: String, and sku: String + - Join all three tables through indexed lookups diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt index c6fcac455d7..f6f9c10b58c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -1 +1,26 @@ -Write a TypeScript SpacetimeDB module with public customer, purchase, and line_item tables. Customer has id: u64 primary key and name: string. Purchase has id: u64 primary key and indexed customerId: u64. Line item has id: u64 primary key, indexed purchaseId: u64, sku: string, and indexed visible: bool. Add a seed() reducer with one complete visible chain. Expose rows containing lineId: u64, customerName: string, and sku: string through a public order_line_detail view by joining all three tables through indexed lookups. +Write a SpacetimeDB backend module in TypeScript that defines a public view joining three normalized tables. + +TABLES +- customer (public) + - Fields: + - id: u64 (primary key) + - name: string +- purchase (public) + - Fields: + - id: u64 (primary key) + - customerId: u64 (btree index) +- line_item (public) + - Fields: + - id: u64 (primary key) + - purchaseId: u64 (btree index) + - sku: string + - visible: bool (btree index) + +REDUCERS +- seed() + - Insert one complete visible customer -> purchase -> line_item chain + +VIEW +- order_line_detail (public) + - Returns rows with lineId: u64, customerName: string, and sku: string + - Join all three tables through indexed lookups diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt index fde1d658389..5fd5a360c02 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt @@ -1 +1,21 @@ -Write a C# SpacetimeDB module with public Member(Id primary key, Name) and Eligibility(Id primary key, indexed MemberId) tables. Add Seed() with two members but eligibility for only member 1. Expose a public EligibleMember view using a query-builder semijoin that returns members only when a matching eligibility row exists. +Write a SpacetimeDB backend module in C# that defines a query-builder semijoin view. + +TABLES +- Member (public, accessor Member) + - Fields: + - Id: ulong (primary key) + - Name: string +- Eligibility (public, accessor Eligibility) + - Fields: + - Id: ulong (primary key) + - MemberId: ulong (btree index) + +REDUCERS +- Seed() + - Insert two Member rows + - Insert an Eligibility row for only member 1 + +VIEW +- EligibleMember (public, query-builder) + - Use a semijoin + - Return members only when a matching Eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt index 90a63f186ec..7d3ec0b6277 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt @@ -1 +1,21 @@ -Write a Rust SpacetimeDB module with public member(id primary key, name) and eligibility(id primary key, indexed member_id) tables. Add seed() with two members but eligibility for only member 1. Expose a public eligible_member view using a query-builder semijoin that returns members only when a matching eligibility row exists. +Write a SpacetimeDB backend module in Rust that defines a query-builder semijoin view. + +TABLES +- member (public), struct Member + - Fields: + - id: u64 (primary key) + - name: String +- eligibility (public), struct Eligibility + - Fields: + - id: u64 (primary key) + - member_id: u64 (btree index) + +REDUCERS +- seed() + - Insert two members + - Insert an eligibility row for only member 1 + +VIEW +- eligible_member (public, query-builder) + - Use a semijoin + - Return members only when a matching eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt index b8a6d1b3cd6..3e6481118bb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt @@ -1 +1,21 @@ -Write a TypeScript SpacetimeDB module with public member(id primary key, name) and eligibility(id primary key, indexed memberId) tables. Add seed() with two members but eligibility for only member 1. Expose a public eligible_member view using a query-builder semijoin that returns members only when a matching eligibility row exists. +Write a SpacetimeDB backend module in TypeScript that defines a query-builder semijoin view. + +TABLES +- member (public) + - Fields: + - id: u64 (primary key) + - name: string +- eligibility (public) + - Fields: + - id: u64 (primary key) + - memberId: u64 (btree index) + +REDUCERS +- seed() + - Insert two member rows + - Insert an eligibility row for only member 1 + +VIEW +- eligible_member (public, query-builder) + - Use a semijoin + - Return members only when a matching eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt index 633a39fd76e..ce63c66ddbf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -1 +1,17 @@ -Write a C# SpacetimeDB module with a public table named Event containing Id: ulong primary key, indexed OccurredAt: Timestamp, and Label: string. Add Seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public WindowEvent procedural view that uses the OccurredAt index to return the inclusive range from 200 through 400 microseconds. +Write a SpacetimeDB backend module in C# that defines a procedural view over an inclusive timestamp range. + +TABLE +- Event (public, accessor Event) + - Fields: + - Id: ulong (primary key) + - OccurredAt: Timestamp (btree index) + - Label: string + +REDUCERS +- Seed() + - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + +VIEW +- WindowEvent (procedural, public) + - Use the OccurredAt index + - Return the inclusive range from 200 through 400 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt index ec65aa328e5..3e2095d944e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -1 +1,17 @@ -Write a Rust SpacetimeDB module with a public table named event containing id: u64 primary key, indexed occurred_at: Timestamp, and label: String. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurred_at index to return the inclusive range from 200 through 400 microseconds. +Write a SpacetimeDB backend module in Rust that defines a procedural view over an inclusive timestamp range. + +TABLE +- event (public), struct Event + - Fields: + - id: u64 (primary key) + - occurred_at: Timestamp (btree index) + - label: String + +REDUCERS +- seed() + - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + +VIEW +- window_event (procedural, public) + - Use the occurred_at index + - Return the inclusive range from 200 through 400 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt index 2f3ec91a5ad..9f0cef78f51 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -1 +1,17 @@ -Write a TypeScript SpacetimeDB module with a public table named event containing id: u64 primary key, indexed occurredAt: Timestamp, and label: string. Add seed() with events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch. Expose a public window_event procedural view that uses the occurredAt index to return the inclusive range from 200 through 400 microseconds. +Write a SpacetimeDB backend module in TypeScript that defines a procedural view over an inclusive timestamp range. + +TABLE +- event (public) + - Fields: + - id: u64 (primary key) + - occurredAt: Timestamp (btree index) + - label: string + +REDUCERS +- seed() + - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + +VIEW +- window_event (procedural, public) + - Use the occurredAt index + - Return the inclusive range from 200 through 400 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt index 97dfb07ebaf..22ff4612518 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt @@ -1 +1,19 @@ -Write a C# SpacetimeDB module with a public Content table containing Id: ulong primary key, indexed Category: string, Active: bool, and Score: int. Add Seed() with rows that separately fail category, active, and score checks plus one passing row. Expose a public FeaturedContent procedural view that first uses the Category index for "news", then filters for active rows with Score at least 10. +Write a SpacetimeDB backend module in C# that defines a procedural view using an index and additional filters. + +TABLE +- Content (public, accessor Content) + - Fields: + - Id: ulong (primary key) + - Category: string (btree index) + - Active: bool + - Score: int + +REDUCERS +- Seed() + - Insert rows that separately fail the Category, Active, and Score checks + - Insert one passing row + +VIEW +- FeaturedContent (procedural, public) + - First use the Category index for "news" + - Then return only Active rows with Score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt index 44edce017a8..f28bd04a3ad 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt @@ -1 +1,19 @@ -Write a Rust SpacetimeDB module with a public content table containing id: u64 primary key, indexed category: String, active: bool, and score: i32. Add seed() with rows that separately fail category, active, and score checks plus one passing row. Expose a public featured_content procedural view that first uses the category index for "news", then filters for active rows with score at least 10. +Write a SpacetimeDB backend module in Rust that defines a procedural view using an index and additional filters. + +TABLE +- content (public), struct Content + - Fields: + - id: u64 (primary key) + - category: String (btree index) + - active: bool + - score: i32 + +REDUCERS +- seed() + - Insert rows that separately fail the category, active, and score checks + - Insert one passing row + +VIEW +- featured_content (procedural, public) + - First use the category index for "news" + - Then return only active rows with score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt index 27ae0bdc4e8..da9cfe624b7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt @@ -1 +1,19 @@ -Write a TypeScript SpacetimeDB module with a public content table containing id: u64 primary key, indexed category: string, active: boolean, and score: i32. Add seed() with rows that separately fail category, active, and score checks plus one passing row. Expose a public featured_content procedural view that first uses the category index for "news", then filters for active rows with score at least 10. +Write a SpacetimeDB backend module in TypeScript that defines a procedural view using an index and additional filters. + +TABLE +- content (public) + - Fields: + - id: u64 (primary key) + - category: string (btree index) + - active: boolean + - score: i32 + +REDUCERS +- seed() + - Insert rows that separately fail the category, active, and score checks + - Insert one passing row + +VIEW +- featured_content (procedural, public) + - First use the category index for "news" + - Then return only active rows with score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt index decc35d6a14..d80a4f173ef 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -1 +1,30 @@ -Write a C# SpacetimeDB module with public Sale(Id: ulong primary key, Category: string, Amount: long) and CategoryTotal(Category: string primary key, TotalAmount: long, SaleCount: ulong) tables. Implement internal shared upsert and delete helpers that keep CategoryTotal transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Export only an Exercise() reducer; it inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose CategoryTotal through a public CategorySummary query-builder view. The final view must contain only books with TotalAmount 25 and SaleCount 1. +Write a SpacetimeDB backend module in C# that maintains and exposes a materialized aggregate. + +TABLES +- Sale (public, accessor Sale) + - Fields: + - Id: ulong (primary key) + - Category: string + - Amount: long +- CategoryTotal (public, accessor CategoryTotal) + - Fields: + - Category: string (primary key) + - TotalAmount: long + - SaleCount: ulong + +HELPERS +- Internal shared upsert and delete helpers + - Keep CategoryTotal transactionally synchronized when a Sale is inserted, changed, moved between categories, or deleted + +REDUCERS +- Exercise() + - This is the only exported reducer + - Insert books Sale rows of 10 and 20 + - Update the second books Sale to 25 + - Insert and then delete a games Sale + - Delete the first books Sale + +VIEW +- CategorySummary (public, query-builder) + - Expose CategoryTotal + - Final result contains only books with TotalAmount 25 and SaleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt index 3179959c75f..7b205201106 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -1 +1,30 @@ -Write a Rust SpacetimeDB module with public sale(id: u64 primary key, category: String, amount: i64) and category_total(category: String primary key, total_amount: i64, sale_count: u64) tables. Implement internal shared upsert and delete helpers that keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Export only an exercise() reducer; it inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with total_amount 25 and sale_count 1. +Write a SpacetimeDB backend module in Rust that maintains and exposes a materialized aggregate. + +TABLES +- sale (public), struct Sale + - Fields: + - id: u64 (primary key) + - category: String + - amount: i64 +- category_total (public), struct CategoryTotal + - Fields: + - category: String (primary key) + - total_amount: i64 + - sale_count: u64 + +HELPERS +- Internal shared upsert and delete helpers + - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted + +REDUCERS +- exercise() + - This is the only exported reducer + - Insert books sales of 10 and 20 + - Update the second books sale to 25 + - Insert and then delete a games sale + - Delete the first books sale + +VIEW +- category_summary (public, query-builder) + - Expose category_total + - Final result contains only books with total_amount 25 and sale_count 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt index 41153d9e415..daccfd20591 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -1 +1,30 @@ -Write a TypeScript SpacetimeDB module with public sale(id: u64 primary key, category: string, amount: i64) and category_total(category: string primary key, totalAmount: i64, saleCount: u64) tables. Implement internal shared upsert and delete helpers that keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted. Export only an exercise() reducer; it inserts books sales of 10 and 20, updates the second to 25, inserts and then deletes a games sale, and deletes the first books sale. Expose category_total through a public category_summary query-builder view. The final view must contain only books with totalAmount 25 and saleCount 1. +Write a SpacetimeDB backend module in TypeScript that maintains and exposes a materialized aggregate. + +TABLES +- sale (public) + - Fields: + - id: u64 (primary key) + - category: string + - amount: i64 +- category_total (public) + - Fields: + - category: string (primary key) + - totalAmount: i64 + - saleCount: u64 + +HELPERS +- Internal shared upsert and delete helpers + - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted + +REDUCERS +- exercise() + - This is the only exported reducer + - Insert books sales of 10 and 20 + - Update the second books sale to 25 + - Insert and then delete a games sale + - Delete the first books sale + +VIEW +- category_summary (public, query-builder) + - Expose category_total + - Final result contains only books with totalAmount 25 and saleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt index de32f779b09..506ddfc7b4c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt @@ -1 +1,17 @@ -Write a C# SpacetimeDB module with a public Ticket table containing Id: ulong primary key, indexed Status: string, and Title: string. Add a Seed() reducer with one open and one closed ticket. Define a public OpenTicket view that returns the query-builder filter for Status "open" rather than materializing rows manually. +Write a SpacetimeDB backend module in C# that defines a query-builder filter view. + +TABLE +- Ticket (public, accessor Ticket) + - Fields: + - Id: ulong (primary key) + - Status: string (btree index) + - Title: string + +REDUCERS +- Seed() + - Insert one open Ticket and one closed Ticket + +VIEW +- OpenTicket (public, query-builder) + - Return the query-builder filter for Status "open" + - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt index 50014678baf..7731fa63708 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt @@ -1 +1,17 @@ -Write a Rust SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: String, and title: String. Add a seed() reducer with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. +Write a SpacetimeDB backend module in Rust that defines a query-builder filter view. + +TABLE +- ticket (public), struct Ticket + - Fields: + - id: u64 (primary key) + - status: String (btree index) + - title: String + +REDUCERS +- seed() + - Insert one open ticket and one closed ticket + +VIEW +- open_ticket (public, query-builder) + - Return the query-builder filter for status "open" + - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt index 32f27b5ce69..a07ad0cbcf7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt @@ -1 +1,17 @@ -Write a TypeScript SpacetimeDB module with a public ticket table containing id: u64 primary key, indexed status: string, and title: string. Add a seed() reducer with one open and one closed ticket. Define a public open_ticket view that returns the query-builder filter for status "open" rather than materializing rows manually. +Write a SpacetimeDB backend module in TypeScript that defines a query-builder filter view. + +TABLE +- ticket (public) + - Fields: + - id: u64 (primary key) + - status: string (btree index) + - title: string + +REDUCERS +- seed() + - Insert one open ticket and one closed ticket + +VIEW +- open_ticket (public, query-builder) + - Return the query-builder filter for status "open" + - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt index 3512de8cd80..9467c5e70c3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt @@ -1 +1,13 @@ -Write a C# SpacetimeDB module with a public SourceRow table containing Id: ulong primary key, Value: string, and indexed Visible: bool. Expose visible rows through a public SourceView procedural view and explicitly declare Id as the view primary key. +Write a SpacetimeDB backend module in C# that defines a procedural view with a primary key. + +TABLE +- SourceRow (public, accessor SourceRow) + - Fields: + - Id: ulong (primary key) + - Value: string + - Visible: bool (btree index) + +VIEW +- SourceView (procedural, public) + - Return visible SourceRow rows + - Explicitly declare Id as the view primary key diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt index 98a8a4aa8eb..89a6317baba 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt @@ -1 +1,13 @@ -Write a Rust SpacetimeDB module with a public source_row table containing id: u64 primary key, value: String, and indexed visible: bool. Expose visible rows through a public source_view procedural view and explicitly declare id as the view primary key. +Write a SpacetimeDB backend module in Rust that defines a procedural view with a primary key. + +TABLE +- source_row (public), struct SourceRow + - Fields: + - id: u64 (primary key) + - value: String + - visible: bool (btree index) + +VIEW +- source_view (procedural, public) + - Return visible source_row rows + - Explicitly declare id as the view primary key diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt index ebc15400c35..48a6c683972 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt @@ -1 +1,13 @@ -Write a TypeScript SpacetimeDB module with a public source_row table containing id: u64 primary key, value: string, and indexed visible: boolean. Expose visible rows through a public source_view procedural view whose returned row type declares id as its primary key. +Write a SpacetimeDB backend module in TypeScript that defines a procedural view with a primary key. + +TABLE +- source_row (public) + - Fields: + - id: u64 (primary key) + - value: string + - visible: boolean (btree index) + +VIEW +- source_view (procedural, public) + - Return visible source_row rows + - The returned row type declares id as its primary key From 5567d916510eb1b2ef1d2b4c92eeda9d88175b5a Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:35:32 -0400 Subject: [PATCH 27/78] Retry transient HTTP eval calls --- .../procedures/t_073_http_fetch/spec.rs | 5 +-- .../procedures/t_074_fetch_and_store/spec.rs | 7 +++-- .../xtask-llm-benchmark/src/eval/defaults.rs | 26 ++++++++++++++++ tools/xtask-llm-benchmark/src/eval/scorers.rs | 31 ++++++++++++++++--- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs index 0abea3ef113..18001ff17ac 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs @@ -1,17 +1,18 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer}; +use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer_with_attempts}; use crate::eval::BenchmarkSpec; use serde_json::json; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_call_output_parity_scorer( + scorers.push(make_call_output_parity_scorer_with_attempts( host_url, file!(), route_tag, "fetch_page_summary", vec![json!("https://example.com")], "http_response_summary", + 3, )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs index ed226ed2319..95ad2472c33 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs @@ -1,4 +1,6 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer_with_attempts, make_sql_count_only_scorer, +}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use serde_json::json; use std::time::Duration; @@ -6,13 +8,14 @@ use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_call_both_scorer( + scorers.push(make_reducer_call_both_scorer_with_attempts( host_url, file!(), route_tag, "fetch_and_store", vec![json!("https://example.com")], "fetch_and_store", + 3, )); let table = table_name("fetched_record", lang); let status = ident("status", casing_for_lang(lang)); diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index 668dba07847..7f3710134cb 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -126,6 +126,18 @@ pub fn make_reducer_call_both_scorer( reducer: &str, args: Vec, id_str: &'static str, +) -> Box { + make_reducer_call_both_scorer_with_attempts(host_url, src_file, route_tag, reducer, args, id_str, 1) +} + +pub fn make_reducer_call_both_scorer_with_attempts( + host_url: &str, + src_file: &str, + route_tag: &str, + reducer: &str, + args: Vec, + id_str: &'static str, + attempts: usize, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); @@ -137,6 +149,7 @@ pub fn make_reducer_call_both_scorer( llm_db, reducer: reducer.to_string(), args, + attempts, id_str, }) as Box } @@ -148,6 +161,18 @@ pub fn make_call_output_parity_scorer( function: &str, args: Vec, id_str: &'static str, +) -> Box { + make_call_output_parity_scorer_with_attempts(host_url, src_file, route_tag, function, args, id_str, 1) +} + +pub fn make_call_output_parity_scorer_with_attempts( + host_url: &str, + src_file: &str, + route_tag: &str, + function: &str, + args: Vec, + id_str: &'static str, + attempts: usize, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); @@ -159,6 +184,7 @@ pub fn make_call_output_parity_scorer( function: function.to_string(), args, collapse_ws: true, + attempts, id_str, }) } diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 23b47e159fd..18dee15adb5 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -833,6 +833,7 @@ pub struct ReducerCallBothScorer { pub llm_db: String, pub reducer: String, pub args: Vec, + pub attempts: usize, pub id_str: &'static str, } @@ -843,9 +844,31 @@ pub struct CallOutputParityScorer { pub function: String, pub args: Vec, pub collapse_ws: bool, + pub attempts: usize, pub id_str: &'static str, } +fn call_with_retries( + db: &str, + function: &str, + args: &[Value], + server: &str, + attempts: usize, +) -> Result { + let attempts = attempts.max(1); + let mut last_error = None; + for attempt in 0..attempts { + match call_reducer_json_out(db, function, args, Some(server)) { + Ok(output) => return Ok(output), + Err(error) => last_error = Some(error), + } + if attempt + 1 < attempts { + thread::sleep(Duration::from_millis(250)); + } + } + Err(last_error.expect("at least one call attempt must run")) +} + pub struct HttpRouteCase { pub method: String, pub path: String, @@ -953,7 +976,7 @@ impl Scorer for CallOutputParityScorer { } fn score(&self, _llm_output: &str) -> ScoreDetails { - let golden = match call_reducer_json_out(&self.golden_db, &self.function, &self.args, Some(&self.server)) { + let golden = match call_with_retries(&self.golden_db, &self.function, &self.args, &self.server, self.attempts) { Ok(output) => output, Err(error) => { return ScoreDetails { @@ -963,7 +986,7 @@ impl Scorer for CallOutputParityScorer { } } }; - let llm = match call_reducer_json_out(&self.llm_db, &self.function, &self.args, Some(&self.server)) { + let llm = match call_with_retries(&self.llm_db, &self.function, &self.args, &self.server, self.attempts) { Ok(output) => output, Err(error) => { return ScoreDetails { @@ -996,14 +1019,14 @@ impl Scorer for ReducerCallBothScorer { self.reducer, self.args, self.golden_db, self.llm_db, self.server ); } - if let Err(e) = call_reducer_json_out(&self.golden_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_with_retries(&self.golden_db, &self.reducer, &self.args, &self.server, self.attempts) { return ScoreDetails { pass: false, partial: 0.0, notes: json!({ "phase":"call_reducer_golden", "error": e, "reducer": self.reducer }), }; } - if let Err(e) = call_reducer_json_out(&self.llm_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_with_retries(&self.llm_db, &self.reducer, &self.args, &self.server, self.attempts) { return ScoreDetails { pass: false, partial: 0.0, From 3c250b9c097bc64f32fbf0d703151beec2bd6e4f Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:52:24 -0400 Subject: [PATCH 28/78] Clarify ambiguous eval task contracts --- .../benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt | 1 + .../benchmarks/reducers/t_058_batched_delete/tasks/rust.txt | 1 + .../reducers/t_058_batched_delete/tasks/typescript.txt | 1 + .../benchmarks/views/t_063_timestamp_window/tasks/csharp.txt | 1 + .../benchmarks/views/t_063_timestamp_window/tasks/rust.txt | 1 + .../views/t_063_timestamp_window/tasks/typescript.txt | 1 + .../views/t_065_materialized_aggregate/tasks/csharp.txt | 4 ++-- .../views/t_065_materialized_aggregate/tasks/rust.txt | 4 ++-- .../views/t_065_materialized_aggregate/tasks/typescript.txt | 4 ++-- 9 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt index cf9d8096d7e..94180f3182d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt @@ -13,6 +13,7 @@ TABLES REDUCERS - SeedGroup(groupId: ulong, count: uint) + - Assign deterministic unique Id values to the inserted rows - RequestDelete(groupId: ulong) - RunDeleteBatch(job: DeleteJob) (scheduled reducer) - Delete at most two WorkItem rows matching the scheduled GroupId diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt index 154e2fea8d5..c88990defec 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt @@ -13,6 +13,7 @@ TABLES REDUCERS - seed_group(group_id: u64, count: u32) + - Assign deterministic unique id values to the inserted rows - request_delete(group_id: u64) - run_delete_batch(job: DeleteJob) (scheduled reducer) - Delete at most two work_item rows matching the scheduled group_id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt index f9d5dd8759c..e34652dfbd1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt @@ -13,6 +13,7 @@ TABLES REDUCERS - seed_group(groupId: u64, count: u32) + - Assign deterministic unique id values to the inserted rows - request_delete(groupId: u64) - runDeleteBatch(timer: delete_job row) (scheduled reducer) - Delete at most two work_item rows matching the scheduled groupId diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt index ce63c66ddbf..87e9f3309aa 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -10,6 +10,7 @@ TABLE REDUCERS - Seed() - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + - Assign Ids 1 through 5 in timestamp order VIEW - WindowEvent (procedural, public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt index 3e2095d944e..e837f83968d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -10,6 +10,7 @@ TABLE REDUCERS - seed() - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + - Assign ids 1 through 5 in timestamp order VIEW - window_event (procedural, public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt index 9f0cef78f51..5edd436dc77 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -10,6 +10,7 @@ TABLE REDUCERS - seed() - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + - Assign ids 1 through 5 in timestamp order VIEW - window_event (procedural, public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt index d80a4f173ef..b2bc80c1b76 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -26,5 +26,5 @@ REDUCERS VIEW - CategorySummary (public, query-builder) - - Expose CategoryTotal - - Final result contains only books with TotalAmount 25 and SaleCount 1 + - Expose all CategoryTotal rows + - After Exercise, the only row is books with TotalAmount 25 and SaleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt index 7b205201106..be026d6c91e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -26,5 +26,5 @@ REDUCERS VIEW - category_summary (public, query-builder) - - Expose category_total - - Final result contains only books with total_amount 25 and sale_count 1 + - Expose all category_total rows + - After exercise, the only row is books with total_amount 25 and sale_count 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt index daccfd20591..fd440865266 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -26,5 +26,5 @@ REDUCERS VIEW - category_summary (public, query-builder) - - Expose category_total - - Final result contains only books with totalAmount 25 and saleCount 1 + - Expose all category_total rows + - After exercise, the only row is books with totalAmount 25 and saleCount 1 From 4a1252ed9c99f9e617c065a5d61a88bf2b0bd73f Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:07:54 -0400 Subject: [PATCH 29/78] Fix reducer schema parity extraction --- tools/xtask-llm-benchmark/src/eval/scorers.rs | 87 +++++++++++++++++-- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 18dee15adb5..63bd3c6c467 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -194,15 +194,28 @@ fn extract_schema( if let Some(rs) = v.get("reducers").and_then(|x| x.as_array()) { for r in rs { let name = r.get("name").and_then(|x| x.as_str()).unwrap_or(""); - let sig = if let Some(args) = r.get("args").and_then(|x| x.as_array()) { - let tys: Vec = args - .iter() - .map(|a| a.get("type").and_then(|x| x.as_str()).unwrap_or("").to_string()) - .collect(); - format!("{}({})", name, tys.join(",")) - } else { - format!("{}()", name) - }; + let params = r + .pointer("/params/elements") + .and_then(Value::as_array) + .or_else(|| r.get("args").and_then(Value::as_array)); + let params = params + .into_iter() + .flatten() + .map(|param| { + let name = schema_name(param.get("name")); + let ty = param + .get("algebraic_type") + .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) + .or_else(|| param.get("type").and_then(Value::as_str).map(str::to_owned)) + .unwrap_or_default(); + if name.is_empty() { + ty + } else { + format!("{name}:{ty}") + } + }) + .collect::>(); + let sig = format!("{}({})", name, params.join(",")); reducers.insert(sig); } } @@ -1114,6 +1127,62 @@ mod tests { assert!(!diff_sets(&golden_rls, &candidate_rls).is_null()); } + #[test] + fn current_schema_extracts_reducer_parameter_names_and_types() { + let mut schema = current_schema(true); + schema["reducers"] = json!([{ + "name": "set_owner", + "params": { + "elements": [ + { "name": { "some": "id" }, "algebraic_type": { "U64": [] } }, + { "name": { "some": "owner" }, "algebraic_type": { "String": [] } } + ] + } + }]); + + let (_, reducers, _) = extract_schema(&schema); + + assert_eq!( + reducers, + BTreeSet::from([r#"set_owner(id:{"U64":[]},owner:{"String":[]})"#.to_owned()]) + ); + } + + #[test] + fn reducer_parameter_type_produces_a_schema_diff() { + let mut golden = current_schema(true); + golden["reducers"] = json!([{ + "name": "set_owner", + "params": { "elements": [ + { "name": { "some": "owner" }, "algebraic_type": { "String": [] } } + ] } + }]); + let mut candidate = current_schema(true); + candidate["reducers"] = json!([{ + "name": "set_owner", + "params": { "elements": [ + { "name": { "some": "owner" }, "algebraic_type": { "U64": [] } } + ] } + }]); + let (_, golden_reducers, _) = extract_schema(&golden); + let (_, candidate_reducers, _) = extract_schema(&candidate); + + assert!(!diff_sets(&golden_reducers, &candidate_reducers).is_null()); + } + + #[test] + fn legacy_reducer_args_remain_supported() { + let mut schema = current_schema(true); + schema["reducers"] = json!([{ + "name": "set_owner", + "args": [{ "name": "owner", "type": "String" }] + }]); + + let (_, reducers, _) = extract_schema(&schema); + + assert_eq!(reducers, BTreeSet::from(["set_owner(owner:String)".to_owned()])); + } + #[test] fn http_route_parity_ignores_unspecified_content_type() { let golden = vec![(201, String::new(), "created".to_string())]; From 081432f7f283b8b19309e39bf3cadf479c8131d4 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:28:53 -0400 Subject: [PATCH 30/78] Strengthen view eval result grading --- .../auth/t_068_secure_projection/spec.rs | 23 +++++++++++------ .../views/t_061_three_table_join/spec.rs | 25 +++++++++++++------ .../t_061_three_table_join/tasks/csharp.txt | 3 +++ .../t_061_three_table_join/tasks/rust.txt | 3 +++ .../tasks/typescript.txt | 3 +++ .../views/t_062_semijoin_intersection/spec.rs | 23 +++++++++++------ .../tasks/csharp.txt | 4 +-- .../tasks/rust.txt | 4 +-- .../tasks/typescript.txt | 4 +-- .../views/t_063_timestamp_window/spec.rs | 23 +++++++++++------ .../t_063_timestamp_window/tasks/csharp.txt | 2 ++ .../t_063_timestamp_window/tasks/rust.txt | 2 ++ .../tasks/typescript.txt | 2 ++ .../views/t_064_index_and_filter/spec.rs | 23 +++++++++++------ .../t_064_index_and_filter/tasks/csharp.txt | 6 +++-- .../t_064_index_and_filter/tasks/rust.txt | 6 +++-- .../tasks/typescript.txt | 6 +++-- .../views/t_066_query_builder_view/spec.rs | 23 +++++++++++------ .../t_066_query_builder_view/tasks/csharp.txt | 3 ++- .../t_066_query_builder_view/tasks/rust.txt | 3 ++- .../tasks/typescript.txt | 3 ++- 21 files changed, 137 insertions(+), 57 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs index a17f6272cb6..0dc215476b1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs @@ -1,15 +1,24 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "seed_private_note".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("my_safe_note", lang)), - expected_count: 1, id_str: "caller_safe_projection", timeout: Duration::from_secs(10), - })); + let view = table_name("my_safe_note", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed_private_note".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "caller_safe_projection", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs index 777e7059caa..8eaee51e5b4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs @@ -1,13 +1,24 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; + pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("order_line_detail", lang)), - expected_count: 1, id_str: "three_table_join", timeout: Duration::from_secs(10), - })); scorers + let view = table_name("order_line_detail", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "three_table_join", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt index 1061f2c5872..099cebe9059 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -19,6 +19,9 @@ TABLES REDUCERS - Seed() - Insert one complete visible Customer -> Purchase -> LineItem chain + - Use Customer Id 1 named "Ada" + - Use Purchase Id 10 for Customer 1 + - Use visible LineItem Id 100 for Purchase 10 with Sku "SKU-1" VIEW - OrderLineDetail (public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt index 1dd2ef4a609..5647ca27dc2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -19,6 +19,9 @@ TABLES REDUCERS - seed() - Insert one complete visible customer -> purchase -> line_item chain + - Use customer id 1 named "Ada" + - Use purchase id 10 for customer 1 + - Use visible line_item id 100 for purchase 10 with sku "SKU-1" VIEW - order_line_detail (public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt index f6f9c10b58c..3ce44511ed1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -19,6 +19,9 @@ TABLES REDUCERS - seed() - Insert one complete visible customer -> purchase -> line_item chain + - Use customer id 1 named "Ada" + - Use purchase id 10 for customer 1 + - Use visible line_item id 100 for purchase 10 with sku "SKU-1" VIEW - order_line_detail (public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs index 28d83f6b87e..c9387e35e2c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs @@ -1,15 +1,24 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("eligible_member", lang)), - expected_count: 1, id_str: "semijoin_intersection", timeout: Duration::from_secs(10), - })); + let view = table_name("eligible_member", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "semijoin_intersection", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt index 5fd5a360c02..c9260a433ca 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt @@ -12,8 +12,8 @@ TABLES REDUCERS - Seed() - - Insert two Member rows - - Insert an Eligibility row for only member 1 + - Insert Member 1 named "Ada" and Member 2 named "Grace" + - Insert Eligibility Id 10 for only Member 1 VIEW - EligibleMember (public, query-builder) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt index 7d3ec0b6277..e3e7bb08817 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt @@ -12,8 +12,8 @@ TABLES REDUCERS - seed() - - Insert two members - - Insert an eligibility row for only member 1 + - Insert member 1 named "Ada" and member 2 named "Grace" + - Insert eligibility id 10 for only member 1 VIEW - eligible_member (public, query-builder) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt index 3e6481118bb..83d299ae021 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt @@ -12,8 +12,8 @@ TABLES REDUCERS - seed() - - Insert two member rows - - Insert an eligibility row for only member 1 + - Insert member 1 named "Ada" and member 2 named "Grace" + - Insert eligibility id 10 for only member 1 VIEW - eligible_member (public, query-builder) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs index 441010dcad0..cde104ab4d4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs @@ -1,15 +1,24 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("window_event", lang)), - expected_count: 3, id_str: "inclusive_timestamp_window", timeout: Duration::from_secs(10), - })); + let view = table_name("window_event", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "inclusive_timestamp_window", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt index 87e9f3309aa..82a3582c809 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -11,8 +11,10 @@ REDUCERS - Seed() - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch - Assign Ids 1 through 5 in timestamp order + - Set each Label to "event-{microseconds}", such as "event-100" VIEW - WindowEvent (procedural, public) - Use the OccurredAt index - Return the inclusive range from 200 through 400 microseconds + - Return rows in ascending OccurredAt order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt index e837f83968d..a0fbb124da6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -11,8 +11,10 @@ REDUCERS - seed() - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch - Assign ids 1 through 5 in timestamp order + - Set each label to "event-{microseconds}", such as "event-100" VIEW - window_event (procedural, public) - Use the occurred_at index - Return the inclusive range from 200 through 400 microseconds + - Return rows in ascending occurred_at order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt index 5edd436dc77..92abe30fa49 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -11,8 +11,10 @@ REDUCERS - seed() - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch - Assign ids 1 through 5 in timestamp order + - Set each label to "event-{microseconds}", such as "event-100" VIEW - window_event (procedural, public) - Use the occurredAt index - Return the inclusive range from 200 through 400 microseconds + - Return rows in ascending occurredAt order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs index 92ef8e979bf..3df3dbfd85e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs @@ -1,15 +1,24 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("featured_content", lang)), - expected_count: 1, id_str: "indexed_candidates_are_filtered", timeout: Duration::from_secs(10), - })); + let view = table_name("featured_content", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "indexed_candidates_are_filtered", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt index 22ff4612518..875da4600ac 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt @@ -10,8 +10,10 @@ TABLE REDUCERS - Seed() - - Insert rows that separately fail the Category, Active, and Score checks - - Insert one passing row + - Insert Id 1 with Category "news", Active true, and Score 20 + - Insert Id 2 with Category "news", Active false, and Score 20 + - Insert Id 3 with Category "news", Active true, and Score 5 + - Insert Id 4 with Category "sports", Active true, and Score 20 VIEW - FeaturedContent (procedural, public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt index f28bd04a3ad..f77f2979310 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt @@ -10,8 +10,10 @@ TABLE REDUCERS - seed() - - Insert rows that separately fail the category, active, and score checks - - Insert one passing row + - Insert id 1 with category "news", active true, and score 20 + - Insert id 2 with category "news", active false, and score 20 + - Insert id 3 with category "news", active true, and score 5 + - Insert id 4 with category "sports", active true, and score 20 VIEW - featured_content (procedural, public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt index da9cfe624b7..eccb7e9e2c2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt @@ -10,8 +10,10 @@ TABLE REDUCERS - seed() - - Insert rows that separately fail the category, active, and score checks - - Insert one passing row + - Insert id 1 with category "news", active true, and score 20 + - Insert id 2 with category "news", active false, and score 20 + - Insert id 3 with category "news", active true, and score 5 + - Insert id 4 with category "sports", active true, and score 20 VIEW - featured_content (procedural, public) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs index ae71428264b..d49c659b8fd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs @@ -1,15 +1,24 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "seed".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("open_ticket", lang)), - expected_count: 1, id_str: "query_builder_filter", timeout: Duration::from_secs(10), - })); + let view = table_name("open_ticket", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "query_builder_filter", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt index 506ddfc7b4c..9d3c3ad556a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt @@ -9,7 +9,8 @@ TABLE REDUCERS - Seed() - - Insert one open Ticket and one closed Ticket + - Insert open Ticket Id 1 titled "A" + - Insert closed Ticket Id 2 titled "B" VIEW - OpenTicket (public, query-builder) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt index 7731fa63708..f309318db3c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt @@ -9,7 +9,8 @@ TABLE REDUCERS - seed() - - Insert one open ticket and one closed ticket + - Insert open ticket id 1 titled "A" + - Insert closed ticket id 2 titled "B" VIEW - open_ticket (public, query-builder) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt index a07ad0cbcf7..8b81902ef3e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt @@ -9,7 +9,8 @@ TABLE REDUCERS - seed() - - Insert one open ticket and one closed ticket + - Insert open ticket id 1 titled "A" + - Insert closed ticket id 2 titled "B" VIEW - open_ticket (public, query-builder) From d39578fc9799368f89cc3588205dd38688e7eeec Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 14:36:42 -0400 Subject: [PATCH 31/78] Ignore reducer parameter names in schema parity --- tools/xtask-llm-benchmark/src/eval/scorers.rs | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 63bd3c6c467..35fa4adcf16 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -202,17 +202,11 @@ fn extract_schema( .into_iter() .flatten() .map(|param| { - let name = schema_name(param.get("name")); - let ty = param + param .get("algebraic_type") .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) .or_else(|| param.get("type").and_then(Value::as_str).map(str::to_owned)) - .unwrap_or_default(); - if name.is_empty() { - ty - } else { - format!("{name}:{ty}") - } + .unwrap_or_default() }) .collect::>(); let sig = format!("{}({})", name, params.join(",")); @@ -1128,7 +1122,7 @@ mod tests { } #[test] - fn current_schema_extracts_reducer_parameter_names_and_types() { + fn current_schema_extracts_reducer_parameter_types() { let mut schema = current_schema(true); schema["reducers"] = json!([{ "name": "set_owner", @@ -1144,7 +1138,7 @@ mod tests { assert_eq!( reducers, - BTreeSet::from([r#"set_owner(id:{"U64":[]},owner:{"String":[]})"#.to_owned()]) + BTreeSet::from([r#"set_owner({"U64":[]},{"String":[]})"#.to_owned()]) ); } @@ -1170,6 +1164,28 @@ mod tests { assert!(!diff_sets(&golden_reducers, &candidate_reducers).is_null()); } + #[test] + fn reducer_parameter_names_do_not_produce_a_schema_diff() { + let mut golden = current_schema(true); + golden["reducers"] = json!([{ + "name": "send_reminder", + "params": { "elements": [ + { "name": { "some": "timer" }, "algebraic_type": { "U64": [] } } + ] } + }]); + let mut candidate = current_schema(true); + candidate["reducers"] = json!([{ + "name": "send_reminder", + "params": { "elements": [ + { "name": { "some": "reminder" }, "algebraic_type": { "U64": [] } } + ] } + }]); + let (_, golden_reducers, _) = extract_schema(&golden); + let (_, candidate_reducers, _) = extract_schema(&candidate); + + assert!(diff_sets(&golden_reducers, &candidate_reducers).is_null()); + } + #[test] fn legacy_reducer_args_remain_supported() { let mut schema = current_schema(true); @@ -1180,7 +1196,7 @@ mod tests { let (_, reducers, _) = extract_schema(&schema); - assert_eq!(reducers, BTreeSet::from(["set_owner(owner:String)".to_owned()])); + assert_eq!(reducers, BTreeSet::from(["set_owner(String)".to_owned()])); } #[test] From 97ba8afd9b7ca9320d324c11375ee567d6f2bd99 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 14:36:55 -0400 Subject: [PATCH 32/78] Clarify private table eval contracts --- .../src/benchmarks/basics/t_010_connect/tasks/csharp.txt | 2 +- .../src/benchmarks/basics/t_010_connect/tasks/rust.txt | 2 +- .../benchmarks/basics/t_010_connect/tasks/typescript.txt | 4 ++-- .../data_modeling/t_025_optional_fields/tasks/csharp.txt | 2 +- .../data_modeling/t_025_optional_fields/tasks/rust.txt | 2 +- .../t_025_optional_fields/tasks/typescript.txt | 2 +- .../data_modeling/t_028_cascade_delete/tasks/csharp.txt | 4 ++-- .../data_modeling/t_028_cascade_delete/tasks/rust.txt | 4 ++-- .../data_modeling/t_028_cascade_delete/tasks/typescript.txt | 4 ++-- .../data_modeling/t_030_two_table_join/tasks/csharp.txt | 6 +++--- .../data_modeling/t_030_two_table_join/tasks/rust.txt | 6 +++--- .../data_modeling/t_030_two_table_join/tasks/typescript.txt | 6 +++--- .../benchmarks/queries/t_032_range_query/tasks/csharp.txt | 4 ++-- .../src/benchmarks/queries/t_032_range_query/tasks/rust.txt | 4 ++-- .../queries/t_032_range_query/tasks/typescript.txt | 4 ++-- .../queries/t_033_sort_and_limit/tasks/csharp.txt | 4 ++-- .../benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt | 4 ++-- .../queries/t_033_sort_and_limit/tasks/typescript.txt | 4 ++-- .../benchmarks/queries/t_034_find_first/tasks/csharp.txt | 4 ++-- .../src/benchmarks/queries/t_034_find_first/tasks/rust.txt | 4 ++-- .../queries/t_034_find_first/tasks/typescript.txt | 4 ++-- .../queries/t_035_select_distinct/tasks/csharp.txt | 4 ++-- .../benchmarks/queries/t_035_select_distinct/tasks/rust.txt | 4 ++-- .../queries/t_035_select_distinct/tasks/typescript.txt | 4 ++-- .../queries/t_036_count_without_collect/tasks/csharp.txt | 4 ++-- .../queries/t_036_count_without_collect/tasks/rust.txt | 4 ++-- .../t_036_count_without_collect/tasks/typescript.txt | 4 ++-- .../queries/t_037_multi_column_filter/tasks/csharp.txt | 4 ++-- .../queries/t_037_multi_column_filter/tasks/rust.txt | 4 ++-- .../queries/t_037_multi_column_filter/tasks/typescript.txt | 4 ++-- 30 files changed, 58 insertions(+), 58 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt index e00dca47a3c..add0113a306 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines one table and two reducers for client lifecycle events. TABLE -- Event +- Event (private) - Struct: Event - Fields: - Id: int (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt index a2fea3f3092..e3497bdd5cb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines one table and two reducers for client lifecycle events. TABLE -- event +- event (private) - Struct: Event - Fields: - id: i32 (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt index 21b4a04b2c4..a9a80b3415d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt @@ -1,11 +1,11 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and two reducers for client lifecycle events. TABLE -- event +- event (private) - Fields: - id: bigint (u64, primary key, auto-increment) - kind: string LIFECYCLE HOOKS (use spacetimedb.clientConnected / spacetimedb.clientDisconnected) - onConnect: insert event with kind="connected" -- onDisconnect: insert event with kind="disconnected" \ No newline at end of file +- onDisconnect: insert event with kind="disconnected" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt index 04c9a2a10b9..371545e5e4b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. TABLES -- Player +- Player (private) - Struct: Player - Fields: - Id: ulong (primary key, AutoInc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt index ac9b1d00a50..165c6d3426f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. TABLES -- player +- player (private) - Struct: Player - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt index 48712f8abea..9960fb64f79 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. TABLES -- player +- player (private) - Fields: - id: number (u64, primary key, autoInc) - name: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt index c3d32fd1d12..9ed82d8e504 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in C# that defines an Author table and a Post table with a foreign key, and a reducer that deletes an author along with all their posts. TABLES -- Author +- Author (private) - Struct: Author - Fields: - Id: ulong (primary key, AutoInc) - Name: string -- Post +- Post (private) - Struct: Post - Fields: - Id: ulong (primary key, AutoInc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt index 8a3c25ba08a..14da8c93bc5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in Rust that defines an author table and a post table with a foreign key, and a reducer that deletes an author along with all their posts. TABLES -- author +- author (private) - Struct: Author - Fields: - id: u64 (primary key, auto_inc) - name: String -- post +- post (private) - Struct: Post - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt index 5cb6d02e7b6..9436daa1913 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt @@ -1,12 +1,12 @@ Write a SpacetimeDB backend module in TypeScript that defines an author table and a post table with a foreign key, and a reducer that deletes an author along with all their posts. TABLES -- author +- author (private) - Fields: - id: number (u64, primary key, autoInc) - name: string -- post +- post (private) - Fields: - id: number (u64, primary key, autoInc) - authorId: number (u64, index btree) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt index 6dfd47c6580..4551a920adf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in C# that defines a Customer table, an Order table with a foreign key to Customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. TABLES -- Customer +- Customer (private) - Struct: Customer - Fields: - Id: ulong (primary key, AutoInc) - Name: string -- Order +- Order (private) - Struct: Order - Fields: - Id: ulong (primary key, AutoInc) @@ -15,7 +15,7 @@ TABLES - Product: string - Amount: uint -- OrderDetail +- OrderDetail (private) - Struct: OrderDetail - Fields: - OrderId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt index 21170c6e3c1..dd5a5c10564 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in Rust that defines a customer table, an order table with a foreign key to customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. TABLES -- customer +- customer (private) - Struct: Customer - Fields: - id: u64 (primary key, auto_inc) - name: String -- order +- order (private) - Struct: Order - Fields: - id: u64 (primary key, auto_inc) @@ -15,7 +15,7 @@ TABLES - product: String - amount: u32 -- order_detail +- order_detail (private) - Struct: OrderDetail - Fields: - order_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt index 589938a7566..3cf12d4233d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt @@ -1,19 +1,19 @@ Write a SpacetimeDB backend module in TypeScript that defines a customer table, an order table with a foreign key to customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. TABLES -- customer +- customer (private) - Fields: - id: number (u64, primary key, autoInc) - name: string -- order +- order (private) - Fields: - id: number (u64, primary key, autoInc) - customerId: number (u64, index btree) - product: string - amount: number (u32) -- order_detail +- order_detail (private) - Fields: - orderId: number (u64, primary key) - customerName: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt index 03b091f6fe2..3d88dc5c931 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a Product table with a BTree-indexed Price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. TABLES -- Product +- Product (private) - Struct: Product - Fields: - Id: ulong (primary key, AutoInc) - Name: string - Price: uint (index BTree) -- PriceRangeResult +- PriceRangeResult (private) - Struct: PriceRangeResult - Fields: - ProductId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt index 7396da179b4..48d8f2f6f3f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a product table with a BTree-indexed price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. TABLES -- product +- product (private) - Struct: Product - Fields: - id: u64 (primary key, auto_inc) - name: String - price: u32 (index btree) -- price_range_result +- price_range_result (private) - Struct: PriceRangeResult - Fields: - product_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt index fdc20994867..0faeeb9f4d3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a product table with a BTree-indexed price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. TABLES -- product +- product (private) - Fields: - id: number (u64, primary key, autoInc) - name: string - price: number (u32, index btree) -- price_range_result +- price_range_result (private) - Fields: - productId: number (u64, primary key) - name: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt index f59eca1f12c..d8b797ed2af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a Player table with scores, a Leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. TABLES -- Player +- Player (private) - Struct: Player - Fields: - Id: ulong (primary key, AutoInc) - Name: string - Score: ulong -- Leaderboard +- Leaderboard (private) - Struct: LeaderboardEntry - Fields: - Rank: uint (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt index cfc38860762..3a6ea9986f8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a player table with scores, a leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. TABLES -- player +- player (private) - Struct: Player - Fields: - id: u64 (primary key, auto_inc) - name: String - score: u64 -- leaderboard +- leaderboard (private) - Struct: LeaderboardEntry - Fields: - rank: u32 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt index d888f39528e..83e8368d6af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a player table with scores, a leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. TABLES -- player +- player (private) - Fields: - id: number (u64, primary key, autoInc) - name: string - score: number (u64) -- leaderboard +- leaderboard (private) - Fields: - rank: number (u32, primary key) - playerName: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt index c89d54ed9a6..59fe2f4a20e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a Task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. TABLES -- Task +- Task (private) - Struct: Task - Fields: - Id: ulong (primary key, AutoInc) - Title: string - Completed: bool -- FirstIncomplete +- FirstIncomplete (private) - Struct: FirstIncomplete - Fields: - TaskId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt index dec8df0b43e..3fe727c5b2b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. TABLES -- task +- task (private) - Struct: Task - Fields: - id: u64 (primary key, auto_inc) - title: String - completed: bool -- first_incomplete +- first_incomplete (private) - Struct: FirstIncomplete - Fields: - task_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt index 940e400a22e..59f899d0829 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. TABLES -- task +- task (private) - Fields: - id: number (u64, primary key, autoInc) - title: string - completed: boolean -- first_incomplete +- first_incomplete (private) - Fields: - taskId: number (u64, primary key) - title: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt index 5ac96c35a88..0fb60362dc5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines an Order table and a result table, with a reducer that collects all distinct Category values from orders into the result table. TABLES -- Order +- Order (private) - Struct: Order - Fields: - Id: ulong (primary key, AutoInc) - Category: string - Amount: uint -- DistinctCategory +- DistinctCategory (private) - Struct: DistinctCategory - Fields: - Category: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt index bc7d9e915cd..44eee546d60 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines an order table and a result table, with a reducer that collects all distinct category values from orders into the result table. TABLES -- order +- order (private) - Struct: Order - Fields: - id: u64 (primary key, auto_inc) - category: String - amount: u32 -- distinct_category +- distinct_category (private) - Struct: DistinctCategory - Fields: - category: String (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt index fe4cd415ed6..809d808c5d5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines an order table and a result table, with a reducer that collects all distinct category values from orders into the result table. TABLES -- order +- order (private) - Fields: - id: number (u64, primary key, autoInc) - category: string - amount: number (u32) -- distinct_category +- distinct_category (private) - Fields: - category: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt index 1508c532fc8..693d4d3faee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a User table and a stats table, with a reducer that counts total users and active users without collecting into a list. TABLES -- User +- User (private) - Struct: User - Fields: - Id: ulong (primary key, AutoInc) - Name: string - Active: bool -- UserStats +- UserStats (private) - Struct: UserStats - Fields: - Key: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt index 24c987810d7..f6a3b3dfbfb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a user table and a stats table, with a reducer that counts total users and active users without collecting into a Vec. TABLES -- user +- user (private) - Struct: User - Fields: - id: u64 (primary key, auto_inc) - name: String - active: bool -- user_stats +- user_stats (private) - Struct: UserStats - Fields: - key: String (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt index 83d9957a584..6018961e127 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a user table and a stats table, with a reducer that counts total users and active users without collecting into an array. TABLES -- user +- user (private) - Fields: - id: number (u64, primary key, autoInc) - name: string - active: boolean -- user_stats +- user_stats (private) - Fields: - key: string (primary key) - count: number (u64) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt index c1f49d42fac..7aefcaaea5b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines an EventLog table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. TABLES -- EventLog +- EventLog (private) - Struct: EventLog - Fields: - Id: ulong (primary key, AutoInc) @@ -10,7 +10,7 @@ TABLES - Message: string - Index: multi-column btree index on (Category, Severity) -- FilteredEvent +- FilteredEvent (private) - Struct: FilteredEvent - Fields: - EventId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt index 88516a7ab47..2e7ea47806f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines an event log table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. TABLES -- event_log +- event_log (private) - Struct: EventLog - Fields: - id: u64 (primary key, auto_inc) @@ -10,7 +10,7 @@ TABLES - message: String - Index: multi-column btree index on (category, severity) -- filtered_event +- filtered_event (private) - Struct: FilteredEvent - Fields: - event_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt index bf9a3845053..70ddcea8163 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines an event log table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. TABLES -- event_log +- event_log (private) - Fields: - id: number (u64, primary key, autoInc) - category: string @@ -9,7 +9,7 @@ TABLES - message: string - Index: multi-column btree index on (category, severity) -- filtered_event +- filtered_event (private) - Fields: - eventId: number (u64, primary key) - message: string From 4af3acd154d99bb34047482490e82063b17e6a04 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 14:54:08 -0400 Subject: [PATCH 33/78] Clarify aggregate table visibility --- .../data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt | 4 ++-- .../data_modeling/t_029_filter_and_aggregate/tasks/rust.txt | 4 ++-- .../t_029_filter_and_aggregate/tasks/typescript.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt index 2f761d52a22..8a84335cd07 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines an Order table and a CategoryStats table, with a reducer that computes aggregate statistics from filtered orders. TABLES -- Order +- Order (private) - Struct: Order - Fields: - Id: ulong (primary key, AutoInc) @@ -9,7 +9,7 @@ TABLES - Amount: ulong - Fulfilled: bool -- CategoryStats +- CategoryStats (private) - Struct: CategoryStats - Fields: - Category: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt index ebe29cf6a76..35cb98d38aa 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines an order table and a stats table, with a reducer that computes aggregate statistics from filtered orders. TABLES -- order +- order (private) - Struct: Order - Fields: - id: u64 (primary key, auto_inc) @@ -9,7 +9,7 @@ TABLES - amount: u64 - fulfilled: bool -- category_stats +- category_stats (private) - Struct: CategoryStats - Fields: - category: String (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt index aad4fbc62d5..c3b1d7cb066 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in TypeScript that defines an order table and a stats table, with a reducer that computes aggregate statistics from filtered orders. TABLES -- order +- order (private) - Fields: - id: number (u64, primary key, autoInc) - category: string (index btree) - amount: number (u64) - fulfilled: boolean -- category_stats +- category_stats (private) - Fields: - category: string (primary key) - totalAmount: number (u64) From 7958c4990dee687997ee098996951234fad586cf Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 14:57:42 -0400 Subject: [PATCH 34/78] Allow complete benchmark module responses --- tools/xtask-llm-benchmark/src/llm/segmentation.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/llm/segmentation.rs b/tools/xtask-llm-benchmark/src/llm/segmentation.rs index 26bc481e52f..8597a470db5 100644 --- a/tools/xtask-llm-benchmark/src/llm/segmentation.rs +++ b/tools/xtask-llm-benchmark/src/llm/segmentation.rs @@ -193,12 +193,13 @@ pub fn xai_ctx_limit_tokens(model: &str) -> usize { 128_000 } -/// Desired output tokens (planning only). Env override: `LLM_DESIRED_OUTPUT_TOKENS` (usize). +/// Desired output tokens for context planning and bounded provider requests. +/// Env override: `LLM_DESIRED_OUTPUT_TOKENS` (usize). pub fn desired_output_tokens() -> usize { std::env::var("LLM_DESIRED_OUTPUT_TOKENS") .ok() .and_then(|s| s.parse::().ok()) - .unwrap_or(1500) + .unwrap_or(4096) } /// Static headroom from env, with sensible defaults and 500-token rounding. From 2d009800cb82fa23ca487d13b19eba4d5744c50b Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 15:02:35 -0400 Subject: [PATCH 35/78] Avoid implicit response caps for optional providers --- tools/xtask-llm-benchmark/src/llm/clients/meta.rs | 4 ++-- .../src/llm/clients/openrouter.rs | 5 ++--- tools/xtask-llm-benchmark/src/llm/segmentation.rs | 12 ++++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 4744361df73..05ea663019e 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -5,7 +5,7 @@ use super::http::HttpClient; use super::oa_compat::OACompatResp; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ - desired_output_tokens, deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, + deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; use crate::llm::types::{LlmOutput, Vendor}; @@ -85,7 +85,7 @@ impl MetaLlamaClient { messages, temperature: 0.0, top_p: None, - max_tokens: Some(desired_output_tokens().max(1) as u32), + max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; // Auth only; optional OpenRouter headers can live in HttpClient if desired diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index 8e8642ada0b..df4c062c1bb 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -6,7 +6,7 @@ use super::http::HttpClient; use super::oa_compat::OACompatResp; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ - desired_output_tokens, deterministic_trim_prefix, non_context_reserve_tokens_env, Segment, + deterministic_trim_prefix, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; use crate::llm::types::{LlmOutput, Vendor}; @@ -218,13 +218,12 @@ impl OpenRouterClient { }); } - let max_tokens = desired_output_tokens().max(1) as u32; let req = Req { model, messages, temperature: 0.0, top_p: None, - max_tokens: Some(max_tokens), + max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/segmentation.rs b/tools/xtask-llm-benchmark/src/llm/segmentation.rs index 8597a470db5..8d8a07ed060 100644 --- a/tools/xtask-llm-benchmark/src/llm/segmentation.rs +++ b/tools/xtask-llm-benchmark/src/llm/segmentation.rs @@ -193,13 +193,17 @@ pub fn xai_ctx_limit_tokens(model: &str) -> usize { 128_000 } -/// Desired output tokens for context planning and bounded provider requests. -/// Env override: `LLM_DESIRED_OUTPUT_TOKENS` (usize). -pub fn desired_output_tokens() -> usize { +/// Explicit output-token limit for providers that accept an optional cap. +pub fn output_token_limit_env() -> Option { std::env::var("LLM_DESIRED_OUTPUT_TOKENS") .ok() .and_then(|s| s.parse::().ok()) - .unwrap_or(4096) +} + +/// Desired output tokens for context planning and providers that require a cap. +/// Env override: `LLM_DESIRED_OUTPUT_TOKENS` (usize). +pub fn desired_output_tokens() -> usize { + output_token_limit_env().unwrap_or(4096) } /// Static headroom from env, with sensible defaults and 500-token rounding. From e040c8d121ff7776b8da26c3ca64eabe30b19a4d Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 15:16:56 -0400 Subject: [PATCH 36/78] Clarify private schema and scheduled exports --- skills/typescript-server/SKILL.md | 2 ++ .../data_modeling/t_031_unique_constraint/tasks/csharp.txt | 2 +- .../data_modeling/t_031_unique_constraint/tasks/rust.txt | 2 +- .../data_modeling/t_031_unique_constraint/tasks/typescript.txt | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index e268bf9aeb7..7f735fb3d2a 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -201,6 +201,8 @@ ctx.db.shipment.deliverBy.filter(new Range( ## Scheduled Tables +The reducer or procedure referenced by a table's `scheduled` option must be exported. + ```typescript import { ScheduleAt } from 'spacetimedb'; // ScheduleAt comes from the root package diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt index 433a4df53aa..f71c512fbe3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. TABLES -- Account +- Account (private) - Struct: Account - Fields: - Id: ulong (primary key, AutoInc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt index 2482f76d578..ab63be2c3a1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. TABLES -- account +- account (private) - Struct: Account - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt index b1e90089c63..9f0d668afde 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. TABLES -- account +- account (private) - Fields: - id: number (u64, primary key, autoInc) - email: string (unique) From ccf14deda23471ad7282a380de19ed1530c46cb4 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 16:20:45 -0400 Subject: [PATCH 37/78] Retry silent Rust compiler exits --- .../src/bench/publishers.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index ba2e38d8f81..f3546932a14 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -204,6 +204,18 @@ fn signal_killed_by(_status: &std::process::ExitStatus) -> Option { fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let combined = format!("{stderr}{stdout}"); let diagnostic = combined.to_ascii_lowercase(); + let silent_rustc_exit = diagnostic.contains("process didn't exit successfully") + && diagnostic.contains("--crate-name") + && diagnostic.contains("(exit code: 1)") + && !diagnostic.lines().any(|line| { + let line = line.trim_start(); + line.starts_with("error[") + || line.strip_prefix("error:").is_some_and(|message| { + let message = message.trim_start(); + !message.starts_with("could not compile `") + && !message.starts_with("process didn't exit successfully") + }) + }); // "Pipe is broken" errors from WASI SDK parallel builds combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") @@ -215,6 +227,9 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { || (combined.contains("MSB3073") && combined.contains("exited with code 2")) // dotnet can crash below spacetime while spacetime exits 1. || combined.contains("code Result<()> { From d188c8f49c4ec45fc0ee4176291366ad95ef9304 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:35:27 -0400 Subject: [PATCH 38/78] Align Rust lifecycle prompt with golden schema --- .../src/benchmarks/basics/t_010_connect/tasks/rust.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt index e3497bdd5cb..e4dcbf0da68 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt @@ -4,7 +4,7 @@ TABLE - event (private) - Struct: Event - Fields: - - id: i32 (primary key, auto-increment) + - id: u64 (primary key, auto-increment) - kind: String REDUCERS From ef4b8b14801ef83e8d06a6ec14db6cc187e03f45 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:46:50 -0400 Subject: [PATCH 39/78] Retry diagnostic-free Cargo exits --- .../src/bench/publishers.rs | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index f3546932a14..ea92aead7b1 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -204,18 +204,23 @@ fn signal_killed_by(_status: &std::process::ExitStatus) -> Option { fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let combined = format!("{stderr}{stdout}"); let diagnostic = combined.to_ascii_lowercase(); + let has_actionable_compiler_diagnostic = diagnostic.lines().any(|line| { + let line = line.trim_start(); + line.starts_with("error[") + || line.strip_prefix("error:").is_some_and(|message| { + let message = message.trim_start(); + !message.starts_with("could not compile `") + && !message.starts_with("process didn't exit successfully") + && !message.starts_with("command [\"cargo\", \"build\"") + }) + }); let silent_rustc_exit = diagnostic.contains("process didn't exit successfully") && diagnostic.contains("--crate-name") && diagnostic.contains("(exit code: 1)") - && !diagnostic.lines().any(|line| { - let line = line.trim_start(); - line.starts_with("error[") - || line.strip_prefix("error:").is_some_and(|message| { - let message = message.trim_start(); - !message.starts_with("could not compile `") - && !message.starts_with("process didn't exit successfully") - }) - }); + && !has_actionable_compiler_diagnostic; + let silent_cargo_exit = diagnostic.contains("error: command [\"cargo\", \"build\"") + && diagnostic.contains("exited with code 1") + && !has_actionable_compiler_diagnostic; // "Pipe is broken" errors from WASI SDK parallel builds combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") @@ -230,6 +235,7 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { // Cargo sometimes reports only a failed rustc command with no compiler // diagnostic. This is a transient child-process failure, not bad source. || silent_rustc_exit + || silent_cargo_exit // A child process can occasionally disappear on Windows after login but // before the CLI emits a compiler/MSBuild diagnostic. Retrying is safe: // benchmark publishes always use their own temporary CLI root and database. @@ -273,6 +279,14 @@ mod tests { "" )); } + + #[test] + fn retries_silent_cargo_exit() { + assert!(is_transient_build_error( + "Error: command [\"cargo\", \"build\", \"--release\"] exited with code 1", + "" + )); + } } fn run(cmd: &mut Command, label: &str) -> Result<()> { From 9e7607f9af9cab81dffb947dcb1c7eda9960243c Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:57:21 -0400 Subject: [PATCH 40/78] Fix Rust external upload golden capture --- .../procedures/t_079_external_upload_flow/answers/rust.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs index c698fccb704..87febdcab04 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs @@ -19,6 +19,6 @@ pub fn upload_and_register(ctx: &mut ProcedureContext, upload_url: String, data: let response = ctx.http.send(request).expect("upload failed"); assert!(response.status().is_success(), "upload failed: {}", response.status()); let row_url = upload_url.clone(); - ctx.with_tx(|tx| { tx.db.uploaded_asset().insert(UploadedAsset { id: 1, url: row_url, size: data.len() as u64 }); }); + ctx.with_tx(|tx| { tx.db.uploaded_asset().insert(UploadedAsset { id: 1, url: row_url.clone(), size: data.len() as u64 }); }); upload_url } From c699171edbd01bb413c992cc51132f0811997dc2 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 18:03:56 -0400 Subject: [PATCH 41/78] Clarify Rust lifecycle and HTTP APIs --- skills/rust-server/SKILL.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index 49b618fdc54..a7106b15ea4 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -149,6 +149,8 @@ pub fn on_connect(ctx: &ReducerContext) { ... } pub fn on_disconnect(ctx: &ReducerContext) { ... } ``` +The current connection ID is available through `ctx.connection_id()` (not a public field) and may be absent outside connection-scoped calls. + ## Views ```rust @@ -277,11 +279,24 @@ pub fn refresh_cache(ctx: &mut ProcedureContext, url: String) { let response = ctx.http.get(url).expect("request failed"); let status = response.status().as_u16(); ctx.with_tx(|tx| { - tx.db.cache_entry().insert(CacheEntry { key: url, status }); + tx.db.cache_entry().insert(CacheEntry { key: url.clone(), status }); }); } ``` +`with_tx` callbacks implement `Fn`; clone captured owned values when storing them rather than moving them out of the closure. + +Consume an HTTP response body as text with `response.into_body().into_string_lossy()`. For methods other than the available convenience calls, build a request and use `send`: + +```rust +let request = Request::builder() + .method("POST") + .uri(url) + .body(Body::from_bytes(data)) + .unwrap(); +let response = ctx.http.send(request).expect("request failed"); +``` + Scheduled procedures use a normal scheduled table, but the scheduled function is marked `#[procedure]` and receives `&mut ProcedureContext` plus the scheduled row. Inbound HTTP uses handler functions and one router: From 414e2b1ec56c76b87dbb424221783d6c9261fe33 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:39:58 -0400 Subject: [PATCH 42/78] Generalize LLM eval guidance and strengthen checks --- skills/csharp-server/SKILL.md | 42 ++++------- skills/rust-server/SKILL.md | 52 +++++--------- skills/typescript-server/SKILL.md | 71 +++++-------------- .../t_001_basic_tables/tasks/csharp.txt | 6 +- .../basics/t_001_basic_tables/tasks/rust.txt | 6 +- .../t_001_basic_tables/tasks/typescript.txt | 8 +-- .../answers/csharp.cs | 2 +- .../answers/rust.rs | 2 +- .../answers/typescript.ts | 4 +- .../t_069_scheduled_materialization/spec.rs | 31 ++++++-- .../tasks/csharp.txt | 4 +- .../tasks/rust.txt | 4 +- .../tasks/typescript.txt | 4 +- .../t_070_connection_scoped_presence/spec.rs | 33 ++++++--- .../tasks/csharp.txt | 9 +-- .../tasks/rust.txt | 9 +-- .../tasks/typescript.txt | 9 +-- .../answers/csharp.cs | 2 +- .../answers/rust.rs | 3 +- .../answers/typescript.ts | 2 +- .../t_059_deterministic_context/spec.rs | 22 ++++-- .../t_052_autoinc_reference/tasks/csharp.txt | 4 +- .../t_052_autoinc_reference/tasks/rust.txt | 4 +- .../tasks/typescript.txt | 4 +- .../t_061_three_table_join/tasks/csharp.txt | 2 +- .../t_061_three_table_join/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 5 +- .../tasks/rust.txt | 5 +- .../tasks/typescript.txt | 5 +- .../xtask-llm-benchmark/src/eval/defaults.rs | 20 ++++++ tools/xtask-llm-benchmark/src/eval/scorers.rs | 38 ++++++++++ 32 files changed, 218 insertions(+), 198 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 074ac58b566..bbceda3dfa8 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -176,19 +176,14 @@ public static Entity? MyProfile(ViewContext ctx) } ``` -Query-builder views use `ViewContext`, `ctx.From`, and return `IQuery` directly: +Query-builder views use `ViewContext`, `ctx.From`, and return `IQuery` directly. Use `Where` for predicates and `RightSemijoin` when the result should contain right-side rows that have a matching left-side row: ```csharp -[SpacetimeDB.View(Accessor = "DiscountedProduct", Public = true)] -public static IQuery DiscountedProduct(ViewContext ctx) => - ctx.From.Product().Where(product => product.Discounted.Eq(true)); - -[SpacetimeDB.View(Accessor = "TaggedProduct", Public = true)] -public static IQuery TaggedProduct(ViewContext ctx) => - ctx.From.ProductTag().RightSemijoin( - ctx.From.Product(), - (tag, product) => tag.ProductId.Eq(product.Id) - ); +ctx.From.Article().Where(article => article.Published.Eq(true)); +ctx.From.Subscription().RightSemijoin( + ctx.From.Account(), + (subscription, account) => subscription.AccountId.Eq(account.Id) +); ``` Declare a procedural view primary key in its attribute: `[SpacetimeDB.View(Accessor = "CatalogEntry", Public = true, PrimaryKey = nameof(CatalogRow.Sku))]`. @@ -200,7 +195,7 @@ Inclusive btree ranges use tuples, for example `ctx.Db.Shipment.DeliverBy.Filter ```csharp [ClientVisibilityFilter] public static readonly Filter PrivateNoteFilter = new Filter.Sql( - "SELECT * FROM PrivateNote WHERE Author = :sender" + "SELECT * FROM OwnedRow WHERE Owner = :sender" ); ``` @@ -260,31 +255,24 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); ``` -Synthetic connection IDs can be constructed from 16 bytes with `ConnectionId.From(bytes)`; the result is nullable and must be checked. +Construct a connection ID from 16 bytes with `ConnectionId.From(bytes)`; the result is nullable and must be checked. ## Procedures and HTTP -Procedures are unstable APIs, so modules using them should include `#pragma warning disable STDB_UNSTABLE`. They receive `ProcedureContext`, may return `[SpacetimeDB.Type]` values, perform outbound HTTP through `ctx.Http`, and open short transactions with `ctx.WithTx`: +Procedures are unstable APIs, so modules using them should include `#pragma warning disable STDB_UNSTABLE`. They receive `ProcedureContext` and may return `[SpacetimeDB.Type]` values: ```csharp [SpacetimeDB.Type] -public partial struct ProductResult { public uint Value; public string Description; } - -[SpacetimeDB.Procedure] -public static ProductResult Multiply(ProcedureContext ctx, uint lhs, uint rhs) => - new() { Value = lhs * rhs, Description = "product" }; +public partial struct ResultValue { public string Value; } [SpacetimeDB.Procedure] -public static void RefreshCache(ProcedureContext ctx, string url) -{ - ctx.Http.Get(url).Match(response => { - ctx.WithTx(tx => { tx.Db.CacheEntry.Insert(new CacheEntry { Key = url, Status = response.StatusCode }); return 0; }); - return 0; - }, error => throw new Exception(error.Message)); -} +public static ResultValue Inspect(ProcedureContext ctx, string input) => + new() { Value = input }; ``` -Scheduled procedures use an ordinary scheduled table whose `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row. Database access inside the procedure goes through `ctx.WithTx`. +Outbound HTTP is available through `ctx.Http`; handle its success/error result before using the response. Open short database transactions with `ctx.WithTx`. Perform network I/O before opening the transaction, and keep only database work inside its callback. + +Scheduled procedures use the ordinary scheduled-table shape. Its `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row, and database access inside that procedure goes through `ctx.WithTx`. Inbound HTTP uses handler attributes and one router. Handler database access also goes through `ctx.WithTx`: diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index a7106b15ea4..551cc143aba 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -178,23 +178,14 @@ Declare a procedural view primary key in the view attribute: fn catalog_entry(ctx: &AnonymousViewContext) -> Vec { ... } ``` -Query-builder views use `ViewContext`, `ctx.from`, and return `impl Query`: +Query-builder views use `ViewContext`, `ctx.from`, and return `impl Query`. Use `filter` for predicates and `right_semijoin` when the result should contain right-side rows that have a matching left-side row: ```rust -use spacetimedb::{view, Query, ViewContext}; - -#[view(accessor = discounted_product, public)] -fn discounted_product(ctx: &ViewContext) -> impl Query { - ctx.from.product().filter(|product| product.discounted.eq(true)) -} - -#[view(accessor = tagged_product, public)] -fn tagged_product(ctx: &ViewContext) -> impl Query { - ctx.from.product_tag().right_semijoin( - ctx.from.product(), - |tag, product| tag.product_id.eq(product.id), - ) -} +ctx.from.article().filter(|article| article.published.eq(true)) +ctx.from.subscription().right_semijoin( + ctx.from.account(), + |subscription, account| subscription.account_id.eq(account.id), +) ``` ## Client Visibility Filters @@ -204,7 +195,7 @@ use spacetimedb::Filter; #[spacetimedb::client_visibility_filter] const PRIVATE_NOTE_FILTER: Filter = - Filter::Sql("SELECT * FROM private_note WHERE author = :sender"); + Filter::Sql("SELECT * FROM owned_row WHERE owner = :sender"); ``` ## Reducer Context API @@ -257,36 +248,29 @@ let at = ScheduleAt::Interval(std::time::Duration::from_secs(5).into()); ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, scheduled_at: at }); ``` -Synthetic connection IDs for module logic/tests can use `ConnectionId::from_u128(1)`. +Construct a connection ID from a numeric representation with `ConnectionId::from_u128(value)`. ## Procedures and HTTP -Procedures use `&mut ProcedureContext`, may return typed values, perform outbound HTTP through `ctx.http`, and open short database transactions with `ctx.with_tx`: +Procedures use `&mut ProcedureContext` and may return typed values: ```rust -use spacetimedb::{procedure, ProcedureContext, SpacetimeType, Table}; +use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; #[derive(SpacetimeType)] -pub struct Product { pub value: u32, pub description: String } +pub struct ResultValue { pub value: String } #[procedure] -pub fn multiply(_ctx: &mut ProcedureContext, lhs: u32, rhs: u32) -> Product { - Product { value: lhs * rhs, description: "product".into() } -} - -#[procedure] -pub fn refresh_cache(ctx: &mut ProcedureContext, url: String) { - let response = ctx.http.get(url).expect("request failed"); - let status = response.status().as_u16(); - ctx.with_tx(|tx| { - tx.db.cache_entry().insert(CacheEntry { key: url.clone(), status }); - }); +pub fn inspect(_ctx: &mut ProcedureContext, input: String) -> ResultValue { + ResultValue { value: input } } ``` -`with_tx` callbacks implement `Fn`; clone captured owned values when storing them rather than moving them out of the closure. +Outbound HTTP is available through `ctx.http`. Convenience methods such as `get` return a response, and other methods use `Request::builder()` with `ctx.http.send(request)`. Consume a response body as text with `response.into_body().into_string_lossy()`. + +Open short database transactions with `ctx.with_tx(|tx| ...)`. The callback implements `Fn`, so clone captured owned values when storing them rather than moving them out of the closure. Perform network I/O before opening the transaction. -Consume an HTTP response body as text with `response.into_body().into_string_lossy()`. For methods other than the available convenience calls, build a request and use `send`: +For an outbound request without a convenience method: ```rust let request = Request::builder() @@ -297,7 +281,7 @@ let request = Request::builder() let response = ctx.http.send(request).expect("request failed"); ``` -Scheduled procedures use a normal scheduled table, but the scheduled function is marked `#[procedure]` and receives `&mut ProcedureContext` plus the scheduled row. +Scheduled procedures use the ordinary scheduled-table shape, but the scheduled function is marked `#[procedure]` and receives `&mut ProcedureContext` plus the scheduled row. Inbound HTTP uses handler functions and one router: diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 7f735fb3d2a..2c92d17705d 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -185,7 +185,7 @@ new Date(Number(row.createdAt.microsSinceUnixEpoch / 1000n)); Do not construct `Identity` values from strings (e.g. `'hex' as Identity`): serialization fails and kills the module. Identities come from `ctx.sender` or `t.identity()` columns. -Synthetic connection IDs for module logic/tests can use `new ConnectionId(1n)` after importing `ConnectionId` from `spacetimedb`. +Construct a `ConnectionId` from its numeric representation with `new ConnectionId(value)` after importing `ConnectionId` from `spacetimedb`. Construct exact timestamps with `new Timestamp(micros)` after importing `Timestamp` from `spacetimedb`. Inclusive index ranges use `Range` from `spacetimedb/server`: @@ -270,30 +270,21 @@ const CatalogKey = t.row('CatalogKey', { }); ``` -Query-builder views use `ctx.from` and return the query directly: +Query-builder views use `ctx.from` and return the query directly. Use `where` for predicates and `rightSemijoin` when the result should contain right-side rows that have a matching left-side row: ```typescript -export const discountedProduct = spacetimedb.view( - { name: 'discounted_product', public: true }, - t.array(product.rowType), - ctx => ctx.from.product.where(product => product.discounted.eq(true)) -); - -export const taggedProduct = spacetimedb.view( - { name: 'tagged_product', public: true }, - t.array(product.rowType), - ctx => ctx.from.productTag.rightSemijoin( - ctx.from.product, - (tag, product) => tag.productId.eq(product.id) - ) -); +ctx => ctx.from.article.where(article => article.published.eq(true)) +ctx => ctx.from.subscription.rightSemijoin( + ctx.from.account, + (subscription, account) => subscription.accountId.eq(account.id) +) ``` ## Client Visibility Filters ```typescript export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( - 'SELECT * FROM private_note WHERE author = :sender' + 'SELECT * FROM owned_row WHERE owner = :sender' ); ``` @@ -301,51 +292,23 @@ export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( `spacetimedb` is the local schema value returned by `schema({...})`; it is not a named export to import from `spacetimedb/server`. -Procedures declare argument and return types. They can perform outbound HTTP through `ctx.http` and open short transactions with `ctx.withTx`: +Procedures declare argument and return types: ```typescript -const Product = t.object('Product', { value: t.u32(), description: t.string() }); +const Result = t.object('Result', { value: t.string() }); -export const multiply = spacetimedb.procedure( - { lhs: t.u32(), rhs: t.u32() }, - Product, - (_ctx, { lhs, rhs }) => ({ value: lhs * rhs, description: 'product' }) -); - -export const refreshCache = spacetimedb.procedure( - { url: t.string() }, - t.unit(), - (ctx, { url }) => { - const response = ctx.http.fetch(url); - ctx.withTx(tx => tx.db.cacheEntry.insert({ key: url, status: response.status })); - return {}; - } +export const inspect = spacetimedb.procedure( + { input: t.string() }, + Result, + (_ctx, { input }) => ({ value: input }) ); ``` -TypeScript outbound HTTP uses `ctx.http.fetch(url, options)`, including for non-GET requests; it does not provide convenience methods such as `get()` or `post()`. Responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. Perform network I/O before `withTx`; only database work belongs inside its callback. - -Scheduled procedures use a normal scheduled table, but its `scheduled` option references a `spacetimedb.procedure(...)` export instead of a reducer. The procedure receives its scheduled row as an argument and uses `withTx` for database access: +TypeScript outbound HTTP uses `ctx.http.fetch(url, options)`, including for non-GET requests; it does not provide convenience methods such as `get()` or `post()`. Responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. -```typescript -const cleanup_timer = table( - { name: 'cleanup_timer', scheduled: (): any => runCleanup }, - { - scheduledId: t.u64().primaryKey().autoInc(), - scheduledAt: t.scheduleAt(), - cacheKey: t.string(), - } -); +Procedures and handlers open short database transactions with `ctx.withTx(tx => ...)`. Perform network I/O before opening the transaction; only database work belongs inside its callback. -export const runCleanup = spacetimedb.procedure( - { timer: cleanup_timer.rowType }, - t.unit(), - (ctx, { timer }) => { - ctx.withTx(tx => tx.db.cacheEntry.key.delete(timer.cacheKey)); - return {}; - } -); -``` +Scheduled procedures use the ordinary scheduled-table shape. Its `scheduled` option references an exported `spacetimedb.procedure(...)` value instead of a reducer, and the procedure accepts the scheduled row as its argument. Inbound HTTP uses `httpHandler`, `httpRouter`, `Router`, and `SyncResponse`: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt index 2dca848cb08..1409089ad36 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines three tables with basic columns. TABLES -- User +- User (private) - Struct: User - Fields: - Id: int (primary key) @@ -9,7 +9,7 @@ TABLES - Age: int - Active: bool -- Product +- Product (private) - Struct: Product - Fields: - Id: int (primary key) @@ -17,7 +17,7 @@ TABLES - Price: float - InStock: bool -- Note +- Note (private) - Struct: Note - Fields: - Id: int (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt index 02d385eb6c9..ab13a01581e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines three tables with basic columns. TABLES -- user +- user (private) - Struct: User - Fields: - id: i32 (primary key) @@ -9,7 +9,7 @@ TABLES - age: i32 - active: bool -- product +- product (private) - Struct: Product - Fields: - id: i32 (primary key) @@ -17,7 +17,7 @@ TABLES - price: f32 - in_stock: bool -- note +- note (private) - Struct: Note - Fields: - id: i32 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt index f8f4666b502..af6e3e1128f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt @@ -1,23 +1,23 @@ Write a SpacetimeDB backend module in TypeScript that defines three tables with basic columns. TABLES -- user +- user (private) - Fields: - id: number (i32, primary key) - name: string - age: number (i32) - active: boolean -- product +- product (private) - Fields: - id: number (i32, primary key) - title: string - price: number (f32) - inStock: boolean -- note +- note (private) - Fields: - id: number (i32, primary key) - body: string - rating: bigint (i64) - - pinned: boolean \ No newline at end of file + - pinned: boolean diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs index cf7fd14353b..ef512f2ec75 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs @@ -22,7 +22,7 @@ public partial struct RefreshJob [Reducer] public static void StartRefresh(ReducerContext ctx) { - var pending = new MaterializedState { Id = 1, Status = "pending", Version = 0, RefreshedAt = ctx.Timestamp }; + var pending = new MaterializedState { Id = 1, Status = "pending", Version = 0, RefreshedAt = new Timestamp(0) }; if (ctx.Db.MaterializedState.Id.Find(1) is null) ctx.Db.MaterializedState.Insert(pending); else ctx.Db.MaterializedState.Id.Update(pending); ctx.Db.RefreshJob.Insert(new RefreshJob { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs index f8e517e1843..1ed33422aa7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs @@ -21,7 +21,7 @@ pub struct RefreshJob { #[reducer] pub fn start_refresh(ctx: &ReducerContext) { - let pending = MaterializedState { id: 1, status: "pending".into(), version: 0, refreshed_at: ctx.timestamp }; + let pending = MaterializedState { id: 1, status: "pending".into(), version: 0, refreshed_at: Timestamp::UNIX_EPOCH }; if ctx.db.materialized_state().id().find(1).is_some() { ctx.db.materialized_state().id().update(pending); } else { ctx.db.materialized_state().insert(pending); } ctx.db.refresh_job().insert(RefreshJob { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts index 789712f83ef..678785f8e83 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts @@ -1,4 +1,4 @@ -import { ScheduleAt } from 'spacetimedb'; +import { ScheduleAt, Timestamp } from 'spacetimedb'; import { schema, table, t } from 'spacetimedb/server'; const materializedState = table({ name: 'materialized_state', public: true }, { @@ -11,7 +11,7 @@ const spacetimedb = schema({ materializedState, refreshJob }); export default spacetimedb; export const start_refresh = spacetimedb.reducer(ctx => { - const pending = { id: 1n, status: 'pending', version: 0n, refreshedAt: ctx.timestamp }; + const pending = { id: 1n, status: 'pending', version: 0n, refreshedAt: new Timestamp(0n) }; if (ctx.db.materializedState.id.find(1n)) ctx.db.materializedState.id.update(pending); else ctx.db.materializedState.insert(pending); ctx.db.refreshJob.insert({ diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs index c7dd0d2103c..e579c919f00 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs @@ -1,18 +1,41 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, + make_sql_output_excludes_scorer, +}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_call_both_scorer(host_url, file!(), route_tag, "start_refresh", vec![], "start_refresh")); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "start_refresh", + vec![], + "start_refresh", + )); let table = table_name("materialized_state", lang); let status = ident("status", casing_for_lang(lang)); let version = ident("version", casing_for_lang(lang)); scorers.push(make_eventually_sql_count_scorer( - host_url, file!(), route_tag, + host_url, + file!(), + route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}='ready' AND {version}=1"), - 1, "scheduled_refresh_completes", Duration::from_secs(10), + 1, + "scheduled_refresh_completes", + Duration::from_secs(10), + )); + let refreshed_at = ident("refreshed_at", casing_for_lang(lang)); + scorers.push(make_sql_output_excludes_scorer( + host_url, + file!(), + route_tag, + format!("SELECT {refreshed_at} FROM {table}"), + vec!["1970-01-01T00:00:00+00:00".into()], + "scheduled_refresh_timestamped", )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt index 223d96fa9f3..1c479fdef11 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt @@ -15,9 +15,9 @@ TABLES REDUCERS - StartRefresh() - - Write Id 1 with Status "pending" and Version 0 + - Write Id 1 with Status "pending", Version 0, and RefreshedAt at the Unix epoch - Schedule state 1 for one millisecond later - RefreshMaterialized(job: RefreshJob) (scheduled reducer) - Update state 1 to Status "ready" and Version 1 - - Set RefreshedAt to the scheduled reducer timestamp + - Replace RefreshedAt with the scheduled reducer timestamp - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt index da3e3c30f99..fc567eb570a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt @@ -15,9 +15,9 @@ TABLES REDUCERS - start_refresh() - - Write id 1 with status "pending" and version 0 + - Write id 1 with status "pending", version 0, and refreshed_at at the Unix epoch - Schedule state 1 for one millisecond later - refresh_materialized(job: RefreshJob) (scheduled reducer) - Update state 1 to status "ready" and version 1 - - Set refreshed_at to the scheduled reducer timestamp + - Replace refreshed_at with the scheduled reducer timestamp - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt index fc5ae4f85a0..91ee348827d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt @@ -15,9 +15,9 @@ TABLES REDUCERS - start_refresh() - - Write id 1 with status "pending" and version 0 + - Write id 1 with status "pending", version 0, and refreshedAt at the Unix epoch - Schedule state 1 for one millisecond later - refreshMaterialized(job: refresh_job row) (scheduled reducer) - Update state 1 to status "ready" and version 1 - - Set refreshedAt to the scheduled reducer timestamp + - Replace refreshedAt with the scheduled reducer timestamp - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs index e6c1fd0f1a8..686cd2c6c00 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs @@ -1,18 +1,31 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); - scorers.push(make_reducer_sql_count_scorer(host_url, ReducerSqlCountConfig { - src_file: file!(), route_tag, reducer: "exercise_presence".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("presence_session", lang)), - // The CLI call itself creates one lifecycle-managed session. The - // helper adds two synthetic sessions and removes one, leaving two - // total rows while still proving connection-scoped deletion. - expected_count: 2, id_str: "disconnect_removes_one_connection", timeout: Duration::from_secs(10), - })); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "exercise_presence", + vec![], + "exercise_presence", + )); + let table = table_name("presence_session", lang); + let connection_id = ident("connection_id", casing_for_lang(lang)); + for (value, expected, id) in [(1, 0, "first_connection_removed"), (2, 1, "second_connection_retained")] { + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {connection_id}=0x{value:032x}"), + expected, + id, + Duration::from_secs(10), + )); + } scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index 5db7f8420ef..ccf9d697a8d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -7,9 +7,6 @@ TABLE - Identity: Identity (btree index) - ConnectedAt: Timestamp -HELPERS -- Share the insert and delete logic between the lifecycle reducers and ExercisePresence - LIFECYCLE HOOKS - ClientConnected - Export with this exact method name @@ -20,6 +17,6 @@ LIFECYCLE HOOKS REDUCERS - ExercisePresence() - - Insert two synthetic connections for the caller - - Remove only the first - - Leave the second synthetic session intact + - Insert synthetic connection IDs 1 and 2 for the caller + - Remove connection ID 1 + - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt index 01af65378e9..79d30710523 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -7,9 +7,6 @@ TABLE - identity: Identity (btree index) - connected_at: Timestamp -HELPERS -- Share the insert and delete logic between the lifecycle reducers and exercise_presence - LIFECYCLE HOOKS - client_connected - Export with this exact identifier @@ -20,6 +17,6 @@ LIFECYCLE HOOKS REDUCERS - exercise_presence() - - Insert two synthetic connections for the caller - - Remove only the first - - Leave the second synthetic session intact + - Insert synthetic connection IDs 1 and 2 for the caller + - Remove connection ID 1 + - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt index 9090ce63274..1779fcd4ef0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -7,9 +7,6 @@ TABLE - identity: Identity (btree index) - connectedAt: Timestamp -HELPERS -- Share the insert and delete logic between the lifecycle hooks and exercise_presence - LIFECYCLE HOOKS - clientConnected - Export with this exact identifier @@ -20,6 +17,6 @@ LIFECYCLE HOOKS REDUCERS - exercise_presence() - - Insert two synthetic connections for the caller - - Remove only the first - - Leave the second synthetic session intact + - Insert synthetic connection IDs 1 and 2 for the caller + - Remove connection ID 1 + - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs index 8bd8c979635..2c27bcb4159 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs @@ -15,6 +15,6 @@ public static void Generate(ReducerContext ctx) => ctx.Db.GeneratedValue.Insert( { Id = 0, CreatedAt = ctx.Timestamp, - RandomValue = ctx.Rng.NextInt64(), + RandomValue = ctx.Rng.NextInt64(1, long.MaxValue), }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs index c1c16b4eca3..50baa498033 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs @@ -11,9 +11,10 @@ pub struct GeneratedValue { #[reducer] pub fn generate(ctx: &ReducerContext) { + let random_value: u64 = ctx.random(); ctx.db.generated_value().insert(GeneratedValue { id: 0, created_at: ctx.timestamp, - random_value: ctx.random(), + random_value: random_value.max(1), }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts index eeae887f791..92b487c6964 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts @@ -11,6 +11,6 @@ export const generate = spacetimedb.reducer(ctx => { ctx.db.generatedValue.insert({ id: 0n, createdAt: ctx.timestamp, - randomValue: BigInt(ctx.random.integerInRange(0, Number.MAX_SAFE_INTEGER)), + randomValue: BigInt(ctx.random.integerInRange(1, Number.MAX_SAFE_INTEGER)), }); }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs index df8acdce33d..fc1af0d45e2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs @@ -1,10 +1,15 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_sql_count_scorer, make_sql_output_excludes_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let table = table_name("generated_value", lang); + let casing = casing_for_lang(lang); + let random_value = ident("random_value", casing); scorers.push(make_reducer_sql_count_scorer( host_url, ReducerSqlCountConfig { @@ -12,12 +17,21 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "generate".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {}", table_name("generated_value", lang)), + sql_count_query: format!("SELECT COUNT(*) AS n FROM {table} WHERE {random_value}<>0"), expected_count: 1, - id_str: "context_value_written", + id_str: "context_values_recorded", timeout: Duration::from_secs(10), }, )); + let created_at = ident("created_at", casing); + scorers.push(make_sql_output_excludes_scorer( + host_url, + file!(), + route_tag, + format!("SELECT {created_at} FROM {table}"), + vec!["1970-01-01T00:00:00+00:00".into()], + "context_timestamp_recorded", + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt index 1b800b26673..2c79b8c0fc0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt @@ -13,6 +13,4 @@ TABLES REDUCERS - CreateFamily(parentName: string, childNames: List) - - Insert the Parent and capture the row returned by Insert - - Use the returned Parent Id for every Child - - Never calculate or assume the next auto-increment value + - Insert the Parent, then insert every Child using the Id actually assigned to that Parent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt index a7ec69b64b3..7351cfd0827 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt @@ -13,6 +13,4 @@ TABLES REDUCERS - create_family(parent_name: String, child_names: Vec) - - Insert the parent and capture the row returned by insert - - Use the returned parent id for every child - - Never calculate or assume the next auto-increment value + - Insert the parent, then insert every child using the ID actually assigned to that parent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt index 7a6b678ecee..334dfaad17c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt @@ -13,6 +13,4 @@ TABLES REDUCERS - create_family(parentName: string, childNames: string array) - - Insert the parent and capture the row returned by insert - - Use the returned parent id for every child - - Never calculate or assume the next auto-increment value + - Insert the parent, then insert every child using the ID actually assigned to that parent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt index 099cebe9059..f9667e88bd4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -26,4 +26,4 @@ REDUCERS VIEW - OrderLineDetail (public) - Returns rows with LineId: ulong, CustomerName: string, and Sku: string - - Join all three tables through indexed lookups + - Return only rows connected through matching Customer, Purchase, and LineItem IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt index 5647ca27dc2..4912f155896 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -26,4 +26,4 @@ REDUCERS VIEW - order_line_detail (public) - Returns rows with line_id: u64, customer_name: String, and sku: String - - Join all three tables through indexed lookups + - Return only rows connected through matching customer, purchase, and line-item IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt index 3ce44511ed1..ceb08d774a0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -26,4 +26,4 @@ REDUCERS VIEW - order_line_detail (public) - Returns rows with lineId: u64, customerName: string, and sku: string - - Join all three tables through indexed lookups + - Return only rows connected through matching customer, purchase, and line-item IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt index b2bc80c1b76..58f533694a6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -12,13 +12,10 @@ TABLES - TotalAmount: long - SaleCount: ulong -HELPERS -- Internal shared upsert and delete helpers - - Keep CategoryTotal transactionally synchronized when a Sale is inserted, changed, moved between categories, or deleted - REDUCERS - Exercise() - This is the only exported reducer + - Keep CategoryTotal transactionally synchronized when a Sale is inserted, changed, moved between categories, or deleted - Insert books Sale rows of 10 and 20 - Update the second books Sale to 25 - Insert and then delete a games Sale diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt index be026d6c91e..c70d7894d36 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -12,13 +12,10 @@ TABLES - total_amount: i64 - sale_count: u64 -HELPERS -- Internal shared upsert and delete helpers - - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted - REDUCERS - exercise() - This is the only exported reducer + - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted - Insert books sales of 10 and 20 - Update the second books sale to 25 - Insert and then delete a games sale diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt index fd440865266..5cb6eb6ff2f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -12,13 +12,10 @@ TABLES - totalAmount: i64 - saleCount: u64 -HELPERS -- Internal shared upsert and delete helpers - - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted - REDUCERS - exercise() - This is the only exported reducer + - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted - Insert books sales of 10 and 20 - Update the second books sale to 25 - Insert and then delete a games sale diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index 7f3710134cb..3b7d6a1e5d6 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -2,6 +2,7 @@ use crate::bench::utils::sanitize_db_name; use crate::eval::scorers::{ CallOutputParityScorer, EventuallySqlCountScorer, HttpRouteCase, HttpRouteParityScorer, ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, + SqlOutputExcludesScorer, }; use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSqlCountConfig}; use std::time::Duration; @@ -130,6 +131,25 @@ pub fn make_reducer_call_both_scorer( make_reducer_call_both_scorer_with_attempts(host_url, src_file, route_tag, reducer, args, id_str, 1) } +pub fn make_sql_output_excludes_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + sql: impl Into, + excluded: Vec, + id_str: &'static str, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(SqlOutputExcludesScorer { + server: host_url.to_string(), + db: llm_db, + sql: sql.into(), + excluded, + id_str, + }) +} + pub fn make_reducer_call_both_scorer_with_attempts( host_url: &str, src_file: &str, diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 35fa4adcf16..bc7d8ae0460 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -725,6 +725,14 @@ pub struct SqlCountOnlyScorer { pub id_str: &'static str, } +pub struct SqlOutputExcludesScorer { + pub server: String, + pub db: String, + pub sql: String, + pub excluded: Vec, + pub id_str: &'static str, +} + pub struct EventuallySqlCountScorer { pub server: String, pub db: String, @@ -788,6 +796,36 @@ impl Scorer for SqlCountOnlyScorer { } } +impl Scorer for SqlOutputExcludesScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + match sql_raw(&self.db, &self.sql, Some(&self.server)) { + Ok(output) => { + let found = self + .excluded + .iter() + .filter(|value| output.contains(value.as_str())) + .cloned() + .collect::>(); + let pass = found.is_empty(); + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "sql": self.sql, "excluded": self.excluded, "found": found, "actual": output }), + } + } + Err(error) => ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "sql", "error": error }), + }, + } + } +} + pub struct SqlExecBothScorer { pub server: String, pub golden_db: String, From 905249ee065e5d36c93a20cdae54435c54eb6a3f Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 22:24:14 -0400 Subject: [PATCH 43/78] Clarify TypeScript eval guidance --- skills/typescript-server/SKILL.md | 6 +++++- .../benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt | 4 ++-- .../benchmarks/basics/t_000_empty_reducers/tasks/rust.txt | 4 ++-- .../basics/t_000_empty_reducers/tasks/typescript.txt | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 2c92d17705d..b4fea3b4668 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -164,7 +164,7 @@ export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ## Reducer Context API -`ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. In helpers, type it as `ReducerCtx>`. +`ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. ```typescript // Auth: ctx.sender is the caller's Identity @@ -245,6 +245,8 @@ const Shape = t.enum('Shape', { `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. + ```typescript // Anonymous view (same for all clients): export const activeUsers = spacetimedb.anonymousView( @@ -304,6 +306,8 @@ export const inspect = spacetimedb.procedure( ); ``` +Procedure callbacks are synchronous. Do not mark them `async` or use `await`; return the declared value directly. A procedure with return type `t.unit()` returns `{}`. + TypeScript outbound HTTP uses `ctx.http.fetch(url, options)`, including for non-GET requests; it does not provide convenience methods such as `get()` or `post()`. Responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. Procedures and handlers open short database transactions with `ctx.withTx(tx => ...)`. Perform network I/O before opening the transaction; only database work belongs inside its callback. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt index 0821f3321a6..0f773c9268b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table and these five empty reducers. TABLES -- EmptyTable +- EmptyTable (private) - Struct: EmptyTable - Fields: - Id: int (primary key) @@ -11,4 +11,4 @@ REDUCERS - EmptyReducer_WithInt: (int count), returns void, empty body - EmptyReducer_WithString: (string name), returns void, empty body - EmptyReducer_WithTwoArgs: (int count, string name), returns void, empty body -- EmptyReducer_WithThreeArgs: (bool active, float ratio, string label), returns void, empty body \ No newline at end of file +- EmptyReducer_WithThreeArgs: (bool active, float ratio, string label), returns void, empty body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt index 7d42898b222..c512823bfbf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table and these five empty reducers. TABLES -- EmptyTable +- EmptyTable (private) - Struct: EmptyTable - Fields: - id: i32 (primary key) @@ -11,4 +11,4 @@ REDUCERS - empty_reducer_with_int: (count: i32), returns (), empty body - empty_reducer_with_string: (name: String), returns (), empty body - empty_reducer_with_two_args: (count: i32, name: String), returns (), empty body -- empty_reducer_with_three_args: (active: bool, ratio: f32, label: String), returns (), empty body \ No newline at end of file +- empty_reducer_with_three_args: (active: bool, ratio: f32, label: String), returns (), empty body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt index 6580e17302f..5be728388b3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table and these five empty reducers. TABLES -- empty_table +- empty_table (private) - Fields: - id: number (i32, primary key) @@ -10,4 +10,4 @@ REDUCERS - emptyReducerWithInt: count:number (i32) - emptyReducerWithString: name:string - emptyReducerWithTwoArgs: count:number (i32), name:string -- emptyReducerWithThreeArgs: active:boolean, ratio:number (f32), label:string \ No newline at end of file +- emptyReducerWithThreeArgs: active:boolean, ratio:number (f32), label:string From 3d21ee79765c336c81e5004faa902bc38c0c229d Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Sat, 18 Jul 2026 22:40:45 -0400 Subject: [PATCH 44/78] Document TypeScript module type contracts --- skills/typescript-server/SKILL.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index b4fea3b4668..6165a0fdc19 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -40,13 +40,17 @@ export const addRecord = spacetimedb.reducer( ); ``` +Named runtime exports are reserved for values registered with SpacetimeDB, such as reducers, lifecycle hooks, views, procedures, HTTP exports, and visibility filters. Keep ordinary helper functions and constants unexported. + ## Imports Schema builders and module exports come from `spacetimedb/server`. Runtime value classes such as `ScheduleAt`, `Timestamp`, and `ConnectionId` come from the root `spacetimedb` package; `Range` comes from `spacetimedb/server`: ```typescript -import { schema, table, t } from 'spacetimedb/server'; -import { SenderError } from 'spacetimedb/server'; +import { + schema, table, t, SenderError, + type InferSchema, type ReducerCtx, +} from 'spacetimedb/server'; import { ConnectionId, ScheduleAt, Timestamp } from 'spacetimedb'; import { Range } from 'spacetimedb/server'; ``` @@ -272,7 +276,7 @@ const CatalogKey = t.row('CatalogKey', { }); ``` -Query-builder views use `ctx.from` and return the query directly. Use `where` for predicates and `rightSemijoin` when the result should contain right-side rows that have a matching left-side row: +Query-builder views use `ctx.from` and return the query directly. Because a query returns a row set, declare its return schema as `t.array(tableName.rowType)`. Use `where` for predicates and `rightSemijoin` when the result should contain right-side rows that have a matching left-side row: ```typescript ctx => ctx.from.article.where(article => article.published.eq(true)) From f67279105be5eab9b322501704160b47ce5635ad Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:49:24 -0400 Subject: [PATCH 45/78] Clarify Rust query and HTTP context contracts --- skills/rust-server/SKILL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index 551cc143aba..bd8f20b1662 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -188,6 +188,8 @@ ctx.from.subscription().right_semijoin( ) ``` +Query-builder comparisons against a `String` column take an owned `String`, for example `row.label.eq("active".to_string())`. + ## Client Visibility Filters ```rust @@ -297,6 +299,8 @@ fn health(_ctx: &mut HandlerContext, _request: Request) -> Response { fn routes() -> Router { Router::new().get("/health", health) } ``` +`HandlerContext` does not expose `db`; open database access with `ctx.with_tx(|tx| ...)`. HTTP response bodies must own their data or borrow `'static` data, so convert dynamic borrowed text to an owned `String` before passing it to `Body::from_bytes`. + ## Logging ```rust From 5fb77843fd3f4e15d4d0e51007b3b9c03ade6838 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:10:19 -0400 Subject: [PATCH 46/78] Correct Rust procedural view guidance --- skills/rust-server/SKILL.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index bd8f20b1662..8364211262e 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -37,6 +37,7 @@ pub struct Entity { pub id: u64, pub owner: Identity, pub name: String, + #[index(btree)] pub active: bool, } ``` @@ -159,7 +160,7 @@ use spacetimedb::{view, AnonymousViewContext}; #[view(accessor = active_users, public)] fn active_users(ctx: &AnonymousViewContext) -> Vec { - ctx.db.entity().iter().filter(|e| e.active).collect() + ctx.db.entity().active().filter(true).collect() } // Per-user view (result varies by sender): @@ -171,6 +172,8 @@ fn my_profile(ctx: &ViewContext) -> Option { } ``` +Procedural-view table handles support indexed `find` and `filter` access, but not full-table `iter()`. Start a procedural view from an appropriate table index. + Declare a procedural view primary key in the view attribute: ```rust From d38f6d91540bfd801229ab417c38ac74dd0f1349 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:30:11 -0400 Subject: [PATCH 47/78] Correct Rust string accessor guidance --- skills/rust-server/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index 8364211262e..d36b3d45eef 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -128,14 +128,14 @@ ctx.db.entity().iter(); // All rows ctx.db.entity().count(); // Count rows ctx.db.entity().id().update(Entity { name: new_name, ..existing }); // Update (override + spread) ctx.db.entity().id().delete(entity_id); // Delete by PK -ctx.db.entity().name().delete("Alice"); // Delete by indexed column +ctx.db.entity().name().delete("Alice".to_string()); // Delete by indexed String column ``` Note: `iter()` and `filter()` return iterators. Collect to Vec if you need `.sort()`, `.filter()`, `.map()`. Range queries on btree indexes: `filter(18..=65)`, `filter(18..)`, `filter(..18)`. -String index filters borrow the key: `ctx.db.product().category().filter(&"hardware".to_string())`. +String column accessors operate on the column's owned `String` type, not `&str`. Pass a `String` or `&String` to `find` and `delete`; index filters borrow the key, as in `ctx.db.product().category().filter(&"hardware".to_string())`. ## Lifecycle Hooks From 3dfba7862afc9f4c211cde673527bbaca75b52e2 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:56:41 -0400 Subject: [PATCH 48/78] Clarify C# view and HTTP contracts --- skills/csharp-server/SKILL.md | 46 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index bbceda3dfa8..5bf41ce7472 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -51,8 +51,10 @@ public partial struct Entity [PrimaryKey] [AutoInc] public ulong Id; + [SpacetimeDB.Index.BTree] public Identity Owner; public string Name; + [SpacetimeDB.Index.BTree] public bool Active; } ``` @@ -158,6 +160,8 @@ public static void OnConnect(ReducerContext ctx) { ... } public static void OnDisconnect(ReducerContext ctx) { ... } ``` +`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. + ## Views ```csharp @@ -165,17 +169,19 @@ public static void OnDisconnect(ReducerContext ctx) { ... } [SpacetimeDB.View(Accessor = "ActiveUsers", Public = true)] public static List ActiveUsers(AnonymousViewContext ctx) { - return ctx.Db.Entity.Iter().Where(e => e.Active).ToList(); + return ctx.Db.Entity.Active.Filter(true).ToList(); } // Per-user view: -[SpacetimeDB.View(Accessor = "MyProfile", Public = true)] -public static Entity? MyProfile(ViewContext ctx) +[SpacetimeDB.View(Accessor = "MyEntities", Public = true)] +public static List MyEntities(ViewContext ctx) { - return ctx.Db.Entity.Identity.Find(ctx.Sender) as Entity?; + return ctx.Db.Entity.Owner.Filter(ctx.Sender).ToList(); } ``` +View contexts expose read-only table handles. These support `Count` plus `Find`/`Filter` on declared indexes, but not full-table `Iter()`. Start procedural view traversal from an indexed lookup or filter. + Query-builder views use `ViewContext`, `ctx.From`, and return `IQuery` directly. Use `Where` for predicates and `RightSemijoin` when the result should contain right-side rows that have a matching left-side row: ```csharp @@ -255,6 +261,8 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); ``` +Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. + Construct a connection ID from 16 bytes with `ConnectionId.From(bytes)`; the result is nullable and must be checked. ## Procedures and HTTP @@ -272,6 +280,36 @@ public static ResultValue Inspect(ProcedureContext ctx, string input) => Outbound HTTP is available through `ctx.Http`; handle its success/error result before using the response. Open short database transactions with `ctx.WithTx`. Perform network I/O before opening the transaction, and keep only database work inside its callback. +```csharp +var result = ctx.Http.Get(uri); +var text = result.Match( + response => response.Body.ToStringUtf8Lossy(), + error => throw new Exception(error.Message) +); + +ctx.WithTx(tx => +{ + tx.Db.ScoreRecord.Insert(new ScoreRecord { Id = 0, Owner = ctx.Sender, Value = 1 }); + return 0; +}); +``` + +`WithTx` is generic: its callback must return a value (use `return 0;` when no result is needed). The callback's generated `tx.Db` exposes module table accessors; `ProcedureContext` and `HandlerContext` do not expose tables directly. + +For non-GET requests, construct and send an `HttpRequest` directly: + +```csharp +var result = ctx.Http.Send(new HttpRequest +{ + Uri = uri, + Method = HttpMethod.Post, + Headers = new() { new HttpHeader("content-type", "text/plain") }, + Body = HttpBody.FromString(payload), +}); +``` + +`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text and `ToStringUtf8Lossy()` to read text; `ToString()` does not return body contents. + Scheduled procedures use the ordinary scheduled-table shape. Its `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row, and database access inside that procedure goes through `ctx.WithTx`. Inbound HTTP uses handler attributes and one router. Handler database access also goes through `ctx.WithTx`: From 4b41dae472856292268c2c89c32f0fc16041d8a6 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:07:45 -0400 Subject: [PATCH 49/78] Clarify C# HTTP types and presence IDs --- skills/csharp-server/SKILL.md | 4 ++-- .../t_070_connection_scoped_presence/tasks/csharp.txt | 1 + .../lifecycle/t_070_connection_scoped_presence/tasks/rust.txt | 1 + .../t_070_connection_scoped_presence/tasks/typescript.txt | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 5bf41ce7472..6a92f8b0872 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -302,13 +302,13 @@ For non-GET requests, construct and send an `HttpRequest` directly: var result = ctx.Http.Send(new HttpRequest { Uri = uri, - Method = HttpMethod.Post, + Method = SpacetimeDB.HttpMethod.Post, Headers = new() { new HttpHeader("content-type", "text/plain") }, Body = HttpBody.FromString(payload), }); ``` -`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text and `ToStringUtf8Lossy()` to read text; `ToString()` does not return body contents. +`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples, and each header's `Value` is `byte[]`; decode text values with `System.Text.Encoding.UTF8.GetString(header.Value)`. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text, `new HttpBody(bytes)` to supply raw bytes, and `ToStringUtf8Lossy()` to read text. There is no `HttpBody.FromBytes`, and `ToString()` does not return body contents. Qualify `SpacetimeDB.HttpMethod` when .NET's implicit `System.Net.Http` imports could make the name ambiguous. Scheduled procedures use the ordinary scheduled-table shape. Its `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row, and database access inside that procedure goes through `ctx.WithTx`. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index ccf9d697a8d..cbea5266563 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -18,5 +18,6 @@ LIFECYCLE HOOKS REDUCERS - ExercisePresence() - Insert synthetic connection IDs 1 and 2 for the caller + - Encode ID N as 16 bytes with byte 0 set to N and all remaining bytes zero - Remove connection ID 1 - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt index 79d30710523..fb7eb4d7c04 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -18,5 +18,6 @@ LIFECYCLE HOOKS REDUCERS - exercise_presence() - Insert synthetic connection IDs 1 and 2 for the caller + - Encode ID N as 16 bytes with byte 0 set to N and all remaining bytes zero - Remove connection ID 1 - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt index 1779fcd4ef0..4d67cd5040a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -18,5 +18,6 @@ LIFECYCLE HOOKS REDUCERS - exercise_presence() - Insert synthetic connection IDs 1 and 2 for the caller + - Encode ID N as 16 bytes with byte 0 set to N and all remaining bytes zero - Remove connection ID 1 - Leave connection ID 2 intact From 005a093fca2b6d68dfaf343a597bcbb6b528b713 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:30:12 -0400 Subject: [PATCH 50/78] Retry transient dotnet linker failures --- .../src/bench/publishers.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index ea92aead7b1..64ccb816fa4 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -221,6 +221,14 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let silent_cargo_exit = diagnostic.contains("error: command [\"cargo\", \"build\"") && diagnostic.contains("exited with code 1") && !has_actionable_compiler_diagnostic; + let has_actionable_dotnet_diagnostic = diagnostic.lines().any(|line| { + line.contains(".cs(") + && (line.contains(": error cs") || line.contains(": error stdb") || line.contains(": error il")) + }); + let dotnet_linker_exit = diagnostic.contains("error msb6006") + && diagnostic.contains("\"dotnet.exe\" exited with code 1") + && diagnostic.contains("error netsdk1144") + && !has_actionable_dotnet_diagnostic; // "Pipe is broken" errors from WASI SDK parallel builds combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") @@ -236,6 +244,10 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { // diagnostic. This is a transient child-process failure, not bad source. || silent_rustc_exit || silent_cargo_exit + // The .NET linker occasionally exits without a source diagnostic and + // MSBuild reports only MSB6006 + NETSDK1144. Do not retry when the + // output also contains a C#, SpacetimeDB codegen, or IL source error. + || dotnet_linker_exit // A child process can occasionally disappear on Windows after login but // before the CLI emits a compiler/MSBuild diagnostic. Retrying is safe: // benchmark publishes always use their own temporary CLI root and database. @@ -287,6 +299,22 @@ mod tests { "" )); } + + #[test] + fn retries_dotnet_linker_exit_without_source_diagnostic() { + assert!(is_transient_build_error( + "Error: command [\"dotnet\", \"publish\"] exited with code 1", + "Microsoft.NET.ILLink.targets(134,5): error MSB6006: \"dotnet.exe\" exited with code 1.\nMicrosoft.NET.ILLink.targets(87,5): error NETSDK1144: Optimizing assemblies for size failed." + )); + } + + #[test] + fn does_not_retry_dotnet_linker_exit_with_source_diagnostic() { + assert!(!is_transient_build_error( + "Error: command [\"dotnet\", \"publish\"] exited with code 1", + "Lib.cs(10,5): error CS0103: The name 'missing' does not exist.\nMicrosoft.NET.ILLink.targets(134,5): error MSB6006: \"dotnet.exe\" exited with code 1.\nMicrosoft.NET.ILLink.targets(87,5): error NETSDK1144: Optimizing assemblies for size failed." + )); + } } fn run(cmd: &mut Command, label: &str) -> Result<()> { From 25e0b500b93f619bf322388913646212373c57ac Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:37:19 -0400 Subject: [PATCH 51/78] updates --- crates/cli/src/api.rs | 13 + crates/cli/src/subcommands/describe.rs | 58 +- tools/xtask-llm-benchmark/Cargo.toml | 3 + tools/xtask-llm-benchmark/src/bench/mod.rs | 2 +- .../src/bench/publishers.rs | 4 +- tools/xtask-llm-benchmark/src/bench/runner.rs | 97 ++-- tools/xtask-llm-benchmark/src/bench/utils.rs | 27 + .../src/bin/llm_benchmark.rs | 11 +- .../xtask-llm-benchmark/src/eval/defaults.rs | 15 +- tools/xtask-llm-benchmark/src/eval/scorers.rs | 494 +++++++++++++----- tools/xtask-llm-benchmark/src/eval/utils.rs | 81 ++- tools/xtask-llm-benchmark/src/llm/prompt.rs | 20 +- 12 files changed, 584 insertions(+), 241 deletions(-) diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index d40b03ee87e..33ca6e547d0 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,6 +4,7 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; @@ -72,6 +73,18 @@ impl ClientApi { Ok(module_def) } + /// Reads the version 10 `ModuleDef` from the `schema` endpoint. + pub async fn module_def_v10(&self) -> anyhow::Result { + let res = self + .client + .get(self.con.db_uri("schema")) + .query(&[("version", "10")]) + .send() + .await?; + let DeserializeWrapper(module_def) = res.json_or_error().await?; + Ok(module_def) + } + pub async fn call(&self, reducer_name: &str, arg_json: String) -> anyhow::Result { Ok(self .client diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index e774224855c..b1de53aedd1 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -38,6 +38,13 @@ pub fn cli() -> clap::Command { .action(ArgAction::SetTrue) .help("Ignore spacetime.json configuration"), ) + .arg( + Arg::new("schema_version") + .long("schema-version") + .value_parser(["9", "10"]) + .default_value("9") + .help("Schema ABI version to request"), + ) .after_help("Run `spacetime help describe` for more detailed information.\n") } @@ -52,6 +59,10 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error let json = args.get_flag("json"); let no_config = args.get_flag("no_config"); + let schema_version = args + .get_one::("schema_version") + .map(String::as_str) + .unwrap_or("9"); let raw_parts: Vec = args .get_many::("describe_parts") .map(|vals| vals.cloned().collect()) @@ -97,30 +108,41 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; - if json { fn sats_to_json(v: &T) -> serde_json::Result { serde_json::to_string_pretty(sats::serde::SerdeWrapper::from_ref(v)) } - let json = match entity { - Some((EntityType::Reducer, reducer_name)) => { - let reducer = module_def - .reducers - .iter() - .find(|r| *r.name == *reducer_name) - .context("no such reducer")?; - sats_to_json(reducer)? + let json = match schema_version { + "9" => { + let module_def = api.module_def().await?; + match entity { + Some((EntityType::Reducer, reducer_name)) => { + let reducer = module_def + .reducers + .iter() + .find(|r| *r.name == *reducer_name) + .context("no such reducer")?; + sats_to_json(reducer)? + } + Some((EntityType::Table, table_name)) => { + let table = module_def + .tables + .iter() + .find(|t| *t.name == *table_name) + .context("no such table")?; + sats_to_json(table)? + } + None => sats_to_json(&module_def)?, + } } - Some((EntityType::Table, table_name)) => { - let table = module_def - .tables - .iter() - .find(|t| *t.name == *table_name) - .context("no such table")?; - sats_to_json(table)? + "10" => { + anyhow::ensure!( + entity.is_none(), + "entity filtering is only supported for schema version 9" + ); + sats_to_json(&api.module_def_v10().await?)? } - None => sats_to_json(&module_def)?, + _ => unreachable!("clap validates schema_version"), }; // TODO: validate the JSON output diff --git a/tools/xtask-llm-benchmark/Cargo.toml b/tools/xtask-llm-benchmark/Cargo.toml index bce7d73e717..a0d70f49af6 100644 --- a/tools/xtask-llm-benchmark/Cargo.toml +++ b/tools/xtask-llm-benchmark/Cargo.toml @@ -30,6 +30,9 @@ regex = "1" heck = "0.5.0" thiserror = "2.0.17" +[dev-dependencies] +spacetimedb-lib = { workspace = true, features = ["serde"] } + [lib] path = "src/lib.rs" diff --git a/tools/xtask-llm-benchmark/src/bench/mod.rs b/tools/xtask-llm-benchmark/src/bench/mod.rs index 6b1ebb1cf98..24fcb04df00 100644 --- a/tools/xtask-llm-benchmark/src/bench/mod.rs +++ b/tools/xtask-llm-benchmark/src/bench/mod.rs @@ -9,4 +9,4 @@ pub(crate) mod utils; pub use publishers::{DotnetPublisher, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; pub use runner::TaskRunner; pub use types::{RunOutcome, TaskPaths}; -pub use utils::bench_route_concurrency; +pub use utils::{bench_route_concurrency, run_scope_tag}; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 64ccb816fa4..c1ce12b6a94 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -1,4 +1,5 @@ use crate::bench::utils::sanitize_db_name; +use crate::eval::spacetime_command; use anyhow::{bail, Context, Result}; use regex::Regex; use std::borrow::Cow; @@ -131,8 +132,7 @@ impl CliRootDir { } fn spacetime_cmd(cli_root: &CliRootDir) -> Command { - let spacetime = env::var_os("LLM_BENCH_SPACETIME_BIN").unwrap_or_else(|| "spacetime".into()); - let mut cmd = Command::new(spacetime); + let mut cmd = spacetime_command(); cmd.arg("--root-dir").arg(cli_root.path()); cmd } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 7a70069d705..bd05e8a28b5 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -16,7 +16,7 @@ use crate::bench::types::{BenchRunContext, PublishParams, RunContext, RunOneErro pub(crate) use crate::bench::types::{RunOutcome, TaskPaths}; use crate::bench::utils::{ bench_concurrency, bench_csharp_concurrency, bench_rust_concurrency, category_slug, debug_llm, fmt_dur, - print_llm_output, sanitize_db_name, task_slug, work_server_dir_scoped, + golden_db_name, print_llm_output, run_scope_tag, sanitize_db_name, task_slug, work_server_dir_scoped, }; use crate::bench::Publisher; use crate::eval::scorers::call_reducer_json_out; @@ -38,7 +38,7 @@ struct PendingRetry { error: anyhow::Error, } -fn build_key(lang: Lang, selectors: Option<&[String]>) -> String { +fn build_key(lang: Lang, selectors: Option<&[String]>, golden_scope: &str) -> String { let v = match selectors { Some(s) if !s.is_empty() => { let mut t = s.to_vec(); @@ -48,18 +48,19 @@ fn build_key(lang: Lang, selectors: Option<&[String]>) -> String { _ => vec!["ALL".to_string()], }; let joined = v.join(","); - format!("{lang:?}:{joined}") + format!("{lang:?}:{golden_scope}:{joined}") } -/// Build goldens **once per (lang, selector-set)** in this process. +/// Build goldens once per language, route/mode scope, and selector set. /// If selectors is None/empty, that means "ALL tasks". pub async fn ensure_goldens_built_once( host: Option, bench_root: &Path, lang: Lang, selectors: Option<&[String]>, + golden_scope: &str, ) -> Result<()> { - let key = build_key(lang, selectors); + let key = build_key(lang, selectors, golden_scope); let set = BUILT_KEYS.get_or_init(|| Mutex::new(HashSet::new())); { let set = set.lock(); @@ -74,7 +75,7 @@ pub async fn ensure_goldens_built_once( } // IMPORTANT: pass selectors through so we only build needed goldens - build_goldens_only_for_lang(host, bench_root, lang, selectors).await?; + build_goldens_only_for_lang(host, bench_root, lang, selectors, golden_scope).await?; // mark as built drop(set_guard); @@ -129,30 +130,8 @@ impl TaskRunner { } } - pub async fn publish_golden_only( - &self, - lang: Lang, - category: &str, - task_id: &str, - golden_src_text: &str, - golden_db: String, - host: Option, - clear_database: bool, - ) -> Result<()> { - self.publish( - PublishParams { - lang, - category, - task_id, - route_tag: "", - source_text: golden_src_text, - db_name: golden_db, - host, - clear_database, - }, - "golden", - ) - .await + pub async fn publish_golden_only(&self, params: PublishParams<'_>) -> Result<()> { + self.publish(params, "golden").await } async fn publish_llm(&self, params: PublishParams<'_>) -> Result<()> { @@ -208,8 +187,8 @@ impl TaskRunner { let category = category_slug(&task.root); let task_id = task_slug(&task.root); - let route_tag = sanitize_db_name(&cfg.route.display_name); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", category, task_id)); + let route_tag = run_scope_tag(cfg.mode, cfg.route.vendor.slug(), &cfg.route.api_model); + let golden_db = golden_db_name(&category, &task_id, &route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", category, task_id, route_tag)); let ctor = resolve_by_path(&task.root)?; @@ -415,7 +394,7 @@ impl TaskRunner { golden_db: Some(golden_db), llm_db: Some(llm_db), work_dir_golden: Some( - work_server_dir_scoped(&category, &task_id, cfg.lang_name, "golden", "") + work_server_dir_scoped(&category, &task_id, cfg.lang_name, "golden", &route_tag) .to_string_lossy() .into_owned(), ), @@ -947,6 +926,7 @@ pub async fn build_goldens_only_for_lang( bench_root: &Path, lang: Lang, selectors: Option<&[String]>, + golden_scope: &str, ) -> Result<()> { let tasks = if let Some(sels) = selectors { let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; @@ -989,38 +969,49 @@ pub async fn build_goldens_only_for_lang( async move { let category = category_slug(&task.root); let task_id = task_slug(&task.root); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", category, task_id)); + let golden_db = golden_db_name(&category, &task_id, golden_scope); let golden_src_text = load_golden_source(&task, lang)?; let migration_setup = load_migration_setup_source(&task, lang)?; println!("→ [{}] build golden {} {}", lang_name, category, task_id); if let Some(setup_source) = migration_setup { runner - .publish_golden_only( + .publish_golden_only(PublishParams { lang, - &category, - &task_id, - &setup_source, - golden_db.clone(), - host_clone.clone(), - true, - ) + category: &category, + task_id: &task_id, + route_tag: golden_scope, + source_text: &setup_source, + db_name: golden_db.clone(), + host: host_clone.clone(), + clear_database: true, + }) .await?; call_reducer_json_out(&golden_db, "seed", &[], Some(host_clone.as_deref().unwrap_or("local"))) .map_err(anyhow::Error::msg)?; runner - .publish_golden_only( + .publish_golden_only(PublishParams { lang, - &category, - &task_id, - &golden_src_text, - golden_db, - host_clone, - false, - ) + category: &category, + task_id: &task_id, + route_tag: golden_scope, + source_text: &golden_src_text, + db_name: golden_db, + host: host_clone, + clear_database: false, + }) .await } else { runner - .publish_golden_only(lang, &category, &task_id, &golden_src_text, golden_db, host_clone, true) + .publish_golden_only(PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag: golden_scope, + source_text: &golden_src_text, + db_name: golden_db, + host: host_clone, + clear_database: true, + }) .await } } @@ -1045,7 +1036,8 @@ pub async fn validate_goldens_for_lang( lang: Lang, selectors: Option<&[String]>, ) -> Result<()> { - build_goldens_only_for_lang(host.clone(), bench_root, lang, selectors).await?; + let route_tag = "golden-validation"; + build_goldens_only_for_lang(host.clone(), bench_root, lang, selectors, route_tag).await?; let tasks = if let Some(sels) = selectors { let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; @@ -1071,7 +1063,6 @@ pub async fn validate_goldens_for_lang( TypeScriptPublisher, ); let lang_name = lang.as_str(); - let route_tag = "golden-validation"; let host_url = host.as_deref().unwrap_or("local"); let mut failures = Vec::new(); diff --git a/tools/xtask-llm-benchmark/src/bench/utils.rs b/tools/xtask-llm-benchmark/src/bench/utils.rs index 6e28315e4f6..4d4ab2f02c3 100644 --- a/tools/xtask-llm-benchmark/src/bench/utils.rs +++ b/tools/xtask-llm-benchmark/src/bench/utils.rs @@ -45,6 +45,14 @@ pub fn sanitize_db_name(raw: &str) -> String { out } +pub fn run_scope_tag(mode: &str, vendor: &str, api_model: &str) -> String { + sanitize_db_name(&format!("{mode}-{vendor}-{api_model}")) +} + +pub fn golden_db_name(category: &str, task: &str, scope: &str) -> String { + sanitize_db_name(&format!("{category}-{task}-{scope}-golden")) +} + pub fn work_server_dir_scoped(category: &str, task: &str, lang: &str, phase: &str, route_tag: &str) -> PathBuf { let target = env::var("CARGO_TARGET_DIR").unwrap_or_else(|_| "target".into()); Path::new(&target) @@ -137,3 +145,22 @@ pub fn fmt_dur(d: Duration) -> String { format!("{}m {:.1}s", m, s) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn run_scope_separates_modes_and_models() { + let guidelines = run_scope_tag("guidelines", "openai", "gpt-5.4-mini"); + let docs = run_scope_tag("docs", "openai", "gpt-5.4-mini"); + let other_model = run_scope_tag("guidelines", "openai", "gpt-5.5"); + + assert_ne!(guidelines, docs); + assert_ne!(guidelines, other_model); + assert_ne!( + golden_db_name("views", "t_067", &guidelines), + golden_db_name("views", "t_067", &docs) + ); + } +} diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 9aec12cd932..801a2e09055 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -313,14 +313,6 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { } let llm_provider = if !config.goldens_only && !config.hash_only { - let rt = runtime.as_ref().expect("failed to initialize runtime for goldens"); - rt.block_on(ensure_goldens_built_once( - config.host.clone(), - &bench_root, - config.lang, - selectors_ref, - ))?; - let provider = make_provider_from_env()?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); @@ -763,6 +755,9 @@ async fn run_many_routes_for_mode( async move { println!("\u{2192} running {}", route.display_name); + let golden_scope = xtask_llm_benchmark::bench::run_scope_tag(mode, route.vendor.slug(), &route.api_model); + ensure_goldens_built_once(host.clone(), bench_root, lang, selectors, &golden_scope).await?; + let per = BenchRunContext { bench_root, mode, diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index 3b7d6a1e5d6..7652426d76e 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -1,4 +1,4 @@ -use crate::bench::utils::sanitize_db_name; +use crate::bench::utils::{golden_db_name, sanitize_db_name}; use crate::eval::scorers::{ CallOutputParityScorer, EventuallySqlCountScorer, HttpRouteCase, HttpRouteParityScorer, ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, @@ -10,7 +10,7 @@ use std::time::Duration; pub fn default_schema_parity_scorers(host_url: &str, src_file: &str, route_tag: &str) -> Vec> { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); vec![Box::new(SchemaParityScorer { @@ -82,7 +82,7 @@ pub fn make_eventually_sql_count_scorer( pub fn make_reducer_data_parity_scorer(host_url: &str, cfg: ReducerDataParityConfig<'_>) -> Box { let (cat, task) = derive_cat_task_from_file(cfg.src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, cfg.route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, cfg.route_tag)); Box::new(ReducerDataParityScorer { @@ -107,7 +107,7 @@ pub fn make_sql_exec_both_scorer( timeout: Duration, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); Box::new(SqlExecBothScorer { @@ -160,7 +160,7 @@ pub fn make_reducer_call_both_scorer_with_attempts( attempts: usize, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); Box::new(ReducerCallBothScorer { @@ -195,7 +195,7 @@ pub fn make_call_output_parity_scorer_with_attempts( attempts: usize, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); Box::new(CallOutputParityScorer { server: host_url.to_string(), @@ -218,7 +218,7 @@ pub fn make_http_route_parity_scorer( id_str: &'static str, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); Box::new(HttpRouteParityScorer { server: host_url.to_string(), @@ -226,6 +226,7 @@ pub fn make_http_route_parity_scorer( llm_db, id_str, compare_content_type, + timeout: Duration::from_secs(10), cases: cases .into_iter() .map(|(method, path, body)| HttpRouteCase { diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index bc7d8ae0460..1cc162b67f3 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -1,9 +1,9 @@ use crate::bench::utils::debug_llm_verbose; -use crate::eval::{normalize, sql_exec, ScoreDetails}; +use crate::eval::utils::{run_with_timeout, sql_exec_with_timeout}; +use crate::eval::{normalize, spacetime_command, ScoreDetails}; use serde_json::{json, Value}; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use std::{io, thread}; @@ -20,6 +20,14 @@ pub struct SchemaParityScorer { pub id_str: &'static str, } +#[derive(Debug, Default, PartialEq, Eq)] +struct SchemaSnapshot { + tables: BTreeMap>, + reducers: BTreeSet, + exports: BTreeSet, + row_level_security: BTreeSet, +} + impl Scorer for SchemaParityScorer { fn id(&self) -> &'static str { self.id_str @@ -44,13 +52,14 @@ impl Scorer for SchemaParityScorer { } } - let (tables_a, reducers_a, rls_a) = extract_schema(&golden); - let (tables_b, reducers_b, rls_b) = extract_schema(&llm); + let schema_a = extract_schema(&golden); + let schema_b = extract_schema(&llm); - let tables_diff = diff_maps(&tables_a, &tables_b); - let reducers_diff = diff_sets(&reducers_a, &reducers_b); - let rls_diff = diff_sets(&rls_a, &rls_b); - let pass = tables_diff.is_null() && reducers_diff.is_null() && rls_diff.is_null(); + let tables_diff = diff_maps(&schema_a.tables, &schema_b.tables); + let reducers_diff = diff_sets(&schema_a.reducers, &schema_b.reducers); + let exports_diff = diff_sets(&schema_a.exports, &schema_b.exports); + let rls_diff = diff_sets(&schema_a.row_level_security, &schema_b.row_level_security); + let pass = tables_diff.is_null() && reducers_diff.is_null() && exports_diff.is_null() && rls_diff.is_null(); ScoreDetails { pass, @@ -61,9 +70,11 @@ impl Scorer for SchemaParityScorer { "llm_db": self.llm_db, "tables_equal": tables_diff.is_null(), "reducers_equal": reducers_diff.is_null(), + "exports_equal": exports_diff.is_null(), "row_level_security_equal": rls_diff.is_null(), "tables_diff": tables_diff, "reducers_diff": reducers_diff, + "exports_diff": exports_diff, "row_level_security_diff": rls_diff, }), } @@ -72,32 +83,12 @@ impl Scorer for SchemaParityScorer { /* helpers */ -fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Result<(i32, Vec, Vec)> { - let mut child = cmd - .current_dir(cwd) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - let start = Instant::now(); - loop { - if let Some(status) = child.try_wait()? { - let out = child.wait_with_output()?; - let code = status.code().unwrap_or(-1); - return Ok((code, out.stdout, out.stderr)); - } - if start.elapsed() > timeout { - let _ = child.kill(); - let _ = child.wait(); - return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); - } - thread::sleep(Duration::from_millis(30)); - } -} - fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { - let mut cmd = Command::new("spacetime"); + let mut cmd = spacetime_command(); cmd.arg("describe") .arg("--json") + .arg("--schema-version") + .arg("10") .arg("-s") .arg(server) .arg("-y") @@ -113,25 +104,18 @@ fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { Ok(v) } -fn extract_schema( - v: &Value, -) -> ( - BTreeMap>, - BTreeSet, - BTreeSet, -) { - let mut tables: BTreeMap> = BTreeMap::new(); - let mut reducers: BTreeSet = BTreeSet::new(); - let mut row_level_security: BTreeSet = BTreeSet::new(); - let types = v - .pointer("/typespace/types") +fn extract_schema(v: &Value) -> SchemaSnapshot { + let mut schema = SchemaSnapshot::default(); + let typespace = v.get("typespace").or_else(|| section(v, "Typespace")); + let types = typespace + .and_then(|value| value.get("types")) .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or_default(); - if let Some(ts) = v.get("tables").and_then(|x| x.as_array()) { + if let Some(ts) = schema_array(v, "tables", "Tables") { for t in ts { - let name = t.get("name").and_then(|x| x.as_str()).unwrap_or("").to_string(); + let name = schema_name(t.get("source_name").or_else(|| t.get("name"))); let mut cols = BTreeMap::new(); // Older CLI descriptions put columns directly on the table. Keep @@ -187,44 +171,141 @@ fn extract_schema( insert_schema_property(&mut cols, "schedule", canonical_value(t.get("schedule"))); insert_schema_property(&mut cols, "table_type", canonical_value(t.get("table_type"))); insert_schema_property(&mut cols, "table_access", canonical_value(t.get("table_access"))); - tables.insert(name, cols); + insert_schema_property(&mut cols, "default_values", canonical_value(t.get("default_values"))); + insert_schema_property(&mut cols, "is_event", canonical_value(t.get("is_event"))); + schema.tables.insert(name, cols); } } - if let Some(rs) = v.get("reducers").and_then(|x| x.as_array()) { + if let Some(rs) = schema_array(v, "reducers", "Reducers") { for r in rs { - let name = r.get("name").and_then(|x| x.as_str()).unwrap_or(""); - let params = r - .pointer("/params/elements") - .and_then(Value::as_array) - .or_else(|| r.get("args").and_then(Value::as_array)); - let params = params - .into_iter() - .flatten() - .map(|param| { - param - .get("algebraic_type") - .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) - .or_else(|| param.get("type").and_then(Value::as_str).map(str::to_owned)) - .unwrap_or_default() - }) - .collect::>(); - let sig = format!("{}({})", name, params.join(",")); - reducers.insert(sig); + let name = schema_name(r.get("source_name").or_else(|| r.get("name"))); + let mut sig = format!("{}({})", name, parameter_types(r, types).join(",")); + append_field(&mut sig, "visibility", r.get("visibility")); + append_type(&mut sig, "ok", r.get("ok_return_type"), types); + append_type(&mut sig, "err", r.get("err_return_type"), types); + append_field(&mut sig, "lifecycle", r.get("lifecycle")); + schema.reducers.insert(sig); } } - for rule in v - .get("row_level_security") - .or_else(|| v.get("rowLevelSecurity")) - .and_then(Value::as_array) + for export in v.get("misc_exports").and_then(Value::as_array).into_iter().flatten() { + if let Some(procedure) = export.get("Procedure") { + schema.exports.insert(function_export("procedure", procedure, types)); + } else if let Some(view) = export.get("View") { + schema.exports.insert(view_export(view, types)); + } else if let Some(default) = export.get("ColumnDefaultValue") { + schema + .exports + .insert(format!("column_default:{}", canonical_value(Some(default)))); + } + } + + for procedure in schema_array(v, "procedures", "Procedures").into_iter().flatten() { + schema.exports.insert(function_export("procedure", procedure, types)); + } + for view in schema_array(v, "views", "Views").into_iter().flatten() { + schema.exports.insert(view_export(view, types)); + } + insert_canonical_exports(&mut schema.exports, v, "schedules", "Schedules", "schedule"); + insert_canonical_exports( + &mut schema.exports, + v, + "life_cycle_reducers", + "LifeCycleReducers", + "lifecycle", + ); + insert_canonical_exports(&mut schema.exports, v, "http_handlers", "HttpHandlers", "http_handler"); + insert_canonical_exports(&mut schema.exports, v, "http_routes", "HttpRoutes", "http_route"); + insert_canonical_exports( + &mut schema.exports, + v, + "view_primary_keys", + "ViewPrimaryKeys", + "view_primary_key", + ); + + for rule in schema_array(v, "row_level_security", "RowLevelSecurity") .into_iter() .flatten() { - row_level_security.insert(canonical_value(Some(rule))); + schema.row_level_security.insert(canonical_value(Some(rule))); + } + + schema +} + +fn section<'a>(v: &'a Value, name: &str) -> Option<&'a Value> { + v.get("sections") + .and_then(Value::as_array) + .into_iter() + .flatten() + .find_map(|item| item.get(name)) +} + +fn schema_array<'a>(v: &'a Value, direct_name: &str, section_name: &str) -> Option<&'a Vec> { + v.get(direct_name) + .or_else(|| section(v, section_name)) + .and_then(Value::as_array) +} + +fn parameter_types(function: &Value, types: &[Value]) -> Vec { + function + .pointer("/params/elements") + .and_then(Value::as_array) + .or_else(|| function.get("args").and_then(Value::as_array)) + .into_iter() + .flatten() + .map(|param| { + param + .get("algebraic_type") + .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) + .or_else(|| param.get("type").and_then(Value::as_str).map(str::to_owned)) + .unwrap_or_default() + }) + .collect() +} + +fn append_field(signature: &mut String, label: &str, value: Option<&Value>) { + if let Some(value) = value { + signature.push_str(&format!("|{label}={}", canonical_value(Some(value)))); + } +} + +fn append_type(signature: &mut String, label: &str, value: Option<&Value>, types: &[Value]) { + if let Some(value) = value { + signature.push_str(&format!( + "|{label}={}", + canonical_type(value, types, &mut BTreeSet::new()) + )); } +} - (tables, reducers, row_level_security) +fn function_export(kind: &str, function: &Value, types: &[Value]) -> String { + let name = schema_name(function.get("source_name").or_else(|| function.get("name"))); + let mut signature = format!("{kind}:{name}({})", parameter_types(function, types).join(",")); + append_type(&mut signature, "return", function.get("return_type"), types); + append_field(&mut signature, "visibility", function.get("visibility")); + signature +} + +fn view_export(view: &Value, types: &[Value]) -> String { + let mut signature = function_export("view", view, types); + append_field(&mut signature, "public", view.get("is_public")); + append_field(&mut signature, "anonymous", view.get("is_anonymous")); + signature +} + +fn insert_canonical_exports( + exports: &mut BTreeSet, + schema: &Value, + direct_name: &str, + section_name: &str, + kind: &str, +) { + for export in schema_array(schema, direct_name, section_name).into_iter().flatten() { + exports.insert(format!("{kind}:{}", canonical_value(Some(export)))); + } } fn schema_name(value: Option<&Value>) -> String { @@ -328,8 +409,13 @@ fn normalize_sequences(value: Option<&Value>, columns: &[String]) -> String { .get("column") .map(|value| column_name(value, columns)) .unwrap_or_default(); - let increment = sequence.get("increment").and_then(Value::as_i64).unwrap_or_default(); - sequences.insert(format!("{column}:{increment}")); + let increment = canonical_value(sequence.get("increment")); + let start = canonical_value(sequence.get("start")); + let min = canonical_value(sequence.get("min_value")); + let max = canonical_value(sequence.get("max_value")); + sequences.insert(format!( + "{column}:increment={increment}:start={start}:min={min}:max={max}" + )); } sequences.into_iter().collect::>().join(";") } @@ -394,7 +480,17 @@ fn err_details(phase: &str, e: io::Error) -> ScoreDetails { /* reducer/sql helpers */ pub fn call_reducer_json_out(db: &str, reducer: &str, args: &[Value], host: Option<&str>) -> Result { - let mut cmd = Command::new("spacetime"); + call_reducer_json_out_with_timeout(db, reducer, args, host, Duration::from_secs(30)) +} + +fn call_reducer_json_out_with_timeout( + db: &str, + reducer: &str, + args: &[Value], + host: Option<&str>, + timeout: Duration, +) -> Result { + let mut cmd = spacetime_command(); cmd.arg("call").arg(db).arg(reducer); for v in args { @@ -409,7 +505,7 @@ pub fn call_reducer_json_out(db: &str, reducer: &str, args: &[Value], host: Opti eprintln!("[dbg] spacetime call: {:?}", cmd); } - let (code, stdout, stderr) = run_with_timeout(cmd, Path::new("."), Duration::from_secs(30)) + let (code, stdout, stderr) = run_with_timeout(cmd, Path::new("."), timeout) .map_err(|e| format!("spacetime call failed or timed out: {e}"))?; if debug_llm_verbose() { eprintln!( @@ -426,7 +522,11 @@ pub fn call_reducer_json_out(db: &str, reducer: &str, args: &[Value], host: Opti } pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result { - let mut cmd = Command::new("spacetime"); + sql_raw_with_timeout(db, query, host, Duration::from_secs(30)) +} + +fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + let mut cmd = spacetime_command(); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -436,28 +536,28 @@ pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result) -> Result { - let mut cmd = Command::new("spacetime"); + sql_count_with_timeout(db, query, host, Duration::from_secs(30)) +} + +fn sql_count_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + let mut cmd = spacetime_command(); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -467,24 +567,20 @@ pub fn sql_count(db: &str, query: &str, host: Option<&str>) -> Result() { if debug_llm_verbose() { @@ -522,7 +618,8 @@ impl Scorer for ReducerSqlEqualsScorer { self.reducer, self.args, self.db, self.server ); } - let call_res = call_reducer_json_out(&self.db, &self.reducer, &self.args, Some(&self.server)); + let call_res = + call_reducer_json_out_with_timeout(&self.db, &self.reducer, &self.args, Some(&self.server), self.timeout); if let Err(e) = call_res { return ScoreDetails { pass: false, @@ -534,7 +631,7 @@ impl Scorer for ReducerSqlEqualsScorer { if debug_llm_verbose() { eprintln!("[dbg] ReducerSqlEqualsScorer: running sql: {}", self.sql); } - match sql_raw(&self.db, &self.sql, Some(&self.server)) { + match sql_raw_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { Ok(out) => { let actual = normalize(&out, self.collapse_ws); let expected = normalize(&self.expected, self.collapse_ws); @@ -591,7 +688,8 @@ impl Scorer for ReducerSqlCountScorer { self.reducer, self.args, self.db, self.server ); } - let call = call_reducer_json_out(&self.db, &self.reducer, &self.args, Some(&self.server)); + let call = + call_reducer_json_out_with_timeout(&self.db, &self.reducer, &self.args, Some(&self.server), self.timeout); if let Err(e) = call { return ScoreDetails { pass: false, @@ -603,7 +701,7 @@ impl Scorer for ReducerSqlCountScorer { if debug_llm_verbose() { eprintln!("[dbg] ReducerSqlCountScorer: running sql: {}", self.sql); } - match sql_count(&self.db, &self.sql, Some(&self.server)) { + match sql_count_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { Ok(n) => { let pass = n == self.expected; if debug_llm_verbose() { @@ -649,14 +747,26 @@ impl Scorer for ReducerDataParityScorer { ); } - if let Err(e) = call_reducer_json_out(&self.golden_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_reducer_json_out_with_timeout( + &self.golden_db, + &self.reducer, + &self.args, + Some(&self.server), + self.timeout, + ) { return ScoreDetails { pass: false, partial: 0.0, notes: json!({"phase":"call_reducer_golden","error":e}), }; } - if let Err(e) = call_reducer_json_out(&self.llm_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_reducer_json_out_with_timeout( + &self.llm_db, + &self.reducer, + &self.args, + Some(&self.server), + self.timeout, + ) { return ScoreDetails { pass: false, partial: 0.0, @@ -667,7 +777,7 @@ impl Scorer for ReducerDataParityScorer { if debug_llm_verbose() { eprintln!("[dbg] query for parity: {}", self.query); } - let g = match sql_raw(&self.golden_db, &self.query, Some(&self.server)) { + let g = match sql_raw_with_timeout(&self.golden_db, &self.query, Some(&self.server), self.timeout) { Ok(s) => s, Err(e) => { return ScoreDetails { @@ -677,7 +787,7 @@ impl Scorer for ReducerDataParityScorer { } } }; - let l = match sql_raw(&self.llm_db, &self.query, Some(&self.server)) { + let l = match sql_raw_with_timeout(&self.llm_db, &self.query, Some(&self.server), self.timeout) { Ok(s) => s, Err(e) => { return ScoreDetails { @@ -750,7 +860,15 @@ impl Scorer for EventuallySqlCountScorer { fn score(&self, _llm_output: &str) -> ScoreDetails { let started = Instant::now(); loop { - let last = match sql_count(&self.db, &self.sql, Some(&self.server)) { + let remaining = self.timeout.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "sql": self.sql, "expected": self.expected, "last": { "error": "timeout" } }), + }; + } + let last = match sql_count_with_timeout(&self.db, &self.sql, Some(&self.server), remaining) { Ok(actual) if actual == self.expected => { return ScoreDetails { pass: true, @@ -778,7 +896,7 @@ impl Scorer for SqlCountOnlyScorer { self.id_str } fn score(&self, _llm_output: &str) -> ScoreDetails { - match sql_count(&self.db, &self.sql, Some(&self.server)) { + match sql_count_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { Ok(n) => { let pass = n == self.expected; ScoreDetails { @@ -847,14 +965,14 @@ impl Scorer for SqlExecBothScorer { self.sql, self.golden_db, self.llm_db, self.server ); } - if let Err(e) = sql_exec(&self.golden_db, &self.sql, Some(&self.server)) { + if let Err(e) = sql_exec_with_timeout(&self.golden_db, &self.sql, Some(&self.server), self.timeout) { return ScoreDetails { pass: false, partial: 0.0, notes: json!({ "phase":"sql_golden", "error": e, "sql": self.sql }), }; } - if let Err(e) = sql_exec(&self.llm_db, &self.sql, Some(&self.server)) { + if let Err(e) = sql_exec_with_timeout(&self.llm_db, &self.sql, Some(&self.server), self.timeout) { return ScoreDetails { pass: false, partial: 0.0, @@ -926,10 +1044,16 @@ pub struct HttpRouteParityScorer { pub llm_db: String, pub cases: Vec, pub compare_content_type: bool, + pub timeout: Duration, pub id_str: &'static str, } -fn call_http_route(server: &str, db: &str, case: &HttpRouteCase) -> Result<(u16, String, String), String> { +fn call_http_route( + server: &str, + db: &str, + case: &HttpRouteCase, + timeout: Duration, +) -> Result<(u16, String, String), String> { let server = server.trim_end_matches('/').to_string(); let db = db.to_string(); let method = case.method.clone(); @@ -940,7 +1064,11 @@ fn call_http_route(server: &str, db: &str, case: &HttpRouteCase) -> Result<(u16, runtime.block_on(async move { let method = reqwest::Method::from_bytes(method.as_bytes()).map_err(|error| error.to_string())?; let url = format!("{server}/v1/database/{db}/route{path}"); - let mut request = reqwest::Client::new().request(method, url); + let client = reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|error| error.to_string())?; + let mut request = client.request(method, url); if let Some(body) = body { request = request.header("content-type", "text/plain").body(body); } @@ -981,7 +1109,7 @@ impl Scorer for HttpRouteParityScorer { let mut golden_results = Vec::new(); let mut llm_results = Vec::new(); for case in &self.cases { - match call_http_route(&self.server, &self.golden_db, case) { + match call_http_route(&self.server, &self.golden_db, case, self.timeout) { Ok(result) => golden_results.push(result), Err(error) => { return ScoreDetails { @@ -991,7 +1119,7 @@ impl Scorer for HttpRouteParityScorer { } } } - match call_http_route(&self.server, &self.llm_db, case) { + match call_http_route(&self.server, &self.llm_db, case, self.timeout) { Ok(result) => llm_results.push(result), Err(error) => { return ScoreDetails { @@ -1092,6 +1220,10 @@ impl Scorer for ReducerCallBothScorer { #[cfg(test)] mod tests { use super::*; + use spacetimedb_lib::db::raw_def::v10::{MethodOrAny, RawModuleDefV10Builder}; + use spacetimedb_lib::db::raw_def::v9::Lifecycle; + use spacetimedb_lib::sats::serde::SerdeWrapper; + use spacetimedb_lib::sats::{AlgebraicType, ProductType}; fn current_schema(include_owner_index: bool) -> Value { let mut indexes = vec![json!({ "algorithm": { "BTree": [0] } })]; @@ -1126,26 +1258,26 @@ mod tests { #[test] fn current_schema_extracts_columns_and_table_properties() { - let (tables, reducers, row_level_security) = extract_schema(¤t_schema(true)); - let child_item = &tables["child_item"]; + let schema = extract_schema(¤t_schema(true)); + let child_item = &schema.tables["child_item"]; assert_eq!(child_item["id"], r#"{"U64":[]}"#); assert_eq!(child_item["owner_id"], r#"{"U64":[]}"#); assert_eq!(child_item["@primary_key"], "id"); assert_eq!(child_item["@indexes"], "BTree(id);BTree(owner_id)"); assert_eq!(child_item["@constraints"], "Unique(id)"); - assert_eq!(child_item["@sequences"], "id:1"); + assert_eq!(child_item["@sequences"], "id:increment=1:start=:min=:max="); assert_eq!(child_item["@table_access"], r#"{"Public":[]}"#); - assert!(reducers.is_empty()); - assert!(row_level_security.is_empty()); + assert!(schema.reducers.is_empty()); + assert!(schema.row_level_security.is_empty()); } #[test] fn missing_index_produces_a_schema_diff() { - let (golden, _, _) = extract_schema(¤t_schema(true)); - let (candidate, _, _) = extract_schema(¤t_schema(false)); + let golden = extract_schema(¤t_schema(true)); + let candidate = extract_schema(¤t_schema(false)); - assert!(!diff_maps(&golden, &candidate).is_null()); + assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); } #[test] @@ -1153,10 +1285,10 @@ mod tests { let mut golden = current_schema(true); golden["row_level_security"] = json!([{ "sql": "SELECT * FROM users WHERE identity = :sender" }]); let candidate = current_schema(true); - let (_, _, golden_rls) = extract_schema(&golden); - let (_, _, candidate_rls) = extract_schema(&candidate); + let golden = extract_schema(&golden); + let candidate = extract_schema(&candidate); - assert!(!diff_sets(&golden_rls, &candidate_rls).is_null()); + assert!(!diff_sets(&golden.row_level_security, &candidate.row_level_security).is_null()); } #[test] @@ -1172,7 +1304,7 @@ mod tests { } }]); - let (_, reducers, _) = extract_schema(&schema); + let reducers = extract_schema(&schema).reducers; assert_eq!( reducers, @@ -1196,8 +1328,8 @@ mod tests { { "name": { "some": "owner" }, "algebraic_type": { "U64": [] } } ] } }]); - let (_, golden_reducers, _) = extract_schema(&golden); - let (_, candidate_reducers, _) = extract_schema(&candidate); + let golden_reducers = extract_schema(&golden).reducers; + let candidate_reducers = extract_schema(&candidate).reducers; assert!(!diff_sets(&golden_reducers, &candidate_reducers).is_null()); } @@ -1218,8 +1350,8 @@ mod tests { { "name": { "some": "reminder" }, "algebraic_type": { "U64": [] } } ] } }]); - let (_, golden_reducers, _) = extract_schema(&golden); - let (_, candidate_reducers, _) = extract_schema(&candidate); + let golden_reducers = extract_schema(&golden).reducers; + let candidate_reducers = extract_schema(&candidate).reducers; assert!(diff_sets(&golden_reducers, &candidate_reducers).is_null()); } @@ -1232,7 +1364,7 @@ mod tests { "args": [{ "name": "owner", "type": "String" }] }]); - let (_, reducers, _) = extract_schema(&schema); + let reducers = extract_schema(&schema).reducers; assert_eq!(reducers, BTreeSet::from(["set_owner(String)".to_owned()])); } @@ -1245,4 +1377,98 @@ mod tests { assert!(http_route_results_equal(&golden, &candidate, false)); assert!(!http_route_results_equal(&golden, &candidate, true)); } + + #[test] + fn v10_schema_extracts_lifecycle_procedures_views_and_http_exports() { + let schema = json!({ + "sections": [ + { "Typespace": { "types": [] } }, + { "Tables": [] }, + { "Reducers": [{ + "source_name": "connected", + "params": { "elements": [] }, + "visibility": { "Private": [] }, + "ok_return_type": { "Unit": [] }, + "err_return_type": { "String": [] } + }] }, + { "Procedures": [{ + "source_name": "fetch", + "params": { "elements": [] }, + "return_type": { "String": [] }, + "visibility": { "ClientCallable": [] } + }] }, + { "Views": [{ + "source_name": "profile", + "params": { "elements": [] }, + "return_type": { "String": [] }, + "is_public": true, + "is_anonymous": false + }] }, + { "LifeCycleReducers": [{ + "lifecycle_spec": { "OnConnect": [] }, + "function_name": "connected" + }] }, + { "HttpHandlers": [{ "source_name": "webhook" }] }, + { "HttpRoutes": [{ + "handler_function": "webhook", + "method": { "Any": [] }, + "path": "/hook" + }] }, + { "ViewPrimaryKeys": [{ "view_source_name": "profile", "columns": ["id"] }] } + ] + }); + + let extracted = extract_schema(&schema); + + assert_eq!(extracted.reducers.len(), 1); + assert!(extracted + .exports + .iter() + .any(|value| value.starts_with("procedure:fetch"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("view:profile"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("lifecycle:"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("http_handler:"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("http_route:"))); + assert!(extracted + .exports + .iter() + .any(|value| value.starts_with("view_primary_key:"))); + } + + #[test] + fn extracts_exports_from_actual_v10_serialization() { + let mut module = RawModuleDefV10Builder::new(); + module.add_lifecycle_reducer(Lifecycle::OnConnect, "connected", ProductType::unit()); + module.add_procedure("fetch", ProductType::unit(), AlgebraicType::unit()); + module.add_view("profile", 0, true, false, ProductType::unit(), AlgebraicType::unit()); + module.add_view_primary_key("profile", ["id"]); + module.add_http_handler("webhook"); + module.add_http_route("webhook", MethodOrAny::Any, "/hook"); + let module = module.finish(); + let json = serde_json::to_value(SerdeWrapper::from_ref(&module)).unwrap(); + + let extracted = extract_schema(&json); + + assert_eq!(extracted.reducers.len(), 1, "serialized schema was {json}"); + assert_eq!(extracted.exports.len(), 6, "serialized schema was {json}"); + assert!( + extracted + .exports + .iter() + .any(|value| value.starts_with("procedure:fetch")), + "serialized schema was {json}" + ); + assert!( + extracted.exports.iter().any(|value| value.starts_with("view:profile")), + "serialized schema was {json}" + ); + + let mut missing_assignments = json.clone(); + missing_assignments["sections"] + .as_array_mut() + .unwrap() + .retain(|section| section.get("LifeCycleReducers").is_none() && section.get("ViewPrimaryKeys").is_none()); + let missing_assignments = extract_schema(&missing_assignments); + assert!(!diff_sets(&extracted.exports, &missing_assignments.exports).is_null()); + } } diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 1fd05c786e4..65023c9f617 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -1,4 +1,9 @@ -use std::process::Command; +use std::env; +use std::io; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; pub fn derive_cat_task_from_file(src: &str) -> (String, String) { let p = std::path::Path::new(src); @@ -18,22 +23,55 @@ pub fn derive_cat_task_from_file(src: &str) -> (String, String) { (cat, task) } +pub(crate) fn spacetime_command() -> Command { + let executable = env::var_os("LLM_BENCH_SPACETIME_BIN").unwrap_or_else(|| "spacetime".into()); + Command::new(executable) +} + pub fn sql_exec(db: &str, query: &str, host: Option<&str>) -> Result<(), String> { - let mut cmd = Command::new("spacetime"); + sql_exec_with_timeout(db, query, host, Duration::from_secs(30)) +} + +pub(crate) fn sql_exec_with_timeout( + db: &str, + query: &str, + host: Option<&str>, + timeout: Duration, +) -> Result<(), String> { + let mut cmd = spacetime_command(); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); } - let out = cmd.output().map_err(|e| format!("spawn spacetime sql failed: {e}"))?; - if !out.status.success() { - return Err(format!( - "spacetime sql failed:\n{}", - String::from_utf8_lossy(&out.stderr) - )); + let (code, _, stderr) = run_with_timeout(cmd, Path::new("."), timeout) + .map_err(|e| format!("spacetime sql failed or timed out: {e}"))?; + if code != 0 { + return Err(format!("spacetime sql failed:\n{}", String::from_utf8_lossy(&stderr))); } Ok(()) } +pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Result<(i32, Vec, Vec)> { + let mut child = cmd + .current_dir(cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let start = Instant::now(); + loop { + if let Some(status) = child.try_wait()? { + let out = child.wait_with_output()?; + return Ok((status.code().unwrap_or(-1), out.stdout, out.stderr)); + } + if start.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); + } + thread::sleep(Duration::from_millis(30)); + } +} + pub fn normalize(s: &str, collapse_ws: bool) -> String { let t = s.trim(); if collapse_ws { @@ -42,3 +80,30 @@ pub fn normalize(s: &str, collapse_ws: bool) -> String { t.to_string() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn process_timeout_kills_a_long_running_command() { + #[cfg(windows)] + let command = { + let mut command = Command::new("powershell"); + command.args(["-NoProfile", "-Command", "Start-Sleep -Seconds 5"]); + command + }; + #[cfg(not(windows))] + let command = { + let mut command = Command::new("sh"); + command.args(["-c", "sleep 5"]); + command + }; + + let started = Instant::now(); + let error = run_with_timeout(command, Path::new("."), Duration::from_millis(100)).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(started.elapsed() < Duration::from_secs(3)); + } +} diff --git a/tools/xtask-llm-benchmark/src/llm/prompt.rs b/tools/xtask-llm-benchmark/src/llm/prompt.rs index bfb15acdc86..6c47bef31c4 100644 --- a/tools/xtask-llm-benchmark/src/llm/prompt.rs +++ b/tools/xtask-llm-benchmark/src/llm/prompt.rs @@ -138,6 +138,16 @@ fn find_tasks_file(task_root: &Path, lang: Lang) -> Option { } } +fn find_setup_file(task_root: &Path, lang: Lang) -> Option { + let file = match lang { + Lang::CSharp => "csharp.cs", + Lang::Rust => "rust.rs", + Lang::TypeScript => "typescript.ts", + }; + let path = task_root.join("setup").join(file); + path.exists().then_some(path) +} + #[cfg(test)] mod tests { use crate::eval::Lang; @@ -177,13 +187,3 @@ mod tests { assert!(prompt.instructions.contains("pub fn touch")); } } - -fn find_setup_file(task_root: &Path, lang: Lang) -> Option { - let file = match lang { - Lang::CSharp => "csharp.cs", - Lang::Rust => "rust.rs", - Lang::TypeScript => "typescript.ts", - }; - let path = task_root.join("setup").join(file); - path.exists().then_some(path) -} From 62aa9e9431c403fd2937afc8e7fd6adc61230dd5 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:50:55 -0400 Subject: [PATCH 52/78] updates --- skills/typescript-server/SKILL.md | 10 ++ .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_061_three_table_join/tasks/csharp.txt | 2 +- .../t_061_three_table_join/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_063_timestamp_window/tasks/csharp.txt | 2 +- .../t_063_timestamp_window/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_064_index_and_filter/tasks/csharp.txt | 2 +- .../t_064_index_and_filter/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_066_query_builder_view/tasks/csharp.txt | 2 +- .../t_066_query_builder_view/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_067_view_primary_key/tasks/csharp.txt | 2 +- .../t_067_view_primary_key/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- tools/xtask-llm-benchmark/src/eval/scorers.rs | 151 ++++++++++++++++-- 26 files changed, 171 insertions(+), 38 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 6165a0fdc19..8a8577e7122 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -40,6 +40,8 @@ export const addRecord = spacetimedb.reducer( ); ``` +Only table definitions belong in `schema({...})`. Row and object builders used as reducer arguments or view return types are not schema entries. + Named runtime exports are reserved for values registered with SpacetimeDB, such as reducers, lifecycle hooks, views, procedures, HTTP exports, and visibility filters. Keep ordinary helper functions and constants unexported. ## Imports @@ -152,6 +154,10 @@ ctx.db.score_record.id.delete(recordId); // Delete by PK Insert through the table accessor (`ctx.db.score_record.insert(...)`). Primary-key, unique, and index accessors support lookup or mutation of existing rows, but do not have `insert(...)`. +The accessor for a primary key or index is the declared column name. For example, a primary key named `eventId` is accessed as `ctx.db.event.eventId`, not `ctx.db.event.id`. + +The schema value registers module exports but does not expose database rows. Pass a context into any helper that needs `ctx.db`. + Note: `iter()` and `filter()` return iterators. Spread to Array for `.sort()`, `.filter()`, `.map()`. ## Lifecycle Hooks @@ -286,6 +292,8 @@ ctx => ctx.from.subscription.rightSemijoin( ) ``` +For `A.leftSemijoin(B, ...)`, the result contains rows from `A`; for `A.rightSemijoin(B, ...)`, it contains rows from `B`. Semijoins do not project combined columns from both tables. Use a procedural view when the result is a custom row assembled from multiple tables. + ## Client Visibility Filters ```typescript @@ -320,6 +328,8 @@ Scheduled procedures use the ordinary scheduled-table shape. Its `scheduled` opt Inbound HTTP uses `httpHandler`, `httpRouter`, `Router`, and `SyncResponse`: +`httpHandler` and `httpRouter` are methods on the local `spacetimedb` schema value, not named imports. + ```typescript import { Router, SyncResponse } from 'spacetimedb/server'; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index cbea5266563..c75fba666f6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -18,6 +18,6 @@ LIFECYCLE HOOKS REDUCERS - ExercisePresence() - Insert synthetic connection IDs 1 and 2 for the caller - - Encode ID N as 16 bytes with byte 0 set to N and all remaining bytes zero + - Use the 128-bit numeric values 1 and 2 - Remove connection ID 1 - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt index fb7eb4d7c04..19f4e53dc7b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -18,6 +18,6 @@ LIFECYCLE HOOKS REDUCERS - exercise_presence() - Insert synthetic connection IDs 1 and 2 for the caller - - Encode ID N as 16 bytes with byte 0 set to N and all remaining bytes zero + - Use the 128-bit numeric values 1 and 2 - Remove connection ID 1 - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt index 4d67cd5040a..53332175766 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -18,6 +18,6 @@ LIFECYCLE HOOKS REDUCERS - exercise_presence() - Insert synthetic connection IDs 1 and 2 for the caller - - Encode ID N as 16 bytes with byte 0 set to N and all remaining bytes zero + - Use the 128-bit numeric values 1 and 2 - Remove connection ID 1 - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt index f9667e88bd4..9fd5ab763e7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -24,6 +24,6 @@ REDUCERS - Use visible LineItem Id 100 for Purchase 10 with Sku "SKU-1" VIEW -- OrderLineDetail (public) +- OrderLineDetail (anonymous procedural, public) - Returns rows with LineId: ulong, CustomerName: string, and Sku: string - Return only rows connected through matching Customer, Purchase, and LineItem IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt index 4912f155896..cd4cf43c491 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -24,6 +24,6 @@ REDUCERS - Use visible line_item id 100 for purchase 10 with sku "SKU-1" VIEW -- order_line_detail (public) +- order_line_detail (anonymous procedural, public) - Returns rows with line_id: u64, customer_name: String, and sku: String - Return only rows connected through matching customer, purchase, and line-item IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt index ceb08d774a0..f930598b039 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -24,6 +24,6 @@ REDUCERS - Use visible line_item id 100 for purchase 10 with sku "SKU-1" VIEW -- order_line_detail (public) +- order_line_detail (anonymous procedural, public) - Returns rows with lineId: u64, customerName: string, and sku: string - Return only rows connected through matching customer, purchase, and line-item IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt index c9260a433ca..4655bb596ce 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt @@ -16,6 +16,6 @@ REDUCERS - Insert Eligibility Id 10 for only Member 1 VIEW -- EligibleMember (public, query-builder) +- EligibleMember (public, non-anonymous query-builder) - Use a semijoin - Return members only when a matching Eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt index e3e7bb08817..cfe7ab28e21 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt @@ -16,6 +16,6 @@ REDUCERS - Insert eligibility id 10 for only member 1 VIEW -- eligible_member (public, query-builder) +- eligible_member (public, non-anonymous query-builder) - Use a semijoin - Return members only when a matching eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt index 83d299ae021..8c0716686c9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt @@ -16,6 +16,6 @@ REDUCERS - Insert eligibility id 10 for only member 1 VIEW -- eligible_member (public, query-builder) +- eligible_member (public, non-anonymous query-builder) - Use a semijoin - Return members only when a matching eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt index 82a3582c809..f7f96c90858 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -14,7 +14,7 @@ REDUCERS - Set each Label to "event-{microseconds}", such as "event-100" VIEW -- WindowEvent (procedural, public) +- WindowEvent (anonymous procedural, public) - Use the OccurredAt index - Return the inclusive range from 200 through 400 microseconds - Return rows in ascending OccurredAt order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt index a0fbb124da6..be8b847f940 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -14,7 +14,7 @@ REDUCERS - Set each label to "event-{microseconds}", such as "event-100" VIEW -- window_event (procedural, public) +- window_event (anonymous procedural, public) - Use the occurred_at index - Return the inclusive range from 200 through 400 microseconds - Return rows in ascending occurred_at order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt index 92abe30fa49..f6dd5b43631 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -14,7 +14,7 @@ REDUCERS - Set each label to "event-{microseconds}", such as "event-100" VIEW -- window_event (procedural, public) +- window_event (anonymous procedural, public) - Use the occurredAt index - Return the inclusive range from 200 through 400 microseconds - Return rows in ascending occurredAt order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt index 875da4600ac..83c69688658 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt @@ -16,6 +16,6 @@ REDUCERS - Insert Id 4 with Category "sports", Active true, and Score 20 VIEW -- FeaturedContent (procedural, public) +- FeaturedContent (anonymous procedural, public) - First use the Category index for "news" - Then return only Active rows with Score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt index f77f2979310..2e30de3af72 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt @@ -16,6 +16,6 @@ REDUCERS - Insert id 4 with category "sports", active true, and score 20 VIEW -- featured_content (procedural, public) +- featured_content (anonymous procedural, public) - First use the category index for "news" - Then return only active rows with score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt index eccb7e9e2c2..4ec5613aae5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt @@ -16,6 +16,6 @@ REDUCERS - Insert id 4 with category "sports", active true, and score 20 VIEW -- featured_content (procedural, public) +- featured_content (anonymous procedural, public) - First use the category index for "news" - Then return only active rows with score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt index 58f533694a6..499cbdcf169 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -22,6 +22,6 @@ REDUCERS - Delete the first books Sale VIEW -- CategorySummary (public, query-builder) +- CategorySummary (public, non-anonymous query-builder) - Expose all CategoryTotal rows - After Exercise, the only row is books with TotalAmount 25 and SaleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt index c70d7894d36..b10e3c27380 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -22,6 +22,6 @@ REDUCERS - Delete the first books sale VIEW -- category_summary (public, query-builder) +- category_summary (public, non-anonymous query-builder) - Expose all category_total rows - After exercise, the only row is books with total_amount 25 and sale_count 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt index 5cb6eb6ff2f..8521b0a41be 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -22,6 +22,6 @@ REDUCERS - Delete the first books sale VIEW -- category_summary (public, query-builder) +- category_summary (public, non-anonymous query-builder) - Expose all category_total rows - After exercise, the only row is books with totalAmount 25 and saleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt index 9d3c3ad556a..1a95f8affc0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt @@ -13,6 +13,6 @@ REDUCERS - Insert closed Ticket Id 2 titled "B" VIEW -- OpenTicket (public, query-builder) +- OpenTicket (public, non-anonymous query-builder) - Return the query-builder filter for Status "open" - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt index f309318db3c..80e07a7546a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt @@ -13,6 +13,6 @@ REDUCERS - Insert closed ticket id 2 titled "B" VIEW -- open_ticket (public, query-builder) +- open_ticket (public, non-anonymous query-builder) - Return the query-builder filter for status "open" - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt index 8b81902ef3e..07872178aa0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt @@ -13,6 +13,6 @@ REDUCERS - Insert closed ticket id 2 titled "B" VIEW -- open_ticket (public, query-builder) +- open_ticket (public, non-anonymous query-builder) - Return the query-builder filter for status "open" - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt index 9467c5e70c3..ed875f18ed7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt @@ -8,6 +8,6 @@ TABLE - Visible: bool (btree index) VIEW -- SourceView (procedural, public) +- SourceView (anonymous procedural, public) - Return visible SourceRow rows - Explicitly declare Id as the view primary key diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt index 89a6317baba..ce4ec9e0fb5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt @@ -8,6 +8,6 @@ TABLE - visible: bool (btree index) VIEW -- source_view (procedural, public) +- source_view (anonymous procedural, public) - Return visible source_row rows - Explicitly declare id as the view primary key diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt index 48a6c683972..357a397d061 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt @@ -8,6 +8,6 @@ TABLE - visible: boolean (btree index) VIEW -- source_view (procedural, public) +- source_view (anonymous procedural, public) - Return visible source_row rows - The returned row type declares id as its primary key diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 1cc162b67f3..1400142353d 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -28,6 +28,59 @@ struct SchemaSnapshot { row_level_security: BTreeSet, } +#[derive(Debug, Default)] +struct SchemaNames { + tables: BTreeMap, + functions: BTreeMap, +} + +impl SchemaNames { + fn from_schema(schema: &Value) -> Self { + let Some(sections) = schema.get("sections") else { + return Self::default(); + }; + let mut names = Self::default(); + + if let Some(entries) = sections + .as_array() + .into_iter() + .flatten() + .find_map(|section| section.get("ExplicitNames")) + .and_then(|names| names.get("entries")) + .and_then(Value::as_array) + { + for entry in entries { + if let Some(mapping) = entry.get("Table") { + insert_name_mapping(&mut names.tables, mapping); + } else if let Some(mapping) = entry.get("Function") { + insert_name_mapping(&mut names.functions, mapping); + } + } + } + names + } + + fn table(&self, name: String) -> String { + self.canonical(name, &self.tables) + } + + fn function(&self, name: String) -> String { + self.canonical(name, &self.functions) + } + + fn canonical(&self, name: String, explicit: &BTreeMap) -> String { + explicit.get(&name).cloned().unwrap_or(name) + } +} + +fn insert_name_mapping(names: &mut BTreeMap, mapping: &Value) { + let source = schema_name(mapping.get("source_name")); + let canonical = schema_name(mapping.get("canonical_name")); + if !source.is_empty() && !canonical.is_empty() { + names.insert(source, canonical); + } +} + impl Scorer for SchemaParityScorer { fn id(&self) -> &'static str { self.id_str @@ -106,6 +159,7 @@ fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { fn extract_schema(v: &Value) -> SchemaSnapshot { let mut schema = SchemaSnapshot::default(); + let names = SchemaNames::from_schema(v); let typespace = v.get("typespace").or_else(|| section(v, "Typespace")); let types = typespace .and_then(|value| value.get("types")) @@ -115,7 +169,7 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { if let Some(ts) = schema_array(v, "tables", "Tables") { for t in ts { - let name = schema_name(t.get("source_name").or_else(|| t.get("name"))); + let name = names.table(schema_name(t.get("source_name").or_else(|| t.get("name")))); let mut cols = BTreeMap::new(); // Older CLI descriptions put columns directly on the table. Keep @@ -179,7 +233,7 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { if let Some(rs) = schema_array(v, "reducers", "Reducers") { for r in rs { - let name = schema_name(r.get("source_name").or_else(|| r.get("name"))); + let name = names.function(schema_name(r.get("source_name").or_else(|| r.get("name")))); let mut sig = format!("{}({})", name, parameter_types(r, types).join(",")); append_field(&mut sig, "visibility", r.get("visibility")); append_type(&mut sig, "ok", r.get("ok_return_type"), types); @@ -191,9 +245,11 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { for export in v.get("misc_exports").and_then(Value::as_array).into_iter().flatten() { if let Some(procedure) = export.get("Procedure") { - schema.exports.insert(function_export("procedure", procedure, types)); + schema + .exports + .insert(function_export("procedure", procedure, types, &names)); } else if let Some(view) = export.get("View") { - schema.exports.insert(view_export(view, types)); + schema.exports.insert(view_export(view, types, &names)); } else if let Some(default) = export.get("ColumnDefaultValue") { schema .exports @@ -202,10 +258,12 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { } for procedure in schema_array(v, "procedures", "Procedures").into_iter().flatten() { - schema.exports.insert(function_export("procedure", procedure, types)); + schema + .exports + .insert(function_export("procedure", procedure, types, &names)); } for view in schema_array(v, "views", "Views").into_iter().flatten() { - schema.exports.insert(view_export(view, types)); + schema.exports.insert(view_export(view, types, &names)); } insert_canonical_exports(&mut schema.exports, v, "schedules", "Schedules", "schedule"); insert_canonical_exports( @@ -215,8 +273,7 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { "LifeCycleReducers", "lifecycle", ); - insert_canonical_exports(&mut schema.exports, v, "http_handlers", "HttpHandlers", "http_handler"); - insert_canonical_exports(&mut schema.exports, v, "http_routes", "HttpRoutes", "http_route"); + insert_http_routes(&mut schema.exports, v); insert_canonical_exports( &mut schema.exports, v, @@ -281,16 +338,18 @@ fn append_type(signature: &mut String, label: &str, value: Option<&Value>, types } } -fn function_export(kind: &str, function: &Value, types: &[Value]) -> String { - let name = schema_name(function.get("source_name").or_else(|| function.get("name"))); +fn function_export(kind: &str, function: &Value, types: &[Value], names: &SchemaNames) -> String { + let name = names.function(schema_name( + function.get("source_name").or_else(|| function.get("name")), + )); let mut signature = format!("{kind}:{name}({})", parameter_types(function, types).join(",")); append_type(&mut signature, "return", function.get("return_type"), types); append_field(&mut signature, "visibility", function.get("visibility")); signature } -fn view_export(view: &Value, types: &[Value]) -> String { - let mut signature = function_export("view", view, types); +fn view_export(view: &Value, types: &[Value], names: &SchemaNames) -> String { + let mut signature = function_export("view", view, types, names); append_field(&mut signature, "public", view.get("is_public")); append_field(&mut signature, "anonymous", view.get("is_anonymous")); signature @@ -308,6 +367,16 @@ fn insert_canonical_exports( } } +fn insert_http_routes(exports: &mut BTreeSet, schema: &Value) { + for route in schema_array(schema, "http_routes", "HttpRoutes").into_iter().flatten() { + let mut route = route.clone(); + if let Some(route) = route.as_object_mut() { + route.remove("handler_function"); + } + exports.insert(format!("http_route:{}", canonical_value(Some(&route)))); + } +} + fn schema_name(value: Option<&Value>) -> String { value .and_then(Value::as_str) @@ -1280,6 +1349,40 @@ mod tests { assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); } + #[test] + fn v10_schema_compares_explicit_canonical_table_names() { + let schema = |source_name: &str| { + json!({ + "sections": [ + { "Typespace": { "types": [{ "Product": { "elements": [] } }] } }, + { "Tables": [{ + "source_name": source_name, + "product_type_ref": 0, + "primary_key": [], + "indexes": [], + "constraints": [], + "sequences": [], + "table_type": { "User": [] }, + "table_access": { "Public": [] }, + "default_values": [], + "is_event": false + }] }, + { "ExplicitNames": { "entries": [{ + "Table": { + "source_name": source_name, + "canonical_name": "child_item" + } + }] } } + ] + }) + }; + + assert_eq!( + extract_schema(&schema("childItem")), + extract_schema(&schema("child_item")) + ); + } + #[test] fn row_level_security_produces_a_schema_diff() { let mut golden = current_schema(true); @@ -1378,6 +1481,27 @@ mod tests { assert!(!http_route_results_equal(&golden, &candidate, true)); } + #[test] + fn http_route_schema_ignores_handler_names() { + let schema = |handler: &str| { + json!({ + "sections": [ + { "HttpHandlers": [{ "source_name": handler }] }, + { "HttpRoutes": [{ + "handler_function": handler, + "method": { "Method": { "Get": [] } }, + "path": "/items" + }] } + ] + }) + }; + + assert_eq!( + extract_schema(&schema("listItems")), + extract_schema(&schema("get_items")) + ); + } + #[test] fn v10_schema_extracts_lifecycle_procedures_views_and_http_exports() { let schema = json!({ @@ -1427,7 +1551,6 @@ mod tests { .any(|value| value.starts_with("procedure:fetch"))); assert!(extracted.exports.iter().any(|value| value.starts_with("view:profile"))); assert!(extracted.exports.iter().any(|value| value.starts_with("lifecycle:"))); - assert!(extracted.exports.iter().any(|value| value.starts_with("http_handler:"))); assert!(extracted.exports.iter().any(|value| value.starts_with("http_route:"))); assert!(extracted .exports @@ -1450,7 +1573,7 @@ mod tests { let extracted = extract_schema(&json); assert_eq!(extracted.reducers.len(), 1, "serialized schema was {json}"); - assert_eq!(extracted.exports.len(), 6, "serialized schema was {json}"); + assert_eq!(extracted.exports.len(), 5, "serialized schema was {json}"); assert!( extracted .exports From 5d63ebe355f21977d8c6738dc2496cbe28c57606 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:04:47 -0400 Subject: [PATCH 53/78] Update SKILL.md --- skills/typescript-server/SKILL.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 8a8577e7122..b7a9ef684cc 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -322,6 +322,8 @@ Procedure callbacks are synchronous. Do not mark them `async` or use `await`; re TypeScript outbound HTTP uses `ctx.http.fetch(url, options)`, including for non-GET requests; it does not provide convenience methods such as `get()` or `post()`. Responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. +`t.array(t.u8())` values are `number[]`. Convert one to `new Uint8Array(value)` before using it as a binary request body. + Procedures and handlers open short database transactions with `ctx.withTx(tx => ...)`. Perform network I/O before opening the transaction; only database work belongs inside its callback. Scheduled procedures use the ordinary scheduled-table shape. Its `scheduled` option references an exported `spacetimedb.procedure(...)` value instead of a reducer, and the procedure accepts the scheduled row as its argument. From 5fe4011e998badbf5ec40169f406736c6c9ca587 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:59:45 -0400 Subject: [PATCH 54/78] Update StdbModule.csproj --- .../src/templates/csharp/server/StdbModule.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj index f286932badd..d401d2c7575 100644 --- a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj +++ b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj @@ -1,7 +1,7 @@ - net8.0 + net8.0;net10.0 wasi-wasm enable enable From 35c5d421e2f0ccaf27fd85f9359e4630384110cc Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:09:44 -0400 Subject: [PATCH 55/78] Update templates.rs --- tools/xtask-llm-benchmark/src/bench/templates.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/xtask-llm-benchmark/src/bench/templates.rs b/tools/xtask-llm-benchmark/src/bench/templates.rs index 35176de8200..3d86a4fdb29 100644 --- a/tools/xtask-llm-benchmark/src/bench/templates.rs +++ b/tools/xtask-llm-benchmark/src/bench/templates.rs @@ -235,6 +235,7 @@ fn write_csharp_nuget_config(root: &Path) -> Result<()> { + @@ -244,6 +245,10 @@ fn write_csharp_nuget_config(root: &Path) -> Result<()> { + + + + From 6cb8317459869870574e72d0ac271d74480a278d Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:59:20 -0400 Subject: [PATCH 56/78] Harden Rust LLM benchmark evaluation --- skills/rust-server/SKILL.md | 6 +- .../src/bench/publishers.rs | 76 +++++++++++++++++-- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index d36b3d45eef..b9698ea63c2 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -181,6 +181,8 @@ Declare a procedural view primary key in the view attribute: fn catalog_entry(ctx: &AnonymousViewContext) -> Vec { ... } ``` +Procedural-view primary keys are explicit schema metadata. Add one only when the view itself is required to expose a primary key; a source table's primary key is not inherited by the view. + Query-builder views use `ViewContext`, `ctx.from`, and return `impl Query`. Use `filter` for predicates and `right_semijoin` when the result should contain right-side rows that have a matching left-side row: ```rust @@ -271,9 +273,9 @@ pub fn inspect(_ctx: &mut ProcedureContext, input: String) -> ResultValue { } ``` -Outbound HTTP is available through `ctx.http`. Convenience methods such as `get` return a response, and other methods use `Request::builder()` with `ctx.http.send(request)`. Consume a response body as text with `response.into_body().into_string_lossy()`. +Outbound HTTP is available through `ctx.http`. Convenience methods such as `get` return a response, and other methods use `Request::builder()` with `ctx.http.send(request)`. Consume a response body as text with `response.into_body().into_string_lossy()`. Header values use the fallible `to_str()` conversion; they do not provide `as_str()`. -Open short database transactions with `ctx.with_tx(|tx| ...)`. The callback implements `Fn`, so clone captured owned values when storing them rather than moving them out of the closure. Perform network I/O before opening the transaction. +Open short database transactions with `ctx.with_tx(|tx| ...)`. It returns the callback's value directly, so do not call `unwrap` or `expect` on the result unless the callback itself returns a `Result`. The callback implements `Fn`, so clone captured owned values when storing them rather than moving them out of the closure. Perform network I/O before opening the transaction. For an outbound request without a convenience method: diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index c1ce12b6a94..15db995c7c3 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -8,6 +8,15 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::LazyLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +static CLI_SESSION_TAG: LazyLock = LazyLock::new(|| { + let started_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + format!("{}-{started_at}", std::process::id()) +}); fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -182,7 +191,7 @@ pub trait Publisher: Send + Sync { fn database_cli_root(module_name: &str) -> Result { let db = sanitize_db_name(module_name); - let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{db}", std::process::id())); + let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{db}", CLI_SESSION_TAG.as_str())); fs::create_dir_all(&path)?; Ok(CliRootDir { path }) } @@ -200,7 +209,7 @@ fn signal_killed_by(_status: &std::process::ExitStatus) -> Option { } /// Check if the failure is a transient error that should be retried. -/// These are resource contention issues in the dotnet WASI SDK. +/// These are resource contention issues and diagnostic-free child-process crashes. fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let combined = format!("{stderr}{stdout}"); let diagnostic = combined.to_ascii_lowercase(); @@ -212,6 +221,7 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { !message.starts_with("could not compile `") && !message.starts_with("process didn't exit successfully") && !message.starts_with("command [\"cargo\", \"build\"") + && !message.starts_with("failed to run custom build command for `") }) }); let silent_rustc_exit = diagnostic.contains("process didn't exit successfully") @@ -221,6 +231,22 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let silent_cargo_exit = diagnostic.contains("error: command [\"cargo\", \"build\"") && diagnostic.contains("exited with code 1") && !has_actionable_compiler_diagnostic; + let silent_rust_lld_exit = diagnostic.contains("error: linking with `rust-lld.exe` failed: exit code: 1") + && !diagnostic.contains("rust-lld: error:") + && !diagnostic.contains("lld-link: error:"); + let silent_build_script_exit = diagnostic.contains("failed to run custom build command for `") + && diagnostic.contains("build-script-build` (exit code: 1)") + && !diagnostic.contains("--- stderr") + && !diagnostic.contains("panicked at") + && !has_actionable_compiler_diagnostic; + let has_terminal_failure_marker = diagnostic.lines().any(|line| { + let line = line.trim_start(); + line.starts_with("error: could not compile `") + || line.starts_with("failed:") + || line.starts_with("exception:") + || line.contains("build failed") + || line.contains("panicked at") + }); let has_actionable_dotnet_diagnostic = diagnostic.lines().any(|line| { line.contains(".cs(") && (line.contains(": error cs") || line.contains(": error stdb") || line.contains(": error il")) @@ -244,6 +270,8 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { // diagnostic. This is a transient child-process failure, not bad source. || silent_rustc_exit || silent_cargo_exit + || silent_rust_lld_exit + || silent_build_script_exit // The .NET linker occasionally exits without a source diagnostic and // MSBuild reports only MSB6006 + NETSDK1144. Do not retry when the // output also contains a C#, SpacetimeDB codegen, or IL source error. @@ -251,9 +279,7 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { // A child process can occasionally disappear on Windows after login but // before the CLI emits a compiler/MSBuild diagnostic. Retrying is safe: // benchmark publishes always use their own temporary CLI root and database. - || (!diagnostic.contains("error") - && !diagnostic.contains("failed") - && !diagnostic.contains("exception")) + || (!has_actionable_compiler_diagnostic && !has_terminal_failure_marker) } #[cfg(test)] @@ -276,6 +302,14 @@ mod tests { )); } + #[test] + fn retries_partial_cargo_output_containing_error_in_crate_name() { + assert!(is_transient_build_error( + "Compiling thiserror v1.0.69\nCompiling anyhow v1.0.104", + "" + )); + } + #[test] fn retries_silent_rustc_exit() { assert!(is_transient_build_error( @@ -300,6 +334,38 @@ mod tests { )); } + #[test] + fn retries_rust_lld_exit_without_linker_diagnostic() { + assert!(is_transient_build_error( + "error: linking with `rust-lld.exe` failed: exit code: 1\n = note: some arguments are omitted\n = note:\n\nerror: could not compile `rustversion` (lib) due to 1 previous error", + "" + )); + } + + #[test] + fn does_not_retry_rust_lld_exit_with_linker_diagnostic() { + assert!(!is_transient_build_error( + "error: linking with `rust-lld.exe` failed: exit code: 1\n = note: rust-lld: error: undefined symbol: missing", + "" + )); + } + + #[test] + fn retries_build_script_exit_without_diagnostic() { + assert!(is_transient_build_error( + "error: failed to run custom build command for `anyhow v1.0.104`\n\nCaused by:\n process didn't exit successfully: `target/release/build/anyhow/build-script-build` (exit code: 1)\n --- stdout\n cargo:rerun-if-changed=src/nightly.rs", + "" + )); + } + + #[test] + fn does_not_retry_build_script_exit_with_diagnostic() { + assert!(!is_transient_build_error( + "error: failed to run custom build command for `native-dependency v1.0.0`\n\nCaused by:\n process didn't exit successfully: `target/release/build/native-dependency/build-script-build` (exit code: 1)\n --- stderr\n error: required native compiler was not found", + "" + )); + } + #[test] fn retries_dotnet_linker_exit_without_source_diagnostic() { assert!(is_transient_build_error( From 6719f9df4dbb1d698d2c407061be7d3e01942b17 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:44:58 -0400 Subject: [PATCH 57/78] fixes --- tools/xtask-llm-benchmark/src/bench/mod.rs | 2 +- .../src/bench/publishers.rs | 69 ++++- tools/xtask-llm-benchmark/src/bench/runner.rs | 253 ++++++++++-------- tools/xtask-llm-benchmark/src/bench/types.rs | 2 + .../src/bin/llm_benchmark.rs | 74 ++++- 5 files changed, 272 insertions(+), 128 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/mod.rs b/tools/xtask-llm-benchmark/src/bench/mod.rs index 6b1ebb1cf98..16f7da94cef 100644 --- a/tools/xtask-llm-benchmark/src/bench/mod.rs +++ b/tools/xtask-llm-benchmark/src/bench/mod.rs @@ -6,7 +6,7 @@ mod templates; pub mod types; pub(crate) mod utils; -pub use publishers::{DotnetPublisher, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; +pub use publishers::{DotnetPublisher, PublishedDatabase, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; pub use runner::TaskRunner; pub use types::{RunOutcome, TaskPaths}; pub use utils::bench_route_concurrency; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index b7fb74c6936..626450ed19a 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -205,8 +205,63 @@ fn strip_ansi_codes(s: &str) -> Cow<'_, str> { /* Shared */ /* -------------------------------------------------------------------------- */ +pub struct PublishedDatabase { + host_url: String, + db: String, + // Keep the publishing CLI session alive so deletion uses the database + // owner's credentials without sharing mutable CLI state across routes. + cli_root: CliRootDir, + deleted: bool, +} + +impl PublishedDatabase { + fn new(host_url: &str, db: String, cli_root: CliRootDir) -> Self { + Self { + host_url: host_url.to_owned(), + db, + cli_root, + deleted: false, + } + } + + fn delete_inner(&self) -> Result<()> { + let mut cmd = spacetime_cmd(&self.cli_root); + cmd.arg("delete") + .arg("-y") + .arg("--no-config") + .arg("--server") + .arg(&self.host_url) + .arg(&self.db); + run(&mut cmd, "spacetime delete").with_context(|| { + format!( + "failed to clean up benchmark database `{}` on {}", + self.db, self.host_url + ) + }) + } + + pub fn delete(mut self) -> Result<()> { + self.delete_inner()?; + self.deleted = true; + Ok(()) + } +} + +impl Drop for PublishedDatabase { + fn drop(&mut self) { + if !self.deleted + && let Err(error) = self.delete_inner() + { + eprintln!( + "[cleanup] failed to delete benchmark database `{}` during fallback cleanup: {error:#}", + self.db + ); + } + } +} + pub trait Publisher: Send + Sync { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()>; + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result; } /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) @@ -342,7 +397,7 @@ impl DotnetPublisher { } impl Publisher for DotnetPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -370,7 +425,7 @@ impl Publisher for DotnetPublisher { Self::configure_dotnet_env(&mut pubcmd); run(&mut pubcmd, "spacetime publish (csharp)")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } /* -------------------------------------------------------------------------- */ @@ -390,7 +445,7 @@ impl SpacetimeRustPublisher { } impl Publisher for SpacetimeRustPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -416,7 +471,7 @@ impl Publisher for SpacetimeRustPublisher { "spacetime publish", )?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } @@ -437,7 +492,7 @@ impl TypeScriptPublisher { } impl Publisher for TypeScriptPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -539,6 +594,6 @@ impl Publisher for TypeScriptPublisher { } run(&mut publish_cmd, "spacetime publish (typescript)")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 2536b5e5fe1..565cd071861 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -1,16 +1,14 @@ use anyhow::{anyhow, bail, Context, Result}; use chrono::Utc; -use futures::{stream, StreamExt}; +use futures::{stream, StreamExt, TryStreamExt}; use serde_json::json; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::OnceLock; -use std::time::Instant; -use tokio::sync::Mutex; +use std::time::{Duration, Instant}; use tokio::task; -use crate::bench::publishers::{DotnetPublisher, SpacetimeRustPublisher, TypeScriptPublisher}; +use crate::bench::publishers::{DotnetPublisher, PublishedDatabase, SpacetimeRustPublisher, TypeScriptPublisher}; use crate::bench::templates::materialize_project; use crate::bench::types::{BenchRunContext, PublishParams, RunContext, RunOneError}; pub(crate) use crate::bench::types::{RunOutcome, TaskPaths}; @@ -30,56 +28,43 @@ pub struct TaskRunner { pub ts_publisher: TypeScriptPublisher, } -static BUILT_KEYS: OnceLock>> = OnceLock::new(); - struct PendingRetry { task: TaskPaths, error: anyhow::Error, } -fn build_key(lang: Lang, selectors: Option<&[String]>) -> String { - let v = match selectors { - Some(s) if !s.is_empty() => { - let mut t = s.to_vec(); - t.sort(); // stable key independent of order - t +fn task_result_or_infrastructure_error( + task: TaskPaths, + result: Result, + started: chrono::DateTime, +) -> Result<(TaskPaths, Result)> { + match result { + Err(RunOneError::Infrastructure { msg, llm_output }) => { + let output_note = llm_output + .as_deref() + .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) + .unwrap_or_default(); + bail!("benchmark infrastructure failure: {msg}{output_note}"); } - _ => vec!["ALL".to_string()], - }; - let joined = v.join(","); - format!("{lang:?}:{joined}") + other => Ok(( + task, + other.map(|mut outcome| { + outcome.started_at.get_or_insert(started); + outcome + }), + )), + } } -/// Build goldens **once per (lang, selector-set)** in this process. -/// If selectors is None/empty, that means "ALL tasks". +/// Build the golden databases and return handles that keep their owning CLI +/// sessions alive until scoring has finished. pub async fn ensure_goldens_built_once( host: Option, bench_root: &Path, lang: Lang, selectors: Option<&[String]>, -) -> Result<()> { - let key = build_key(lang, selectors); - let set = BUILT_KEYS.get_or_init(|| Mutex::new(HashSet::new())); - { - let set = set.lock(); - if set.await.contains(&key) { - return Ok(()); - } - } - // single-flight for this key - let set_guard = set.lock().await; - if set_guard.contains(&key) { - return Ok(()); - } - - // IMPORTANT: pass selectors through so we only build needed goldens - build_goldens_only_for_lang(host, bench_root, lang, selectors).await?; - - // mark as built - drop(set_guard); - let mut set = BUILT_KEYS.get().unwrap().lock().await; - set.insert(key); - Ok(()) +) -> Result> { + build_goldens_only_for_lang(host, bench_root, lang, selectors).await } async fn publish_rust_async( @@ -87,16 +72,57 @@ async fn publish_rust_async( host_url: String, wdir: PathBuf, db: String, -) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await? +} +async fn publish_cs_async( + publisher: DotnetPublisher, + host_url: String, + wdir: PathBuf, + db: String, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await? +} +async fn publish_ts_async( + publisher: TypeScriptPublisher, + host_url: String, + wdir: PathBuf, + db: String, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await? +} + +async fn delete_database_async(database: PublishedDatabase) -> Result<()> { + task::spawn_blocking(move || database.delete()).await??; Ok(()) } -async fn publish_cs_async(publisher: DotnetPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + +pub async fn delete_databases(databases: Vec) -> Result<()> { + stream::iter(databases.into_iter().map(delete_database_async)) + .buffer_unordered(8) + .try_collect::>() + .await?; Ok(()) } -async fn publish_ts_async(publisher: TypeScriptPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + +async fn ensure_server_healthy(host: Option<&str>) -> Result<()> { + let Some(host) = host else { + return Ok(()); + }; + let ping_url = format!("{}/v1/ping", host.trim_end_matches('/')); + let response = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build()? + .get(&ping_url) + .send() + .await + .with_context(|| format!("benchmark server at {host} is unavailable; refusing to upload results"))?; + if !response.status().is_success() { + bail!( + "benchmark server health check at {ping_url} returned {}; refusing to upload results", + response.status() + ); + } Ok(()) } @@ -123,7 +149,7 @@ impl TaskRunner { golden_src_text: &str, golden_db: String, host: Option, - ) -> Result<()> { + ) -> Result { self.publish( PublishParams { lang, @@ -139,11 +165,11 @@ impl TaskRunner { .await } - async fn publish_llm(&self, params: PublishParams<'_>) -> Result<()> { + async fn publish_llm(&self, params: PublishParams<'_>) -> Result { self.publish(params, "llm").await } - async fn publish(&self, params: PublishParams<'_>, phase: &str) -> Result<()> { + async fn publish(&self, params: PublishParams<'_>, phase: &str) -> Result { let lang_name = match params.lang { Lang::Rust => "rust", Lang::CSharp => "csharp", @@ -164,13 +190,13 @@ impl TaskRunner { )?; let host_url = params.host.unwrap_or_else(|| "local".to_owned()); - match params.lang { + let database = match params.lang { Lang::Rust => publish_rust_async(self.rust_publisher, host_url, wdir, params.db_name).await?, Lang::CSharp => publish_cs_async(self.cs_publisher, host_url, wdir, params.db_name).await?, Lang::TypeScript => publish_ts_async(self.ts_publisher, host_url, wdir, params.db_name).await?, - } + }; - Ok(()) + Ok(database) } pub async fn run_one(&self, task: &TaskPaths, cfg: &RunContext<'_>) -> Result { @@ -267,7 +293,7 @@ impl TaskRunner { print_llm_output(&cfg.route.display_name, &task_id, &llm_output); } - let publish_error: Option = self + let publish_result = self .publish_llm(PublishParams { lang: cfg.lang, category: &category, @@ -277,15 +303,26 @@ impl TaskRunner { db_name: llm_db.clone(), host: cfg.host.clone(), }) - .await - .err() - .map(|e| { + .await; + let (published_database, publish_error) = match publish_result { + Ok(database) => (Some(database), None), + Err(error) => { eprintln!( - "⚠️ publish failed for {}/{}/{}: {e:#}", + "⚠️ publish failed for {}/{}/{}: {error:#}", category, task_id, cfg.route.display_name ); - format!("{:#}", e) + (None, Some(format!("{error:#}"))) + } + }; + + if publish_error.is_some() + && let Err(error) = ensure_server_healthy(cfg.host.as_deref()).await + { + return Err(RunOneError::Infrastructure { + msg: format!("publish failed and the benchmark server is unavailable: {error:#}"), + llm_output: Some(llm_output), }); + } let mut passed = 0usize; let mut partial_sum = 0f32; @@ -319,6 +356,15 @@ impl TaskRunner { ); } + if let Some(database) = published_database + && let Err(error) = delete_database_async(database).await + { + return Err(RunOneError::Infrastructure { + msg: format!("database cleanup failed for `{llm_db}`: {error:#}"), + llm_output: Some(llm_output), + }); + } + let score_pct = if total_tasks == 0 { 0.0 } else { @@ -381,7 +427,7 @@ fn partition_results( lang_name: &str, route: &ModelRoute, hash: &str, -) -> (Vec, Vec) { +) -> Result<(Vec, Vec)> { let mut good = Vec::new(); let mut retry = Vec::new(); @@ -400,6 +446,13 @@ fn partition_results( Some(llm_output), )); } + Err(RunOneError::Infrastructure { msg, llm_output }) => { + let output_note = llm_output + .as_deref() + .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) + .unwrap_or_default(); + bail!("benchmark infrastructure failure: {msg}{output_note}"); + } Err(RunOneError::Other(e)) => { // No output at all — provider error, retry eprintln!("\u{26a0}\u{fe0f} provider error, will retry: {e:?}"); @@ -408,7 +461,7 @@ fn partition_results( } } - (good, retry) + Ok((good, retry)) } fn pending_retries_to_outcomes( @@ -532,21 +585,15 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); + let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress const MAX_RETRY_ROUNDS: usize = 3; @@ -586,21 +633,15 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu llm, host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; if new_outcomes.is_empty() && !still_failing.is_empty() { // No progress — provider is likely down. Keep the failures and stop retrying. @@ -632,6 +673,10 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu println!("[runner] completed batch: {} results", outcomes.len()); + // A dead local server turns later publish failures into misleading zeroes. + // Refuse to analyze or upload any batch that cannot pass a final health check. + ensure_server_healthy(cfg.host.as_deref()).await?; + if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { Ok(text) => text, @@ -728,21 +773,15 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> host: cfg.host.clone(), }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); + let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress let dropped; @@ -785,21 +824,15 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> llm, host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; if new_outcomes.is_empty() && !still_failing.is_empty() { eprintln!( @@ -827,6 +860,8 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> ); } + ensure_server_healthy(cfg.host.as_deref()).await?; + if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { Ok(text) => text, @@ -886,7 +921,7 @@ pub async fn build_goldens_only_for_lang( bench_root: &Path, lang: Lang, selectors: Option<&[String]>, -) -> Result<()> { +) -> Result> { let tasks = if let Some(sels) = selectors { let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; let all = discover_tasks(bench_root)?; @@ -922,7 +957,7 @@ pub async fn build_goldens_only_for_lang( _ => bench_concurrency(), }; - stream::iter(tasks.into_iter().map(|task| { + let databases = stream::iter(tasks.into_iter().map(|task| { let runner = &runner; let host_clone = host.clone(); async move { @@ -943,7 +978,7 @@ pub async fn build_goldens_only_for_lang( .collect::>>()?; println!("✓ [{}] goldens build/publish: complete", lang_name); - Ok(()) + Ok(databases) } fn discover_tasks(benchmarks_root: &Path) -> Result> { diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index e54df0d4902..16a8336a61d 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -134,6 +134,8 @@ pub struct RouteRun { pub enum RunOneError { #[error("{msg}")] WithOutput { msg: String, llm_output: String }, + #[error("{msg}")] + Infrastructure { msg: String, llm_output: Option }, #[error(transparent)] Other(#[from] anyhow::Error), } diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 6ec030a49e8..db9148fa337 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -13,7 +13,8 @@ use tokio::runtime::Runtime; use xtask_llm_benchmark::api::ApiClient; use xtask_llm_benchmark::bench::bench_route_concurrency; use xtask_llm_benchmark::bench::runner::{ - build_goldens_only_for_lang, ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, + build_goldens_only_for_lang, delete_databases, ensure_goldens_built_once, + run_selected_or_all_for_model_async_for_lang, }; use xtask_llm_benchmark::bench::types::{BenchRunContext, RouteRun, RunConfig, RunOutcome}; use xtask_llm_benchmark::context::constants::ALL_MODES; @@ -279,7 +280,7 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { eprintln!("[warn] failed to upload task catalog: {e}"); } - let RuntimeInit { runtime, guard } = initialize_runtime(config.hash_only)?; + let RuntimeInit { runtime, mut guard } = initialize_runtime(config.hash_only)?; config.host = guard.as_ref().map(|g| g.host_url.clone()); @@ -295,48 +296,88 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { if config.goldens_only { let rt = runtime.as_ref().expect("runtime required for --goldens-only"); - rt.block_on(build_goldens_only_for_lang( + let result = rt.block_on(build_goldens_only_for_lang( config.host.clone(), &bench_root, config.lang, selectors_ref, - ))?; + )); + let databases = match result { + Ok(databases) => databases, + Err(error) => { + report_server_status(guard.as_mut()); + return Err(error); + } + }; + if let Err(error) = rt.block_on(delete_databases(databases)) { + report_server_status(guard.as_mut()); + return Err(error); + } println!("[{}] goldens-only build complete", config.lang.as_str()); return Ok(()); } - let llm_provider = if !config.goldens_only && !config.hash_only { + // Goldens are live parity fixtures: scorers describe them, call their + // reducers, and compare their data with generated modules. Keep them + // published through every model and mode, then delete them at teardown. + let (llm_provider, golden_databases) = if !config.goldens_only && !config.hash_only { let rt = runtime.as_ref().expect("failed to initialize runtime for goldens"); - rt.block_on(ensure_goldens_built_once( + let result = rt.block_on(ensure_goldens_built_once( config.host.clone(), &bench_root, config.lang, selectors_ref, - ))?; + )); + let databases = match result { + Ok(databases) => databases, + Err(error) => { + report_server_status(guard.as_mut()); + return Err(error); + } + }; let provider = make_provider_from_env()?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); preflight_llm_routes(rt, provider.as_ref(), &routes, &modes)?; - Some(provider) + (Some(provider), databases) } else { - None + (None, Vec::new()) }; let mut all_outcomes: Vec = Vec::new(); for mode in modes { - let outcomes = run_mode_benchmarks( + let result = run_mode_benchmarks( &mode, config.lang, &config, &bench_root, runtime.as_ref(), llm_provider.as_ref(), - )?; + ); + let outcomes = match result { + Ok(outcomes) => outcomes, + Err(error) => { + if let Some(rt) = runtime.as_ref() + && let Err(cleanup_error) = rt.block_on(delete_databases(golden_databases)) + { + eprintln!("[cleanup] failed to delete golden databases after benchmark failure: {cleanup_error:#}"); + } + report_server_status(guard.as_mut()); + return Err(error); + } + }; all_outcomes.extend(outcomes); } + if let Some(rt) = runtime.as_ref() + && let Err(error) = rt.block_on(delete_databases(golden_databases)) + { + report_server_status(guard.as_mut()); + return Err(error); + } + // Write local run log on --dry-run so results aren't lost if dry_run && !all_outcomes.is_empty() { let runs_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("runs"); @@ -358,6 +399,17 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { Ok(()) } +fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { + let Some(guard) = guard else { + return; + }; + match guard.child.try_wait() { + Ok(Some(status)) => eprintln!("[server] local SpacetimeDB exited unexpectedly: {status}"), + Ok(None) => eprintln!("[server] local SpacetimeDB is still running after benchmark failure"), + Err(error) => eprintln!("[server] failed to read local SpacetimeDB exit status: {error}"), + } +} + /* ------------------------------ analyze ------------------------------ */ fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { From b59d00d63c3df3f5873d71c7af0ea4df22a8219c Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:57:49 -0400 Subject: [PATCH 58/78] Improve C# benchmark reliability --- skills/csharp-server/SKILL.md | 4 +++ .../src/bench/publishers.rs | 26 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 6a92f8b0872..00a20abf2af 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -17,6 +17,8 @@ metadata: Reducers are static methods in a `static partial class`; tables are `public partial struct`s. This reference keeps everything in one `public static partial class Module`, which needs only `using SpacetimeDB;`: +Methods exported through SpacetimeDB attributes, including reducers, procedures, views, HTTP handlers, and routers, must be `public static`; generated bindings invoke them from another class. + ```csharp using SpacetimeDB; @@ -194,6 +196,8 @@ ctx.From.Subscription().RightSemijoin( Declare a procedural view primary key in its attribute: `[SpacetimeDB.View(Accessor = "CatalogEntry", Public = true, PrimaryKey = nameof(CatalogRow.Sku))]`. +Procedural-view primary keys are explicit schema metadata. Add one only when the view itself is required to expose a primary key; a source table's primary key is not inherited by the view. + Inclusive btree ranges use tuples, for example `ctx.Db.Shipment.DeliverBy.Filter((new Timestamp(1_000), new Timestamp(2_000)))`. ## Client Visibility Filters diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 15db995c7c3..b245f70d916 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -221,6 +221,7 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { !message.starts_with("could not compile `") && !message.starts_with("process didn't exit successfully") && !message.starts_with("command [\"cargo\", \"build\"") + && !message.starts_with("command [\"dotnet\", \"publish\"") && !message.starts_with("failed to run custom build command for `") }) }); @@ -251,6 +252,10 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { line.contains(".cs(") && (line.contains(": error cs") || line.contains(": error stdb") || line.contains(": error il")) }); + let silent_dotnet_exit = diagnostic.contains("error: command [\"dotnet\", \"publish\"") + && diagnostic.contains("exited with code 1") + && !has_actionable_compiler_diagnostic + && !has_actionable_dotnet_diagnostic; let dotnet_linker_exit = diagnostic.contains("error msb6006") && diagnostic.contains("\"dotnet.exe\" exited with code 1") && diagnostic.contains("error netsdk1144") @@ -272,6 +277,7 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { || silent_cargo_exit || silent_rust_lld_exit || silent_build_script_exit + || silent_dotnet_exit // The .NET linker occasionally exits without a source diagnostic and // MSBuild reports only MSB6006 + NETSDK1144. Do not retry when the // output also contains a C#, SpacetimeDB codegen, or IL source error. @@ -279,7 +285,9 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { // A child process can occasionally disappear on Windows after login but // before the CLI emits a compiler/MSBuild diagnostic. Retrying is safe: // benchmark publishes always use their own temporary CLI root and database. - || (!has_actionable_compiler_diagnostic && !has_terminal_failure_marker) + || (!has_actionable_compiler_diagnostic + && !has_actionable_dotnet_diagnostic + && !has_terminal_failure_marker) } #[cfg(test)] @@ -374,6 +382,22 @@ mod tests { )); } + #[test] + fn retries_dotnet_publish_exit_without_diagnostic() { + assert!(is_transient_build_error( + "Error: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1", + "" + )); + } + + #[test] + fn does_not_retry_dotnet_publish_exit_with_source_diagnostic() { + assert!(!is_transient_build_error( + "Error: command [\"dotnet\", \"publish\", \"-c\", \"Release\"] exited with code 1", + "src/Module.cs(10,5): error CS0122: method is inaccessible due to its protection level" + )); + } + #[test] fn does_not_retry_dotnet_linker_exit_with_source_diagnostic() { assert!(!is_transient_build_error( From 219d8668a5a4952ee983614940277c4621ba35cd Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:57:42 -0400 Subject: [PATCH 59/78] Clarify C# connection ID encoding eval --- .../lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index c75fba666f6..ec0df7626d3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -19,5 +19,6 @@ REDUCERS - ExercisePresence() - Insert synthetic connection IDs 1 and 2 for the caller - Use the 128-bit numeric values 1 and 2 + - Construct the IDs from 16-byte little-endian values - Remove connection ID 1 - Leave connection ID 2 intact From e96a9fc5c42115e62451cb62e4bfa8f3969c040f Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:08:20 -0400 Subject: [PATCH 60/78] Harden LLM benchmark validation and cleanup --- .../Runtime/build/SpacetimeDB.Runtime.targets | 2 +- .../src/bench/publishers.rs | 73 +++++++++++++++++-- .../answers/csharp.cs | 21 +++++- .../answers/rust.rs | 21 +++++- .../answers/typescript.ts | 7 +- .../t_079_external_upload_flow/spec.rs | 14 +++- .../tasks/csharp.txt | 4 +- .../t_079_external_upload_flow/tasks/rust.txt | 4 +- .../tasks/typescript.txt | 4 +- tools/xtask-llm-benchmark/src/eval/utils.rs | 60 +++++++++++++-- 10 files changed, 185 insertions(+), 25 deletions(-) diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index e3b4bd0e942..88744fc21fe 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -2,7 +2,7 @@ + Condition="'$(_UseNativeAotLlvm)' == 'true' and '$(ILCompilerTargetsPath)' == '' and '$(PkgMicrosoft_DotNet_ILCompiler_LLVM)' != '' and Exists('$(PkgMicrosoft_DotNet_ILCompiler_LLVM)\build\Microsoft.DotNet.ILCompiler.LLVM.targets')" /> diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 71f5cae1a65..c83faff4090 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -132,12 +132,39 @@ fn resolve_node_exe(nodejs_dir: Option<&Path>) -> Option { struct CliRootDir { path: PathBuf, + remove_on_drop: bool, } impl CliRootDir { fn path(&self) -> &Path { &self.path } + + fn remove(&self) -> Result<()> { + match fs::remove_dir_all(&self.path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("failed to remove CLI root {}", self.path.display())), + } + } + + fn claim_cleanup(&mut self) { + self.remove_on_drop = true; + } + + fn relinquish_cleanup(&mut self) { + self.remove_on_drop = false; + } +} + +impl Drop for CliRootDir { + fn drop(&mut self) { + if self.remove_on_drop + && let Err(error) = self.remove() + { + eprintln!("[cleanup] failed to remove CLI root {}: {error:#}", self.path.display()); + } + } } fn spacetime_cmd(cli_root: &CliRootDir) -> Command { @@ -195,7 +222,8 @@ pub struct PublishedDatabase { } impl PublishedDatabase { - fn new(host_url: &str, db: String, cli_root: CliRootDir) -> Self { + fn new(host_url: &str, db: String, mut cli_root: CliRootDir) -> Self { + cli_root.claim_cleanup(); Self { host_url: host_url.to_owned(), db, @@ -223,6 +251,8 @@ impl PublishedDatabase { pub fn delete(mut self) -> Result<()> { self.delete_inner()?; self.deleted = true; + self.cli_root.remove()?; + self.cli_root.relinquish_cleanup(); Ok(()) } @@ -230,6 +260,7 @@ impl PublishedDatabase { /// has been created by a migration republish. pub fn relinquish(mut self) { self.deleted = true; + self.cli_root.relinquish_cleanup(); } } @@ -256,11 +287,11 @@ pub trait Publisher: Send + Sync { ) -> Result; } -fn database_cli_root(module_name: &str) -> Result { +fn database_cli_root(module_name: &str, remove_on_drop: bool) -> Result { let db = sanitize_db_name(module_name); let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{db}", CLI_SESSION_TAG.as_str())); fs::create_dir_all(&path)?; - Ok(CliRootDir { path }) + Ok(CliRootDir { path, remove_on_drop }) } /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) @@ -359,7 +390,35 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { #[cfg(test)] mod tests { - use super::is_transient_build_error; + use super::{is_transient_build_error, CliRootDir}; + use std::fs; + + #[test] + fn cli_root_cleans_up_when_it_owns_the_directory() { + let path = std::env::temp_dir().join(format!("stdb-cli-root-cleanup-test-{}", std::process::id())); + fs::create_dir_all(&path).unwrap(); + + drop(CliRootDir { + path: path.clone(), + remove_on_drop: true, + }); + + assert!(!path.exists()); + } + + #[test] + fn cli_root_can_relinquish_cleanup_to_a_republish() { + let path = std::env::temp_dir().join(format!("stdb-cli-root-republish-test-{}", std::process::id())); + fs::create_dir_all(&path).unwrap(); + + drop(CliRootDir { + path: path.clone(), + remove_on_drop: false, + }); + + assert!(path.exists()); + fs::remove_dir_all(path).unwrap(); + } #[test] fn retries_publish_exit_without_actionable_diagnostic() { @@ -596,7 +655,7 @@ impl Publisher for DotnetPublisher { let source = source .canonicalize() .with_context(|| format!("failed to resolve C# source path {}", source.display()))?; - let cli_root = database_cli_root(module_name)?; + let cli_root = database_cli_root(module_name, clear_database)?; let mut pubcmd = spacetime_cmd(&cli_root); pubcmd.arg("publish"); @@ -651,7 +710,7 @@ impl Publisher for SpacetimeRustPublisher { // sanitize db + server let db = sanitize_db_name(module_name); - let cli_root = database_cli_root(module_name)?; + let cli_root = database_cli_root(module_name, clear_database)?; // 2) Publish let mut pubcmd = spacetime_cmd(&cli_root); @@ -705,7 +764,7 @@ impl Publisher for TypeScriptPublisher { Self::ensure_package_json(source)?; let db = sanitize_db_name(module_name); - let cli_root = database_cli_root(module_name)?; + let cli_root = database_cli_root(module_name, clear_database)?; // Install dependencies (--ignore-workspace to avoid parent workspace interference). let nodejs_dir = configured_nodejs_dir(); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs index 453021cf307..1970290d374 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs @@ -4,7 +4,14 @@ public static partial class Module { [Table(Accessor = "UploadedAsset", Public = true)] - public partial struct UploadedAsset { [PrimaryKey] public ulong Id; public string Url; public ulong Size; } + public partial struct UploadedAsset + { + [PrimaryKey] public ulong Id; + public string Url; + public ulong Size; + public ushort Status; + public bool ResponseBodyPresent; + } [SpacetimeDB.HttpHandler] public static HttpResponse Upload(HandlerContext ctx, HttpRequest request) => new( @@ -25,7 +32,17 @@ public static string UploadAndRegister(ProcedureContext ctx, string uploadUrl, b }; return ctx.Http.Send(request).Match(response => { if (response.StatusCode < 200 || response.StatusCode >= 300) throw new Exception($"upload failed: {response.StatusCode}"); - ctx.WithTx(tx => { tx.Db.UploadedAsset.Insert(new UploadedAsset { Id = 1, Url = uploadUrl, Size = (ulong)data.Length }); return 0; }); + var responseBodyPresent = response.Body.ToBytes().Length > 0; + ctx.WithTx(tx => { + tx.Db.UploadedAsset.Insert(new UploadedAsset { + Id = 1, + Url = uploadUrl, + Size = (ulong)data.Length, + Status = response.StatusCode, + ResponseBodyPresent = responseBodyPresent, + }); + return 0; + }); return uploadUrl; }, error => throw new Exception(error.Message)); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs index 87febdcab04..31bcef63db3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs @@ -2,7 +2,14 @@ use spacetimedb::{procedure, table, ProcedureContext, Table}; use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; #[table(accessor = uploaded_asset, public)] -pub struct UploadedAsset { #[primary_key] pub id: u64, pub url: String, pub size: u64 } +pub struct UploadedAsset { + #[primary_key] + pub id: u64, + pub url: String, + pub size: u64, + pub status: u16, + pub response_body_present: bool, +} #[handler] fn upload(_ctx: &mut HandlerContext, _request: Request) -> Response { @@ -18,7 +25,17 @@ pub fn upload_and_register(ctx: &mut ProcedureContext, upload_url: String, data: .body(Body::from_bytes(data.clone())).unwrap(); let response = ctx.http.send(request).expect("upload failed"); assert!(response.status().is_success(), "upload failed: {}", response.status()); + let status = response.status().as_u16(); + let response_body_present = !response.into_body().into_bytes().is_empty(); let row_url = upload_url.clone(); - ctx.with_tx(|tx| { tx.db.uploaded_asset().insert(UploadedAsset { id: 1, url: row_url.clone(), size: data.len() as u64 }); }); + ctx.with_tx(|tx| { + tx.db.uploaded_asset().insert(UploadedAsset { + id: 1, + url: row_url.clone(), + size: data.len() as u64, + status, + response_body_present, + }); + }); upload_url } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts index 13f52a86438..56e6e096c62 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts @@ -1,7 +1,7 @@ import { Router, SyncResponse, schema, table, t } from 'spacetimedb/server'; const uploadedAsset = table({ name: 'uploaded_asset', public: true }, { - id: t.u64().primaryKey(), url: t.string(), size: t.u64(), + id: t.u64().primaryKey(), url: t.string(), size: t.u64(), status: t.u16(), responseBodyPresent: t.bool(), }); const spacetimedb = schema({ uploadedAsset }); export default spacetimedb; @@ -19,7 +19,10 @@ export const upload_and_register = spacetimedb.procedure( { method: 'POST', headers: { 'content-type': 'application/octet-stream' }, body: new Uint8Array(data) } ); if (response.status < 200 || response.status >= 300) throw new Error(`upload failed: ${response.status}`); - ctx.withTx(tx => tx.db.uploadedAsset.insert({ id: 1n, url: uploadUrl, size: BigInt(data.length) })); + const responseBodyPresent = response.bytes().length > 0; + ctx.withTx(tx => tx.db.uploadedAsset.insert({ + id: 1n, url: uploadUrl, size: BigInt(data.length), status: response.status, responseBodyPresent, + })); return uploadUrl; } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs index 2fb34d7c179..efa06980138 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs @@ -10,8 +10,12 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_call_output_parity_scorer( - host_url, file!(), route_tag, "upload_and_register", - vec![json!("https://postman-echo.com/post"), json!([1, 2, 3, 4])], "upload_return_url", + host_url, + file!(), + route_tag, + "upload_and_register", + vec![json!("https://postman-echo.com/post"), json!([1, 2, 3, 4])], + "upload_return_url", )); scorers.push(make_http_route_parity_scorer( host_url, @@ -24,9 +28,13 @@ pub fn spec() -> BenchmarkSpec { let table = table_name("uploaded_asset", lang); let url = ident("url", casing_for_lang(lang)); let size = ident("size", casing_for_lang(lang)); + let status = ident("status", casing_for_lang(lang)); + let response_body_present = ident("response_body_present", casing_for_lang(lang)); scorers.push(make_sql_count_only_scorer( host_url, file!(), route_tag, - format!("SELECT COUNT(*) AS n FROM {table} WHERE {url}='https://postman-echo.com/post' AND {size}=4"), + format!( + "SELECT COUNT(*) AS n FROM {table} WHERE {url}='https://postman-echo.com/post' AND {size}=4 AND {status}=200 AND {response_body_present}=true" + ), 1, "upload_metadata_stored", Duration::from_secs(10), )); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt index 5514c722006..5817b13ce75 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -6,6 +6,8 @@ TABLE - Id: ulong (primary key) - Url: string - Size: ulong + - Status: ushort + - ResponseBodyPresent: bool HANDLER - POST /upload @@ -16,7 +18,7 @@ PROCEDURE - UploadAndRegister(uploadUrl: string, data: byte[]) -> string - POST the binary data to the supplied public URL - Require a successful response - - Open a transaction and store Id 1, the supplied URL, and the byte length + - Open a transaction and store Id 1, the supplied URL, the byte length, the response status, and whether the response body is non-empty - Return the supplied URL ROUTER diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt index c3bb0c861fc..f528ede25ac 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -6,6 +6,8 @@ TABLE - id: u64 (primary key) - url: String - size: u64 + - status: u16 + - response_body_present: bool HANDLER - POST /upload @@ -16,7 +18,7 @@ PROCEDURE - upload_and_register(upload_url: String, data: Vec) -> String - POST the binary data to the supplied public URL - Require a successful response - - Open a transaction and store id 1, the supplied URL, and the byte length + - Open a transaction and store id 1, the supplied URL, the byte length, the response status, and whether the response body is non-empty - Return the supplied URL ROUTER diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt index 1a2e9e067d3..3f06b4e2dcb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -6,6 +6,8 @@ TABLE - id: u64 (primary key) - url: string - size: u64 + - status: u16 + - responseBodyPresent: bool HANDLER - POST /upload @@ -16,7 +18,7 @@ PROCEDURE - upload_and_register(uploadUrl: string, data: u8[]) -> string - POST the binary data to the supplied public URL - Require a successful response - - Open a transaction and store id 1, the supplied URL, and the byte length + - Open a transaction and store id 1, the supplied URL, the byte length, the response status, and whether the response body is non-empty - Return the supplied URL ROUTER diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 65023c9f617..1f7a6945431 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -1,5 +1,5 @@ use std::env; -use std::io; +use std::io::{self, Read}; use std::path::Path; use std::process::{Command, Stdio}; use std::thread; @@ -57,19 +57,39 @@ pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; + let mut stdout = child.stdout.take().expect("stdout was configured as piped"); + let mut stderr = child.stderr.take().expect("stderr was configured as piped"); + let stdout_reader = thread::spawn(move || { + let mut output = Vec::new(); + stdout.read_to_end(&mut output)?; + Ok::<_, io::Error>(output) + }); + let stderr_reader = thread::spawn(move || { + let mut output = Vec::new(); + stderr.read_to_end(&mut output)?; + Ok::<_, io::Error>(output) + }); let start = Instant::now(); - loop { + let status = loop { if let Some(status) = child.try_wait()? { - let out = child.wait_with_output()?; - return Ok((status.code().unwrap_or(-1), out.stdout, out.stderr)); + break status; } if start.elapsed() >= timeout { let _ = child.kill(); let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); } thread::sleep(Duration::from_millis(30)); - } + }; + let stdout = stdout_reader + .join() + .map_err(|_| io::Error::other("stdout reader thread panicked"))??; + let stderr = stderr_reader + .join() + .map_err(|_| io::Error::other("stderr reader thread panicked"))??; + Ok((status.code().unwrap_or(-1), stdout, stderr)) } pub fn normalize(s: &str, collapse_ws: bool) -> String { @@ -106,4 +126,34 @@ mod tests { assert_eq!(error.kind(), io::ErrorKind::TimedOut); assert!(started.elapsed() < Duration::from_secs(3)); } + + #[test] + fn process_output_is_drained_while_the_command_runs() { + #[cfg(windows)] + let command = { + let mut command = Command::new("powershell"); + command.args([ + "-NoProfile", + "-Command", + "$chunk = 'x' * 4096; 1..256 | ForEach-Object { [Console]::Out.Write($chunk); [Console]::Error.Write($chunk) }", + ]); + command + }; + #[cfg(not(windows))] + let command = { + let mut command = Command::new("sh"); + command.args([ + "-c", + "dd if=/dev/zero bs=4096 count=256 2>/dev/null; dd if=/dev/zero bs=4096 count=256 1>&2 2>/dev/null", + ]); + command + }; + + let (code, stdout, stderr) = run_with_timeout(command, Path::new("."), Duration::from_secs(10)) + .expect("large output should not deadlock"); + + assert_eq!(code, 0); + assert_eq!(stdout.len(), 4096 * 256); + assert_eq!(stderr.len(), 4096 * 256); + } } From 9bcdc72326612fc6cc672458840af8a2ca47aaab Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:29:31 -0400 Subject: [PATCH 61/78] Clarify HTTP APIs in server benchmark skills --- skills/csharp-server/SKILL.md | 4 +++- skills/rust-server/SKILL.md | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 00a20abf2af..5d82f7d1b23 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -312,7 +312,7 @@ var result = ctx.Http.Send(new HttpRequest }); ``` -`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples, and each header's `Value` is `byte[]`; decode text values with `System.Text.Encoding.UTF8.GetString(header.Value)`. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text, `new HttpBody(bytes)` to supply raw bytes, and `ToStringUtf8Lossy()` to read text. There is no `HttpBody.FromBytes`, and `ToString()` does not return body contents. Qualify `SpacetimeDB.HttpMethod` when .NET's implicit `System.Net.Http` imports could make the name ambiguous. +`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples, and each header's `Value` is `byte[]`; decode text values with `System.Text.Encoding.UTF8.GetString(header.Value)`. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text, `new HttpBody(bytes)` to supply raw bytes, `ToBytes()` to read raw bytes, and `ToStringUtf8Lossy()` to read text. There is no `HttpBody.FromBytes`, and `ToString()` does not return body contents. Qualify `SpacetimeDB.HttpMethod` when .NET's implicit `System.Net.Http` imports could make the name ambiguous. Scheduled procedures use the ordinary scheduled-table shape. Its `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row, and database access inside that procedure goes through `ctx.WithTx`. @@ -328,6 +328,8 @@ public static HttpResponse Health(HandlerContext ctx, HttpRequest request) => ne public static Router Routes() => SpacetimeDB.Router.New().Get("/health", Handlers.Health); ``` +`Handlers` is generated from methods marked `[SpacetimeDB.HttpHandler]`; do not declare it yourself. Reference an attributed method in a router as `Handlers.MethodName`. + ## Custom Types ```csharp diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index b9698ea63c2..f10e8caf615 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -273,9 +273,9 @@ pub fn inspect(_ctx: &mut ProcedureContext, input: String) -> ResultValue { } ``` -Outbound HTTP is available through `ctx.http`. Convenience methods such as `get` return a response, and other methods use `Request::builder()` with `ctx.http.send(request)`. Consume a response body as text with `response.into_body().into_string_lossy()`. Header values use the fallible `to_str()` conversion; they do not provide `as_str()`. +Outbound HTTP is available through `ctx.http`. Convenience methods such as `get` return a response, and other methods use `Request::builder()` with `ctx.http.send(request)`. `response.status()` returns a `StatusCode`; use `is_success()` to test it or `as_u16()` when a numeric status is needed. Responses are not cloneable, so inspect status and headers before consuming the body with `response.into_body().into_string_lossy()`. Header values use the fallible `to_str()` conversion; they do not provide `as_str()`. -Open short database transactions with `ctx.with_tx(|tx| ...)`. It returns the callback's value directly, so do not call `unwrap` or `expect` on the result unless the callback itself returns a `Result`. The callback implements `Fn`, so clone captured owned values when storing them rather than moving them out of the closure. Perform network I/O before opening the transaction. +Open short database transactions with `ctx.with_tx(|tx| ...)`. Access tables inside the callback through `tx.db`, not directly on `tx`. It returns the callback's value directly, so do not call `unwrap` or `expect` on the result unless the callback itself returns a `Result`. The callback implements `Fn`, so clone captured owned values when storing them rather than moving them out of the closure. Perform network I/O before opening the transaction. For an outbound request without a convenience method: From 438f71c4f2e1f7b674307a4729d1f0d40046153f Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:22:36 -0400 Subject: [PATCH 62/78] Clarify TypeScript benchmark guidance --- skills/typescript-server/SKILL.md | 14 ++++++++++++-- .../t_065_materialized_aggregate/tasks/csharp.txt | 4 ++-- .../t_065_materialized_aggregate/tasks/rust.txt | 4 ++-- .../tasks/typescript.txt | 4 ++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index d35774d1288..6a809e151fe 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -108,7 +108,7 @@ Treat requested table options and column modifiers as part of the schema contrac ## Indexes -Prefer inline `.index('btree')` for single-column. Use named indexes only for multi-column: +Use inline `.index('btree')` when a single-column index does not need a named accessor. Use an `indexes` entry when the accessor is named explicitly or the index spans multiple columns. Every `indexes` entry requires `columns`; do not also add `.index('btree')` to the same column. ```typescript // Inline (preferred for single-column): @@ -184,6 +184,14 @@ export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); `ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. +```typescript +type Ctx = ReducerCtx>; + +function findRecord(ctx: Ctx, id: bigint) { + return ctx.db.score_record.id.find(id); +} +``` + ```typescript // Auth: ctx.sender is the caller's Identity if (!row.owner.equals(ctx.sender)) throw new SenderError('unauthorized'); @@ -310,7 +318,9 @@ ctx => ctx.from.subscription.rightSemijoin( ) ``` -For `A.leftSemijoin(B, ...)`, the result contains rows from `A`; for `A.rightSemijoin(B, ...)`, it contains rows from `B`. Semijoins do not project combined columns from both tables. Use a procedural view when the result is a custom row assembled from multiple tables. +The method name identifies which side is returned: `A.leftSemijoin(B, ...)` returns rows from `A`, while `A.rightSemijoin(B, ...)` returns rows from `B`. To return one table's rows when another table has a match, put the returned table on the corresponding side. Semijoins do not project combined columns from both tables. + +Procedural views read through `ctx.db` and return materialized values such as arrays. Query-builder values from `ctx.from` are returned directly; they are not iterators and cannot be spread, looped over, or mixed with array methods. Use a procedural view when the result is a custom row assembled from multiple tables. ## Client Visibility Filters diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt index 499cbdcf169..0be092a9636 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -16,9 +16,9 @@ REDUCERS - Exercise() - This is the only exported reducer - Keep CategoryTotal transactionally synchronized when a Sale is inserted, changed, moved between categories, or deleted - - Insert books Sale rows of 10 and 20 + - Insert books Sale rows with Ids 1 and 2 and Amounts 10 and 20 - Update the second books Sale to 25 - - Insert and then delete a games Sale + - Insert and then delete a games Sale with Id 3 - Delete the first books Sale VIEW diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt index b10e3c27380..29043396e57 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -16,9 +16,9 @@ REDUCERS - exercise() - This is the only exported reducer - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted - - Insert books sales of 10 and 20 + - Insert books sales with ids 1 and 2 and amounts 10 and 20 - Update the second books sale to 25 - - Insert and then delete a games sale + - Insert and then delete a games sale with id 3 - Delete the first books sale VIEW diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt index 8521b0a41be..4f0d4616c77 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -16,9 +16,9 @@ REDUCERS - exercise() - This is the only exported reducer - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted - - Insert books sales of 10 and 20 + - Insert books sales with ids 1 and 2 and amounts 10 and 20 - Update the second books sale to 25 - - Insert and then delete a games sale + - Insert and then delete a games sale with id 3 - Delete the first books sale VIEW From 5aac86619d7170058ada602a00dd7b6d3628ca26 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:22:50 -0400 Subject: [PATCH 63/78] Harden benchmark regression tests --- tools/xtask-llm-benchmark/src/eval/scorers.rs | 14 +++++++++++ tools/xtask-llm-benchmark/src/eval/utils.rs | 24 +++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 1400142353d..fb23572ea7f 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -1349,6 +1349,20 @@ mod tests { assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); } + #[test] + fn missing_default_value_produces_a_schema_diff() { + let mut golden = current_schema(true); + golden["tables"][0]["default_values"] = json!([{ + "col_id": 1, + "value": [1] + }]); + + let golden = extract_schema(&golden); + let candidate = extract_schema(¤t_schema(true)); + + assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); + } + #[test] fn v10_schema_compares_explicit_canonical_table_names() { let schema = |source_name: &str| { diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 1f7a6945431..9da184a33a5 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -104,9 +104,16 @@ pub fn normalize(s: &str, collapse_ws: bool) -> String { #[cfg(test)] mod tests { use super::*; + use std::sync::Mutex; + + static PROCESS_TEST_LOCK: Mutex<()> = Mutex::new(()); #[test] fn process_timeout_kills_a_long_running_command() { + let _guard = PROCESS_TEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + #[cfg(windows)] let command = { let mut command = Command::new("powershell"); @@ -129,13 +136,17 @@ mod tests { #[test] fn process_output_is_drained_while_the_command_runs() { + let _guard = PROCESS_TEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + #[cfg(windows)] let command = { let mut command = Command::new("powershell"); command.args([ "-NoProfile", "-Command", - "$chunk = 'x' * 4096; 1..256 | ForEach-Object { [Console]::Out.Write($chunk); [Console]::Error.Write($chunk) }", + "$chunk = 'x' * 4096; 1..16 | ForEach-Object { [Console]::Out.Write($chunk); [Console]::Error.Write($chunk) }", ]); command }; @@ -149,11 +160,16 @@ mod tests { command }; - let (code, stdout, stderr) = run_with_timeout(command, Path::new("."), Duration::from_secs(10)) + #[cfg(windows)] + let expected_len = 4096 * 16; + #[cfg(not(windows))] + let expected_len = 4096 * 256; + + let (code, stdout, stderr) = run_with_timeout(command, Path::new("."), Duration::from_secs(15)) .expect("large output should not deadlock"); assert_eq!(code, 0); - assert_eq!(stdout.len(), 4096 * 256); - assert_eq!(stderr.len(), 4096 * 256); + assert_eq!(stdout.len(), expected_len); + assert_eq!(stderr.len(), expected_len); } } From bdbeb968872d362151cfe86d06102476a4d53262 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:39:50 -0400 Subject: [PATCH 64/78] Remove bias --- skills/csharp-server/SKILL.md | 2 +- .../benchmarks/auth/t_083_row_level_security/tasks/csharp.txt | 3 +-- .../benchmarks/auth/t_083_row_level_security/tasks/rust.txt | 3 +-- .../auth/t_083_row_level_security/tasks/typescript.txt | 3 +-- .../t_070_connection_scoped_presence/tasks/csharp.txt | 1 - 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 5d82f7d1b23..bacfceba14d 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -267,7 +267,7 @@ ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. -Construct a connection ID from 16 bytes with `ConnectionId.From(bytes)`; the result is nullable and must be checked. +To construct a `ConnectionId` from a 128-bit numeric representation, encode it as exactly 16 little-endian bytes and call `ConnectionId.From(bytes)`. The result is nullable and must be checked before use. ## Procedures and HTTP diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt index 7bfa3c2e3d9..4a5868806d9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt @@ -7,8 +7,7 @@ TABLE - Name: string CLIENT VISIBILITY FILTER -- SQL exactly: `SELECT * FROM UserRecord WHERE Identity = :sender` -- Each caller can see only its own row +- Restrict UserRecord so each caller can see only rows whose Identity matches the caller REDUCERS - RegisterSelf(name: string) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt index cf8d6854729..a87cf289dda 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt @@ -7,8 +7,7 @@ TABLE - name: String CLIENT VISIBILITY FILTER -- SQL exactly: `SELECT * FROM user_record WHERE identity = :sender` -- Each caller can see only its own row +- Restrict user_record so each caller can see only rows whose identity matches the caller REDUCERS - register_self(name: String) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt index 45ec9688a86..5a6ea23495e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt @@ -7,8 +7,7 @@ TABLE - name: string CLIENT VISIBILITY FILTER -- SQL exactly: `SELECT * FROM user_record WHERE identity = :sender` -- Each caller can see only its own row +- Restrict user_record so each caller can see only rows whose identity matches the caller REDUCERS - register_self(name: string) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt index ec0df7626d3..c75fba666f6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -19,6 +19,5 @@ REDUCERS - ExercisePresence() - Insert synthetic connection IDs 1 and 2 for the caller - Use the 128-bit numeric values 1 and 2 - - Construct the IDs from 16-byte little-endian values - Remove connection ID 1 - Leave connection ID 2 intact From 21125cb5a598eb233761ba0cd705ea864e38e643 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:01:31 -0400 Subject: [PATCH 65/78] remove bias --- skills/csharp-server/SKILL.md | 2 +- skills/rust-server/SKILL.md | 2 +- skills/typescript-server/SKILL.md | 4 +--- .../benchmarks/tables/t_053_default_values/tasks/csharp.txt | 1 - .../src/benchmarks/tables/t_053_default_values/tasks/rust.txt | 1 - .../tables/t_053_default_values/tasks/typescript.txt | 1 - .../src/benchmarks/tables/t_054_special_types/tasks/rust.txt | 1 - 7 files changed, 3 insertions(+), 9 deletions(-) diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index bacfceba14d..1d08ba89fe7 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -97,7 +97,7 @@ The complete set of column attributes: [Default(true)] // migration-safe default for a newly appended field ``` -Defaults are for compatible schema upgrades. Preserve existing fields and reducers, append the defaulted field, and do not apply `[Default(...)]` to primary-key, unique, or auto-increment fields. +Defaults support compatible addition of a newly appended field. Do not apply `[Default(...)]` to primary-key, unique, or auto-increment fields. ## Indexes diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index f10e8caf615..0e282242263 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -74,7 +74,7 @@ Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, #[default(true)] // migration-safe default for a newly appended column ``` -Defaults are for compatible schema upgrades. Append the defaulted field, preserve all existing fields and reducers exactly, and do not place `#[default(...)]` on primary-key, unique, or auto-increment columns. +Defaults support compatible addition of a newly appended field. Do not place `#[default(...)]` on primary-key, unique, or auto-increment columns. ## Indexes diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 6a809e151fe..9c7a44408c5 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -98,14 +98,12 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. -Use `.default(value)` only for a newly appended migration-safe field. Preserve existing fields and reducers exactly, and do not put defaults on primary-key, unique, or auto-increment columns. +Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` Schema builders describe the database's wire types; they are not TypeScript type names. For example, a `t.u16()` value is a TypeScript `number`, not a value cast to a type named `u16`. -Treat requested table options and column modifiers as part of the schema contract. Do not add `event`, `autoInc`, `unique`, indexes, or defaults unless the module requirements call for them. - ## Indexes Use inline `.index('btree')` when a single-column index does not need a named accessor. Use an `indexes` entry when the accessor is named explicitly or the index spans multiple columns. Every `indexes` entry requires `columns`; do not also add `.index('btree')` to the same column. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt index f00c940541f..8499bad93c9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt @@ -13,4 +13,3 @@ REDUCERS MIGRATION - A compatible republish must backfill existing rows -- Do not put the default annotation on a primary-key, unique, or auto-increment column diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt index 20ab31dcb0b..37ecc05516f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt @@ -13,4 +13,3 @@ REDUCERS MIGRATION - A compatible republish must backfill existing rows -- Do not put the default annotation on a primary-key, unique, or auto-increment column diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt index 3ee41ae4228..61d1e32ef14 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt @@ -13,4 +13,3 @@ REDUCERS MIGRATION - A compatible republish must backfill existing rows -- Do not put the default on a primary-key, unique, or auto-increment column diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt index fd0a5b9a3ec..7264c4157f4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt @@ -11,6 +11,5 @@ TABLE - signed_128: i128 - unsigned_256: u256 - signed_256: i256 - - Use the SpacetimeDB integer types for 256-bit values No reducers are required. From a4a4a3af0210611e0d1c3cdda49ddd89a40f43ed Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:45:16 -0400 Subject: [PATCH 66/78] cleanup --- .../Runtime/build/SpacetimeDB.Runtime.targets | 2 +- crates/cli/src/api.rs | 13 ---- crates/cli/src/subcommands/describe.rs | 58 +++++---------- tools/xtask-llm-benchmark/Cargo.toml | 2 +- .../src/bench/publishers.rs | 72 ++++++------------- tools/xtask-llm-benchmark/src/eval/scorers.rs | 47 +++++++----- tools/xtask-llm-benchmark/src/eval/utils.rs | 35 +++++++-- 7 files changed, 99 insertions(+), 130 deletions(-) diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index 88744fc21fe..e3b4bd0e942 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -2,7 +2,7 @@ + Condition="'$(EXPERIMENTAL_WASM_AOT)' == '1' and '$(ILCompilerTargetsPath)' == '' and '$(PkgMicrosoft_DotNet_ILCompiler_LLVM)' != '' and Exists('$(PkgMicrosoft_DotNet_ILCompiler_LLVM)\build\Microsoft.DotNet.ILCompiler.LLVM.targets')" /> diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index 33ca6e547d0..d40b03ee87e 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,7 +4,6 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; -use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; @@ -73,18 +72,6 @@ impl ClientApi { Ok(module_def) } - /// Reads the version 10 `ModuleDef` from the `schema` endpoint. - pub async fn module_def_v10(&self) -> anyhow::Result { - let res = self - .client - .get(self.con.db_uri("schema")) - .query(&[("version", "10")]) - .send() - .await?; - let DeserializeWrapper(module_def) = res.json_or_error().await?; - Ok(module_def) - } - pub async fn call(&self, reducer_name: &str, arg_json: String) -> anyhow::Result { Ok(self .client diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index b1de53aedd1..e774224855c 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -38,13 +38,6 @@ pub fn cli() -> clap::Command { .action(ArgAction::SetTrue) .help("Ignore spacetime.json configuration"), ) - .arg( - Arg::new("schema_version") - .long("schema-version") - .value_parser(["9", "10"]) - .default_value("9") - .help("Schema ABI version to request"), - ) .after_help("Run `spacetime help describe` for more detailed information.\n") } @@ -59,10 +52,6 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error let json = args.get_flag("json"); let no_config = args.get_flag("no_config"); - let schema_version = args - .get_one::("schema_version") - .map(String::as_str) - .unwrap_or("9"); let raw_parts: Vec = args .get_many::("describe_parts") .map(|vals| vals.cloned().collect()) @@ -108,41 +97,30 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error }; let api = ClientApi::new(conn); + let module_def = api.module_def().await?; + if json { fn sats_to_json(v: &T) -> serde_json::Result { serde_json::to_string_pretty(sats::serde::SerdeWrapper::from_ref(v)) } - let json = match schema_version { - "9" => { - let module_def = api.module_def().await?; - match entity { - Some((EntityType::Reducer, reducer_name)) => { - let reducer = module_def - .reducers - .iter() - .find(|r| *r.name == *reducer_name) - .context("no such reducer")?; - sats_to_json(reducer)? - } - Some((EntityType::Table, table_name)) => { - let table = module_def - .tables - .iter() - .find(|t| *t.name == *table_name) - .context("no such table")?; - sats_to_json(table)? - } - None => sats_to_json(&module_def)?, - } + let json = match entity { + Some((EntityType::Reducer, reducer_name)) => { + let reducer = module_def + .reducers + .iter() + .find(|r| *r.name == *reducer_name) + .context("no such reducer")?; + sats_to_json(reducer)? } - "10" => { - anyhow::ensure!( - entity.is_none(), - "entity filtering is only supported for schema version 9" - ); - sats_to_json(&api.module_def_v10().await?)? + Some((EntityType::Table, table_name)) => { + let table = module_def + .tables + .iter() + .find(|t| *t.name == *table_name) + .context("no such table")?; + sats_to_json(table)? } - _ => unreachable!("clap validates schema_version"), + None => sats_to_json(&module_def)?, }; // TODO: validate the JSON output diff --git a/tools/xtask-llm-benchmark/Cargo.toml b/tools/xtask-llm-benchmark/Cargo.toml index a0d70f49af6..6565fa9723e 100644 --- a/tools/xtask-llm-benchmark/Cargo.toml +++ b/tools/xtask-llm-benchmark/Cargo.toml @@ -24,7 +24,7 @@ dotenvy = "0.15" async-trait = "0.1.89" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } urlencoding = "2.1.3" -reqwest = { version = "0.12", features = ["json"] } +reqwest = { version = "0.12", features = ["blocking", "json"] } futures = "0.3.31" regex = "1" heck = "0.5.0" diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index c83faff4090..15047e71d70 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -310,10 +310,16 @@ fn signal_killed_by(_status: &std::process::ExitStatus) -> Option { /// These are resource contention issues and diagnostic-free child-process crashes. fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let combined = format!("{stderr}{stdout}"); - let diagnostic = combined.to_ascii_lowercase(); - let has_actionable_compiler_diagnostic = diagnostic.lines().any(|line| { + let output = combined.to_ascii_lowercase(); + let has_source_diagnostic = output.lines().any(|line| { let line = line.trim_start(); line.starts_with("error[") + || line.contains(": error cs") + || line.contains(": error stdb") + || line.contains(": error il") + || line.starts_with("error ts") + || line.contains("rust-lld: error:") + || line.contains("lld-link: error:") || line.strip_prefix("error:").is_some_and(|message| { let message = message.trim_start(); !message.starts_with("could not compile `") @@ -321,24 +327,14 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { && !message.starts_with("command [\"cargo\", \"build\"") && !message.starts_with("command [\"dotnet\", \"publish\"") && !message.starts_with("failed to run custom build command for `") + && !message.starts_with("linking with `rust-lld.exe` failed") }) }); - let silent_rustc_exit = diagnostic.contains("process didn't exit successfully") - && diagnostic.contains("--crate-name") - && diagnostic.contains("(exit code: 1)") - && !has_actionable_compiler_diagnostic; - let silent_cargo_exit = diagnostic.contains("error: command [\"cargo\", \"build\"") - && diagnostic.contains("exited with code 1") - && !has_actionable_compiler_diagnostic; - let silent_rust_lld_exit = diagnostic.contains("error: linking with `rust-lld.exe` failed: exit code: 1") - && !diagnostic.contains("rust-lld: error:") - && !diagnostic.contains("lld-link: error:"); - let silent_build_script_exit = diagnostic.contains("failed to run custom build command for `") - && diagnostic.contains("build-script-build` (exit code: 1)") - && !diagnostic.contains("--- stderr") - && !diagnostic.contains("panicked at") - && !has_actionable_compiler_diagnostic; - let has_terminal_failure_marker = diagnostic.lines().any(|line| { + if has_source_diagnostic { + return false; + } + + let has_terminal_failure = output.lines().any(|line| { let line = line.trim_start(); line.starts_with("error: could not compile `") || line.starts_with("failed:") @@ -346,46 +342,18 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { || line.contains("build failed") || line.contains("panicked at") }); - let has_actionable_dotnet_diagnostic = diagnostic.lines().any(|line| { - line.contains(".cs(") - && (line.contains(": error cs") || line.contains(": error stdb") || line.contains(": error il")) - }); - let silent_dotnet_exit = diagnostic.contains("error: command [\"dotnet\", \"publish\"") - && diagnostic.contains("exited with code 1") - && !has_actionable_compiler_diagnostic - && !has_actionable_dotnet_diagnostic; - let dotnet_linker_exit = diagnostic.contains("error msb6006") - && diagnostic.contains("\"dotnet.exe\" exited with code 1") - && diagnostic.contains("error netsdk1144") - && !has_actionable_dotnet_diagnostic; - // "Pipe is broken" errors from WASI SDK parallel builds + let diagnostic_free_rust_exit = (output.contains("process didn't exit successfully") + && output.contains("--crate-name")) + || output.contains("linking with `rust-lld.exe` failed: exit code: 1"); + combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") - // Other transient resource errors || combined.contains("Unable to read data from the transport connection") - // WASI SDK tar extraction race condition - multiple parallel builds - // trying to extract the same tarball simultaneously || (combined.contains("wasi-sdk") && combined.contains("tar")) || (combined.contains("MSB3073") && combined.contains("exited with code 2")) - // dotnet can crash below spacetime while spacetime exits 1. || combined.contains("code io::Result { - let mut cmd = spacetime_command(); - cmd.arg("describe") - .arg("--json") - .arg("--schema-version") - .arg("10") - .arg("-s") - .arg(server) - .arg("-y") - .arg(db); - let (code, out, err) = run_with_timeout(cmd, Path::new("."), timeout)?; - if code != 0 { - return Err(io::Error::other(format!( - "describe failed: {}", - String::from_utf8_lossy(&err) - ))); - } - let v: Value = serde_json::from_slice(&out).map_err(|e| io::Error::other(format!("parse json: {}", e)))?; - Ok(v) + thread::scope(|scope| { + scope + .spawn(|| describe_db_blocking(server, db, timeout)) + .join() + .map_err(|_| io::Error::other("schema request thread panicked"))? + }) +} + +fn describe_db_blocking(server: &str, db: &str, timeout: Duration) -> io::Result { + let url = format!( + "{}/v1/database/{}/schema", + server.trim_end_matches('/'), + urlencoding::encode(db) + ); + let client = reqwest::blocking::Client::builder() + .timeout(timeout) + .build() + .map_err(io::Error::other)?; + let response = client + .get(url) + .query(&[("version", "10")]) + .send() + .map_err(io::Error::other)?; + let status = response.status(); + if !status.is_success() { + let body = response.text().unwrap_or_default(); + return Err(io::Error::other(format!("schema request failed ({status}): {body}"))); + } + response.json().map_err(io::Error::other) } fn extract_schema(v: &Value) -> SchemaSnapshot { diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 9da184a33a5..93dbbc32e93 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -1,7 +1,7 @@ use std::env; use std::io::{self, Read}; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Child, Command, Stdio}; use std::thread; use std::time::{Duration, Instant}; @@ -52,6 +52,12 @@ pub(crate) fn sql_exec_with_timeout( } pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Result<(i32, Vec, Vec)> { + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let mut child = cmd .current_dir(cwd) .stdout(Stdio::piped()) @@ -75,10 +81,7 @@ pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) break status; } if start.elapsed() >= timeout { - let _ = child.kill(); - let _ = child.wait(); - let _ = stdout_reader.join(); - let _ = stderr_reader.join(); + kill_process_tree(&mut child); return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); } thread::sleep(Duration::from_millis(30)); @@ -92,6 +95,28 @@ pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) Ok((status.code().unwrap_or(-1), stdout, stderr)) } +fn kill_process_tree(child: &mut Child) { + #[cfg(windows)] + let killed = Command::new("taskkill") + .args(["/F", "/T", "/PID", &child.id().to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .is_ok(); + + #[cfg(unix)] + let killed = Command::new("kill") + .args(["-KILL", &format!("-{}", child.id())]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()); + + if !killed { + let _ = child.kill(); + } +} + pub fn normalize(s: &str, collapse_ws: bool) -> String { let t = s.trim(); if collapse_ws { From 07ad9552d9ba221352f2f834aa352c912f1d0b68 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:57:53 -0400 Subject: [PATCH 67/78] Use workspace CLI for LLM benchmark runs --- tools/xtask-llm-benchmark/src/eval/utils.rs | 35 ++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 93dbbc32e93..93d911e844a 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -1,6 +1,6 @@ use std::env; use std::io::{self, Read}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::thread; use std::time::{Duration, Instant}; @@ -24,8 +24,35 @@ pub fn derive_cat_task_from_file(src: &str) -> (String, String) { } pub(crate) fn spacetime_command() -> Command { - let executable = env::var_os("LLM_BENCH_SPACETIME_BIN").unwrap_or_else(|| "spacetime".into()); - Command::new(executable) + if let Some(executable) = env::var_os("LLM_BENCH_SPACETIME_BIN") { + return Command::new(executable); + } + + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("xtask-llm-benchmark is under tools/xtask-llm-benchmark"); + let target = env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| workspace.join("target")); + let target = if target.is_absolute() { + target + } else { + workspace.join(target) + }; + let executable = if cfg!(windows) { + "spacetimedb-cli.exe" + } else { + "spacetimedb-cli" + }; + for profile in ["release", "debug"] { + let candidate = target.join(profile).join(executable); + if candidate.is_file() { + return Command::new(candidate); + } + } + + Command::new("spacetime") } pub fn sql_exec(db: &str, query: &str, host: Option<&str>) -> Result<(), String> { @@ -190,7 +217,7 @@ mod tests { #[cfg(not(windows))] let expected_len = 4096 * 256; - let (code, stdout, stderr) = run_with_timeout(command, Path::new("."), Duration::from_secs(15)) + let (code, stdout, stderr) = run_with_timeout(command, Path::new("."), Duration::from_secs(60)) .expect("large output should not deadlock"); assert_eq!(code, 0); From 173fa669fc7f0fdcc08bf850c3fb590ffc2845af Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:16:06 -0400 Subject: [PATCH 68/78] Parallelize periodic benchmarks by language --- .github/workflows/llm-benchmark-periodic.yml | 98 ++++++++++---------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 8e46d9ebc2b..44b57031088 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -48,9 +48,29 @@ concurrency: cancel-in-progress: true jobs: + prepare_matrix: + name: Prepare benchmark matrix + runs-on: ubuntu-latest + outputs: + languages: ${{ steps.matrix.outputs.languages }} + steps: + - id: matrix + env: + INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }} + run: | + languages="$(jq -cn --arg value "$INPUT_LANGUAGES" '$value | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')" + echo "languages=$languages" >> "$GITHUB_OUTPUT" + run-benchmarks: + name: Run benchmarks (${{ matrix.lang }}) + needs: prepare_matrix runs-on: spacetimedb-new-runner-2 + strategy: + fail-fast: false + matrix: + lang: ${{ fromJSON(needs.prepare_matrix.outputs.languages) }} + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -61,11 +81,13 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Setup .NET SDK + if: matrix.lang == 'csharp' uses: actions/setup-dotnet@v4 with: global-json-file: global.json - name: Install WASI workload + if: matrix.lang == 'csharp' env: DOTNET_MULTILEVEL_LOOKUP: "0" DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home @@ -74,25 +96,25 @@ jobs: dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel - name: Pack C# runtime packages - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'csharp') }} + if: matrix.lang == 'csharp' run: | dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime dotnet pack -c Release crates/bindings-csharp/Runtime - name: Set up Node.js - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }} + if: matrix.lang == 'typescript' uses: actions/setup-node@v4 with: node-version: 22 - name: Install pnpm - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }} + if: matrix.lang == 'typescript' uses: ./.github/actions/setup-pnpm with: run_install: true - name: Build TypeScript SDK - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }} + if: matrix.lang == 'typescript' run: pnpm build working-directory: crates/bindings-typescript @@ -119,8 +141,12 @@ jobs: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" MSBUILDDISABLENODEREUSE: "1" DOTNET_CLI_USE_MSBUILD_SERVER: "0" - LLM_BENCH_CSHARP_CONCURRENCY: "1" - INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }} + LLM_BENCH_CONCURRENCY: "8" + LLM_BENCH_RUST_CONCURRENCY: "4" + LLM_BENCH_CSHARP_CONCURRENCY: "2" + LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }} + LLM_BENCH_RUST_TARGET_DIR: ${{ runner.temp }}/llm-benchmark-rust-target + INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} INPUT_MODES: ${{ inputs.modes || 'guidelines,no_context' }} @@ -128,7 +154,7 @@ jobs: INPUT_TASKS: ${{ inputs.tasks || '' }} INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }} run: | - LANGS="$INPUT_LANGUAGES" + LANG="$INPUT_LANGUAGE" MODEL_SET="$INPUT_MODEL_SET" MODELS="$INPUT_MODELS" MODES="$INPUT_MODES" @@ -162,45 +188,21 @@ jobs: ;; esac - SUCCEEDED=0 - FAILED=0 - for LANG in $(echo "$LANGS" | tr ',' ' '); do - EXTRA_ARGS=() - if [ -n "$CATEGORIES" ]; then - EXTRA_ARGS+=(--categories "$CATEGORIES") - fi - if [ -n "$TASKS" ]; then - EXTRA_ARGS+=(--tasks "$TASKS") - fi - if [ "$DRY_RUN" = "true" ]; then - EXTRA_ARGS+=(--dry-run) - fi - - if [ "$MODEL_SET" = "website_active" ]; then - if llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"; then - SUCCEEDED=$((SUCCEEDED + 1)) - else - echo "::warning::Benchmark run failed for lang=$LANG" - FAILED=$((FAILED + 1)) - fi - elif [ "$MODEL_SET" = "local_defaults" ]; then - if llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}"; then - SUCCEEDED=$((SUCCEEDED + 1)) - else - echo "::warning::Benchmark run failed for lang=$LANG" - FAILED=$((FAILED + 1)) - fi - else - if llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"; then - SUCCEEDED=$((SUCCEEDED + 1)) - else - echo "::warning::Benchmark run failed for lang=$LANG models=$MODELS" - FAILED=$((FAILED + 1)) - fi - fi - done - echo "Benchmark runs: $SUCCEEDED succeeded, $FAILED failed" - if [ "$FAILED" -gt 0 ]; then - echo "::error::$FAILED benchmark run(s) failed" - exit 1 + EXTRA_ARGS=() + if [ -n "$CATEGORIES" ]; then + EXTRA_ARGS+=(--categories "$CATEGORIES") + fi + if [ -n "$TASKS" ]; then + EXTRA_ARGS+=(--tasks "$TASKS") + fi + if [ "$DRY_RUN" = "true" ]; then + EXTRA_ARGS+=(--dry-run) + fi + + if [ "$MODEL_SET" = "website_active" ]; then + llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}" + elif [ "$MODEL_SET" = "local_defaults" ]; then + llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}" + else + llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}" fi From 3483bd6e15af9712b658a043f9145e73740eb1a9 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:40:46 -0400 Subject: [PATCH 69/78] Keep parallel Rust benchmark builds isolated --- .github/workflows/llm-benchmark-periodic.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 44b57031088..587cfdafd42 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -145,7 +145,6 @@ jobs: LLM_BENCH_RUST_CONCURRENCY: "4" LLM_BENCH_CSHARP_CONCURRENCY: "2" LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }} - LLM_BENCH_RUST_TARGET_DIR: ${{ runner.temp }}/llm-benchmark-rust-target INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} From 30dbf5a15842bdeec0bd53e6335590be775cadc0 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:52:11 -0400 Subject: [PATCH 70/78] Refine TypeScript eval guidance and randomness scoring --- skills/typescript-server/SKILL.md | 2 ++ .../benchmarks/reducers/t_059_deterministic_context/spec.rs | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 9c7a44408c5..c9f2e7343fd 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -160,6 +160,8 @@ ctx.db.score_record.id.delete(recordId); // Delete by PK Insert through the table accessor (`ctx.db.score_record.insert(...)`). Primary-key, unique, and index accessors support lookup or mutation of existing rows, but do not have `insert(...)`. +`insert(...)` returns the inserted row, including database-assigned auto-increment fields. + The accessor for a primary key or index is the declared column name. For example, a primary key named `eventId` is accessed as `ctx.db.event.eventId`, not `ctx.db.event.id`. The schema value registers module exports but does not expose database rows. Pass a context into any helper that needs `ctx.db`. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs index fc1af0d45e2..beab260bc4a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs @@ -9,7 +9,6 @@ pub fn spec() -> BenchmarkSpec { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); let table = table_name("generated_value", lang); let casing = casing_for_lang(lang); - let random_value = ident("random_value", casing); scorers.push(make_reducer_sql_count_scorer( host_url, ReducerSqlCountConfig { @@ -17,9 +16,9 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "generate".into(), args: vec![], - sql_count_query: format!("SELECT COUNT(*) AS n FROM {table} WHERE {random_value}<>0"), + sql_count_query: format!("SELECT COUNT(*) AS n FROM {table}"), expected_count: 1, - id_str: "context_values_recorded", + id_str: "generated_value_recorded", timeout: Duration::from_secs(10), }, )); From 3571338d9dc64bd391cbb2356ef94656dcfa82e2 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 23 Jul 2026 12:32:13 -0400 Subject: [PATCH 71/78] updates --- .../src/bench/publishers.rs | 52 +++++++++++- .../auth/t_083_row_level_security/spec.rs | 3 +- .../t_057_nested_cascade_delete/spec.rs | 27 +++++-- .../t_059_deterministic_context/spec.rs | 26 +++++- .../t_067_view_primary_key/answers/csharp.cs | 20 ++++- .../t_067_view_primary_key/answers/rust.rs | 30 ++++++- .../answers/typescript.ts | 4 + .../views/t_067_view_primary_key/spec.rs | 26 +++++- .../t_067_view_primary_key/tasks/csharp.txt | 5 ++ .../t_067_view_primary_key/tasks/rust.txt | 5 ++ .../tasks/typescript.txt | 5 ++ .../xtask-llm-benchmark/src/eval/defaults.rs | 25 +++++- tools/xtask-llm-benchmark/src/eval/scorers.rs | 79 +++++++++++++++++++ tools/xtask-llm-benchmark/src/eval/utils.rs | 22 ++++-- 14 files changed, 301 insertions(+), 28 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 15047e71d70..e6e05bbefec 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -345,6 +345,34 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let diagnostic_free_rust_exit = (output.contains("process didn't exit successfully") && output.contains("--crate-name")) || output.contains("linking with `rust-lld.exe` failed: exit code: 1"); + let wrapper_command_exit = output.contains("error: command [\"cargo\", \"build\"") + || output.contains("error: command [\"dotnet\", \"publish\"") + || (output.contains("error: failed to run custom build command for") + && output.contains("process didn't exit successfully")); + let progress_only_exit = output.trim().is_empty() + || output.lines().filter(|line| !line.trim().is_empty()).all(|line| { + let line = line.trim_start(); + [ + "building ", + "compiling ", + "downloaded ", + "downloading ", + "finished ", + "logged in with identity ", + "optimizing ", + "packages: ", + "progress: ", + "recreating ", + "restored ", + "restoring ", + "saving config ", + "warning: this login will not work for any other servers.", + "we have logged in directly to your target server.", + ] + .iter() + .any(|prefix| line.starts_with(prefix)) + || line.chars().all(|character| matches!(character, '+' | '-')) + }); combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") @@ -353,7 +381,8 @@ fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { || (combined.contains("MSB3073") && combined.contains("exited with code 2")) || combined.contains("code Result<()> { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs index 34c450f9f7b..ec545ee4033 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs @@ -7,6 +7,7 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); let table = table_name("user_record", lang); + let identity = ident("identity", casing_for_lang(lang)); let name = ident("name", casing_for_lang(lang)); scorers.push(make_reducer_data_parity_scorer( host_url, @@ -15,7 +16,7 @@ pub fn spec() -> BenchmarkSpec { route_tag, reducer: "register_self".into(), args: vec![json!("caller")], - select_query: format!("SELECT {name} FROM {table}"), + select_query: format!("SELECT {identity}, {name} FROM {table}"), collapse_ws: true, timeout: Duration::from_secs(10), id_str: "caller_sees_own_row", diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs index a51402e4f3f..d45b4740450 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs @@ -1,7 +1,7 @@ use crate::eval::defaults::{ default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer, }; -use crate::eval::{table_name, BenchmarkSpec}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use serde_json::json; use std::time::Duration; @@ -24,19 +24,30 @@ pub fn spec() -> BenchmarkSpec { vec![json!(1)], "delete_workspace", )); - for (table, scorer_id) in [ - ("workspace", "workspace_count"), - ("project", "project_count"), - ("task_item", "task_count"), - ("task_note", "note_count"), + let id = ident("id", casing_for_lang(lang)); + for (table, count_id, preserved_id) in [ + ("workspace", "workspace_count", "workspace_two_preserved"), + ("project", "project_count", "project_two_preserved"), + ("task_item", "task_count", "task_two_preserved"), + ("task_note", "note_count", "note_two_preserved"), ] { + let table = table_name(table, lang); scorers.push(make_sql_count_only_scorer( host_url, file!(), route_tag, - format!("SELECT COUNT(*) AS n FROM {}", table_name(table, lang)), + format!("SELECT COUNT(*) AS n FROM {table}"), 1, - scorer_id, + count_id, + Duration::from_secs(10), + )); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=2"), + 1, + preserved_id, Duration::from_secs(10), )); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs index beab260bc4a..af961a7c99a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs @@ -1,5 +1,6 @@ use crate::eval::defaults::{ - default_schema_parity_scorers, make_reducer_sql_count_scorer, make_sql_output_excludes_scorer, + default_schema_parity_scorers, make_reducer_sql_count_scorer, make_sql_distinct_rows_scorer, + make_sql_output_excludes_scorer, }; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig}; use std::time::Duration; @@ -22,6 +23,29 @@ pub fn spec() -> BenchmarkSpec { timeout: Duration::from_secs(10), }, )); + scorers.push(make_reducer_sql_count_scorer( + host_url, + ReducerSqlCountConfig { + src_file: file!(), + route_tag, + reducer: "generate".into(), + args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {table}"), + expected_count: 2, + id_str: "second_generated_value_recorded", + timeout: Duration::from_secs(10), + }, + )); + let random_value = ident("random_value", casing); + scorers.push(make_sql_distinct_rows_scorer( + host_url, + file!(), + route_tag, + format!("SELECT {random_value} FROM {table}"), + 2, + "successive_random_values_differ", + Duration::from_secs(10), + )); let created_at = ident("created_at", casing); scorers.push(make_sql_output_excludes_scorer( host_url, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs index df432aa7bf9..a86a546b493 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs @@ -1,7 +1,23 @@ using SpacetimeDB; + public static partial class Module { - [Table(Accessor = "SourceRow", Public = true)] public partial struct SourceRow { [PrimaryKey] public ulong Id; public string Value; [SpacetimeDB.Index.BTree] public bool Visible; } + [Table(Accessor = "SourceRow", Public = true)] + public partial struct SourceRow + { + [PrimaryKey] public ulong Id; + public string Value; + [SpacetimeDB.Index.BTree] public bool Visible; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.SourceRow.Insert(new SourceRow { Id = 1, Value = "shown", Visible = true }); + ctx.Db.SourceRow.Insert(new SourceRow { Id = 2, Value = "hidden", Visible = false }); + } + [SpacetimeDB.View(Accessor = "SourceView", Public = true, PrimaryKey = nameof(SourceRow.Id))] - public static IEnumerable SourceView(AnonymousViewContext ctx) => ctx.Db.SourceRow.Visible.Filter(true); + public static IEnumerable SourceView(AnonymousViewContext ctx) => + ctx.Db.SourceRow.Visible.Filter(true); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs index 4604879a3e6..3115c51ddd2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs @@ -1,5 +1,29 @@ -use spacetimedb::{table, view, AnonymousViewContext}; +use spacetimedb::{table, view, AnonymousViewContext, ReducerContext, Table}; + #[table(accessor = source_row, public)] -pub struct SourceRow { #[primary_key] pub id: u64, pub value: String, #[index(btree)] pub visible: bool } +pub struct SourceRow { + #[primary_key] + pub id: u64, + pub value: String, + #[index(btree)] + pub visible: bool, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.source_row().insert(SourceRow { + id: 1, + value: "shown".into(), + visible: true, + }); + ctx.db.source_row().insert(SourceRow { + id: 2, + value: "hidden".into(), + visible: false, + }); +} + #[view(accessor = source_view, public, primary_key = id)] -pub fn source_view(ctx: &AnonymousViewContext) -> Vec { ctx.db.source_row().visible().filter(true).collect() } +pub fn source_view(ctx: &AnonymousViewContext) -> Vec { + ctx.db.source_row().visible().filter(true).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts index b1f3b99b417..7bfffd801fb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts @@ -2,4 +2,8 @@ import { schema, table, t } from 'spacetimedb/server'; const SourceViewRow = t.row('SourceViewRow', { id: t.u64().primaryKey(), value: t.string(), visible: t.bool().index('btree') }); const sourceRow = table({ name: 'source_row', public: true }, SourceViewRow); const spacetimedb = schema({ sourceRow }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { + ctx.db.sourceRow.insert({ id: 1n, value: 'shown', visible: true }); + ctx.db.sourceRow.insert({ id: 2n, value: 'hidden', visible: false }); +}); export const source_view = spacetimedb.anonymousView({ name: 'source_view', public: true }, t.array(SourceViewRow), ctx => Array.from(ctx.db.sourceRow.visible.filter(true))); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs index 8fc7c4dda44..778ba91fefb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs @@ -1,3 +1,23 @@ -use crate::eval::defaults::default_schema_parity_scorers; -use crate::eval::BenchmarkSpec; -pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| default_schema_parity_scorers(host_url, file!(), route_tag)) } +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {}", table_name("source_view", lang)), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "visible_rows_only", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt index ed875f18ed7..f0917addc3f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt @@ -7,6 +7,11 @@ TABLE - Value: string - Visible: bool (btree index) +REDUCERS +- Seed() + - Insert row 1 with Value "shown" and Visible true + - Insert row 2 with Value "hidden" and Visible false + VIEW - SourceView (anonymous procedural, public) - Return visible SourceRow rows diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt index ce4ec9e0fb5..a1e0797c3bd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt @@ -7,6 +7,11 @@ TABLE - value: String - visible: bool (btree index) +REDUCERS +- seed() + - Insert row 1 with value "shown" and visible true + - Insert row 2 with value "hidden" and visible false + VIEW - source_view (anonymous procedural, public) - Return visible source_row rows diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt index 357a397d061..b328e4ac382 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt @@ -7,6 +7,11 @@ TABLE - value: string - visible: boolean (btree index) +REDUCERS +- seed() + - Insert row 1 with value "shown" and visible true + - Insert row 2 with value "hidden" and visible false + VIEW - source_view (anonymous procedural, public) - Return visible source_row rows diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index 7652426d76e..f78fd6c1b5b 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -1,8 +1,8 @@ use crate::bench::utils::{golden_db_name, sanitize_db_name}; use crate::eval::scorers::{ CallOutputParityScorer, EventuallySqlCountScorer, HttpRouteCase, HttpRouteParityScorer, ReducerCallBothScorer, - ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, - SqlOutputExcludesScorer, + ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, + SqlDistinctRowsScorer, SqlExecBothScorer, SqlOutputExcludesScorer, }; use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSqlCountConfig}; use std::time::Duration; @@ -59,6 +59,27 @@ pub fn make_sql_count_only_scorer( }) } +pub fn make_sql_distinct_rows_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + sql: impl Into, + expected: usize, + id_str: &'static str, + timeout: Duration, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(SqlDistinctRowsScorer { + server: host_url.to_string(), + db: llm_db, + sql: sql.into(), + expected, + timeout, + id_str, + }) +} + pub fn make_eventually_sql_count_scorer( host_url: &str, src_file: &str, diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 4cd68bebc9c..a38e23b7308 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -632,6 +632,31 @@ fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Dura Ok(String::from_utf8_lossy(&stdout).to_string()) } +fn sql_json_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + let mut cmd = spacetime_command(); + cmd.arg("sql").arg(db).arg(query).args(["--format", "json"]); + if let Some(h) = host { + cmd.arg("--server").arg(h); + } + + let (code, stdout, stderr) = run_with_timeout(cmd, Path::new("."), timeout) + .map_err(|e| format!("spacetime sql failed or timed out: {e}"))?; + if code != 0 { + return Err(format!("spacetime sql failed:\n{}", String::from_utf8_lossy(&stderr))); + } + serde_json::from_slice(&stdout).map_err(|error| format!("invalid JSON SQL output: {error}")) +} + +fn distinct_sql_row_count(output: &Value) -> Result { + let rows = output + .as_array() + .and_then(|statements| statements.first()) + .and_then(|statement| statement.get("rows")) + .and_then(Value::as_array) + .ok_or_else(|| "JSON SQL output is missing rows".to_string())?; + Ok(rows.iter().map(Value::to_string).collect::>().len()) +} + pub fn sql_count(db: &str, query: &str, host: Option<&str>) -> Result { sql_count_with_timeout(db, query, host, Duration::from_secs(30)) } @@ -915,6 +940,15 @@ pub struct SqlCountOnlyScorer { pub id_str: &'static str, } +pub struct SqlDistinctRowsScorer { + pub server: String, + pub db: String, + pub sql: String, + pub expected: usize, + pub timeout: Duration, + pub id_str: &'static str, +} + pub struct SqlOutputExcludesScorer { pub server: String, pub db: String, @@ -994,6 +1028,40 @@ impl Scorer for SqlCountOnlyScorer { } } +impl Scorer for SqlDistinctRowsScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let output = match sql_json_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { + Ok(output) => output, + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "sql", "error": error }), + } + } + }; + match distinct_sql_row_count(&output) { + Ok(actual) => { + let pass = actual == self.expected; + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "sql": self.sql, "expected": self.expected, "actual": actual }), + } + } + Err(error) => ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "parse_sql", "error": error, "output": output }), + }, + } + } +} + impl Scorer for SqlOutputExcludesScorer { fn id(&self) -> &'static str { self.id_str @@ -1305,6 +1373,17 @@ mod tests { use spacetimedb_lib::sats::serde::SerdeWrapper; use spacetimedb_lib::sats::{AlgebraicType, ProductType}; + #[test] + fn counts_distinct_rows_in_json_sql_output() { + let output = json!([{ + "rows": [[1], [2], [1]], + "schema": { "elements": [] }, + "total_duration_micros": 1 + }]); + + assert_eq!(distinct_sql_row_count(&output).unwrap(), 2); + } + fn current_schema(include_owner_index: bool) -> Value { let mut indexes = vec![json!({ "algorithm": { "BTree": [0] } })]; if include_owner_index { diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 93d911e844a..c35b1fb2e99 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -108,8 +108,14 @@ pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) break status; } if start.elapsed() >= timeout { - kill_process_tree(&mut child); - return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); + let termination_error = kill_process_tree(&mut child).err(); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + let message = termination_error + .map(|error| format!("process timeout; failed to terminate process tree: {error}")) + .unwrap_or_else(|| "process timeout".to_string()); + return Err(io::Error::new(io::ErrorKind::TimedOut, message)); } thread::sleep(Duration::from_millis(30)); }; @@ -122,14 +128,14 @@ pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) Ok((status.code().unwrap_or(-1), stdout, stderr)) } -fn kill_process_tree(child: &mut Child) { +fn kill_process_tree(child: &mut Child) -> io::Result<()> { #[cfg(windows)] let killed = Command::new("taskkill") .args(["/F", "/T", "/PID", &child.id().to_string()]) .stdout(Stdio::null()) .stderr(Stdio::null()) - .spawn() - .is_ok(); + .status() + .is_ok_and(|status| status.success()); #[cfg(unix)] let killed = Command::new("kill") @@ -139,8 +145,10 @@ fn kill_process_tree(child: &mut Child) { .status() .is_ok_and(|status| status.success()); - if !killed { - let _ = child.kill(); + if killed { + Ok(()) + } else { + child.kill() } } From 5b46b1395b8de3d53635f3eeda55ecc181e73cbb Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:14:41 -0400 Subject: [PATCH 72/78] lints --- .../t_068_secure_projection/answers/rust.rs | 15 ++++- .../t_083_row_level_security/answers/rust.rs | 8 ++- .../answers/rust.rs | 21 ++++-- .../answers/rust.rs | 12 +++- .../t_071_scheduled_private/answers/rust.rs | 18 ++++- .../lifecycle/t_071_scheduled_private/spec.rs | 19 ++++-- .../t_080_automatic_migration/answers/rust.rs | 5 +- .../t_080_automatic_migration/setup/rust.rs | 5 +- .../answers/rust.rs | 16 ++++- .../t_081_incremental_migration/setup/rust.rs | 5 +- .../t_072_procedure_return/answers/rust.rs | 10 ++- .../procedures/t_072_procedure_return/spec.rs | 7 +- .../t_073_http_fetch/answers/rust.rs | 19 ++++-- .../t_074_fetch_and_store/answers/rust.rs | 15 ++++- .../procedures/t_074_fetch_and_store/spec.rs | 8 ++- .../t_075_scheduled_procedure/answers/rust.rs | 19 +++++- .../t_075_scheduled_procedure/spec.rs | 20 ++++-- .../t_076_http_handler/answers/rust.rs | 11 ++- .../procedures/t_076_http_handler/spec.rs | 7 +- .../t_077_http_router/answers/rust.rs | 13 +++- .../procedures/t_077_http_router/spec.rs | 4 +- .../t_078_idempotent_webhook/answers/rust.rs | 47 ++++++++++--- .../t_078_idempotent_webhook/spec.rs | 12 +++- .../answers/rust.rs | 19 ++++-- .../answers/rust.rs | 7 +- .../answers/rust.rs | 30 +++++++-- .../t_057_nested_cascade_delete/spec.rs | 4 +- .../t_058_batched_delete/answers/rust.rs | 17 +++-- .../t_053_default_values/answers/rust.rs | 6 +- .../tables/t_053_default_values/setup/rust.rs | 5 +- .../t_061_three_table_join/answers/rust.rs | 67 ++++++++++++++++--- .../answers/rust.rs | 29 ++++++-- .../t_064_index_and_filter/answers/rust.rs | 48 +++++++++++-- .../answers/rust.rs | 67 ++++++++++++++++--- .../t_065_materialized_aggregate/spec.rs | 18 +++-- .../t_066_query_builder_view/answers/rust.rs | 24 +++++-- 36 files changed, 533 insertions(+), 124 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs index b7c75d6b7a2..e1be4fa8325 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs @@ -11,7 +11,10 @@ pub struct SecretNote { } #[derive(SpacetimeType)] -pub struct SafeNote { pub id: u64, pub title: String } +pub struct SafeNote { + pub id: u64, + pub title: String, +} #[reducer] pub fn seed_private_note(ctx: &ReducerContext) { @@ -25,5 +28,13 @@ pub fn seed_private_note(ctx: &ReducerContext) { #[view(accessor = my_safe_note, public)] pub fn my_safe_note(ctx: &ViewContext) -> Vec { - ctx.db.secret_note().owner().filter(ctx.sender()).map(|note| SafeNote { id: note.id, title: note.title }).collect() + ctx.db + .secret_note() + .owner() + .filter(ctx.sender()) + .map(|note| SafeNote { + id: note.id, + title: note.title, + }) + .collect() } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs index 8ebfd7e9ff4..563b979cd17 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs @@ -8,10 +8,12 @@ pub struct UserRecord { } #[spacetimedb::client_visibility_filter] -const USER_RECORD_FILTER: Filter = - Filter::Sql("SELECT * FROM user_record WHERE identity = :sender"); +const USER_RECORD_FILTER: Filter = Filter::Sql("SELECT * FROM user_record WHERE identity = :sender"); #[spacetimedb::reducer] pub fn register_self(ctx: &ReducerContext, name: String) { - ctx.db.user_record().insert(UserRecord { identity: ctx.sender(), name }); + ctx.db.user_record().insert(UserRecord { + identity: ctx.sender(), + name, + }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs index 1ed33422aa7..78d44301106 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs @@ -21,9 +21,17 @@ pub struct RefreshJob { #[reducer] pub fn start_refresh(ctx: &ReducerContext) { - let pending = MaterializedState { id: 1, status: "pending".into(), version: 0, refreshed_at: Timestamp::UNIX_EPOCH }; - if ctx.db.materialized_state().id().find(1).is_some() { ctx.db.materialized_state().id().update(pending); } - else { ctx.db.materialized_state().insert(pending); } + let pending = MaterializedState { + id: 1, + status: "pending".into(), + version: 0, + refreshed_at: Timestamp::UNIX_EPOCH, + }; + if ctx.db.materialized_state().id().find(1).is_some() { + ctx.db.materialized_state().id().update(pending); + } else { + ctx.db.materialized_state().insert(pending); + } ctx.db.refresh_job().insert(RefreshJob { scheduled_id: 0, scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), @@ -33,7 +41,12 @@ pub fn start_refresh(ctx: &ReducerContext) { #[reducer] pub fn refresh_materialized(ctx: &ReducerContext, job: RefreshJob) { - let mut state = ctx.db.materialized_state().id().find(job.state_id).expect("materialized state missing"); + let mut state = ctx + .db + .materialized_state() + .id() + .find(job.state_id) + .expect("materialized state missing"); state.status = "ready".into(); state.version = 1; state.refreshed_at = ctx.timestamp; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs index 0bc4637029e..5121a6286d5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs @@ -11,7 +11,9 @@ pub struct PresenceSession { fn add_session(ctx: &ReducerContext, connection_id: ConnectionId) { ctx.db.presence_session().insert(PresenceSession { - connection_id, identity: ctx.sender(), connected_at: ctx.timestamp, + connection_id, + identity: ctx.sender(), + connected_at: ctx.timestamp, }); } @@ -20,10 +22,14 @@ fn remove_session(ctx: &ReducerContext, connection_id: ConnectionId) { } #[reducer(client_connected)] -pub fn client_connected(ctx: &ReducerContext) { add_session(ctx, ctx.connection_id().expect("connection id missing")); } +pub fn client_connected(ctx: &ReducerContext) { + add_session(ctx, ctx.connection_id().expect("connection id missing")); +} #[reducer(client_disconnected)] -pub fn client_disconnected(ctx: &ReducerContext) { remove_session(ctx, ctx.connection_id().expect("connection id missing")); } +pub fn client_disconnected(ctx: &ReducerContext) { + remove_session(ctx, ctx.connection_id().expect("connection id missing")); +} #[reducer] pub fn exercise_presence(ctx: &ReducerContext) { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs index 12b086f0b75..85fc0f9bcab 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs @@ -2,7 +2,11 @@ use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; #[table(accessor = job_result, public)] -pub struct JobResult { #[primary_key] pub id: u64, pub status: String } +pub struct JobResult { + #[primary_key] + pub id: u64, + pub status: String, +} #[table(accessor = private_job, scheduled(run_private_job))] pub struct PrivateJob { @@ -15,7 +19,10 @@ pub struct PrivateJob { #[reducer] pub fn enqueue_private(ctx: &ReducerContext, id: u64) { - ctx.db.job_result().insert(JobResult { id, status: "queued".into() }); + ctx.db.job_result().insert(JobResult { + id, + status: "queued".into(), + }); ctx.db.private_job().insert(PrivateJob { scheduled_id: 0, scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), @@ -25,7 +32,12 @@ pub fn enqueue_private(ctx: &ReducerContext, id: u64) { #[reducer] pub fn run_private_job(ctx: &ReducerContext, job: PrivateJob) { - let mut result = ctx.db.job_result().id().find(job.result_id).expect("job result missing"); + let mut result = ctx + .db + .job_result() + .id() + .find(job.result_id) + .expect("job result missing"); result.status = "complete".into(); ctx.db.job_result().id().update(result); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs index c6693feff71..06e6d8f0b2d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs @@ -1,4 +1,6 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, +}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use serde_json::json; use std::time::Duration; @@ -7,15 +9,24 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_reducer_call_both_scorer( - host_url, file!(), route_tag, "enqueue_private", vec![json!(7)], "enqueue_private", + host_url, + file!(), + route_tag, + "enqueue_private", + vec![json!(7)], + "enqueue_private", )); let table = table_name("job_result", lang); let id = ident("id", casing_for_lang(lang)); let status = ident("status", casing_for_lang(lang)); scorers.push(make_eventually_sql_count_scorer( - host_url, file!(), route_tag, + host_url, + file!(), + route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=7 AND {status}='complete'"), - 1, "private_scheduled_job_completes", Duration::from_secs(10), + 1, + "private_scheduled_job_completes", + Duration::from_secs(10), )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs index 375aece6f7c..4609b6f9bc9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs @@ -16,7 +16,10 @@ pub struct Category { #[spacetimedb::reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.product().insert(Product { id: 1, name: "legacy".into() }); + ctx.db.product().insert(Product { + id: 1, + name: "legacy".into(), + }); } #[spacetimedb::reducer] diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs index 862af19aa02..7eb1cc95792 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs @@ -9,7 +9,10 @@ pub struct Product { #[spacetimedb::reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.product().insert(Product { id: 1, name: "legacy".into() }); + ctx.db.product().insert(Product { + id: 1, + name: "legacy".into(), + }); } #[spacetimedb::reducer] diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs index 6a81e46129b..0ac6567d852 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs @@ -17,20 +17,30 @@ pub struct ItemV2 { #[spacetimedb::reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.legacy_item().insert(LegacyItem { id: 1, value: "old".into() }); + ctx.db.legacy_item().insert(LegacyItem { + id: 1, + value: "old".into(), + }); } #[spacetimedb::reducer] pub fn migrate(ctx: &ReducerContext) { for row in ctx.db.legacy_item().iter() { if ctx.db.item_v2().id().find(row.id).is_none() { - ctx.db.item_v2().insert(ItemV2 { id: row.id, value: row.value, version: 2 }); + ctx.db.item_v2().insert(ItemV2 { + id: row.id, + value: row.value, + version: 2, + }); } } } #[spacetimedb::reducer] pub fn dual_write(ctx: &ReducerContext, id: u64, value: String) { - ctx.db.legacy_item().insert(LegacyItem { id, value: value.clone() }); + ctx.db.legacy_item().insert(LegacyItem { + id, + value: value.clone(), + }); ctx.db.item_v2().insert(ItemV2 { id, value, version: 2 }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs index 182924f4c81..68e9f6f1db4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs @@ -9,5 +9,8 @@ pub struct LegacyItem { #[spacetimedb::reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.legacy_item().insert(LegacyItem { id: 1, value: "old".into() }); + ctx.db.legacy_item().insert(LegacyItem { + id: 1, + value: "old".into(), + }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs index 7942c99720c..298a0ef5802 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs @@ -1,9 +1,15 @@ use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; #[derive(SpacetimeType)] -pub struct Summary { pub total: u32, pub label: String } +pub struct Summary { + pub total: u32, + pub label: String, +} #[procedure] pub fn calculate_summary(_ctx: &mut ProcedureContext, lhs: u32, rhs: u32) -> Summary { - Summary { total: lhs + rhs, label: "calculated".into() } + Summary { + total: lhs + rhs, + label: "calculated".into(), + } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs index 87f2cdefd74..b0fa5340997 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs @@ -6,7 +6,12 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_call_output_parity_scorer( - host_url, file!(), route_tag, "calculate_summary", vec![json!(7), json!(5)], "typed_procedure_return", + host_url, + file!(), + route_tag, + "calculate_summary", + vec![json!(7), json!(5)], + "typed_procedure_return", )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs index 98585213d74..eeef78345b1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs @@ -1,14 +1,25 @@ use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; #[derive(SpacetimeType)] -pub struct FetchSummary { pub status: u16, pub html_content_type: bool, pub has_example_domain: bool } +pub struct FetchSummary { + pub status: u16, + pub html_content_type: bool, + pub has_example_domain: bool, +} #[procedure] pub fn fetch_page_summary(ctx: &mut ProcedureContext, url: String) -> FetchSummary { let response = ctx.http.get(url).expect("page request failed"); let status = response.status().as_u16(); - let html_content_type = response.headers().get("content-type") - .and_then(|value| value.to_str().ok()).is_some_and(|value| value.contains("text/html")); + let html_content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("text/html")); let body = response.into_body().into_string_lossy(); - FetchSummary { status, html_content_type, has_example_domain: body.contains("Example Domain") } + FetchSummary { + status, + html_content_type, + has_example_domain: body.contains("Example Domain"), + } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs index 561c93390c0..57aa7883f7b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs @@ -1,12 +1,23 @@ use spacetimedb::{procedure, table, ProcedureContext, Table}; #[table(accessor = fetched_record, public)] -pub struct FetchedRecord { #[primary_key] pub id: u64, pub status: u16, pub valid_body: bool } +pub struct FetchedRecord { + #[primary_key] + pub id: u64, + pub status: u16, + pub valid_body: bool, +} #[procedure] pub fn fetch_and_store(ctx: &mut ProcedureContext, url: String) { let response = ctx.http.get(url).expect("page request failed"); let status = response.status().as_u16(); let valid_body = response.into_body().into_string_lossy().contains("Example Domain"); - ctx.with_tx(|tx| { tx.db.fetched_record().insert(FetchedRecord { id: 1, status, valid_body }); }); + ctx.with_tx(|tx| { + tx.db.fetched_record().insert(FetchedRecord { + id: 1, + status, + valid_body, + }); + }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs index 95ad2472c33..148548fa342 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs @@ -21,9 +21,13 @@ pub fn spec() -> BenchmarkSpec { let status = ident("status", casing_for_lang(lang)); let valid = ident("valid_body", casing_for_lang(lang)); scorers.push(make_sql_count_only_scorer( - host_url, file!(), route_tag, + host_url, + file!(), + route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}=200 AND {valid}=true"), - 1, "fetched_row_stored", Duration::from_secs(10), + 1, + "fetched_row_stored", + Duration::from_secs(10), )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs index 7c9bff5e95a..fb3904b20eb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs @@ -2,7 +2,11 @@ use spacetimedb::{procedure, reducer, table, ProcedureContext, ReducerContext, S use std::time::Duration; #[table(accessor = procedure_result, public)] -pub struct ProcedureResult { #[primary_key] pub id: u64, pub value: u32 } +pub struct ProcedureResult { + #[primary_key] + pub id: u64, + pub value: u32, +} #[table(accessor = procedure_job, scheduled(run_scheduled_procedure))] pub struct ProcedureJob { @@ -18,11 +22,20 @@ pub struct ProcedureJob { #[reducer] pub fn schedule_procedure(ctx: &ReducerContext, id: u64, lhs: u32, rhs: u32) { ctx.db.procedure_job().insert(ProcedureJob { - scheduled_id: 0, scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), id, lhs, rhs, + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + id, + lhs, + rhs, }); } #[procedure] pub fn run_scheduled_procedure(ctx: &mut ProcedureContext, job: ProcedureJob) { - ctx.with_tx(|tx| { tx.db.procedure_result().insert(ProcedureResult { id: job.id, value: job.lhs + job.rhs }); }); + ctx.with_tx(|tx| { + tx.db.procedure_result().insert(ProcedureResult { + id: job.id, + value: job.lhs + job.rhs, + }); + }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs index 62ba0f29ca8..378c61676fd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs @@ -1,4 +1,6 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, +}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use serde_json::json; use std::time::Duration; @@ -7,14 +9,24 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_reducer_call_both_scorer( - host_url, file!(), route_tag, "schedule_procedure", vec![json!(9), json!(7), json!(5)], "schedule_procedure", + host_url, + file!(), + route_tag, + "schedule_procedure", + vec![json!(9), json!(7), json!(5)], + "schedule_procedure", )); let table = table_name("procedure_result", lang); let id = ident("id", casing_for_lang(lang)); let value = ident("value", casing_for_lang(lang)); scorers.push(make_eventually_sql_count_scorer( - host_url, file!(), route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=9 AND {value}=12"), - 1, "scheduled_procedure_completes", Duration::from_secs(10), + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=9 AND {value}=12"), + 1, + "scheduled_procedure_completes", + Duration::from_secs(10), )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs index 5d0f0f1952a..5114d5fd4b4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs @@ -3,9 +3,14 @@ use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response #[handler] fn echo(_ctx: &mut HandlerContext, request: Request) -> Response { let body = request.into_body().into_string_lossy(); - Response::builder().status(201).header("content-type", "text/plain") - .body(Body::from_bytes(format!("echo:{body}"))).unwrap() + Response::builder() + .status(201) + .header("content-type", "text/plain") + .body(Body::from_bytes(format!("echo:{body}"))) + .unwrap() } #[router] -fn routes() -> Router { Router::new().post("/echo", echo) } +fn routes() -> Router { + Router::new().post("/echo", echo) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs index f765e8d410a..5d3c96379dc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs @@ -5,7 +5,12 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_http_route_parity_scorer( - host_url, file!(), route_tag, vec![("POST", "/echo", Some("hello"))], true, "http_handler_response", + host_url, + file!(), + route_tag, + vec![("POST", "/echo", Some("hello"))], + true, + "http_handler_response", )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs index 50441c35fe5..83cabcf594a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs @@ -1,13 +1,20 @@ use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; #[handler] -fn list_items(_ctx: &mut HandlerContext, _request: Request) -> Response { Response::new(Body::from_bytes("list")) } +fn list_items(_ctx: &mut HandlerContext, _request: Request) -> Response { + Response::new(Body::from_bytes("list")) +} #[handler] fn create_item(_ctx: &mut HandlerContext, request: Request) -> Response { let body = request.into_body().into_string_lossy(); - Response::builder().status(201).body(Body::from_bytes(format!("created:{body}"))).unwrap() + Response::builder() + .status(201) + .body(Body::from_bytes(format!("created:{body}"))) + .unwrap() } #[router] -fn routes() -> Router { Router::new().get("/items", list_items).post("/items", create_item) } +fn routes() -> Router { + Router::new().get("/items", list_items).post("/items", create_item) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs index 3b4b74aff41..740c6cc7419 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs @@ -5,7 +5,9 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_http_route_parity_scorer( - host_url, file!(), route_tag, + host_url, + file!(), + route_tag, vec![ ("GET", "/items", None), ("POST", "/items", Some("book")), diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs index 0d4921ba5d2..8be8e888c2b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs @@ -1,29 +1,54 @@ -use spacetimedb::{table, Table}; use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; +use spacetimedb::{table, Table}; #[table(accessor = processed_event)] -pub struct ProcessedEvent { #[primary_key] pub event_id: String } +pub struct ProcessedEvent { + #[primary_key] + pub event_id: String, +} #[table(accessor = webhook_state, public)] -pub struct WebhookState { #[primary_key] pub key: String, pub last_sequence: u64, pub value: String } +pub struct WebhookState { + #[primary_key] + pub key: String, + pub last_sequence: u64, + pub value: String, +} #[handler] fn webhook(ctx: &mut HandlerContext, request: Request) -> Response { let body = request.into_body().into_string_lossy(); let parts: Vec<_> = body.splitn(3, '|').collect(); - if parts.len() != 3 { return Response::builder().status(400).body(Body::from_bytes("invalid")).unwrap(); } + if parts.len() != 3 { + return Response::builder() + .status(400) + .body(Body::from_bytes("invalid")) + .unwrap(); + } let event_id = parts[0].to_string(); let sequence: u64 = parts[1].parse().expect("invalid sequence"); let value = parts[2].to_string(); let outcome = ctx.with_tx(|tx| { - if tx.db.processed_event().event_id().find(&event_id).is_some() { return "duplicate"; } - tx.db.processed_event().insert(ProcessedEvent { event_id: event_id.clone() }); + if tx.db.processed_event().event_id().find(&event_id).is_some() { + return "duplicate"; + } + tx.db.processed_event().insert(ProcessedEvent { + event_id: event_id.clone(), + }); let key = "account".to_string(); if let Some(mut state) = tx.db.webhook_state().key().find(&key) { - if sequence <= state.last_sequence { return "stale"; } - state.last_sequence = sequence; state.value = value.clone(); tx.db.webhook_state().key().update(state); + if sequence <= state.last_sequence { + return "stale"; + } + state.last_sequence = sequence; + state.value = value.clone(); + tx.db.webhook_state().key().update(state); } else { - tx.db.webhook_state().insert(WebhookState { key, last_sequence: sequence, value: value.clone() }); + tx.db.webhook_state().insert(WebhookState { + key, + last_sequence: sequence, + value: value.clone(), + }); } "applied" }); @@ -31,4 +56,6 @@ fn webhook(ctx: &mut HandlerContext, request: Request) -> Response { } #[router] -fn routes() -> Router { Router::new().post("/webhook", webhook) } +fn routes() -> Router { + Router::new().post("/webhook", webhook) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs index 794697a54c2..178befb0192 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs @@ -6,7 +6,9 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_http_route_parity_scorer( - host_url, file!(), route_tag, + host_url, + file!(), + route_tag, vec![ ("POST", "/webhook", Some("evt-1|2|new")), ("POST", "/webhook", Some("evt-1|2|new")), @@ -20,9 +22,13 @@ pub fn spec() -> BenchmarkSpec { let sequence = ident("last_sequence", casing_for_lang(lang)); let value = ident("value", casing_for_lang(lang)); scorers.push(make_sql_count_only_scorer( - host_url, file!(), route_tag, + host_url, + file!(), + route_tag, format!("SELECT COUNT(*) AS n FROM {table} WHERE {key}='account' AND {sequence}=2 AND {value}='new'"), - 1, "webhook_state_is_current", Duration::from_secs(10), + 1, + "webhook_state_is_current", + Duration::from_secs(10), )); scorers }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs index 31bcef63db3..6e389a9df60 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs @@ -1,5 +1,5 @@ -use spacetimedb::{procedure, table, ProcedureContext, Table}; use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; +use spacetimedb::{procedure, table, ProcedureContext, Table}; #[table(accessor = uploaded_asset, public)] pub struct UploadedAsset { @@ -13,16 +13,25 @@ pub struct UploadedAsset { #[handler] fn upload(_ctx: &mut HandlerContext, _request: Request) -> Response { - Response::builder().status(201).body(Body::from_bytes("https://files.local/object-1")).unwrap() + Response::builder() + .status(201) + .body(Body::from_bytes("https://files.local/object-1")) + .unwrap() } #[router] -fn routes() -> Router { Router::new().post("/upload", upload) } +fn routes() -> Router { + Router::new().post("/upload", upload) +} #[procedure] pub fn upload_and_register(ctx: &mut ProcedureContext, upload_url: String, data: Vec) -> String { - let request = Request::builder().method("POST").uri(upload_url.clone()).header("content-type", "application/octet-stream") - .body(Body::from_bytes(data.clone())).unwrap(); + let request = Request::builder() + .method("POST") + .uri(upload_url.clone()) + .header("content-type", "application/octet-stream") + .body(Body::from_bytes(data.clone())) + .unwrap(); let response = ctx.http.send(request).expect("upload failed"); assert!(response.status().is_success(), "upload failed: {}", response.status()); let status = response.status().as_u16(); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs index 8054a55f80c..71d0741fe2d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs @@ -30,7 +30,12 @@ pub fn transfer(ctx: &ReducerContext, request_id: String, from_id: u64, to_id: u return Err("invalid transfer".into()); } let mut from = ctx.db.account().id().find(from_id).ok_or("source account not found")?; - let mut to = ctx.db.account().id().find(to_id).ok_or("destination account not found")?; + let mut to = ctx + .db + .account() + .id() + .find(to_id) + .ok_or("destination account not found")?; if from.balance < amount { return Err("insufficient balance".into()); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs index 685d18baecc..910f00814d6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs @@ -1,13 +1,31 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = workspace, public)] -pub struct Workspace { #[primary_key] pub id: u64 } +pub struct Workspace { + #[primary_key] + pub id: u64, +} #[table(accessor = project, public)] -pub struct Project { #[primary_key] pub id: u64, #[index(btree)] pub workspace_id: u64 } +pub struct Project { + #[primary_key] + pub id: u64, + #[index(btree)] + pub workspace_id: u64, +} #[table(accessor = task_item, public)] -pub struct TaskItem { #[primary_key] pub id: u64, #[index(btree)] pub project_id: u64 } +pub struct TaskItem { + #[primary_key] + pub id: u64, + #[index(btree)] + pub project_id: u64, +} #[table(accessor = task_note, public)] -pub struct TaskNote { #[primary_key] pub id: u64, #[index(btree)] pub task_id: u64 } +pub struct TaskNote { + #[primary_key] + pub id: u64, + #[index(btree)] + pub task_id: u64, +} #[reducer] pub fn seed(ctx: &ReducerContext) { @@ -26,7 +44,9 @@ pub fn delete_workspace(ctx: &ReducerContext, id: u64) { let tasks: Vec<_> = ctx.db.task_item().project_id().filter(project.id).collect(); for task in tasks { let notes: Vec<_> = ctx.db.task_note().task_id().filter(task.id).collect(); - for note in notes { ctx.db.task_note().id().delete(note.id); } + for note in notes { + ctx.db.task_note().id().delete(note.id); + } ctx.db.task_item().id().delete(task.id); } ctx.db.project().id().delete(project.id); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs index d45b4740450..03ab60fe4ad 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs @@ -1,6 +1,4 @@ -use crate::eval::defaults::{ - default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer, -}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; use serde_json::json; use std::time::Duration; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs index 40f21e21d96..d4418cc5539 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs @@ -29,16 +29,25 @@ fn enqueue(ctx: &ReducerContext, group_id: u64) { #[reducer] pub fn seed_group(ctx: &ReducerContext, group_id: u64, count: u32) { for offset in 0..count { - ctx.db.work_item().insert(WorkItem { id: group_id * 100 + offset as u64, group_id }); + ctx.db.work_item().insert(WorkItem { + id: group_id * 100 + offset as u64, + group_id, + }); } } #[reducer] -pub fn request_delete(ctx: &ReducerContext, group_id: u64) { enqueue(ctx, group_id); } +pub fn request_delete(ctx: &ReducerContext, group_id: u64) { + enqueue(ctx, group_id); +} #[reducer] pub fn run_delete_batch(ctx: &ReducerContext, job: DeleteJob) { let rows: Vec<_> = ctx.db.work_item().group_id().filter(job.group_id).take(2).collect(); - for row in rows { ctx.db.work_item().id().delete(row.id); } - if ctx.db.work_item().group_id().filter(job.group_id).next().is_some() { enqueue(ctx, job.group_id); } + for row in rows { + ctx.db.work_item().id().delete(row.id); + } + if ctx.db.work_item().group_id().filter(job.group_id).next().is_some() { + enqueue(ctx, job.group_id); + } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs index 2e77e2393c0..76963b7a165 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs @@ -11,7 +11,11 @@ pub struct Widget { #[spacetimedb::reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.widget().insert(Widget { id: 1, name: "legacy".into(), enabled: true }); + ctx.db.widget().insert(Widget { + id: 1, + name: "legacy".into(), + enabled: true, + }); } #[spacetimedb::reducer] diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs index ff3b23c6283..221f8362303 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs @@ -9,7 +9,10 @@ pub struct Widget { #[spacetimedb::reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.widget().insert(Widget { id: 1, name: "legacy".into() }); + ctx.db.widget().insert(Widget { + id: 1, + name: "legacy".into(), + }); } #[spacetimedb::reducer] diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs index dc0ee315cd2..a8760d3d8eb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs @@ -1,14 +1,61 @@ use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, SpacetimeType, Table}; -#[table(accessor = customer, public)] pub struct Customer { #[primary_key] pub id: u64, pub name: String } -#[table(accessor = purchase, public)] pub struct Purchase { #[primary_key] pub id: u64, #[index(btree)] pub customer_id: u64 } -#[table(accessor = line_item, public)] pub struct LineItem { #[primary_key] pub id: u64, #[index(btree)] pub purchase_id: u64, pub sku: String, #[index(btree)] pub visible: bool } -#[derive(SpacetimeType)] pub struct OrderLineDetail { pub line_id: u64, pub customer_name: String, pub sku: String } -#[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.customer().insert(Customer { id: 1, name: "Ada".into() }); ctx.db.purchase().insert(Purchase { id: 10, customer_id: 1 }); ctx.db.line_item().insert(LineItem { id: 100, purchase_id: 10, sku: "SKU-1".into(), visible: true }); } +#[table(accessor = customer, public)] +pub struct Customer { + #[primary_key] + pub id: u64, + pub name: String, +} +#[table(accessor = purchase, public)] +pub struct Purchase { + #[primary_key] + pub id: u64, + #[index(btree)] + pub customer_id: u64, +} +#[table(accessor = line_item, public)] +pub struct LineItem { + #[primary_key] + pub id: u64, + #[index(btree)] + pub purchase_id: u64, + pub sku: String, + #[index(btree)] + pub visible: bool, +} +#[derive(SpacetimeType)] +pub struct OrderLineDetail { + pub line_id: u64, + pub customer_name: String, + pub sku: String, +} +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.customer().insert(Customer { + id: 1, + name: "Ada".into(), + }); + ctx.db.purchase().insert(Purchase { id: 10, customer_id: 1 }); + ctx.db.line_item().insert(LineItem { + id: 100, + purchase_id: 10, + sku: "SKU-1".into(), + visible: true, + }); +} #[view(accessor = order_line_detail, public)] pub fn order_line_detail(ctx: &AnonymousViewContext) -> Vec { - ctx.db.line_item().visible().filter(true).filter_map(|line| { - let purchase = ctx.db.purchase().id().find(line.purchase_id)?; - let customer = ctx.db.customer().id().find(purchase.customer_id)?; - Some(OrderLineDetail { line_id: line.id, customer_name: customer.name, sku: line.sku }) - }).collect() + ctx.db + .line_item() + .visible() + .filter(true) + .filter_map(|line| { + let purchase = ctx.db.purchase().id().find(line.purchase_id)?; + let customer = ctx.db.customer().id().find(purchase.customer_id)?; + Some(OrderLineDetail { + line_id: line.id, + customer_name: customer.name, + sku: line.sku, + }) + }) + .collect() } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs index 601a25f1070..b0324b642d5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs @@ -1,18 +1,37 @@ use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; #[table(accessor = member, public)] -pub struct Member { #[primary_key] pub id: u64, pub name: String } +pub struct Member { + #[primary_key] + pub id: u64, + pub name: String, +} #[table(accessor = eligibility, public)] -pub struct Eligibility { #[primary_key] pub id: u64, #[index(btree)] pub member_id: u64 } +pub struct Eligibility { + #[primary_key] + pub id: u64, + #[index(btree)] + pub member_id: u64, +} #[reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.member().insert(Member { id: 1, name: "Ada".into() }); - ctx.db.member().insert(Member { id: 2, name: "Grace".into() }); + ctx.db.member().insert(Member { + id: 1, + name: "Ada".into(), + }); + ctx.db.member().insert(Member { + id: 2, + name: "Grace".into(), + }); ctx.db.eligibility().insert(Eligibility { id: 10, member_id: 1 }); } #[view(accessor = eligible_member, public)] pub fn eligible_member(ctx: &ViewContext) -> impl Query { - ctx.from.eligibility().right_semijoin(ctx.from.member(), |eligibility, member| eligibility.member_id.eq(member.id)) + ctx.from + .eligibility() + .right_semijoin(ctx.from.member(), |eligibility, member| { + eligibility.member_id.eq(member.id) + }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs index 853331e9fb9..f4337a01fe7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs @@ -1,16 +1,50 @@ use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, Table}; #[table(accessor = content, public)] -pub struct Content { #[primary_key] pub id: u64, #[index(btree)] pub category: String, pub active: bool, pub score: i32 } +pub struct Content { + #[primary_key] + pub id: u64, + #[index(btree)] + pub category: String, + pub active: bool, + pub score: i32, +} #[reducer] pub fn seed(ctx: &ReducerContext) { for row in [ - Content { id: 1, category: "news".into(), active: true, score: 20 }, - Content { id: 2, category: "news".into(), active: false, score: 20 }, - Content { id: 3, category: "news".into(), active: true, score: 5 }, - Content { id: 4, category: "sports".into(), active: true, score: 20 }, - ] { ctx.db.content().insert(row); } + Content { + id: 1, + category: "news".into(), + active: true, + score: 20, + }, + Content { + id: 2, + category: "news".into(), + active: false, + score: 20, + }, + Content { + id: 3, + category: "news".into(), + active: true, + score: 5, + }, + Content { + id: 4, + category: "sports".into(), + active: true, + score: 20, + }, + ] { + ctx.db.content().insert(row); + } } #[view(accessor = featured_content, public)] pub fn featured_content(ctx: &AnonymousViewContext) -> Vec { - ctx.db.content().category().filter(&"news".to_string()).filter(|row| row.active && row.score >= 10).collect() + ctx.db + .content() + .category() + .filter(&"news".to_string()) + .filter(|row| row.active && row.score >= 10) + .collect() } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs index e01c0e461d7..8e18db470bf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs @@ -1,10 +1,20 @@ use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; #[table(accessor = sale, public)] -pub struct Sale { #[primary_key] pub id: u64, pub category: String, pub amount: i64 } +pub struct Sale { + #[primary_key] + pub id: u64, + pub category: String, + pub amount: i64, +} #[table(accessor = category_total, public)] -pub struct CategoryTotal { #[primary_key] pub category: String, pub total_amount: i64, pub sale_count: u64 } +pub struct CategoryTotal { + #[primary_key] + pub category: String, + pub total_amount: i64, + pub sale_count: u64, +} fn add_to_total(ctx: &ReducerContext, category: &String, amount: i64) { if let Some(mut total) = ctx.db.category_total().category().find(category) { @@ -12,12 +22,21 @@ fn add_to_total(ctx: &ReducerContext, category: &String, amount: i64) { total.sale_count += 1; ctx.db.category_total().category().update(total); } else { - ctx.db.category_total().insert(CategoryTotal { category: category.clone(), total_amount: amount, sale_count: 1 }); + ctx.db.category_total().insert(CategoryTotal { + category: category.clone(), + total_amount: amount, + sale_count: 1, + }); } } fn remove_from_total(ctx: &ReducerContext, category: &String, amount: i64) { - let mut total = ctx.db.category_total().category().find(category).expect("missing category total"); + let mut total = ctx + .db + .category_total() + .category() + .find(category) + .expect("missing category total"); if total.sale_count == 1 { ctx.db.category_total().category().delete(category); } else { @@ -50,13 +69,43 @@ fn delete_sale(ctx: &ReducerContext, id: u64) { #[reducer] pub fn exercise(ctx: &ReducerContext) { - upsert_sale(ctx, Sale { id: 1, category: "books".into(), amount: 10 }); - upsert_sale(ctx, Sale { id: 2, category: "books".into(), amount: 20 }); - upsert_sale(ctx, Sale { id: 2, category: "books".into(), amount: 25 }); - upsert_sale(ctx, Sale { id: 3, category: "games".into(), amount: 40 }); + upsert_sale( + ctx, + Sale { + id: 1, + category: "books".into(), + amount: 10, + }, + ); + upsert_sale( + ctx, + Sale { + id: 2, + category: "books".into(), + amount: 20, + }, + ); + upsert_sale( + ctx, + Sale { + id: 2, + category: "books".into(), + amount: 25, + }, + ); + upsert_sale( + ctx, + Sale { + id: 3, + category: "games".into(), + amount: 40, + }, + ); delete_sale(ctx, 3); delete_sale(ctx, 1); } #[view(accessor = category_summary, public)] -pub fn category_summary(ctx: &ViewContext) -> impl Query { ctx.from.category_total() } +pub fn category_summary(ctx: &ViewContext) -> impl Query { + ctx.from.category_total() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs index 8202ff0bc15..4939ac6682d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs @@ -7,11 +7,19 @@ pub fn spec() -> BenchmarkSpec { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); let sql = SqlBuilder::new(casing_for_lang(lang)); let columns = sql.cols(&["category", "total_amount", "sale_count"]).join(", "); - scorers.push(make_reducer_data_parity_scorer(host_url, ReducerDataParityConfig { - src_file: file!(), route_tag, reducer: "exercise".into(), args: vec![], - select_query: format!("SELECT {columns} FROM {}", table_name("category_summary", lang)), - id_str: "aggregate_is_synchronized", collapse_ws: true, timeout: Duration::from_secs(10), - })); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "exercise".into(), + args: vec![], + select_query: format!("SELECT {columns} FROM {}", table_name("category_summary", lang)), + id_str: "aggregate_is_synchronized", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs index 030479f11af..d607333d84d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs @@ -1,10 +1,26 @@ use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; #[table(accessor = ticket, public)] -pub struct Ticket { #[primary_key] pub id: u64, #[index(btree)] pub status: String, pub title: String } +pub struct Ticket { + #[primary_key] + pub id: u64, + #[index(btree)] + pub status: String, + pub title: String, +} #[reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.ticket().insert(Ticket { id: 1, status: "open".into(), title: "A".into() }); - ctx.db.ticket().insert(Ticket { id: 2, status: "closed".into(), title: "B".into() }); + ctx.db.ticket().insert(Ticket { + id: 1, + status: "open".into(), + title: "A".into(), + }); + ctx.db.ticket().insert(Ticket { + id: 2, + status: "closed".into(), + title: "B".into(), + }); } #[view(accessor = open_ticket, public)] -pub fn open_ticket(ctx: &ViewContext) -> impl Query { ctx.from.ticket().filter(|ticket| ticket.status.eq("open".to_string())) } +pub fn open_ticket(ctx: &ViewContext) -> impl Query { + ctx.from.ticket().filter(|ticket| ticket.status.eq("open".to_string())) +} From 0db8781c2582b27072a76368a110646285fac8a4 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:25:32 -0400 Subject: [PATCH 73/78] analysis + post to discord --- .github/workflows/llm-benchmark-periodic.yml | 118 +++++++++ .../xtask-llm-benchmark/src/bench/analysis.rs | 243 +++++++++++++++++- tools/xtask-llm-benchmark/src/bench/runner.rs | 45 +++- 3 files changed, 381 insertions(+), 25 deletions(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 587cfdafd42..3d18ce9eeb3 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -39,6 +39,11 @@ on: description: 'Run benchmarks without uploading results' required: false default: 'false' + post_discord: + description: 'Post the analysis summary to Discord' + required: false + type: boolean + default: false permissions: contents: read @@ -145,6 +150,7 @@ jobs: LLM_BENCH_RUST_CONCURRENCY: "4" LLM_BENCH_CSHARP_CONCURRENCY: "2" LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }} + LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} @@ -205,3 +211,115 @@ jobs: else llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}" fi + + - name: Upload analysis reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: llm-benchmark-analysis-${{ matrix.lang }} + path: ${{ runner.temp }}/llm-benchmark-reports + if-no-files-found: ignore + retention-days: 14 + + summarize: + name: Summarize benchmark analysis + if: always() + needs: run-benchmarks + runs-on: ubuntu-latest + + steps: + - name: Download analysis reports + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: llm-benchmark-analysis-* + path: reports + merge-multiple: true + + - name: Publish workflow summary + run: | + echo "# LLM benchmark analysis" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + + if [ ! -d reports ] || ! find reports -type f -name '*.md' -print -quit | grep -q .; then + echo "No analysis reports were produced." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "Full per-model reports are available in the workflow artifacts." >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + + find reports -type f -name '*.md' -print | sort | while IFS= read -r report; do + awk '/^## Failure patterns/{exit} NR <= 80 {print}' "$report" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "---" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + done + + - name: Prepare Discord summary + if: github.event_name == 'schedule' || inputs.post_discord + env: + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + python3 - <<'PY' + import os + import re + from pathlib import Path + + reports = sorted(Path("reports").rglob("*.md")) + rates = [] + actions = [] + + for report in reports: + text = report.read_text(encoding="utf-8") + + def field(name): + match = re.search(rf"^- {name}: (.+)$", text, re.MULTILINE) + return match.group(1) if match else "unknown" + + rates.append( + f"• {field('Language')} / {field('Mode')} / {field('Model')}: " + f"{field('Tasks')} tasks, {field('Scorers')} scorers" + ) + + recommended = text.partition("## Recommended actions")[2].partition("## Failure patterns")[0] + for line in recommended.splitlines(): + if line.startswith("- **[") and line not in actions: + actions.append(line.replace("- ", "• ", 1)) + + lines = [ + "**LLM benchmark analysis**", + f"[View workflow run]({os.environ['RUN_URL']})", + "", + "**Pass rates**", + *(rates[:8] or ["• No analysis reports were produced."]), + ] + if len(rates) > 8: + lines.append(f"• …and {len(rates) - 8} more report(s)") + + lines.extend(["", "**Recommended actions**", *(actions[:5] or ["• None"])]) + if len(actions) > 5: + lines.append(f"• …and {len(actions) - 5} more action(s)") + + message = "\n".join(lines) + if len(message) > 1900: + message = message[:1860].rstrip() + "\n… See the workflow for the full report." + Path("discord-summary.txt").write_text(message, encoding="utf-8") + PY + + - name: Post Discord summary + if: github.event_name == 'schedule' || inputs.post_discord + continue-on-error: true + env: + DISCORD_WEBHOOK_URL: ${{ secrets.LLM_BENCHMARK_DISCORD_WEBHOOK_URL }} + run: | + if [ -z "$DISCORD_WEBHOOK_URL" ]; then + echo "::warning::LLM_BENCHMARK_DISCORD_WEBHOOK_URL is not configured" + exit 0 + fi + + payload="$(jq -n --rawfile content discord-summary.txt '{content: $content}')" + curl --fail --silent --show-error \ + --header 'Content-Type: application/json' \ + --data "$payload" \ + "$DISCORD_WEBHOOK_URL" diff --git a/tools/xtask-llm-benchmark/src/bench/analysis.rs b/tools/xtask-llm-benchmark/src/bench/analysis.rs index cb23fbb6cf5..177d85cb815 100644 --- a/tools/xtask-llm-benchmark/src/bench/analysis.rs +++ b/tools/xtask-llm-benchmark/src/bench/analysis.rs @@ -7,11 +7,14 @@ use anyhow::Result; use spacetimedb_data_structures::map::HashMap; use std::path::Path; +const MAX_CONTEXT_CHARS: usize = 20_000; + pub async fn run_analysis( outcomes: &[RunOutcome], lang: &str, mode: &str, model_name: &str, + context: &str, bench_root: &Path, llm: &dyn LlmProvider, ) -> Result> { @@ -24,7 +27,7 @@ pub async fn run_analysis( return Ok(None); } - let prompt = build_prompt(lang, mode, model_name, bench_root, &failures); + let prompt = build_prompt(lang, mode, model_name, context, bench_root, &failures); let route = ModelRoute::new( "gpt-5.4-mini", @@ -49,9 +52,10 @@ pub fn system_prompt() -> String { } pub const SYSTEM_PROMPT: &str = "\ -You summarize LLM benchmark failures for SpacetimeDB into structured markdown. \ +You turn LLM benchmark failures for SpacetimeDB into evidence-based, structured markdown. \ Each failure includes the model's generated code, the scorer error, and the golden (correct) answer when available. \ -Write in third person for a public benchmark page. Do not address the reader."; +Write in third person for a public benchmark page. Do not address the reader. \ +Recommend repository changes only when the supplied evidence supports them."; fn context_description(mode: &str) -> &'static str { match mode { @@ -103,7 +107,14 @@ fn read_golden(bench_root: &Path, task_id: &str, lang: &str) -> Option { None } -pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, failures: &[&RunOutcome]) -> String { +pub fn build_prompt( + lang: &str, + mode: &str, + model_name: &str, + context: &str, + bench_root: &Path, + failures: &[&RunOutcome], +) -> String { let lang_display = match lang { "rust" => "Rust", "csharp" => "C#", @@ -118,6 +129,16 @@ pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, count = failures.len(), ); + let context_is_complete = context.chars().count() <= MAX_CONTEXT_CHARS; + if has_context(mode) { + prompt.push_str("### Context supplied to the model\n\n"); + prompt.push_str(&format!("```\n{}\n```\n", truncate(context, MAX_CONTEXT_CHARS))); + if !context_is_complete { + prompt.push_str("The context excerpt was truncated. Treat context-gap conclusions as low confidence.\n"); + } + prompt.push('\n'); + } + for f in failures.iter().take(15) { prompt.push_str(&format!("### {} ({}/{})\n", f.task, f.passed_tests, f.total_tests)); @@ -141,40 +162,133 @@ pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, prompt.push_str(&format!("({} more failures not shown)\n\n", failures.len() - 15)); } - prompt.push_str(&analysis_instructions(mode)); + prompt.push_str(&analysis_instructions_with_context(mode, context_is_complete)); prompt } pub fn analysis_instructions(mode: &str) -> String { - let fix_line = if has_context(mode) { + analysis_instructions_with_context(mode, false) +} + +fn analysis_instructions_with_context(mode: &str, context_is_complete: bool) -> String { + let context_gap_line = if has_context(mode) { let name = context_name(mode); - format!("5. **{name} gap:** What's missing or unclear in the {name} that led to this mistake\n") + if context_is_complete { + format!("- **{name} gap:** What is missing or unclear in the {name}, or `None`\n") + } else { + format!("- **{name} gap:** `Needs manual review`; the complete {name} was not supplied\n") + } } else { String::new() }; + let context_rule = if has_context(mode) && !context_is_complete { + "- Because the complete context was not supplied, do not classify a group as `Skill problem` or `Documentation problem`.\n" + } else { + "" + }; format!( "\ --- -Group failures by root cause pattern. Use this exact structure for each group: +Begin with this section: + +## Recommended actions + +List at most five distinct, repository-owned actions supported by the evidence. Use: + +- **[Classification | Confidence] Short title** — Plain-language action. Evidence: task IDs. + +If the failures are isolated model mistakes, provider failures, or otherwise do not justify a repository change, write: +`No repository changes recommended.` + +Then group failures by root cause using this exact structure: + +## Failure patterns ### [Pattern Name] (N tasks) -1. **What the model wrote:** Show the relevant incorrect lines from the generated code -2. **What was expected:** Show the relevant lines from the golden answer -3. **What the error says:** Quote the scorer error that identifies the problem -4. **Why this happened:** Why the model likely made this mistake (e.g. confused with another framework, hallucinated API, singular vs plural naming) -5. **Affected tasks:** list of task IDs -{fix_line} +- **Classification:** One of `Eval problem`, `Skill problem`, `Documentation problem`, `API/ergonomics problem`, `Model limitation`, `Infrastructure/provider problem`, or `No action` +- **Confidence:** `High`, `Medium`, or `Low` +- **What the model wrote:** Relevant incorrect lines from the generated code +- **What was expected:** Relevant lines from the golden answer +- **What the error says:** The scorer error that identifies the problem +- **Why this happened:** The likely root cause +- **Suggested action:** A plain-language repository change, or `None` +- **Suggested area:** The likely repository area, or `None` +- **Affected tasks:** Task IDs +{context_gap_line} Rules: - Group tasks that fail for the same reason. Do not repeat the same analysis per task. - Show only the relevant lines, not entire files. - Skip provider errors (timeouts, 429s) with a brief note. +- Do not recommend changing an eval, skill, documentation, or API merely because one model made an isolated mistake. +- Do not invent evidence, repository paths, or implementation details that were not supplied. +- Prefer `Model limitation`, `Infrastructure/provider problem`, or `No action` when the evidence does not support a repository change. +- Classify a context gap as `Skill problem` or `Documentation problem` only when the supplied context directly supports that conclusion. +- `Eval problem`, `Skill problem`, `Documentation problem`, and `API/ergonomics problem` require a non-`None` suggested action and a matching entry under `Recommended actions`. +- When `Suggested action` is `None`, use `Model limitation`, `Infrastructure/provider problem`, or `No action`. +- Write `No repository changes recommended.` only when no failure group recommends a repository change. +{context_rule}\ " ) } +pub fn build_report( + outcomes: &[RunOutcome], + lang: &str, + mode: &str, + model_name: &str, + analysis: Option<&str>, +) -> String { + let passed_tasks = outcomes + .iter() + .filter(|outcome| outcome.total_tests > 0 && outcome.passed_tests == outcome.total_tests) + .count(); + let total_tasks = outcomes.len(); + let passed_scorers: u32 = outcomes.iter().map(|outcome| outcome.passed_tests).sum(); + let total_scorers: u32 = outcomes.iter().map(|outcome| outcome.total_tests).sum(); + + let mut report = format!( + "\ +# LLM Benchmark Analysis + +- Language: {lang} +- Mode: {mode} +- Model: {model_name} +- Tasks: {passed_tasks}/{total_tasks} ({task_percent:.1}%) +- Scorers: {passed_scorers}/{total_scorers} ({scorer_percent:.1}%) + +", + task_percent = percent(passed_tasks as u32, total_tasks as u32), + scorer_percent = percent(passed_scorers, total_scorers), + ); + + match analysis { + Some(analysis) => report.push_str(analysis.trim()), + None => report.push_str( + "\ +## Recommended actions + +No repository changes recommended. + +## Failure patterns + +No failures detected.", + ), + } + report.push('\n'); + report +} + +fn percent(passed: u32, total: u32) -> f64 { + if total == 0 { + 0.0 + } else { + f64::from(passed) * 100.0 / f64::from(total) + } +} + fn extract_reasons(details: &HashMap) -> Vec { details .iter() @@ -192,3 +306,104 @@ fn truncate(s: &str, max: usize) -> &str { None => s, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn outcome(task: &str, passed_tests: u32, total_tests: u32) -> RunOutcome { + RunOutcome { + hash: String::new(), + task: task.to_string(), + lang: "typescript".to_string(), + golden_published: true, + model_name: "test-model".to_string(), + total_tests, + passed_tests, + llm_output: None, + category: None, + route_api_model: None, + golden_db: None, + llm_db: None, + work_dir_golden: None, + work_dir_llm: None, + scorer_details: None, + vendor: String::new(), + input_tokens: None, + output_tokens: None, + generation_duration_ms: None, + started_at: None, + finished_at: None, + } + } + + #[test] + fn instructions_request_actionable_evidence_based_output() { + let instructions = analysis_instructions("guidelines"); + + assert!(instructions.contains("## Recommended actions")); + assert!(instructions.contains("Eval problem")); + assert!(instructions.contains("Skill problem")); + assert!(instructions.contains("Model limitation")); + assert!(instructions.contains("Do not invent evidence")); + assert!(instructions.contains("AI guidelines gap")); + assert!(instructions.contains("Needs manual review")); + assert!(instructions.contains("do not classify a group as `Skill problem` or `Documentation problem`")); + } + + #[test] + fn no_context_does_not_request_a_context_gap() { + let instructions = analysis_instructions("no_context"); + + assert!(!instructions.contains("context gap:")); + assert!(!instructions.contains("guidelines gap:")); + } + + #[test] + fn report_includes_task_and_scorer_pass_rates() { + let outcomes = vec![outcome("t_001", 3, 3), outcome("t_002", 1, 2), outcome("t_003", 0, 0)]; + let report = build_report( + &outcomes, + "typescript", + "guidelines", + "test-model", + Some("## Recommended actions\n\n- Fix the skill."), + ); + + assert!(report.contains("- Tasks: 1/3 (33.3%)")); + assert!(report.contains("- Scorers: 4/5 (80.0%)")); + assert!(report.contains("- Fix the skill.")); + } + + #[test] + fn live_prompt_supplies_context_for_gap_analysis() { + let failed = outcome("t_001", 0, 1); + let prompt = build_prompt( + "typescript", + "guidelines", + "test-model", + "Use transactions for procedure writes.", + Path::new("missing-benchmark-root"), + &[&failed], + ); + + assert!(prompt.contains("### Context supplied to the model")); + assert!(prompt.contains("Use transactions for procedure writes.")); + assert!(prompt.contains("What is missing or unclear in the AI guidelines")); + assert!(!prompt.contains("complete AI guidelines was not supplied")); + } + + #[test] + fn passing_report_recommends_no_changes() { + let report = build_report( + &[outcome("t_001", 3, 3)], + "typescript", + "guidelines", + "test-model", + None, + ); + + assert!(report.contains("No repository changes recommended.")); + assert!(report.contains("No failures detected.")); + } +} diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index fc6da3884af..c4394ff97a4 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -522,11 +522,31 @@ fn dry_run_analysis_path(run_id: &str, lang_name: &str, mode: &str, route: &Mode .join(format!("run-{run_id}-{lang_name}-{mode}-{route_tag}-analysis.md")) } +fn configured_analysis_path(lang_name: &str, mode: &str, route: &ModelRoute) -> Option { + let report_dir = std::env::var_os("LLM_BENCHMARK_REPORT_DIR").filter(|value| !value.is_empty())?; + let route_tag = sanitize_db_name(&route.display_name); + Some(PathBuf::from(report_dir).join(format!("{lang_name}-{mode}-{route_tag}.md"))) +} + +fn write_analysis_report( + path: &Path, + cfg: &BenchRunContext<'_>, + outcomes: &[RunOutcome], + analysis: Option<&str>, +) -> Result<()> { + fs::create_dir_all(path.parent().unwrap_or_else(|| Path::new(".")))?; + let report = + crate::bench::analysis::build_report(outcomes, cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, analysis); + fs::write(path, report)?; + Ok(()) +} + async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutcome]) -> Result> { + let configured_path = configured_analysis_path(cfg.lang.as_str(), cfg.mode, cfg.route); let should_run = if cfg.dry_run { cfg.local_analysis } else { - cfg.api_client.is_some() + cfg.api_client.is_some() || configured_path.is_some() }; if !should_run { @@ -538,27 +558,30 @@ async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutco cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, + cfg.context, cfg.bench_root, cfg.llm, ) .await?; if cfg.dry_run - && let (Some(text), Some(run_id)) = (analysis.as_deref(), cfg.dry_run_id.as_deref()) + && let Some(run_id) = cfg.dry_run_id.as_deref() { let path = dry_run_analysis_path(run_id, cfg.lang.as_str(), cfg.mode, cfg.route); - let _ = fs::create_dir_all(path.parent().unwrap_or_else(|| Path::new("."))); - let contents = format!( - "# Local Benchmark Analysis\n\n- Lang: {}\n- Mode: {}\n- Model: {}\n\n{}", - cfg.lang.as_str(), - cfg.mode, - cfg.route.display_name, - text - ); - match fs::write(&path, contents) { + match write_analysis_report(&path, cfg, outcomes, analysis.as_deref()) { Ok(()) => println!("Local analysis: {}", path.display()), Err(e) => eprintln!("[warn] failed to write local analysis: {e}"), } + } + + if let Some(path) = configured_path { + match write_analysis_report(&path, cfg, outcomes, analysis.as_deref()) { + Ok(()) => println!("Analysis report: {}", path.display()), + Err(e) => eprintln!("[warn] failed to write analysis report: {e}"), + } + } + + if cfg.dry_run { return Ok(None); } From c3e0fb6f407c8960294e8a60c336a6674262278e Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:51:46 -0400 Subject: [PATCH 74/78] Update llm-benchmark-periodic.yml --- .github/workflows/llm-benchmark-periodic.yml | 132 +++++++++++++++---- 1 file changed, 105 insertions(+), 27 deletions(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 3d18ce9eeb3..470e4a70741 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -260,15 +260,27 @@ jobs: if: github.event_name == 'schedule' || inputs.post_discord env: RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUN_LABEL: ${{ github.event_name == 'schedule' && 'Weekly scheduled run' || 'Manual run' }} run: | python3 - <<'PY' + import json import os import re + from collections import defaultdict from pathlib import Path reports = sorted(Path("reports").rglob("*.md")) - rates = [] + totals = [0, 0] + by_language = defaultdict(lambda: [0, 0]) actions = [] + other_findings = [] + + def language_name(language): + return { + "csharp": "C#", + "rust": "Rust", + "typescript": "TypeScript", + }.get(language, language) for report in reports: text = report.read_text(encoding="utf-8") @@ -277,34 +289,101 @@ jobs: match = re.search(rf"^- {name}: (.+)$", text, re.MULTILINE) return match.group(1) if match else "unknown" - rates.append( - f"• {field('Language')} / {field('Mode')} / {field('Model')}: " - f"{field('Tasks')} tasks, {field('Scorers')} scorers" - ) + language = field("Language") + mode = field("Mode") + model = field("Model") + tasks = re.match(r"(\d+)/(\d+)", field("Tasks")) + if tasks: + passed, total = map(int, tasks.groups()) + totals[0] += passed + totals[1] += total + by_language[language][0] += passed + by_language[language][1] += total recommended = text.partition("## Recommended actions")[2].partition("## Failure patterns")[0] for line in recommended.splitlines(): - if line.startswith("- **[") and line not in actions: - actions.append(line.replace("- ", "• ", 1)) - - lines = [ - "**LLM benchmark analysis**", - f"[View workflow run]({os.environ['RUN_URL']})", - "", - "**Pass rates**", - *(rates[:8] or ["• No analysis reports were produced."]), + action = f"- {language_name(language)} / {mode}: {line[2:]}" + if line.startswith("- **[") and action not in actions: + actions.append(action) + + failures = text.partition("## Failure patterns")[2] + for section in failures.split("\n### ")[1:]: + title = section.splitlines()[0].rsplit(" (", 1)[0] + classification = re.search(r"^- \*\*Classification:\*\* (.+)$", section, re.MULTILINE) + if classification and classification.group(1) in { + "Model limitation", + "Infrastructure/provider problem", + "No action", + }: + finding = ( + f"- {language_name(language)} / {mode} / {model}: " + f"{title} - {classification.group(1)}" + ) + if finding not in other_findings: + other_findings.append(finding) + + def rate(passed, total): + percent = passed * 100 / total if total else 0 + return f"{passed}/{total} ({percent:.1f}%)" + + def field_value(lines, empty, overflow_label): + visible = lines[:5] or [empty] + if len(lines) > 5: + visible.append(f"- ...and {len(lines) - 5} more {overflow_label}(s)") + value = "\n".join(visible) + return value if len(value) <= 1000 else value[:970].rstrip() + "\n... View the full analysis." + + passed, total = totals + pass_percent = passed * 100 / total if total else 0 + has_infrastructure_failure = any("Infrastructure/provider problem" in item for item in other_findings) + if not total or pass_percent < 90 or has_infrastructure_failure: + color = 0xED4245 + elif actions or pass_percent < 95: + color = 0xFEE75C + else: + color = 0x57F287 + + language_rates = [ + f"- **{language_name(language)}:** {rate(*counts)}" + for language, counts in sorted(by_language.items()) ] - if len(rates) > 8: - lines.append(f"• …and {len(rates) - 8} more report(s)") - - lines.extend(["", "**Recommended actions**", *(actions[:5] or ["• None"])]) - if len(actions) > 5: - lines.append(f"• …and {len(actions) - 5} more action(s)") - - message = "\n".join(lines) - if len(message) > 1900: - message = message[:1860].rstrip() + "\n… See the workflow for the full report." - Path("discord-summary.txt").write_text(message, encoding="utf-8") + payload = { + "username": "SpacetimeDB LLM Benchmarks", + "embeds": [ + { + "title": "LLM Benchmark Analysis", + "url": os.environ["RUN_URL"], + "description": f"**{rate(*totals)}** task runs passed", + "color": color, + "fields": [ + { + "name": "By language", + "value": field_value( + language_rates, + "No analysis reports were produced.", + "language", + ), + "inline": False, + }, + { + "name": "Action items", + "value": field_value(actions, "None", "action"), + "inline": False, + }, + { + "name": "Other failures", + "value": field_value(other_findings, "None", "finding"), + "inline": False, + }, + ], + "footer": {"text": os.environ["RUN_LABEL"]}, + } + ], + } + Path("discord-payload.json").write_text( + json.dumps(payload, ensure_ascii=False), + encoding="utf-8", + ) PY - name: Post Discord summary @@ -318,8 +397,7 @@ jobs: exit 0 fi - payload="$(jq -n --rawfile content discord-summary.txt '{content: $content}')" curl --fail --silent --show-error \ --header 'Content-Type: application/json' \ - --data "$payload" \ + --data-binary @discord-payload.json \ "$DISCORD_WEBHOOK_URL" From c018a4485a5b94e521a1ee76e1ce45e266d36c9d Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:55:50 -0400 Subject: [PATCH 75/78] discord summary script --- .github/workflows/llm-benchmark-periodic.yml | 136 ++------------ .../scripts/discord_summary.py | 172 ++++++++++++++++++ .../scripts/test_discord_summary.py | 111 +++++++++++ 3 files changed, 295 insertions(+), 124 deletions(-) create mode 100644 tools/xtask-llm-benchmark/scripts/discord_summary.py create mode 100644 tools/xtask-llm-benchmark/scripts/test_discord_summary.py diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 470e4a70741..2626bb84573 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -228,6 +228,12 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Test Discord formatter + run: python3 tools/xtask-llm-benchmark/scripts/test_discord_summary.py + - name: Download analysis reports continue-on-error: true uses: actions/download-artifact@v4 @@ -261,130 +267,12 @@ jobs: env: RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} RUN_LABEL: ${{ github.event_name == 'schedule' && 'Weekly scheduled run' || 'Manual run' }} - run: | - python3 - <<'PY' - import json - import os - import re - from collections import defaultdict - from pathlib import Path - - reports = sorted(Path("reports").rglob("*.md")) - totals = [0, 0] - by_language = defaultdict(lambda: [0, 0]) - actions = [] - other_findings = [] - - def language_name(language): - return { - "csharp": "C#", - "rust": "Rust", - "typescript": "TypeScript", - }.get(language, language) - - for report in reports: - text = report.read_text(encoding="utf-8") - - def field(name): - match = re.search(rf"^- {name}: (.+)$", text, re.MULTILINE) - return match.group(1) if match else "unknown" - - language = field("Language") - mode = field("Mode") - model = field("Model") - tasks = re.match(r"(\d+)/(\d+)", field("Tasks")) - if tasks: - passed, total = map(int, tasks.groups()) - totals[0] += passed - totals[1] += total - by_language[language][0] += passed - by_language[language][1] += total - - recommended = text.partition("## Recommended actions")[2].partition("## Failure patterns")[0] - for line in recommended.splitlines(): - action = f"- {language_name(language)} / {mode}: {line[2:]}" - if line.startswith("- **[") and action not in actions: - actions.append(action) - - failures = text.partition("## Failure patterns")[2] - for section in failures.split("\n### ")[1:]: - title = section.splitlines()[0].rsplit(" (", 1)[0] - classification = re.search(r"^- \*\*Classification:\*\* (.+)$", section, re.MULTILINE) - if classification and classification.group(1) in { - "Model limitation", - "Infrastructure/provider problem", - "No action", - }: - finding = ( - f"- {language_name(language)} / {mode} / {model}: " - f"{title} - {classification.group(1)}" - ) - if finding not in other_findings: - other_findings.append(finding) - - def rate(passed, total): - percent = passed * 100 / total if total else 0 - return f"{passed}/{total} ({percent:.1f}%)" - - def field_value(lines, empty, overflow_label): - visible = lines[:5] or [empty] - if len(lines) > 5: - visible.append(f"- ...and {len(lines) - 5} more {overflow_label}(s)") - value = "\n".join(visible) - return value if len(value) <= 1000 else value[:970].rstrip() + "\n... View the full analysis." - - passed, total = totals - pass_percent = passed * 100 / total if total else 0 - has_infrastructure_failure = any("Infrastructure/provider problem" in item for item in other_findings) - if not total or pass_percent < 90 or has_infrastructure_failure: - color = 0xED4245 - elif actions or pass_percent < 95: - color = 0xFEE75C - else: - color = 0x57F287 - - language_rates = [ - f"- **{language_name(language)}:** {rate(*counts)}" - for language, counts in sorted(by_language.items()) - ] - payload = { - "username": "SpacetimeDB LLM Benchmarks", - "embeds": [ - { - "title": "LLM Benchmark Analysis", - "url": os.environ["RUN_URL"], - "description": f"**{rate(*totals)}** task runs passed", - "color": color, - "fields": [ - { - "name": "By language", - "value": field_value( - language_rates, - "No analysis reports were produced.", - "language", - ), - "inline": False, - }, - { - "name": "Action items", - "value": field_value(actions, "None", "action"), - "inline": False, - }, - { - "name": "Other failures", - "value": field_value(other_findings, "None", "finding"), - "inline": False, - }, - ], - "footer": {"text": os.environ["RUN_LABEL"]}, - } - ], - } - Path("discord-payload.json").write_text( - json.dumps(payload, ensure_ascii=False), - encoding="utf-8", - ) - PY + run: > + python3 tools/xtask-llm-benchmark/scripts/discord_summary.py + --reports-dir reports + --run-url "$RUN_URL" + --run-label "$RUN_LABEL" + --output discord-payload.json - name: Post Discord summary if: github.event_name == 'schedule' || inputs.post_discord diff --git a/tools/xtask-llm-benchmark/scripts/discord_summary.py b/tools/xtask-llm-benchmark/scripts/discord_summary.py new file mode 100644 index 00000000000..dfd211aeb2e --- /dev/null +++ b/tools/xtask-llm-benchmark/scripts/discord_summary.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +GREEN = 0x57F287 +YELLOW = 0xFEE75C +RED = 0xED4245 +MAX_ITEMS = 5 +MAX_FIELD_LENGTH = 1000 + + +@dataclass +class Report: + language: str + mode: str + model: str + passed_tasks: int + total_tasks: int + actions: list[str] + other_findings: list[str] + + +def field(text: str, name: str) -> str: + match = re.search(rf"^- {re.escape(name)}: (.+)$", text, re.MULTILINE) + return match.group(1) if match else "unknown" + + +def language_name(language: str) -> str: + return { + "csharp": "C#", + "rust": "Rust", + "typescript": "TypeScript", + }.get(language, language) + + +def parse_report(text: str) -> Report: + language = field(text, "Language") + mode = field(text, "Mode") + model = field(text, "Model") + task_match = re.match(r"(\d+)/(\d+)", field(text, "Tasks")) + passed_tasks, total_tasks = map(int, task_match.groups()) if task_match else (0, 0) + + actions = [] + recommended = text.partition("## Recommended actions")[2].partition("## Failure patterns")[0] + for line in recommended.splitlines(): + if line.startswith("- **["): + actions.append(f"- {language_name(language)} / {mode}: {line[2:]}") + + other_findings = [] + failures = text.partition("## Failure patterns")[2] + for section in re.split(r"^### ", failures, flags=re.MULTILINE)[1:]: + heading = section.splitlines()[0] + title_match = re.match(r"(.+) \(\d+ tasks?\)$", heading) + title = title_match.group(1) if title_match else heading + classification = re.search(r"^- \*\*Classification:\*\* (.+)$", section, re.MULTILINE) + if classification and classification.group(1) in { + "Model limitation", + "Infrastructure/provider problem", + "No action", + }: + other_findings.append( + f"- {language_name(language)} / {mode} / {model}: {title} - {classification.group(1)}" + ) + + return Report(language, mode, model, passed_tasks, total_tasks, actions, other_findings) + + +def load_reports(reports_dir: Path) -> list[Report]: + return [ + parse_report(path.read_text(encoding="utf-8")) + for path in sorted(reports_dir.rglob("*.md")) + ] + + +def rate(passed: int, total: int) -> str: + percent = passed * 100 / total if total else 0 + return f"{passed}/{total} ({percent:.1f}%)" + + +def field_value(lines: list[str], empty: str, overflow_label: str) -> str: + unique = list(dict.fromkeys(lines)) + visible = unique[:MAX_ITEMS] or [empty] + if len(unique) > MAX_ITEMS: + visible.append(f"- ...and {len(unique) - MAX_ITEMS} more {overflow_label}(s)") + value = "\n".join(visible) + if len(value) > MAX_FIELD_LENGTH: + return value[: MAX_FIELD_LENGTH - 30].rstrip() + "\n... View the full analysis." + return value + + +def build_payload(reports: list[Report], run_url: str, run_label: str) -> dict: + totals = [0, 0] + by_language: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + actions = [] + other_findings = [] + + for report in reports: + totals[0] += report.passed_tasks + totals[1] += report.total_tasks + by_language[report.language][0] += report.passed_tasks + by_language[report.language][1] += report.total_tasks + actions.extend(report.actions) + other_findings.extend(report.other_findings) + + passed, total = totals + pass_percent = passed * 100 / total if total else 0 + has_infrastructure_failure = any("Infrastructure/provider problem" in item for item in other_findings) + if not total or pass_percent < 90 or has_infrastructure_failure: + color = RED + elif actions or pass_percent < 95: + color = YELLOW + else: + color = GREEN + + language_rates = [ + f"- **{language_name(language)}:** {rate(*counts)}" + for language, counts in sorted(by_language.items()) + ] + return { + "username": "SpacetimeDB LLM Benchmarks", + "embeds": [ + { + "title": "LLM Benchmark Analysis", + "url": run_url, + "description": f"**{rate(*totals)}** task runs passed", + "color": color, + "fields": [ + { + "name": "By language", + "value": field_value( + language_rates, + "No analysis reports were produced.", + "language", + ), + "inline": False, + }, + { + "name": "Action items", + "value": field_value(actions, "None", "action"), + "inline": False, + }, + { + "name": "Other failures", + "value": field_value(other_findings, "None", "finding"), + "inline": False, + }, + ], + "footer": {"text": run_label}, + } + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build a Discord embed from LLM benchmark analysis reports.") + parser.add_argument("--reports-dir", type=Path, required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--run-label", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + payload = build_payload(load_reports(args.reports_dir), args.run_url, args.run_label) + args.output.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tools/xtask-llm-benchmark/scripts/test_discord_summary.py b/tools/xtask-llm-benchmark/scripts/test_discord_summary.py new file mode 100644 index 00000000000..6f7c13a5949 --- /dev/null +++ b/tools/xtask-llm-benchmark/scripts/test_discord_summary.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import tempfile +import unittest +from pathlib import Path + +from discord_summary import GREEN, RED, YELLOW, build_payload, field_value, load_reports + + +def report( + *, + language: str = "csharp", + mode: str = "guidelines", + model: str = "test-model", + tasks: str = "36/37 (97.3%)", + action: str | None = None, + pattern: str | None = None, + classification: str = "Model limitation", +) -> str: + recommended = action or "No repository changes recommended." + failures = ( + f"### {pattern} (1 task)\n\n- **Classification:** {classification}\n" + if pattern + else "No failures detected.\n" + ) + return f"""# LLM Benchmark Analysis + +- Language: {language} +- Mode: {mode} +- Model: {model} +- Tasks: {tasks} +- Scorers: 100/100 (100.0%) + +## Recommended actions + +{recommended} + +## Failure patterns + +{failures}""" + + +class DiscordSummaryTests(unittest.TestCase): + def load(self, *contents: str): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index, content in enumerate(contents): + (root / f"report-{index}.md").write_text(content, encoding="utf-8") + return load_reports(root) + + def test_aggregates_rates_and_formats_findings(self): + reports = self.load( + report(tasks="36/37 (97.3%)"), + report( + language="rust", + tasks="34/37 (91.9%)", + pattern="Incorrect sum-type syntax", + ), + ) + + embed = build_payload(reports, "https://example.com/run", "Weekly run")["embeds"][0] + + self.assertEqual(embed["description"], "**70/74 (94.6%)** task runs passed") + self.assertEqual(embed["color"], YELLOW) + self.assertIn("**C#:** 36/37 (97.3%)", embed["fields"][0]["value"]) + self.assertIn("Rust / guidelines / test-model", embed["fields"][2]["value"]) + + def test_action_items_force_review_status(self): + reports = self.load( + report( + action="- **[Skill problem | High] Clarify transactions** — Update the skill. Evidence: t_075.", + ) + ) + + embed = build_payload(reports, "https://example.com/run", "Weekly run")["embeds"][0] + + self.assertEqual(embed["color"], YELLOW) + self.assertIn("C# / guidelines", embed["fields"][1]["value"]) + + def test_healthy_and_infrastructure_colors(self): + healthy = build_payload( + self.load(report(tasks="37/37 (100.0%)")), + "https://example.com/run", + "Weekly run", + )["embeds"][0] + infrastructure = build_payload( + self.load( + report( + tasks="37/37 (100.0%)", + pattern="Provider timeout", + classification="Infrastructure/provider problem", + ) + ), + "https://example.com/run", + "Weekly run", + )["embeds"][0] + + self.assertEqual(healthy["color"], GREEN) + self.assertEqual(infrastructure["color"], RED) + + def test_empty_and_long_fields_stay_valid(self): + empty = build_payload([], "https://example.com/run", "Manual run")["embeds"][0] + long_value = field_value([f"- {index} {'x' * 400}" for index in range(6)], "None", "finding") + + self.assertEqual(empty["color"], RED) + self.assertIn("No analysis reports", empty["fields"][0]["value"]) + self.assertLessEqual(len(long_value), 1000) + + +if __name__ == "__main__": + unittest.main() From 7c265e552a8fbd8010123170e63f6db6ad093e09 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:15:22 -0400 Subject: [PATCH 76/78] reasoning arg --- .../src/bin/llm_benchmark.rs | 33 ++++++++++++++----- .../src/llm/clients/anthropic.rs | 16 +++++++-- .../src/llm/clients/deepseek.rs | 12 +++++-- .../src/llm/clients/google.rs | 23 +++++++++++-- .../src/llm/clients/meta.rs | 4 +-- .../src/llm/clients/mod.rs | 17 ++++++---- .../src/llm/clients/openai.rs | 11 +++++-- .../src/llm/clients/openrouter.rs | 11 +++++-- .../src/llm/clients/xai.rs | 6 ++-- tools/xtask-llm-benchmark/src/llm/config.rs | 8 +++-- tools/xtask-llm-benchmark/src/llm/mod.rs | 2 +- tools/xtask-llm-benchmark/src/llm/provider.rs | 12 +++++-- tools/xtask-llm-benchmark/src/llm/types.rs | 20 +++++++++++ 13 files changed, 139 insertions(+), 36 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index be76423286d..84be5faaed5 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -21,7 +21,9 @@ use xtask_llm_benchmark::context::constants::ALL_MODES; use xtask_llm_benchmark::context::{build_context, compute_processed_context_hash}; use xtask_llm_benchmark::eval::Lang; use xtask_llm_benchmark::llm::types::Vendor; -use xtask_llm_benchmark::llm::{default_model_routes, make_provider_from_env, LlmProvider, ModelRoute}; +use xtask_llm_benchmark::llm::{ + default_model_routes, make_provider_from_env, LlmProvider, ModelRoute, ReasoningEffort, +}; #[derive(Clone, Debug)] struct ModelGroup { @@ -68,6 +70,10 @@ impl std::str::FromStr for ModelGroup { after_help = "Notes:\n • Anthropic ids: claude-sonnet-4-5, claude-sonnet-4, claude-3-7-sonnet-latest, claude-3-5-sonnet-latest\n • Base URLs must not include /v1; models must be valid for the chosen provider.\n" )] struct Cli { + /// Reasoning effort used for every model request + #[arg(long, value_enum, default_value_t = ReasoningEffort::Medium, global = true)] + reasoning: ReasoningEffort, + #[command(subcommand)] command: Commands, } @@ -215,20 +221,20 @@ fn main() -> Result<()> { let cli = Cli::parse(); match cli.command { - Commands::Run(args) => cmd_run(args), - Commands::Analyze(args) => cmd_analyze(args), + Commands::Run(args) => cmd_run(args, cli.reasoning), + Commands::Analyze(args) => cmd_analyze(args, cli.reasoning), } } /* ------------------------------ run ------------------------------ */ -fn cmd_run(args: RunArgs) -> Result<()> { - run_benchmarks(args)?; +fn cmd_run(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { + run_benchmarks(args, reasoning)?; Ok(()) } /// Core benchmark runner used by both `run` and `ci-quickfix` -fn run_benchmarks(args: RunArgs) -> Result<()> { +fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { let dry_run = args.dry_run; let local_analysis = args.local_analysis; let dry_run_id = dry_run.then(|| { @@ -314,7 +320,7 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { } let llm_provider = if !config.goldens_only && !config.hash_only { - let provider = make_provider_from_env()?; + let provider = make_provider_from_env(reasoning)?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); preflight_llm_routes(rt, provider.as_ref(), &routes, &modes)?; @@ -378,7 +384,7 @@ fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { /* ------------------------------ analyze ------------------------------ */ -fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { +fn cmd_analyze(args: AnalyzeArgs, reasoning: ReasoningEffort) -> Result<()> { let api = ApiClient::from_env() .context("failed to initialize API client")? .context("LLM_BENCHMARK_UPLOAD_URL required for analyze")?; @@ -434,7 +440,7 @@ fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { // Initialize LLM provider for analysis let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; - let provider = make_provider_from_env()?; + let provider = make_provider_from_env(reasoning)?; let analysis_route = ModelRoute::new( "gpt-5.4-mini", @@ -989,6 +995,15 @@ mod tests { } } + #[test] + fn reasoning_defaults_to_medium_and_accepts_an_override() { + let default = Cli::try_parse_from(["llm", "run", "--hash-only"]).unwrap(); + assert_eq!(default.reasoning, ReasoningEffort::Medium); + + let overridden = Cli::try_parse_from(["llm", "run", "--hash-only", "--reasoning", "high"]).unwrap(); + assert_eq!(overridden.reasoning, ReasoningEffort::High); + } + #[test] fn explicit_models_bypass_remote_model_source() { let mut args = base_run_args(); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs index 8bb0d1ac734..b8bd6cd55e6 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ anthropic_ctx_limit_tokens, build_anthropic_messages, desired_output_tokens, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{anyhow, bail, Context, Result}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE}; use reqwest::{Client, StatusCode}; @@ -30,7 +30,7 @@ impl AnthropicClient { format!("{}/v1/messages", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); let mut static_prefix = prompt.static_prefix.clone().unwrap_or_default(); @@ -73,13 +73,25 @@ impl AnthropicClient { struct Req { model: String, max_tokens: u32, + thinking: ThinkingConfig, + output_config: OutputConfig, #[serde(skip_serializing_if = "Option::is_none")] system: Option, messages: Vec, } + #[derive(Serialize)] + struct ThinkingConfig { + r#type: &'static str, + } + #[derive(Serialize)] + struct OutputConfig { + effort: ReasoningEffort, + } let req = Req { model: model_norm.to_string(), max_tokens, + thinking: ThinkingConfig { r#type: "adaptive" }, + output_config: OutputConfig { effort: reasoning }, system: system_json, messages, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs index 5b7bc7b8636..ccf3164d856 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs @@ -7,7 +7,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deepseek_ctx_limit_tokens, deterministic_trim_prefix, estimate_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct DeepSeekClient { @@ -21,7 +21,7 @@ impl DeepSeekClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -47,6 +47,12 @@ impl DeepSeekClient { model: &'a str, messages: Vec>, temperature: f32, + thinking: ThinkingConfig, + reasoning_effort: ReasoningEffort, + } + #[derive(Serialize)] + struct ThinkingConfig { + r#type: &'static str, } #[derive(Serialize)] struct Msg<'a> { @@ -78,6 +84,8 @@ impl DeepSeekClient { model, messages, temperature: 0.0, + thinking: ThinkingConfig { r#type: "enabled" }, + reasoning_effort: reasoning, }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/google.rs b/tools/xtask-llm-benchmark/src/llm/clients/google.rs index ad309a88477..f822be51e5f 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/google.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/google.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, gemini_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; /// Google uses API key in the query string rather than Authorization header. #[derive(Clone)] @@ -23,7 +23,7 @@ impl GoogleGeminiClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { // ---- Never trim system or dynamic segments ---- let system = prompt.system.clone(); let segs: Vec> = prompt.segments.clone(); @@ -49,10 +49,24 @@ impl GoogleGeminiClient { #[serde(skip_serializing_if = "Option::is_none")] system_instruction: Option>, contents: Vec>, + #[serde(rename = "generationConfig")] + generation_config: GenerationConfig, #[serde(skip_serializing_if = "Option::is_none")] safety_settings: Option>, } + #[derive(Serialize)] + struct GenerationConfig { + #[serde(rename = "thinkingConfig")] + thinking_config: ThinkingConfig, + } + + #[derive(Serialize)] + struct ThinkingConfig { + #[serde(rename = "thinkingLevel")] + thinking_level: ReasoningEffort, + } + #[derive(Serialize)] struct SystemInstruction<'a> { parts: [Part<'a>; 1], @@ -100,6 +114,11 @@ impl GoogleGeminiClient { let req = Req { system_instruction, contents, + generation_config: GenerationConfig { + thinking_config: ThinkingConfig { + thinking_level: reasoning, + }, + }, safety_settings: None, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 05ea663019e..9bf4d87df9d 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -7,7 +7,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct MetaLlamaClient { @@ -22,7 +22,7 @@ impl MetaLlamaClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, _reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); // Build input like other clients diff --git a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs index 254fe5b8f63..604afed9c60 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs @@ -20,7 +20,7 @@ pub use openrouter::OpenRouterClient; pub use xai::XaiGrokClient; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::LlmOutput; +use crate::llm::types::{LlmOutput, ReasoningEffort}; #[derive(Debug, Clone)] pub struct ClientPreflight { @@ -51,7 +51,7 @@ pub trait LlmClient: Send + Sync { ))) } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result; + async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result; } macro_rules! impl_direct_llm_client { @@ -62,8 +62,13 @@ macro_rules! impl_direct_llm_client { $provider_name } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { - <$ty>::generate(self, model, prompt).await + async fn generate( + &self, + model: &str, + prompt: &BuiltPrompt, + reasoning: ReasoningEffort, + ) -> Result { + <$ty>::generate(self, model, prompt, reasoning).await } } }; @@ -87,7 +92,7 @@ impl LlmClient for OpenRouterClient { Ok(ClientPreflight::new(status.summary())) } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { - OpenRouterClient::generate(self, model, prompt).await + async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { + OpenRouterClient::generate(self, model, prompt, reasoning).await } } diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs index 9fed933d3fb..537b4a6b84a 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ build_openai_responses_input, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, openai_ctx_limit_tokens, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{bail, Context, Result}; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ impl OpenAiClient { format!("{}/v1/responses", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); @@ -85,14 +85,21 @@ impl OpenAiClient { struct Req<'a> { model: &'a str, input: Vec, + reasoning: ReasoningConfig, #[serde(skip_serializing_if = "Option::is_none")] max_output_tokens: Option, } + #[derive(Serialize)] + struct ReasoningConfig { + effort: ReasoningEffort, + } + let url = self.responses_url(); let payload = Req { model, input, + reasoning: ReasoningConfig { effort: reasoning }, max_output_tokens: None, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index df4c062c1bb..c4e806666ee 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; const OPENROUTER_BASE: &str = "https://openrouter.ai/api/v1"; @@ -162,7 +162,7 @@ impl OpenRouterClient { Ok(()) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -183,6 +183,7 @@ impl OpenRouterClient { model: &'a str, messages: Vec>, temperature: f32, + reasoning: ReasoningConfig, #[serde(skip_serializing_if = "Option::is_none")] top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -195,6 +196,11 @@ impl OpenRouterClient { content: &'a str, } + #[derive(Serialize)] + struct ReasoningConfig { + effort: ReasoningEffort, + } + let mut messages: Vec = Vec::new(); if let Some(sys) = system.as_deref() @@ -222,6 +228,7 @@ impl OpenRouterClient { model, messages, temperature: 0.0, + reasoning: ReasoningConfig { effort: reasoning }, top_p: None, max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs index be345c82b32..46062c6d9bc 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs @@ -6,7 +6,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, xai_ctx_limit_tokens, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct XaiGrokClient { @@ -21,7 +21,7 @@ impl XaiGrokClient { } /// Uses BuiltPrompt (system, static_prefix, segments) and maps to xAI /chat/completions. - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/v1/chat/completions", self.base.trim_end_matches('/')); // Never trim system or dynamic segments @@ -41,6 +41,7 @@ impl XaiGrokClient { model: &'a str, messages: Vec>, temperature: f32, + reasoning_effort: ReasoningEffort, } #[derive(Serialize)] @@ -75,6 +76,7 @@ impl XaiGrokClient { model, messages, temperature: 0.0, + reasoning_effort: reasoning, }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/config.rs b/tools/xtask-llm-benchmark/src/llm/config.rs index 5eacdf40248..2e76dcbfdef 100644 --- a/tools/xtask-llm-benchmark/src/llm/config.rs +++ b/tools/xtask-llm-benchmark/src/llm/config.rs @@ -6,7 +6,7 @@ use crate::llm::clients::{ AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, OpenRouterClient, XaiGrokClient, }; use crate::llm::provider::{LlmProvider, RouterProvider}; -use crate::llm::types::Vendor; +use crate::llm::types::{ReasoningEffort, Vendor}; fn force_vendor_from_env() -> Option { match env::var("LLM_VENDOR").ok().as_deref() { @@ -34,7 +34,7 @@ fn force_vendor_from_env() -> Option { /// When OPENROUTER_API_KEY is set, it acts as a fallback for any vendor that doesn't /// have its own direct API key configured. This means you can set just OPENROUTER_API_KEY /// to run all models through OpenRouter, or mix direct keys with OpenRouter fallback. -pub fn make_provider_from_env() -> Result> { +pub fn make_provider_from_env(reasoning: ReasoningEffort) -> Result> { let http = HttpClient::new()?; // Filter out empty strings so an empty env var falls through to OpenRouter. @@ -84,6 +84,8 @@ pub fn make_provider_from_env() -> Result> { let openrouter = openrouter_key.map(|k| OpenRouterClient::new(http.clone(), k)); let force = force_vendor_from_env(); - let router = RouterProvider::new(openai, anthropic, google, xai, deepseek, meta, openrouter, force); + let router = RouterProvider::new( + openai, anthropic, google, xai, deepseek, meta, openrouter, force, reasoning, + ); Ok(Arc::new(router)) } diff --git a/tools/xtask-llm-benchmark/src/llm/mod.rs b/tools/xtask-llm-benchmark/src/llm/mod.rs index 4b72185c760..9e6548ec55a 100644 --- a/tools/xtask-llm-benchmark/src/llm/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/mod.rs @@ -10,4 +10,4 @@ pub use config::make_provider_from_env; pub use model_routes::{default_model_routes, ModelRoute}; pub use prompt::PromptBuilder; pub use provider::{LlmProvider, RouterProvider}; -pub use types::LlmOutput; +pub use types::{LlmOutput, ReasoningEffort}; diff --git a/tools/xtask-llm-benchmark/src/llm/provider.rs b/tools/xtask-llm-benchmark/src/llm/provider.rs index 355f2e19a3e..1dba17764b9 100644 --- a/tools/xtask-llm-benchmark/src/llm/provider.rs +++ b/tools/xtask-llm-benchmark/src/llm/provider.rs @@ -8,7 +8,7 @@ use crate::llm::clients::{ }; use crate::llm::model_routes::ModelRoute; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[async_trait] pub trait LlmProvider: Send + Sync { @@ -19,6 +19,7 @@ pub trait LlmProvider: Send + Sync { pub struct RouterProvider { clients: HashMap>, pub force: Option, + reasoning: ReasoningEffort, } impl RouterProvider { @@ -32,6 +33,7 @@ impl RouterProvider { meta: Option, openrouter: Option, force: Option, + reasoning: ReasoningEffort, ) -> Self { let mut clients: HashMap> = HashMap::new(); @@ -57,7 +59,11 @@ impl RouterProvider { clients.insert(Vendor::OpenRouter, Box::new(client)); } - Self { clients, force } + Self { + clients, + force, + reasoning, + } } } @@ -105,7 +111,7 @@ impl LlmProvider for RouterProvider { ); } - resolved.client.generate(&resolved.model, prompt).await + resolved.client.generate(&resolved.model, prompt, self.reasoning).await } } diff --git a/tools/xtask-llm-benchmark/src/llm/types.rs b/tools/xtask-llm-benchmark/src/llm/types.rs index 5ccfd14f339..f83a45b3d81 100644 --- a/tools/xtask-llm-benchmark/src/llm/types.rs +++ b/tools/xtask-llm-benchmark/src/llm/types.rs @@ -1,6 +1,16 @@ +use clap::ValueEnum; use serde::{Deserialize, Serialize}; use std::fmt; +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningEffort { + Low, + #[default] + Medium, + High, +} + /// Output from an LLM generation call, including token usage. #[derive(Debug, Clone, Default)] pub struct LlmOutput { @@ -68,3 +78,13 @@ impl fmt::Display for Vendor { f.write_str(self.slug()) } } + +#[cfg(test)] +mod tests { + use super::ReasoningEffort; + + #[test] + fn reasoning_effort_uses_provider_wire_values() { + assert_eq!(serde_json::to_string(&ReasoningEffort::Medium).unwrap(), r#""medium""#); + } +} From ef59cc22f9456bc9ddb249e1364306c4a0cf3e41 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:05:17 -0400 Subject: [PATCH 77/78] polish --- .../scripts/discord_summary.py | 1 + .../scripts/test_discord_summary.py | 4 +- .../xtask-llm-benchmark/src/bench/analysis.rs | 79 ++++++++++++++++++- tools/xtask-llm-benchmark/src/bench/runner.rs | 6 +- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/tools/xtask-llm-benchmark/scripts/discord_summary.py b/tools/xtask-llm-benchmark/scripts/discord_summary.py index dfd211aeb2e..c76f418f73a 100644 --- a/tools/xtask-llm-benchmark/scripts/discord_summary.py +++ b/tools/xtask-llm-benchmark/scripts/discord_summary.py @@ -123,6 +123,7 @@ def build_payload(reports: list[Report], run_url: str, run_label: str) -> dict: ] return { "username": "SpacetimeDB LLM Benchmarks", + "allowed_mentions": {"parse": []}, "embeds": [ { "title": "LLM Benchmark Analysis", diff --git a/tools/xtask-llm-benchmark/scripts/test_discord_summary.py b/tools/xtask-llm-benchmark/scripts/test_discord_summary.py index 6f7c13a5949..bf1137aa80a 100644 --- a/tools/xtask-llm-benchmark/scripts/test_discord_summary.py +++ b/tools/xtask-llm-benchmark/scripts/test_discord_summary.py @@ -99,9 +99,11 @@ def test_healthy_and_infrastructure_colors(self): self.assertEqual(infrastructure["color"], RED) def test_empty_and_long_fields_stay_valid(self): - empty = build_payload([], "https://example.com/run", "Manual run")["embeds"][0] + payload = build_payload([], "https://example.com/run", "Manual run") + empty = payload["embeds"][0] long_value = field_value([f"- {index} {'x' * 400}" for index in range(6)], "None", "finding") + self.assertEqual(payload["allowed_mentions"], {"parse": []}) self.assertEqual(empty["color"], RED) self.assertIn("No analysis reports", empty["fields"][0]["value"]) self.assertLessEqual(len(long_value), 1000) diff --git a/tools/xtask-llm-benchmark/src/bench/analysis.rs b/tools/xtask-llm-benchmark/src/bench/analysis.rs index 177d85cb815..2b2f014b772 100644 --- a/tools/xtask-llm-benchmark/src/bench/analysis.rs +++ b/tools/xtask-llm-benchmark/src/bench/analysis.rs @@ -248,6 +248,31 @@ pub fn build_report( let total_tasks = outcomes.len(); let passed_scorers: u32 = outcomes.iter().map(|outcome| outcome.passed_tests).sum(); let total_scorers: u32 = outcomes.iter().map(|outcome| outcome.total_tests).sum(); + let failures_without_output: Vec<&str> = outcomes + .iter() + .filter(|outcome| outcome.passed_tests < outcome.total_tests && outcome.llm_output.is_none()) + .map(|outcome| outcome.task.as_str()) + .collect(); + let unavailable_output_section = if failures_without_output.is_empty() { + None + } else { + let task_count = failures_without_output.len(); + let task_label = if task_count == 1 { "task" } else { "tasks" }; + let tasks = failures_without_output + .iter() + .map(|task| format!("`{task}`")) + .collect::>() + .join(", "); + Some(format!( + "\ +### Model output unavailable ({task_count} {task_label}) + +- **Classification:** Infrastructure/provider problem +- **Tasks:** {tasks} +- **What happened:** The model request failed before producing output, so source-level failure analysis was not possible. +- **Suggested action:** Retry the affected tasks and inspect the benchmark logs if the failure persists." + )) + }; let mut report = format!( "\ @@ -265,7 +290,26 @@ pub fn build_report( ); match analysis { - Some(analysis) => report.push_str(analysis.trim()), + Some(analysis) => { + report.push_str(analysis.trim()); + if let Some(section) = unavailable_output_section { + report.push_str("\n\n"); + report.push_str(§ion); + } + } + None if unavailable_output_section.is_some() => { + report.push_str(&format!( + "\ +## Recommended actions + +No repository changes recommended. + +## Failure patterns + +{}", + unavailable_output_section.unwrap() + )); + } None => report.push_str( "\ ## Recommended actions @@ -406,4 +450,37 @@ mod tests { assert!(report.contains("No repository changes recommended.")); assert!(report.contains("No failures detected.")); } + + #[test] + fn failed_request_without_output_is_reported_as_infrastructure() { + let report = build_report( + &[outcome("t_001", 0, 1)], + "typescript", + "guidelines", + "test-model", + None, + ); + + assert!(report.contains("Model output unavailable (1 task)")); + assert!(report.contains("Infrastructure/provider problem")); + assert!(report.contains("`t_001`")); + assert!(!report.contains("No failures detected.")); + } + + #[test] + fn failed_request_without_output_is_added_to_model_analysis() { + let mut generated_failure = outcome("t_001", 0, 1); + generated_failure.llm_output = Some("generated source".to_string()); + let report = build_report( + &[generated_failure, outcome("t_002", 0, 1)], + "typescript", + "guidelines", + "test-model", + Some("## Recommended actions\n\nNo repository changes recommended.\n\n## Failure patterns\n\n### Invalid API"), + ); + + assert!(report.contains("### Invalid API")); + assert!(report.contains("Model output unavailable (1 task)")); + assert!(report.contains("`t_002`")); + } } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index c4394ff97a4..f993f80c883 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -543,11 +543,7 @@ fn write_analysis_report( async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutcome]) -> Result> { let configured_path = configured_analysis_path(cfg.lang.as_str(), cfg.mode, cfg.route); - let should_run = if cfg.dry_run { - cfg.local_analysis - } else { - cfg.api_client.is_some() || configured_path.is_some() - }; + let should_run = cfg.local_analysis || configured_path.is_some() || (!cfg.dry_run && cfg.api_client.is_some()); if !should_run { return Ok(None); From 3e5666f09aa546273584cc1ea81ea07adc8feef6 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:23:00 -0400 Subject: [PATCH 78/78] Allow benchmark runs to skip task catalog upload --- .github/workflows/llm-benchmark-periodic.yml | 10 ++++++++++ .../src/bin/llm_benchmark.rs | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 2626bb84573..bdf50c0fae5 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -39,6 +39,11 @@ on: description: 'Run benchmarks without uploading results' required: false default: 'false' + skip_task_catalog_upload: + description: 'Skip uploading the benchmark task catalog' + required: false + type: boolean + default: false post_discord: description: 'Post the analysis summary to Discord' required: false @@ -158,6 +163,7 @@ jobs: INPUT_CATEGORIES: ${{ inputs.categories || '' }} INPUT_TASKS: ${{ inputs.tasks || '' }} INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }} + INPUT_SKIP_TASK_CATALOG_UPLOAD: ${{ inputs.skip_task_catalog_upload || 'false' }} run: | LANG="$INPUT_LANGUAGE" MODEL_SET="$INPUT_MODEL_SET" @@ -166,6 +172,7 @@ jobs: CATEGORIES="$INPUT_CATEGORIES" TASKS="$INPUT_TASKS" DRY_RUN="$INPUT_DRY_RUN" + SKIP_TASK_CATALOG_UPLOAD="$INPUT_SKIP_TASK_CATALOG_UPLOAD" case "$MODEL_SET" in website_active) @@ -203,6 +210,9 @@ jobs: if [ "$DRY_RUN" = "true" ]; then EXTRA_ARGS+=(--dry-run) fi + if [ "$SKIP_TASK_CATALOG_UPLOAD" = "true" ]; then + EXTRA_ARGS+=(--skip-task-catalog-upload) + fi if [ "$MODEL_SET" = "website_active" ]; then llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}" diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 84be5faaed5..fae506b4813 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -145,6 +145,10 @@ struct RunArgs { #[arg(long)] dry_run: bool, + /// Skip uploading the benchmark task catalog + #[arg(long)] + skip_task_catalog_upload: bool, + /// When used with --dry-run, also generate local markdown analysis files #[arg(long, requires = "dry_run")] local_analysis: bool, @@ -287,7 +291,8 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { let bench_root = find_bench_root(); // Upload task catalog before running benchmarks - if let Some(ref api) = upload_client + if !args.skip_task_catalog_upload + && let Some(ref api) = upload_client && let Err(e) = api.upload_task_catalog(&bench_root) { eprintln!("[warn] failed to upload task catalog: {e}"); @@ -970,6 +975,7 @@ mod tests { models: None, model_source: ModelSource::Static, dry_run: false, + skip_task_catalog_upload: false, local_analysis: false, route_overrides: None, } @@ -1004,6 +1010,15 @@ mod tests { assert_eq!(overridden.reasoning, ReasoningEffort::High); } + #[test] + fn task_catalog_upload_can_be_skipped() { + let cli = Cli::try_parse_from(["llm", "run", "--hash-only", "--skip-task-catalog-upload"]).unwrap(); + let Commands::Run(args) = cli.command else { + panic!("expected run command"); + }; + assert!(args.skip_task_catalog_upload); + } + #[test] fn explicit_models_bypass_remote_model_source() { let mut args = base_run_args();