From f1a20a9e9b9e54ac0027e483111704e7bc648f10 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Sun, 12 Jul 2026 14:15:01 +0530 Subject: [PATCH 1/6] migrations in dst cleanup proptest like gen more cleanup cleanups in migrations consistent choices remove ensure_ helpers common new table code redability better error messages unreviewed --- .../src/locking_tx_datastore/replay.rs | 2 +- crates/dst/Cargo.toml | 1 + crates/dst/src/engine.rs | 366 ++++++- crates/dst/src/engine/generation.rs | 394 ++++++++ crates/dst/src/engine/migrations.rs | 906 ++++++++++++++++++ crates/dst/src/engine/model.rs | 434 ++++++++- crates/dst/src/engine/properties.rs | 63 +- crates/dst/src/engine/row.rs | 13 + crates/dst/src/engine/state.rs | 237 +++++ crates/dst/src/engine/workload.rs | 260 +++-- crates/dst/src/lib.rs | 1 + crates/dst/src/rng.rs | 62 ++ crates/dst/src/schema.rs | 638 ++++++++---- crates/dst/src/traits.rs | 56 +- 14 files changed, 3039 insertions(+), 394 deletions(-) create mode 100644 crates/dst/src/engine/generation.rs create mode 100644 crates/dst/src/engine/migrations.rs create mode 100644 crates/dst/src/engine/row.rs create mode 100644 crates/dst/src/engine/state.rs create mode 100644 crates/dst/src/rng.rs diff --git a/crates/datastore/src/locking_tx_datastore/replay.rs b/crates/datastore/src/locking_tx_datastore/replay.rs index d803360551d..d439553c01f 100644 --- a/crates/datastore/src/locking_tx_datastore/replay.rs +++ b/crates/datastore/src/locking_tx_datastore/replay.rs @@ -1085,7 +1085,7 @@ impl StateView for ReplayCommittedState<'_> { /// then falling back to [`CommittedState::iter_by_col_eq`]. fn find_st_table_row(&self, table_id: TableId) -> Result { if let Some(row_ptr) = self.replay_table_updated.get(&table_id) { - let (table, blob_store, _) = self.state.get_table_and_blob_store(table_id)?; + let (table, blob_store, _) = self.state.get_table_and_blob_store(ST_TABLE_ID)?; // SAFETY: `row_ptr` is stored in `self.replay_table_updated`, // meaning it was inserted into `st_table` by `replay_insert` // and has not yet been deleted by `replay_delete_by_rel`. diff --git a/crates/dst/Cargo.toml b/crates/dst/Cargo.toml index 7d47c9a1b47..0fa65aa24d7 100644 --- a/crates/dst/Cargo.toml +++ b/crates/dst/Cargo.toml @@ -11,6 +11,7 @@ fallocate = ["spacetimedb-commitlog/fallocate"] [dependencies] anyhow.workspace = true clap.workspace = true +futures.workspace = true spacetimedb-datastore.workspace = true spacetimedb-commitlog.workspace = true spacetimedb-durability.workspace = true diff --git a/crates/dst/src/engine.rs b/crates/dst/src/engine.rs index 451b7417e41..ff90cd0a233 100644 --- a/crates/dst/src/engine.rs +++ b/crates/dst/src/engine.rs @@ -6,22 +6,31 @@ use spacetimedb_datastore::traits::{IsolationLevel, TxData}; use spacetimedb_engine::error::{DBError, DatastoreError, IndexError}; use spacetimedb_engine::persistence::{DiskSizeFn, Durability as EngineDurability, Persistence}; use spacetimedb_engine::relational_db::{MutTx, RelationalDB}; +use spacetimedb_engine::update::{update_database, UpdateLogger}; +use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::{Identity, RawModuleDef}; use spacetimedb_primitives::TableId; use spacetimedb_runtime::sim::{Rng, Runtime as SimRuntime}; use spacetimedb_runtime::Handle; +use spacetimedb_schema::auto_migrate::{ponder_migrate, MigratePlan}; use spacetimedb_schema::def::ModuleDef; use spacetimedb_schema::schema::{Schema, TableSchema}; use spacetimedb_table::page_pool::PagePool; +mod generation; +mod migrations; mod model; mod properties; +mod row; +mod state; mod workload; -use self::workload::{ - normalize_rows, row_to_bytes, CommitDelta, CountState, InsertOutcome, Interaction, Observation, TableDelta, - TableRowCount, +use self::migrations::{Migration, MigrationExpectation, MigrationRejection}; +use self::row::{normalize_rows, row_to_bytes}; +use self::state::{ + table_schema_state_for_schema, CommitDelta, CountState, SchemaState, TableDelta, TableRowCount, TableRows, }; +use self::workload::{InsertOutcome, Interaction, Observation}; use crate::engine::model::Model; use crate::engine::properties::EngineProperties; @@ -36,6 +45,7 @@ pub struct EngineTarget { active_mut_tx: Option, commitlog: InMemoryCommitlog, runtime_handle: Handle, + schema: SchemaPlan, } impl EngineTarget { @@ -54,6 +64,7 @@ impl EngineTarget { active_mut_tx: None, commitlog, runtime_handle, + schema, }) } @@ -88,11 +99,14 @@ impl EngineTarget { } } - fn install_schema(db: &RelationalDB, schema: &SchemaPlan) -> anyhow::Result<()> { + fn module_def(schema: &SchemaPlan) -> anyhow::Result { let raw = to_raw_def(schema); let raw_module_def = RawModuleDef::V10(raw); - let module_def = - ModuleDef::try_from(raw_module_def).map_err(|e| anyhow::anyhow!("schema validation failed: {e}"))?; + ModuleDef::try_from(raw_module_def).map_err(|e| anyhow::anyhow!("schema validation failed: {e}")) + } + + fn install_schema(db: &RelationalDB, schema: &SchemaPlan) -> anyhow::Result<()> { + let module_def = Self::module_def(schema)?; db.with_auto_commit(Workload::Internal, |tx| -> Result<(), DBError> { for table_def in module_def.tables() { @@ -138,27 +152,46 @@ impl EngineTarget { .ok_or_else(|| anyhow::anyhow!("database is not open"))?; let tx = db.begin_tx(Workload::Internal); let mut row_counts = Vec::with_capacity(self.table_ids.len()); + let mut table_rows = Vec::with_capacity(self.table_ids.len()); + let mut schema_tables = Vec::with_capacity(self.table_ids.len()); for (table, table_id) in self.table_ids.iter().enumerate() { - let count = match db.iter(&tx, *table_id) { - Ok(iter) => iter.count() as u64, + let rows = match db.iter(&tx, *table_id) { + Ok(iter) => normalize_rows(iter.map(|row| row.to_product_value()).collect()), Err(err) => { let _ = db.release_tx(tx); return Err(err.into()); } }; + let count = rows.len() as u64; row_counts.push(TableRowCount { table, count }); + table_rows.push(TableRows { table, rows }); + + let schema = match db.schema_for_table(&tx, *table_id) { + Ok(schema) => schema, + Err(err) => { + let _ = db.release_tx(tx); + return Err(err.into()); + } + }; + schema_tables.push(table_schema_state_for_schema(table, &schema)); } let _ = db.release_tx(tx); - Ok(CountState { row_counts }) + Ok(CountState { + row_counts, + table_rows, + schema: SchemaState { tables: schema_tables }, + }) } - fn is_unique_constraint_violation(error: &DBError) -> bool { - matches!( - error, - DBError::Datastore(DatastoreError::Index(IndexError::UniqueConstraintViolation(_))) - ) + fn unique_constraint_violation_details(error: &DBError) -> Option { + match error { + DBError::Datastore(DatastoreError::Index(IndexError::UniqueConstraintViolation(violation))) => { + Some(violation.to_string()) + } + _ => None, + } } fn commit_delta_from_tx_data(&self, tx_data: &TxData) -> CommitDelta { @@ -187,6 +220,99 @@ impl EngineTarget { CommitDelta { tables } } + fn migrate(&mut self, migration: &Migration) -> anyhow::Result { + anyhow::ensure!( + self.active_mut_tx.is_none(), + "migration while mutable transaction is active" + ); + anyhow::ensure!( + migration.schema() != &self.schema, + "engine DST generated a no-op migration" + ); + + let old_module_def = Self::module_def(&self.schema)?; + let new_module_def = match Self::module_def(migration.schema()) { + Ok(module_def) => module_def, + Err(error) => { + return match migration.expectation() { + MigrationExpectation::Rejected(MigrationRejection::InvalidDefinition) => { + Ok(Observation::MigrationRejected { + reason: MigrationRejection::InvalidDefinition, + }) + } + _ => Err(error), + }; + } + }; + + let plan = match ponder_migrate(&old_module_def, &new_module_def) { + Ok(plan) => plan, + Err(error) => { + return match migration.expectation() { + MigrationExpectation::Rejected(MigrationRejection::InvalidDefinition) => { + Ok(Observation::MigrationRejected { + reason: MigrationRejection::InvalidDefinition, + }) + } + _ => Err(error.into()), + }; + } + }; + + match migration.expectation() { + MigrationExpectation::Accepted => { + ensure_auto_plan(&plan)?; + self.apply_auto_migration(migration, plan)?; + Ok(Observation::Migrated) + } + MigrationExpectation::Rejected(MigrationRejection::ManualRequired) => match plan { + MigratePlan::Manual(_) => Ok(Observation::MigrationRejected { + reason: MigrationRejection::ManualRequired, + }), + MigratePlan::Auto(_) => { + anyhow::bail!("engine unexpectedly auto-accepted migration expected to require manual migration") + } + }, + MigrationExpectation::Rejected(MigrationRejection::PrecheckFailure) => { + ensure_auto_plan(&plan)?; + self.expect_precheck_failure(plan) + } + MigrationExpectation::Rejected(MigrationRejection::InvalidDefinition) => { + anyhow::bail!("engine produced a migration plan for a schema expected to be invalid") + } + } + } + + fn apply_auto_migration(&mut self, migration: &Migration, plan: MigratePlan<'_>) -> anyhow::Result<()> { + let db = self + .db + .as_ref() + .ok_or_else(|| anyhow::anyhow!("database is not open"))?; + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let _ = update_database(db, &mut tx, AuthCtx::for_testing(), plan, &DstUpdateLogger)?; + let Some((_tx_offset, _tx_data, _tx_metrics, _reducer)) = db.commit_tx(tx)? else { + anyhow::bail!("migration commit produced no transaction data"); + }; + + self.schema = migration.schema().clone(); + self.table_ids = Self::load_table_ids(db, &self.schema)?; + Ok(()) + } + + fn expect_precheck_failure(&self, plan: MigratePlan<'_>) -> anyhow::Result { + let db = self + .db + .as_ref() + .ok_or_else(|| anyhow::anyhow!("database is not open"))?; + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + match update_database(db, &mut tx, AuthCtx::for_testing(), plan, &DstUpdateLogger) { + Ok(_) => anyhow::bail!("engine applied migration expected to fail precheck"), + Err(_) => Ok(Observation::MigrationRejected { + reason: MigrationRejection::PrecheckFailure, + }), + } + } + pub fn execute(&mut self, interaction: &Interaction) -> anyhow::Result { tracing::debug!(?interaction, "executing interaction"); @@ -216,11 +342,14 @@ impl EngineTarget { .ok_or_else(|| anyhow::anyhow!("insert without active mutable transaction"))?; let outcome = match db.insert(tx, table_id, &bytes) { Ok((_generated_columns, row, _flags)) => InsertOutcome::Accepted(row.to_product_value()), - // Generated rows can intentionally hit unique constraints; the oracle validates that rejection. - Err(error) if Self::is_unique_constraint_violation(&error) => { - InsertOutcome::UniqueConstraintViolation + Err(error) => { + // Generated rows can intentionally hit unique constraints; the oracle validates that rejection. + if let Some(details) = Self::unique_constraint_violation_details(&error) { + InsertOutcome::UniqueConstraintViolation { details } + } else { + return Err(error.into()); + } } - Err(error) => return Err(error.into()), }; Ok(Observation::Inserted { outcome }) } @@ -253,6 +382,7 @@ impl EngineTarget { delta: self.commit_delta_from_tx_data(&tx_data), }) } + Interaction::Migrate(migration) => self.migrate(migration), Interaction::Replay => { let _ = self.active_mut_tx.take(); self.reopen_from_commitlog()?; @@ -271,6 +401,21 @@ impl EngineTarget { } } +fn ensure_auto_plan(plan: &MigratePlan<'_>) -> anyhow::Result<()> { + let MigratePlan::Auto(_) = plan else { + anyhow::bail!("engine DST generated a manual migration plan"); + }; + Ok(()) +} + +struct DstUpdateLogger; + +impl UpdateLogger for DstUpdateLogger { + fn info(&self, msg: &str) { + tracing::debug!(%msg, "engine DST migration update"); + } +} + impl TargetDriver for EngineTarget { type Observation = Observation; @@ -302,3 +447,188 @@ impl TestSuite for EngineTest { Ok((interactions, target, properties)) } } + +#[cfg(test)] +mod tests { + use spacetimedb_lib::AlgebraicValue; + use spacetimedb_runtime::sim::Runtime as SimRuntime; + use spacetimedb_sats::product; + + use super::migrations::{Migration, SchemaRewrite, TableMigrationOp}; + use super::*; + use crate::schema::{ColumnPlan, IndexAlgorithm, IndexPlan, TablePlan, Type, UniqueConstraintPlan}; + + fn migration_replay_schema() -> SchemaPlan { + SchemaPlan { + tables: vec![TablePlan { + name: "items".into(), + columns: vec![ + ColumnPlan { + name: "id".into(), + ty: Type::U64, + }, + ColumnPlan { + name: "kind".into(), + ty: Type::Sum { variants: 1 }, + }, + ], + primary_key: Some(0), + indexes: vec![IndexPlan { + columns: vec![0], + algorithm: IndexAlgorithm::BTree, + }], + unique_constraints: vec![UniqueConstraintPlan { columns: vec![0] }], + sequences: vec![], + is_public: true, + is_event: false, + }], + } + } + + fn add_column_replay_schema() -> SchemaPlan { + SchemaPlan { + tables: vec![TablePlan { + name: "items".into(), + columns: vec![ + ColumnPlan { + name: "id".into(), + ty: Type::U64, + }, + ColumnPlan { + name: "score".into(), + ty: Type::U64, + }, + ], + primary_key: Some(0), + indexes: vec![IndexPlan { + columns: vec![0], + algorithm: IndexAlgorithm::BTree, + }], + unique_constraints: vec![UniqueConstraintPlan { columns: vec![0] }], + sequences: vec![], + is_public: true, + is_event: false, + }], + } + } + + fn change_index_replay_schema() -> SchemaPlan { + SchemaPlan { + tables: vec![TablePlan { + name: "items".into(), + columns: vec![ + ColumnPlan { + name: "id".into(), + ty: Type::U64, + }, + ColumnPlan { + name: "score".into(), + ty: Type::U64, + }, + ], + primary_key: Some(0), + indexes: vec![ + IndexPlan { + columns: vec![0], + algorithm: IndexAlgorithm::BTree, + }, + IndexPlan { + columns: vec![1], + algorithm: IndexAlgorithm::BTree, + }, + ], + unique_constraints: vec![UniqueConstraintPlan { columns: vec![0] }], + sequences: vec![], + is_public: true, + is_event: false, + }], + } + } + + fn insert_u64_rows(target: &mut EngineTarget) -> anyhow::Result<()> { + target.execute(&Interaction::BeginMutTx)?; + for id in 0..128u64 { + target.execute(&Interaction::Insert { + table: 0, + row: product![id, id * 10], + })?; + } + target.execute(&Interaction::CommitTx)?; + Ok(()) + } + + #[test] + fn engine_dst_smoke_runs_random_workload() -> anyhow::Result<()> { + let mut runtime = SimRuntime::new(0); + runtime.block_on(EngineTest.run(Rng::new(0), 1_000))?; + Ok(()) + } + + #[test] + fn add_column_migration_replays_with_existing_rows() -> anyhow::Result<()> { + let mut target = EngineTarget::init(add_column_replay_schema(), 0)?; + insert_u64_rows(&mut target)?; + + target.execute(&Interaction::Migrate(Migration::from_rewrites( + &target.schema, + vec![SchemaRewrite::AlterTable { + table: "items".into(), + ops: vec![ + TableMigrationOp::ChangeAccess, + TableMigrationOp::AddColumn { ty: Type::U64 }, + ], + }], + )?))?; + target.execute(&Interaction::Replay)?; + + Ok(()) + } + + #[test] + fn change_index_migration_replays_with_existing_rows() -> anyhow::Result<()> { + let mut target = EngineTarget::init(change_index_replay_schema(), 0)?; + insert_u64_rows(&mut target)?; + + target.execute(&Interaction::Migrate(Migration::from_rewrites( + &target.schema, + vec![SchemaRewrite::AlterTable { + table: "items".into(), + ops: vec![ + TableMigrationOp::ChangeAccess, + TableMigrationOp::ChangeIndex { columns: vec![1] }, + ], + }], + )?))?; + target.execute(&Interaction::Replay)?; + + Ok(()) + } + + #[test] + fn migration_that_updates_st_table_and_st_column_replays() -> anyhow::Result<()> { + let mut target = EngineTarget::init(migration_replay_schema(), 0)?; + + target.execute(&Interaction::BeginMutTx)?; + for id in 0..128u64 { + target.execute(&Interaction::Insert { + table: 0, + row: product![id, AlgebraicValue::sum(0, AlgebraicValue::U8(1))], + })?; + } + target.execute(&Interaction::CommitTx)?; + + target.execute(&Interaction::Migrate(Migration::from_rewrites( + &target.schema, + vec![SchemaRewrite::AlterTable { + table: "items".into(), + ops: vec![ + TableMigrationOp::ChangeAccess, + TableMigrationOp::ChangeColumnType { column: 1 }, + ], + }], + )?))?; + target.execute(&Interaction::Replay)?; + + Ok(()) + } +} diff --git a/crates/dst/src/engine/generation.rs b/crates/dst/src/engine/generation.rs new file mode 100644 index 00000000000..899b048ecff --- /dev/null +++ b/crates/dst/src/engine/generation.rs @@ -0,0 +1,394 @@ +//! Workload value and migration generation for the engine DST driver. + +use spacetimedb_lib::{AlgebraicValue, ProductValue}; +use spacetimedb_runtime::sim::Rng; +use spacetimedb_sats::ArrayValue; + +use super::migrations::{Migration, MigrationMode}; +use super::model::{ColumnDomain, Model}; +use super::row::Row; +use crate::rng::{choice, Choice, WeightedChoice}; +use crate::schema::Type; + +/// Read-only generation context for one model state. +pub(crate) struct GenCtx<'a> { + rng: &'a Rng, + model: &'a Model, +} + +impl<'a> GenCtx<'a> { + pub(crate) fn new(rng: &'a Rng, model: &'a Model) -> Self { + Self { rng, model } + } + + pub(crate) fn gen_insert_row(&self, table: usize) -> Row { + ValueGen::new(self.rng, self.model).gen_insert_row(table) + } + + pub(crate) fn gen_migration(&self) -> Option { + MigrationGen::new(self.rng, self.model).choose() + } +} + +#[derive(Clone, Copy)] +enum ValueCase { + Random, + Small, + Edge, + NearExisting, + Existing, + Weird, +} + +impl WeightedChoice for ValueCase { + const CHOICES: &'static [Choice] = &[ + choice(45, Self::Random), + choice(15, Self::Small), + choice(15, Self::Edge), + choice(10, Self::NearExisting), + choice(10, Self::Existing), + choice(5, Self::Weird), + ]; +} + +#[derive(Clone, Copy)] +enum FreshValueCase { + Random, + Small, + Edge, + Weird, +} + +impl WeightedChoice for FreshValueCase { + const CHOICES: &'static [Choice] = &[ + choice(50, Self::Random), + choice(20, Self::Small), + choice(20, Self::Edge), + choice(10, Self::Weird), + ]; +} + +#[derive(Clone, Copy)] +enum I64Case { + Random, + Small, + Edge, +} + +#[derive(Clone, Copy)] +enum U64Case { + Random, + Small, + Edge, +} + +#[derive(Clone, Copy)] +enum StringCase { + RandomTagged, + Empty, + SmallAscii, + OrderedPrefix, + SqlEscaped, + NullByte, + Long, +} + +impl StringCase { + const WEIRD_CHOICES: &'static [Choice] = &[ + choice(35, Self::SqlEscaped), + choice(25, Self::NullByte), + choice(25, Self::OrderedPrefix), + choice(15, Self::Empty), + ]; + + fn pick_weird(rng: &Rng) -> Self { + crate::rng::pick_choice(rng, Self::WEIRD_CHOICES) + } +} + +#[derive(Clone, Copy)] +enum BytesCase { + Random, + Empty, + Small, + RepeatedZero, + RepeatedMax, + Alternating, +} + +impl BytesCase { + const WEIRD_CHOICES: &'static [Choice] = &[ + choice(25, Self::Empty), + choice(20, Self::RepeatedZero), + choice(20, Self::RepeatedMax), + choice(20, Self::Alternating), + choice(15, Self::Small), + ]; + + fn pick_weird(rng: &Rng) -> Self { + crate::rng::pick_choice(rng, Self::WEIRD_CHOICES) + } +} + +struct ValueGen<'a> { + rng: &'a Rng, + model: &'a Model, +} + +impl<'a> ValueGen<'a> { + fn new(rng: &'a Rng, model: &'a Model) -> Self { + Self { rng, model } + } + + fn gen_insert_row(&self, table: usize) -> Row { + self.model.schema().tables[table] + .columns + .iter() + .enumerate() + .map(|(column, _)| { + let domain = self.model.column_domain(table, column); + self.gen_insert_value(&domain) + }) + .collect::() + } + + fn gen_insert_value(&self, domain: &ColumnDomain) -> AlgebraicValue { + if domain.sequenced { + return sequence_placeholder(domain.ty); + } + + if domain.unique { + return self.gen_fresh_value(domain); + } + + self.gen_value(domain) + } + + fn gen_value(&self, domain: &ColumnDomain) -> AlgebraicValue { + match ValueCase::pick(self.rng) { + ValueCase::Random => self.gen_random_value(domain.ty), + ValueCase::Small => self.gen_small_value(domain.ty), + ValueCase::Edge => self.gen_edge_value(domain.ty), + ValueCase::NearExisting => self + .near_existing_value(domain) + .unwrap_or_else(|| self.gen_random_value(domain.ty)), + ValueCase::Existing => self + .existing_value(domain) + .unwrap_or_else(|| self.gen_random_value(domain.ty)), + ValueCase::Weird => self.gen_weird_value(domain.ty), + } + } + + fn gen_fresh_value(&self, domain: &ColumnDomain) -> AlgebraicValue { + for _ in 0..32 { + let value = self.gen_fresh_candidate(domain.ty); + if !domain.values.contains(&value) { + return value; + } + } + + self.gen_counter_value(domain.ty, self.rng.next_u64()) + } + + fn gen_fresh_candidate(&self, ty: Type) -> AlgebraicValue { + match FreshValueCase::pick(self.rng) { + FreshValueCase::Random => self.gen_random_value(ty), + FreshValueCase::Small => self.gen_small_value(ty), + FreshValueCase::Edge => self.gen_edge_value(ty), + FreshValueCase::Weird => self.gen_weird_value(ty), + } + } + + fn existing_value(&self, domain: &ColumnDomain) -> Option { + (!domain.values.is_empty()).then(|| domain.values[self.rng.index(domain.values.len())].clone()) + } + + fn near_existing_value(&self, domain: &ColumnDomain) -> Option { + let existing = self.existing_value(domain)?; + Some(match (domain.ty, existing) { + (Type::Bool, AlgebraicValue::Bool(value)) => AlgebraicValue::Bool(!value), + (Type::I64, AlgebraicValue::I64(value)) => AlgebraicValue::I64(value.saturating_add(1)), + (Type::U64, AlgebraicValue::U64(value)) => AlgebraicValue::U64(value.saturating_add(1)), + (Type::String, AlgebraicValue::String(value)) => { + let mut value = value.to_string(); + value.push('a'); + AlgebraicValue::String(value.into()) + } + (Type::Bytes, AlgebraicValue::Array(ArrayValue::U8(value))) => { + let mut value = value.to_vec(); + value.push(0); + AlgebraicValue::Array(ArrayValue::U8(value.into())) + } + (Type::Sum { .. }, value) => value, + (_, value) => value, + }) + } + + fn gen_random_value(&self, ty: Type) -> AlgebraicValue { + match ty { + Type::Bool => AlgebraicValue::Bool(self.rng.next_u64().is_multiple_of(2)), + Type::I64 => AlgebraicValue::I64(self.gen_i64_value(I64Case::Random)), + Type::U64 => AlgebraicValue::U64(self.gen_u64_value(U64Case::Random)), + Type::String => AlgebraicValue::String(self.gen_string_value(StringCase::RandomTagged).into()), + Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(BytesCase::Random).into())), + Type::Sum { variants } => { + let tag = self.rng.index(variants as usize) as u8; + AlgebraicValue::sum(tag, AlgebraicValue::U8(self.rng.next_u64() as u8)) + } + } + } + + fn gen_small_value(&self, ty: Type) -> AlgebraicValue { + match ty { + Type::Bool => AlgebraicValue::Bool(false), + Type::I64 => AlgebraicValue::I64(self.gen_i64_value(I64Case::Small)), + Type::U64 => AlgebraicValue::U64(self.gen_u64_value(U64Case::Small)), + Type::String => AlgebraicValue::String(self.gen_string_value(StringCase::SmallAscii).into()), + Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(BytesCase::Small).into())), + Type::Sum { variants } => { + let tag = if variants <= 1 { + 0 + } else { + self.rng.index(variants as usize) as u8 + }; + AlgebraicValue::sum(tag, AlgebraicValue::U8(0)) + } + } + } + + fn gen_edge_value(&self, ty: Type) -> AlgebraicValue { + match ty { + Type::Bool => AlgebraicValue::Bool(true), + Type::I64 => AlgebraicValue::I64(self.gen_i64_value(I64Case::Edge)), + Type::U64 => AlgebraicValue::U64(self.gen_u64_value(U64Case::Edge)), + Type::String => AlgebraicValue::String(self.gen_string_value(StringCase::Long).into()), + Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(BytesCase::RepeatedMax).into())), + Type::Sum { variants } => AlgebraicValue::sum(variants.saturating_sub(1), AlgebraicValue::U8(u8::MAX)), + } + } + + fn gen_weird_value(&self, ty: Type) -> AlgebraicValue { + match ty { + Type::String => { + let case = StringCase::pick_weird(self.rng); + AlgebraicValue::String(self.gen_string_value(case).into()) + } + Type::Bytes => { + let case = BytesCase::pick_weird(self.rng); + AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(case).into())) + } + _ => self.gen_edge_value(ty), + } + } + + fn gen_i64_value(&self, case: I64Case) -> i64 { + match case { + I64Case::Random => self.rng.next_u64() as i64, + I64Case::Small => self.sample(&[-3, -2, -1, 0, 1, 2, 3]), + I64Case::Edge => self.sample(&[i64::MIN, i64::MIN + 1, -1, 0, 1, i64::MAX - 1, i64::MAX]), + } + } + + fn gen_u64_value(&self, case: U64Case) -> u64 { + match case { + U64Case::Random => self.rng.next_u64(), + U64Case::Small => self.sample(&[0, 1, 2, 3, 4, 5]), + U64Case::Edge => self.sample(&[0, 1, 2, u64::MAX - 1, u64::MAX]), + } + } + + fn gen_string_value(&self, case: StringCase) -> String { + match case { + StringCase::RandomTagged => format!("v_{}", self.rng.next_u64()), + StringCase::Empty => String::new(), + StringCase::SmallAscii => self.sample(&["a", "aa", "ab", "b", "z", "v_0", "v_1"]).to_owned(), + StringCase::OrderedPrefix => self.sample(&["a", "aa", "aaa", "ab", "aba", "abb", "b"]).to_owned(), + StringCase::SqlEscaped => self + .sample(&["quote'", "double\"quote", "back\\slash", "line\nbreak"]) + .to_owned(), + StringCase::NullByte => "nul\0byte".to_owned(), + StringCase::Long => "x".repeat(128), + } + } + + fn gen_bytes_value(&self, case: BytesCase) -> Vec { + match case { + BytesCase::Random => { + let len = (self.rng.next_u64() % 16) as usize; + (0..len).map(|_| self.rng.next_u64() as u8).collect() + } + BytesCase::Empty => Vec::new(), + BytesCase::Small => self.sample(&[&[][..], &[0][..], &[1][..], &[0, 255][..]]).to_vec(), + BytesCase::RepeatedZero => vec![0; 32], + BytesCase::RepeatedMax => vec![255; 32], + BytesCase::Alternating => vec![0, 255, 0, 255, 0, 255], + } + } + + fn gen_counter_value(&self, ty: Type, counter: u64) -> AlgebraicValue { + match ty { + Type::Bool => AlgebraicValue::Bool(counter.is_multiple_of(2)), + Type::I64 => AlgebraicValue::I64(counter as i64), + Type::U64 => AlgebraicValue::U64(counter), + Type::String => AlgebraicValue::String(format!("fresh_{counter}").into()), + Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(counter.to_le_bytes().to_vec().into())), + Type::Sum { variants } => AlgebraicValue::sum( + if variants == 0 { + 0 + } else { + (counter % variants as u64) as u8 + }, + AlgebraicValue::U8(counter as u8), + ), + } + } + + fn sample(&self, values: &[T]) -> T { + values[self.rng.index(values.len())] + } +} + +fn sequence_placeholder(ty: Type) -> AlgebraicValue { + match ty { + Type::I64 => AlgebraicValue::I64(0), + Type::U64 => AlgebraicValue::U64(0), + _ => unreachable!("sequence columns are integral"), + } +} + +struct MigrationGen<'a> { + rng: &'a Rng, + model: &'a Model, +} + +impl<'a> MigrationGen<'a> { + fn new(rng: &'a Rng, model: &'a Model) -> Self { + Self { rng, model } + } + + fn choose(&self) -> Option { + match MigrationMode::pick(self.rng) { + MigrationMode::Accepted => self.choose_accepted(), + MigrationMode::Rejected => { + Migration::choose_rejected(self.rng, self.model.schema(), self.model).or_else(|| self.choose_accepted()) + } + } + } + + fn choose_accepted(&self) -> Option { + let original = self.model.schema(); + let mut schema = original.clone(); + let steps = 1 + self.rng.index(10); + + for _ in 0..steps { + let Some(rewrite) = Migration::choose_rewrite(self.rng, &schema, self.model) else { + break; + }; + rewrite + .apply_to(&mut schema) + .expect("generated rewrite must be valid for the draft schema"); + } + + (schema != *original).then(|| Migration::from_schema(schema)) + } +} diff --git a/crates/dst/src/engine/migrations.rs b/crates/dst/src/engine/migrations.rs new file mode 100644 index 00000000000..f3bb76b9339 --- /dev/null +++ b/crates/dst/src/engine/migrations.rs @@ -0,0 +1,906 @@ +//! Schema migration candidates for the engine DST driver. +//! +//! This module generates schema changes with an explicit expected outcome. +//! Accepted migrations exercise engine auto-migration. Rejected migrations are +//! boundary probes where the engine should refuse the change for a known reason. + +use super::model::{ColumnDomain, Model}; +use crate::rng::{choice, choose_index, Choice, WeightedChoice}; +use crate::schema::{ + ColumnPlan, IndexAlgorithm, IndexPlan, SchemaGenerator, SchemaNames, SchemaPlan, SchemaProfile, SequencePlan, + TablePlan, Type, UniqueConstraintPlan, +}; +use spacetimedb_runtime::sim::Rng; + +const MAX_SUM_VARIANTS: u8 = 32; +const MAX_EVENT_COLUMNS: usize = 32; +const MAX_TABLE_COLUMNS: usize = 32; +const MAX_TABLES: usize = 128; + +/// A fully materialized target schema for one migration interaction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Migration { + schema: SchemaPlan, + expectation: MigrationExpectation, +} + +/// Whether the target schema should be accepted or refused by the engine. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MigrationExpectation { + Accepted, + Rejected(MigrationRejection), +} + +/// The specific boundary a rejected migration is expected to hit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MigrationRejection { + ManualRequired, + PrecheckFailure, + InvalidDefinition, +} + +/// A deterministic schema edit used while building a migration candidate. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SchemaRewrite { + AddTable { table: TablePlan }, + RemoveTable { table: String }, + AlterTable { table: String, ops: Vec }, +} + +/// Table-local migration operations generated by the DST harness. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TableMigrationOp { + ChangeAccess, + AddColumn { + ty: Type, + }, + AddIndex { + columns: Vec, + algorithm: IndexAlgorithm, + }, + RemoveIndex { + columns: Vec, + }, + AddSequence { + sequence: SequencePlan, + }, + RemoveSequence { + column: usize, + }, + AddUniqueConstraint { + columns: Vec, + }, + RemoveUniqueConstraint { + columns: Vec, + }, + ChangePrimaryKey { + column: Option, + }, + ChangeIndex { + columns: Vec, + }, + ChangeColumnType { + column: usize, + }, + ReschemaEventTable, +} + +/// Top-level migration outcome bucket. Keep this gate separate so adding more +/// rejected rules does not silently increase total rejection frequency. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MigrationMode { + Accepted, + Rejected, +} + +impl WeightedChoice for MigrationMode { + const CHOICES: &'static [Choice] = &[choice(95, Self::Accepted), choice(5, Self::Rejected)]; +} + +/// Weighted categories of auto-migration surfaces to probe. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MigrationChoice { + AddTable, + RemoveTable, + AddColumn, + AddIndex, + RemoveIndex, + ChangeIndex, + AddSequence, + RemoveSequence, + AddUniqueConstraint, + RemoveUniqueConstraint, + DropPrimaryKeyAndWidenSum, + WidenSumColumn, + ReschemaEventTable, +} + +impl WeightedChoice for MigrationChoice { + const CHOICES: &'static [Choice] = &[ + choice(2, Self::AddTable), + choice(1, Self::RemoveTable), + choice(14, Self::AddColumn), + choice(10, Self::AddIndex), + choice(8, Self::RemoveIndex), + choice(10, Self::ChangeIndex), + choice(18, Self::AddSequence), + choice(8, Self::RemoveSequence), + choice(12, Self::AddUniqueConstraint), + choice(8, Self::RemoveUniqueConstraint), + choice(6, Self::DropPrimaryKeyAndWidenSum), + choice(12, Self::WidenSumColumn), + choice(8, Self::ReschemaEventTable), + ]; +} + +/// Weighted rejected migration boundaries to probe. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RejectedMigrationChoice { + AddSequencePrecheckFailure, + AddI64DefaultMaxSequencePrecheckFailure, +} + +impl WeightedChoice for RejectedMigrationChoice { + const CHOICES: &'static [Choice] = &[ + choice(1, Self::AddSequencePrecheckFailure), + choice(1, Self::AddI64DefaultMaxSequencePrecheckFailure), + ]; +} + +impl Migration { + pub(crate) fn from_schema(schema: SchemaPlan) -> Self { + Self::from_schema_with_expectation(schema, MigrationExpectation::Accepted) + } + + pub(crate) fn from_schema_with_expectation(schema: SchemaPlan, expectation: MigrationExpectation) -> Self { + Self { schema, expectation } + } + + fn from_rewrite_with_expectation( + base: &SchemaPlan, + rewrite: SchemaRewrite, + expectation: MigrationExpectation, + ) -> anyhow::Result { + let mut schema = base.clone(); + rewrite.apply_to(&mut schema)?; + Ok(Self::from_schema_with_expectation(schema, expectation)) + } + + #[cfg(test)] + pub(crate) fn from_rewrites(base: &SchemaPlan, rewrites: Vec) -> anyhow::Result { + let mut schema = base.clone(); + for rewrite in &rewrites { + rewrite.apply_to(&mut schema)?; + } + Ok(Self::from_schema(schema)) + } + + pub(crate) fn schema(&self) -> &SchemaPlan { + &self.schema + } + + pub(crate) fn expectation(&self) -> MigrationExpectation { + self.expectation + } + + pub(crate) fn choose_rewrite(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { + for _ in 0..16 { + let choice = MigrationChoice::pick(rng); + if let Some(rewrite) = pick_rewrite(rng, Self::candidates_for(rng, schema, model, choice)) { + return Some(rewrite); + } + } + + pick_rewrite(rng, Self::candidates(rng, schema, model)) + } + + pub(crate) fn choose_rejected(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { + for _ in 0..16 { + let choice = RejectedMigrationChoice::pick(rng); + if let Some(migration) = pick_migration(Self::rejected_candidates_for(schema, model, choice), rng) { + return Some(migration); + } + } + + pick_migration(Self::rejected_candidates(schema, model), rng) + } + + pub(crate) fn candidates(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Vec { + ::CHOICES + .iter() + .flat_map(|choice| Self::candidates_for(rng, schema, model, choice.value())) + .collect() + } + + fn candidates_for(rng: &Rng, schema: &SchemaPlan, model: &Model, choice: MigrationChoice) -> Vec { + match choice { + MigrationChoice::AddTable => add_table_rewrites(rng, schema), + MigrationChoice::RemoveTable => remove_table_rewrites(schema, model), + MigrationChoice::AddColumn => add_column_rewrites(schema), + MigrationChoice::AddIndex => add_index_rewrites(schema), + MigrationChoice::RemoveIndex => remove_index_rewrites(schema), + MigrationChoice::ChangeIndex => change_index_rewrites(schema), + MigrationChoice::AddSequence => add_sequence_rewrites(schema, model), + MigrationChoice::RemoveSequence => remove_sequence_rewrites(schema), + MigrationChoice::AddUniqueConstraint => add_unique_constraint_rewrites(schema, model), + MigrationChoice::RemoveUniqueConstraint => remove_unique_constraint_rewrites(schema), + MigrationChoice::DropPrimaryKeyAndWidenSum => drop_primary_key_and_widen_sum_rewrites(schema), + MigrationChoice::WidenSumColumn => widen_sum_column_rewrites(schema), + MigrationChoice::ReschemaEventTable => reschema_event_table_rewrites(schema, model), + } + } + + fn rejected_candidates(schema: &SchemaPlan, model: &Model) -> Vec { + ::CHOICES + .iter() + .flat_map(|choice| Self::rejected_candidates_for(schema, model, choice.value())) + .collect() + } + + fn rejected_candidates_for(schema: &SchemaPlan, model: &Model, choice: RejectedMigrationChoice) -> Vec { + let expectation = MigrationExpectation::Rejected(MigrationRejection::PrecheckFailure); + let rewrites = match choice { + RejectedMigrationChoice::AddSequencePrecheckFailure => { + add_sequence_precheck_failure_rewrites(schema, model) + } + RejectedMigrationChoice::AddI64DefaultMaxSequencePrecheckFailure => { + add_i64_default_max_sequence_precheck_failure_rewrites(schema, model) + } + }; + + rewrites + .into_iter() + .filter_map(|rewrite| Self::from_rewrite_with_expectation(schema, rewrite, expectation).ok()) + .collect() + } +} + +fn pick_rewrite(rng: &Rng, mut candidates: Vec) -> Option { + let idx = choose_index(rng, candidates.len())?; + Some(candidates.swap_remove(idx)) +} + +fn pick_migration(mut candidates: Vec, rng: &Rng) -> Option { + let idx = choose_index(rng, candidates.len())?; + Some(candidates.swap_remove(idx)) +} + +fn add_table_rewrites(rng: &Rng, schema: &SchemaPlan) -> Vec { + if schema.tables.len() >= MAX_TABLES { + return Vec::new(); + } + + let generator = SchemaGenerator::new(rng.clone(), SchemaProfile::engine_dst()); + [false, true] + .into_iter() + .map(|is_event| SchemaRewrite::AddTable { + table: generator.gen_table_for_schema(schema, is_event), + }) + .collect() +} + +fn remove_table_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { + let non_event_tables = schema.tables.iter().filter(|table| !table.is_event).count(); + + schema + .tables + .iter() + .filter_map(|table| { + let row_count = model.row_count_by_table_name(&table.name); + let pristine = row_count == 0 && !model.ever_inserted_by_table_name(&table.name); + ((table.is_event && row_count == 0) || (!table.is_event && pristine && non_event_tables > 1)).then(|| { + SchemaRewrite::RemoveTable { + table: table.name.clone(), + } + }) + }) + .collect() +} + +fn add_column_rewrites(schema: &SchemaPlan) -> Vec { + let types = addable_column_types(schema); + schema + .tables + .iter() + .filter(|table| !table.is_event && table.columns.len() < MAX_TABLE_COLUMNS) + .flat_map(|table| { + types.iter().copied().map(|ty| { + SchemaRewrite::alter_table( + table, + [TableMigrationOp::ChangeAccess, TableMigrationOp::AddColumn { ty }], + ) + }) + }) + .collect() +} + +fn addable_column_types(schema: &SchemaPlan) -> Vec { + Type::ALL + .iter() + .copied() + .filter(|ty| !matches!(ty, Type::Sum { .. }) || !schema_has_sum_column(schema)) + .collect() +} + +fn schema_has_sum_column(schema: &SchemaPlan) -> bool { + schema + .tables + .iter() + .flat_map(|table| table.columns.iter()) + .any(|column| matches!(column.ty, Type::Sum { .. })) +} + +fn add_index_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event) + .flat_map(|table| { + addable_indexes(table).into_iter().map(|(columns, algorithm)| { + SchemaRewrite::alter_table(table, [TableMigrationOp::AddIndex { columns, algorithm }]) + }) + }) + .collect() +} + +fn remove_index_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event) + .flat_map(|table| { + changeable_index_columns(table) + .into_iter() + .map(|columns| SchemaRewrite::alter_table(table, [TableMigrationOp::RemoveIndex { columns }])) + }) + .collect() +} + +fn change_index_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event) + .flat_map(|table| { + changeable_index_columns(table).into_iter().map(|columns| { + SchemaRewrite::alter_table( + table, + [ + TableMigrationOp::ChangeAccess, + TableMigrationOp::ChangeIndex { columns }, + ], + ) + }) + }) + .collect() +} + +fn add_sequence_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { + let mut rewrites = Vec::new(); + + for table in schema.tables.iter().filter(|table| !table.is_event) { + let row_count = model.row_count_by_table_name(&table.name); + let pristine = row_count == 0 && !model.ever_inserted_by_table_name(&table.name); + + if pristine { + rewrites.extend( + addable_sequences(table) + .into_iter() + .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])), + ); + } + + if row_count > 0 { + rewrites.extend( + addable_sequence_boundary_probes(table, |column| column_domain(model, table, column)) + .into_iter() + .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])), + ); + } + } + + rewrites +} + +fn add_sequence_precheck_failure_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event && model.row_count_by_table_name(&table.name) > 0) + .flat_map(|table| { + sequence_precheck_failure_probes(table, |column| column_domain(model, table, column)) + .into_iter() + .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])) + }) + .collect() +} + +fn add_i64_default_max_sequence_precheck_failure_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event && model.row_count_by_table_name(&table.name) > 0) + .flat_map(|table| { + i64_default_max_sequence_precheck_failure_probes(table, |column| column_domain(model, table, column)) + .into_iter() + .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])) + }) + .collect() +} + +fn remove_sequence_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event) + .flat_map(|table| { + removable_sequence_columns(table) + .into_iter() + .map(|column| SchemaRewrite::alter_table(table, [TableMigrationOp::RemoveSequence { column }])) + }) + .collect() +} + +fn add_unique_constraint_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { + let mut rewrites = Vec::new(); + + for table in schema.tables.iter().filter(|table| !table.is_event) { + let row_count = model.row_count_by_table_name(&table.name); + let pristine = row_count == 0 && !model.ever_inserted_by_table_name(&table.name); + if !pristine { + continue; + } + + for columns in addable_unique_constraint_columns(table) { + let mut ops = Vec::new(); + if !has_index(table, &columns) { + ops.push(TableMigrationOp::AddIndex { + columns: columns.clone(), + algorithm: IndexAlgorithm::BTree, + }); + } + ops.push(TableMigrationOp::AddUniqueConstraint { columns }); + rewrites.push(SchemaRewrite::alter_table(table, ops)); + } + } + + rewrites +} + +fn remove_unique_constraint_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event) + .flat_map(|table| { + removable_unique_constraint_columns(table).into_iter().map(|columns| { + SchemaRewrite::alter_table(table, [TableMigrationOp::RemoveUniqueConstraint { columns }]) + }) + }) + .collect() +} + +fn drop_primary_key_and_widen_sum_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event && table.primary_key.is_some() && table.sequences.is_empty()) + .flat_map(|table| { + widenable_sum_columns(table).into_iter().map(|column| { + SchemaRewrite::alter_table( + table, + [ + TableMigrationOp::ChangePrimaryKey { column: None }, + TableMigrationOp::ChangeColumnType { column }, + ], + ) + }) + }) + .collect() +} + +fn widen_sum_column_rewrites(schema: &SchemaPlan) -> Vec { + schema + .tables + .iter() + .filter(|table| !table.is_event) + .flat_map(|table| { + widenable_sum_columns(table).into_iter().map(|column| { + SchemaRewrite::alter_table( + table, + [ + TableMigrationOp::ChangeAccess, + TableMigrationOp::ChangeColumnType { column }, + ], + ) + }) + }) + .collect() +} + +fn reschema_event_table_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { + schema + .tables + .iter() + .filter(|table| table.is_event && model.row_count_by_table_name(&table.name) == 0) + .filter(|table| table.columns.len() < MAX_EVENT_COLUMNS) + .map(|table| { + SchemaRewrite::alter_table( + table, + [TableMigrationOp::ChangeAccess, TableMigrationOp::ReschemaEventTable], + ) + }) + .collect() +} + +impl SchemaRewrite { + fn alter_table(table: &TablePlan, ops: impl Into>) -> Self { + Self::AlterTable { + table: table.name.clone(), + ops: ops.into(), + } + } + + pub(crate) fn apply_to(&self, schema: &mut SchemaPlan) -> anyhow::Result<()> { + match self { + Self::AddTable { table } => { + schema.tables.push(table.clone()); + } + Self::RemoveTable { table } => { + let table = table_position(schema, table)?; + schema.tables.remove(table); + } + Self::AlterTable { table, ops } => { + let table = table_position(schema, table)?; + let table_plan = &mut schema.tables[table]; + for op in ops { + apply_table_op(table_plan, op.clone())?; + } + } + } + Ok(()) + } +} + +fn table_position(schema: &SchemaPlan, table: &str) -> anyhow::Result { + schema + .tables + .iter() + .position(|table_plan| table_plan.name == table) + .ok_or_else(|| anyhow::anyhow!("migration references missing table {table}")) +} + +fn apply_table_op(table: &mut TablePlan, op: TableMigrationOp) -> anyhow::Result<()> { + match op { + TableMigrationOp::ChangeAccess => { + table.is_public = !table.is_public; + } + TableMigrationOp::AddColumn { ty } => { + anyhow::ensure!(!table.is_event, "add-column migration selected event table"); + anyhow::ensure!( + table.columns.len() < MAX_TABLE_COLUMNS, + "table already has the configured maximum number of columns" + ); + table.columns.push(ColumnPlan { + name: SchemaNames::fresh_column_name(table, "added_col"), + ty, + }); + } + TableMigrationOp::AddIndex { columns, algorithm } => { + ensure_columns_exist(table, &columns)?; + anyhow::ensure!( + !has_index(table, &columns), + "add-index migration selected existing index columns" + ); + table.indexes.push(IndexPlan { columns, algorithm }); + } + TableMigrationOp::RemoveIndex { columns } => { + ensure_columns_exist(table, &columns)?; + let Some(index) = table.indexes.iter().position(|index| index.columns == columns) else { + anyhow::bail!("remove-index migration references missing index on columns {columns:?}"); + }; + anyhow::ensure!( + !is_required_index(table, &columns), + "remove-index migration selected a required index" + ); + table.indexes.remove(index); + } + TableMigrationOp::AddSequence { sequence } => { + let column = sequence.column; + let Some(column_plan) = table.columns.get(column) else { + anyhow::bail!("add-sequence migration references missing column {column}"); + }; + anyhow::ensure!( + column_plan.ty.is_integral(), + "add-sequence migration selected non-integral column" + ); + anyhow::ensure!( + table.sequences.iter().all(|existing| existing.column != column), + "add-sequence migration selected an already sequenced column" + ); + table.sequences.push(sequence); + } + TableMigrationOp::RemoveSequence { column } => { + anyhow::ensure!( + column < table.columns.len(), + "remove-sequence migration references missing column" + ); + let Some(sequence) = table.sequences.iter().position(|sequence| sequence.column == column) else { + anyhow::bail!("remove-sequence migration references missing sequence on column {column}"); + }; + table.sequences.remove(sequence); + } + TableMigrationOp::AddUniqueConstraint { columns } => { + ensure_columns_exist(table, &columns)?; + anyhow::ensure!( + !table + .unique_constraints + .iter() + .any(|constraint| constraint.columns == columns), + "add-constraint migration selected existing constraint columns" + ); + anyhow::ensure!( + has_index(table, &columns), + "add-constraint migration requires a matching index" + ); + table.unique_constraints.push(UniqueConstraintPlan { columns }); + } + TableMigrationOp::RemoveUniqueConstraint { columns } => { + ensure_columns_exist(table, &columns)?; + let Some(constraint) = table + .unique_constraints + .iter() + .position(|constraint| constraint.columns == columns) + else { + anyhow::bail!("remove-constraint migration references missing constraint on columns {columns:?}"); + }; + anyhow::ensure!( + !table.primary_key.is_some_and(|primary_key| columns == [primary_key]), + "remove-constraint migration selected primary-key constraint" + ); + table.unique_constraints.remove(constraint); + } + TableMigrationOp::ChangePrimaryKey { column } => { + if let Some(column) = column { + anyhow::ensure!( + column < table.columns.len(), + "primary-key migration references missing column" + ); + } + table.primary_key = column; + } + TableMigrationOp::ChangeIndex { columns } => { + ensure_columns_exist(table, &columns)?; + anyhow::ensure!( + !is_required_index(table, &columns), + "index migration selected a required index" + ); + let Some(index_plan) = table.indexes.iter_mut().find(|index| index.columns == columns) else { + anyhow::bail!("index migration references missing index on columns {columns:?}"); + }; + index_plan.algorithm = match index_plan.algorithm { + IndexAlgorithm::BTree => IndexAlgorithm::Hash, + IndexAlgorithm::Hash => IndexAlgorithm::BTree, + }; + } + TableMigrationOp::ChangeColumnType { column } => { + widen_sum_column(table, Some(column))?; + } + TableMigrationOp::ReschemaEventTable => { + anyhow::ensure!(table.is_event, "event-table migration selected non-event table"); + anyhow::ensure!( + table.columns.len() < MAX_EVENT_COLUMNS, + "event table already has the configured maximum number of columns" + ); + table.columns.push(ColumnPlan { + name: SchemaNames::fresh_column_name(table, "reschema_payload"), + ty: Type::U64, + }); + } + } + + Ok(()) +} + +fn widenable_sum_columns(table: &TablePlan) -> Vec { + table + .columns + .iter() + .enumerate() + .filter_map(|(column, column_plan)| match column_plan.ty { + Type::Sum { variants } if variants < MAX_SUM_VARIANTS => Some(column), + _ => None, + }) + .collect() +} + +fn addable_indexes(table: &TablePlan) -> Vec<(Vec, IndexAlgorithm)> { + (0..table.columns.len()) + .filter_map(|column| { + let columns = vec![column]; + (!has_index(table, &columns)).then_some((columns, IndexAlgorithm::BTree)) + }) + .collect() +} + +fn changeable_index_columns(table: &TablePlan) -> Vec> { + table + .indexes + .iter() + .filter_map(|index| (!is_required_index(table, &index.columns)).then_some(index.columns.clone())) + .collect() +} + +fn addable_sequences(table: &TablePlan) -> Vec { + table + .columns + .iter() + .enumerate() + .filter_map(|(column, column_plan)| { + (column_plan.ty.is_integral() && table.sequences.iter().all(|sequence| sequence.column != column)) + .then(|| SequencePlan::new(column, column_plan.ty).expect("column type checked above")) + }) + .collect() +} + +fn addable_sequence_boundary_probes( + table: &TablePlan, + column_domain: impl Fn(usize) -> Option, +) -> Vec { + table + .columns + .iter() + .enumerate() + .filter_map(|(column, column_plan)| { + let domain = column_domain(column)?; + if !column_plan.ty.is_integral() + || table.sequences.iter().any(|sequence| sequence.column == column) + || domain.sequenced + || !domain.single_column_indexed + || !domain.single_column_unique + { + return None; + } + + domain + .integral_values() + .filter_map(|max_value| SequencePlan::with_existing_value_as_max(column, column_plan.ty, max_value)) + .find(|sequence| added_sequence_precheck_range_is_clear(&domain, sequence)) + }) + .collect() +} + +fn i64_default_max_sequence_precheck_failure_probes( + table: &TablePlan, + column_domain: impl Fn(usize) -> Option, +) -> Vec { + table + .columns + .iter() + .enumerate() + .filter_map(|(column, column_plan)| { + let domain = column_domain(column)?; + if column_plan.ty != Type::I64 + || table.sequences.iter().any(|sequence| sequence.column == column) + || domain.sequenced + || !domain + .integral_values() + .any(|value| (1..i64::MAX as i128).contains(&value)) + { + return None; + } + + SequencePlan::new(column, column_plan.ty) + }) + .collect() +} + +fn sequence_precheck_failure_probes( + table: &TablePlan, + column_domain: impl Fn(usize) -> Option, +) -> Vec { + table + .columns + .iter() + .enumerate() + .filter_map(|(column, column_plan)| { + let domain = column_domain(column)?; + if !column_plan.ty.is_integral() + || table.sequences.iter().any(|sequence| sequence.column == column) + || domain.sequenced + || !domain.single_column_indexed + || !domain.single_column_unique + { + return None; + } + + domain + .integral_values() + .filter_map(|max_value| SequencePlan::with_existing_value_as_max(column, column_plan.ty, max_value)) + .find(|sequence| !added_sequence_precheck_range_is_clear(&domain, sequence)) + }) + .collect() +} + +fn added_sequence_precheck_range_is_clear(domain: &ColumnDomain, sequence: &SequencePlan) -> bool { + let min = sequence.min_value.unwrap_or(1); + let max = sequence.max_value.unwrap_or(i128::MAX); + + // The engine's add-sequence precheck rejects existing values in `min..max`. + // The boundary probe intentionally places an existing value at `max`, so + // only values below the exclusive upper bound make the migration invalid. + domain.integral_values().all(|value| value < min || value >= max) +} + +fn column_domain(model: &Model, table: &TablePlan, column: usize) -> Option { + let column_name = &table.columns.get(column)?.name; + model.column_domain_by_name(&table.name, column_name) +} + +fn removable_sequence_columns(table: &TablePlan) -> Vec { + table.sequences.iter().map(|sequence| sequence.column).collect() +} + +fn addable_unique_constraint_columns(table: &TablePlan) -> Vec> { + (0..table.columns.len()) + .filter_map(|column| { + let columns = vec![column]; + (!table + .unique_constraints + .iter() + .any(|constraint| constraint.columns == columns)) + .then_some(columns) + }) + .collect() +} + +fn removable_unique_constraint_columns(table: &TablePlan) -> Vec> { + table + .unique_constraints + .iter() + .filter_map(|constraint| { + (!table + .primary_key + .is_some_and(|primary_key| constraint.columns == [primary_key])) + .then_some(constraint.columns.clone()) + }) + .collect() +} + +fn has_index(table: &TablePlan, columns: &[usize]) -> bool { + table.indexes.iter().any(|index| index.columns == columns) +} + +fn is_required_index(table: &TablePlan, columns: &[usize]) -> bool { + table.primary_key.is_some_and(|primary_key| columns == [primary_key]) + || table + .unique_constraints + .iter() + .any(|constraint| constraint.columns == columns) +} + +fn ensure_columns_exist(table: &TablePlan, columns: &[usize]) -> anyhow::Result<()> { + anyhow::ensure!(!columns.is_empty(), "migration selected empty column list"); + anyhow::ensure!( + columns.iter().all(|&column| column < table.columns.len()), + "migration references missing column" + ); + Ok(()) +} + +fn widen_sum_column(table: &mut TablePlan, column: Option) -> anyhow::Result<()> { + let column = column.ok_or_else(|| anyhow::anyhow!("sum-widening migration missing column"))?; + let Some(column_plan) = table.columns.get_mut(column) else { + anyhow::bail!("sum-widening migration references missing column {column}"); + }; + + let Type::Sum { variants } = &mut column_plan.ty else { + anyhow::bail!("sum-widening migration selected a non-sum column"); + }; + anyhow::ensure!( + *variants < MAX_SUM_VARIANTS, + "sum-widening migration selected maxed-out sum column" + ); + *variants += 1; + Ok(()) +} diff --git a/crates/dst/src/engine/model.rs b/crates/dst/src/engine/model.rs index 5d6913212e0..ff4b14470cb 100644 --- a/crates/dst/src/engine/model.rs +++ b/crates/dst/src/engine/model.rs @@ -1,23 +1,138 @@ -use super::workload::{ - normalize_rows, CommitDelta, CountState, InsertOutcome, Interaction, Observation, Row, TableDelta, TableRowCount, -}; -use crate::schema::SchemaPlan; +use spacetimedb_lib::{db::raw_def::SEQUENCE_ALLOCATION_STEP, AlgebraicValue, ProductValue}; + +use super::migrations::MigrationExpectation; +use super::row::{normalize_rows, Row}; +use super::state::{schema_state_for_plan, CommitDelta, CountState, TableDelta, TableRowCount, TableRows}; +use super::workload::{InsertOutcome, Interaction, Observation}; +use crate::schema::{ColumnPlan, SchemaPlan, SequencePlan, TablePlan, Type}; #[derive(Debug)] pub struct Model { schema: SchemaPlan, committed_tables: Vec, + sequences: Vec>, pending_tx: Option, } #[derive(Debug)] struct TableState { rows: Vec, + ever_inserted: bool, } #[derive(Debug)] struct PendingTx { tables: Vec, + sequence_allocations: Vec>>, +} + +#[derive(Debug, Clone)] +struct ModelSequence { + column: usize, + ty: Type, + value: i128, + allocated: i128, + durable_allocated: i128, + min_value: i128, + max_value: i128, + increment: i128, +} + +impl ModelSequence { + fn new(plan: &SequencePlan, ty: Type) -> Self { + let start = plan.start.unwrap_or(1); + Self { + column: plan.column, + ty, + value: start, + allocated: start, + durable_allocated: start, + min_value: plan.min_value.unwrap_or(1), + max_value: plan.max_value.unwrap_or(i128::MAX), + increment: plan.increment, + } + } + + fn same_definition(&self, plan: &SequencePlan, ty: Type) -> bool { + self.ty == ty + && self.min_value == plan.min_value.unwrap_or(1) + && self.max_value == plan.max_value.unwrap_or(i128::MAX) + && self.increment == plan.increment + } + + fn with_column(mut self, column: usize) -> Self { + self.column = column; + self + } + + fn generate(&mut self) -> (AlgebraicValue, Option) { + let old_allocated = self.allocated; + if self.needs_allocation() { + self.allocate_steps(SEQUENCE_ALLOCATION_STEP as usize); + } + + let value = self.value; + self.value = self.next_value(); + let allocation = (self.allocated != old_allocated).then_some(self.allocated); + (self.value_to_algebraic(value), allocation) + } + + fn reset_to_durable(&mut self) { + self.value = self.durable_allocated; + self.allocated = self.durable_allocated; + } + + fn needs_allocation(&self) -> bool { + self.value == self.allocated + } + + fn allocate_steps(&mut self, steps: usize) { + if !self.needs_allocation() { + return; + } + + let original_allocation = self.allocated; + for _ in 0..steps { + let next = next_sequence_value(self.min_value, self.max_value, self.increment, self.allocated); + if next == original_allocation { + break; + } + self.allocated = next; + } + debug_assert!(!self.needs_allocation(), "sequence allocation should make progress"); + } + + fn next_value(&self) -> i128 { + next_sequence_value(self.min_value, self.max_value, self.increment, self.value) + } + + fn value_to_algebraic(&self, value: i128) -> AlgebraicValue { + match self.ty { + Type::I64 => AlgebraicValue::I64(value as i64), + Type::U64 => AlgebraicValue::U64(value as u64), + _ => unreachable!("sequence columns are integral in the DST schema"), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ColumnDomain { + pub(crate) ty: Type, + pub(crate) values: Vec, + pub(crate) unique: bool, + pub(crate) single_column_unique: bool, + pub(crate) single_column_indexed: bool, + pub(crate) sequenced: bool, +} + +impl ColumnDomain { + pub(crate) fn integral_values(&self) -> impl Iterator + '_ { + self.values.iter().filter_map(|value| match value { + AlgebraicValue::U64(value) => Some(*value as i128), + AlgebraicValue::I64(value) => Some(*value as i128), + _ => None, + }) + } } // Keep mutable transactions as an overlay: committed rows stay shared, while @@ -35,19 +150,32 @@ impl PendingTable { } impl PendingTx { - fn new(table_count: usize) -> Self { + fn new(sequences: &[Vec]) -> Self { Self { - tables: (0..table_count).map(|_| PendingTable::default()).collect(), + tables: (0..sequences.len()).map(|_| PendingTable::default()).collect(), + sequence_allocations: sequences + .iter() + .map(|table_sequences| vec![None; table_sequences.len()]) + .collect(), } } } impl Model { pub fn new(schema: SchemaPlan) -> Self { - let committed_tables = schema.tables.iter().map(|_| TableState { rows: vec![] }).collect(); + let committed_tables = schema + .tables + .iter() + .map(|_| TableState { + rows: vec![], + ever_inserted: false, + }) + .collect(); + let sequences = sequence_states_for_schema(&schema); Self { schema, committed_tables, + sequences, pending_tx: None, } } @@ -65,6 +193,15 @@ impl Model { &mut self.pending_tx.as_mut().expect("active transaction").tables[table] } + fn pending_sequence_allocation_mut(&mut self, table: usize, sequence: usize) -> &mut Option { + debug_assert!(self.pending_tx.is_some()); + &mut self + .pending_tx + .as_mut() + .expect("active transaction") + .sequence_allocations[table][sequence] + } + fn visible_committed_rows(&self, table: usize) -> impl Iterator + '_ { let pending_table = self.pending_table(table); self.committed_tables[table] @@ -105,32 +242,55 @@ impl Model { false } + fn apply_sequence_values(&mut self, table: usize, row: &Row) -> Row { + let mut row = row.clone(); + let sequence_count = self.sequences[table].len(); + + for sequence_idx in 0..sequence_count { + let column = self.sequences[table][sequence_idx].column; + if !row.elements[column].is_numeric_zero() { + continue; + } + + let (value, allocation) = self.sequences[table][sequence_idx].generate(); + row.elements[column] = value; + if let Some(allocation) = allocation { + *self.pending_sequence_allocation_mut(table, sequence_idx) = Some(allocation); + } + } + + row + } + pub fn apply(&mut self, interaction: &Interaction) -> Observation { match interaction { Interaction::BeginMutTx => { debug_assert!(self.pending_tx.is_none()); - self.pending_tx = Some(PendingTx::new(self.committed_tables.len())); + self.pending_tx = Some(PendingTx::new(&self.sequences)); Observation::BeganMutTx } Interaction::Insert { table, row } => { debug_assert!(self.pending_tx.is_some()); - // Properties feed the target-returned row here, so sequence-generated - // values become part of the oracle before commit/replay checks run. - if self.any_visible_row(*table, |visible_row| visible_row == row) { + self.committed_tables[*table].ever_inserted = true; + let row = self.apply_sequence_values(*table, row); + + if self.any_visible_row(*table, |visible_row| visible_row == &row) { return Observation::Inserted { - outcome: InsertOutcome::Accepted(row.clone()), + outcome: InsertOutcome::Accepted(row), }; } - if self.violates_unique_constraint(*table, row) { + if self.violates_unique_constraint(*table, &row) { return Observation::Inserted { - outcome: InsertOutcome::UniqueConstraintViolation, + outcome: InsertOutcome::UniqueConstraintViolation { + details: "model unique constraint".into(), + }, }; } self.pending_table_mut(*table).inserts.push(row.clone()); Observation::Inserted { - outcome: InsertOutcome::Accepted(row.clone()), + outcome: InsertOutcome::Accepted(row), } } Interaction::Delete { table, row } => { @@ -151,8 +311,23 @@ impl Model { let delta = self.commit_pending(pending_tx); Observation::Committed { delta } } + Interaction::Migrate(migration) => { + debug_assert!(self.pending_tx.is_none()); + match migration.expectation() { + MigrationExpectation::Accepted => { + let old_schema = std::mem::replace(&mut self.schema, migration.schema().clone()); + let old_tables = std::mem::take(&mut self.committed_tables); + let old_sequences = std::mem::take(&mut self.sequences); + self.committed_tables = remap_table_states(&old_schema, &self.schema, old_tables); + self.sequences = remap_sequence_states(&old_schema, &self.schema, old_sequences); + Observation::Migrated + } + MigrationExpectation::Rejected(reason) => Observation::MigrationRejected { reason }, + } + } Interaction::Replay => { self.pending_tx = None; + self.reset_sequences_to_durable(); Observation::Replayed { state: self.count_state(), } @@ -161,9 +336,13 @@ impl Model { } fn commit_pending(&mut self, pending_tx: PendingTx) -> CommitDelta { + let PendingTx { + tables: pending_tables, + sequence_allocations, + } = pending_tx; let mut tables = Vec::new(); - for (table, pending_table) in pending_tx.tables.into_iter().enumerate() { + for (table, pending_table) in pending_tables.into_iter().enumerate() { if pending_table.inserts.is_empty() && pending_table.deletes.is_empty() { continue; } @@ -206,9 +385,23 @@ impl Model { committed_rows.extend(pending_table.inserts); } + for (table, allocations) in sequence_allocations.into_iter().enumerate() { + for (sequence, allocation) in allocations.into_iter().enumerate() { + if let Some(allocation) = allocation { + self.sequences[table][sequence].durable_allocated = allocation; + } + } + } + CommitDelta { tables } } + fn reset_sequences_to_durable(&mut self) { + for sequence in self.sequences.iter_mut().flatten() { + sequence.reset_to_durable(); + } + } + pub fn in_mut_tx(&self) -> bool { self.pending_tx.is_some() } @@ -217,6 +410,56 @@ impl Model { self.visible_count(table) as usize } + pub fn ever_inserted(&self, table: usize) -> bool { + self.committed_tables[table].ever_inserted + } + + pub(crate) fn row_count_by_table_name(&self, table: &str) -> usize { + self.table_index(table).map_or(0, |table| self.row_count(table)) + } + + pub(crate) fn ever_inserted_by_table_name(&self, table: &str) -> bool { + self.table_index(table) + .is_some_and(|table| self.committed_tables[table].ever_inserted) + } + + pub(crate) fn column_domain_by_name(&self, table: &str, column: &str) -> Option { + let table = self.table_index(table)?; + let column = self.schema.tables[table] + .columns + .iter() + .position(|column_plan| column_plan.name == column)?; + Some(self.column_domain(table, column)) + } + + fn table_index(&self, table: &str) -> Option { + self.schema + .tables + .iter() + .position(|table_plan| table_plan.name == table) + } + + pub(crate) fn column_domain(&self, table: usize, column: usize) -> ColumnDomain { + let table_plan = &self.schema.tables[table]; + ColumnDomain { + ty: table_plan.columns[column].ty, + values: self + .visible_rows(table) + .map(|row| row.elements[column].clone()) + .collect(), + unique: table_plan + .unique_constraints + .iter() + .any(|constraint| constraint.columns.contains(&column)), + single_column_unique: table_plan + .unique_constraints + .iter() + .any(|constraint| constraint.columns == [column]), + single_column_indexed: table_plan.indexes.iter().any(|index| index.columns == [column]), + sequenced: table_plan.sequences.iter().any(|sequence| sequence.column == column), + } + } + pub fn row(&self, table: usize, row: usize) -> Option<&Row> { self.visible_rows(table).nth(row) } @@ -233,10 +476,166 @@ impl Model { count: self.visible_count(table), }) .collect(); - CountState { row_counts } + let table_rows = (0..self.schema.tables.len()) + .map(|table| TableRows { + table, + rows: normalize_rows(self.visible_rows(table).cloned().collect()), + }) + .collect(); + + CountState { + row_counts, + table_rows, + schema: schema_state_for_plan(&self.schema), + } } } +fn sequence_states_for_schema(schema: &SchemaPlan) -> Vec> { + schema.tables.iter().map(sequence_states_for_table).collect() +} + +fn sequence_states_for_table(table: &TablePlan) -> Vec { + table + .sequences + .iter() + .map(|sequence| ModelSequence::new(sequence, table.columns[sequence.column].ty)) + .collect() +} + +fn remap_sequence_states( + old_schema: &SchemaPlan, + new_schema: &SchemaPlan, + old_sequences: Vec>, +) -> Vec> { + new_schema + .tables + .iter() + .map(|new_table| { + let Some(old_table_idx) = old_schema + .tables + .iter() + .position(|old_table| old_table.name == new_table.name) + else { + return sequence_states_for_table(new_table); + }; + + let old_table = &old_schema.tables[old_table_idx]; + new_table + .sequences + .iter() + .map(|new_sequence| { + remap_sequence_state(old_table, new_table, &old_sequences[old_table_idx], new_sequence) + }) + .collect() + }) + .collect() +} + +fn remap_sequence_state( + old_table: &TablePlan, + new_table: &TablePlan, + old_sequences: &[ModelSequence], + new_sequence: &SequencePlan, +) -> ModelSequence { + let new_column = &new_table.columns[new_sequence.column]; + let Some(old_column_idx) = old_table + .columns + .iter() + .position(|old_column| old_column.name == new_column.name) + else { + return ModelSequence::new(new_sequence, new_column.ty); + }; + + let Some(old_sequence_idx) = old_table + .sequences + .iter() + .position(|old_sequence| old_sequence.column == old_column_idx) + else { + return ModelSequence::new(new_sequence, new_column.ty); + }; + + let old_sequence = &old_sequences[old_sequence_idx]; + if old_sequence.same_definition(new_sequence, new_column.ty) { + old_sequence.clone().with_column(new_sequence.column) + } else { + ModelSequence::new(new_sequence, new_column.ty) + } +} + +fn next_sequence_value(min: i128, max: i128, increment: i128, value: i128) -> i128 { + let mut next = value + increment; + if increment > 0 { + if next > max { + next = min + (next - max - 1) % (max - min + 1); + } + } else if next < min { + next = max - (min - next - 1) % (max - min + 1); + } + next +} + +fn remap_table_states( + old_schema: &SchemaPlan, + new_schema: &SchemaPlan, + old_tables: Vec, +) -> Vec { + let mut old_tables = old_tables.into_iter().map(Some).collect::>(); + new_schema + .tables + .iter() + .map(|new_table| { + let Some(old_table_idx) = old_schema + .tables + .iter() + .position(|old_table| old_table.name == new_table.name) + else { + return TableState { + rows: vec![], + ever_inserted: false, + }; + }; + + let old_table = &old_schema.tables[old_table_idx]; + let old_state = old_tables[old_table_idx] + .take() + .expect("old table state is consumed once"); + remap_table_state(old_table, new_table, old_state) + }) + .collect() +} + +fn remap_table_state(old_table: &TablePlan, new_table: &TablePlan, state: TableState) -> TableState { + TableState { + rows: state + .rows + .into_iter() + .map(|row| remap_row(old_table, new_table, row)) + .collect(), + ever_inserted: state.ever_inserted, + } +} + +fn remap_row(old_table: &TablePlan, new_table: &TablePlan, row: Row) -> Row { + let elements = new_table + .columns + .iter() + .map(|new_column| remap_value(old_table, new_column, &row)) + .collect::>(); + ProductValue { + elements: elements.into_boxed_slice(), + } +} + +fn remap_value(old_table: &TablePlan, new_column: &ColumnPlan, row: &Row) -> AlgebraicValue { + old_table + .columns + .iter() + .position(|old_column| old_column.name == new_column.name) + .map(|old_column| row.elements[old_column].clone()) + .unwrap_or_else(|| new_column.ty.default_value()) +} + #[cfg(test)] mod tests { use spacetimedb_lib::AlgebraicValue; @@ -260,6 +659,7 @@ mod tests { unique_constraints: vec![UniqueConstraintPlan { columns: vec![0] }], sequences: vec![], is_public: true, + is_event: false, }], } } diff --git a/crates/dst/src/engine/properties.rs b/crates/dst/src/engine/properties.rs index 667eec09510..0a0b6e19abd 100644 --- a/crates/dst/src/engine/properties.rs +++ b/crates/dst/src/engine/properties.rs @@ -1,5 +1,5 @@ use super::model::Model; -use super::workload::{InsertOutcome, Interaction, Observation, Row}; +use super::workload::{InsertOutcome, Interaction, Observation}; use crate::schema::SchemaPlan; use crate::traits::Properties; @@ -15,6 +15,7 @@ impl EngineProperties { properties: vec![ Box::new(InsertMatches), Box::new(CommitMatches), + Box::new(MigrateMatches), Box::new(ReplayMatchesModel), ], } @@ -55,31 +56,13 @@ impl EngineOracle { fn apply(&mut self, interaction: &Interaction, observation: &Observation) -> anyhow::Result { let observation = match (interaction, observation) { - ( - Interaction::Insert { table, .. }, - Observation::Inserted { - outcome: InsertOutcome::Accepted(row), - }, - ) => self.apply_insert(*table, row), - ( - Interaction::Insert { .. }, - Observation::Inserted { - outcome: InsertOutcome::UniqueConstraintViolation, - }, - ) => self.model.apply(interaction), + (Interaction::Insert { .. }, Observation::Inserted { .. }) => self.model.apply(interaction), (Interaction::Insert { .. }, _) => anyhow::bail!("insert produced unexpected observation"), _ => self.model.apply(interaction), }; Ok(observation) } - - fn apply_insert(&mut self, table: usize, row: &Row) -> Observation { - self.model.apply(&Interaction::Insert { - table, - row: row.clone(), - }) - } } struct InsertMatches; @@ -91,7 +74,7 @@ impl EngineProperty for InsertMatches { fn check( &self, - _interaction: &Interaction, + interaction: &Interaction, observation: &Observation, expected: &Observation, ) -> anyhow::Result<()> { @@ -106,12 +89,16 @@ impl EngineProperty for InsertMatches { (InsertOutcome::Accepted(row), InsertOutcome::Accepted(expected)) => { anyhow::ensure!(row == expected, "insert_matches: accepted row diverged from model"); } - (InsertOutcome::UniqueConstraintViolation, InsertOutcome::UniqueConstraintViolation) => {} - (InsertOutcome::Accepted(_), InsertOutcome::UniqueConstraintViolation) => { - anyhow::bail!("insert_matches: target accepted row rejected by model"); + (InsertOutcome::UniqueConstraintViolation { .. }, InsertOutcome::UniqueConstraintViolation { .. }) => {} + (InsertOutcome::Accepted(_), InsertOutcome::UniqueConstraintViolation { .. }) => { + anyhow::bail!( + "insert_matches: target accepted row rejected by model\ninteraction: {interaction:#?}\ntarget: {observation:#?}\nmodel: {expected:#?}" + ); } - (InsertOutcome::UniqueConstraintViolation, InsertOutcome::Accepted(_)) => { - anyhow::bail!("insert_matches: target rejected row accepted by model"); + (InsertOutcome::UniqueConstraintViolation { .. }, InsertOutcome::Accepted(_)) => { + anyhow::bail!( + "insert_matches: target rejected row accepted by model\ninteraction: {interaction:#?}\ntarget: {observation:#?}\nmodel: {expected:#?}" + ); } } @@ -144,6 +131,30 @@ impl EngineProperty for CommitMatches { } } +struct MigrateMatches; + +impl EngineProperty for MigrateMatches { + fn observes(&self, interaction: &Interaction) -> bool { + matches!(interaction, Interaction::Migrate(_)) + } + + fn check( + &self, + interaction: &Interaction, + observation: &Observation, + expected: &Observation, + ) -> anyhow::Result<()> { + anyhow::ensure!( + observation == expected, + "migrate_matches: migration outcome diverged from model +interaction: {interaction:#?} +target: {observation:#?} +model: {expected:#?}" + ); + Ok(()) + } +} + struct ReplayMatchesModel; impl EngineProperty for ReplayMatchesModel { diff --git a/crates/dst/src/engine/row.rs b/crates/dst/src/engine/row.rs new file mode 100644 index 00000000000..a3008d48d42 --- /dev/null +++ b/crates/dst/src/engine/row.rs @@ -0,0 +1,13 @@ +use spacetimedb_lib::bsatn::to_vec; +use spacetimedb_lib::ProductValue; + +pub type Row = ProductValue; + +pub fn row_to_bytes(row: &Row) -> Vec { + to_vec(row).expect("row serialization must not fail") +} + +pub fn normalize_rows(mut rows: Vec) -> Vec { + rows.sort_by_key(row_to_bytes); + rows +} diff --git a/crates/dst/src/engine/state.rs b/crates/dst/src/engine/state.rs new file mode 100644 index 00000000000..db579034ebb --- /dev/null +++ b/crates/dst/src/engine/state.rs @@ -0,0 +1,237 @@ +use spacetimedb_lib::db::auth::StAccess; +use spacetimedb_lib::AlgebraicType; +use spacetimedb_primitives::ColId; +use spacetimedb_schema::def::IndexAlgorithm as SchemaIndexAlgorithm; +use spacetimedb_schema::schema::TableSchema; + +use super::row::Row; +use crate::schema::{ + IndexAlgorithm as PlanIndexAlgorithm, IndexPlan, SchemaPlan, SequencePlan, TablePlan, UniqueConstraintPlan, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CountState { + pub row_counts: Vec, + pub table_rows: Vec, + pub schema: SchemaState, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TableRowCount { + pub table: usize, + pub count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableRows { + pub table: usize, + pub rows: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SchemaState { + pub tables: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableSchemaState { + pub table: usize, + pub name: String, + pub is_public: bool, + pub is_event: bool, + pub primary_key: Option, + pub columns: Vec, + pub indexes: Vec, + pub unique_constraints: Vec, + pub sequences: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ColumnState { + pub name: String, + pub ty: AlgebraicType, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct IndexState { + pub columns: Vec, + pub algorithm: IndexAlgorithmState, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum IndexAlgorithmState { + BTree, + Hash, + Direct, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct UniqueConstraintState { + pub columns: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct SequenceState { + pub column: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitDelta { + pub tables: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableDelta { + pub table: usize, + pub inserts: Vec, + pub deletes: Vec, + pub truncated: bool, +} + +impl From for IndexAlgorithmState { + fn from(algorithm: PlanIndexAlgorithm) -> Self { + match algorithm { + PlanIndexAlgorithm::BTree => Self::BTree, + PlanIndexAlgorithm::Hash => Self::Hash, + } + } +} + +impl From<&SchemaIndexAlgorithm> for IndexAlgorithmState { + fn from(algorithm: &SchemaIndexAlgorithm) -> Self { + match algorithm { + SchemaIndexAlgorithm::BTree(_) => Self::BTree, + SchemaIndexAlgorithm::Hash(_) => Self::Hash, + SchemaIndexAlgorithm::Direct(_) => Self::Direct, + _ => Self::Unknown, + } + } +} + +impl IndexState { + fn from_plan(index: &IndexPlan) -> Self { + Self { + columns: index.columns.clone(), + algorithm: index.algorithm.into(), + } + } + + fn from_schema(algorithm: &SchemaIndexAlgorithm) -> Self { + Self { + columns: schema_index_columns(algorithm), + algorithm: algorithm.into(), + } + } +} + +impl UniqueConstraintState { + fn from_plan(constraint: &UniqueConstraintPlan) -> Self { + Self { + columns: constraint.columns.clone(), + } + } + + fn from_schema_columns(columns: impl IntoIterator) -> Self { + Self { + columns: columns.into_iter().map(|col| col.0 as usize).collect(), + } + } +} + +impl SequenceState { + fn from_plan(sequence: &SequencePlan) -> Self { + Self { + column: sequence.column, + } + } + + fn from_schema_column(column: ColId) -> Self { + Self { + column: column.0 as usize, + } + } +} + +pub fn schema_state_for_plan(schema: &SchemaPlan) -> SchemaState { + SchemaState { + tables: schema + .tables + .iter() + .enumerate() + .map(|(table, table_plan)| table_schema_state_for_plan(table, table_plan)) + .collect(), + } +} + +pub fn table_schema_state_for_schema(table: usize, schema: &TableSchema) -> TableSchemaState { + TableSchemaState { + table, + name: schema.table_name.to_string(), + is_public: schema.table_access == StAccess::Public, + is_event: schema.is_event, + primary_key: schema.primary_key.map(|col| col.0 as usize), + columns: schema + .columns + .iter() + .map(|column| ColumnState { + name: column.col_name.to_string(), + ty: column.col_type.clone(), + }) + .collect(), + indexes: sorted( + schema + .indexes + .iter() + .map(|index| IndexState::from_schema(&index.index_algorithm)), + ), + unique_constraints: sorted(schema.constraints.iter().filter_map(|constraint| { + constraint + .data + .unique_columns() + .map(|columns| UniqueConstraintState::from_schema_columns(columns.iter())) + })), + sequences: sorted( + schema + .sequences + .iter() + .map(|sequence| SequenceState::from_schema_column(sequence.col_pos)), + ), + } +} + +fn table_schema_state_for_plan(table: usize, table_plan: &TablePlan) -> TableSchemaState { + TableSchemaState { + table, + name: table_plan.name.clone(), + is_public: table_plan.is_public, + is_event: table_plan.is_event, + primary_key: table_plan.primary_key, + columns: table_plan + .columns + .iter() + .map(|column| ColumnState { + name: column.name.clone(), + ty: column.ty.to_algebraic(), + }) + .collect(), + indexes: sorted(table_plan.indexes.iter().map(IndexState::from_plan)), + unique_constraints: sorted( + table_plan + .unique_constraints + .iter() + .map(UniqueConstraintState::from_plan), + ), + sequences: sorted(table_plan.sequences.iter().map(SequenceState::from_plan)), + } +} + +fn schema_index_columns(algorithm: &SchemaIndexAlgorithm) -> Vec { + algorithm.columns().iter().map(|col| col.0 as usize).collect() +} + +fn sorted(values: impl IntoIterator) -> Vec { + let mut values = values.into_iter().collect::>(); + values.sort(); + values +} diff --git a/crates/dst/src/engine/workload.rs b/crates/dst/src/engine/workload.rs index bc9e4fcc36e..aca7b96b7d3 100644 --- a/crates/dst/src/engine/workload.rs +++ b/crates/dst/src/engine/workload.rs @@ -1,24 +1,30 @@ -use std::fmt::{Debug, Error, Formatter}; +//! Workload interaction generation for the engine DST driver. -use spacetimedb_lib::bsatn::to_vec; -use spacetimedb_lib::{AlgebraicValue, ProductValue}; -use spacetimedb_runtime::sim::Rng; -use spacetimedb_sats::ArrayValue; +use std::fmt::{Debug, Error as FmtError, Formatter}; +use super::generation::GenCtx; +use super::migrations::{Migration, MigrationRejection}; use super::model::Model; -use crate::schema::{SchemaPlan, TablePlan, Type}; - -pub type Row = ProductValue; +use super::row::Row; +use super::state::{CommitDelta, CountState}; +use crate::rng::{choice, pick_choice, Choice}; +use crate::schema::SchemaPlan; +use crate::traits::InteractionGen; +use anyhow::Error; +use spacetimedb_runtime::sim::Rng; +/// One generated action for the engine target to execute. #[derive(Debug, Clone)] pub enum Interaction { BeginMutTx, Insert { table: usize, row: Row }, Delete { table: usize, row: Row }, CommitTx, + Migrate(Migration), Replay, } +/// Counts of emitted workload interactions, reported at the end of each run. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct InteractionCounts { pub total: usize, @@ -26,6 +32,7 @@ pub struct InteractionCounts { pub insert: usize, pub delete: usize, pub commit_tx: usize, + pub migrate: usize, pub replay: usize, } @@ -38,53 +45,53 @@ impl InteractionCounts { Interaction::Insert { .. } => self.insert += 1, Interaction::Delete { .. } => self.delete += 1, Interaction::CommitTx => self.commit_tx += 1, + Interaction::Migrate(_) => self.migrate += 1, Interaction::Replay => self.replay += 1, } } } +/// Observable result of executing an interaction against the engine. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Observation { BeganMutTx, Inserted { outcome: InsertOutcome }, Deleted, Committed { delta: CommitDelta }, + Migrated, + MigrationRejected { reason: MigrationRejection }, Replayed { state: CountState }, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum InsertOutcome { Accepted(Row), - UniqueConstraintViolation, + UniqueConstraintViolation { details: String }, } +/// Runtime-tunable weights for top-level workload actions. #[derive(Debug, Clone, Copy)] pub struct InteractionWeights { pub insert: u64, pub delete: u64, pub commit_tx: u64, + pub migrate: u64, pub replay: u64, } impl Default for InteractionWeights { fn default() -> Self { Self { - insert: 50, - delete: 20, - commit_tx: 29, + insert: 5_000, + delete: 2_000, + commit_tx: 2_800, + migrate: 100, replay: 1, } } } -#[derive(Debug, Clone, Copy)] -enum InteractionChoice { - Insert, - Delete, - CommitTx, - Replay, -} - +/// Stateful interaction source that mirrors target observations into the model. pub struct WorkloadGen { rng: Rng, model: Model, @@ -94,11 +101,15 @@ pub struct WorkloadGen { impl WorkloadGen { pub fn new(rng: Rng, model: Model) -> Self { + Self::with_weights(rng, model, InteractionWeights::default()) + } + + pub fn with_weights(rng: Rng, model: Model, weights: InteractionWeights) -> Self { Self { rng, model, stats: InteractionCounts::default(), - weights: InteractionWeights::default(), + weights, } } @@ -106,70 +117,71 @@ impl WorkloadGen { self.stats } - fn schema(&self) -> &SchemaPlan { - self.model.schema() - } + pub fn next_interaction(&mut self) -> Interaction { + let choice = self.pick_interaction_choice(); + let interaction = self.interaction_from_choice(choice); - fn gen_value(&self, ty: Type) -> AlgebraicValue { - match ty { - Type::Bool => AlgebraicValue::Bool(self.rng.next_u64().is_multiple_of(2)), - Type::I64 => AlgebraicValue::I64(self.rng.next_u64() as i64), - Type::U64 => AlgebraicValue::U64(self.rng.next_u64()), - Type::String => AlgebraicValue::String(format!("v_{}", self.rng.next_u64()).into()), - Type::Bytes => { - let len = (self.rng.next_u64() % 16) as usize; - let bytes: Vec = (0..len).map(|_| self.rng.next_u64() as u8).collect(); - AlgebraicValue::Array(ArrayValue::U8(bytes.into())) - } - } - } + self.stats.record(&interaction); - fn gen_row(&self, table: &TablePlan) -> Row { - table - .columns - .iter() - .map(|c| self.gen_value(c.ty)) - .collect::() + interaction } - fn gen_insert_row(&self, table_idx: usize) -> Row { - let table = &self.schema().tables[table_idx]; - let mut row = self.gen_row(table); - - if let Some(sequence) = table.sequences.first() { - row.elements[sequence.column] = match table.columns[sequence.column].ty { - Type::I64 => AlgebraicValue::I64(0), - Type::U64 => AlgebraicValue::U64(0), - _ => unreachable!("sequence columns are integral"), - }; + fn observe_interaction(&mut self, interaction: &Interaction, observation: &Observation) -> Result<(), Error> { + match (interaction, observation) { + (Interaction::Insert { .. }, Observation::Inserted { .. }) => { + self.model.apply(interaction); + } + (Interaction::Insert { .. }, _) => anyhow::bail!("insert produced unexpected observation"), + _ => { + self.model.apply(interaction); + } } - row + Ok(()) } +} - fn non_auto_inc_table_idx(&self) -> Option { - let auto_inc_table = self - .schema() - .auto_inc_table_and_column() - .map(|(table_idx, _)| table_idx); +#[derive(Debug, Clone, Copy)] +enum InteractionChoice { + Insert, + Delete, + CommitTx, + Migrate, + Replay, +} - (0..self.schema().tables.len()).find(|&table_idx| Some(table_idx) != auto_inc_table) +impl InteractionWeights { + fn choices(self) -> [Choice; 5] { + [ + choice(self.insert, InteractionChoice::Insert), + choice(self.delete, InteractionChoice::Delete), + choice(self.commit_tx, InteractionChoice::CommitTx), + choice(self.migrate, InteractionChoice::Migrate), + choice(self.replay, InteractionChoice::Replay), + ] } +} - pub fn next_interaction(&mut self) -> Interaction { - let choice = self.pick_interaction_choice(); - let interaction = self.interaction_from_choice(choice); - - self.model.apply(&interaction); - self.stats.record(&interaction); +impl WorkloadGen { + fn schema(&self) -> &SchemaPlan { + self.model.schema() + } - interaction + fn non_sequenced_table_idx(&self) -> Option { + (0..self.schema().tables.len()).find(|&table_idx| { + let table = &self.schema().tables[table_idx]; + !table.is_event && table.sequences.is_empty() + }) } fn interaction_from_choice(&mut self, choice: InteractionChoice) -> Interaction { if !self.model.in_mut_tx() { return match choice { InteractionChoice::Replay => Interaction::Replay, + InteractionChoice::Migrate => self + .gen_migration() + .map(Interaction::Migrate) + .unwrap_or(Interaction::Replay), // Insert/Delete/CommitTx are not legal outside a mutable tx. // Treat those weighted choices as pressure to start one. @@ -182,12 +194,14 @@ impl WorkloadGen { match choice { InteractionChoice::Replay => Interaction::Replay, + InteractionChoice::Migrate => Interaction::CommitTx, + InteractionChoice::Insert => { let table = self.insert_table_idx(); Interaction::Insert { table, - row: self.gen_insert_row(table), + row: GenCtx::new(&self.rng, &self.model).gen_insert_row(table), } } @@ -213,96 +227,70 @@ impl WorkloadGen { } fn pick_interaction_choice(&mut self) -> InteractionChoice { - let weights = self.weights; - - match self.pick_weighted(&[weights.insert, weights.delete, weights.commit_tx, weights.replay]) { - 0 => InteractionChoice::Insert, - 1 => InteractionChoice::Delete, - 2 => InteractionChoice::CommitTx, - 3 => InteractionChoice::Replay, - _ => unreachable!(), - } + let choices = self.weights.choices(); + pick_choice(&self.rng, &choices) } - fn pick_weighted(&mut self, weights: &[u64]) -> usize { - let total: u64 = weights.iter().sum(); - - assert!(total > 0, "at least one interaction weight must be non-zero"); - - let mut selected = self.rng.next_u64() % total; - - for (idx, weight) in weights.iter().copied().enumerate() { - if selected < weight { - return idx; - } + fn insert_table_idx(&self) -> usize { + let sequenced_tables = self.sequenced_table_indices(); + let data_tables = self.data_table_indices(); - selected -= weight; + if !sequenced_tables.is_empty() && !self.rng.next_u64().is_multiple_of(3) { + sequenced_tables[self.rng.index(sequenced_tables.len())] + } else { + data_tables[self.rng.index(data_tables.len())] } + } - unreachable!("selected value is always inside total weight") + fn sequenced_table_indices(&self) -> Vec { + self.schema() + .tables + .iter() + .enumerate() + .filter_map(|(table_idx, table)| (!table.is_event && !table.sequences.is_empty()).then_some(table_idx)) + .collect() } - fn insert_table_idx(&self) -> usize { - let auto_inc_table_idx = self + fn data_table_indices(&self) -> Vec { + let data_tables: Vec<_> = self .schema() - .auto_inc_table_and_column() - .map(|(table_idx, _)| table_idx); + .tables + .iter() + .enumerate() + .filter_map(|(table_idx, table)| (!table.is_event).then_some(table_idx)) + .collect(); + assert!( + !data_tables.is_empty(), + "engine DST schema must include a non-event table" + ); + data_tables + } - match auto_inc_table_idx { - Some(table_idx) if !self.rng.next_u64().is_multiple_of(3) => table_idx, - _ => self.rng.index(self.schema().tables.len()), - } + fn gen_migration(&self) -> Option { + GenCtx::new(&self.rng, &self.model).gen_migration() } fn deletable_table_idx(&self) -> Option { - self.non_auto_inc_table_idx() + self.non_sequenced_table_idx() .filter(|&table_idx| self.model.row_count(table_idx) > 0) } } impl Debug for WorkloadGen { - fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> { write!(f, "{:?}", self.stats()) } } -impl Iterator for WorkloadGen { - type Item = Interaction; +impl InteractionGen for WorkloadGen { + type Interaction = Interaction; + type Observation = Observation; - fn next(&mut self) -> Option { - Some(self.next_interaction()) + fn next_interaction(&mut self) -> Option { + Some(WorkloadGen::next_interaction(self)) } -} -pub fn row_to_bytes(row: &Row) -> Vec { - to_vec(row).expect("row serialization must not fail") -} - -pub fn normalize_rows(mut rows: Vec) -> Vec { - rows.sort_by_key(row_to_bytes); - rows -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CountState { - pub row_counts: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct TableRowCount { - pub table: usize, - pub count: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CommitDelta { - pub tables: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TableDelta { - pub table: usize, - pub inserts: Vec, - pub deletes: Vec, - pub truncated: bool, + fn observe(&mut self, interaction: &Self::Interaction, observation: &Self::Observation) -> Result<(), Error> { + self.observe_interaction(interaction, observation) + } } diff --git a/crates/dst/src/lib.rs b/crates/dst/src/lib.rs index 8d12c575e4c..ef8e754f6de 100644 --- a/crates/dst/src/lib.rs +++ b/crates/dst/src/lib.rs @@ -1,4 +1,5 @@ pub mod engine; +mod rng; pub mod schema; pub mod sim; pub mod traits; diff --git a/crates/dst/src/rng.rs b/crates/dst/src/rng.rs new file mode 100644 index 00000000000..626d53b87d5 --- /dev/null +++ b/crates/dst/src/rng.rs @@ -0,0 +1,62 @@ +//! Deterministic random-selection helpers shared by DST generators. + +use spacetimedb_runtime::sim::Rng; + +/// A weighted value in a deterministic choice table. +#[derive(Clone, Copy)] +pub(crate) struct Choice { + weight: u64, + value: T, +} + +impl Choice { + pub(crate) const fn value(self) -> T { + self.value + } +} + +/// Construct a weighted choice entry. +pub(crate) const fn choice(weight: u64, value: T) -> Choice { + Choice { weight, value } +} + +/// Pick one value from `choices`, using each entry's relative weight. +pub(crate) fn pick_choice(rng: &Rng, choices: &[Choice]) -> T { + let total: u64 = choices.iter().map(|choice| choice.weight).sum(); + + assert!(total > 0, "at least one choice weight must be non-zero"); + + let mut selected = rng.next_u64() % total; + + for choice in choices.iter().copied() { + if selected < choice.weight { + return choice.value; + } + + selected -= choice.weight; + } + + unreachable!("selected value is always inside total weight") +} + +/// Static weighted choices for enum-like generator cases. +pub(crate) trait WeightedChoice: Copy + 'static { + const CHOICES: &'static [Choice]; + + fn pick(rng: &Rng) -> Self { + pick_choice(rng, Self::CHOICES) + } +} + +/// Pick an index from `0..len`, or return `None` for an empty collection. +pub(crate) fn choose_index(rng: &Rng, len: usize) -> Option { + (len > 0).then(|| rng.index(len)) +} + +/// Pick a value in the inclusive range `lo..=hi`. +pub(crate) fn range_inclusive(rng: &Rng, lo: usize, hi: usize) -> usize { + if lo >= hi { + return lo; + } + lo + (rng.next_u64() as usize % (hi - lo + 1)) +} diff --git a/crates/dst/src/schema.rs b/crates/dst/src/schema.rs index 641281db3c3..b8d70463de2 100644 --- a/crates/dst/src/schema.rs +++ b/crates/dst/src/schema.rs @@ -1,14 +1,35 @@ +//! Schema plans and raw module lowering for the engine DST harness. + +use crate::rng; use spacetimedb_lib::db::raw_def::v10::*; use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, TableAccess, TableType}; use spacetimedb_primitives::{ColId, ColList}; use spacetimedb_runtime::sim::Rng; -use spacetimedb_sats::{AlgebraicType, ArrayType, ProductType, ProductTypeElement}; - +use spacetimedb_sats::{ + AlgebraicType, AlgebraicValue, ArrayType, ArrayValue, ProductType, ProductTypeElement, SumType, SumTypeVariant, +}; + +/// Generate the default engine DST schema. +/// +/// The initial schema is intentionally random and valid, not repaired into a +/// fixed coverage fixture. Long runs are expected to discover more surfaces via +/// migrations. pub fn default_schema(rng: Rng) -> SchemaPlan { - let profile = SchemaProfile::default(); - let mut plan = SchemaGenerator::new(rng, profile).gen_schema(); - plan.ensure_auto_inc_table(); - plan + SchemaGenerator::new(rng, SchemaProfile::engine_dst()).gen_schema() +} + +/// Lower a generated schema plan into the raw module format used by the engine. +pub fn to_raw_def(schema: &SchemaPlan) -> RawModuleDefV10 { + let mut builder = RawModuleDefV10Builder::new(); + builder.set_case_conversion_policy(CaseConversionPolicy::None); + + for table in &schema.tables { + to_raw_def_table(&mut builder, table); + } + + let mut raw = builder.finish(); + apply_sequence_bounds(schema, &mut raw); + raw } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -18,10 +39,19 @@ pub enum Type { U64, String, Bytes, + Sum { variants: u8 }, } impl Type { - pub const ALL: &'static [Type] = &[Type::Bool, Type::I64, Type::U64, Type::String, Type::Bytes]; + /// Representative column types used by migration candidate generation. + pub const ALL: &'static [Type] = &[ + Type::Bool, + Type::I64, + Type::U64, + Type::String, + Type::Bytes, + Type::Sum { variants: 1 }, + ]; pub fn to_algebraic(self) -> AlgebraicType { match self { @@ -32,6 +62,26 @@ impl Type { Type::Bytes => AlgebraicType::Array(ArrayType { elem_ty: Box::new(AlgebraicType::U8), }), + Type::Sum { variants } => { + debug_assert!(variants > 0); + AlgebraicType::Sum(SumType::new( + (0..variants) + .map(|variant| SumTypeVariant::new_named(AlgebraicType::U8, format!("variant_{variant}"))) + .collect::>() + .into_boxed_slice(), + )) + } + } + } + + pub fn default_value(self) -> AlgebraicValue { + match self { + Type::Bool => AlgebraicValue::Bool(false), + Type::I64 => AlgebraicValue::I64(0), + Type::U64 => AlgebraicValue::U64(0), + Type::String => AlgebraicValue::String("".into()), + Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(Vec::new().into())), + Type::Sum { .. } => AlgebraicValue::sum(0, AlgebraicValue::U8(0)), } } @@ -40,55 +90,13 @@ impl Type { } } -// Schema plan — the canonical source of truth. -// This Schema should be able to translate to valid `RawModuleDefV10`. -#[derive(Debug, Clone)] +/// Canonical schema representation for the DST model and engine target. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct SchemaPlan { pub tables: Vec, } -impl SchemaPlan { - pub fn auto_inc_table_and_column(&self) -> Option<(usize, usize)> { - self.tables - .iter() - .enumerate() - .find_map(|(table_idx, table)| table.sequences.first().map(|sequence| (table_idx, sequence.column))) - } - - pub fn ensure_auto_inc_table(&mut self) { - if self.auto_inc_table_and_column().is_some() { - return; - } - - let table = self.tables.first_mut().expect("schema must contain at least one table"); - if table.columns.is_empty() { - table.columns.push(ColumnPlan { - name: "id".into(), - ty: Type::U64, - }); - } else { - table.columns[0].ty = Type::U64; - } - - table.primary_key = Some(0); - if !table - .unique_constraints - .iter() - .any(|constraint| constraint.columns == [0]) - { - table.unique_constraints.push(UniqueConstraintPlan { columns: vec![0] }); - } - if !table.indexes.iter().any(|index| index.columns == [0]) { - table.indexes.push(IndexPlan { - columns: vec![0], - algorithm: IndexAlgorithm::BTree, - }); - } - table.sequences = vec![SequencePlan::new(0, Type::U64).expect("u64 is integral")]; - } -} - -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct TablePlan { pub name: String, pub columns: Vec, @@ -97,15 +105,16 @@ pub struct TablePlan { pub unique_constraints: Vec, pub sequences: Vec, pub is_public: bool, + pub is_event: bool, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ColumnPlan { pub name: String, pub ty: Type, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct IndexPlan { /// Indices into `TablePlan.columns`. pub columns: Vec, @@ -118,17 +127,21 @@ pub enum IndexAlgorithm { Hash, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct UniqueConstraintPlan { /// Indices into `TablePlan.columns`. Non-empty. pub columns: Vec, } /// A sequence on a specific integral column. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct SequencePlan { /// Index into `TablePlan.columns`. pub column: usize, + pub start: Option, + pub min_value: Option, + pub max_value: Option, + pub increment: i128, } impl SequencePlan { @@ -137,82 +150,50 @@ impl SequencePlan { if !ty.is_integral() { return None; } - Some(Self { column }) + Some(Self { + column, + start: None, + min_value: None, + max_value: None, + increment: 1, + }) } -} -// Lowering into RawModuleDefV10. -pub fn to_raw_def(schema: &SchemaPlan) -> RawModuleDefV10 { - let mut builder = RawModuleDefV10Builder::new(); - builder.set_case_conversion_policy(CaseConversionPolicy::None); - - for table in &schema.tables { - to_raw_def_table(&mut builder, table); - } - - builder.finish() -} - -fn to_raw_def_table(builder: &mut RawModuleDefV10Builder, table: &TablePlan) { - let product_type = ProductType { - elements: table - .columns - .iter() - .map(|col| ProductTypeElement { - name: Some(col.name.clone().into()), - algebraic_type: col.ty.to_algebraic(), - }) - .collect(), - }; - - let mut tbl = builder.build_table_with_new_type(table.name.clone(), product_type, true); - - tbl = tbl.with_type(TableType::User); - tbl = tbl.with_access(if table.is_public { - TableAccess::Public - } else { - TableAccess::Private - }); - // Primary key. - if let Some(pk) = table.primary_key { - tbl = tbl.with_primary_key(ColId(pk as u16)); - } - - // Unique constraints — all of them, including PK-matching. - for constraint in &table.unique_constraints { - let col_list: ColList = constraint.columns.iter().map(|&c| ColId(c as u16)).collect(); - tbl = tbl.with_unique_constraint(col_list); + pub fn with_bounds( + column: usize, + ty: Type, + start: i128, + min_value: i128, + max_value: i128, + increment: i128, + ) -> Option { + if !ty.is_integral() || increment == 0 || min_value >= max_value || start < min_value || start > max_value { + return None; + } + Some(Self { + column, + start: Some(start), + min_value: Some(min_value), + max_value: Some(max_value), + increment, + }) } - // Indexes. - for index in &table.indexes { - let col_list: ColList = index.columns.iter().map(|&c| ColId(c as u16)).collect(); + /// Build a small bounded sequence whose max is an already-observed value. + /// + /// This intentionally creates a high-risk migration surface: if migration + /// prechecks accept an unsafe sequence, later inserts can collide with + /// existing unique values. + pub fn with_existing_value_as_max(column: usize, ty: Type, existing_value: i128) -> Option { + const DOMAIN_SIZE: i128 = 3; - let algorithm = match index.algorithm { - IndexAlgorithm::BTree => RawIndexAlgorithm::BTree { columns: col_list }, - IndexAlgorithm::Hash => RawIndexAlgorithm::Hash { columns: col_list }, - }; - - let source_name = format!( - "{}_{}_idx", - table.name, - index - .columns - .iter() - .map(|&c| table.columns[c].name.as_str()) - .collect::>() - .join("_") - ); - - tbl = tbl.with_index_no_accessor_name(algorithm, source_name); - } + if existing_value < DOMAIN_SIZE { + return None; + } - // Sequences — all of them. - for seq in &table.sequences { - tbl = tbl.with_column_sequence(ColId(seq.column as u16)); + let min_value = existing_value - (DOMAIN_SIZE - 1); + Self::with_bounds(column, ty, min_value, min_value, existing_value, 1) } - - tbl.finish(); } /// Controls the shape of generated schemas. @@ -220,6 +201,9 @@ fn to_raw_def_table(builder: &mut RawModuleDefV10Builder, table: &TablePlan) { pub struct SchemaProfile { pub table_count: (usize, usize), pub columns: (usize, usize), + pub table_kind_weights: TableKindWeights, + pub type_weights: TypeWeights, + pub sum_variants: (usize, usize), pub pk_prob: f64, pub auto_inc_prob: f64, pub indexes: (usize, usize), @@ -228,21 +212,66 @@ pub struct SchemaProfile { pub private_prob: f64, } +impl SchemaProfile { + pub fn engine_dst() -> Self { + Self { + table_count: (3, 10), + columns: (1, 20), + table_kind_weights: TableKindWeights::default(), + type_weights: TypeWeights::default(), + sum_variants: (1, 4), + pk_prob: 0.65, + auto_inc_prob: 0.20, + indexes: (0, 5), + unique_constraints: (0, 3), + btree_prob: 0.65, + private_prob: 0.10, + } + } +} + impl Default for SchemaProfile { + fn default() -> Self { + Self::engine_dst() + } +} + +#[derive(Debug, Clone, Copy)] +pub struct TableKindWeights { + pub data: u64, + pub event: u64, +} + +impl Default for TableKindWeights { + fn default() -> Self { + Self { data: 9, event: 1 } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct TypeWeights { + pub bool_: u64, + pub i64_: u64, + pub u64_: u64, + pub string: u64, + pub bytes: u64, + pub sum: u64, +} + +impl Default for TypeWeights { fn default() -> Self { Self { - table_count: (1, 100), - columns: (1, 10), - pk_prob: 0.7, - auto_inc_prob: 0.3, - indexes: (0, 3), - unique_constraints: (0, 2), - btree_prob: 0.7, - private_prob: 0.1, + bool_: 12, + i64_: 24, + u64_: 28, + string: 16, + bytes: 12, + sum: 8, } } } +/// Random schema generator used by initial schema creation and add-table migrations. pub struct SchemaGenerator { rng: Rng, profile: SchemaProfile, @@ -253,60 +282,206 @@ impl SchemaGenerator { Self { rng, profile } } - fn range(&self, (lo, hi): (usize, usize)) -> usize { - if lo >= hi { - return lo; + /// Generate one table compatible with an existing schema. + /// + /// This is used by migration generation so add-table migrations use the same + /// shape policy as initial schema generation. Randomness happens before the + /// rewrite is applied; the rewrite itself remains deterministic. + pub fn gen_table_for_schema(&self, schema: &SchemaPlan, is_event: bool) -> TablePlan { + let mut sum_available = !schema_has_sum_column(schema); + self.gen_table(&schema.tables, is_event, &mut sum_available) + } + + /// Generate a complete schema from this generator's profile. + pub fn gen_schema(&self) -> SchemaPlan { + let table_count = SchemaDecisions::range(&self.rng, self.profile.table_count); + let mut tables: Vec = Vec::with_capacity(table_count); + let mut sum_available = true; + for table_idx in 0..table_count { + let must_be_data = table_idx + 1 == table_count && !tables.iter().any(|table| !table.is_event); + let is_event = !must_be_data && matches!(self.gen_table_kind(), TableKind::Event); + tables.push(self.gen_table(&tables, is_event, &mut sum_available)); + } + SchemaPlan { tables } + } +} + +/// Stable naming helpers for generated migration artifacts. +pub struct SchemaNames; + +impl SchemaNames { + pub fn fresh_column_name(table: &TablePlan, base: &str) -> String { + if table.columns.iter().all(|column| column.name != base) { + return base.into(); + } + + for suffix in 0.. { + let candidate = format!("{base}_{suffix}"); + if table.columns.iter().all(|column| column.name != candidate) { + return candidate; + } + } + + unreachable!("unbounded suffix search must find a unique column name") + } + + pub fn index_name(table: &TablePlan, index: &IndexPlan) -> String { + format!( + "{}_{}_idx", + table.name, + index + .columns + .iter() + .map(|&c| table.columns[c].name.as_str()) + .collect::>() + .join("_") + ) + } +} + +#[derive(Debug, Clone, Copy)] +enum TableKind { + Data, + Event, +} + +impl TableKindWeights { + fn choices(self) -> [rng::Choice; 2] { + [ + rng::choice(self.data, TableKind::Data), + rng::choice(self.event, TableKind::Event), + ] + } +} + +#[derive(Debug, Clone, Copy)] +enum TypeKind { + Bool, + I64, + U64, + String, + Bytes, + Sum, +} + +impl TypeWeights { + fn choices(self) -> [rng::Choice; 6] { + [ + rng::choice(self.bool_, TypeKind::Bool), + rng::choice(self.i64_, TypeKind::I64), + rng::choice(self.u64_, TypeKind::U64), + rng::choice(self.string, TypeKind::String), + rng::choice(self.bytes, TypeKind::Bytes), + rng::choice(self.sum, TypeKind::Sum), + ] + } + + fn non_sum_choices(self) -> [rng::Choice; 5] { + [ + rng::choice(self.bool_, TypeKind::Bool), + rng::choice(self.i64_, TypeKind::I64), + rng::choice(self.u64_, TypeKind::U64), + rng::choice(self.string, TypeKind::String), + rng::choice(self.bytes, TypeKind::Bytes), + ] + } +} + +struct SchemaDecisions; + +impl SchemaDecisions { + fn range(rng: &Rng, (lo, hi): (usize, usize)) -> usize { + rng::range_inclusive(rng, lo, hi) + } + + fn index(rng: &Rng, len: usize) -> usize { + rng::choose_index(rng, len).expect("len must be non-zero") + } + + fn sample_probability(rng: &Rng, probability: f64) -> bool { + rng.sample_probability(probability) + } + + fn gen_table_name(rng: &Rng, tables: &[TablePlan]) -> String { + loop { + let name = format!("tbl_{}", Self::gen_ident(rng)); + if tables.iter().all(|table| table.name != name) { + return name; + } } - lo + (self.rng.next_u64() as usize % (hi - lo + 1)) } - fn gen_type(&self) -> Type { - Type::ALL[self.rng.index(Type::ALL.len())] + fn gen_column_name(rng: &Rng, seen: &[String]) -> String { + loop { + let name = Self::gen_ident(rng); + if !seen.contains(&name) { + return name; + } + } } - fn gen_columns(&self) -> Vec { - let n = self.range(self.profile.columns); + fn gen_ident(rng: &Rng) -> String { + const CHARS: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789_"; + const FIRST: &[u8] = b"abcdefghijklmnopqrstuvwxyz_"; + let len = 4 + (rng.next_u64() as usize % 12); + let mut s = String::with_capacity(len); + s.push(FIRST[Self::index(rng, FIRST.len())] as char); + for _ in 1..len { + s.push(CHARS[Self::index(rng, CHARS.len())] as char); + } + s + } +} + +impl SchemaGenerator { + fn gen_columns(&self, sum_available: &mut bool) -> Vec { + let n = SchemaDecisions::range(&self.rng, self.profile.columns); let mut names = Vec::with_capacity(n); let mut seen = Vec::with_capacity(n); for _ in 0..n { - let name = self.gen_column_name(&seen); + let name = SchemaDecisions::gen_column_name(&self.rng, &seen); seen.push(name.clone()); names.push(ColumnPlan { name, - ty: self.gen_type(), + ty: self.gen_type(sum_available), }); } names } - fn gen_ident(&self) -> String { - const CHARS: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789_"; - const FIRST: &[u8] = b"abcdefghijklmnopqrstuvwxyz_"; - let len = 4 + (self.rng.next_u64() as usize % 12); - let mut s = String::with_capacity(len); - s.push(FIRST[self.rng.index(FIRST.len())] as char); - for _ in 1..len { - s.push(CHARS[self.rng.index(CHARS.len())] as char); - } - s - } + fn gen_type(&self, sum_available: &mut bool) -> Type { + let kind = if *sum_available { + let choices = self.profile.type_weights.choices(); + rng::pick_choice(&self.rng, &choices) + } else { + let choices = self.profile.type_weights.non_sum_choices(); + rng::pick_choice(&self.rng, &choices) + }; - fn gen_column_name(&self, seen: &[String]) -> String { - loop { - let name = self.gen_ident(); - if !seen.contains(&name) { - return name; + match kind { + TypeKind::Bool => Type::Bool, + TypeKind::I64 => Type::I64, + TypeKind::U64 => Type::U64, + TypeKind::String => Type::String, + TypeKind::Bytes => Type::Bytes, + TypeKind::Sum => { + *sum_available = false; + Type::Sum { + variants: SchemaDecisions::range(&self.rng, self.profile.sum_variants) as u8, + } } } } fn gen_unique_constraints(&self, columns: &[ColumnPlan], pk: &Option) -> Vec { - let n = self.range(self.profile.unique_constraints); + let n = SchemaDecisions::range(&self.rng, self.profile.unique_constraints); let mut seen: Vec> = Vec::new(); let mut result = Vec::new(); for _ in 0..n { - let num_cols = 1 + self.rng.index(columns.len().min(3)); - let mut cols: Vec = (0..num_cols).map(|_| self.rng.index(columns.len())).collect(); + let num_cols = 1 + SchemaDecisions::index(&self.rng, columns.len().min(3)); + let mut cols: Vec = (0..num_cols) + .map(|_| SchemaDecisions::index(&self.rng, columns.len())) + .collect(); cols.sort(); cols.dedup(); if !cols.is_empty() && !seen.contains(&cols) { @@ -314,11 +489,11 @@ impl SchemaGenerator { result.push(UniqueConstraintPlan { columns: cols }); } } - // Ensure PK has a matching unique constraint. - if let Some(pk) = pk - && !seen.iter().any(|cols| cols.len() == 1 && cols[0] == *pk) - { - result.push(UniqueConstraintPlan { columns: vec![*pk] }); + // A primary key always has a matching unique constraint. + if let Some(pk) = pk { + if !seen.iter().any(|cols| cols.len() == 1 && cols[0] == *pk) { + result.push(UniqueConstraintPlan { columns: vec![*pk] }); + } } result } @@ -329,11 +504,9 @@ impl SchemaGenerator { unique_constraints: &[UniqueConstraintPlan], pk: &Option, ) -> Vec { - // Every unique constraint and PK needs a matching index. let mut seen_cols: Vec> = Vec::new(); let mut indexes: Vec = Vec::new(); - // Index for PK. if let Some(pk) = pk { seen_cols.push(vec![*pk]); indexes.push(IndexPlan { @@ -342,7 +515,6 @@ impl SchemaGenerator { }); } - // Indexes for unique constraints. for constraint in unique_constraints { if seen_cols.contains(&constraint.columns) { continue; @@ -354,18 +526,19 @@ impl SchemaGenerator { }); } - // Additional random indexes. - let n = self.range(self.profile.indexes); + let n = SchemaDecisions::range(&self.rng, self.profile.indexes); for _ in 0..n { - let num_cols = 1 + self.rng.index(columns.len().min(3)); - let mut cols: Vec = (0..num_cols).map(|_| self.rng.index(columns.len())).collect(); + let num_cols = 1 + SchemaDecisions::index(&self.rng, columns.len().min(3)); + let mut cols: Vec = (0..num_cols) + .map(|_| SchemaDecisions::index(&self.rng, columns.len())) + .collect(); cols.sort(); cols.dedup(); if cols.is_empty() || seen_cols.contains(&cols) { continue; } seen_cols.push(cols.clone()); - let algorithm = if self.rng.sample_probability(self.profile.btree_prob) { + let algorithm = if SchemaDecisions::sample_probability(&self.rng, self.profile.btree_prob) { IndexAlgorithm::BTree } else { IndexAlgorithm::Hash @@ -379,11 +552,27 @@ impl SchemaGenerator { indexes } - fn gen_table(&self, _table_index: usize) -> TablePlan { - let columns = self.gen_columns(); + fn gen_table(&self, existing_tables: &[TablePlan], is_event: bool, sum_available: &mut bool) -> TablePlan { + let columns = self.gen_columns(sum_available); + let name = SchemaDecisions::gen_table_name(&self.rng, existing_tables); + let is_public = !SchemaDecisions::sample_probability(&self.rng, self.profile.private_prob); - let primary_key = if self.rng.sample_probability(self.profile.pk_prob) && !columns.is_empty() { - Some(self.rng.index(columns.len())) + if is_event { + return TablePlan { + name, + columns, + primary_key: None, + indexes: vec![], + unique_constraints: vec![], + sequences: vec![], + is_public, + is_event: true, + }; + } + + let primary_key = if SchemaDecisions::sample_probability(&self.rng, self.profile.pk_prob) && !columns.is_empty() + { + Some(SchemaDecisions::index(&self.rng, columns.len())) } else { None }; @@ -391,7 +580,9 @@ impl SchemaGenerator { let unique_constraints = self.gen_unique_constraints(&columns, &primary_key); let sequences = if let Some(pk) = primary_key { - if columns[pk].ty.is_integral() && self.rng.sample_probability(self.profile.auto_inc_prob) { + if columns[pk].ty.is_integral() + && SchemaDecisions::sample_probability(&self.rng, self.profile.auto_inc_prob) + { SequencePlan::new(pk, columns[pk].ty).into_iter().collect() } else { vec![] @@ -402,8 +593,6 @@ impl SchemaGenerator { let indexes = self.gen_indexes(&columns, &unique_constraints, &primary_key); - let name = format!("tbl_{}", self.gen_ident()); - TablePlan { name, columns, @@ -411,15 +600,89 @@ impl SchemaGenerator { indexes, unique_constraints, sequences, - is_public: !self.rng.sample_probability(self.profile.private_prob), + is_public, + is_event: false, } } - pub fn gen_schema(&self) -> SchemaPlan { - let table_count = self.range(self.profile.table_count); - let tables = (0..table_count).map(|i| self.gen_table(i)).collect(); - SchemaPlan { tables } + fn gen_table_kind(&self) -> TableKind { + let choices = self.profile.table_kind_weights.choices(); + rng::pick_choice(&self.rng, &choices) + } +} + +fn apply_sequence_bounds(schema: &SchemaPlan, raw: &mut RawModuleDefV10) { + for (table_plan, raw_table) in schema.tables.iter().zip(raw.tables_mut_for_tests().iter_mut()) { + for (sequence_plan, raw_sequence) in table_plan.sequences.iter().zip(raw_table.sequences.iter_mut()) { + raw_sequence.start = sequence_plan.start; + raw_sequence.min_value = sequence_plan.min_value; + raw_sequence.max_value = sequence_plan.max_value; + raw_sequence.increment = sequence_plan.increment; + } + } +} + +fn to_raw_def_table(builder: &mut RawModuleDefV10Builder, table: &TablePlan) { + let product_type = ProductType { + elements: table + .columns + .iter() + .map(|col| ProductTypeElement { + name: Some(col.name.clone().into()), + algebraic_type: col.ty.to_algebraic(), + }) + .collect(), + }; + + let mut tbl = builder.build_table_with_new_type_for_tests(table.name.clone(), product_type, true); + + tbl = tbl.with_type(TableType::User); + tbl = tbl.with_event(table.is_event); + tbl = tbl.with_access(if table.is_public { + TableAccess::Public + } else { + TableAccess::Private + }); + + if let Some(pk) = table.primary_key { + tbl = tbl.with_primary_key(ColId(pk as u16)); } + + for constraint in &table.unique_constraints { + let col_list: ColList = constraint.columns.iter().map(|&c| ColId(c as u16)).collect(); + tbl = tbl.with_unique_constraint(col_list); + } + + for index in &table.indexes { + let col_list: ColList = index.columns.iter().map(|&c| ColId(c as u16)).collect(); + + let algorithm = match index.algorithm { + IndexAlgorithm::BTree => RawIndexAlgorithm::BTree { columns: col_list }, + IndexAlgorithm::Hash => RawIndexAlgorithm::Hash { columns: col_list }, + }; + + tbl = tbl.with_index_no_accessor_name(algorithm, SchemaNames::index_name(table, index)); + } + + for seq in &table.sequences { + tbl = tbl.with_column_sequence(ColId(seq.column as u16)); + } + + // AddColumns needs defaults when existing rows are present. Supplying stable + // defaults for all columns lets the engine keep only the newly-added tail. + for (col_id, column) in table.columns.iter().enumerate() { + tbl = tbl.with_default_column_value(ColId(col_id as u16), column.ty.default_value()); + } + + tbl.finish(); +} + +fn schema_has_sum_column(schema: &SchemaPlan) -> bool { + schema + .tables + .iter() + .flat_map(|table| table.columns.iter()) + .any(|column| matches!(column.ty, Type::Sum { .. })) } #[cfg(test)] @@ -453,6 +716,7 @@ mod tests { unique_constraints: vec![UniqueConstraintPlan { columns: vec![0] }], sequences: vec![SequencePlan::new(0, Type::U64).unwrap()], is_public: true, + is_event: false, }], }; diff --git a/crates/dst/src/traits.rs b/crates/dst/src/traits.rs index 2185e0ec918..3941e9c0cc9 100644 --- a/crates/dst/src/traits.rs +++ b/crates/dst/src/traits.rs @@ -1,4 +1,10 @@ -use anyhow::Error; +use std::{ + fmt::Debug, + panic::{resume_unwind, AssertUnwindSafe}, +}; + +use anyhow::{Context, Error}; +use futures::FutureExt; use spacetimedb_runtime::sim::Rng; /// This should be implemented by System under test. @@ -16,6 +22,16 @@ pub trait Properties { fn observe(&mut self, interaction: &I, observation: &O) -> Result<(), Error>; } +/// Stateful source of interactions that can incorporate target feedback. +pub trait InteractionGen { + type Interaction: Debug; + type Observation; + + fn next_interaction(&mut self) -> Option; + + fn observe(&mut self, interaction: &Self::Interaction, observation: &Self::Observation) -> Result<(), Error>; +} + pub type TestSuiteParts = ( ::Interactions, ::Target, @@ -23,8 +39,11 @@ pub type TestSuiteParts = ( ); pub trait TestSuite { - type Interaction: std::fmt::Debug; - type Interactions: Iterator + std::fmt::Debug; + type Interaction: Debug; + type Interactions: InteractionGen< + Interaction = Self::Interaction, + Observation = >::Observation, + > + Debug; type Target: TargetDriver; type Properties: Properties>::Observation>; @@ -39,19 +58,38 @@ pub trait TestSuite { async move { let (mut interactions, mut target, mut properties) = self.build(rng).await?; - let result = async { - for interaction in interactions.by_ref().take(max_interactions) { - let observation = target.execute(&interaction).await?; - properties.observe(&interaction, &observation)?; + let result = AssertUnwindSafe(async { + for step in 0..max_interactions { + let Some(interaction) = interactions.next_interaction() else { + break; + }; + + let observation = target + .execute(&interaction) + .await + .with_context(|| format!("DST target failed at interaction #{step}: {interaction:?}"))?; + + properties + .observe(&interaction, &observation) + .with_context(|| format!("DST property failed at interaction #{step}: {interaction:?}"))?; + + interactions.observe(&interaction, &observation).with_context(|| { + format!("DST generator feedback failed at interaction #{step}: {interaction:?}") + })?; } Ok(()) - } + }) + .catch_unwind() .await; + eprintln!("final interaction counts: {interactions:?}"); tracing::info!(interaction_counts = ?interactions, "final interaction counts"); - result + match result { + Ok(result) => result, + Err(payload) => resume_unwind(payload), + } } } } From d441285f09a9fc4bfa5034474fa8fa64be5065dc Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 29 Jul 2026 13:11:02 +0530 Subject: [PATCH 2/6] snake case --- crates/dst/src/engine/generation.rs | 744 +++++++++++++++++++--------- crates/dst/src/engine/migrations.rs | 26 +- crates/dst/src/engine/model.rs | 83 +++- crates/dst/src/rng.rs | 37 +- crates/dst/src/schema.rs | 3 +- 5 files changed, 640 insertions(+), 253 deletions(-) diff --git a/crates/dst/src/engine/generation.rs b/crates/dst/src/engine/generation.rs index 899b048ecff..3a83f7249ee 100644 --- a/crates/dst/src/engine/generation.rs +++ b/crates/dst/src/engine/generation.rs @@ -1,14 +1,26 @@ //! Workload value and migration generation for the engine DST driver. +//! +//! Read this file as policy first, mechanics second: +//! - `ValueGen::gen_insert_row` chooses the row-level insert shape: valid row, +//! whole-row duplicate, arbitrary candidate, or targeted uniqueness conflict. +//! - `ValueGen::gen_value_for_case` chooses one generated column value. The +//! type-specific helpers below it are mostly curated sample sets. +//! - `MigrationGen` only chooses accepted vs. rejected migration work; concrete +//! schema rewrite rules live in `migrations.rs`. use spacetimedb_lib::{AlgebraicValue, ProductValue}; use spacetimedb_runtime::sim::Rng; use spacetimedb_sats::ArrayValue; use super::migrations::{Migration, MigrationMode}; -use super::model::{ColumnDomain, Model}; +use super::model::Model; use super::row::Row; use crate::rng::{choice, Choice, WeightedChoice}; -use crate::schema::Type; +use crate::schema::{TablePlan, Type}; + +// Bound the valid-insert search: random generation may collide with existing +// unique constraints, but a failed search must not stall the workload. +const INSERT_CANDIDATE_ATTEMPTS: usize = 32; /// Read-only generation context for one model state. pub(crate) struct GenCtx<'a> { @@ -30,106 +42,348 @@ impl<'a> GenCtx<'a> { } } +// Row-level insert cases choose what kind of insert operation to try. #[derive(Clone, Copy)] -enum ValueCase { +enum InsertRowCase { + Valid, + AnyCandidate, + ExistingRow, + UniqueConflict, +} + +impl InsertRowCase { + const CHOICES: [Choice; 4] = [ + choice(80, Self::Valid), + choice(10, Self::AnyCandidate), + choice(5, Self::ExistingRow), + choice(5, Self::UniqueConflict), + ]; +} + +impl WeightedChoice for InsertRowCase {} + +// Column-level cases choose how to synthesize each non-sequence column value. +#[derive(Clone, Copy)] +enum ColumnValueCase { Random, Small, Edge, - NearExisting, - Existing, Weird, + Existing, + NearExisting, } -impl WeightedChoice for ValueCase { - const CHOICES: &'static [Choice] = &[ +impl ColumnValueCase { + const CHOICES: [Choice; 6] = [ choice(45, Self::Random), choice(15, Self::Small), choice(15, Self::Edge), - choice(10, Self::NearExisting), - choice(10, Self::Existing), choice(5, Self::Weird), + choice(10, Self::Existing), + choice(10, Self::NearExisting), ]; } +impl WeightedChoice for ColumnValueCase {} + #[derive(Clone, Copy)] -enum FreshValueCase { - Random, - Small, - Edge, - Weird, +enum ExistingValueScope { + SameColumn, + SameTable, + AnyTable, } -impl WeightedChoice for FreshValueCase { - const CHOICES: &'static [Choice] = &[ - choice(50, Self::Random), - choice(20, Self::Small), - choice(20, Self::Edge), - choice(10, Self::Weird), +impl ExistingValueScope { + const CHOICES: [Choice; 3] = [ + choice(55, Self::SameColumn), + choice(25, Self::SameTable), + choice(20, Self::AnyTable), ]; } -#[derive(Clone, Copy)] -enum I64Case { - Random, - Small, - Edge, +impl WeightedChoice for ExistingValueScope {} + +trait TypeValueGen { + fn random(&self, rng: &Rng) -> AlgebraicValue; + fn small(&self, rng: &Rng) -> AlgebraicValue; + fn edge(&self, rng: &Rng) -> AlgebraicValue; + fn weird(&self, rng: &Rng) -> AlgebraicValue; + fn counter(&self, counter: u64) -> AlgebraicValue; + fn near(&self, value: AlgebraicValue) -> AlgebraicValue; + fn matches(&self, value: &AlgebraicValue) -> bool; } -#[derive(Clone, Copy)] -enum U64Case { - Random, - Small, - Edge, +struct BoolGen; +struct I64Gen; +struct U64Gen; +struct StringGen; +struct BytesGen; +struct SumGen { + variants: u8, } -#[derive(Clone, Copy)] -enum StringCase { - RandomTagged, - Empty, - SmallAscii, - OrderedPrefix, - SqlEscaped, - NullByte, - Long, +impl TypeValueGen for BoolGen { + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::Bool(rng.next_u64().is_multiple_of(2)) + } + + fn small(&self, _rng: &Rng) -> AlgebraicValue { + AlgebraicValue::Bool(false) + } + + fn edge(&self, _rng: &Rng) -> AlgebraicValue { + AlgebraicValue::Bool(true) + } + + fn weird(&self, rng: &Rng) -> AlgebraicValue { + self.edge(rng) + } + + fn counter(&self, counter: u64) -> AlgebraicValue { + AlgebraicValue::Bool(counter.is_multiple_of(2)) + } + + fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + match value { + AlgebraicValue::Bool(value) => AlgebraicValue::Bool(!value), + other => other, + } + } + + fn matches(&self, value: &AlgebraicValue) -> bool { + matches!(value, AlgebraicValue::Bool(_)) + } } -impl StringCase { - const WEIRD_CHOICES: &'static [Choice] = &[ - choice(35, Self::SqlEscaped), - choice(25, Self::NullByte), - choice(25, Self::OrderedPrefix), - choice(15, Self::Empty), - ]; +impl TypeValueGen for I64Gen { + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::I64(rng.next_u64() as i64) + } + + fn small(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::I64(sample(rng, &[-3, -2, -1, 0, 1, 2, 3])) + } + + fn edge(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::I64(sample(rng, &[i64::MIN, i64::MIN + 1, -1, 0, 1, i64::MAX - 1, i64::MAX])) + } + + fn weird(&self, rng: &Rng) -> AlgebraicValue { + self.edge(rng) + } - fn pick_weird(rng: &Rng) -> Self { - crate::rng::pick_choice(rng, Self::WEIRD_CHOICES) + fn counter(&self, counter: u64) -> AlgebraicValue { + AlgebraicValue::I64(counter as i64) + } + + fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + match value { + AlgebraicValue::I64(value) => AlgebraicValue::I64(value.saturating_add(1)), + other => other, + } + } + + fn matches(&self, value: &AlgebraicValue) -> bool { + matches!(value, AlgebraicValue::I64(_)) } } -#[derive(Clone, Copy)] -enum BytesCase { - Random, - Empty, - Small, - RepeatedZero, - RepeatedMax, - Alternating, +impl TypeValueGen for U64Gen { + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::U64(rng.next_u64()) + } + + fn small(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::U64(sample(rng, &[0, 1, 2, 3, 4, 5])) + } + + fn edge(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::U64(sample(rng, &[0, 1, 2, u64::MAX - 1, u64::MAX])) + } + + fn weird(&self, rng: &Rng) -> AlgebraicValue { + self.edge(rng) + } + + fn counter(&self, counter: u64) -> AlgebraicValue { + AlgebraicValue::U64(counter) + } + + fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + match value { + AlgebraicValue::U64(value) => AlgebraicValue::U64(value.saturating_add(1)), + other => other, + } + } + + fn matches(&self, value: &AlgebraicValue) -> bool { + matches!(value, AlgebraicValue::U64(_)) + } } -impl BytesCase { - const WEIRD_CHOICES: &'static [Choice] = &[ - choice(25, Self::Empty), - choice(20, Self::RepeatedZero), - choice(20, Self::RepeatedMax), - choice(20, Self::Alternating), - choice(15, Self::Small), - ]; +impl TypeValueGen for StringGen { + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::String(format!("v_{}", rng.next_u64()).into()) + } + + fn small(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::String( + sample(rng, &["a", "aa", "ab", "b", "z", "v_0", "v_1"]) + .to_owned() + .into(), + ) + } + + fn edge(&self, _rng: &Rng) -> AlgebraicValue { + AlgebraicValue::String("x".repeat(128).into()) + } - fn pick_weird(rng: &Rng) -> Self { - crate::rng::pick_choice(rng, Self::WEIRD_CHOICES) + fn weird(&self, rng: &Rng) -> AlgebraicValue { + let value = match rng.index(100) { + 0..35 => sample(rng, &["quote'", "double\"quote", "back\\slash", "line\nbreak"]).to_owned(), + 35..60 => "nul\0byte".to_owned(), + 60..85 => sample(rng, &["a", "aa", "aaa", "ab", "aba", "abb", "b"]).to_owned(), + _ => String::new(), + }; + AlgebraicValue::String(value.into()) + } + + fn counter(&self, counter: u64) -> AlgebraicValue { + AlgebraicValue::String(format!("fresh_{counter}").into()) + } + + fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + match value { + AlgebraicValue::String(value) => { + let mut value = value.to_string(); + value.push('a'); + AlgebraicValue::String(value.into()) + } + other => other, + } + } + + fn matches(&self, value: &AlgebraicValue) -> bool { + matches!(value, AlgebraicValue::String(_)) } } +impl TypeValueGen for BytesGen { + fn random(&self, rng: &Rng) -> AlgebraicValue { + let len = (rng.next_u64() % 16) as usize; + let value = (0..len).map(|_| rng.next_u64() as u8).collect::>(); + AlgebraicValue::Array(ArrayValue::U8(value.into())) + } + + fn small(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::Array(ArrayValue::U8( + sample(rng, &[&[][..], &[0][..], &[1][..], &[0, 255][..]]) + .to_vec() + .into(), + )) + } + + fn edge(&self, _rng: &Rng) -> AlgebraicValue { + AlgebraicValue::Array(ArrayValue::U8(vec![255; 32].into())) + } + + fn weird(&self, rng: &Rng) -> AlgebraicValue { + let value = match rng.index(100) { + 0..25 => Vec::new(), + 25..45 => vec![0; 32], + 45..65 => vec![255; 32], + 65..85 => vec![0, 255, 0, 255, 0, 255], + _ => sample(rng, &[&[][..], &[0][..], &[1][..], &[0, 255][..]]).to_vec(), + }; + AlgebraicValue::Array(ArrayValue::U8(value.into())) + } + + fn counter(&self, counter: u64) -> AlgebraicValue { + AlgebraicValue::Array(ArrayValue::U8(counter.to_le_bytes().to_vec().into())) + } + + fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + match value { + AlgebraicValue::Array(ArrayValue::U8(value)) => { + let mut value = value.to_vec(); + value.push(0); + AlgebraicValue::Array(ArrayValue::U8(value.into())) + } + other => other, + } + } + + fn matches(&self, value: &AlgebraicValue) -> bool { + matches!(value, AlgebraicValue::Array(ArrayValue::U8(_))) + } +} + +impl TypeValueGen for SumGen { + fn random(&self, rng: &Rng) -> AlgebraicValue { + let tag = rng.index(self.variants as usize) as u8; + AlgebraicValue::sum(tag, AlgebraicValue::U8(rng.next_u64() as u8)) + } + + fn small(&self, rng: &Rng) -> AlgebraicValue { + let tag = if self.variants <= 1 { + 0 + } else { + rng.index(self.variants as usize) as u8 + }; + AlgebraicValue::sum(tag, AlgebraicValue::U8(0)) + } + + fn edge(&self, _rng: &Rng) -> AlgebraicValue { + AlgebraicValue::sum(self.variants.saturating_sub(1), AlgebraicValue::U8(u8::MAX)) + } + + fn weird(&self, rng: &Rng) -> AlgebraicValue { + self.edge(rng) + } + + fn counter(&self, counter: u64) -> AlgebraicValue { + AlgebraicValue::sum( + if self.variants == 0 { + 0 + } else { + (counter % self.variants as u64) as u8 + }, + AlgebraicValue::U8(counter as u8), + ) + } + + fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + match value { + AlgebraicValue::Sum(sum) if self.variants > 0 => { + AlgebraicValue::sum(sum.tag.wrapping_add(1) % self.variants, *sum.value) + } + other => other, + } + } + + fn matches(&self, value: &AlgebraicValue) -> bool { + matches!(value, AlgebraicValue::Sum(sum) if sum.tag < self.variants) + } +} + +fn generator_for(ty: Type) -> Box { + match ty { + Type::Bool => Box::new(BoolGen), + Type::I64 => Box::new(I64Gen), + Type::U64 => Box::new(U64Gen), + Type::String => Box::new(StringGen), + Type::Bytes => Box::new(BytesGen), + Type::Sum { variants } => Box::new(SumGen { variants }), + } +} + +fn sample(rng: &Rng, values: &[T]) -> T { + values[rng.index(values.len())] +} + +// Row generation deliberately mixes normal traffic with rows the engine should +// reject. The model is the oracle for whether a candidate is valid; this module +// only decides which kind of pressure to apply. struct ValueGen<'a> { rng: &'a Rng, model: &'a Model, @@ -141,221 +395,257 @@ impl<'a> ValueGen<'a> { } fn gen_insert_row(&self, table: usize) -> Row { - self.model.schema().tables[table] + // Start here when tuning insert behavior; everything below materializes + // one arm of this match. + match InsertRowCase::pick(self.rng, &InsertRowCase::CHOICES) { + InsertRowCase::Valid => self.gen_valid_insert_row(table), + InsertRowCase::AnyCandidate => self.gen_row_candidate(table), + InsertRowCase::ExistingRow => self + .existing_row(table) + .unwrap_or_else(|| self.gen_valid_insert_row(table)), + InsertRowCase::UniqueConflict => self + .unique_conflict_row(table) + .unwrap_or_else(|| self.gen_valid_insert_row(table)), + } + } + + fn gen_valid_insert_row(&self, table: usize) -> Row { + // The valid path still begins with ordinary candidates so it samples the + // same value domain as rejected paths. Counter rows are only the + // progress fallback. + for _ in 0..INSERT_CANDIDATE_ATTEMPTS { + let row = self.gen_row_candidate(table); + if !self.model.insert_would_violate_unique_constraint(table, &row) { + return row; + } + } + + self.gen_counter_row(table, self.rng.next_u64()) + } + + fn gen_row_candidate(&self, table: usize) -> Row { + self.table(table) + .columns + .iter() + .enumerate() + .map(|(column, column_plan)| self.gen_insert_value(table, column, column_plan.ty)) + .collect::() + } + + fn gen_counter_row(&self, table: usize, counter: u64) -> Row { + self.table(table) .columns .iter() .enumerate() - .map(|(column, _)| { - let domain = self.model.column_domain(table, column); - self.gen_insert_value(&domain) + .map(|(column, column_plan)| { + if self.is_sequence_column(table, column) { + sequence_placeholder(column_plan.ty) + } else { + self.gen_counter_value(column_plan.ty, counter.wrapping_add(column as u64)) + } }) .collect::() } - fn gen_insert_value(&self, domain: &ColumnDomain) -> AlgebraicValue { - if domain.sequenced { - return sequence_placeholder(domain.ty); + fn gen_insert_value(&self, table: usize, column: usize, ty: Type) -> AlgebraicValue { + if self.is_sequence_column(table, column) { + return sequence_placeholder(ty); } - if domain.unique { - return self.gen_fresh_value(domain); - } + self.gen_value(table, column, ty) + } - self.gen_value(domain) - } - - fn gen_value(&self, domain: &ColumnDomain) -> AlgebraicValue { - match ValueCase::pick(self.rng) { - ValueCase::Random => self.gen_random_value(domain.ty), - ValueCase::Small => self.gen_small_value(domain.ty), - ValueCase::Edge => self.gen_edge_value(domain.ty), - ValueCase::NearExisting => self - .near_existing_value(domain) - .unwrap_or_else(|| self.gen_random_value(domain.ty)), - ValueCase::Existing => self - .existing_value(domain) - .unwrap_or_else(|| self.gen_random_value(domain.ty)), - ValueCase::Weird => self.gen_weird_value(domain.ty), - } + fn gen_value(&self, table: usize, column: usize, ty: Type) -> AlgebraicValue { + let case = ColumnValueCase::pick(self.rng, &ColumnValueCase::CHOICES); + self.gen_value_for_case(table, column, ty, case).unwrap_or_else(|| { + // Existing-value cases can fail when the model is empty or no + // visible value matches the requested type. + self.gen_value_for_case(table, column, ty, ColumnValueCase::Random) + .expect("random value generation cannot fail") + }) } - fn gen_fresh_value(&self, domain: &ColumnDomain) -> AlgebraicValue { - for _ in 0..32 { - let value = self.gen_fresh_candidate(domain.ty); - if !domain.values.contains(&value) { - return value; - } + fn gen_value_for_case( + &self, + table: usize, + column: usize, + ty: Type, + case: ColumnValueCase, + ) -> Option { + let type_gen = generator_for(ty); + match case { + ColumnValueCase::Random => Some(type_gen.random(self.rng)), + ColumnValueCase::Small => Some(type_gen.small(self.rng)), + ColumnValueCase::Edge => Some(type_gen.edge(self.rng)), + ColumnValueCase::Weird => Some(type_gen.weird(self.rng)), + ColumnValueCase::Existing => self.existing_value(table, column, ty), + ColumnValueCase::NearExisting => self.near_existing_value(table, column, ty), } + } - self.gen_counter_value(domain.ty, self.rng.next_u64()) + fn existing_row(&self, table: usize) -> Option { + let count = self.model.row_count(table); + (count > 0).then(|| { + self.model + .row(table, self.rng.index(count)) + .expect("sampled row index is in bounds") + .clone() + }) } - fn gen_fresh_candidate(&self, ty: Type) -> AlgebraicValue { - match FreshValueCase::pick(self.rng) { - FreshValueCase::Random => self.gen_random_value(ty), - FreshValueCase::Small => self.gen_small_value(ty), - FreshValueCase::Edge => self.gen_edge_value(ty), - FreshValueCase::Weird => self.gen_weird_value(ty), + fn unique_conflict_row(&self, table: usize) -> Option { + // Target a specific unique constraint without turning the operation into + // an exact-row duplicate: copy constrained columns from an existing row, + // then perturb an unconstrained column if necessary. + let constraints = &self.table(table).unique_constraints; + if constraints.is_empty() || self.model.row_count(table) == 0 { + return None; } - } - fn existing_value(&self, domain: &ColumnDomain) -> Option { - (!domain.values.is_empty()).then(|| domain.values[self.rng.index(domain.values.len())].clone()) - } + let start = self.rng.index(constraints.len()); + for offset in 0..constraints.len() { + let constraint = &constraints[(start + offset) % constraints.len()]; + let base = self.existing_row(table)?; + let mut row = self.gen_row_candidate(table); - fn near_existing_value(&self, domain: &ColumnDomain) -> Option { - let existing = self.existing_value(domain)?; - Some(match (domain.ty, existing) { - (Type::Bool, AlgebraicValue::Bool(value)) => AlgebraicValue::Bool(!value), - (Type::I64, AlgebraicValue::I64(value)) => AlgebraicValue::I64(value.saturating_add(1)), - (Type::U64, AlgebraicValue::U64(value)) => AlgebraicValue::U64(value.saturating_add(1)), - (Type::String, AlgebraicValue::String(value)) => { - let mut value = value.to_string(); - value.push('a'); - AlgebraicValue::String(value.into()) + for &column in &constraint.columns { + row.elements[column] = base.elements[column].clone(); } - (Type::Bytes, AlgebraicValue::Array(ArrayValue::U8(value))) => { - let mut value = value.to_vec(); - value.push(0); - AlgebraicValue::Array(ArrayValue::U8(value.into())) + + if row == base { + let Some(column) = + (0..self.table(table).columns.len()).find(|column| !constraint.columns.contains(column)) + else { + continue; + }; + row.elements[column] = + self.near_value(self.table(table).columns[column].ty, base.elements[column].clone()); } - (Type::Sum { .. }, value) => value, - (_, value) => value, - }) - } - fn gen_random_value(&self, ty: Type) -> AlgebraicValue { - match ty { - Type::Bool => AlgebraicValue::Bool(self.rng.next_u64().is_multiple_of(2)), - Type::I64 => AlgebraicValue::I64(self.gen_i64_value(I64Case::Random)), - Type::U64 => AlgebraicValue::U64(self.gen_u64_value(U64Case::Random)), - Type::String => AlgebraicValue::String(self.gen_string_value(StringCase::RandomTagged).into()), - Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(BytesCase::Random).into())), - Type::Sum { variants } => { - let tag = self.rng.index(variants as usize) as u8; - AlgebraicValue::sum(tag, AlgebraicValue::U8(self.rng.next_u64() as u8)) + if row != base && self.model.insert_would_violate_unique_constraint(table, &row) { + return Some(row); } } + + None } - fn gen_small_value(&self, ty: Type) -> AlgebraicValue { - match ty { - Type::Bool => AlgebraicValue::Bool(false), - Type::I64 => AlgebraicValue::I64(self.gen_i64_value(I64Case::Small)), - Type::U64 => AlgebraicValue::U64(self.gen_u64_value(U64Case::Small)), - Type::String => AlgebraicValue::String(self.gen_string_value(StringCase::SmallAscii).into()), - Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(BytesCase::Small).into())), - Type::Sum { variants } => { - let tag = if variants <= 1 { - 0 - } else { - self.rng.index(variants as usize) as u8 - }; - AlgebraicValue::sum(tag, AlgebraicValue::U8(0)) - } - } + fn existing_value(&self, table: usize, column: usize, ty: Type) -> Option { + // Existing-value generation is runtime-value based, not column-domain + // based. After migrations, the interesting reusable value may live in a + // different column or table as long as its SATS shape still fits `ty`. + let scope = ExistingValueScope::pick(self.rng, &ExistingValueScope::CHOICES); + self.existing_value_in_scope(table, column, ty, scope) + .or_else(|| self.existing_value_in_scope(table, column, ty, ExistingValueScope::SameColumn)) + .or_else(|| self.existing_value_in_scope(table, column, ty, ExistingValueScope::SameTable)) + .or_else(|| self.existing_value_in_scope(table, column, ty, ExistingValueScope::AnyTable)) } - fn gen_edge_value(&self, ty: Type) -> AlgebraicValue { - match ty { - Type::Bool => AlgebraicValue::Bool(true), - Type::I64 => AlgebraicValue::I64(self.gen_i64_value(I64Case::Edge)), - Type::U64 => AlgebraicValue::U64(self.gen_u64_value(U64Case::Edge)), - Type::String => AlgebraicValue::String(self.gen_string_value(StringCase::Long).into()), - Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(BytesCase::RepeatedMax).into())), - Type::Sum { variants } => AlgebraicValue::sum(variants.saturating_sub(1), AlgebraicValue::U8(u8::MAX)), + fn existing_value_in_scope( + &self, + table: usize, + column: usize, + ty: Type, + scope: ExistingValueScope, + ) -> Option { + match scope { + ExistingValueScope::SameColumn => self.existing_column_value(table, column, ty), + ExistingValueScope::SameTable => self.existing_table_value(table, ty), + ExistingValueScope::AnyTable => self.existing_any_table_value(ty), } } - fn gen_weird_value(&self, ty: Type) -> AlgebraicValue { - match ty { - Type::String => { - let case = StringCase::pick_weird(self.rng); - AlgebraicValue::String(self.gen_string_value(case).into()) - } - Type::Bytes => { - let case = BytesCase::pick_weird(self.rng); - AlgebraicValue::Array(ArrayValue::U8(self.gen_bytes_value(case).into())) - } - _ => self.gen_edge_value(ty), + fn existing_column_value(&self, table: usize, column: usize, ty: Type) -> Option { + let count = self.model.row_count(table); + if count == 0 { + return None; } - } - fn gen_i64_value(&self, case: I64Case) -> i64 { - match case { - I64Case::Random => self.rng.next_u64() as i64, - I64Case::Small => self.sample(&[-3, -2, -1, 0, 1, 2, 3]), - I64Case::Edge => self.sample(&[i64::MIN, i64::MIN + 1, -1, 0, 1, i64::MAX - 1, i64::MAX]), + let start = self.rng.index(count); + for offset in 0..count { + let row = self.model.row(table, (start + offset) % count)?; + let value = &row.elements[column]; + if value_matches_type(ty, value) { + return Some(value.clone()); + } } + + None } - fn gen_u64_value(&self, case: U64Case) -> u64 { - match case { - U64Case::Random => self.rng.next_u64(), - U64Case::Small => self.sample(&[0, 1, 2, 3, 4, 5]), - U64Case::Edge => self.sample(&[0, 1, 2, u64::MAX - 1, u64::MAX]), - } + fn existing_table_value(&self, table: usize, ty: Type) -> Option { + let values = self.visible_values_in_tables(ty, table..table + 1); + (!values.is_empty()).then(|| values[self.rng.index(values.len())].clone()) } - fn gen_string_value(&self, case: StringCase) -> String { - match case { - StringCase::RandomTagged => format!("v_{}", self.rng.next_u64()), - StringCase::Empty => String::new(), - StringCase::SmallAscii => self.sample(&["a", "aa", "ab", "b", "z", "v_0", "v_1"]).to_owned(), - StringCase::OrderedPrefix => self.sample(&["a", "aa", "aaa", "ab", "aba", "abb", "b"]).to_owned(), - StringCase::SqlEscaped => self - .sample(&["quote'", "double\"quote", "back\\slash", "line\nbreak"]) - .to_owned(), - StringCase::NullByte => "nul\0byte".to_owned(), - StringCase::Long => "x".repeat(128), - } + fn existing_any_table_value(&self, ty: Type) -> Option { + let values = self.visible_values_in_tables(ty, 0..self.model.schema().tables.len()); + (!values.is_empty()).then(|| values[self.rng.index(values.len())].clone()) } - fn gen_bytes_value(&self, case: BytesCase) -> Vec { - match case { - BytesCase::Random => { - let len = (self.rng.next_u64() % 16) as usize; - (0..len).map(|_| self.rng.next_u64() as u8).collect() + fn visible_values_in_tables(&self, ty: Type, tables: impl Iterator) -> Vec { + // Scan stored rows by runtime type compatibility. This is what lets + // existing-value reuse survive schema rewrites and sum-variant changes. + let mut values = Vec::new(); + for table in tables { + let table_plan = self.table(table); + for row_idx in 0..self.model.row_count(table) { + let row = self.model.row(table, row_idx).expect("row index is in bounds"); + for (column, _column_plan) in table_plan.columns.iter().enumerate() { + let value = &row.elements[column]; + if value_matches_type(ty, value) { + values.push(value.clone()); + } + } } - BytesCase::Empty => Vec::new(), - BytesCase::Small => self.sample(&[&[][..], &[0][..], &[1][..], &[0, 255][..]]).to_vec(), - BytesCase::RepeatedZero => vec![0; 32], - BytesCase::RepeatedMax => vec![255; 32], - BytesCase::Alternating => vec![0, 255, 0, 255, 0, 255], } + values + } + + fn near_existing_value(&self, table: usize, column: usize, ty: Type) -> Option { + self.existing_value(table, column, ty) + .map(|value| self.near_value(ty, value)) + } + + fn near_value(&self, ty: Type, value: AlgebraicValue) -> AlgebraicValue { + generator_for(ty).near(value) } fn gen_counter_value(&self, ty: Type, counter: u64) -> AlgebraicValue { - match ty { - Type::Bool => AlgebraicValue::Bool(counter.is_multiple_of(2)), - Type::I64 => AlgebraicValue::I64(counter as i64), - Type::U64 => AlgebraicValue::U64(counter), - Type::String => AlgebraicValue::String(format!("fresh_{counter}").into()), - Type::Bytes => AlgebraicValue::Array(ArrayValue::U8(counter.to_le_bytes().to_vec().into())), - Type::Sum { variants } => AlgebraicValue::sum( - if variants == 0 { - 0 - } else { - (counter % variants as u64) as u8 - }, - AlgebraicValue::U8(counter as u8), - ), - } + generator_for(ty).counter(counter) } - fn sample(&self, values: &[T]) -> T { - values[self.rng.index(values.len())] + fn is_sequence_column(&self, table: usize, column: usize) -> bool { + self.table(table) + .sequences + .iter() + .any(|sequence| sequence.column == column) + } + + fn table(&self, table: usize) -> &TablePlan { + &self.model.schema().tables[table] } } +// Sequence columns are engine-filled on insert. The command still needs a SATS +// value with the right shape, so this placeholder should be ignored downstream. fn sequence_placeholder(ty: Type) -> AlgebraicValue { - match ty { - Type::I64 => AlgebraicValue::I64(0), - Type::U64 => AlgebraicValue::U64(0), + let value = generator_for(ty).counter(0); + match value { + AlgebraicValue::I64(_) | AlgebraicValue::U64(_) => value, _ => unreachable!("sequence columns are integral"), } } +// Reuse checks the materialized value, not the current column definition. Sum +// values need an extra tag bound because migrations can narrow variant counts. +fn value_matches_type(ty: Type, value: &AlgebraicValue) -> bool { + generator_for(ty).matches(value) +} + +// Migration generation is intentionally shallow here: accepted migrations are +// short rewrite chains, while rejected migration construction lives in +// `migrations.rs` and falls back to accepted work when no rejection applies. struct MigrationGen<'a> { rng: &'a Rng, model: &'a Model, @@ -367,7 +657,7 @@ impl<'a> MigrationGen<'a> { } fn choose(&self) -> Option { - match MigrationMode::pick(self.rng) { + match MigrationMode::pick(self.rng, &MigrationMode::CHOICES) { MigrationMode::Accepted => self.choose_accepted(), MigrationMode::Rejected => { Migration::choose_rejected(self.rng, self.model.schema(), self.model).or_else(|| self.choose_accepted()) diff --git a/crates/dst/src/engine/migrations.rs b/crates/dst/src/engine/migrations.rs index f3bb76b9339..963e4783f61 100644 --- a/crates/dst/src/engine/migrations.rs +++ b/crates/dst/src/engine/migrations.rs @@ -93,10 +93,12 @@ pub(crate) enum MigrationMode { Rejected, } -impl WeightedChoice for MigrationMode { - const CHOICES: &'static [Choice] = &[choice(95, Self::Accepted), choice(5, Self::Rejected)]; +impl MigrationMode { + pub(super) const CHOICES: [Choice; 2] = [choice(95, Self::Accepted), choice(5, Self::Rejected)]; } +impl WeightedChoice for MigrationMode {} + /// Weighted categories of auto-migration surfaces to probe. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum MigrationChoice { @@ -115,8 +117,8 @@ enum MigrationChoice { ReschemaEventTable, } -impl WeightedChoice for MigrationChoice { - const CHOICES: &'static [Choice] = &[ +impl MigrationChoice { + const CHOICES: [Choice; 13] = [ choice(2, Self::AddTable), choice(1, Self::RemoveTable), choice(14, Self::AddColumn), @@ -133,6 +135,8 @@ impl WeightedChoice for MigrationChoice { ]; } +impl WeightedChoice for MigrationChoice {} + /// Weighted rejected migration boundaries to probe. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum RejectedMigrationChoice { @@ -140,13 +144,15 @@ enum RejectedMigrationChoice { AddI64DefaultMaxSequencePrecheckFailure, } -impl WeightedChoice for RejectedMigrationChoice { - const CHOICES: &'static [Choice] = &[ +impl RejectedMigrationChoice { + const CHOICES: [Choice; 2] = [ choice(1, Self::AddSequencePrecheckFailure), choice(1, Self::AddI64DefaultMaxSequencePrecheckFailure), ]; } +impl WeightedChoice for RejectedMigrationChoice {} + impl Migration { pub(crate) fn from_schema(schema: SchemaPlan) -> Self { Self::from_schema_with_expectation(schema, MigrationExpectation::Accepted) @@ -185,7 +191,7 @@ impl Migration { pub(crate) fn choose_rewrite(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { for _ in 0..16 { - let choice = MigrationChoice::pick(rng); + let choice = MigrationChoice::pick(rng, &MigrationChoice::CHOICES); if let Some(rewrite) = pick_rewrite(rng, Self::candidates_for(rng, schema, model, choice)) { return Some(rewrite); } @@ -196,7 +202,7 @@ impl Migration { pub(crate) fn choose_rejected(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { for _ in 0..16 { - let choice = RejectedMigrationChoice::pick(rng); + let choice = RejectedMigrationChoice::pick(rng, &RejectedMigrationChoice::CHOICES); if let Some(migration) = pick_migration(Self::rejected_candidates_for(schema, model, choice), rng) { return Some(migration); } @@ -206,7 +212,7 @@ impl Migration { } pub(crate) fn candidates(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Vec { - ::CHOICES + MigrationChoice::CHOICES .iter() .flat_map(|choice| Self::candidates_for(rng, schema, model, choice.value())) .collect() @@ -231,7 +237,7 @@ impl Migration { } fn rejected_candidates(schema: &SchemaPlan, model: &Model) -> Vec { - ::CHOICES + RejectedMigrationChoice::CHOICES .iter() .flat_map(|choice| Self::rejected_candidates_for(schema, model, choice.value())) .collect() diff --git a/crates/dst/src/engine/model.rs b/crates/dst/src/engine/model.rs index ff4b14470cb..1f574c3dbf7 100644 --- a/crates/dst/src/engine/model.rs +++ b/crates/dst/src/engine/model.rs @@ -117,9 +117,7 @@ impl ModelSequence { #[derive(Debug, Clone)] pub(crate) struct ColumnDomain { - pub(crate) ty: Type, pub(crate) values: Vec, - pub(crate) unique: bool, pub(crate) single_column_unique: bool, pub(crate) single_column_indexed: bool, pub(crate) sequenced: bool, @@ -242,6 +240,26 @@ impl Model { false } + pub(crate) fn insert_would_violate_unique_constraint(&self, table: usize, row: &Row) -> bool { + let row = self.project_sequence_values(table, row); + !self.any_visible_row(table, |visible_row| visible_row == &row) && self.violates_unique_constraint(table, &row) + } + + fn project_sequence_values(&self, table: usize, row: &Row) -> Row { + let mut row = row.clone(); + let mut sequences = self.sequences[table].clone(); + + for sequence in &mut sequences { + let column = sequence.column; + if row.elements[column].is_numeric_zero() { + let (value, _allocation) = sequence.generate(); + row.elements[column] = value; + } + } + + row + } + fn apply_sequence_values(&mut self, table: usize, row: &Row) -> Row { let mut row = row.clone(); let sequence_count = self.sequences[table].len(); @@ -442,15 +460,10 @@ impl Model { pub(crate) fn column_domain(&self, table: usize, column: usize) -> ColumnDomain { let table_plan = &self.schema.tables[table]; ColumnDomain { - ty: table_plan.columns[column].ty, values: self .visible_rows(table) .map(|row| row.elements[column].clone()) .collect(), - unique: table_plan - .unique_constraints - .iter() - .any(|constraint| constraint.columns.contains(&column)), single_column_unique: table_plan .unique_constraints .iter() @@ -641,7 +654,7 @@ mod tests { use spacetimedb_lib::AlgebraicValue; use super::*; - use crate::schema::{ColumnPlan, IndexAlgorithm, IndexPlan, TablePlan, Type, UniqueConstraintPlan}; + use crate::schema::{ColumnPlan, IndexAlgorithm, IndexPlan, SequencePlan, TablePlan, Type, UniqueConstraintPlan}; fn schema() -> SchemaPlan { SchemaPlan { @@ -670,6 +683,60 @@ mod tests { } } + fn keyed_payload_schema(sequence: bool) -> SchemaPlan { + SchemaPlan { + tables: vec![TablePlan { + name: "items".into(), + columns: vec![ + ColumnPlan { + name: "id".into(), + ty: Type::U64, + }, + ColumnPlan { + name: "payload".into(), + ty: Type::String, + }, + ], + primary_key: Some(0), + indexes: vec![IndexPlan { + columns: vec![0], + algorithm: IndexAlgorithm::BTree, + }], + unique_constraints: vec![UniqueConstraintPlan { columns: vec![0] }], + sequences: sequence + .then(|| SequencePlan::new(0, Type::U64).expect("u64 sequence")) + .into_iter() + .collect(), + is_public: true, + is_event: false, + }], + } + } + + fn payload_row(id: u64, payload: &str) -> Row { + Row { + elements: vec![AlgebraicValue::U64(id), AlgebraicValue::String(payload.into())].into(), + } + } + + #[test] + fn insert_would_violate_unique_constraint_distinguishes_duplicates_from_conflicts() { + let mut model = Model::new(keyed_payload_schema(false)); + model.committed_tables[0].rows.push(payload_row(1, "a")); + + assert!(!model.insert_would_violate_unique_constraint(0, &payload_row(1, "a"))); + assert!(model.insert_would_violate_unique_constraint(0, &payload_row(1, "b"))); + assert!(!model.insert_would_violate_unique_constraint(0, &payload_row(2, "b"))); + } + + #[test] + fn insert_would_violate_unique_constraint_projects_sequence_values() { + let mut model = Model::new(keyed_payload_schema(true)); + model.committed_tables[0].rows.push(payload_row(1, "a")); + + assert!(model.insert_would_violate_unique_constraint(0, &payload_row(0, "b"))); + } + #[test] fn begin_mut_tx_does_not_clone_committed_tables() { let mut model = Model::new(schema()); diff --git a/crates/dst/src/rng.rs b/crates/dst/src/rng.rs index 626d53b87d5..5fd494b6d3e 100644 --- a/crates/dst/src/rng.rs +++ b/crates/dst/src/rng.rs @@ -39,12 +39,10 @@ pub(crate) fn pick_choice(rng: &Rng, choices: &[Choice]) -> T { unreachable!("selected value is always inside total weight") } -/// Static weighted choices for enum-like generator cases. -pub(crate) trait WeightedChoice: Copy + 'static { - const CHOICES: &'static [Choice]; - - fn pick(rng: &Rng) -> Self { - pick_choice(rng, Self::CHOICES) +/// Weighted choice helper for enum-like generator cases. +pub(crate) trait WeightedChoice: Copy { + fn pick(rng: &Rng, choices: &[Choice]) -> Self { + pick_choice(rng, choices) } } @@ -60,3 +58,30 @@ pub(crate) fn range_inclusive(rng: &Rng, lo: usize, hi: usize) -> usize { } lo + (rng.next_u64() as usize % (hi - lo + 1)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Case { + Default, + Mutated, + } + + impl Case { + const CHOICES: [Choice; 2] = [choice(1, Self::Default), choice(0, Self::Mutated)]; + } + + impl WeightedChoice for Case {} + + #[test] + fn fixed_choice_arrays_can_be_locally_mutated() { + let mut choices = Case::CHOICES; + choices[0] = choice(0, Case::Default); + choices[1] = choice(1, Case::Mutated); + + assert_eq!(Case::pick(&Rng::new(0), &choices), Case::Mutated); + assert_eq!(Case::pick(&Rng::new(0), &Case::CHOICES), Case::Default); + } +} diff --git a/crates/dst/src/schema.rs b/crates/dst/src/schema.rs index b8d70463de2..79b9bb753e9 100644 --- a/crates/dst/src/schema.rs +++ b/crates/dst/src/schema.rs @@ -21,7 +21,7 @@ pub fn default_schema(rng: Rng) -> SchemaPlan { /// Lower a generated schema plan into the raw module format used by the engine. pub fn to_raw_def(schema: &SchemaPlan) -> RawModuleDefV10 { let mut builder = RawModuleDefV10Builder::new(); - builder.set_case_conversion_policy(CaseConversionPolicy::None); + builder.set_case_conversion_policy(CaseConversionPolicy::SnakeCase); for table in &schema.tables { to_raw_def_table(&mut builder, table); @@ -257,7 +257,6 @@ pub struct TypeWeights { pub bytes: u64, pub sum: u64, } - impl Default for TypeWeights { fn default() -> Self { Self { From 31bdba99e1e623f4797c43a4bb19196b20b85f0a Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 29 Jul 2026 14:07:03 +0530 Subject: [PATCH 3/6] pools --- crates/dst/src/engine/generation.rs | 707 +++++++++++++++------------- crates/dst/src/engine/workload.rs | 50 +- 2 files changed, 412 insertions(+), 345 deletions(-) diff --git a/crates/dst/src/engine/generation.rs b/crates/dst/src/engine/generation.rs index 3a83f7249ee..34629f39dad 100644 --- a/crates/dst/src/engine/generation.rs +++ b/crates/dst/src/engine/generation.rs @@ -3,12 +3,14 @@ //! Read this file as policy first, mechanics second: //! - `ValueGen::gen_insert_row` chooses the row-level insert shape: valid row, //! whole-row duplicate, arbitrary candidate, or targeted uniqueness conflict. -//! - `ValueGen::gen_value_for_case` chooses one generated column value. The -//! type-specific helpers below it are mostly curated sample sets. +//! - `ValueGen::gen_value_for_case` chooses one generated column value: fresh +//! random, sampled from the accumulated pool, or mutated from that pool. //! - `MigrationGen` only chooses accepted vs. rejected migration work; concrete //! schema rewrite rules live in `migrations.rs`. -use spacetimedb_lib::{AlgebraicValue, ProductValue}; +use std::collections::BTreeMap; + +use spacetimedb_lib::{bsatn, AlgebraicValue, ProductValue}; use spacetimedb_runtime::sim::Rng; use spacetimedb_sats::ArrayValue; @@ -16,25 +18,28 @@ use super::migrations::{Migration, MigrationMode}; use super::model::Model; use super::row::Row; use crate::rng::{choice, Choice, WeightedChoice}; -use crate::schema::{TablePlan, Type}; +use crate::schema::{SchemaPlan, TablePlan, Type}; // Bound the valid-insert search: random generation may collide with existing // unique constraints, but a failed search must not stall the workload. const INSERT_CANDIDATE_ATTEMPTS: usize = 32; +const VALUE_POOL_VALUES_PER_TYPE: usize = 4096; +const MUTATION_BYTE_DELTAS: [u8; 4] = [1, u8::MAX, 0x80, 0x7f]; -/// Read-only generation context for one model state. +/// Generation context for one model state plus accumulated generator memory. pub(crate) struct GenCtx<'a> { rng: &'a Rng, model: &'a Model, + generation: &'a mut GenerationState, } impl<'a> GenCtx<'a> { - pub(crate) fn new(rng: &'a Rng, model: &'a Model) -> Self { - Self { rng, model } + pub(crate) fn new(rng: &'a Rng, model: &'a Model, generation: &'a mut GenerationState) -> Self { + Self { rng, model, generation } } - pub(crate) fn gen_insert_row(&self, table: usize) -> Row { - ValueGen::new(self.rng, self.model).gen_insert_row(table) + pub(crate) fn gen_insert_row(&mut self, table: usize) -> Row { + ValueGen::new(self.rng, self.model, self.generation).gen_insert_row(table) } pub(crate) fn gen_migration(&self) -> Option { @@ -66,84 +71,210 @@ impl WeightedChoice for InsertRowCase {} #[derive(Clone, Copy)] enum ColumnValueCase { Random, - Small, - Edge, - Weird, - Existing, - NearExisting, + Pooled, + Mutated, } impl ColumnValueCase { - const CHOICES: [Choice; 6] = [ - choice(45, Self::Random), - choice(15, Self::Small), - choice(15, Self::Edge), - choice(5, Self::Weird), - choice(10, Self::Existing), - choice(10, Self::NearExisting), + const CHOICES: [Choice; 3] = [ + choice(50, Self::Random), + choice(30, Self::Pooled), + choice(20, Self::Mutated), ]; } impl WeightedChoice for ColumnValueCase {} -#[derive(Clone, Copy)] -enum ExistingValueScope { - SameColumn, - SameTable, - AnyTable, +pub(crate) struct GenerationState { + generators: BTreeMap>, } -impl ExistingValueScope { - const CHOICES: [Choice; 3] = [ - choice(55, Self::SameColumn), - choice(25, Self::SameTable), - choice(20, Self::AnyTable), - ]; +impl GenerationState { + pub(crate) fn seeded(schema: &SchemaPlan) -> Self { + let mut state = Self { + generators: BTreeMap::new(), + }; + state.seed_schema(schema); + state + } + + pub(crate) fn seed_schema(&mut self, schema: &SchemaPlan) { + for table in &schema.tables { + for column in &table.columns { + self.generator_mut(column.ty); + } + } + } + + pub(crate) fn observe_row(&mut self, table: &TablePlan, row: &Row) { + for (column, value) in table.columns.iter().zip(row.elements.iter()) { + self.generator_mut(column.ty).observe(value.clone()); + } + } + + fn generator_mut(&mut self, ty: Type) -> &mut dyn TypeValueGen { + self.generators + .entry(TypeKey::from(ty)) + .or_insert_with(|| new_generator_for(ty)) + .as_mut() + } +} + +#[derive(Default)] +struct ValueBucket { + values: Vec, + observed: usize, +} + +impl ValueBucket { + fn contains(&self, value: &AlgebraicValue) -> bool { + self.values.contains(value) + } + + fn observe(&mut self, value: AlgebraicValue) { + if self.values.len() < VALUE_POOL_VALUES_PER_TYPE { + self.values.push(value); + } else { + let slot = self.observed % VALUE_POOL_VALUES_PER_TYPE; + self.values[slot] = value; + } + self.observed = self.observed.wrapping_add(1); + } + + fn sample(&self, rng: &Rng) -> Option { + (!self.values.is_empty()).then(|| self.values[rng.index(self.values.len())].clone()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum TypeKey { + Bool, + I64, + U64, + String, + Bytes, + Sum { variants: u8 }, } -impl WeightedChoice for ExistingValueScope {} +impl From for TypeKey { + fn from(ty: Type) -> Self { + match ty { + Type::Bool => Self::Bool, + Type::I64 => Self::I64, + Type::U64 => Self::U64, + Type::String => Self::String, + Type::Bytes => Self::Bytes, + Type::Sum { variants } => Self::Sum { variants }, + } + } +} +/// Stateful value generator for one exact DST column type. +/// +/// Each implementation owns its accumulated value pool. `seeds` supplies the +/// stable starting corpus installed when the generator is first stored; +/// `observe` appends runtime values; `near` samples the pool itself and applies +/// the type-local perturbation. Keeping the pool behind this trait makes pooled +/// and near-value generation follow the same exact type key. trait TypeValueGen { + fn pool(&self) -> &ValueBucket; + fn pool_mut(&mut self) -> &mut ValueBucket; + + /// Stable boundary values installed once when this exact type first appears. + fn seeds(&self) -> Vec; + + /// Produce a fresh valid value without consulting accumulated history. fn random(&self, rng: &Rng) -> AlgebraicValue; - fn small(&self, rng: &Rng) -> AlgebraicValue; - fn edge(&self, rng: &Rng) -> AlgebraicValue; - fn weird(&self, rng: &Rng) -> AlgebraicValue; - fn counter(&self, counter: u64) -> AlgebraicValue; - fn near(&self, value: AlgebraicValue) -> AlgebraicValue; + + /// Make a type-valid nearby value from an already selected value. + /// Returning `value` unchanged is allowed when no sensible nearby value + /// exists or the input does not match this generator's type. + fn near_from(&self, rng: &Rng, value: AlgebraicValue) -> AlgebraicValue; + + /// Check whether a materialized value is valid for this exact type. fn matches(&self, value: &AlgebraicValue) -> bool; -} -struct BoolGen; -struct I64Gen; -struct U64Gen; -struct StringGen; -struct BytesGen; -struct SumGen { - variants: u8, -} + fn observe(&mut self, value: AlgebraicValue) { + if self.matches(&value) { + self.pool_mut().observe(value); + } + } -impl TypeValueGen for BoolGen { - fn random(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::Bool(rng.next_u64().is_multiple_of(2)) + fn pooled(&self, rng: &Rng) -> Option { + self.pool().sample(rng) } - fn small(&self, _rng: &Rng) -> AlgebraicValue { - AlgebraicValue::Bool(false) + fn near(&self, rng: &Rng) -> Option { + let value = self.pooled(rng)?; + let original = value.clone(); + let near = self.near_from(rng, value); + Some(if self.matches(&near) { near } else { original }) } +} - fn edge(&self, _rng: &Rng) -> AlgebraicValue { - AlgebraicValue::Bool(true) +fn seed_generator(generator: &mut dyn TypeValueGen) { + for value in generator.seeds() { + let should_seed = generator.matches(&value) && !generator.pool().contains(&value); + if should_seed { + generator.pool_mut().observe(value); + } } +} + +macro_rules! value_pool_methods { + () => { + fn pool(&self) -> &ValueBucket { + &self.pool + } + + fn pool_mut(&mut self) -> &mut ValueBucket { + &mut self.pool + } + }; +} + +#[derive(Default)] +struct BoolGen { + pool: ValueBucket, +} + +#[derive(Default)] +struct I64Gen { + pool: ValueBucket, +} + +#[derive(Default)] +struct U64Gen { + pool: ValueBucket, +} - fn weird(&self, rng: &Rng) -> AlgebraicValue { - self.edge(rng) +#[derive(Default)] +struct StringGen { + pool: ValueBucket, +} + +#[derive(Default)] +struct BytesGen { + pool: ValueBucket, +} + +struct SumGen { + variants: u8, + pool: ValueBucket, +} + +impl TypeValueGen for BoolGen { + value_pool_methods!(); + + fn seeds(&self) -> Vec { + vec![AlgebraicValue::Bool(false), AlgebraicValue::Bool(true)] } - fn counter(&self, counter: u64) -> AlgebraicValue { - AlgebraicValue::Bool(counter.is_multiple_of(2)) + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::Bool(rng.next_u64().is_multiple_of(2)) } - fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + fn near_from(&self, _rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { match value { AlgebraicValue::Bool(value) => AlgebraicValue::Bool(!value), other => other, @@ -156,29 +287,25 @@ impl TypeValueGen for BoolGen { } impl TypeValueGen for I64Gen { - fn random(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::I64(rng.next_u64() as i64) - } - - fn small(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::I64(sample(rng, &[-3, -2, -1, 0, 1, 2, 3])) - } - - fn edge(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::I64(sample(rng, &[i64::MIN, i64::MIN + 1, -1, 0, 1, i64::MAX - 1, i64::MAX])) - } + value_pool_methods!(); - fn weird(&self, rng: &Rng) -> AlgebraicValue { - self.edge(rng) + fn seeds(&self) -> Vec { + [-3, -2, -1, 0, 1, 2, 3, i64::MIN, i64::MIN + 1, i64::MAX - 1, i64::MAX] + .into_iter() + .map(AlgebraicValue::I64) + .collect() } - fn counter(&self, counter: u64) -> AlgebraicValue { - AlgebraicValue::I64(counter as i64) + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::I64(rng.next_u64() as i64) } - fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + fn near_from(&self, rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { match value { - AlgebraicValue::I64(value) => AlgebraicValue::I64(value.saturating_add(1)), + AlgebraicValue::I64(value) if rng.next_u64().is_multiple_of(2) => { + AlgebraicValue::I64(value.wrapping_add(1)) + } + AlgebraicValue::I64(value) => AlgebraicValue::I64(value.wrapping_sub(1)), other => other, } } @@ -189,29 +316,25 @@ impl TypeValueGen for I64Gen { } impl TypeValueGen for U64Gen { - fn random(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::U64(rng.next_u64()) - } - - fn small(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::U64(sample(rng, &[0, 1, 2, 3, 4, 5])) - } + value_pool_methods!(); - fn edge(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::U64(sample(rng, &[0, 1, 2, u64::MAX - 1, u64::MAX])) + fn seeds(&self) -> Vec { + [0, 1, 2, 3, 4, 5, u64::MAX - 1, u64::MAX] + .into_iter() + .map(AlgebraicValue::U64) + .collect() } - fn weird(&self, rng: &Rng) -> AlgebraicValue { - self.edge(rng) - } - - fn counter(&self, counter: u64) -> AlgebraicValue { - AlgebraicValue::U64(counter) + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::U64(rng.next_u64()) } - fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + fn near_from(&self, rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { match value { - AlgebraicValue::U64(value) => AlgebraicValue::U64(value.saturating_add(1)), + AlgebraicValue::U64(value) if rng.next_u64().is_multiple_of(2) => { + AlgebraicValue::U64(value.wrapping_add(1)) + } + AlgebraicValue::U64(value) => AlgebraicValue::U64(value.wrapping_sub(1)), other => other, } } @@ -222,41 +345,46 @@ impl TypeValueGen for U64Gen { } impl TypeValueGen for StringGen { - fn random(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::String(format!("v_{}", rng.next_u64()).into()) - } - - fn small(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::String( - sample(rng, &["a", "aa", "ab", "b", "z", "v_0", "v_1"]) - .to_owned() - .into(), - ) + value_pool_methods!(); + + fn seeds(&self) -> Vec { + [ + "", + "a", + "aa", + "aaa", + "ab", + "aba", + "abb", + "b", + "z", + "v_0", + "v_1", + "quote'", + "double\"quote", + "back\\slash", + "line\nbreak", + "nul\0byte", + ] + .into_iter() + .map(|value| AlgebraicValue::String(value.into())) + .chain(std::iter::once(AlgebraicValue::String("x".repeat(128).into()))) + .collect() } - fn edge(&self, _rng: &Rng) -> AlgebraicValue { - AlgebraicValue::String("x".repeat(128).into()) - } - - fn weird(&self, rng: &Rng) -> AlgebraicValue { - let value = match rng.index(100) { - 0..35 => sample(rng, &["quote'", "double\"quote", "back\\slash", "line\nbreak"]).to_owned(), - 35..60 => "nul\0byte".to_owned(), - 60..85 => sample(rng, &["a", "aa", "aaa", "ab", "aba", "abb", "b"]).to_owned(), - _ => String::new(), - }; - AlgebraicValue::String(value.into()) - } - - fn counter(&self, counter: u64) -> AlgebraicValue { - AlgebraicValue::String(format!("fresh_{counter}").into()) + fn random(&self, rng: &Rng) -> AlgebraicValue { + AlgebraicValue::String(format!("v_{}", rng.next_u64()).into()) } - fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + fn near_from(&self, rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { match value { AlgebraicValue::String(value) => { - let mut value = value.to_string(); - value.push('a'); + let mut value = value.into_string(); + if !value.is_empty() && rng.next_u64().is_multiple_of(4) { + value.pop(); + } else { + value.push((b'a' + rng.index(26) as u8) as char); + } AlgebraicValue::String(value.into()) } other => other, @@ -269,44 +397,35 @@ impl TypeValueGen for StringGen { } impl TypeValueGen for BytesGen { - fn random(&self, rng: &Rng) -> AlgebraicValue { - let len = (rng.next_u64() % 16) as usize; - let value = (0..len).map(|_| rng.next_u64() as u8).collect::>(); - AlgebraicValue::Array(ArrayValue::U8(value.into())) - } - - fn small(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::Array(ArrayValue::U8( - sample(rng, &[&[][..], &[0][..], &[1][..], &[0, 255][..]]) - .to_vec() - .into(), - )) - } + value_pool_methods!(); - fn edge(&self, _rng: &Rng) -> AlgebraicValue { - AlgebraicValue::Array(ArrayValue::U8(vec![255; 32].into())) + fn seeds(&self) -> Vec { + vec![ + AlgebraicValue::Array(ArrayValue::U8(Vec::new().into())), + AlgebraicValue::Array(ArrayValue::U8(vec![0].into())), + AlgebraicValue::Array(ArrayValue::U8(vec![1].into())), + AlgebraicValue::Array(ArrayValue::U8(vec![0, 255].into())), + AlgebraicValue::Array(ArrayValue::U8(vec![0; 32].into())), + AlgebraicValue::Array(ArrayValue::U8(vec![255; 32].into())), + AlgebraicValue::Array(ArrayValue::U8(vec![0, 255, 0, 255, 0, 255].into())), + ] } - fn weird(&self, rng: &Rng) -> AlgebraicValue { - let value = match rng.index(100) { - 0..25 => Vec::new(), - 25..45 => vec![0; 32], - 45..65 => vec![255; 32], - 65..85 => vec![0, 255, 0, 255, 0, 255], - _ => sample(rng, &[&[][..], &[0][..], &[1][..], &[0, 255][..]]).to_vec(), - }; + fn random(&self, rng: &Rng) -> AlgebraicValue { + let len = (rng.next_u64() % 16) as usize; + let value = (0..len).map(|_| rng.next_u64() as u8).collect::>(); AlgebraicValue::Array(ArrayValue::U8(value.into())) } - fn counter(&self, counter: u64) -> AlgebraicValue { - AlgebraicValue::Array(ArrayValue::U8(counter.to_le_bytes().to_vec().into())) - } - - fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + fn near_from(&self, rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { match value { AlgebraicValue::Array(ArrayValue::U8(value)) => { let mut value = value.to_vec(); - value.push(0); + if value.is_empty() { + value.push(sample_delta(rng)); + } else { + mutate_bytes(rng, &mut value); + } AlgebraicValue::Array(ArrayValue::U8(value.into())) } other => other, @@ -319,40 +438,23 @@ impl TypeValueGen for BytesGen { } impl TypeValueGen for SumGen { - fn random(&self, rng: &Rng) -> AlgebraicValue { - let tag = rng.index(self.variants as usize) as u8; - AlgebraicValue::sum(tag, AlgebraicValue::U8(rng.next_u64() as u8)) - } - - fn small(&self, rng: &Rng) -> AlgebraicValue { - let tag = if self.variants <= 1 { - 0 - } else { - rng.index(self.variants as usize) as u8 - }; - AlgebraicValue::sum(tag, AlgebraicValue::U8(0)) - } - - fn edge(&self, _rng: &Rng) -> AlgebraicValue { - AlgebraicValue::sum(self.variants.saturating_sub(1), AlgebraicValue::U8(u8::MAX)) - } + value_pool_methods!(); - fn weird(&self, rng: &Rng) -> AlgebraicValue { - self.edge(rng) + fn seeds(&self) -> Vec { + let mut values = vec![AlgebraicValue::sum(0, AlgebraicValue::U8(0))]; + if self.variants > 1 { + values.push(AlgebraicValue::sum(self.variants - 1, AlgebraicValue::U8(u8::MAX))); + } + values } - fn counter(&self, counter: u64) -> AlgebraicValue { - AlgebraicValue::sum( - if self.variants == 0 { - 0 - } else { - (counter % self.variants as u64) as u8 - }, - AlgebraicValue::U8(counter as u8), - ) + fn random(&self, rng: &Rng) -> AlgebraicValue { + debug_assert!(self.variants > 0); + let tag = rng.index(self.variants as usize) as u8; + AlgebraicValue::sum(tag, AlgebraicValue::U8(rng.next_u64() as u8)) } - fn near(&self, value: AlgebraicValue) -> AlgebraicValue { + fn near_from(&self, _rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { match value { AlgebraicValue::Sum(sum) if self.variants > 0 => { AlgebraicValue::sum(sum.tag.wrapping_add(1) % self.variants, *sum.value) @@ -366,19 +468,20 @@ impl TypeValueGen for SumGen { } } -fn generator_for(ty: Type) -> Box { - match ty { - Type::Bool => Box::new(BoolGen), - Type::I64 => Box::new(I64Gen), - Type::U64 => Box::new(U64Gen), - Type::String => Box::new(StringGen), - Type::Bytes => Box::new(BytesGen), - Type::Sum { variants } => Box::new(SumGen { variants }), - } -} - -fn sample(rng: &Rng, values: &[T]) -> T { - values[rng.index(values.len())] +fn new_generator_for(ty: Type) -> Box { + let mut generator: Box = match ty { + Type::Bool => Box::::default(), + Type::I64 => Box::::default(), + Type::U64 => Box::::default(), + Type::String => Box::::default(), + Type::Bytes => Box::::default(), + Type::Sum { variants } => Box::new(SumGen { + variants, + pool: ValueBucket::default(), + }), + }; + seed_generator(generator.as_mut()); + generator } // Row generation deliberately mixes normal traffic with rows the engine should @@ -387,14 +490,15 @@ fn sample(rng: &Rng, values: &[T]) -> T { struct ValueGen<'a> { rng: &'a Rng, model: &'a Model, + generation: &'a mut GenerationState, } impl<'a> ValueGen<'a> { - fn new(rng: &'a Rng, model: &'a Model) -> Self { - Self { rng, model } + fn new(rng: &'a Rng, model: &'a Model, generation: &'a mut GenerationState) -> Self { + Self { rng, model, generation } } - fn gen_insert_row(&self, table: usize) -> Row { + fn gen_insert_row(&mut self, table: usize) -> Row { // Start here when tuning insert behavior; everything below materializes // one arm of this match. match InsertRowCase::pick(self.rng, &InsertRowCase::CHOICES) { @@ -409,10 +513,10 @@ impl<'a> ValueGen<'a> { } } - fn gen_valid_insert_row(&self, table: usize) -> Row { - // The valid path still begins with ordinary candidates so it samples the - // same value domain as rejected paths. Counter rows are only the - // progress fallback. + fn gen_valid_insert_row(&mut self, table: usize) -> Row { + // The valid path samples the same value domain as rejected paths. After + // bounded attempts, prefer an existing row as an accepted no-op over a + // hand-built per-type escape value. for _ in 0..INSERT_CANDIDATE_ATTEMPTS { let row = self.gen_row_candidate(table); if !self.model.insert_would_violate_unique_constraint(table, &row) { @@ -420,66 +524,47 @@ impl<'a> ValueGen<'a> { } } - self.gen_counter_row(table, self.rng.next_u64()) - } - - fn gen_row_candidate(&self, table: usize) -> Row { - self.table(table) - .columns - .iter() - .enumerate() - .map(|(column, column_plan)| self.gen_insert_value(table, column, column_plan.ty)) - .collect::() + self.existing_row(table) + .unwrap_or_else(|| self.gen_row_candidate(table)) } - fn gen_counter_row(&self, table: usize, counter: u64) -> Row { - self.table(table) + fn gen_row_candidate(&mut self, table: usize) -> Row { + let column_types = self + .table(table) .columns .iter() + .map(|column| column.ty) + .collect::>(); + column_types + .into_iter() .enumerate() - .map(|(column, column_plan)| { - if self.is_sequence_column(table, column) { - sequence_placeholder(column_plan.ty) - } else { - self.gen_counter_value(column_plan.ty, counter.wrapping_add(column as u64)) - } - }) + .map(|(column, ty)| self.gen_insert_value(table, column, ty)) .collect::() } - fn gen_insert_value(&self, table: usize, column: usize, ty: Type) -> AlgebraicValue { + fn gen_insert_value(&mut self, table: usize, column: usize, ty: Type) -> AlgebraicValue { if self.is_sequence_column(table, column) { return sequence_placeholder(ty); } - self.gen_value(table, column, ty) + self.gen_value(ty) } - fn gen_value(&self, table: usize, column: usize, ty: Type) -> AlgebraicValue { + fn gen_value(&mut self, ty: Type) -> AlgebraicValue { let case = ColumnValueCase::pick(self.rng, &ColumnValueCase::CHOICES); - self.gen_value_for_case(table, column, ty, case).unwrap_or_else(|| { - // Existing-value cases can fail when the model is empty or no - // visible value matches the requested type. - self.gen_value_for_case(table, column, ty, ColumnValueCase::Random) - .expect("random value generation cannot fail") - }) + if let Some(value) = self.gen_value_for_case(ty, case) { + return value; + } + + self.gen_value_for_case(ty, ColumnValueCase::Random) + .expect("random value generation cannot fail") } - fn gen_value_for_case( - &self, - table: usize, - column: usize, - ty: Type, - case: ColumnValueCase, - ) -> Option { - let type_gen = generator_for(ty); + fn gen_value_for_case(&mut self, ty: Type, case: ColumnValueCase) -> Option { match case { - ColumnValueCase::Random => Some(type_gen.random(self.rng)), - ColumnValueCase::Small => Some(type_gen.small(self.rng)), - ColumnValueCase::Edge => Some(type_gen.edge(self.rng)), - ColumnValueCase::Weird => Some(type_gen.weird(self.rng)), - ColumnValueCase::Existing => self.existing_value(table, column, ty), - ColumnValueCase::NearExisting => self.near_existing_value(table, column, ty), + ColumnValueCase::Random => Some(self.generation.generator_mut(ty).random(self.rng)), + ColumnValueCase::Pooled => self.pooled_value(ty), + ColumnValueCase::Mutated => self.mutated_pooled_value(ty), } } @@ -493,11 +578,11 @@ impl<'a> ValueGen<'a> { }) } - fn unique_conflict_row(&self, table: usize) -> Option { + fn unique_conflict_row(&mut self, table: usize) -> Option { // Target a specific unique constraint without turning the operation into // an exact-row duplicate: copy constrained columns from an existing row, // then perturb an unconstrained column if necessary. - let constraints = &self.table(table).unique_constraints; + let constraints = self.table(table).unique_constraints.clone(); if constraints.is_empty() || self.model.row_count(table) == 0 { return None; } @@ -518,8 +603,8 @@ impl<'a> ValueGen<'a> { else { continue; }; - row.elements[column] = - self.near_value(self.table(table).columns[column].ty, base.elements[column].clone()); + let ty = self.table(table).columns[column].ty; + row.elements[column] = self.near_value(ty); } if row != base && self.model.insert_would_violate_unique_constraint(table, &row) { @@ -530,89 +615,18 @@ impl<'a> ValueGen<'a> { None } - fn existing_value(&self, table: usize, column: usize, ty: Type) -> Option { - // Existing-value generation is runtime-value based, not column-domain - // based. After migrations, the interesting reusable value may live in a - // different column or table as long as its SATS shape still fits `ty`. - let scope = ExistingValueScope::pick(self.rng, &ExistingValueScope::CHOICES); - self.existing_value_in_scope(table, column, ty, scope) - .or_else(|| self.existing_value_in_scope(table, column, ty, ExistingValueScope::SameColumn)) - .or_else(|| self.existing_value_in_scope(table, column, ty, ExistingValueScope::SameTable)) - .or_else(|| self.existing_value_in_scope(table, column, ty, ExistingValueScope::AnyTable)) - } - - fn existing_value_in_scope( - &self, - table: usize, - column: usize, - ty: Type, - scope: ExistingValueScope, - ) -> Option { - match scope { - ExistingValueScope::SameColumn => self.existing_column_value(table, column, ty), - ExistingValueScope::SameTable => self.existing_table_value(table, ty), - ExistingValueScope::AnyTable => self.existing_any_table_value(ty), - } - } - - fn existing_column_value(&self, table: usize, column: usize, ty: Type) -> Option { - let count = self.model.row_count(table); - if count == 0 { - return None; - } - - let start = self.rng.index(count); - for offset in 0..count { - let row = self.model.row(table, (start + offset) % count)?; - let value = &row.elements[column]; - if value_matches_type(ty, value) { - return Some(value.clone()); - } - } - - None - } - - fn existing_table_value(&self, table: usize, ty: Type) -> Option { - let values = self.visible_values_in_tables(ty, table..table + 1); - (!values.is_empty()).then(|| values[self.rng.index(values.len())].clone()) - } - - fn existing_any_table_value(&self, ty: Type) -> Option { - let values = self.visible_values_in_tables(ty, 0..self.model.schema().tables.len()); - (!values.is_empty()).then(|| values[self.rng.index(values.len())].clone()) - } - - fn visible_values_in_tables(&self, ty: Type, tables: impl Iterator) -> Vec { - // Scan stored rows by runtime type compatibility. This is what lets - // existing-value reuse survive schema rewrites and sum-variant changes. - let mut values = Vec::new(); - for table in tables { - let table_plan = self.table(table); - for row_idx in 0..self.model.row_count(table) { - let row = self.model.row(table, row_idx).expect("row index is in bounds"); - for (column, _column_plan) in table_plan.columns.iter().enumerate() { - let value = &row.elements[column]; - if value_matches_type(ty, value) { - values.push(value.clone()); - } - } - } - } - values - } - - fn near_existing_value(&self, table: usize, column: usize, ty: Type) -> Option { - self.existing_value(table, column, ty) - .map(|value| self.near_value(ty, value)) + fn pooled_value(&mut self, ty: Type) -> Option { + self.generation.generator_mut(ty).pooled(self.rng) } - fn near_value(&self, ty: Type, value: AlgebraicValue) -> AlgebraicValue { - generator_for(ty).near(value) + fn mutated_pooled_value(&mut self, ty: Type) -> Option { + let type_gen = self.generation.generator_mut(ty); + mutate_value(self.rng, type_gen, ty) } - fn gen_counter_value(&self, ty: Type, counter: u64) -> AlgebraicValue { - generator_for(ty).counter(counter) + fn near_value(&mut self, ty: Type) -> AlgebraicValue { + let type_gen = self.generation.generator_mut(ty); + type_gen.near(self.rng).unwrap_or_else(|| type_gen.random(self.rng)) } fn is_sequence_column(&self, table: usize, column: usize) -> bool { @@ -627,22 +641,47 @@ impl<'a> ValueGen<'a> { } } +fn mutate_value(rng: &Rng, type_gen: &dyn TypeValueGen, ty: Type) -> Option { + let value = type_gen.pooled(rng)?; + if let Some(mutated) = mutate_bsatn_value(rng, ty, &value).filter(|mutated| type_gen.matches(mutated)) { + return Some(mutated); + } + + type_gen.near(rng) +} + +fn mutate_bsatn_value(rng: &Rng, ty: Type, value: &AlgebraicValue) -> Option { + let mut bytes = bsatn::to_vec(value).ok()?; + mutate_bytes(rng, &mut bytes); + let ty = ty.to_algebraic(); + AlgebraicValue::decode(&ty, &mut &bytes[..]).ok() +} + +fn mutate_bytes(rng: &Rng, bytes: &mut [u8]) { + if bytes.is_empty() { + return; + } + let edits = 1 + rng.index(3); + for _ in 0..edits { + let index = rng.index(bytes.len()); + bytes[index] = bytes[index].wrapping_add(sample_delta(rng)); + } +} + +fn sample_delta(rng: &Rng) -> u8 { + MUTATION_BYTE_DELTAS[rng.index(MUTATION_BYTE_DELTAS.len())] +} + // Sequence columns are engine-filled on insert. The command still needs a SATS // value with the right shape, so this placeholder should be ignored downstream. fn sequence_placeholder(ty: Type) -> AlgebraicValue { - let value = generator_for(ty).counter(0); - match value { - AlgebraicValue::I64(_) | AlgebraicValue::U64(_) => value, + match ty { + Type::I64 => AlgebraicValue::I64(0), + Type::U64 => AlgebraicValue::U64(0), _ => unreachable!("sequence columns are integral"), } } -// Reuse checks the materialized value, not the current column definition. Sum -// values need an extra tag bound because migrations can narrow variant counts. -fn value_matches_type(ty: Type, value: &AlgebraicValue) -> bool { - generator_for(ty).matches(value) -} - // Migration generation is intentionally shallow here: accepted migrations are // short rewrite chains, while rejected migration construction lives in // `migrations.rs` and falls back to accepted work when no rejection applies. diff --git a/crates/dst/src/engine/workload.rs b/crates/dst/src/engine/workload.rs index aca7b96b7d3..e37418bbdea 100644 --- a/crates/dst/src/engine/workload.rs +++ b/crates/dst/src/engine/workload.rs @@ -2,7 +2,7 @@ use std::fmt::{Debug, Error as FmtError, Formatter}; -use super::generation::GenCtx; +use super::generation::{GenCtx, GenerationState}; use super::migrations::{Migration, MigrationRejection}; use super::model::Model; use super::row::Row; @@ -95,6 +95,7 @@ impl Default for InteractionWeights { pub struct WorkloadGen { rng: Rng, model: Model, + generation: GenerationState, stats: InteractionCounts, weights: InteractionWeights, } @@ -105,9 +106,11 @@ impl WorkloadGen { } pub fn with_weights(rng: Rng, model: Model, weights: InteractionWeights) -> Self { + let generation = GenerationState::seeded(model.schema()); Self { rng, model, + generation, stats: InteractionCounts::default(), weights, } @@ -127,17 +130,41 @@ impl WorkloadGen { } fn observe_interaction(&mut self, interaction: &Interaction, observation: &Observation) -> Result<(), Error> { - match (interaction, observation) { - (Interaction::Insert { .. }, Observation::Inserted { .. }) => { - self.model.apply(interaction); - } + let model_observation = match (interaction, observation) { + (Interaction::Insert { .. }, Observation::Inserted { .. }) => self.model.apply(interaction), (Interaction::Insert { .. }, _) => anyhow::bail!("insert produced unexpected observation"), - _ => { - self.model.apply(interaction); + _ => self.model.apply(interaction), + }; + + self.observe_generation(interaction, &model_observation); + Ok(()) + } + + fn observe_generation(&mut self, interaction: &Interaction, model_observation: &Observation) { + match (interaction, model_observation) { + (Interaction::Insert { table, row }, Observation::Inserted { outcome }) => { + let table_plan = self.model.schema().tables[*table].clone(); + self.generation.observe_row(&table_plan, row); + if let InsertOutcome::Accepted(row) = outcome { + self.generation.observe_row(&table_plan, row); + } } + (_, Observation::Migrated) => self.observe_schema_values(), + _ => {} } + } - Ok(()) + fn observe_schema_values(&mut self) { + let schema = self.model.schema().clone(); + self.generation.seed_schema(&schema); + for (table, table_plan) in schema.tables.iter().enumerate() { + let rows = (0..self.model.row_count(table)) + .map(|row| self.model.row(table, row).expect("row index is in bounds").clone()) + .collect::>(); + for row in rows { + self.generation.observe_row(table_plan, &row); + } + } } } @@ -199,9 +226,10 @@ impl WorkloadGen { InteractionChoice::Insert => { let table = self.insert_table_idx(); + let mut ctx = GenCtx::new(&self.rng, &self.model, &mut self.generation); Interaction::Insert { table, - row: GenCtx::new(&self.rng, &self.model).gen_insert_row(table), + row: ctx.gen_insert_row(table), } } @@ -266,8 +294,8 @@ impl WorkloadGen { data_tables } - fn gen_migration(&self) -> Option { - GenCtx::new(&self.rng, &self.model).gen_migration() + fn gen_migration(&mut self) -> Option { + GenCtx::new(&self.rng, &self.model, &mut self.generation).gen_migration() } fn deletable_table_idx(&self) -> Option { From 04d81838aac6c44b743258c05dac322f72ff518d Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 29 Jul 2026 15:22:41 +0530 Subject: [PATCH 4/6] migrations --- crates/dst/src/engine/generation.rs | 92 +- crates/dst/src/engine/migrations.rs | 1269 +++++++++++++++++++++++---- crates/dst/src/rng.rs | 6 - 3 files changed, 1137 insertions(+), 230 deletions(-) diff --git a/crates/dst/src/engine/generation.rs b/crates/dst/src/engine/generation.rs index 34629f39dad..b2724ea4987 100644 --- a/crates/dst/src/engine/generation.rs +++ b/crates/dst/src/engine/generation.rs @@ -4,13 +4,13 @@ //! - `ValueGen::gen_insert_row` chooses the row-level insert shape: valid row, //! whole-row duplicate, arbitrary candidate, or targeted uniqueness conflict. //! - `ValueGen::gen_value_for_case` chooses one generated column value: fresh -//! random, sampled from the accumulated pool, or mutated from that pool. +//! random, sampled from the accumulated pool, or near an accumulated pool value. //! - `MigrationGen` only chooses accepted vs. rejected migration work; concrete //! schema rewrite rules live in `migrations.rs`. use std::collections::BTreeMap; -use spacetimedb_lib::{bsatn, AlgebraicValue, ProductValue}; +use spacetimedb_lib::{AlgebraicValue, ProductValue}; use spacetimedb_runtime::sim::Rng; use spacetimedb_sats::ArrayValue; @@ -24,7 +24,6 @@ use crate::schema::{SchemaPlan, TablePlan, Type}; // unique constraints, but a failed search must not stall the workload. const INSERT_CANDIDATE_ATTEMPTS: usize = 32; const VALUE_POOL_VALUES_PER_TYPE: usize = 4096; -const MUTATION_BYTE_DELTAS: [u8; 4] = [1, u8::MAX, 0x80, 0x7f]; /// Generation context for one model state plus accumulated generator memory. pub(crate) struct GenCtx<'a> { @@ -72,14 +71,14 @@ impl WeightedChoice for InsertRowCase {} enum ColumnValueCase { Random, Pooled, - Mutated, + Near, } impl ColumnValueCase { const CHOICES: [Choice; 3] = [ choice(50, Self::Random), choice(30, Self::Pooled), - choice(20, Self::Mutated), + choice(20, Self::Near), ]; } @@ -373,7 +372,21 @@ impl TypeValueGen for StringGen { } fn random(&self, rng: &Rng) -> AlgebraicValue { - AlgebraicValue::String(format!("v_{}", rng.next_u64()).into()) + let mut state = rng.next_u64(); + let len = match state % 100 { + 0..=9 => 0, + 10..=79 => (state as usize >> 8) % 32, + 80..=97 => 32 + ((state as usize >> 8) % 224), + _ => 256 + ((state as usize >> 8) % 768), + }; + let alphabet = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-'\"\\\n\t\0 %.,:;()[]{}"; + let value = (0..len) + .map(|_| { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + alphabet[(state as usize >> 32) % alphabet.len()] as char + }) + .collect::(); + AlgebraicValue::String(value.into()) } fn near_from(&self, rng: &Rng, value: AlgebraicValue) -> AlgebraicValue { @@ -422,9 +435,19 @@ impl TypeValueGen for BytesGen { AlgebraicValue::Array(ArrayValue::U8(value)) => { let mut value = value.to_vec(); if value.is_empty() { - value.push(sample_delta(rng)); + value.push(rng.next_u64() as u8); } else { - mutate_bytes(rng, &mut value); + let edits = 1 + rng.index(3); + for _ in 0..edits { + let index = rng.index(value.len()); + let delta = match rng.index(4) { + 0 => 1, + 1 => u8::MAX, + 2 => 0x80, + _ => 0x7f, + }; + value[index] = value[index].wrapping_add(delta); + } } AlgebraicValue::Array(ArrayValue::U8(value.into())) } @@ -564,7 +587,7 @@ impl<'a> ValueGen<'a> { match case { ColumnValueCase::Random => Some(self.generation.generator_mut(ty).random(self.rng)), ColumnValueCase::Pooled => self.pooled_value(ty), - ColumnValueCase::Mutated => self.mutated_pooled_value(ty), + ColumnValueCase::Near => Some(self.near_value(ty)), } } @@ -619,11 +642,6 @@ impl<'a> ValueGen<'a> { self.generation.generator_mut(ty).pooled(self.rng) } - fn mutated_pooled_value(&mut self, ty: Type) -> Option { - let type_gen = self.generation.generator_mut(ty); - mutate_value(self.rng, type_gen, ty) - } - fn near_value(&mut self, ty: Type) -> AlgebraicValue { let type_gen = self.generation.generator_mut(ty); type_gen.near(self.rng).unwrap_or_else(|| type_gen.random(self.rng)) @@ -641,37 +659,6 @@ impl<'a> ValueGen<'a> { } } -fn mutate_value(rng: &Rng, type_gen: &dyn TypeValueGen, ty: Type) -> Option { - let value = type_gen.pooled(rng)?; - if let Some(mutated) = mutate_bsatn_value(rng, ty, &value).filter(|mutated| type_gen.matches(mutated)) { - return Some(mutated); - } - - type_gen.near(rng) -} - -fn mutate_bsatn_value(rng: &Rng, ty: Type, value: &AlgebraicValue) -> Option { - let mut bytes = bsatn::to_vec(value).ok()?; - mutate_bytes(rng, &mut bytes); - let ty = ty.to_algebraic(); - AlgebraicValue::decode(&ty, &mut &bytes[..]).ok() -} - -fn mutate_bytes(rng: &Rng, bytes: &mut [u8]) { - if bytes.is_empty() { - return; - } - let edits = 1 + rng.index(3); - for _ in 0..edits { - let index = rng.index(bytes.len()); - bytes[index] = bytes[index].wrapping_add(sample_delta(rng)); - } -} - -fn sample_delta(rng: &Rng) -> u8 { - MUTATION_BYTE_DELTAS[rng.index(MUTATION_BYTE_DELTAS.len())] -} - // Sequence columns are engine-filled on insert. The command still needs a SATS // value with the right shape, so this placeholder should be ignored downstream. fn sequence_placeholder(ty: Type) -> AlgebraicValue { @@ -705,19 +692,6 @@ impl<'a> MigrationGen<'a> { } fn choose_accepted(&self) -> Option { - let original = self.model.schema(); - let mut schema = original.clone(); - let steps = 1 + self.rng.index(10); - - for _ in 0..steps { - let Some(rewrite) = Migration::choose_rewrite(self.rng, &schema, self.model) else { - break; - }; - rewrite - .apply_to(&mut schema) - .expect("generated rewrite must be valid for the draft schema"); - } - - (schema != *original).then(|| Migration::from_schema(schema)) + Migration::choose_accepted(self.rng, self.model.schema(), self.model) } } diff --git a/crates/dst/src/engine/migrations.rs b/crates/dst/src/engine/migrations.rs index 963e4783f61..47de1200f49 100644 --- a/crates/dst/src/engine/migrations.rs +++ b/crates/dst/src/engine/migrations.rs @@ -11,6 +11,7 @@ use crate::schema::{ TablePlan, Type, UniqueConstraintPlan, }; use spacetimedb_runtime::sim::Rng; +use std::collections::HashSet; const MAX_SUM_VARIANTS: u8 = 32; const MAX_EVENT_COLUMNS: usize = 32; @@ -99,59 +100,219 @@ impl MigrationMode { impl WeightedChoice for MigrationMode {} -/// Weighted categories of auto-migration surfaces to probe. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum MigrationChoice { +/// Stable identity for migration rules. +/// +/// Keep this ID separate from weights and rule implementation. It is the +/// greppable handle for triage, swarm toggles, and future campaign profiles. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum RuleId { AddTable, - RemoveTable, + RemovePristineNonEventTable, + RemoveEmptyEventTable, AddColumn, AddIndex, RemoveIndex, ChangeIndex, - AddSequence, + AddSequenceOnPristineTable, + AddSequenceBoundaryProbe, RemoveSequence, - AddUniqueConstraint, + AddUniqueConstraintOnPristineTable, RemoveUniqueConstraint, DropPrimaryKeyAndWidenSum, WidenSumColumn, - ReschemaEventTable, + ReschemaEmptyEventTable, + AddI64DefaultMaxSequencePrecheckFailure, } -impl MigrationChoice { - const CHOICES: [Choice; 13] = [ - choice(2, Self::AddTable), - choice(1, Self::RemoveTable), - choice(14, Self::AddColumn), - choice(10, Self::AddIndex), - choice(8, Self::RemoveIndex), - choice(10, Self::ChangeIndex), - choice(18, Self::AddSequence), - choice(8, Self::RemoveSequence), - choice(12, Self::AddUniqueConstraint), - choice(8, Self::RemoveUniqueConstraint), - choice(6, Self::DropPrimaryKeyAndWidenSum), - choice(12, Self::WidenSumColumn), - choice(8, Self::ReschemaEventTable), - ]; +#[derive(Clone, Copy)] +struct RuleEntry { + id: RuleId, + rule: &'static dyn MigrationRule, } -impl WeightedChoice for MigrationChoice {} +struct RuleWeights { + accepted: &'static [(RuleId, u32)], + rejected: &'static [(RuleId, u32)], +} -/// Weighted rejected migration boundaries to probe. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RejectedMigrationChoice { - AddSequencePrecheckFailure, - AddI64DefaultMaxSequencePrecheckFailure, +static MIGRATION_RULES: &[RuleEntry] = &[ + RuleEntry { + id: RuleId::AddTable, + rule: &AddTableRule, + }, + RuleEntry { + id: RuleId::RemovePristineNonEventTable, + rule: &RemovePristineNonEventTableRule, + }, + RuleEntry { + id: RuleId::RemoveEmptyEventTable, + rule: &RemoveEmptyEventTableRule, + }, + RuleEntry { + id: RuleId::AddColumn, + rule: &AddColumnRule, + }, + RuleEntry { + id: RuleId::AddIndex, + rule: &AddIndexRule, + }, + RuleEntry { + id: RuleId::RemoveIndex, + rule: &RemoveIndexRule, + }, + RuleEntry { + id: RuleId::ChangeIndex, + rule: &ChangeIndexRule, + }, + RuleEntry { + id: RuleId::AddSequenceOnPristineTable, + rule: &AddSequenceOnPristineTableRule, + }, + RuleEntry { + id: RuleId::AddSequenceBoundaryProbe, + rule: &AddSequenceBoundaryProbeRule, + }, + RuleEntry { + id: RuleId::RemoveSequence, + rule: &RemoveSequenceRule, + }, + RuleEntry { + id: RuleId::AddUniqueConstraintOnPristineTable, + rule: &AddUniqueConstraintOnPristineTableRule, + }, + RuleEntry { + id: RuleId::RemoveUniqueConstraint, + rule: &RemoveUniqueConstraintRule, + }, + RuleEntry { + id: RuleId::DropPrimaryKeyAndWidenSum, + rule: &DropPrimaryKeyAndWidenSumRule, + }, + RuleEntry { + id: RuleId::WidenSumColumn, + rule: &WidenSumColumnRule, + }, + RuleEntry { + id: RuleId::ReschemaEmptyEventTable, + rule: &ReschemaEmptyEventTableRule, + }, + RuleEntry { + id: RuleId::AddI64DefaultMaxSequencePrecheckFailure, + rule: &AddI64DefaultMaxSequencePrecheckFailureRule, + }, +]; + +static DEFAULT_RULE_WEIGHTS: RuleWeights = RuleWeights { + accepted: &[ + (RuleId::AddTable, 2), + (RuleId::RemovePristineNonEventTable, 1), + (RuleId::RemoveEmptyEventTable, 1), + (RuleId::AddColumn, 14), + (RuleId::AddIndex, 10), + (RuleId::RemoveIndex, 8), + (RuleId::ChangeIndex, 10), + (RuleId::AddSequenceOnPristineTable, 8), + (RuleId::AddSequenceBoundaryProbe, 10), + (RuleId::RemoveSequence, 8), + (RuleId::AddUniqueConstraintOnPristineTable, 12), + (RuleId::RemoveUniqueConstraint, 8), + (RuleId::DropPrimaryKeyAndWidenSum, 6), + (RuleId::WidenSumColumn, 12), + (RuleId::ReschemaEmptyEventTable, 8), + ], + rejected: &[ + (RuleId::AddSequenceBoundaryProbe, 1), + (RuleId::AddI64DefaultMaxSequencePrecheckFailure, 1), + ], +}; + +fn migration_rules() -> &'static [RuleEntry] { + MIGRATION_RULES +} + +#[derive(Debug, Clone, Copy)] +struct RuleCtx<'a> { + schema: &'a SchemaPlan, + model: &'a Model, +} + +/// A positive case emitted by a migration rule. +/// +/// `writes` are derived from the rewrite and used by the batch validator. +/// `protects` are rule-owned semantic dependencies such as "this table was +/// pristine" or "this column domain cleared the add-sequence precheck." +#[derive(Debug, Clone, PartialEq, Eq)] +struct AcceptedCase { + rule: RuleId, + rewrite: SchemaRewrite, + writes: Vec, + protects: Vec, +} + +impl AcceptedCase { + fn new(rule: RuleId, rewrite: SchemaRewrite) -> Self { + let writes = write_resources(&rewrite); + Self { + rule, + rewrite, + writes, + protects: Vec::new(), + } + } + + fn protecting(mut self, protects: impl Into>) -> Self { + self.protects = protects.into(); + self + } +} + +/// A negative case emitted by a migration rule. +/// +/// These are not "anything that failed an accepted rule." They are known +/// single-boundary violations with a precise expected oracle result. +#[derive(Debug, Clone, PartialEq, Eq)] +struct RejectedCase { + rewrite: SchemaRewrite, + expected: MigrationRejection, } -impl RejectedMigrationChoice { - const CHOICES: [Choice; 2] = [ - choice(1, Self::AddSequencePrecheckFailure), - choice(1, Self::AddI64DefaultMaxSequencePrecheckFailure), - ]; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum Resource { + Table(String), + Column { table: String, column: usize }, + Index { table: String, columns: Vec }, + Sequence { table: String, column: usize }, + UniqueConstraint { table: String, columns: Vec }, + PrimaryKey { table: String }, +} + +trait MigrationRule: Sync { + fn accepted_cases(&self, _rng: &Rng, _ctx: RuleCtx<'_>, _rule: RuleId) -> Vec { + Vec::new() + } + + fn rejected_cases(&self, _rng: &Rng, _ctx: RuleCtx<'_>) -> Vec { + Vec::new() + } + + /// Rule-owned semantic validation for accepted cases. + /// + /// Structural validation is shared by applying the rewrite batch to a draft + /// schema. Data/model assumptions stay with the rule which made them. + fn validate_accepted( + &self, + _case: &AcceptedCase, + _original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + Ok(()) + } } -impl WeightedChoice for RejectedMigrationChoice {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AcceptedComposition { + Capped { max_rules: usize }, +} impl Migration { pub(crate) fn from_schema(schema: SchemaPlan) -> Self { @@ -189,86 +350,691 @@ impl Migration { self.expectation } - pub(crate) fn choose_rewrite(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { - for _ in 0..16 { - let choice = MigrationChoice::pick(rng, &MigrationChoice::CHOICES); - if let Some(rewrite) = pick_rewrite(rng, Self::candidates_for(rng, schema, model, choice)) { - return Some(rewrite); - } + pub(crate) fn choose_accepted(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { + let max_rules = 1 + rng.index(10); + build_accepted_migration( + rng, + RuleCtx { schema, model }, + AcceptedComposition::Capped { max_rules }, + ) + } + + pub(crate) fn choose_rejected(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { + build_rejected_migration(rng, RuleCtx { schema, model }) + } +} + +fn build_accepted_migration(rng: &Rng, ctx: RuleCtx<'_>, mode: AcceptedComposition) -> Option { + let AcceptedComposition::Capped { max_rules } = mode; + let max_rules = max_rules.max(1); + + for _ in 0..16 { + let cases = pick_accepted_cases(rng, ctx, max_rules)?; + if let Ok(migration) = build_validated_accepted_batch(ctx, cases) { + return Some(migration); } + } - pick_rewrite(rng, Self::candidates(rng, schema, model)) + let case = pick_accepted_case(rng, ctx, &DEFAULT_RULE_WEIGHTS)?; + build_validated_accepted_batch(ctx, vec![case]).ok() +} + +fn build_validated_accepted_batch(ctx: RuleCtx<'_>, cases: Vec) -> anyhow::Result { + anyhow::ensure!(!cases.is_empty(), "empty accepted migration batch"); + validate_no_resource_conflicts(&cases)?; + + let mut draft = ctx.schema.clone(); + for case in &cases { + case.rewrite.apply_to(&mut draft)?; } - pub(crate) fn choose_rejected(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Option { - for _ in 0..16 { - let choice = RejectedMigrationChoice::pick(rng, &RejectedMigrationChoice::CHOICES); - if let Some(migration) = pick_migration(Self::rejected_candidates_for(schema, model, choice), rng) { - return Some(migration); - } + for case in &cases { + let rule = rule_by_id(case.rule) + .ok_or_else(|| anyhow::anyhow!("accepted migration case references missing rule {:?}", case.rule))?; + rule.rule.validate_accepted(case, ctx, &draft)?; + } + + anyhow::ensure!(draft != *ctx.schema, "accepted migration batch was no-op"); + Ok(Migration::from_schema(draft)) +} + +fn build_rejected_migration(rng: &Rng, ctx: RuleCtx<'_>) -> Option { + let rejected = pick_rejected_case(rng, ctx)?; + Migration::from_rewrite_with_expectation( + ctx.schema, + rejected.rewrite, + MigrationExpectation::Rejected(rejected.expected), + ) + .ok() +} + +fn rule_by_id(id: RuleId) -> Option<&'static RuleEntry> { + migration_rules().iter().find(|entry| entry.id == id) +} + +fn pick_accepted_cases(rng: &Rng, ctx: RuleCtx<'_>, max_rules: usize) -> Option> { + let mut cases = Vec::new(); + for _ in 0..max_rules { + let Some(case) = pick_accepted_case(rng, ctx, &DEFAULT_RULE_WEIGHTS) else { + break; + }; + cases.push(case); + } + (!cases.is_empty()).then_some(cases) +} + +fn pick_accepted_case(rng: &Rng, ctx: RuleCtx<'_>, weights: &RuleWeights) -> Option { + for _ in 0..16 { + let entry = pick_weighted_rule(rng, weights.accepted)?; + if let Some(case) = pick_candidate(rng, entry.rule.accepted_cases(rng, ctx, entry.id)) { + return Some(case); + } + } + + pick_candidate(rng, accepted_cases_for_weights(rng, ctx, weights.accepted)) +} + +fn pick_rejected_case(rng: &Rng, ctx: RuleCtx<'_>) -> Option { + for _ in 0..16 { + let entry = pick_weighted_rule(rng, DEFAULT_RULE_WEIGHTS.rejected)?; + if let Some(case) = pick_candidate(rng, entry.rule.rejected_cases(rng, ctx)) { + return Some(case); + } + } + + pick_candidate(rng, rejected_cases_for_weights(rng, ctx, DEFAULT_RULE_WEIGHTS.rejected)) +} + +fn pick_weighted_rule(rng: &Rng, weights: &[(RuleId, u32)]) -> Option<&'static RuleEntry> { + let total = weights.iter().map(|(_, weight)| *weight).sum::(); + if total == 0 { + return None; + } + + let mut ticket = rng.index(total as usize) as u32; + for &(id, weight) in weights { + if ticket < weight { + return Some(rule_by_id(id).expect("rule weight references unregistered migration rule")); + } + ticket -= weight; + } + + None +} + +fn accepted_cases_for_weights(rng: &Rng, ctx: RuleCtx<'_>, weights: &[(RuleId, u32)]) -> Vec { + weights + .iter() + .filter(|(_, weight)| *weight > 0) + .filter_map(|(id, _)| rule_by_id(*id)) + .flat_map(|entry| entry.rule.accepted_cases(rng, ctx, entry.id)) + .collect() +} + +fn rejected_cases_for_weights(rng: &Rng, ctx: RuleCtx<'_>, weights: &[(RuleId, u32)]) -> Vec { + weights + .iter() + .filter(|(_, weight)| *weight > 0) + .filter_map(|(id, _)| rule_by_id(*id)) + .flat_map(|entry| entry.rule.rejected_cases(rng, ctx)) + .collect() +} + +fn pick_candidate(rng: &Rng, mut candidates: Vec) -> Option { + let idx = choose_index(rng, candidates.len())?; + Some(candidates.swap_remove(idx)) +} + +fn validate_no_resource_conflicts(cases: &[AcceptedCase]) -> anyhow::Result<()> { + for (left_idx, left) in cases.iter().enumerate() { + for right in &cases[left_idx + 1..] { + validate_resource_set_is_disjoint(&left.writes, &right.writes)?; + validate_resource_set_is_disjoint(&left.writes, &right.protects)?; + validate_resource_set_is_disjoint(&left.protects, &right.writes)?; } + } + Ok(()) +} + +fn validate_resource_set_is_disjoint(left: &[Resource], right: &[Resource]) -> anyhow::Result<()> { + for left in left { + for right in right { + anyhow::ensure!( + !resources_conflict(left, right), + "accepted migration batch has conflicting resources {left:?} and {right:?}" + ); + } + } + Ok(()) +} + +fn write_resources(rewrite: &SchemaRewrite) -> Vec { + let mut resources = match rewrite { + SchemaRewrite::AddTable { table } => vec![Resource::Table(table.name.clone())], + SchemaRewrite::RemoveTable { table } => vec![Resource::Table(table.clone())], + SchemaRewrite::AlterTable { table, ops } => ops + .iter() + .map(|op| match op { + TableMigrationOp::ChangeAccess + | TableMigrationOp::AddColumn { .. } + | TableMigrationOp::ReschemaEventTable => Resource::Table(table.clone()), + TableMigrationOp::AddIndex { columns, .. } + | TableMigrationOp::RemoveIndex { columns } + | TableMigrationOp::ChangeIndex { columns } => Resource::Index { + table: table.clone(), + columns: columns.clone(), + }, + TableMigrationOp::AddSequence { sequence } => Resource::Sequence { + table: table.clone(), + column: sequence.column, + }, + TableMigrationOp::RemoveSequence { column } => Resource::Sequence { + table: table.clone(), + column: *column, + }, + TableMigrationOp::AddUniqueConstraint { columns } + | TableMigrationOp::RemoveUniqueConstraint { columns } => Resource::UniqueConstraint { + table: table.clone(), + columns: columns.clone(), + }, + TableMigrationOp::ChangePrimaryKey { .. } => Resource::PrimaryKey { table: table.clone() }, + TableMigrationOp::ChangeColumnType { column } => Resource::Column { + table: table.clone(), + column: *column, + }, + }) + .collect(), + }; + dedup_resources(&mut resources); + resources +} + +fn dedup_resources(resources: &mut Vec) { + let mut seen = HashSet::new(); + resources.retain(|resource| seen.insert(resource.clone())); +} + +fn resources_conflict(left: &Resource, right: &Resource) -> bool { + if left == right { + return true; + } + + match (left, right) { + (Resource::Table(left), right) => resource_table(right).is_some_and(|right| right == left), + (left, Resource::Table(right)) => resource_table(left).is_some_and(|left| left == right), + ( + Resource::Column { + table: left_table, + column: left_column, + }, + Resource::Sequence { + table: right_table, + column: right_column, + }, + ) + | ( + Resource::Sequence { + table: left_table, + column: left_column, + }, + Resource::Column { + table: right_table, + column: right_column, + }, + ) => left_table == right_table && left_column == right_column, + _ => false, + } +} - pick_migration(Self::rejected_candidates(schema, model), rng) +fn resource_table(resource: &Resource) -> Option<&str> { + match resource { + Resource::Table(table) + | Resource::Column { table, .. } + | Resource::Index { table, .. } + | Resource::Sequence { table, .. } + | Resource::UniqueConstraint { table, .. } + | Resource::PrimaryKey { table } => Some(table), } +} + +fn accepted_cases_from_rewrites(rule: RuleId, rewrites: Vec) -> Vec { + rewrites + .into_iter() + .map(|rewrite| AcceptedCase::new(rule, rewrite)) + .collect() +} + +fn table_is_pristine(model: &Model, table: &TablePlan) -> bool { + model.row_count_by_table_name(&table.name) == 0 && !model.ever_inserted_by_table_name(&table.name) +} + +fn original_table<'a>(ctx: RuleCtx<'a>, table: &str) -> anyhow::Result<&'a TablePlan> { + ctx.schema + .tables + .iter() + .find(|table_plan| table_plan.name == table) + .ok_or_else(|| anyhow::anyhow!("migration rule references missing original table {table}")) +} + +fn alter_table_sequence(case: &AcceptedCase) -> anyhow::Result<(&str, &SequencePlan)> { + let SchemaRewrite::AlterTable { table, ops } = &case.rewrite else { + anyhow::bail!("add-sequence rule emitted non-table rewrite"); + }; + let Some(sequence) = ops.iter().find_map(|op| match op { + TableMigrationOp::AddSequence { sequence } => Some(sequence), + _ => None, + }) else { + anyhow::bail!("add-sequence rule emitted rewrite without add-sequence op"); + }; + Ok((table, sequence)) +} + +struct AddTableRule; - pub(crate) fn candidates(rng: &Rng, schema: &SchemaPlan, model: &Model) -> Vec { - MigrationChoice::CHOICES +impl MigrationRule for AddTableRule { + fn accepted_cases(&self, rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, add_table_rewrites(rng, ctx.schema)) + } +} + +struct RemovePristineNonEventTableRule; + +impl MigrationRule for RemovePristineNonEventTableRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + let non_event_tables = ctx.schema.tables.iter().filter(|table| !table.is_event).count(); + ctx.schema + .tables .iter() - .flat_map(|choice| Self::candidates_for(rng, schema, model, choice.value())) + .filter(|table| !table.is_event && non_event_tables > 1 && table_is_pristine(ctx.model, table)) + .map(|table| { + AcceptedCase::new( + rule, + SchemaRewrite::RemoveTable { + table: table.name.clone(), + }, + ) + .protecting(vec![Resource::Table(table.name.clone())]) + }) .collect() } - fn candidates_for(rng: &Rng, schema: &SchemaPlan, model: &Model, choice: MigrationChoice) -> Vec { - match choice { - MigrationChoice::AddTable => add_table_rewrites(rng, schema), - MigrationChoice::RemoveTable => remove_table_rewrites(schema, model), - MigrationChoice::AddColumn => add_column_rewrites(schema), - MigrationChoice::AddIndex => add_index_rewrites(schema), - MigrationChoice::RemoveIndex => remove_index_rewrites(schema), - MigrationChoice::ChangeIndex => change_index_rewrites(schema), - MigrationChoice::AddSequence => add_sequence_rewrites(schema, model), - MigrationChoice::RemoveSequence => remove_sequence_rewrites(schema), - MigrationChoice::AddUniqueConstraint => add_unique_constraint_rewrites(schema, model), - MigrationChoice::RemoveUniqueConstraint => remove_unique_constraint_rewrites(schema), - MigrationChoice::DropPrimaryKeyAndWidenSum => drop_primary_key_and_widen_sum_rewrites(schema), - MigrationChoice::WidenSumColumn => widen_sum_column_rewrites(schema), - MigrationChoice::ReschemaEventTable => reschema_event_table_rewrites(schema, model), - } + fn validate_accepted( + &self, + case: &AcceptedCase, + original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + let SchemaRewrite::RemoveTable { table } = &case.rewrite else { + anyhow::bail!("remove-pristine-table rule emitted non-remove rewrite"); + }; + let table_plan = original_table(original, table)?; + let non_event_tables = original.schema.tables.iter().filter(|table| !table.is_event).count(); + anyhow::ensure!(!table_plan.is_event, "remove-pristine-table rule selected event table"); + anyhow::ensure!( + non_event_tables > 1, + "remove-pristine-table rule selected last non-event table" + ); + anyhow::ensure!( + table_is_pristine(original.model, table_plan), + "remove-pristine-table rule selected non-pristine table" + ); + Ok(()) } +} - fn rejected_candidates(schema: &SchemaPlan, model: &Model) -> Vec { - RejectedMigrationChoice::CHOICES +struct RemoveEmptyEventTableRule; + +impl MigrationRule for RemoveEmptyEventTableRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + ctx.schema + .tables .iter() - .flat_map(|choice| Self::rejected_candidates_for(schema, model, choice.value())) + .filter(|table| table.is_event && ctx.model.row_count_by_table_name(&table.name) == 0) + .map(|table| { + AcceptedCase::new( + rule, + SchemaRewrite::RemoveTable { + table: table.name.clone(), + }, + ) + .protecting(vec![Resource::Table(table.name.clone())]) + }) .collect() } - fn rejected_candidates_for(schema: &SchemaPlan, model: &Model, choice: RejectedMigrationChoice) -> Vec { - let expectation = MigrationExpectation::Rejected(MigrationRejection::PrecheckFailure); - let rewrites = match choice { - RejectedMigrationChoice::AddSequencePrecheckFailure => { - add_sequence_precheck_failure_rewrites(schema, model) - } - RejectedMigrationChoice::AddI64DefaultMaxSequencePrecheckFailure => { - add_i64_default_max_sequence_precheck_failure_rewrites(schema, model) - } + fn validate_accepted( + &self, + case: &AcceptedCase, + original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + let SchemaRewrite::RemoveTable { table } = &case.rewrite else { + anyhow::bail!("remove-empty-event-table rule emitted non-remove rewrite"); }; + let table_plan = original_table(original, table)?; + anyhow::ensure!( + table_plan.is_event, + "remove-empty-event-table rule selected non-event table" + ); + anyhow::ensure!( + original.model.row_count_by_table_name(table) == 0, + "remove-empty-event-table rule selected non-empty table" + ); + Ok(()) + } +} - rewrites +struct AddColumnRule; + +impl MigrationRule for AddColumnRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, add_column_rewrites(ctx.schema)) + } +} + +struct AddIndexRule; + +impl MigrationRule for AddIndexRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, add_index_rewrites(ctx.schema)) + } +} + +struct RemoveIndexRule; + +impl MigrationRule for RemoveIndexRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, remove_index_rewrites(ctx.schema)) + } +} + +struct ChangeIndexRule; + +impl MigrationRule for ChangeIndexRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, change_index_rewrites(ctx.schema)) + } +} + +struct AddSequenceOnPristineTableRule; + +impl MigrationRule for AddSequenceOnPristineTableRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + ctx.schema + .tables + .iter() + .filter(|table| !table.is_event && table_is_pristine(ctx.model, table)) + .flat_map(|table| { + addable_sequences(table).into_iter().map(move |sequence| { + AcceptedCase::new( + rule, + SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }]), + ) + .protecting(vec![Resource::Table(table.name.clone())]) + }) + }) + .collect() + } + + fn validate_accepted( + &self, + case: &AcceptedCase, + original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + let (table, sequence) = alter_table_sequence(case)?; + let table_plan = original_table(original, table)?; + anyhow::ensure!(!table_plan.is_event, "add-sequence-pristine rule selected event table"); + anyhow::ensure!( + table_is_pristine(original.model, table_plan), + "add-sequence-pristine rule selected non-pristine table" + ); + let column_plan = table_plan + .columns + .get(sequence.column) + .ok_or_else(|| anyhow::anyhow!("add-sequence-pristine rule references missing column"))?; + anyhow::ensure!( + column_plan.ty.is_integral(), + "add-sequence-pristine rule selected non-integral column" + ); + anyhow::ensure!( + table_plan + .sequences + .iter() + .all(|existing| existing.column != sequence.column), + "add-sequence-pristine rule selected already sequenced column" + ); + Ok(()) + } +} + +struct AddSequenceBoundaryProbeRule; + +impl MigrationRule for AddSequenceBoundaryProbeRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + ctx.schema + .tables + .iter() + .filter(|table| !table.is_event && ctx.model.row_count_by_table_name(&table.name) > 0) + .flat_map(|table| { + addable_sequence_boundary_probes(table, |column| column_domain(ctx.model, table, column)) + .into_iter() + .map(move |sequence| { + let column = sequence.column; + AcceptedCase::new( + rule, + SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }]), + ) + .protecting(vec![Resource::Column { + table: table.name.clone(), + column, + }]) + }) + }) + .collect() + } + + fn rejected_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>) -> Vec { + add_sequence_precheck_failure_rewrites(ctx.schema, ctx.model) .into_iter() - .filter_map(|rewrite| Self::from_rewrite_with_expectation(schema, rewrite, expectation).ok()) + .map(|rewrite| RejectedCase { + rewrite, + expected: MigrationRejection::PrecheckFailure, + }) .collect() } + + fn validate_accepted( + &self, + case: &AcceptedCase, + original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + let (table, sequence) = alter_table_sequence(case)?; + let table_plan = original_table(original, table)?; + anyhow::ensure!(!table_plan.is_event, "add-sequence-boundary rule selected event table"); + anyhow::ensure!( + original.model.row_count_by_table_name(table) > 0, + "add-sequence-boundary rule selected empty table" + ); + let column_plan = table_plan + .columns + .get(sequence.column) + .ok_or_else(|| anyhow::anyhow!("add-sequence-boundary rule references missing column"))?; + let domain = column_domain(original.model, table_plan, sequence.column) + .ok_or_else(|| anyhow::anyhow!("add-sequence-boundary rule references missing column domain"))?; + anyhow::ensure!( + column_plan.ty.is_integral(), + "add-sequence-boundary rule selected non-integral column" + ); + anyhow::ensure!( + table_plan + .sequences + .iter() + .all(|existing| existing.column != sequence.column), + "add-sequence-boundary rule selected already sequenced column" + ); + anyhow::ensure!( + !domain.sequenced, + "add-sequence-boundary rule selected model-sequenced column" + ); + anyhow::ensure!( + domain.single_column_indexed, + "add-sequence-boundary rule selected unindexed column" + ); + anyhow::ensure!( + domain.single_column_unique, + "add-sequence-boundary rule selected non-unique column" + ); + anyhow::ensure!( + added_sequence_precheck_range_is_clear(&domain, sequence), + "accepted add-sequence boundary case no longer clears precheck range" + ); + Ok(()) + } } -fn pick_rewrite(rng: &Rng, mut candidates: Vec) -> Option { - let idx = choose_index(rng, candidates.len())?; - Some(candidates.swap_remove(idx)) +struct RemoveSequenceRule; + +impl MigrationRule for RemoveSequenceRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, remove_sequence_rewrites(ctx.schema)) + } } -fn pick_migration(mut candidates: Vec, rng: &Rng) -> Option { - let idx = choose_index(rng, candidates.len())?; - Some(candidates.swap_remove(idx)) +struct AddUniqueConstraintOnPristineTableRule; + +impl MigrationRule for AddUniqueConstraintOnPristineTableRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + ctx.schema + .tables + .iter() + .filter(|table| !table.is_event && table_is_pristine(ctx.model, table)) + .flat_map(|table| { + addable_unique_constraint_columns(table) + .into_iter() + .map(move |columns| { + let mut ops = Vec::new(); + if !has_index(table, &columns) { + ops.push(TableMigrationOp::AddIndex { + columns: columns.clone(), + algorithm: IndexAlgorithm::BTree, + }); + } + ops.push(TableMigrationOp::AddUniqueConstraint { columns }); + AcceptedCase::new(rule, SchemaRewrite::alter_table(table, ops)) + .protecting(vec![Resource::Table(table.name.clone())]) + }) + }) + .collect() + } + + fn validate_accepted( + &self, + case: &AcceptedCase, + original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + let SchemaRewrite::AlterTable { table, ops } = &case.rewrite else { + anyhow::bail!("add-unique-constraint rule emitted non-table rewrite"); + }; + let table_plan = original_table(original, table)?; + anyhow::ensure!(!table_plan.is_event, "add-unique-constraint rule selected event table"); + anyhow::ensure!( + table_is_pristine(original.model, table_plan), + "add-unique-constraint rule selected non-pristine table" + ); + anyhow::ensure!( + ops.iter() + .any(|op| matches!(op, TableMigrationOp::AddUniqueConstraint { .. })), + "add-unique-constraint rule emitted rewrite without unique-constraint op" + ); + Ok(()) + } +} + +struct RemoveUniqueConstraintRule; + +impl MigrationRule for RemoveUniqueConstraintRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, remove_unique_constraint_rewrites(ctx.schema)) + } +} + +struct DropPrimaryKeyAndWidenSumRule; + +impl MigrationRule for DropPrimaryKeyAndWidenSumRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, drop_primary_key_and_widen_sum_rewrites(ctx.schema)) + } +} + +struct WidenSumColumnRule; + +impl MigrationRule for WidenSumColumnRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, widen_sum_column_rewrites(ctx.schema)) + } +} + +struct ReschemaEmptyEventTableRule; + +impl MigrationRule for ReschemaEmptyEventTableRule { + fn accepted_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { + accepted_cases_from_rewrites(rule, reschema_event_table_rewrites(ctx.schema, ctx.model)) + .into_iter() + .map(|case| { + let protected_table = match &case.rewrite { + SchemaRewrite::AlterTable { table, .. } => Some(table.clone()), + _ => None, + }; + if let Some(table) = protected_table { + case.protecting(vec![Resource::Table(table)]) + } else { + case + } + }) + .collect() + } + + fn validate_accepted( + &self, + case: &AcceptedCase, + original: RuleCtx<'_>, + _final_schema: &SchemaPlan, + ) -> anyhow::Result<()> { + let SchemaRewrite::AlterTable { table, .. } = &case.rewrite else { + anyhow::bail!("reschema-empty-event-table rule emitted non-table rewrite"); + }; + let table_plan = original_table(original, table)?; + anyhow::ensure!( + table_plan.is_event, + "reschema-empty-event-table rule selected non-event table" + ); + anyhow::ensure!( + original.model.row_count_by_table_name(table) == 0, + "reschema-empty-event-table rule selected non-empty table" + ); + anyhow::ensure!( + table_plan.columns.len() < MAX_EVENT_COLUMNS, + "reschema-empty-event-table rule selected full event table" + ); + Ok(()) + } +} + +struct AddI64DefaultMaxSequencePrecheckFailureRule; + +impl MigrationRule for AddI64DefaultMaxSequencePrecheckFailureRule { + fn rejected_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>) -> Vec { + add_i64_default_max_sequence_precheck_failure_rewrites(ctx.schema, ctx.model) + .into_iter() + .map(|rewrite| RejectedCase { + rewrite, + expected: MigrationRejection::PrecheckFailure, + }) + .collect() + } } fn add_table_rewrites(rng: &Rng, schema: &SchemaPlan) -> Vec { @@ -285,24 +1051,6 @@ fn add_table_rewrites(rng: &Rng, schema: &SchemaPlan) -> Vec { .collect() } -fn remove_table_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { - let non_event_tables = schema.tables.iter().filter(|table| !table.is_event).count(); - - schema - .tables - .iter() - .filter_map(|table| { - let row_count = model.row_count_by_table_name(&table.name); - let pristine = row_count == 0 && !model.ever_inserted_by_table_name(&table.name); - ((table.is_event && row_count == 0) || (!table.is_event && pristine && non_event_tables > 1)).then(|| { - SchemaRewrite::RemoveTable { - table: table.name.clone(), - } - }) - }) - .collect() -} - fn add_column_rewrites(schema: &SchemaPlan) -> Vec { let types = addable_column_types(schema); schema @@ -329,11 +1077,11 @@ fn addable_column_types(schema: &SchemaPlan) -> Vec { } fn schema_has_sum_column(schema: &SchemaPlan) -> bool { - schema - .tables - .iter() - .flat_map(|table| table.columns.iter()) - .any(|column| matches!(column.ty, Type::Sum { .. })) + schema.tables.iter().any(table_has_sum_column) +} + +fn table_has_sum_column(table: &TablePlan) -> bool { + table.columns.iter().any(|column| matches!(column.ty, Type::Sum { .. })) } fn add_index_rewrites(schema: &SchemaPlan) -> Vec { @@ -381,33 +1129,6 @@ fn change_index_rewrites(schema: &SchemaPlan) -> Vec { .collect() } -fn add_sequence_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { - let mut rewrites = Vec::new(); - - for table in schema.tables.iter().filter(|table| !table.is_event) { - let row_count = model.row_count_by_table_name(&table.name); - let pristine = row_count == 0 && !model.ever_inserted_by_table_name(&table.name); - - if pristine { - rewrites.extend( - addable_sequences(table) - .into_iter() - .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])), - ); - } - - if row_count > 0 { - rewrites.extend( - addable_sequence_boundary_probes(table, |column| column_domain(model, table, column)) - .into_iter() - .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])), - ); - } - } - - rewrites -} - fn add_sequence_precheck_failure_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { schema .tables @@ -447,32 +1168,6 @@ fn remove_sequence_rewrites(schema: &SchemaPlan) -> Vec { .collect() } -fn add_unique_constraint_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { - let mut rewrites = Vec::new(); - - for table in schema.tables.iter().filter(|table| !table.is_event) { - let row_count = model.row_count_by_table_name(&table.name); - let pristine = row_count == 0 && !model.ever_inserted_by_table_name(&table.name); - if !pristine { - continue; - } - - for columns in addable_unique_constraint_columns(table) { - let mut ops = Vec::new(); - if !has_index(table, &columns) { - ops.push(TableMigrationOp::AddIndex { - columns: columns.clone(), - algorithm: IndexAlgorithm::BTree, - }); - } - ops.push(TableMigrationOp::AddUniqueConstraint { columns }); - rewrites.push(SchemaRewrite::alter_table(table, ops)); - } - } - - rewrites -} - fn remove_unique_constraint_rewrites(schema: &SchemaPlan) -> Vec { schema .tables @@ -550,16 +1245,38 @@ impl SchemaRewrite { pub(crate) fn apply_to(&self, schema: &mut SchemaPlan) -> anyhow::Result<()> { match self { Self::AddTable { table } => { + anyhow::ensure!( + schema.tables.len() < MAX_TABLES, + "migration would exceed the configured maximum number of tables" + ); + anyhow::ensure!( + schema.tables.iter().all(|existing| existing.name != table.name), + "add-table migration selected existing table name" + ); + anyhow::ensure!( + !table_has_sum_column(table) || !schema_has_sum_column(schema), + "migration would add a second sum column" + ); schema.tables.push(table.clone()); } Self::RemoveTable { table } => { let table = table_position(schema, table)?; + if !schema.tables[table].is_event { + let non_event_tables = schema.tables.iter().filter(|table| !table.is_event).count(); + anyhow::ensure!(non_event_tables > 1, "migration would remove the last non-event table"); + } schema.tables.remove(table); } Self::AlterTable { table, ops } => { let table = table_position(schema, table)?; - let table_plan = &mut schema.tables[table]; for op in ops { + if matches!(op, TableMigrationOp::AddColumn { ty: Type::Sum { .. } }) { + anyhow::ensure!( + !schema_has_sum_column(schema), + "migration would add a second sum column" + ); + } + let table_plan = &mut schema.tables[table]; apply_table_op(table_plan, op.clone())?; } } @@ -910,3 +1627,225 @@ fn widen_sum_column(table: &mut TablePlan, column: Option) -> anyhow::Res *variants += 1; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::workload::Interaction; + use spacetimedb_sats::product; + + fn i64_table(name: impl Into, is_event: bool) -> TablePlan { + TablePlan { + name: name.into(), + columns: vec![ColumnPlan { + name: "id".into(), + ty: Type::I64, + }], + primary_key: None, + indexes: vec![], + unique_constraints: vec![], + sequences: vec![], + is_public: true, + is_event, + } + } + + fn indexed_unique_i64_schema() -> SchemaPlan { + let mut table = i64_table("items", false); + table.indexes.push(IndexPlan { + columns: vec![0], + algorithm: IndexAlgorithm::BTree, + }); + table.unique_constraints.push(UniqueConstraintPlan { columns: vec![0] }); + SchemaPlan { tables: vec![table] } + } + + #[test] + fn migration_rule_registry_has_unique_ids_and_default_weights_resolve() { + let mut seen = HashSet::new(); + for entry in migration_rules() { + assert!(seen.insert(entry.id), "duplicate registered RuleId: {:?}", entry.id); + } + + for &(id, weight) in DEFAULT_RULE_WEIGHTS + .accepted + .iter() + .chain(DEFAULT_RULE_WEIGHTS.rejected.iter()) + { + assert!(weight > 0, "default rule weights should not contain zero entries"); + assert!( + rule_by_id(id).is_some(), + "default weights reference unregistered rule {id:?}" + ); + } + } + + #[test] + fn accepted_batch_rejects_write_to_protected_table() { + let schema = indexed_unique_i64_schema(); + let model = Model::new(schema.clone()); + let table = &schema.tables[0]; + let sequence = SequencePlan::new(0, Type::I64).expect("i64 sequence"); + + let add_sequence = AcceptedCase::new( + RuleId::AddSequenceOnPristineTable, + SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }]), + ) + .protecting(vec![Resource::Table(table.name.clone())]); + let add_column = AcceptedCase::new( + RuleId::AddColumn, + SchemaRewrite::alter_table(table, [TableMigrationOp::AddColumn { ty: Type::U64 }]), + ); + + let err = build_validated_accepted_batch( + RuleCtx { + schema: &schema, + model: &model, + }, + vec![add_sequence, add_column], + ) + .expect_err("protected table write should be rejected before emission"); + + assert!(err.to_string().contains("conflicting resources")); + } + + #[test] + fn add_sequence_boundary_rule_emits_explicit_precheck_rejections() { + let schema = indexed_unique_i64_schema(); + let mut model = Model::new(schema.clone()); + model.apply(&Interaction::BeginMutTx); + model.apply(&Interaction::Insert { + table: 0, + row: product![4i64], + }); + model.apply(&Interaction::Insert { + table: 0, + row: product![5i64], + }); + model.apply(&Interaction::CommitTx); + + let cases = AddSequenceBoundaryProbeRule.rejected_cases( + &Rng::new(0), + RuleCtx { + schema: &schema, + model: &model, + }, + ); + + assert!(!cases.is_empty(), "expected at least one precheck-failure case"); + for case in cases { + assert_eq!(case.expected, MigrationRejection::PrecheckFailure); + let SchemaRewrite::AlterTable { ops, .. } = &case.rewrite else { + panic!("precheck rejection should alter one table"); + }; + assert_eq!(ops.len(), 1); + assert!(matches!(ops[0], TableMigrationOp::AddSequence { .. })); + Migration::from_rewrite_with_expectation( + &schema, + case.rewrite, + MigrationExpectation::Rejected(MigrationRejection::PrecheckFailure), + ) + .expect("rejected case should still be structurally valid"); + } + } + + #[test] + fn accepted_batch_rejects_removing_every_non_event_table() { + let schema = SchemaPlan { + tables: vec![i64_table("left", false), i64_table("right", false)], + }; + let model = Model::new(schema.clone()); + + let remove_left = AcceptedCase::new( + RuleId::RemovePristineNonEventTable, + SchemaRewrite::RemoveTable { table: "left".into() }, + ); + let remove_right = AcceptedCase::new( + RuleId::RemovePristineNonEventTable, + SchemaRewrite::RemoveTable { table: "right".into() }, + ); + + let err = build_validated_accepted_batch( + RuleCtx { + schema: &schema, + model: &model, + }, + vec![remove_left, remove_right], + ) + .expect_err("batch should not remove all non-event tables"); + + assert!(err.to_string().contains("last non-event table")); + } + + #[test] + fn accepted_batch_rejects_exceeding_max_tables() { + let schema = SchemaPlan { + tables: (0..MAX_TABLES - 1) + .map(|idx| i64_table(format!("table_{idx}"), false)) + .collect(), + }; + let model = Model::new(schema.clone()); + + let add_first = AcceptedCase::new( + RuleId::AddTable, + SchemaRewrite::AddTable { + table: i64_table("new_a", false), + }, + ); + let add_second = AcceptedCase::new( + RuleId::AddTable, + SchemaRewrite::AddTable { + table: i64_table("new_b", false), + }, + ); + + let err = build_validated_accepted_batch( + RuleCtx { + schema: &schema, + model: &model, + }, + vec![add_first, add_second], + ) + .expect_err("batch should not exceed table cap"); + + assert!(err.to_string().contains("maximum number of tables")); + } + + #[test] + fn accepted_batch_rejects_adding_two_sum_columns() { + let schema = SchemaPlan { + tables: vec![i64_table("left", false), i64_table("right", false)], + }; + let model = Model::new(schema.clone()); + + let add_left_sum = AcceptedCase::new( + RuleId::AddColumn, + SchemaRewrite::alter_table( + &schema.tables[0], + [TableMigrationOp::AddColumn { + ty: Type::Sum { variants: 1 }, + }], + ), + ); + let add_right_sum = AcceptedCase::new( + RuleId::AddColumn, + SchemaRewrite::alter_table( + &schema.tables[1], + [TableMigrationOp::AddColumn { + ty: Type::Sum { variants: 1 }, + }], + ), + ); + + let err = build_validated_accepted_batch( + RuleCtx { + schema: &schema, + model: &model, + }, + vec![add_left_sum, add_right_sum], + ) + .expect_err("batch should not add two sum columns"); + + assert!(err.to_string().contains("second sum column")); + } +} diff --git a/crates/dst/src/rng.rs b/crates/dst/src/rng.rs index 5fd494b6d3e..8c1360e5b4f 100644 --- a/crates/dst/src/rng.rs +++ b/crates/dst/src/rng.rs @@ -9,12 +9,6 @@ pub(crate) struct Choice { value: T, } -impl Choice { - pub(crate) const fn value(self) -> T { - self.value - } -} - /// Construct a weighted choice entry. pub(crate) const fn choice(weight: u64, value: T) -> Choice { Choice { weight, value } From 05d789d8fea19fcd3dc63db28bcc4c851e886a9e Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 29 Jul 2026 16:16:58 +0530 Subject: [PATCH 5/6] snakecase --- crates/dst/src/engine.rs | 45 +++++++++--- crates/dst/src/engine/migrations.rs | 51 ++++++++++---- crates/dst/src/engine/properties.rs | 102 +++++++++++++++++++++++++++- crates/dst/src/engine/state.rs | 91 +++++++------------------ crates/dst/src/schema.rs | 70 ++++++++++++++++++- 5 files changed, 268 insertions(+), 91 deletions(-) diff --git a/crates/dst/src/engine.rs b/crates/dst/src/engine.rs index ff90cd0a233..77ce8826491 100644 --- a/crates/dst/src/engine.rs +++ b/crates/dst/src/engine.rs @@ -8,7 +8,7 @@ use spacetimedb_engine::persistence::{DiskSizeFn, Durability as EngineDurability use spacetimedb_engine::relational_db::{MutTx, RelationalDB}; use spacetimedb_engine::update::{update_database, UpdateLogger}; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::{Identity, RawModuleDef}; +use spacetimedb_lib::Identity; use spacetimedb_primitives::TableId; use spacetimedb_runtime::sim::{Rng, Runtime as SimRuntime}; use spacetimedb_runtime::Handle; @@ -35,7 +35,7 @@ use self::workload::{InsertOutcome, Interaction, Observation}; use crate::engine::model::Model; use crate::engine::properties::EngineProperties; use crate::engine::workload::WorkloadGen; -use crate::schema::{default_schema, to_raw_def, SchemaPlan}; +use crate::schema::{canonical_schema, default_schema, to_module_def, SchemaPlan}; use crate::sim::commitlog::{InMemoryCommitlog, InMemoryCommitlogHandle}; use crate::traits::{TargetDriver, TestSuite}; @@ -50,13 +50,14 @@ pub struct EngineTarget { impl EngineTarget { pub fn init(schema: SchemaPlan, runtime_seed: u64) -> anyhow::Result { + let canonical = canonical_schema(&schema)?; let runtime = SimRuntime::new(runtime_seed); let runtime_handle = Handle::simulation(runtime.handle()); let commitlog = InMemoryCommitlog::new(); let db = Self::open_db(&commitlog, runtime_handle.clone())?; Self::install_schema(&db, &schema)?; - let table_ids = Self::load_table_ids(&db, &schema)?; + let table_ids = Self::load_table_ids(&db, &canonical)?; Ok(Self { db: Some(db), @@ -64,7 +65,7 @@ impl EngineTarget { active_mut_tx: None, commitlog, runtime_handle, - schema, + schema: canonical, }) } @@ -100,9 +101,7 @@ impl EngineTarget { } fn module_def(schema: &SchemaPlan) -> anyhow::Result { - let raw = to_raw_def(schema); - let raw_module_def = RawModuleDef::V10(raw); - ModuleDef::try_from(raw_module_def).map_err(|e| anyhow::anyhow!("schema validation failed: {e}")) + to_module_def(schema) } fn install_schema(db: &RelationalDB, schema: &SchemaPlan) -> anyhow::Result<()> { @@ -231,7 +230,7 @@ impl EngineTarget { ); let old_module_def = Self::module_def(&self.schema)?; - let new_module_def = match Self::module_def(migration.schema()) { + let new_module_def = match Self::module_def(migration.target_schema()) { Ok(module_def) => module_def, Err(error) => { return match migration.expectation() { @@ -438,7 +437,8 @@ impl TestSuite for EngineTest { async fn build(&self, rng: Rng) -> Result<(Self::Interactions, Self::Target, Self::Properties), anyhow::Error> { let schema = default_schema(rng.clone()); let runtime_seed = rng.next_u64(); - let target = EngineTarget::init(schema.clone(), runtime_seed)?; + let target = EngineTarget::init(schema, runtime_seed)?; + let schema = target.schema.clone(); let properties = EngineProperties::new(schema.clone()); let model = Model::new(schema); @@ -564,6 +564,33 @@ mod tests { Ok(()) } + #[test] + fn init_installs_raw_schema_and_stores_canonical_schema() -> anyhow::Result<()> { + let raw_schema = SchemaPlan { + tables: vec![TablePlan { + name: "RawTable".into(), + columns: vec![ColumnPlan { + name: "SomeValue".into(), + ty: Type::U64, + }], + primary_key: None, + indexes: vec![], + unique_constraints: vec![], + sequences: vec![], + is_public: true, + is_event: false, + }], + }; + + let expected = canonical_schema(&raw_schema)?; + assert_ne!(expected, raw_schema); + + let target = EngineTarget::init(raw_schema, 0)?; + assert_eq!(target.schema, expected); + + Ok(()) + } + #[test] fn add_column_migration_replays_with_existing_rows() -> anyhow::Result<()> { let mut target = EngineTarget::init(add_column_replay_schema(), 0)?; diff --git a/crates/dst/src/engine/migrations.rs b/crates/dst/src/engine/migrations.rs index 47de1200f49..03a33b31dd4 100644 --- a/crates/dst/src/engine/migrations.rs +++ b/crates/dst/src/engine/migrations.rs @@ -7,8 +7,8 @@ use super::model::{ColumnDomain, Model}; use crate::rng::{choice, choose_index, Choice, WeightedChoice}; use crate::schema::{ - ColumnPlan, IndexAlgorithm, IndexPlan, SchemaGenerator, SchemaNames, SchemaPlan, SchemaProfile, SequencePlan, - TablePlan, Type, UniqueConstraintPlan, + canonical_schema, ColumnPlan, IndexAlgorithm, IndexPlan, SchemaGenerator, SchemaNames, SchemaPlan, SchemaProfile, + SequencePlan, TablePlan, Type, UniqueConstraintPlan, }; use spacetimedb_runtime::sim::Rng; use std::collections::HashSet; @@ -18,9 +18,16 @@ const MAX_EVENT_COLUMNS: usize = 32; const MAX_TABLE_COLUMNS: usize = 32; const MAX_TABLES: usize = 128; -/// A fully materialized target schema for one migration interaction. +/// A fully materialized schema migration interaction. +/// +/// `target_schema` is the raw schema handed to engine validation. `schema` is +/// the canonical schema the model should observe after the engine applies the +/// raw definition and `SnakeCase` conversion. Rejected migrations may not have a +/// valid canonical form, so they keep both fields identical and never advance +/// model state. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Migration { + target_schema: SchemaPlan, schema: SchemaPlan, expectation: MigrationExpectation, } @@ -315,12 +322,21 @@ enum AcceptedComposition { } impl Migration { - pub(crate) fn from_schema(schema: SchemaPlan) -> Self { - Self::from_schema_with_expectation(schema, MigrationExpectation::Accepted) + fn accepted(target_schema: SchemaPlan) -> anyhow::Result { + let schema = canonical_schema(&target_schema)?; + Ok(Self { + target_schema, + schema, + expectation: MigrationExpectation::Accepted, + }) } - pub(crate) fn from_schema_with_expectation(schema: SchemaPlan, expectation: MigrationExpectation) -> Self { - Self { schema, expectation } + fn rejected(target_schema: SchemaPlan, rejection: MigrationRejection) -> Self { + Self { + schema: target_schema.clone(), + target_schema, + expectation: MigrationExpectation::Rejected(rejection), + } } fn from_rewrite_with_expectation( @@ -330,7 +346,10 @@ impl Migration { ) -> anyhow::Result { let mut schema = base.clone(); rewrite.apply_to(&mut schema)?; - Ok(Self::from_schema_with_expectation(schema, expectation)) + match expectation { + MigrationExpectation::Accepted => Self::accepted(schema), + MigrationExpectation::Rejected(rejection) => Ok(Self::rejected(schema, rejection)), + } } #[cfg(test)] @@ -339,13 +358,19 @@ impl Migration { for rewrite in &rewrites { rewrite.apply_to(&mut schema)?; } - Ok(Self::from_schema(schema)) + Self::accepted(schema) } + /// Canonical schema visible to the DST model after an accepted migration. pub(crate) fn schema(&self) -> &SchemaPlan { &self.schema } + /// Raw target schema handed to engine validation and auto-migration. + pub(crate) fn target_schema(&self) -> &SchemaPlan { + &self.target_schema + } + pub(crate) fn expectation(&self) -> MigrationExpectation { self.expectation } @@ -388,14 +413,16 @@ fn build_validated_accepted_batch(ctx: RuleCtx<'_>, cases: Vec) -> case.rewrite.apply_to(&mut draft)?; } + let final_schema = canonical_schema(&draft)?; + for case in &cases { let rule = rule_by_id(case.rule) .ok_or_else(|| anyhow::anyhow!("accepted migration case references missing rule {:?}", case.rule))?; - rule.rule.validate_accepted(case, ctx, &draft)?; + rule.rule.validate_accepted(case, ctx, &final_schema)?; } - anyhow::ensure!(draft != *ctx.schema, "accepted migration batch was no-op"); - Ok(Migration::from_schema(draft)) + anyhow::ensure!(final_schema != *ctx.schema, "accepted migration batch was no-op"); + Migration::accepted(draft) } fn build_rejected_migration(rng: &Rng, ctx: RuleCtx<'_>) -> Option { diff --git a/crates/dst/src/engine/properties.rs b/crates/dst/src/engine/properties.rs index 0a0b6e19abd..3369ffa25c3 100644 --- a/crates/dst/src/engine/properties.rs +++ b/crates/dst/src/engine/properties.rs @@ -1,4 +1,5 @@ use super::model::Model; +use super::state::CountState; use super::workload::{InsertOutcome, Interaction, Observation}; use crate::schema::SchemaPlan; use crate::traits::Properties; @@ -177,8 +178,107 @@ impl EngineProperty for ReplayMatchesModel { anyhow::ensure!( state == expected, - "replay_matches_model: replayed state diverged from model" + "replay_matches_model: replayed state diverged from model: {}", + count_state_mismatch(state, expected) ); Ok(()) } } + +fn count_state_mismatch(target: &CountState, model: &CountState) -> String { + if target.schema != model.schema { + if target.schema.tables.len() != model.schema.tables.len() { + return format!( + "schema table count target={} model={}", + target.schema.tables.len(), + model.schema.tables.len() + ); + } + + for (table_idx, (target_table, model_table)) in + target.schema.tables.iter().zip(&model.schema.tables).enumerate() + { + if target_table == model_table { + continue; + } + + if target_table.name != model_table.name { + return format!( + "schema table {table_idx} name target={:?} model={:?}", + target_table.name, model_table.name + ); + } + if target_table.columns != model_table.columns { + return format!( + "schema table {table_idx} columns target={:?} model={:?}", + target_table.columns, model_table.columns + ); + } + if target_table.indexes != model_table.indexes { + return format!( + "schema table {table_idx} indexes target={:?} model={:?}", + target_table.indexes, model_table.indexes + ); + } + if target_table.unique_constraints != model_table.unique_constraints { + return format!( + "schema table {table_idx} unique constraints target={:?} model={:?}", + target_table.unique_constraints, model_table.unique_constraints + ); + } + if target_table.sequences != model_table.sequences { + return format!( + "schema table {table_idx} sequences target={:?} model={:?}", + target_table.sequences, model_table.sequences + ); + } + return format!("schema table {table_idx} target={target_table:?} model={model_table:?}"); + } + } + + if target.row_counts != model.row_counts { + return format!("row counts target={:?} model={:?}", target.row_counts, model.row_counts); + } + + if target.table_rows.len() != model.table_rows.len() { + return format!( + "table row set count target={} model={}", + target.table_rows.len(), + model.table_rows.len() + ); + } + + for (table_idx, (target_rows, model_rows)) in target.table_rows.iter().zip(&model.table_rows).enumerate() { + if target_rows == model_rows { + continue; + } + if target_rows.table != model_rows.table { + return format!( + "table row entry {table_idx} id target={} model={}", + target_rows.table, model_rows.table + ); + } + if target_rows.rows.len() != model_rows.rows.len() { + return format!( + "table {} row count target={} model={}", + target_rows.table, + target_rows.rows.len(), + model_rows.rows.len() + ); + } + if let Some(row_idx) = target_rows + .rows + .iter() + .zip(&model_rows.rows) + .position(|(target_row, model_row)| target_row != model_row) + { + return format!( + "table {} row {row_idx} target={:?} model={:?}", + target_rows.table, target_rows.rows[row_idx], model_rows.rows[row_idx] + ); + } + return format!("table {} rows differ", target_rows.table); + } + + "unknown mismatch".into() +} diff --git a/crates/dst/src/engine/state.rs b/crates/dst/src/engine/state.rs index db579034ebb..73f10a9d120 100644 --- a/crates/dst/src/engine/state.rs +++ b/crates/dst/src/engine/state.rs @@ -1,13 +1,11 @@ use spacetimedb_lib::db::auth::StAccess; use spacetimedb_lib::AlgebraicType; -use spacetimedb_primitives::ColId; +use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_schema::def::IndexAlgorithm as SchemaIndexAlgorithm; -use spacetimedb_schema::schema::TableSchema; +use spacetimedb_schema::schema::{Schema, TableSchema}; use super::row::Row; -use crate::schema::{ - IndexAlgorithm as PlanIndexAlgorithm, IndexPlan, SchemaPlan, SequencePlan, TablePlan, UniqueConstraintPlan, -}; +use crate::schema::{to_module_def, SchemaPlan}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CountState { @@ -89,15 +87,6 @@ pub struct TableDelta { pub truncated: bool, } -impl From for IndexAlgorithmState { - fn from(algorithm: PlanIndexAlgorithm) -> Self { - match algorithm { - PlanIndexAlgorithm::BTree => Self::BTree, - PlanIndexAlgorithm::Hash => Self::Hash, - } - } -} - impl From<&SchemaIndexAlgorithm> for IndexAlgorithmState { fn from(algorithm: &SchemaIndexAlgorithm) -> Self { match algorithm { @@ -110,13 +99,6 @@ impl From<&SchemaIndexAlgorithm> for IndexAlgorithmState { } impl IndexState { - fn from_plan(index: &IndexPlan) -> Self { - Self { - columns: index.columns.clone(), - algorithm: index.algorithm.into(), - } - } - fn from_schema(algorithm: &SchemaIndexAlgorithm) -> Self { Self { columns: schema_index_columns(algorithm), @@ -126,12 +108,6 @@ impl IndexState { } impl UniqueConstraintState { - fn from_plan(constraint: &UniqueConstraintPlan) -> Self { - Self { - columns: constraint.columns.clone(), - } - } - fn from_schema_columns(columns: impl IntoIterator) -> Self { Self { columns: columns.into_iter().map(|col| col.0 as usize).collect(), @@ -140,12 +116,6 @@ impl UniqueConstraintState { } impl SequenceState { - fn from_plan(sequence: &SequencePlan) -> Self { - Self { - column: sequence.column, - } - } - fn from_schema_column(column: ColId) -> Self { Self { column: column.0 as usize, @@ -154,14 +124,27 @@ impl SequenceState { } pub fn schema_state_for_plan(schema: &SchemaPlan) -> SchemaState { - SchemaState { - tables: schema - .tables - .iter() - .enumerate() - .map(|(table, table_plan)| table_schema_state_for_plan(table, table_plan)) - .collect(), - } + schema_state_for_valid_plan(schema).expect("model schema must lower to a valid module definition") +} + +fn schema_state_for_valid_plan(schema: &SchemaPlan) -> anyhow::Result { + let module_def = to_module_def(schema)?; + + let tables = schema + .tables + .iter() + .enumerate() + .map(|(table, table_plan)| { + let table_def = module_def + .tables() + .find(|table_def| &*table_def.accessor_name == table_plan.name) + .ok_or_else(|| anyhow::anyhow!("validated schema is missing table accessor {:?}", table_plan.name))?; + let table_schema = TableSchema::from_module_def(&module_def, table_def, (), TableId::SENTINEL); + Ok(table_schema_state_for_schema(table, &table_schema)) + }) + .collect::>>()?; + + Ok(SchemaState { tables }) } pub fn table_schema_state_for_schema(table: usize, schema: &TableSchema) -> TableSchemaState { @@ -200,32 +183,6 @@ pub fn table_schema_state_for_schema(table: usize, schema: &TableSchema) -> Tabl } } -fn table_schema_state_for_plan(table: usize, table_plan: &TablePlan) -> TableSchemaState { - TableSchemaState { - table, - name: table_plan.name.clone(), - is_public: table_plan.is_public, - is_event: table_plan.is_event, - primary_key: table_plan.primary_key, - columns: table_plan - .columns - .iter() - .map(|column| ColumnState { - name: column.name.clone(), - ty: column.ty.to_algebraic(), - }) - .collect(), - indexes: sorted(table_plan.indexes.iter().map(IndexState::from_plan)), - unique_constraints: sorted( - table_plan - .unique_constraints - .iter() - .map(UniqueConstraintState::from_plan), - ), - sequences: sorted(table_plan.sequences.iter().map(SequenceState::from_plan)), - } -} - fn schema_index_columns(algorithm: &SchemaIndexAlgorithm) -> Vec { algorithm.columns().iter().map(|col| col.0 as usize).collect() } diff --git a/crates/dst/src/schema.rs b/crates/dst/src/schema.rs index 79b9bb753e9..a817f9397bb 100644 --- a/crates/dst/src/schema.rs +++ b/crates/dst/src/schema.rs @@ -1,13 +1,14 @@ //! Schema plans and raw module lowering for the engine DST harness. use crate::rng; -use spacetimedb_lib::db::raw_def::v10::*; use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, TableAccess, TableType}; +use spacetimedb_lib::{db::raw_def::v10::*, RawModuleDef}; use spacetimedb_primitives::{ColId, ColList}; use spacetimedb_runtime::sim::Rng; use spacetimedb_sats::{ AlgebraicType, AlgebraicValue, ArrayType, ArrayValue, ProductType, ProductTypeElement, SumType, SumTypeVariant, }; +use spacetimedb_schema::def::ModuleDef; /// Generate the default engine DST schema. /// @@ -32,6 +33,54 @@ pub fn to_raw_def(schema: &SchemaPlan) -> RawModuleDefV10 { raw } +/// Lower and validate a schema plan exactly as the engine update path will see it. +pub fn to_module_def(schema: &SchemaPlan) -> anyhow::Result { + ModuleDef::try_from(RawModuleDef::V10(to_raw_def(schema))) + .map_err(|error| anyhow::anyhow!("schema validation failed: {error}")) +} + +/// Return the schema as the engine will see it after raw definition validation. +/// +/// The DST generator intentionally goes through the same `ModuleDef` validation +/// path as the engine instead of duplicating snake-case conversion rules in the +/// model. This keeps migration generation deterministic while making the model +/// use the canonical names that replay and auto-migration observe. +pub fn canonical_schema(schema: &SchemaPlan) -> anyhow::Result { + let module_def = to_module_def(schema)?; + + let mut tables = Vec::with_capacity(schema.tables.len()); + for table_plan in &schema.tables { + let table_def = module_def + .tables() + .find(|table_def| &*table_def.accessor_name == table_plan.name) + .ok_or_else(|| anyhow::anyhow!("validated schema is missing table accessor {:?}", table_plan.name))?; + + anyhow::ensure!( + table_def.columns.len() == table_plan.columns.len(), + "validated table {:?} has {} columns, plan has {}", + table_plan.name, + table_def.columns.len(), + table_plan.columns.len() + ); + + let mut table = table_plan.clone(); + table.name = table_def.name.to_string(); + for (column, column_def) in table.columns.iter_mut().zip(&table_def.columns) { + anyhow::ensure!( + &*column_def.accessor_name == column.name, + "validated table {:?} column accessor mismatch: expected {:?}, got {:?}", + table_plan.name, + column.name, + column_def.accessor_name.to_string() + ); + column.name = column_def.name.to_string(); + } + tables.push(table); + } + + Ok(SchemaPlan { tables }) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Type { Bool, @@ -421,7 +470,7 @@ impl SchemaDecisions { fn gen_ident(rng: &Rng) -> String { const CHARS: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789_"; - const FIRST: &[u8] = b"abcdefghijklmnopqrstuvwxyz_"; + const FIRST: &[u8] = b"abcdefghijklmnopqrstuvwxyz"; let len = 4 + (rng.next_u64() as usize % 12); let mut s = String::with_capacity(len); s.push(FIRST[Self::index(rng, FIRST.len())] as char); @@ -687,6 +736,23 @@ fn schema_has_sum_column(schema: &SchemaPlan) -> bool { #[cfg(test)] mod tests { use super::*; + #[test] + fn seed_3000_schema_canonicalizes_to_valid_snake_case_model_schema() { + let raw_schema = default_schema(Rng::new(3000)); + let schema = canonical_schema(&raw_schema).unwrap(); + let module_def = to_module_def(&schema).unwrap(); + + assert_eq!(canonical_schema(&schema).unwrap(), schema); + + for table_plan in &schema.tables { + let table_def = module_def.table(table_plan.name.as_str()).unwrap(); + assert_eq!(table_def.name.to_string(), table_plan.name); + + for (column_def, column_plan) in table_def.columns.iter().zip(&table_plan.columns) { + assert_eq!(column_def.name.to_string(), column_plan.name); + } + } + } #[test] fn lower_single_table() { From 4bb4f9b55db24485abf024033a5a642b060aab04 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 29 Jul 2026 17:01:44 +0530 Subject: [PATCH 6/6] improve unreviewed migration --- crates/dst/src/engine.rs | 46 ++--- crates/dst/src/engine/migrations.rs | 283 ++++++++++++++++++---------- crates/dst/src/engine/model.rs | 2 +- crates/dst/src/engine/workload.rs | 4 +- 4 files changed, 196 insertions(+), 139 deletions(-) diff --git a/crates/dst/src/engine.rs b/crates/dst/src/engine.rs index 77ce8826491..2fcb0cc8138 100644 --- a/crates/dst/src/engine.rs +++ b/crates/dst/src/engine.rs @@ -25,7 +25,7 @@ mod row; mod state; mod workload; -use self::migrations::{Migration, MigrationExpectation, MigrationRejection}; +use self::migrations::{Migration, MigrationExpectation}; use self::row::{normalize_rows, row_to_bytes}; use self::state::{ table_schema_state_for_schema, CommitDelta, CountState, SchemaState, TableDelta, TableRowCount, TableRows, @@ -234,12 +234,8 @@ impl EngineTarget { Ok(module_def) => module_def, Err(error) => { return match migration.expectation() { - MigrationExpectation::Rejected(MigrationRejection::InvalidDefinition) => { - Ok(Observation::MigrationRejected { - reason: MigrationRejection::InvalidDefinition, - }) - } - _ => Err(error), + MigrationExpectation::Rejected => Ok(Observation::MigrationRejected), + MigrationExpectation::Accepted => Err(error), }; } }; @@ -248,12 +244,8 @@ impl EngineTarget { Ok(plan) => plan, Err(error) => { return match migration.expectation() { - MigrationExpectation::Rejected(MigrationRejection::InvalidDefinition) => { - Ok(Observation::MigrationRejected { - reason: MigrationRejection::InvalidDefinition, - }) - } - _ => Err(error.into()), + MigrationExpectation::Rejected => Ok(Observation::MigrationRejected), + MigrationExpectation::Accepted => Err(error.into()), }; } }; @@ -264,21 +256,7 @@ impl EngineTarget { self.apply_auto_migration(migration, plan)?; Ok(Observation::Migrated) } - MigrationExpectation::Rejected(MigrationRejection::ManualRequired) => match plan { - MigratePlan::Manual(_) => Ok(Observation::MigrationRejected { - reason: MigrationRejection::ManualRequired, - }), - MigratePlan::Auto(_) => { - anyhow::bail!("engine unexpectedly auto-accepted migration expected to require manual migration") - } - }, - MigrationExpectation::Rejected(MigrationRejection::PrecheckFailure) => { - ensure_auto_plan(&plan)?; - self.expect_precheck_failure(plan) - } - MigrationExpectation::Rejected(MigrationRejection::InvalidDefinition) => { - anyhow::bail!("engine produced a migration plan for a schema expected to be invalid") - } + MigrationExpectation::Rejected => self.expect_rejected_migration(plan), } } @@ -298,17 +276,19 @@ impl EngineTarget { Ok(()) } - fn expect_precheck_failure(&self, plan: MigratePlan<'_>) -> anyhow::Result { + fn expect_rejected_migration(&self, plan: MigratePlan<'_>) -> anyhow::Result { + if matches!(&plan, MigratePlan::Manual(_)) { + return Ok(Observation::MigrationRejected); + } + let db = self .db .as_ref() .ok_or_else(|| anyhow::anyhow!("database is not open"))?; let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); match update_database(db, &mut tx, AuthCtx::for_testing(), plan, &DstUpdateLogger) { - Ok(_) => anyhow::bail!("engine applied migration expected to fail precheck"), - Err(_) => Ok(Observation::MigrationRejected { - reason: MigrationRejection::PrecheckFailure, - }), + Ok(_) => anyhow::bail!("engine applied migration expected to be rejected"), + Err(_) => Ok(Observation::MigrationRejected), } } diff --git a/crates/dst/src/engine/migrations.rs b/crates/dst/src/engine/migrations.rs index 03a33b31dd4..931e7a37388 100644 --- a/crates/dst/src/engine/migrations.rs +++ b/crates/dst/src/engine/migrations.rs @@ -2,7 +2,7 @@ //! //! This module generates schema changes with an explicit expected outcome. //! Accepted migrations exercise engine auto-migration. Rejected migrations are -//! boundary probes where the engine should refuse the change for a known reason. +//! rule-local counterexamples where the engine should refuse the change. use super::model::{ColumnDomain, Model}; use crate::rng::{choice, choose_index, Choice, WeightedChoice}; @@ -36,15 +36,7 @@ pub struct Migration { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MigrationExpectation { Accepted, - Rejected(MigrationRejection), -} - -/// The specific boundary a rejected migration is expected to hit. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MigrationRejection { - ManualRequired, - PrecheckFailure, - InvalidDefinition, + Rejected, } /// A deterministic schema edit used while building a migration candidate. @@ -128,7 +120,6 @@ enum RuleId { DropPrimaryKeyAndWidenSum, WidenSumColumn, ReschemaEmptyEventTable, - AddI64DefaultMaxSequencePrecheckFailure, } #[derive(Clone, Copy)] @@ -138,8 +129,7 @@ struct RuleEntry { } struct RuleWeights { - accepted: &'static [(RuleId, u32)], - rejected: &'static [(RuleId, u32)], + rules: &'static [(RuleId, u32)], } static MIGRATION_RULES: &[RuleEntry] = &[ @@ -203,14 +193,10 @@ static MIGRATION_RULES: &[RuleEntry] = &[ id: RuleId::ReschemaEmptyEventTable, rule: &ReschemaEmptyEventTableRule, }, - RuleEntry { - id: RuleId::AddI64DefaultMaxSequencePrecheckFailure, - rule: &AddI64DefaultMaxSequencePrecheckFailureRule, - }, ]; static DEFAULT_RULE_WEIGHTS: RuleWeights = RuleWeights { - accepted: &[ + rules: &[ (RuleId::AddTable, 2), (RuleId::RemovePristineNonEventTable, 1), (RuleId::RemoveEmptyEventTable, 1), @@ -227,10 +213,6 @@ static DEFAULT_RULE_WEIGHTS: RuleWeights = RuleWeights { (RuleId::WidenSumColumn, 12), (RuleId::ReschemaEmptyEventTable, 8), ], - rejected: &[ - (RuleId::AddSequenceBoundaryProbe, 1), - (RuleId::AddI64DefaultMaxSequencePrecheckFailure, 1), - ], }; fn migration_rules() -> &'static [RuleEntry] { @@ -273,14 +255,30 @@ impl AcceptedCase { } } -/// A negative case emitted by a migration rule. -/// -/// These are not "anything that failed an accepted rule." They are known -/// single-boundary violations with a precise expected oracle result. +/// A rule-local counterexample emitted by a migration rule. #[derive(Debug, Clone, PartialEq, Eq)] struct RejectedCase { + rule: RuleId, rewrite: SchemaRewrite, - expected: MigrationRejection, + writes: Vec, + protects: Vec, +} + +impl RejectedCase { + fn new(rule: RuleId, rewrite: SchemaRewrite) -> Self { + let writes = write_resources(&rewrite); + Self { + rule, + rewrite, + writes, + protects: Vec::new(), + } + } + + fn protecting(mut self, protects: impl Into>) -> Self { + self.protects = protects.into(); + self + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -298,7 +296,7 @@ trait MigrationRule: Sync { Vec::new() } - fn rejected_cases(&self, _rng: &Rng, _ctx: RuleCtx<'_>) -> Vec { + fn rejected_cases(&self, _rng: &Rng, _ctx: RuleCtx<'_>, _rule: RuleId) -> Vec { Vec::new() } @@ -331,24 +329,11 @@ impl Migration { }) } - fn rejected(target_schema: SchemaPlan, rejection: MigrationRejection) -> Self { + fn rejected(target_schema: SchemaPlan) -> Self { Self { schema: target_schema.clone(), target_schema, - expectation: MigrationExpectation::Rejected(rejection), - } - } - - fn from_rewrite_with_expectation( - base: &SchemaPlan, - rewrite: SchemaRewrite, - expectation: MigrationExpectation, - ) -> anyhow::Result { - let mut schema = base.clone(); - rewrite.apply_to(&mut schema)?; - match expectation { - MigrationExpectation::Accepted => Self::accepted(schema), - MigrationExpectation::Rejected(rejection) => Ok(Self::rejected(schema, rejection)), + expectation: MigrationExpectation::Rejected, } } @@ -415,24 +400,51 @@ fn build_validated_accepted_batch(ctx: RuleCtx<'_>, cases: Vec) -> let final_schema = canonical_schema(&draft)?; - for case in &cases { - let rule = rule_by_id(case.rule) - .ok_or_else(|| anyhow::anyhow!("accepted migration case references missing rule {:?}", case.rule))?; - rule.rule.validate_accepted(case, ctx, &final_schema)?; - } + validate_accepted_cases(ctx, &cases, &final_schema)?; anyhow::ensure!(final_schema != *ctx.schema, "accepted migration batch was no-op"); Migration::accepted(draft) } +fn validate_accepted_cases(ctx: RuleCtx<'_>, cases: &[AcceptedCase], final_schema: &SchemaPlan) -> anyhow::Result<()> { + for case in cases { + let rule = rule_by_id(case.rule) + .ok_or_else(|| anyhow::anyhow!("accepted migration case references missing rule {:?}", case.rule))?; + rule.rule.validate_accepted(case, ctx, final_schema)?; + } + Ok(()) +} + fn build_rejected_migration(rng: &Rng, ctx: RuleCtx<'_>) -> Option { + for _ in 0..16 { + let rejected = pick_rejected_case(rng, ctx)?; + let accepted = pick_accepted_cases_around_rejected(rng, ctx, &rejected); + if let Ok(migration) = build_validated_rejected_batch(ctx, accepted, rejected) { + return Some(migration); + } + } + let rejected = pick_rejected_case(rng, ctx)?; - Migration::from_rewrite_with_expectation( - ctx.schema, - rejected.rewrite, - MigrationExpectation::Rejected(rejected.expected), - ) - .ok() + build_validated_rejected_batch(ctx, Vec::new(), rejected).ok() +} + +fn build_validated_rejected_batch( + ctx: RuleCtx<'_>, + accepted: Vec, + rejected: RejectedCase, +) -> anyhow::Result { + validate_no_resource_conflicts(&accepted)?; + validate_rejected_case_isolated(&accepted, &rejected)?; + + let mut draft = ctx.schema.clone(); + for case in &accepted { + case.rewrite.apply_to(&mut draft)?; + } + rejected.rewrite.apply_to(&mut draft)?; + + validate_accepted_cases(ctx, &accepted, &draft)?; + anyhow::ensure!(draft != *ctx.schema, "rejected migration batch was no-op"); + Ok(Migration::rejected(draft)) } fn rule_by_id(id: RuleId) -> Option<&'static RuleEntry> { @@ -450,26 +462,50 @@ fn pick_accepted_cases(rng: &Rng, ctx: RuleCtx<'_>, max_rules: usize) -> Option< (!cases.is_empty()).then_some(cases) } +fn pick_accepted_cases_around_rejected(rng: &Rng, ctx: RuleCtx<'_>, rejected: &RejectedCase) -> Vec { + let mut cases = Vec::new(); + + for &(id, weight) in DEFAULT_RULE_WEIGHTS.rules { + if weight == 0 || id == rejected.rule { + continue; + } + let Some(entry) = rule_by_id(id) else { + continue; + }; + let Some(case) = pick_candidate(rng, entry.rule.accepted_cases(rng, ctx, entry.id)) else { + continue; + }; + + let mut trial = cases.clone(); + trial.push(case.clone()); + if validate_no_resource_conflicts(&trial).is_ok() && validate_rejected_case_isolated(&trial, rejected).is_ok() { + cases.push(case); + } + } + + cases +} + fn pick_accepted_case(rng: &Rng, ctx: RuleCtx<'_>, weights: &RuleWeights) -> Option { for _ in 0..16 { - let entry = pick_weighted_rule(rng, weights.accepted)?; + let entry = pick_weighted_rule(rng, weights.rules)?; if let Some(case) = pick_candidate(rng, entry.rule.accepted_cases(rng, ctx, entry.id)) { return Some(case); } } - pick_candidate(rng, accepted_cases_for_weights(rng, ctx, weights.accepted)) + pick_candidate(rng, accepted_cases_for_weights(rng, ctx, weights.rules)) } fn pick_rejected_case(rng: &Rng, ctx: RuleCtx<'_>) -> Option { for _ in 0..16 { - let entry = pick_weighted_rule(rng, DEFAULT_RULE_WEIGHTS.rejected)?; - if let Some(case) = pick_candidate(rng, entry.rule.rejected_cases(rng, ctx)) { + let entry = pick_weighted_rule(rng, DEFAULT_RULE_WEIGHTS.rules)?; + if let Some(case) = pick_candidate(rng, entry.rule.rejected_cases(rng, ctx, entry.id)) { return Some(case); } } - pick_candidate(rng, rejected_cases_for_weights(rng, ctx, DEFAULT_RULE_WEIGHTS.rejected)) + pick_candidate(rng, rejected_cases_for_weights(rng, ctx, DEFAULT_RULE_WEIGHTS.rules)) } fn pick_weighted_rule(rng: &Rng, weights: &[(RuleId, u32)]) -> Option<&'static RuleEntry> { @@ -503,7 +539,7 @@ fn rejected_cases_for_weights(rng: &Rng, ctx: RuleCtx<'_>, weights: &[(RuleId, u .iter() .filter(|(_, weight)| *weight > 0) .filter_map(|(id, _)| rule_by_id(*id)) - .flat_map(|entry| entry.rule.rejected_cases(rng, ctx)) + .flat_map(|entry| entry.rule.rejected_cases(rng, ctx, entry.id)) .collect() } @@ -512,6 +548,16 @@ fn pick_candidate(rng: &Rng, mut candidates: Vec) -> Option { Some(candidates.swap_remove(idx)) } +fn validate_rejected_case_isolated(accepted: &[AcceptedCase], rejected: &RejectedCase) -> anyhow::Result<()> { + for case in accepted { + validate_resource_set_is_disjoint(&case.writes, &rejected.writes)?; + validate_resource_set_is_disjoint(&case.writes, &rejected.protects)?; + validate_resource_set_is_disjoint(&case.protects, &rejected.writes)?; + validate_resource_set_is_disjoint(&case.protects, &rejected.protects)?; + } + Ok(()) +} + fn validate_no_resource_conflicts(cases: &[AcceptedCase]) -> anyhow::Result<()> { for (left_idx, left) in cases.iter().enumerate() { for right in &cases[left_idx + 1..] { @@ -656,6 +702,18 @@ fn alter_table_sequence(case: &AcceptedCase) -> anyhow::Result<(&str, &SequenceP Ok((table, sequence)) } +fn rejected_sequence_case(rule: RuleId, rewrite: SchemaRewrite) -> RejectedCase { + let protects = match &rewrite { + SchemaRewrite::AlterTable { table, ops } + if ops.iter().any(|op| matches!(op, TableMigrationOp::AddSequence { .. })) => + { + vec![Resource::Table(table.clone())] + } + _ => Vec::new(), + }; + RejectedCase::new(rule, rewrite).protecting(protects) +} + struct AddTableRule; impl MigrationRule for AddTableRule { @@ -847,27 +905,23 @@ impl MigrationRule for AddSequenceBoundaryProbeRule { addable_sequence_boundary_probes(table, |column| column_domain(ctx.model, table, column)) .into_iter() .map(move |sequence| { - let column = sequence.column; AcceptedCase::new( rule, SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }]), ) - .protecting(vec![Resource::Column { - table: table.name.clone(), - column, - }]) + .protecting(vec![Resource::Table(table.name.clone())]) }) }) .collect() } - fn rejected_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>) -> Vec { + fn rejected_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>, rule: RuleId) -> Vec { add_sequence_precheck_failure_rewrites(ctx.schema, ctx.model) .into_iter() - .map(|rewrite| RejectedCase { - rewrite, - expected: MigrationRejection::PrecheckFailure, - }) + .chain(add_i64_explicit_max_sequence_precheck_failure_rewrites( + ctx.schema, ctx.model, + )) + .map(|rewrite| rejected_sequence_case(rule, rewrite)) .collect() } @@ -1050,20 +1104,6 @@ impl MigrationRule for ReschemaEmptyEventTableRule { } } -struct AddI64DefaultMaxSequencePrecheckFailureRule; - -impl MigrationRule for AddI64DefaultMaxSequencePrecheckFailureRule { - fn rejected_cases(&self, _rng: &Rng, ctx: RuleCtx<'_>) -> Vec { - add_i64_default_max_sequence_precheck_failure_rewrites(ctx.schema, ctx.model) - .into_iter() - .map(|rewrite| RejectedCase { - rewrite, - expected: MigrationRejection::PrecheckFailure, - }) - .collect() - } -} - fn add_table_rewrites(rng: &Rng, schema: &SchemaPlan) -> Vec { if schema.tables.len() >= MAX_TABLES { return Vec::new(); @@ -1169,13 +1209,13 @@ fn add_sequence_precheck_failure_rewrites(schema: &SchemaPlan, model: &Model) -> .collect() } -fn add_i64_default_max_sequence_precheck_failure_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { +fn add_i64_explicit_max_sequence_precheck_failure_rewrites(schema: &SchemaPlan, model: &Model) -> Vec { schema .tables .iter() .filter(|table| !table.is_event && model.row_count_by_table_name(&table.name) > 0) .flat_map(|table| { - i64_default_max_sequence_precheck_failure_probes(table, |column| column_domain(model, table, column)) + i64_explicit_max_sequence_precheck_failure_probes(table, |column| column_domain(model, table, column)) .into_iter() .map(|sequence| SchemaRewrite::alter_table(table, [TableMigrationOp::AddSequence { sequence }])) }) @@ -1520,7 +1560,7 @@ fn addable_sequence_boundary_probes( .collect() } -fn i64_default_max_sequence_precheck_failure_probes( +fn i64_explicit_max_sequence_precheck_failure_probes( table: &TablePlan, column_domain: impl Fn(usize) -> Option, ) -> Vec { @@ -1540,7 +1580,7 @@ fn i64_default_max_sequence_precheck_failure_probes( return None; } - SequencePlan::new(column, column_plan.ty) + SequencePlan::with_bounds(column, column_plan.ty, 1, 1, i64::MAX as i128, 1) }) .collect() } @@ -1694,11 +1734,7 @@ mod tests { assert!(seen.insert(entry.id), "duplicate registered RuleId: {:?}", entry.id); } - for &(id, weight) in DEFAULT_RULE_WEIGHTS - .accepted - .iter() - .chain(DEFAULT_RULE_WEIGHTS.rejected.iter()) - { + for &(id, weight) in DEFAULT_RULE_WEIGHTS.rules { assert!(weight > 0, "default rule weights should not contain zero entries"); assert!( rule_by_id(id).is_some(), @@ -1737,7 +1773,7 @@ mod tests { } #[test] - fn add_sequence_boundary_rule_emits_explicit_precheck_rejections() { + fn add_sequence_boundary_rule_emits_rejected_counterexamples() { let schema = indexed_unique_i64_schema(); let mut model = Model::new(schema.clone()); model.apply(&Interaction::BeginMutTx); @@ -1757,25 +1793,66 @@ mod tests { schema: &schema, model: &model, }, + RuleId::AddSequenceBoundaryProbe, ); - assert!(!cases.is_empty(), "expected at least one precheck-failure case"); + assert!(!cases.is_empty(), "expected at least one rejected counterexample"); for case in cases { - assert_eq!(case.expected, MigrationRejection::PrecheckFailure); + assert_eq!(case.rule, RuleId::AddSequenceBoundaryProbe); let SchemaRewrite::AlterTable { ops, .. } = &case.rewrite else { - panic!("precheck rejection should alter one table"); + panic!("sequence rejection should alter one table"); }; assert_eq!(ops.len(), 1); assert!(matches!(ops[0], TableMigrationOp::AddSequence { .. })); - Migration::from_rewrite_with_expectation( - &schema, - case.rewrite, - MigrationExpectation::Rejected(MigrationRejection::PrecheckFailure), - ) - .expect("rejected case should still be structurally valid"); + let mut target = schema.clone(); + case.rewrite + .apply_to(&mut target) + .expect("rejected case should still be structurally valid"); + assert_eq!( + Migration::rejected(target).expectation(), + MigrationExpectation::Rejected + ); } } + #[test] + fn i64_explicit_max_sequence_rejection_uses_typed_bounds() { + let schema = SchemaPlan { + tables: vec![i64_table("items", false)], + }; + let mut model = Model::new(schema.clone()); + model.apply(&Interaction::BeginMutTx); + model.apply(&Interaction::Insert { + table: 0, + row: product![4i64], + }); + model.apply(&Interaction::CommitTx); + + let cases = AddSequenceBoundaryProbeRule.rejected_cases( + &Rng::new(0), + RuleCtx { + schema: &schema, + model: &model, + }, + RuleId::AddSequenceBoundaryProbe, + ); + + assert_eq!(cases.len(), 1); + let SchemaRewrite::AlterTable { ops, .. } = &cases[0].rewrite else { + panic!("i64 precheck rejection should alter one table"); + }; + let TableMigrationOp::AddSequence { sequence } = &ops[0] else { + panic!("i64 precheck rejection should add a sequence"); + }; + assert_eq!(sequence.start, Some(1)); + assert_eq!(sequence.min_value, Some(1)); + assert_eq!(sequence.max_value, Some(i64::MAX as i128)); + assert!(!added_sequence_precheck_range_is_clear( + &model.column_domain(0, sequence.column), + sequence + )); + } + #[test] fn accepted_batch_rejects_removing_every_non_event_table() { let schema = SchemaPlan { diff --git a/crates/dst/src/engine/model.rs b/crates/dst/src/engine/model.rs index 1f574c3dbf7..0753c20a7eb 100644 --- a/crates/dst/src/engine/model.rs +++ b/crates/dst/src/engine/model.rs @@ -340,7 +340,7 @@ impl Model { self.sequences = remap_sequence_states(&old_schema, &self.schema, old_sequences); Observation::Migrated } - MigrationExpectation::Rejected(reason) => Observation::MigrationRejected { reason }, + MigrationExpectation::Rejected => Observation::MigrationRejected, } } Interaction::Replay => { diff --git a/crates/dst/src/engine/workload.rs b/crates/dst/src/engine/workload.rs index e37418bbdea..7ab9412b894 100644 --- a/crates/dst/src/engine/workload.rs +++ b/crates/dst/src/engine/workload.rs @@ -3,7 +3,7 @@ use std::fmt::{Debug, Error as FmtError, Formatter}; use super::generation::{GenCtx, GenerationState}; -use super::migrations::{Migration, MigrationRejection}; +use super::migrations::Migration; use super::model::Model; use super::row::Row; use super::state::{CommitDelta, CountState}; @@ -59,7 +59,7 @@ pub enum Observation { Deleted, Committed { delta: CommitDelta }, Migrated, - MigrationRejected { reason: MigrationRejection }, + MigrationRejected, Replayed { state: CountState }, }