From e15fc19bd6959442cf1ee97ba5e374b16167b3ff Mon Sep 17 00:00:00 2001 From: AppzDev Date: Fri, 16 Oct 2026 22:04:00 +0530 Subject: [PATCH 1/2] feat(schedule): durable workflow scheduling, atomic start-once, daemon ticker (item #369 PR-B) Schedules/schedule_runs/workflow_start_receipts migration; SQLite-backed start_workflow_once_with_params + create_start_once under BEGIN IMMEDIATE; generic terminal-outcome callback; cancel_if_orphaned with disposition; src/schedule store/tick/tests; spawn_schedule_ticker booted beside the binary staleness watchdog. Spec artifact MICPSjcR3V83J2v_AXHkm rev 4. Agentflare-Agent: opencode Agentflare-Branch: task/369-pr-b-durable-workflow-scheduling-migrati Agentflare-Item: 369 --- crates/flare-workflow/src/engine.rs | 13 +- crates/flare-workflow/src/engine_control.rs | 2 + .../flare-workflow/src/engine_start_once.rs | 195 ++++ crates/flare-workflow/src/json.rs | 21 +- crates/flare-workflow/src/lib.rs | 8 +- .../flare-workflow/src/sqlite_start_once.rs | 178 ++++ crates/flare-workflow/src/sqlite_store.rs | 78 +- .../src/tests_schedule_start.rs | 600 ++++++++++++ crates/flare-workflow/src/types.rs | 22 + src/dashboard/dispatch_boot.rs | 76 ++ src/dashboard/mod.rs | 2 +- src/dashboard/server.rs | 3 +- src/main.rs | 1 + src/schedule/mod.rs | 384 ++++++++ src/schedule/store.rs | 884 ++++++++++++++++++ src/schedule/tests.rs | 741 +++++++++++++++ src/schedule/tests_lifecycle.rs | 601 ++++++++++++ src/schedule/tests_recovery.rs | 568 +++++++++++ src/schedule/tick.rs | 428 +++++++++ src/workflow.rs | 38 +- 20 files changed, 4824 insertions(+), 19 deletions(-) create mode 100644 crates/flare-workflow/src/engine_start_once.rs create mode 100644 crates/flare-workflow/src/sqlite_start_once.rs create mode 100644 crates/flare-workflow/src/tests_schedule_start.rs create mode 100644 src/schedule/mod.rs create mode 100644 src/schedule/store.rs create mode 100644 src/schedule/tests.rs create mode 100644 src/schedule/tests_lifecycle.rs create mode 100644 src/schedule/tests_recovery.rs create mode 100644 src/schedule/tick.rs diff --git a/crates/flare-workflow/src/engine.rs b/crates/flare-workflow/src/engine.rs index de2686d06..d81cdf59c 100644 --- a/crates/flare-workflow/src/engine.rs +++ b/crates/flare-workflow/src/engine.rs @@ -32,6 +32,9 @@ use crate::waits::WakeAt; mod control; #[path = "engine_lease.rs"] mod lease; +#[path = "engine_start_once.rs"] +mod start_once; +pub use start_once::{OrphanCancelDisposition, TerminalCallback}; pub use lease::DEFAULT_LEASE_TTL; use lease::process_executor_id; @@ -223,6 +226,8 @@ pub struct WorkflowEngine = InMemoryStore> superseded: RunIdSet, /// See [`patch_run_data`](Self::patch_run_data). data_patches: DataPatches, + /// Eager terminal-outcome hook (see `TerminalCallback`); `None` by default. + terminal_callback: Arc>>, } impl WorkflowEngine> { @@ -248,6 +253,7 @@ impl + 'static> WorkflowEngine { driving: Arc::new(parking_lot::Mutex::new(HashSet::new())), superseded: Arc::new(parking_lot::Mutex::new(HashSet::new())), data_patches: Arc::new(parking_lot::Mutex::new(HashMap::new())), + terminal_callback: Arc::new(parking_lot::Mutex::new(None)), } } @@ -1067,6 +1073,8 @@ impl + 'static> WorkflowEngine { self.event_bus .publish(WorkflowEvent::WorkflowCompleted { run_id, duration }) .await; + self.notify_terminal(run_id, WorkflowStatus::Completed, None) + .await; } } @@ -1125,9 +1133,11 @@ impl + 'static> WorkflowEngine { .publish(WorkflowEvent::WorkflowFailed { run_id, failed_step: failed_step.unwrap_or_else(|| StepId::new("internal_scheduler")), - error: error_message, + error: error_message.clone(), }) .await; + self.notify_terminal(run_id, WorkflowStatus::Failed, Some(error_message)) + .await; Ok(()) } @@ -1459,6 +1469,7 @@ impl + 'static> WorkflowEngine { driving: Arc::clone(&self.driving), superseded: Arc::clone(&self.superseded), data_patches: Arc::clone(&self.data_patches), + terminal_callback: Arc::clone(&self.terminal_callback), } } } diff --git a/crates/flare-workflow/src/engine_control.rs b/crates/flare-workflow/src/engine_control.rs index 833e600f1..a42cefff6 100644 --- a/crates/flare-workflow/src/engine_control.rs +++ b/crates/flare-workflow/src/engine_control.rs @@ -122,6 +122,8 @@ impl + 'static> WorkflowEngine { self.event_bus .publish(WorkflowEvent::WorkflowCancelled { run_id }) .await; + self.notify_terminal(run_id, WorkflowStatus::Cancelled, None) + .await; } Ok(()) } diff --git a/crates/flare-workflow/src/engine_start_once.rs b/crates/flare-workflow/src/engine_start_once.rs new file mode 100644 index 000000000..61e783146 --- /dev/null +++ b/crates/flare-workflow/src/engine_start_once.rs @@ -0,0 +1,195 @@ +//! Idempotent start-once, conditional orphan cancellation, and the generic +//! terminal-outcome callback for [`WorkflowEngine`](super::WorkflowEngine). +//! SQLite-specialized (durable receipts need SQL transactions); unrelated +//! `StateStore` implementations keep the existing non-idempotent start API. +//! Child module of `engine` (split out for size) so it can reach the +//! engine's private fields. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; + +use super::{JournaledInput, StartGuard, WorkflowEngine}; +use crate::events::WorkflowEvent; +use crate::sqlite_store::SqliteStore; +use crate::store::StateStore; +use crate::types::*; + +/// Callback invoked once per terminal driver settlement (`Completed`, +/// `Failed`, or an actual `Cancelled` transition) with the run's final +/// status and error. Registered once per engine; `flare-workflow` knows +/// nothing about the caller's tables — the scheduler's callback writes its +/// own `schedule_runs` row. Must never fail the workflow: keep it to a short +/// fallible write that logs internally, since the tick fallback converges +/// anything it drops. +pub type TerminalCallback = Arc< + dyn Fn( + WorkflowRunId, + WorkflowStatus, + Option, + ) -> Pin + Send>> + + Send + + Sync, +>; + +/// Disposition of a conditional orphan-cancellation check. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OrphanCancelDisposition { + /// Stale/missing lease on a live-state run: now `Cancelled`. + Interrupted, + /// Paused run or fresh lease: untouched. + Unchanged, + /// Already terminal: the real outcome (persisted by the caller). + Terminal(WorkflowStatus), + /// No such workflow row. + Missing, +} + +impl + 'static> WorkflowEngine { + /// Register the terminal-outcome callback (at most one; replaces any + /// previous). Fires on every terminal driver settlement from here on. + pub fn on_terminal(&self, cb: TerminalCallback) { + *self.terminal_callback.lock() = Some(cb); + } + + /// Invoke the registered terminal callback, if any. Awaited on the + /// driver's task right after the settle commits, so an eager write lands + /// before any later eviction can remove the workflow row. + pub(crate) async fn notify_terminal( + &self, + run_id: WorkflowRunId, + status: WorkflowStatus, + error: Option, + ) { + let cb = self.terminal_callback.lock().clone(); + if let Some(cb) = cb { + cb(run_id, status, error).await; + } + } +} + +impl WorkflowEngine> { + /// Idempotent start backed by a durable receipt: the receipt, the run's + /// initial state, and its input journal entry commit in one SQLite + /// transaction, and only the caller that created the receipt launches + /// the driver. A repeated start for `start_key` returns the existing run + /// ID — even after workflow-history eviction — without spawning, + /// resuming, or rebuilding anything. + #[must_use = "run ID should be stored or awaited"] + pub async fn start_workflow_once_with_params( + &self, + start_key: &str, + definition_id: WorkflowId, + data: D, + input: String, + params: serde_json::Value, + ) -> WorkflowResult { + // Fast path first: an existing receipt wins over everything, + // including a missing workflow row or definition. + if let Some(run_id) = self.state_store.lookup_start(start_key).await? { + return Ok(StartOnceResult::Existing(run_id)); + } + + let guard = StartGuard::new(self); + if self.is_shutting_down() { + return Err(WorkflowError::ShuttingDown); + } + + let definition = self + .definitions + .read() + .get(&definition_id) + .cloned() + .ok_or_else(|| WorkflowError::DefinitionNotFound(definition_id.clone()))?; + + let run_id = WorkflowRunId::new(); + let mut state = WorkflowState::new(run_id, definition_id.clone(), data); + state.status = WorkflowStatus::Running; + state.lease = Some(self.new_lease()); + state.input = input.clone(); + state.context.params = params.clone(); + state.step_states.reserve(definition.steps.len()); + for step in &definition.steps { + state + .step_states + .insert(step.id.clone(), StepState::default()); + } + let driving = self + .claim_driving(run_id) + .ok_or(WorkflowError::InvalidStateTransition { + from: WorkflowStatus::Running, + to: WorkflowStatus::Running, + })?; + let journaled_input = JournaledInput { input, params }; + let value = serde_json::to_vec(&journaled_input) + .map_err(|e| WorkflowError::Journal(format!("serialize input: {e}")))?; + + match self + .state_store + .create_start_once(start_key.to_string(), state, JournalEntry::Input { value }) + .await? + { + // Lost the race (or the key predates us): someone else committed + // first. Drop the candidate's guards; never launch. + StartOnceResult::Existing(run_id) => Ok(StartOnceResult::Existing(run_id)), + StartOnceResult::Created(run_id) => { + // Post-commit maintenance must not turn this committed start + // into an apparent failure: warn and carry on. + if let Err(e) = self.evict_old_runs_if_needed().await { + tracing::warn!(run_id = %run_id, error = ?e, "start-once: post-commit eviction failed; run is still committed"); + } + self.event_bus + .publish(WorkflowEvent::WorkflowStarted { + run_id, + definition_id, + }) + .await; + guard.commit(); + self.spawn_driven( + run_id, + Arc::clone(&definition), + self.active_workflow_guard(), + driving, + ); + Ok(StartOnceResult::Created(run_id)) + } + } + } + + /// Conditionally cancel `run_id` iff it is an eligible orphan (live-state + /// run whose executor lease is stale or missing at `now`): rechecks + /// status and lease atomically under the store's mutation protection, + /// persists the disposition consistently, and publishes a cancellation + /// event plus the terminal callback only when state actually changed. + /// Never use unconditional `cancel_workflow` here and assume + /// interruption — a paused run or a fresh lease must survive. + pub async fn cancel_if_orphaned( + &self, + run_id: WorkflowRunId, + now: DateTime, + ) -> WorkflowResult { + use crate::sqlite_store::OrphanCancelOutcome; + let outcome = self + .state_store + .cancel_orphaned_run(run_id, now, self.lease_ttl()) + .await?; + Ok(match outcome { + // No terminal-callback notification here: the caller initiated + // this transition synchronously and maps the disposition itself + // (the scheduler records `interrupted`, not `cancelled`). Firing + // the eager hook as well would race that mapping and mislabel + // the occurrence. Driver-driven settlements still notify. + OrphanCancelOutcome::Cancelled => { + self.event_bus + .publish(WorkflowEvent::WorkflowCancelled { run_id }) + .await; + OrphanCancelDisposition::Interrupted + } + OrphanCancelOutcome::Unchanged => OrphanCancelDisposition::Unchanged, + OrphanCancelOutcome::Terminal(status) => OrphanCancelDisposition::Terminal(status), + OrphanCancelOutcome::Missing => OrphanCancelDisposition::Missing, + }) + } +} diff --git a/crates/flare-workflow/src/json.rs b/crates/flare-workflow/src/json.rs index 844d19d29..9833bd0c0 100644 --- a/crates/flare-workflow/src/json.rs +++ b/crates/flare-workflow/src/json.rs @@ -204,6 +204,18 @@ pub enum JsonErrorMode { pub fn compile_workflow( json: &JsonWorkflow, send: SendMessage, +) -> Result, String> { + compile_workflow_in(json, send, None) +} + +/// Like [`compile_workflow`], but every agent and command step runs in `cwd` +/// when set: agent steps carry it in `StepInvocation::cwd` for the send hook, +/// command steps spawn with it as their process working directory. `None` +/// preserves the ambient-cwd behavior. +pub fn compile_workflow_in( + json: &JsonWorkflow, + send: SendMessage, + cwd: Option, ) -> Result, String> { let mut wf = WorkflowDefinition::new(json.name.clone(), json.name.clone()); @@ -250,6 +262,7 @@ pub fn compile_workflow( } Arc::new(CommandExecutor { command: command.clone(), + cwd: cwd.clone(), }) } (None, Some(agent)) => Arc::new(PromptExecutor { @@ -260,6 +273,7 @@ pub fn compile_workflow( args: s.args.clone(), hard_cap_secs: s.hard_cap_secs, idle_timeout_secs: s.idle_timeout_secs, + cwd: cwd.clone(), persona: s.persona.clone(), }), }; @@ -373,6 +387,7 @@ struct PromptExecutor { args: Vec, hard_cap_secs: Option, idle_timeout_secs: Option, + cwd: Option, persona: Option, } @@ -387,7 +402,7 @@ impl StepExecutor for PromptExecutor { args: self.args.clone(), hard_cap_secs: self.hard_cap_secs, idle_timeout_secs: self.idle_timeout_secs, - cwd: None, + cwd: self.cwd.clone(), owner: None, persona: self.persona.clone(), }; @@ -416,6 +431,7 @@ impl StepExecutor for PromptExecutor { /// fails the step with stderr attached. struct CommandExecutor { command: Vec, + cwd: Option, } #[async_trait] @@ -439,6 +455,9 @@ impl StepExecutor for CommandExecutor { // completion in the background. let mut std_cmd = flare_process::command(&program); std_cmd.args(&args); + if let Some(cwd) = &self.cwd { + std_cmd.current_dir(cwd); + } let output = tokio::process::Command::from(std_cmd) .stdin(std::process::Stdio::null()) .kill_on_drop(true) diff --git a/crates/flare-workflow/src/lib.rs b/crates/flare-workflow/src/lib.rs index cb50e4795..1013f6168 100644 --- a/crates/flare-workflow/src/lib.rs +++ b/crates/flare-workflow/src/lib.rs @@ -28,15 +28,19 @@ pub mod variables; pub mod waits; pub use definition::{StepCondition, StepDefinition, ValidationError, WorkflowDefinition}; -pub use engine::WorkflowEngine; +pub use engine::{OrphanCancelDisposition, TerminalCallback, WorkflowEngine}; pub use events::{EventBus, EventSubscriber, LoggingSubscriber, WorkflowEvent}; pub use executor::{FunctionStep, StepExecutor, noop_executor}; pub use json::{ JsonErrorMode, JsonMode, JsonStep, JsonWorkflow, PipelineData, SendMessage, StepInvocation, - compile_workflow, + compile_workflow, compile_workflow_in, }; pub use retry::{Backoff, Retryable, apply_jitter}; pub use sqlite_store::{SqliteStore, SqliteStoreError}; pub use store::{InMemoryStore, StateStore, smoke_test}; pub use types::*; pub use variables::{capture_output, expand_variables}; + +#[cfg(test)] +#[path = "tests_schedule_start.rs"] +mod tests_schedule_start; diff --git a/crates/flare-workflow/src/sqlite_start_once.rs b/crates/flare-workflow/src/sqlite_start_once.rs new file mode 100644 index 000000000..e160550fb --- /dev/null +++ b/crates/flare-workflow/src/sqlite_start_once.rs @@ -0,0 +1,178 @@ +//! Atomic start-once persistence for [`SqliteStore`](super::SqliteStore): +//! durable start receipts committed in one `BEGIN IMMEDIATE` transaction +//! with the run's initial state and input journal entry, plus the atomic +//! read/check/write behind conditional orphan cancellation. Child module of +//! `sqlite_store` (split out for size) so it can reuse the transaction- +//! neutral row writer and the store's locks. + +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use rusqlite::{OptionalExtension, TransactionBehavior, params}; + +use super::{SqliteStore, blocking}; +use crate::journal; +use crate::types::{ + JournalEntry, StartOnceResult, WorkflowData, WorkflowError, WorkflowResult, WorkflowRunId, + WorkflowState, WorkflowStatus, +}; + +/// Store-level outcome of a conditional orphan-cancellation check. The engine +/// maps this to the public [`OrphanCancelDisposition`](crate::OrphanCancelDisposition). +pub(crate) enum OrphanCancelOutcome { + Cancelled, + Unchanged, + Terminal(WorkflowStatus), + Missing, +} + +impl SqliteStore { + /// Look up the durable start receipt for `start_key`. + pub async fn lookup_start(&self, start_key: &str) -> WorkflowResult> { + let conn = Arc::clone(&self.conn); + let key = start_key.to_string(); + blocking(move || { + let conn = conn + .lock() + .map_err(|e| WorkflowError::Store(format!("lock: {e}")))?; + conn.query_row( + "SELECT run_id FROM workflow_start_receipts WHERE start_key = ?1", + params![key], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|e| WorkflowError::Store(format!("lookup_start: {e}")))? + .map(|s| { + s.parse::() + .map_err(|e| WorkflowError::Store(format!("bad receipt run_id: {e}"))) + }) + .transpose() + }) + .await + } + + /// Atomically commit a start receipt, the run's initial state (+ child + /// rows), and its initial input journal entry — or return the pre-existing + /// receipt's run when this key already started. Runs under the store's + /// `update_lock` (same serialization as every `update()` writer) and a + /// SQLite `BEGIN IMMEDIATE` transaction: any pre-commit error rolls back + /// all three writes together, and a concurrent claimant for the same key + /// rechecks under the transaction and gets `Existing`. + pub async fn create_start_once( + &self, + start_key: String, + state: WorkflowState, + input_entry: JournalEntry, + ) -> WorkflowResult { + // Serialize with the other load-mutate-write paths (lease renewals, + // driver settles) so none of them can interleave this commit. + let _guard = self.update_lock.lock().await; + let conn = Arc::clone(&self.conn); + let run_id = state.run_id; + blocking(move || { + let mut conn = conn + .lock() + .map_err(|e| WorkflowError::Store(format!("lock: {e}")))?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| WorkflowError::Store(format!("begin start-once tx: {e}")))?; + // Recheck under the transaction: a concurrent start may have + // committed while this caller prepared its candidate state. + let existing: Option = tx + .query_row( + "SELECT run_id FROM workflow_start_receipts WHERE start_key = ?1", + params![start_key], + |row| row.get(0), + ) + .optional() + .map_err(|e| WorkflowError::Store(format!("recheck receipt: {e}")))?; + if let Some(s) = existing { + let id = s + .parse::() + .map_err(|e| WorkflowError::Store(format!("bad receipt run_id: {e}")))?; + // Roll back (nothing written yet) and report the winner. + tx.rollback() + .map_err(|e| WorkflowError::Store(format!("rollback: {e}")))?; + return Ok(StartOnceResult::Existing(id)); + } + let now = Utc::now().to_rfc3339(); + tx.execute( + "INSERT INTO workflow_start_receipts (start_key, run_id, created_at) + VALUES (?1, ?2, ?3)", + params![start_key, run_id.to_string(), now], + ) + .map_err(|e| WorkflowError::Store(format!("insert receipt: {e}")))?; + Self::write_state_rows(&tx, &state)?; + journal::append(&tx, &run_id.to_string(), &input_entry) + .map_err(|e| WorkflowError::Store(format!("start-once journal: {e}")))?; + tx.commit() + .map_err(|e| WorkflowError::Store(format!("commit start-once: {e}")))?; + Ok(StartOnceResult::Created(run_id)) + }) + .await + } + + /// Conditional orphan cancellation's atomic read/check/write: recheck the + /// run's live status and lease under the store's mutation protection + /// (never the caller's possibly-stale scan snapshot), and only then + /// decide. Paused runs and fresh leases are never touched; already- + /// terminal runs report their real outcome; a missing row reports + /// `Missing`; anything else becomes `Cancelled` with its lease cleared. + pub(crate) async fn cancel_orphaned_run( + &self, + run_id: WorkflowRunId, + now: DateTime, + lease_ttl: std::time::Duration, + ) -> WorkflowResult { + let _guard = self.update_lock.lock().await; + let conn = Arc::clone(&self.conn); + blocking(move || { + let mut conn = conn + .lock() + .map_err(|e| WorkflowError::Store(format!("lock: {e}")))?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| WorkflowError::Store(format!("begin orphan-cancel tx: {e}")))?; + let json: Option = tx + .query_row( + "SELECT state_json FROM workflow_runs WHERE id = ?1", + params![run_id.to_string()], + |row| row.get(0), + ) + .optional() + .map_err(|e| WorkflowError::Store(format!("orphan-cancel load: {e}")))?; + let Some(json) = json else { + tx.rollback() + .map_err(|e| WorkflowError::Store(format!("rollback: {e}")))?; + return Ok(OrphanCancelOutcome::Missing); + }; + let mut state = Self::deserialize(&json)?; + if state.status.is_terminal() { + tx.rollback() + .map_err(|e| WorkflowError::Store(format!("rollback: {e}")))?; + return Ok(OrphanCancelOutcome::Terminal(state.status)); + } + let lease_fresh = state + .lease + .as_ref() + .is_some_and(|l| l.is_fresh(now, lease_ttl)); + if state.status == WorkflowStatus::Paused || lease_fresh { + tx.rollback() + .map_err(|e| WorkflowError::Store(format!("rollback: {e}")))?; + return Ok(OrphanCancelOutcome::Unchanged); + } + state.status = WorkflowStatus::Cancelled; + state.lease = None; + state.updated_at = now; + state.error = Some( + "interrupted: executor lease stale or missing; run cancelled as an orphan" + .to_string(), + ); + Self::write_state_rows(&tx, &state)?; + tx.commit() + .map_err(|e| WorkflowError::Store(format!("commit orphan-cancel: {e}")))?; + Ok(OrphanCancelOutcome::Cancelled) + }) + .await + } +} diff --git a/crates/flare-workflow/src/sqlite_store.rs b/crates/flare-workflow/src/sqlite_store.rs index bccc9b7d6..f21d5a12c 100644 --- a/crates/flare-workflow/src/sqlite_store.rs +++ b/crates/flare-workflow/src/sqlite_store.rs @@ -16,10 +16,13 @@ use rusqlite_migration::{M, Migrations}; use crate::journal; use crate::store::StateStore; +#[path = "sqlite_start_once.rs"] +mod start_once; use crate::types::{ JournalEntry, MetricsFilter, StepId, StepMetrics, StepStatus, WorkflowContext, WorkflowData, WorkflowError, WorkflowMetrics, WorkflowResult, WorkflowRunId, WorkflowState, WorkflowStatus, }; +pub(crate) use start_once::OrphanCancelOutcome; /// Error opening/creating a SQLite store. #[derive(Debug, thiserror::Error)] @@ -112,6 +115,57 @@ impl SqliteStore { ALTER TABLE step_state ADD COLUMN input_tokens INTEGER; ALTER TABLE step_state ADD COLUMN output_tokens INTEGER;", ), + // Workflow schedules (Phase 1, item #366 PR-B): interval + // schedules, occurrence history, and durable start receipts share + // this file, so they migrate here — never a second migration + // list on `workflows.db`. + M::up( + "CREATE TABLE schedules ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + repo_root TEXT NOT NULL, + workflow_name TEXT NOT NULL, + params TEXT, + every_secs INTEGER NOT NULL CHECK (every_secs >= 60), + grace_secs INTEGER CHECK (grace_secs IS NULL OR grace_secs >= 0), + overlap TEXT NOT NULL DEFAULT 'skip' CHECK (overlap IN ('skip','allow')), + enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0,1)), + next_fire_at INTEGER NOT NULL, + last_fire_at INTEGER, + frozen_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE (repo_root, name) + ); + CREATE INDEX schedules_due ON schedules(enabled, next_fire_at); + CREATE TABLE schedule_runs ( + id TEXT PRIMARY KEY, + schedule_id TEXT NOT NULL REFERENCES schedules(id) ON DELETE CASCADE, + slot INTEGER NOT NULL, + trigger TEXT NOT NULL CHECK (trigger IN ('scheduled','manual')), + status TEXT NOT NULL CHECK (status IN ( + 'queued','dispatching','dispatched', + 'completed','failed','cancelled','interrupted','evicted', + 'dispatch_failed','skipped_missed','skipped_overlap' + )), + launch_spec TEXT, + start_deadline_at INTEGER, + dispatch_started_at INTEGER, + run_id TEXT, + error TEXT, + created_at INTEGER NOT NULL, + finished_at INTEGER + ); + CREATE UNIQUE INDEX schedule_runs_scheduled_slot + ON schedule_runs(schedule_id, slot) WHERE trigger = 'scheduled'; + CREATE INDEX schedule_runs_open ON schedule_runs(schedule_id, status); + CREATE INDEX schedule_runs_queue ON schedule_runs(status, created_at); + CREATE TABLE workflow_start_receipts ( + start_key TEXT PRIMARY KEY, + run_id TEXT NOT NULL UNIQUE, + created_at INTEGER NOT NULL + );", + ), ]) } @@ -142,6 +196,14 @@ impl SqliteStore { } } + /// Open the workflow database with the full migration list and hand back + /// the raw connection, for sibling tables (`schedules`, `schedule_runs`, + /// `workflow_start_receipts`) sharing the file. One migration list owns + /// the file; this is how other owners open it without forking that list. + pub fn open_shared_connection(path: &std::path::Path) -> Result { + db_kit::open_file(path, &Self::migrations()).map_err(SqliteStoreError::Open) + } + /// Write a state row plus its step_state/run_vars projections atomically. /// /// `step_states`/`variables` are fixed-growing maps within a run's @@ -150,11 +212,23 @@ impl SqliteStore { /// this crate removes entries), so a targeted UPSERT of each row is /// always sufficient — no per-write DELETE-all pass is needed. fn write_state(conn: &Connection, state: &WorkflowState) -> WorkflowResult<()> { - let json = Self::serialize(state)?; let tx = conn .unchecked_transaction() .map_err(|e| WorkflowError::Store(format!("begin tx: {e}")))?; + Self::write_state_rows(&tx, state)?; + tx.commit() + .map_err(|e| WorkflowError::Store(format!("commit: {e}")))?; + Ok(()) + } + /// Transaction-neutral row writer behind [`Self::write_state`]: runs on + /// the caller's transaction so atomic multi-table commits (start receipt + /// + run + journal) can reuse it without nesting `BEGIN`s. + fn write_state_rows( + tx: &rusqlite::Transaction<'_>, + state: &WorkflowState, + ) -> WorkflowResult<()> { + let json = Self::serialize(state)?; tx.execute( "INSERT INTO workflow_runs (id, workflow_id, status, current_step, input, output, error, state_json, created_at, updated_at) @@ -215,8 +289,6 @@ impl SqliteStore { .map_err(|e| WorkflowError::Store(format!("upsert run_vars: {e}")))?; } - tx.commit() - .map_err(|e| WorkflowError::Store(format!("commit: {e}")))?; Ok(()) } diff --git a/crates/flare-workflow/src/tests_schedule_start.rs b/crates/flare-workflow/src/tests_schedule_start.rs new file mode 100644 index 000000000..4eb7d9204 --- /dev/null +++ b/crates/flare-workflow/src/tests_schedule_start.rs @@ -0,0 +1,600 @@ +//! Start-once, receipt, and orphan-cancel tests (item #366 PR-B: T16–T21, +//! T24, plus the terminal-callback and disposition matrix). Real SQLite +//! transactions throughout; barriers and polling, never wall-clock sleeps +//! for lease expiry (stored leases are backdated instead). + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use chrono::{DateTime, Utc}; + +use crate::OrphanCancelDisposition; +use crate::engine::WorkflowEngine; +use crate::json::{PipelineData, SendMessage, StepInvocation, compile_workflow}; +use crate::sqlite_store::SqliteStore; +use crate::store::StateStore; +use crate::types::*; + +/// A `WorkflowData` whose serialization always fails: faults the state write +/// inside `create_start_once` to prove the receipt/journal roll back with it. +#[derive(Debug, Clone)] +struct FailSerialize; + +impl serde::Serialize for FailSerialize { + fn serialize(&self, _: S) -> Result { + Err(serde::ser::Error::custom("boom")) + } +} + +impl<'de> serde::Deserialize<'de> for FailSerialize { + fn deserialize>(d: D) -> Result { + let _: () = serde::Deserialize::deserialize(d)?; + Ok(FailSerialize) + } +} + +impl WorkflowData for FailSerialize { + fn workflow_type() -> &'static str { + "fail-serialize" + } +} + +fn counting_send(counter: Arc) -> SendMessage { + Arc::new(move |inv: StepInvocation| { + let counter = Arc::clone(&counter); + Box::pin(async move { + counter.fetch_add(1, Ordering::SeqCst); + Ok((format!("got {}", inv.prompt), 0, 0)) + }) + }) +} + +/// Never resolves: parks the driver in a live step without finishing. +fn blocked_send() -> SendMessage { + Arc::new(|_: StepInvocation| { + Box::pin(async move { + let never: (String, u64, u64) = std::future::pending().await; + Ok(never) + }) + }) +} + +fn one_step_def(send: SendMessage) -> crate::definition::WorkflowDefinition { + let json: crate::json::JsonWorkflow = serde_json::from_str( + r#"{"name": "w", "steps": [{"name": "s", "agent": "a", "prompt": "hi"}]}"#, + ) + .unwrap(); + let mut def = compile_workflow(&json, send).unwrap(); + def.id = WorkflowId::new("w"); + def +} + +/// Poll the store until `run_id` is terminal (real async work settling, not +/// clock manipulation). +async fn wait_terminal( + store: &SqliteStore, + run_id: WorkflowRunId, +) -> WorkflowState { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let state = store.load(run_id).await.unwrap(); + if state.status.is_terminal() { + return state; + } + assert!( + tokio::time::Instant::now() < deadline, + "run {run_id} never settled" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn run_count(store: &SqliteStore) -> usize { + store.list_all().await.unwrap().len() +} + +/// T16: concurrent starts for one key commit one receipt/run/input journal +/// and one driver; losers get `Existing`. +#[tokio::test] +async fn concurrent_starts_for_one_key_start_once() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + let counter = Arc::new(AtomicUsize::new(0)); + engine + .register_workflow(one_step_def(counting_send(counter.clone()))) + .unwrap(); + + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut tasks = Vec::new(); + for _ in 0..2 { + let (engine, barrier) = (engine.clone(), Arc::clone(&barrier)); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap() + })); + } + let mut ids = Vec::new(); + for t in tasks { + ids.push(t.await.unwrap().run_id()); + } + assert_eq!(ids[0], ids[1], "both starts resolve the same run"); + assert_eq!(run_count(&store).await, 1); + assert_eq!(store.lookup_start("k").await.unwrap(), Some(ids[0])); + assert_eq!(store.journal(ids[0]).await.unwrap().len(), 1); + + wait_terminal(&store, ids[0]).await; + // One driver executed the single step exactly once. + assert_eq!(counter.load(Ordering::SeqCst), 1); +} + +/// T17: faults at each of the three atomic writes roll back all three. +#[tokio::test] +async fn start_once_faults_roll_back_receipt_state_and_journal() { + // State-write fault (unserializable data). + { + let store = SqliteStore::::open_memory().unwrap(); + let run = WorkflowRunId::new(); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), FailSerialize); + state.status = WorkflowStatus::Running; + let err = store + .create_start_once( + "k-state".to_string(), + state, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("serialize"), "got {err:?}"); + assert_eq!(store.lookup_start("k-state").await.unwrap(), None); + assert!(matches!( + store.load(run).await.unwrap_err(), + WorkflowError::NotFound(_) + )); + assert_eq!(store.journal(run).await.unwrap().len(), 0); + } + // Receipt-write fault (run_id UNIQUE conflict with a foreign key). + { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t17.db"); + let store = SqliteStore::::open_file(&path).unwrap(); + let run = WorkflowRunId::new(); + let setup = rusqlite::Connection::open(&path).unwrap(); + setup + .execute( + "INSERT INTO workflow_start_receipts (start_key, run_id, created_at) \ + VALUES ('other', ?1, 'now')", + rusqlite::params![run.to_string()], + ) + .unwrap(); + drop(setup); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + state.status = WorkflowStatus::Running; + let err = store + .create_start_once( + "k-receipt".to_string(), + state, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("receipt"), "got {err:?}"); + assert_eq!(store.lookup_start("k-receipt").await.unwrap(), None); + assert!(matches!( + store.load(run).await.unwrap_err(), + WorkflowError::NotFound(_) + )); + assert_eq!(store.journal(run).await.unwrap().len(), 0); + } + // Journal-write fault: a trigger aborting journal inserts for this run. + // Receipt + state commit first in the same tx, so this proves the whole + // transaction rolls back (no receipt, no run row). + { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t17j.db"); + let store = SqliteStore::::open_file(&path).unwrap(); + let run = WorkflowRunId::new(); + let setup = rusqlite::Connection::open(&path).unwrap(); + setup + .execute_batch(&format!( + "CREATE TRIGGER fault_journal BEFORE INSERT ON journal \ + WHEN NEW.run_id = '{}' BEGIN SELECT RAISE(ABORT, 'journal fault'); END;", + run + )) + .unwrap(); + drop(setup); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + state.status = WorkflowStatus::Running; + let err = store + .create_start_once( + "k-journal".to_string(), + state, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("journal") || err.to_string().contains("UNIQUE"), + "got {err:?}" + ); + assert_eq!(store.lookup_start("k-journal").await.unwrap(), None); + assert!(matches!( + store.load(run).await.unwrap_err(), + WorkflowError::NotFound(_) + )); + } +} + +/// T18 (engine half): a live first start blocks duplicates — a concurrent +/// second start returns `Existing` without launching another driver. +#[tokio::test] +async fn concurrent_duplicate_start_while_first_in_flight_returns_existing() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + engine + .register_workflow(one_step_def(blocked_send())) + .unwrap(); + + let first = engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap(); + assert!(first.was_created()); + // Driver is parked in its step; the lease is fresh. + let second = engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap(); + assert_eq!(second, StartOnceResult::Existing(first.run_id())); + assert_eq!(run_count(&store).await, 1); + assert_eq!(engine.active_workflow_count(), 1); +} + +/// T19 (engine half): after a commit without a driver, the next start for +/// the key returns the receipt's run and launches nothing. +#[tokio::test] +async fn start_after_commit_without_driver_returns_existing_without_launch() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + let counter = Arc::new(AtomicUsize::new(0)); + engine + .register_workflow(one_step_def(counting_send(counter.clone()))) + .unwrap(); + + // Simulate "crash after commit before driver launch": durable + // receipt/run/journal, no driver. + let run = WorkflowRunId::new(); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + state.status = WorkflowStatus::Running; + state.lease = Some(ExecutorLease { + owner: "dead:1:x".into(), + renewed_at: Utc::now(), + }); + store + .create_start_once( + "k".to_string(), + state, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap(); + + let again = engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap(); + assert_eq!(again, StartOnceResult::Existing(run)); + assert_eq!(counter.load(Ordering::SeqCst), 0, "no driver launched"); + assert_eq!(run_count(&store).await, 1); +} + +/// T20: an evicted-then-looked-up key still resolves the receipt — no new +/// driver, receipt preserved. +#[tokio::test] +async fn evicted_run_still_resolves_receipt_without_new_driver() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + let counter = Arc::new(AtomicUsize::new(0)); + engine + .register_workflow(one_step_def(counting_send(counter.clone()))) + .unwrap(); + + let created = engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap() + .run_id(); + wait_terminal(&store, created).await; + // Evict through the real deletion path. + store.delete(created).await.unwrap(); + assert!(matches!( + store.load(created).await.unwrap_err(), + WorkflowError::NotFound(_) + )); + + let again = engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap(); + assert_eq!(again, StartOnceResult::Existing(created)); + assert_eq!(store.lookup_start("k").await.unwrap(), Some(created)); + assert_eq!(run_count(&store).await, 0, "no replacement run"); + assert_eq!(counter.load(Ordering::SeqCst), 1, "no new driver"); +} + +/// T21 (engine half): a committed-but-never-driven run with a stale lease is +/// interrupted by conditional cancel — exactly one run, never a replacement +/// start. +#[tokio::test] +async fn committed_undriven_run_with_stale_lease_is_interrupted_not_replaced() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + engine + .register_workflow(one_step_def(blocked_send())) + .unwrap(); + + let run = WorkflowRunId::new(); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + state.status = WorkflowStatus::Running; + state.lease = Some(ExecutorLease { + owner: "dead:1:x".into(), + renewed_at: Utc::now() - chrono::Duration::hours(1), + }); + store + .create_start_once( + "k".to_string(), + state, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap(); + + let disp = engine.cancel_if_orphaned(run, Utc::now()).await.unwrap(); + assert_eq!(disp, OrphanCancelDisposition::Interrupted); + assert_eq!( + store.load(run).await.unwrap().status, + WorkflowStatus::Cancelled + ); + assert_eq!(run_count(&store).await, 1); +} + +/// T24: post-commit eviction pressure cannot turn a committed start into an +/// apparent failure — `Created` still resolves, launches, and completes. +#[tokio::test] +async fn post_commit_eviction_pressure_still_yields_created_and_completion() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + let counter = Arc::new(AtomicUsize::new(0)); + engine + .register_workflow(one_step_def(counting_send(counter.clone()))) + .unwrap(); + + // Push the store past the 200-run eviction cap with terminal runs. + for _ in 0..205 { + let run = WorkflowRunId::new(); + let mut s = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + s.status = WorkflowStatus::Completed; + store.save(s).await.unwrap(); + } + let created = engine + .start_workflow_once_with_params( + "k", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap(); + assert!(created.was_created()); + let run_id = created.run_id(); + assert_eq!(store.lookup_start("k").await.unwrap(), Some(run_id)); + wait_terminal(&store, run_id).await; + assert_eq!(counter.load(Ordering::SeqCst), 1); + assert!(run_count(&store).await <= 201); +} + +/// Terminal callback: exactly one `Completed` notification per settled run, +/// and a `Cancelled` one on real cancellation. +#[tokio::test] +async fn terminal_callback_fires_once_per_settlement() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + let counter = Arc::new(AtomicUsize::new(0)); + engine + .register_workflow(one_step_def(counting_send(counter.clone()))) + .unwrap(); + let seen: Arc>> = + Arc::new(std::sync::Mutex::new(Vec::new())); + let seen_cb = Arc::clone(&seen); + engine.on_terminal(Arc::new(move |run_id, status, _| { + let seen_cb = Arc::clone(&seen_cb); + Box::pin(async move { + seen_cb.lock().unwrap().push((run_id, status)); + }) + })); + + let run = engine + .start_workflow_once_with_params( + "k1", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap() + .run_id(); + wait_terminal(&store, run).await; + // Let the awaited callback land (it runs before the settle returns, but + // assert without assuming scheduling). + assert_eq!( + *seen.lock().unwrap(), + vec![(run, WorkflowStatus::Completed)] + ); + + let run2 = engine + .start_workflow_once_with_params( + "k2", + WorkflowId::new("w"), + PipelineData, + String::new(), + serde_json::Value::Null, + ) + .await + .unwrap() + .run_id(); + engine.cancel_workflow(run2).await.unwrap(); + assert_eq!( + *seen.lock().unwrap(), + vec![ + (run, WorkflowStatus::Completed), + (run2, WorkflowStatus::Cancelled) + ] + ); +} + +/// `cancel_if_orphaned` disposition matrix: fresh leases and paused runs +/// survive; terminal runs report; missing rows report; only stale orphans +/// flip to `Cancelled`. +#[tokio::test] +async fn orphan_cancel_dispositions() { + let store = SqliteStore::::open_memory().unwrap(); + let engine = WorkflowEngine::with_store(store.clone()); + let now = Utc::now(); + let past = now - chrono::Duration::hours(1); + + async fn put( + store: &SqliteStore, + status: WorkflowStatus, + lease: Option, + ) -> WorkflowRunId { + let run = WorkflowRunId::new(); + let mut s = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + s.status = status; + s.lease = lease; + store.save(s).await.unwrap(); + run + } + let lease_at = |t: DateTime| { + Some(ExecutorLease { + owner: "dead:1:x".into(), + renewed_at: t, + }) + }; + + // Stale lease on a live run: interrupted. + let stale = put(&store, WorkflowStatus::Running, lease_at(past)).await; + assert_eq!( + engine.cancel_if_orphaned(stale, now).await.unwrap(), + OrphanCancelDisposition::Interrupted + ); + assert_eq!( + store.load(stale).await.unwrap().status, + WorkflowStatus::Cancelled + ); + assert!(store.load(stale).await.unwrap().lease.is_none()); + + // Fresh lease: untouched. + let fresh = put(&store, WorkflowStatus::Running, lease_at(now)).await; + assert_eq!( + engine.cancel_if_orphaned(fresh, now).await.unwrap(), + OrphanCancelDisposition::Unchanged + ); + assert_eq!( + store.load(fresh).await.unwrap().status, + WorkflowStatus::Running + ); + + // Missing lease on a live run: interrupted. + let no_lease = put(&store, WorkflowStatus::Waiting, None).await; + assert_eq!( + engine.cancel_if_orphaned(no_lease, now).await.unwrap(), + OrphanCancelDisposition::Interrupted + ); + + // Paused with a stale lease: intentional inactivity, never cancelled. + let paused = put(&store, WorkflowStatus::Paused, lease_at(past)).await; + assert_eq!( + engine.cancel_if_orphaned(paused, now).await.unwrap(), + OrphanCancelDisposition::Unchanged + ); + assert_eq!( + store.load(paused).await.unwrap().status, + WorkflowStatus::Paused + ); + + // Already terminal: real outcome reported, row untouched. + let done = put(&store, WorkflowStatus::Failed, lease_at(past)).await; + assert_eq!( + engine.cancel_if_orphaned(done, now).await.unwrap(), + OrphanCancelDisposition::Terminal(WorkflowStatus::Failed) + ); + + // Missing row. + assert_eq!( + engine + .cancel_if_orphaned(WorkflowRunId::new(), now) + .await + .unwrap(), + OrphanCancelDisposition::Missing + ); +} + +/// Engine eviction/deletion paths must never delete start receipts. +#[tokio::test] +async fn deletion_never_removes_start_receipts() { + let store = SqliteStore::::open_memory().unwrap(); + let run = WorkflowRunId::new(); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + state.status = WorkflowStatus::Running; + store + .create_start_once( + "k".to_string(), + state, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap(); + store.delete(run).await.unwrap(); + assert_eq!(store.lookup_start("k").await.unwrap(), Some(run)); +} diff --git a/crates/flare-workflow/src/types.rs b/crates/flare-workflow/src/types.rs index cf7fcd2c1..081b92e30 100644 --- a/crates/flare-workflow/src/types.rs +++ b/crates/flare-workflow/src/types.rs @@ -497,6 +497,28 @@ impl WorkflowState { } } +/// Outcome of an idempotent start attempt (`start_workflow_once_with_params` / +/// `create_start_once`): whether this call committed a fresh run (and may +/// launch its driver) or found a durable receipt from an earlier start (and +/// must not launch anything). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StartOnceResult { + Created(WorkflowRunId), + Existing(WorkflowRunId), +} + +impl StartOnceResult { + pub fn run_id(self) -> WorkflowRunId { + match self { + Self::Created(id) | Self::Existing(id) => id, + } + } + + pub fn was_created(self) -> bool { + matches!(self, Self::Created(_)) + } +} + /// Result returned by a step execution. #[derive(Debug, Clone, PartialEq, Eq)] pub enum StepResult { diff --git a/src/dashboard/dispatch_boot.rs b/src/dashboard/dispatch_boot.rs index f40049f56..c221b86d0 100644 --- a/src/dashboard/dispatch_boot.rs +++ b/src/dashboard/dispatch_boot.rs @@ -39,6 +39,82 @@ pub(super) fn start_work_dispatch(queue: &agentflare_jobs::Queue) { ); } +/// Daemon-owned workflow-schedule ticker (item #366 Phase 1): boots the +/// `Scheduler` on its own `WorkflowEngine` + store pair over +/// `default_db_path()` and ticks every 60 s. Deliberately independent of +/// `start_work_dispatch`'s smoke-test / register / recover gate — a +/// work-item recovery failure leaves schedules firing. Never calls +/// `recover`/`resume_run`/`try_resume` on the scheduler engine: occurrence +/// linkage goes through start receipts, and resuming here would break +/// at-most-once fire semantics. +pub(super) fn spawn_schedule_ticker() -> tokio::task::JoinHandle<()> { + spawn_schedule_ticker_every( + crate::workflow::default_db_path(), + crate::workflow::agent_send_hook(), + crate::schedule::TICK_INTERVAL_SECS, + ) +} + +/// Test seam: same ticker over an isolated database with an injectable send +/// hook and tick interval. Production always passes the 60 s constant. +pub(crate) fn spawn_schedule_ticker_every( + db_path: std::path::PathBuf, + send: flare_workflow::json::SendMessage, + interval_secs: u64, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let store = match crate::schedule::ScheduleStore::open(&db_path) { + Ok(s) => s, + Err(e) => { + eprintln!("agentflare-schedule: ticker off (store open failed): {e}"); + return; + } + }; + let wf_store = match flare_workflow::sqlite_store::SqliteStore::open_file(&db_path) { + Ok(s) => s, + Err(e) => { + eprintln!("agentflare-schedule: ticker off (workflow store open failed): {e}"); + return; + } + }; + let engine = flare_workflow::engine::WorkflowEngine::with_store(wf_store) + .with_runtime_handle(crate::workflow::blocking_runtime_handle()); + let scheduler = crate::schedule::Scheduler::new(store, engine, send); + let mut ticker = + tokio::time::interval(std::time::Duration::from_secs(interval_secs.max(1))); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + ticker.tick().await; + match scheduler.tick_once(chrono::Utc::now()).await { + Ok(summary) => { + if summary.admitted > 0 + || summary.skipped_missed > 0 + || summary.skipped_overlap > 0 + || summary.reconciled > 0 + || summary.fired > 0 + || !summary.errors.is_empty() + { + eprintln!( + "agentflare-schedule: tick admitted={} missed={} overlap={} \ + reconciled={} fired={} pruned={} errors={:?}", + summary.admitted, + summary.skipped_missed, + summary.skipped_overlap, + summary.reconciled, + summary.fired, + summary.pruned, + summary.errors, + ); + } + } + Err(e) => { + eprintln!("agentflare-schedule: tick failed: {e}"); + } + } + } + }) +} + /// Retries `engine().recover()` every `PIPELINE_RECOVERY_RETRY_INTERVAL` /// after it failed at boot, logging each failure, and starts work dispatch /// the first time it succeeds. diff --git a/src/dashboard/mod.rs b/src/dashboard/mod.rs index 988919e3a..88ae312f0 100644 --- a/src/dashboard/mod.rs +++ b/src/dashboard/mod.rs @@ -5,7 +5,7 @@ mod data; mod gate; mod messages; mod orphan_reconcile; -mod server; +pub(crate) mod server; pub fn serve(host: &str, port: u16, open: bool, yes_expose: bool) { let runtime = tokio::runtime::Runtime::new().expect("failed to build tokio runtime"); diff --git a/src/dashboard/server.rs b/src/dashboard/server.rs index 99feec2c4..5b19e3d83 100644 --- a/src/dashboard/server.rs +++ b/src/dashboard/server.rs @@ -691,7 +691,7 @@ pub fn router(queue: Queue) -> Router { } #[path = "dispatch_boot.rs"] -mod dispatch_boot; +pub(crate) mod dispatch_boot; use dispatch_boot::*; fn is_local_bind(host: &str) -> bool { @@ -849,6 +849,7 @@ pub async fn run(host: &str, port: u16, open: bool, yes_expose: bool) { } } spawn_binary_staleness_watchdog(crate::daemon::BinarySnapshot::capture()); + spawn_schedule_ticker(); spawn_job_cleanup(queue.clone()); let listener = tokio::net::TcpListener::bind((host, port)) diff --git a/src/main.rs b/src/main.rs index fa98b271e..711113f8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,6 +71,7 @@ mod quota; mod review; mod rollup; mod rule_text; +mod schedule; mod sessions; mod shell; mod shim_install; diff --git a/src/schedule/mod.rs b/src/schedule/mod.rs new file mode 100644 index 000000000..fe0388134 --- /dev/null +++ b/src/schedule/mod.rs @@ -0,0 +1,384 @@ +//! Durable interval scheduling for project-local JSON workflows (item #366 +//! Phase 1, PR-B): `schedules` / `schedule_runs` / `workflow_start_receipts` +//! storage, a daemon-owned 60 s ticker, and the eager terminal-outcome hook. +//! CLI/MCP surfaces (PR-C) enqueue and administer here; only the daemon's +//! `Scheduler` executes. + +// The admin/CRUD half of this subtree (schedule CRUD, validation, manual +// receipts) is PR-C (#370) CLI/MCP surface: it ships here with full test +// coverage so the daemon half can build on it, and lights up in production +// when PR-C wires the commands. Silence `dead_code` for the subtree until +// then — PR-C removes this once the commands land — rather than scattering +// per-item allows. +#![allow(dead_code)] + +pub(crate) mod store; +pub(crate) mod tick; + +pub use store::ScheduleStore; + +#[cfg(test)] +mod tests; +#[cfg(test)] +#[path = "tests_lifecycle.rs"] +mod tests_lifecycle; +#[cfg(test)] +#[path = "tests_recovery.rs"] +mod tests_recovery; + +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex}; + +use chrono::{DateTime, Utc}; +use flare_workflow::json::{PipelineData, SendMessage}; +use flare_workflow::sqlite_store::SqliteStore; +use flare_workflow::{StateStore, WorkflowEngine, WorkflowRunId}; + +/// Seconds between daemon scheduler ticks. +pub const TICK_INTERVAL_SECS: u64 = 60; +/// Default grace when a schedule sets none: `min(every_secs, 12 h)`. +pub const MAX_GRACE_DEFAULT: i64 = 43200; +/// How long a claimed-but-unlinked dispatch waits for its start receipt +/// before recovery requeues it. A retry threshold only — never proof the +/// start did not happen (the receipt check decides that). +pub const START_RECOVERY_DELAY_SECS: i64 = 120; +/// Per-schedule retention: newest N non-skip terminal occurrences… +pub const RETENTION_NON_SKIP: i64 = 100; +/// …plus, under a separate cap, newest N skip rows. +pub const RETENTION_SKIP: i64 = 100; +/// Minimum schedule interval: the ticker only runs every 60 s. +pub const MIN_EVERY_SECS: i64 = 60; + +/// Terminal statuses that are real outcomes (not skips). +pub const TERMINAL_NON_SKIP_STATUSES: [&str; 6] = [ + "completed", + "failed", + "cancelled", + "interrupted", + "evicted", + "dispatch_failed", +]; +/// Terminal skip statuses, retained under their own cap. +pub const SKIP_STATUSES: [&str; 2] = ["skipped_missed", "skipped_overlap"]; + +/// One `schedules` row. Timestamps are UTC epoch seconds; `params_raw` is +/// the stored JSON object (None = absent at create). +#[derive(Debug, Clone)] +pub struct ScheduleRow { + pub id: String, + pub name: String, + pub repo_root: String, + pub workflow_name: String, + pub params_raw: Option, + pub every_secs: i64, + pub grace_secs: Option, + pub overlap: String, + pub enabled: bool, + pub next_fire_at: i64, + pub last_fire_at: Option, + pub frozen_error: Option, + pub created_at: i64, + pub updated_at: i64, +} + +impl ScheduleRow { + /// Stored params as JSON: absent means `{}`, anything else must already + /// be an object (enforced at create/update; a garbage row freezes the + /// schedule instead of failing the tick). + pub fn params_json(&self) -> Result { + match &self.params_raw { + None => Ok(serde_json::json!({})), + Some(raw) => { + let v: serde_json::Value = + serde_json::from_str(raw).map_err(|e| format!("invalid params JSON: {e}"))?; + if !v.is_object() { + return Err("params must be a JSON object".to_string()); + } + Ok(v) + } + } + } + + /// Effective grace: explicit `grace_secs`, else `min(every_secs, 12 h)`. + pub fn effective_grace_secs(&self) -> i64 { + self.grace_secs + .unwrap_or_else(|| self.every_secs.min(MAX_GRACE_DEFAULT)) + } +} + +/// One `schedule_runs` row. +#[derive(Debug, Clone)] +pub struct OccurrenceRow { + pub id: String, + pub schedule_id: String, + pub slot: i64, + pub trigger: String, + pub status: String, + pub launch_spec: Option, + pub start_deadline_at: Option, + pub dispatch_started_at: Option, + pub run_id: Option, + pub error: Option, + pub created_at: i64, + pub finished_at: Option, +} + +impl OccurrenceRow { + pub fn is_terminal(&self) -> bool { + TERMINAL_NON_SKIP_STATUSES.contains(&self.status.as_str()) + || SKIP_STATUSES.contains(&self.status.as_str()) + } + + pub fn parsed_launch_spec(&self) -> Result { + let raw = self + .launch_spec + .as_deref() + .ok_or_else(|| "occurrence has no launch_spec".to_string())?; + serde_json::from_str(raw).map_err(|e| format!("invalid launch_spec: {e}")) + } +} + +/// Frozen request snapshot an admitted occurrence executes from: the accepted +/// schedule state at admission time. Workflow file *contents* resolve at +/// execution; everything here is fixed across updates/restarts. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LaunchSpec { + pub repo_root: String, + pub workflow_name: String, + pub params: serde_json::Value, + pub schedule_name: String, +} + +/// Durable manual-fire acceptance: the request is queued, never executed by +/// the client. +#[derive(Debug, Clone)] +pub struct FireReceipt { + pub occurrence_id: String, + pub status: String, +} + +/// Per-tick accounting for logs and tests. +#[derive(Debug, Default)] +pub struct TickSummary { + pub admitted: u64, + pub skipped_missed: u64, + pub skipped_overlap: u64, + pub fired: u64, + pub reconciled: u64, + pub pruned: u64, + pub errors: Vec, +} + +/// Durable start key for an occurrence: the receipt is what makes a repeated +/// start return the existing run instead of launching a second driver. +pub fn start_key_for(occurrence_id: &str) -> String { + format!("schedule:{occurrence_id}") +} + +/// `^[A-Za-z0-9._-]{1,64}$` without a regex dependency. +pub fn validate_name(name: &str) -> Result<(), String> { + if name.is_empty() || name.len() > 64 { + return Err(format!( + "invalid schedule name '{name}': 1-64 characters required" + )); + } + if !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'_' || b == b'-') + { + return Err(format!( + "invalid schedule name '{name}': only A-Z a-z 0-9 . _ - allowed" + )); + } + Ok(()) +} + +pub fn validate_every_secs(every_secs: i64) -> Result<(), String> { + if every_secs < MIN_EVERY_SECS { + return Err(format!( + "invalid interval {every_secs}s: minimum is {MIN_EVERY_SECS}s" + )); + } + Ok(()) +} + +pub fn validate_grace_secs(grace_secs: Option) -> Result<(), String> { + if grace_secs.is_some_and(|g| g < 0) { + return Err("invalid grace: must be >= 0".to_string()); + } + Ok(()) +} + +pub fn validate_overlap(overlap: &str) -> Result<(), String> { + if overlap != "skip" && overlap != "allow" { + return Err(format!( + "invalid overlap '{overlap}': must be 'skip' or 'allow'" + )); + } + Ok(()) +} + +/// Params must be absent or a JSON object, and must not claim the reserved +/// top-level `schedule` key the fire path injects. +pub fn validate_params(params: &serde_json::Value) -> Result<(), String> { + let obj = params + .as_object() + .ok_or_else(|| "params must be a JSON object".to_string())?; + if obj.contains_key("schedule") { + return Err("params reserves top-level key `schedule`".to_string()); + } + Ok(()) +} + +/// Canonicalize a repo root for storage/comparison; reject nonexistent roots. +pub fn canonical_repo_root(root: &str) -> Result { + let path = std::path::Path::new(root); + if !path.is_dir() { + return Err(format!("repo root does not exist: '{root}'")); + } + std::fs::canonicalize(path) + .map(|p| p.to_string_lossy().into_owned()) + .map_err(|e| format!("cannot canonicalize repo root '{root}': {e}")) +} + +/// Daemon-owned scheduler: one long-lived engine/store pair, per-schedule +/// register-plus-start locks, an in-flight occurrence set, and a tick mutex. +/// Construction registers the eager terminal-outcome callback on the engine +/// so driver completion writes `schedule_runs` directly (tick reconciliation +/// is the crash fallback, not the primary path). +pub struct Scheduler { + engine: WorkflowEngine>, + store: ScheduleStore, + send: SendMessage, + schedule_locks: Arc>>>>, + in_flight: Arc>>, + tick_lock: tokio::sync::Mutex<()>, + fires: Mutex>>, +} + +impl Scheduler { + pub fn new( + store: ScheduleStore, + engine: WorkflowEngine>, + send: SendMessage, + ) -> Self { + let sched = Self { + engine, + store: store.clone(), + send, + schedule_locks: Arc::new(Mutex::new(HashMap::new())), + in_flight: Arc::new(Mutex::new(HashSet::new())), + tick_lock: tokio::sync::Mutex::new(()), + fires: Mutex::new(Vec::new()), + }; + // Eager path: driver completion lands in `schedule_runs` before any + // later eviction can remove the workflow row. Write failures only + // log; the tick fallback converges them. + let cb_store = sched.store.clone(); + sched + .engine + .on_terminal(Arc::new(move |run_id, status, error| { + let store = cb_store.clone(); + Box::pin(async move { + let outcome = match status { + flare_workflow::WorkflowStatus::Completed => "completed", + flare_workflow::WorkflowStatus::Failed => "failed", + flare_workflow::WorkflowStatus::Cancelled => "cancelled", + _ => return, + }; + if let Err(e) = store + .complete_linked_run( + &run_id.to_string(), + outcome, + error.as_deref(), + Utc::now().timestamp(), + ) + .await + { + eprintln!("agentflare-schedule: eager outcome write failed: {e}"); + } + }) + })); + sched + } + + /// Open both halves (schedule tables + workflow store) on `db_path` with + /// an ambient-runtime engine. Tests use temp paths; production passes + /// `default_db_path()` with a `WORKFLOW_RT`-pinned engine via `new`. + pub fn open(db_path: &std::path::Path, send: SendMessage) -> Result { + let store = ScheduleStore::open(db_path)?; + let wf_store = + SqliteStore::open_file(db_path).map_err(|e| format!("open workflow store: {e}"))?; + let engine = WorkflowEngine::::with_store(wf_store); + Ok(Self::new(store, engine, send)) + } + + pub fn store(&self) -> &ScheduleStore { + &self.store + } + + fn schedule_lock(&self, schedule_id: &str) -> Arc> { + let mut locks = self.schedule_locks.lock().expect("schedule locks"); + Arc::clone( + locks + .entry(schedule_id.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))), + ) + } + + fn mark_in_flight(&self, occurrence_id: &str) -> bool { + self.in_flight + .lock() + .expect("in-flight set") + .insert(occurrence_id.to_string()) + } + + fn unmark_in_flight(&self, occurrence_id: &str) { + self.in_flight + .lock() + .expect("in-flight set") + .remove(occurrence_id); + } + + fn is_in_flight(&self, occurrence_id: &str) -> bool { + self.in_flight + .lock() + .expect("in-flight set") + .contains(occurrence_id) + } + + /// Drop finished fire tasks so the handle list cannot grow without bound. + /// Live tasks are untouched — claiming never waits on them. + fn reap_fires(&self) { + self.fires + .lock() + .expect("fire handles") + .retain(|h| !h.is_finished()); + } + + /// Await every outstanding fire task. Tests use this after `tick_once` + /// to converge asynchronously spawned fires without wall-clock sleeps. + pub async fn drain_fires(&self) { + let handles: Vec<_> = self.fires.lock().expect("fire handles").drain(..).collect(); + for h in handles { + let _ = h.await; + } + } + + /// Current engine run lookup for tests and the ticker (receipt authority + /// stays with `lookup_start`; this is diagnostic linkage only). + pub async fn lookup_run( + &self, + run_id: WorkflowRunId, + ) -> Result>, String> { + match self.engine.state_store().load(run_id).await { + Ok(s) => Ok(Some(s)), + Err(flare_workflow::WorkflowError::NotFound(_)) => Ok(None), + Err(e) => Err(e.to_string()), + } + } + + pub fn now_timestamp(now: DateTime) -> i64 { + now.timestamp() + } +} diff --git a/src/schedule/store.rs b/src/schedule/store.rs new file mode 100644 index 000000000..e5143fa8b --- /dev/null +++ b/src/schedule/store.rs @@ -0,0 +1,884 @@ +//! Blocking schedule storage: `schedules` CRUD, admission, manual queue, +//! linkage, retention. Every method is `async` but runs its SQLite section on +//! a blocking thread; transactions never span an `.await`, a filesystem +//! read, or an agent launch. + +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use rusqlite::{Connection, OptionalExtension, Row, TransactionBehavior, params}; + +use super::{ + FireReceipt, LaunchSpec, OccurrenceRow, RETENTION_NON_SKIP, RETENTION_SKIP, SKIP_STATUSES, + ScheduleRow, TERMINAL_NON_SKIP_STATUSES, canonical_repo_root, validate_every_secs, + validate_grace_secs, validate_name, validate_overlap, validate_params, +}; +use flare_workflow::json::{JsonWorkflow, PipelineData, compile_workflow}; +use flare_workflow::sqlite_store::SqliteStore; + +/// Outcome of one transactional slot claim. +#[derive(Debug, Clone)] +pub enum SlotOutcome { + Admitted { + occurrence_id: String, + }, + SkippedMissed { + occurrence_id: String, + }, + SkippedOverlap { + occurrence_id: String, + }, + /// Row gone, disabled, frozen, or no longer due (lost race / no-op). + NotDue, + /// Malformed row (bad params, overflow): frozen, clock untouched. + Frozen, +} + +/// Inputs for [`ScheduleStore::create_schedule`]. +#[derive(Debug, Clone)] +pub struct CreateInput { + pub name: String, + pub repo_root: String, + pub workflow_name: String, + pub params: Option, + pub every_secs: i64, + pub grace_secs: Option, + pub overlap: String, +} + +/// Inputs for [`ScheduleStore::update_schedule`]: `None` = no change. +/// `params = Some(Null)` clears to absent; `grace_secs = Some(None)` clears +/// to the default grace. +#[derive(Debug, Clone, Default)] +pub struct UpdateInput { + pub workflow_name: Option, + pub params: Option, + pub every_secs: Option, + pub grace_secs: Option>, + pub overlap: Option, +} + +async fn blocking(f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + tokio::task::spawn_blocking(f) + .await + .map_err(|e| format!("schedule store task panicked: {e}"))? +} + +fn row_to_schedule(row: &Row<'_>) -> rusqlite::Result { + Ok(ScheduleRow { + id: row.get("id")?, + name: row.get("name")?, + repo_root: row.get("repo_root")?, + workflow_name: row.get("workflow_name")?, + params_raw: row.get("params")?, + every_secs: row.get("every_secs")?, + grace_secs: row.get("grace_secs")?, + overlap: row.get("overlap")?, + enabled: row.get::<_, i64>("enabled")? != 0, + next_fire_at: row.get("next_fire_at")?, + last_fire_at: row.get("last_fire_at")?, + frozen_error: row.get("frozen_error")?, + created_at: row.get("created_at")?, + updated_at: row.get("updated_at")?, + }) +} + +fn row_to_occurrence(row: &Row<'_>) -> rusqlite::Result { + Ok(OccurrenceRow { + id: row.get("id")?, + schedule_id: row.get("schedule_id")?, + slot: row.get("slot")?, + trigger: row.get("trigger")?, + status: row.get("status")?, + launch_spec: row.get("launch_spec")?, + start_deadline_at: row.get("start_deadline_at")?, + dispatch_started_at: row.get("dispatch_started_at")?, + run_id: row.get("run_id")?, + error: row.get("error")?, + created_at: row.get("created_at")?, + finished_at: row.get("finished_at")?, + }) +} + +fn select_schedule(tx: &Connection, id: &str) -> Result, String> { + tx.query_row( + "SELECT id,name,repo_root,workflow_name,params,every_secs,grace_secs,overlap,\ + enabled,next_fire_at,last_fire_at,frozen_error,created_at,updated_at \ + FROM schedules WHERE id = ?1", + params![id], + row_to_schedule, + ) + .optional() + .map_err(|e| format!("load schedule: {e}")) +} + +/// Resolve, parse, and structurally validate the named workflow: the create/ +/// update gate and the execution-time revalidation share it. +fn check_workflow(repo_root: &str, workflow_name: &str) -> Result<(), String> { + let definition = + crate::workflow::resolve_named_definition(std::path::Path::new(repo_root), workflow_name)?; + let json: JsonWorkflow = + serde_json::from_str(&definition).map_err(|e| format!("invalid workflow JSON: {e}"))?; + // Structural compile only; the dummy hook is never invoked by it. + let dummy = std::sync::Arc::new(|_: flare_workflow::json::StepInvocation| { + Box::pin(async move { Err::<(String, u64, u64), String>("unused".to_string()) }) + as std::pin::Pin + Send>> + }); + compile_workflow(&json, dummy)?; + Ok(()) +} + +fn launch_snapshot( + repo_root: &str, + workflow_name: &str, + params: serde_json::Value, + schedule_name: &str, +) -> Result { + serde_json::to_string(&LaunchSpec { + repo_root: repo_root.to_string(), + workflow_name: workflow_name.to_string(), + params, + schedule_name: schedule_name.to_string(), + }) + .map_err(|e| format!("serialize launch_spec: {e}")) +} + +/// Blocking schedule/occurrence storage on the shared `workflows.db` file +/// (opened with the full migration list — never a second list). +#[derive(Clone)] +pub struct ScheduleStore { + conn: Arc>, +} + +impl ScheduleStore { + pub fn open(path: &Path) -> Result { + let conn = SqliteStore::::open_shared_connection(path) + .map_err(|e| format!("open schedule store: {e}"))?; + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + }) + } + + pub async fn create_schedule( + &self, + input: CreateInput, + now: i64, + ) -> Result { + validate_name(&input.name)?; + validate_every_secs(input.every_secs)?; + validate_grace_secs(input.grace_secs)?; + validate_overlap(&input.overlap)?; + let params = input.params.unwrap_or(serde_json::json!({})); + validate_params(¶ms)?; + let repo_root = canonical_repo_root(&input.repo_root)?; + check_workflow(&repo_root, &input.workflow_name)?; + let next_fire_at = now + .checked_add(input.every_secs) + .ok_or_else(|| "timestamp overflow: next_fire_at".to_string())?; + let conn = Arc::clone(&self.conn); + blocking(move || { + let conn = conn_lock(&conn)?; + let id = flare_workflow::WorkflowRunId::new().to_string(); + let params_raw = serde_json::to_string(¶ms).map_err(|e| e.to_string())?; + let r = conn.execute( + "INSERT INTO schedules (id,name,repo_root,workflow_name,params,every_secs,\ + grace_secs,overlap,enabled,next_fire_at,last_fire_at,frozen_error,\ + created_at,updated_at) \ + VALUES (?1,?2,?3,?4,?5,?6,?7,?8,1,?9,NULL,NULL,?10,?10)", + params![ + id, + input.name, + repo_root, + input.workflow_name, + params_raw, + input.every_secs, + input.grace_secs, + input.overlap, + next_fire_at, + now, + ], + ); + if let Err(e) = r { + if e.to_string().contains("UNIQUE constraint") { + return Err(format!( + "schedule '{}' already exists in this repo", + input.name + )); + } + return Err(format!("create schedule: {e}")); + } + select_schedule(&conn, &id)?.ok_or_else(|| "create schedule: row missing".to_string()) + }) + .await + } + + pub async fn get_by_id(&self, id: &str) -> Result, String> { + let conn = Arc::clone(&self.conn); + let id = id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + select_schedule(&conn, &id) + }) + .await + } + + pub async fn get_by_name( + &self, + repo_root: &str, + name: &str, + ) -> Result, String> { + let conn = Arc::clone(&self.conn); + let (repo_root, name) = (repo_root.to_string(), name.to_string()); + blocking(move || { + let conn = conn_lock(&conn)?; + conn.query_row( + "SELECT id,name,repo_root,workflow_name,params,every_secs,grace_secs,overlap,\ + enabled,next_fire_at,last_fire_at,frozen_error,created_at,updated_at \ + FROM schedules WHERE repo_root = ?1 AND name = ?2", + params![repo_root, name], + row_to_schedule, + ) + .optional() + .map_err(|e| format!("load schedule: {e}")) + }) + .await + } + + pub async fn list_for_repo(&self, repo_root: &str) -> Result, String> { + let conn = Arc::clone(&self.conn); + let repo_root = repo_root.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let mut stmt = conn + .prepare( + "SELECT id,name,repo_root,workflow_name,params,every_secs,grace_secs,overlap,\ + enabled,next_fire_at,last_fire_at,frozen_error,created_at,updated_at \ + FROM schedules WHERE repo_root = ?1 ORDER BY name", + ) + .map_err(|e| format!("prepare list: {e}"))?; + let rows = stmt + .query_map(params![repo_root], row_to_schedule) + .map_err(|e| format!("list schedules: {e}"))?; + rows.collect::>>() + .map_err(|e| format!("list schedules: {e}")) + }) + .await + } + + pub async fn update_schedule( + &self, + id: &str, + update: UpdateInput, + now: i64, + ) -> Result { + if update.workflow_name.as_deref().is_some_and(str::is_empty) { + return Err("workflow name must not be empty".to_string()); + } + if let Some(p) = &update.params + && !p.is_null() + { + validate_params(p)?; + } + if let Some(e) = update.every_secs { + validate_every_secs(e)?; + } + if let Some(g) = update.grace_secs { + validate_grace_secs(g)?; + } + if let Some(o) = &update.overlap { + validate_overlap(o)?; + } + let conn = Arc::clone(&self.conn); + let (id, update) = (id.to_string(), update); + blocking(move || { + let conn = conn_lock(&conn)?; + let mut sched = + select_schedule(&conn, &id)?.ok_or_else(|| format!("unknown schedule '{id}'"))?; + if let Some(w) = update.workflow_name { + check_workflow(&sched.repo_root, &w)?; + sched.workflow_name = w; + } + if let Some(p) = update.params { + sched.params_raw = if p.is_null() { + None + } else { + Some(serde_json::to_string(&p).map_err(|e| e.to_string())?) + }; + } + let mut next_fire_at = sched.next_fire_at; + if let Some(e) = update.every_secs { + next_fire_at = now + .checked_add(e) + .ok_or_else(|| "timestamp overflow: next_fire_at".to_string())?; + sched.every_secs = e; + } + if let Some(g) = update.grace_secs { + sched.grace_secs = g; + } + if let Some(o) = update.overlap { + sched.overlap = o; + } + // Every successful update clears a freeze; accepted occurrences + // keep their launch snapshots. + conn.execute( + "UPDATE schedules SET workflow_name=?1, params=?2, every_secs=?3, grace_secs=?4,\ + overlap=?5, next_fire_at=?6, frozen_error=NULL, updated_at=?7 WHERE id=?8", + params![ + sched.workflow_name, + sched.params_raw, + sched.every_secs, + sched.grace_secs, + sched.overlap, + next_fire_at, + now, + id, + ], + ) + .map_err(|e| format!("update schedule: {e}"))?; + select_schedule(&conn, &id)?.ok_or_else(|| "update schedule: row missing".to_string()) + }) + .await + } + + pub async fn set_enabled( + &self, + id: &str, + enabled: bool, + now: i64, + ) -> Result { + let conn = Arc::clone(&self.conn); + let id = id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let sched = + select_schedule(&conn, &id)?.ok_or_else(|| format!("unknown schedule '{id}'"))?; + // Enabling resets the slot without catch-up; disabling only stops + // new scheduled admissions. + let next_fire_at = if enabled && !sched.enabled { + now.checked_add(sched.every_secs) + .ok_or_else(|| "timestamp overflow: next_fire_at".to_string())? + } else { + sched.next_fire_at + }; + conn.execute( + "UPDATE schedules SET enabled=?1, next_fire_at=?2, updated_at=?3 WHERE id=?4", + params![i64::from(enabled), next_fire_at, now, id], + ) + .map_err(|e| format!("set enabled: {e}"))?; + select_schedule(&conn, &id)?.ok_or_else(|| "set enabled: row missing".to_string()) + }) + .await + } + + /// Delete the schedule and its occurrence history/queue via cascade. + /// Never cancels already-claimed workflow work; receipts survive. + pub async fn remove_schedule(&self, id: &str) -> Result { + let conn = Arc::clone(&self.conn); + let id = id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let n = conn + .execute("DELETE FROM schedules WHERE id = ?1", params![id]) + .map_err(|e| format!("remove schedule: {e}"))?; + Ok(n > 0) + }) + .await + } + + /// Manual fire: transactionally queue a `manual` occurrence and return + /// its receipt. Bypasses enabled/grace/overlap, never expires, and never + /// moves the schedule clock or creates an engine. + pub async fn enqueue_manual(&self, schedule_id: &str, now: i64) -> Result { + let conn = Arc::clone(&self.conn); + let schedule_id = schedule_id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let sched = select_schedule(&conn, &schedule_id)? + .ok_or_else(|| format!("unknown schedule '{schedule_id}'"))?; + let params = sched + .params_json() + .map_err(|e| format!("schedule '{}' has corrupt params: {e}", sched.name))?; + let spec = + launch_snapshot(&sched.repo_root, &sched.workflow_name, params, &sched.name)?; + let occurrence_id = flare_workflow::WorkflowRunId::new().to_string(); + conn.execute( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,launch_spec,\ + start_deadline_at,dispatch_started_at,run_id,error,created_at,finished_at) \ + VALUES (?1,?2,?3,'manual','queued',?4,NULL,NULL,NULL,NULL,?5,NULL)", + params![occurrence_id, schedule_id, now, spec, now], + ) + .map_err(|e| format!("enqueue manual: {e}"))?; + Ok(FireReceipt { + occurrence_id, + status: "queued".to_string(), + }) + }) + .await + } + + pub async fn due_schedules(&self, now: i64) -> Result, String> { + let conn = Arc::clone(&self.conn); + blocking(move || { + let conn = conn_lock(&conn)?; + let mut stmt = conn + .prepare( + "SELECT id,name,repo_root,workflow_name,params,every_secs,grace_secs,overlap,\ + enabled,next_fire_at,last_fire_at,frozen_error,created_at,updated_at \ + FROM schedules WHERE enabled = 1 AND frozen_error IS NULL \ + AND next_fire_at <= ?1 ORDER BY next_fire_at, id", + ) + .map_err(|e| format!("prepare due: {e}"))?; + let rows = stmt + .query_map(params![now], row_to_schedule) + .map_err(|e| format!("due schedules: {e}"))?; + rows.collect::>>() + .map_err(|e| format!("due schedules: {e}")) + }) + .await + } + + /// One transaction per schedule (§6.2): grace/overlap admission or a + /// terminal skip, then the slot-derived clock advance. The partial unique + /// index is the final duplicate-claim guard. + pub async fn claim_due_slot(&self, schedule_id: &str, now: i64) -> Result { + let conn = Arc::clone(&self.conn); + let schedule_id = schedule_id.to_string(); + blocking(move || { + let mut conn = conn_lock(&conn)?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| format!("begin claim tx: {e}"))?; + let sched = select_schedule(&tx, &schedule_id)?; + let Some(sched) = sched else { + tx.rollback().map_err(|e| e.to_string())?; + return Ok(SlotOutcome::NotDue); + }; + if !sched.enabled || sched.frozen_error.is_some() || sched.next_fire_at > now { + tx.rollback().map_err(|e| e.to_string())?; + return Ok(SlotOutcome::NotDue); + } + let slot = sched.next_fire_at; + // Malformed params or an invalid interval freezes only this + // schedule until an update clears it. + let params = match sched.params_json() { + Ok(p) => p, + Err(e) => { + freeze(&tx, &schedule_id, &format!("frozen: {e}"), now)?; + tx.commit().map_err(|e| format!("commit freeze: {e}"))?; + return Ok(SlotOutcome::Frozen); + } + }; + // Slot-derived advance first: a failed insert or advance rolls + // back the whole transaction, and overflow freezes instead of + // wrapping. + let next_fire_at = match advance_slot(slot, now, sched.every_secs) { + Some(n) => n, + None => { + freeze(&tx, &schedule_id, "frozen: slot advance overflow", now)?; + tx.commit().map_err(|e| format!("commit freeze: {e}"))?; + return Ok(SlotOutcome::Frozen); + } + }; + let grace = sched.effective_grace_secs(); + let outcome = if now.saturating_sub(slot) > grace { + let id = insert_skip(&tx, &schedule_id, slot, "skipped_missed", now)?; + SlotOutcome::SkippedMissed { occurrence_id: id } + } else if sched.overlap == "skip" && has_open(&tx, &schedule_id)? { + let id = insert_skip(&tx, &schedule_id, slot, "skipped_overlap", now)?; + SlotOutcome::SkippedOverlap { occurrence_id: id } + } else { + let deadline = match slot.checked_add(grace) { + Some(d) => d, + None => { + freeze(&tx, &schedule_id, "frozen: start deadline overflow", now)?; + tx.commit().map_err(|e| format!("commit freeze: {e}"))?; + return Ok(SlotOutcome::Frozen); + } + }; + let spec = + launch_snapshot(&sched.repo_root, &sched.workflow_name, params, &sched.name)?; + let occurrence_id = flare_workflow::WorkflowRunId::new().to_string(); + let r = tx.execute( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,launch_spec,\ + start_deadline_at,dispatch_started_at,run_id,error,created_at,finished_at) \ + VALUES (?1,?2,?3,'scheduled','queued',?4,?5,NULL,NULL,NULL,?6,NULL)", + params![occurrence_id, schedule_id, slot, spec, deadline, now], + ); + match r { + Ok(_) => SlotOutcome::Admitted { occurrence_id }, + Err(e) if e.to_string().contains("UNIQUE constraint") => { + // A competing claimant won this slot; it advances the + // clock below exactly once per slot. + SlotOutcome::NotDue + } + Err(e) => return Err(format!("admit occurrence: {e}")), + } + }; + tx.execute( + "UPDATE schedules SET next_fire_at=?1, last_fire_at=?2, updated_at=?3 WHERE id=?4", + params![next_fire_at, slot, now, schedule_id], + ) + .map_err(|e| format!("advance clock: {e}"))?; + tx.commit() + .map_err(|e| format!("commit claim: {e}"))?; + Ok(outcome) + }) + .await + } + + /// Claim every queued occurrence, oldest first: scheduled entries past + /// their initial deadline become `skipped_missed` (manuals never expire), + /// the rest flip to `dispatching` with a fresh attempt timestamp. + /// Already-accepted queued work is never re-admitted. + pub async fn claim_queued(&self, now: i64) -> Result, String> { + let conn = Arc::clone(&self.conn); + blocking(move || { + let mut conn = conn_lock(&conn)?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| format!("begin queue-claim tx: {e}"))?; + let queued: Vec = { + let mut stmt = tx + .prepare( + "SELECT id,schedule_id,slot,trigger,status,launch_spec,start_deadline_at,\ + dispatch_started_at,run_id,error,created_at,finished_at \ + FROM schedule_runs WHERE status = 'queued' ORDER BY created_at, id", + ) + .map_err(|e| format!("prepare queue: {e}"))?; + stmt.query_map([], row_to_occurrence) + .map_err(|e| format!("read queue: {e}"))? + .collect::>>() + .map_err(|e| format!("read queue: {e}"))? + }; + let mut claimed = Vec::new(); + for occ in queued { + let expired = + occ.trigger == "scheduled" && occ.start_deadline_at.is_some_and(|d| now > d); + if expired { + tx.execute( + "UPDATE schedule_runs SET status='skipped_missed', finished_at=?1 \ + WHERE id=?2 AND status='queued'", + params![now, occ.id], + ) + .map_err(|e| format!("expire queued: {e}"))?; + continue; + } + tx.execute( + "UPDATE schedule_runs SET status='dispatching', dispatch_started_at=?1 \ + WHERE id=?2 AND status='queued'", + params![now, occ.id], + ) + .map_err(|e| format!("claim queued: {e}"))?; + let mut occ = occ; + occ.status = "dispatching".to_string(); + occ.dispatch_started_at = Some(now); + claimed.push(occ); + } + tx.commit() + .map_err(|e| format!("commit queue-claim: {e}"))?; + Ok(claimed) + }) + .await + } + + pub async fn get_occurrence(&self, id: &str) -> Result, String> { + let conn = Arc::clone(&self.conn); + let id = id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + select_occurrence(&conn, &id) + }) + .await + } + + pub async fn occurrences_for_schedule( + &self, + schedule_id: &str, + limit: i64, + ) -> Result, String> { + let conn = Arc::clone(&self.conn); + let schedule_id = schedule_id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let mut stmt = conn + .prepare( + "SELECT id,schedule_id,slot,trigger,status,launch_spec,start_deadline_at,\ + dispatch_started_at,run_id,error,created_at,finished_at \ + FROM schedule_runs WHERE schedule_id = ?1 \ + ORDER BY created_at DESC, id DESC LIMIT ?2", + ) + .map_err(|e| format!("prepare runs: {e}"))?; + let rows = stmt + .query_map(params![schedule_id, limit], row_to_occurrence) + .map_err(|e| format!("list runs: {e}"))?; + rows.collect::>>() + .map_err(|e| format!("list runs: {e}")) + }) + .await + } + + /// Every non-terminal occurrence in `status`, oldest first (reconcile scan). + pub async fn list_open(&self, status: &str) -> Result, String> { + let conn = Arc::clone(&self.conn); + let status = status.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let mut stmt = conn + .prepare( + "SELECT id,schedule_id,slot,trigger,status,launch_spec,start_deadline_at,\ + dispatch_started_at,run_id,error,created_at,finished_at \ + FROM schedule_runs WHERE status = ?1 ORDER BY created_at, id", + ) + .map_err(|e| format!("prepare open: {e}"))?; + let rows = stmt + .query_map(params![status], row_to_occurrence) + .map_err(|e| format!("list open: {e}"))?; + rows.collect::>>() + .map_err(|e| format!("list open: {e}")) + }) + .await + } + + /// Idempotent fire linkage: records the run on an open occurrence and + /// moves it to `dispatched`. Never regresses a terminal row. + pub async fn link_run(&self, occurrence_id: &str, run_id: &str) -> Result { + let conn = Arc::clone(&self.conn); + let (occurrence_id, run_id) = (occurrence_id.to_string(), run_id.to_string()); + blocking(move || { + let conn = conn_lock(&conn)?; + let n = conn + .execute( + "UPDATE schedule_runs SET run_id=?1, status='dispatched' \ + WHERE id=?2 AND status IN ('dispatching','dispatched')", + params![run_id, occurrence_id], + ) + .map_err(|e| format!("link run: {e}"))?; + Ok(n > 0) + }) + .await + } + + /// Eager terminal write (driver completion): only a `dispatched` + /// occurrence linked to this run moves; anything else (already terminal, + /// unlinked, concurrently reconciled) is left for the tick fallback. + /// Returns whether a row moved. + pub async fn complete_linked_run( + &self, + run_id: &str, + outcome: &str, + error: Option<&str>, + now: i64, + ) -> Result { + let conn = Arc::clone(&self.conn); + let (run_id, outcome, error) = ( + run_id.to_string(), + outcome.to_string(), + error.map(str::to_string), + ); + blocking(move || { + let conn = conn_lock(&conn)?; + let n = conn + .execute( + "UPDATE schedule_runs SET status=?1, error=?2, finished_at=?3 \ + WHERE run_id=?4 AND status='dispatched'", + params![outcome, error, now, run_id], + ) + .map_err(|e| format!("eager complete: {e}"))?; + Ok(n > 0) + }) + .await + } + + /// Tick-fallback settle: move an open (`dispatching`/`dispatched`) + /// occurrence to a terminal status. Never touches terminal rows. + /// Returns whether a row moved. + pub async fn settle_open( + &self, + occurrence_id: &str, + status: &str, + error: Option<&str>, + now: i64, + ) -> Result { + let conn = Arc::clone(&self.conn); + let (occurrence_id, status, error) = ( + occurrence_id.to_string(), + status.to_string(), + error.map(str::to_string), + ); + blocking(move || { + let conn = conn_lock(&conn)?; + let n = conn + .execute( + "UPDATE schedule_runs SET status=?1, error=?2, finished_at=?3 \ + WHERE id=?4 AND status IN ('dispatching','dispatched')", + params![status, error, now, occurrence_id], + ) + .map_err(|e| format!("settle occurrence: {e}"))?; + Ok(n > 0) + }) + .await + } + + /// Recovery: return a claimed-but-unlinked occurrence to the queue with a + /// cleared attempt timestamp. Returns whether a row moved. + pub async fn mark_queued(&self, occurrence_id: &str) -> Result { + let conn = Arc::clone(&self.conn); + let occurrence_id = occurrence_id.to_string(); + blocking(move || { + let conn = conn_lock(&conn)?; + let n = conn + .execute( + "UPDATE schedule_runs SET status='queued', dispatch_started_at=NULL \ + WHERE id=?1 AND status='dispatching'", + params![occurrence_id], + ) + .map_err(|e| format!("requeue occurrence: {e}"))?; + Ok(n > 0) + }) + .await + } + + /// Definitive pre-start failure (or a claimed attempt lost past grace): + /// terminal, with the reason kept on the retained row. + pub async fn mark_dispatch_failed( + &self, + occurrence_id: &str, + error: &str, + now: i64, + ) -> Result { + let conn = Arc::clone(&self.conn); + let (occurrence_id, error) = (occurrence_id.to_string(), error.to_string()); + blocking(move || { + let conn = conn_lock(&conn)?; + let n = conn + .execute( + "UPDATE schedule_runs SET status='dispatch_failed', error=?1, finished_at=?2 \ + WHERE id=?3 AND status='dispatching'", + params![error, now, occurrence_id], + ) + .map_err(|e| format!("mark dispatch_failed: {e}"))?; + Ok(n > 0) + }) + .await + } + + /// Retention per touched schedule, in its own transaction: newest 100 + /// non-skip terminal rows plus, independently, newest 100 skip rows. + /// Non-terminal rows are never counted or deleted. Returns rows removed. + pub async fn prune_schedule(&self, schedule_id: &str) -> Result { + let conn = Arc::clone(&self.conn); + let schedule_id = schedule_id.to_string(); + blocking(move || { + let mut conn = conn_lock(&conn)?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| format!("begin prune tx: {e}"))?; + let mut removed = 0u64; + for statuses in [TERMINAL_NON_SKIP_STATUSES.as_slice(), SKIP_STATUSES.as_slice()] { + let placeholders = statuses.iter().map(|_| "?").collect::>().join(","); + let sql = format!( + "DELETE FROM schedule_runs WHERE schedule_id = ? AND status IN ({placeholders}) \ + AND id NOT IN (SELECT id FROM schedule_runs WHERE schedule_id = ? \ + AND status IN ({placeholders}) ORDER BY created_at DESC, id DESC LIMIT ?)" + ); + let mut params_vec: Vec<&dyn rusqlite::types::ToSql> = vec![&schedule_id]; + for s in statuses { + params_vec.push(s); + } + params_vec.push(&schedule_id); + for s in statuses { + params_vec.push(s); + } + // Both caps are 100 today; the bound param keeps the shape + // once they diverge. + params_vec.push(&RETENTION_NON_SKIP); + let _ = RETENTION_SKIP; + removed += tx + .execute(&sql, params_vec.as_slice()) + .map_err(|e| format!("prune: {e}"))? as u64; + } + tx.commit() + .map_err(|e| format!("commit prune: {e}"))?; + Ok(removed) + }) + .await + } + + #[cfg(test)] + pub async fn test_exec(&self, sql: String) -> Result { + let conn = Arc::clone(&self.conn); + blocking(move || { + let conn = conn_lock(&conn)?; + conn.execute_batch(&sql) + .map(|_| 0) + .map_err(|e| format!("test exec: {e}")) + }) + .await + } +} + +fn conn_lock( + conn: &Arc>, +) -> Result, String> { + conn.lock().map_err(|e| format!("lock: {e}")) +} + +fn select_occurrence(conn: &Connection, id: &str) -> Result, String> { + conn.query_row( + "SELECT id,schedule_id,slot,trigger,status,launch_spec,start_deadline_at,\ + dispatch_started_at,run_id,error,created_at,finished_at \ + FROM schedule_runs WHERE id = ?1", + params![id], + row_to_occurrence, + ) + .optional() + .map_err(|e| format!("load occurrence: {e}")) +} + +fn freeze(tx: &Connection, schedule_id: &str, msg: &str, now: i64) -> Result<(), String> { + tx.execute( + "UPDATE schedules SET frozen_error=?1, updated_at=?2 WHERE id=?3", + params![msg, now, schedule_id], + ) + .map_err(|e| format!("freeze schedule: {e}"))?; + Ok(()) +} + +/// `next = slot + (floor((now - slot) / every) + 1) * every`, all checked. +/// Never backfills; advances from the original slot. +fn advance_slot(slot: i64, now: i64, every: i64) -> Option { + let k = (now.saturating_sub(slot) / every).checked_add(1)?; + slot.checked_add(k.checked_mul(every)?) +} + +fn has_open(tx: &Connection, schedule_id: &str) -> Result { + tx.query_row( + "SELECT EXISTS(SELECT 1 FROM schedule_runs WHERE schedule_id = ?1 \ + AND status IN ('queued','dispatching','dispatched'))", + params![schedule_id], + |row| row.get::<_, bool>(0), + ) + .map_err(|e| format!("overlap check: {e}")) +} + +fn insert_skip( + tx: &Connection, + schedule_id: &str, + slot: i64, + status: &str, + now: i64, +) -> Result { + let id = flare_workflow::WorkflowRunId::new().to_string(); + tx.execute( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,launch_spec,\ + start_deadline_at,dispatch_started_at,run_id,error,created_at,finished_at) \ + VALUES (?1,?2,?3,'scheduled',?4,NULL,NULL,NULL,NULL,NULL,?5,?5)", + params![id, schedule_id, slot, status, now], + ) + .map_err(|e| format!("insert skip: {e}"))?; + Ok(id) +} diff --git a/src/schedule/tests.rs b/src/schedule/tests.rs new file mode 100644 index 000000000..e245ca950 --- /dev/null +++ b/src/schedule/tests.rs @@ -0,0 +1,741 @@ +//! Scheduler admission/queue/manual tests (item #366 PR-B: T1–T15). Shared +//! harness lives here (`Harness`, mock sends); recovery and lifecycle suites +//! are sibling modules. One temp `workflows.db` per test, injected `now`. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use chrono::{DateTime, Utc}; +use flare_workflow::WorkflowRunId; +use flare_workflow::json::{PipelineData, SendMessage}; +use flare_workflow::types::*; + +use super::store::{CreateInput, UpdateInput}; +use super::{OccurrenceRow, ScheduleRow, Scheduler, start_key_for}; + +pub(super) fn mock_send(counter: Arc) -> SendMessage { + Arc::new(move |inv: flare_workflow::json::StepInvocation| { + let counter = Arc::clone(&counter); + Box::pin(async move { + counter.fetch_add(1, Ordering::SeqCst); + Ok((format!("mock:{}", inv.prompt), 0, 0)) + }) + }) +} + +pub(super) fn capture_send( + counter: Arc, + cwds: Arc>>>, +) -> SendMessage { + Arc::new(move |inv: flare_workflow::json::StepInvocation| { + let (counter, cwds) = (Arc::clone(&counter), Arc::clone(&cwds)); + Box::pin(async move { + counter.fetch_add(1, Ordering::SeqCst); + cwds.lock().unwrap().push(inv.cwd); + Ok(("mock".to_string(), 0, 0)) + }) + }) +} + +pub(super) struct Harness { + pub(super) dir: tempfile::TempDir, + pub(super) sched: Scheduler, +} + +impl Harness { + pub(super) fn new() -> Self { + let counter = Arc::new(AtomicUsize::new(0)); + Self::with_send(mock_send(Arc::clone(&counter))) + } + + pub(super) fn with_send(send: SendMessage) -> Self { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("workflows.db"); + let sched = Scheduler::open(&db, send).unwrap(); + Self { dir, sched } + } + + pub(super) fn repo(&self, name: &str) -> PathBuf { + let p = self.dir.path().join(name); + std::fs::create_dir_all(p.join(".agentflare").join("workflows")).unwrap(); + p + } + + pub(super) fn write_wf(&self, repo: &Path, wf: &str, body: &str) { + std::fs::write( + repo.join(".agentflare") + .join("workflows") + .join(format!("{wf}.json")), + body, + ) + .unwrap(); + } + + pub(super) fn echo_wf(&self, repo: &Path, wf: &str, msg: &str) { + self.write_wf( + repo, + wf, + &format!(r#"{{"name":"{wf}","steps":[{{"name":"s","command":["echo","{msg}"]}}]}}"#), + ); + } + + pub(super) fn canon(&self, repo: &Path) -> String { + std::fs::canonicalize(repo) + .unwrap() + .to_string_lossy() + .into_owned() + } + + pub(super) async fn mk( + &self, + repo: &Path, + name: &str, + wf: &str, + every: i64, + now: i64, + ) -> ScheduleRow { + self.sched + .store() + .create_schedule( + CreateInput { + name: name.into(), + repo_root: self.canon(repo), + workflow_name: wf.into(), + params: None, + every_secs: every, + grace_secs: None, + overlap: "skip".into(), + }, + now, + ) + .await + .unwrap() + } + + pub(super) fn at(&self, base: i64, off: i64) -> DateTime { + DateTime::from_timestamp(base + off, 0).unwrap() + } + + /// Tick + converge spawned fires (links land; drivers keep running). + pub(super) async fn tick(&self, now: DateTime) -> super::TickSummary { + let s = self.sched.tick_once(now).await.unwrap(); + self.sched.drain_fires().await; + s + } + + pub(super) async fn occ(&self, id: &str) -> OccurrenceRow { + self.sched + .store() + .get_occurrence(id) + .await + .unwrap() + .unwrap() + } + + pub(super) async fn wait_occ(&self, id: &str) -> OccurrenceRow { + let end = std::time::Instant::now() + std::time::Duration::from_secs(15); + loop { + let o = self.occ(id).await; + if o.is_terminal() { + return o; + } + assert!( + std::time::Instant::now() < end, + "occurrence {id} never settled" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + } + + /// Craft a driverless run + receipt under the occurrence's start key. + pub(super) async fn craft_run( + &self, + occ_id: &str, + status: WorkflowStatus, + lease_ago_secs: Option, + ) -> WorkflowRunId { + let run = WorkflowRunId::new(); + let mut s = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + s.status = status; + s.lease = lease_ago_secs.map(|ago| ExecutorLease { + owner: "dead:1:x".into(), + renewed_at: Utc::now() - chrono::Duration::seconds(ago), + }); + self.sched + .engine + .state_store() + .create_start_once( + start_key_for(occ_id), + s, + JournalEntry::Input { value: vec![] }, + ) + .await + .unwrap(); + run + } + + pub(super) async fn backdate_dispatch(&self, occ_id: &str, ts: i64) { + self.sched + .store() + .test_exec(format!( + "UPDATE schedule_runs SET dispatch_started_at = {ts} WHERE id = '{occ_id}';" + )) + .await + .unwrap(); + } +} + +/// T1: due/disabled/frozen filtering. +#[tokio::test] +async fn t01_due_disabled_frozen_filtering() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let due = h.mk(&repo, "due", "w", 300, t0).await; + let mut dis = h.mk(&repo, "dis", "w", 300, t0).await; + dis = h + .sched + .store() + .set_enabled(&dis.id, false, t0) + .await + .unwrap(); + assert!(!dis.enabled); + let frozen = h.mk(&repo, "froz", "w", 300, t0).await; + h.sched + .store() + .test_exec(format!( + "UPDATE schedules SET frozen_error = 'bad' WHERE id = '{}';", + frozen.id + )) + .await + .unwrap(); + let future = h.mk(&repo, "future", "w", 300, t0 + 10_000).await; + let _ = future; + let due_list = h.sched.store().due_schedules(t0 + 310).await.unwrap(); + assert_eq!( + due_list.iter().map(|s| s.name.clone()).collect::>(), + vec!["due"] + ); + let _ = due; +} + +/// T2: within grace admits exactly one occurrence; `last_fire_at == slot`. +#[tokio::test] +async fn t02_within_grace_admits_once_with_slot_last_fire() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let slot = t0 + 300; + let out = h + .sched + .store() + .claim_due_slot(&s.id, slot + 10) + .await + .unwrap(); + let id = match out { + super::store::SlotOutcome::Admitted { occurrence_id } => occurrence_id, + o => panic!("expected admission, got {o:?}"), + }; + let s2 = h.sched.store().get_by_id(&s.id).await.unwrap().unwrap(); + assert_eq!(s2.last_fire_at, Some(slot)); + assert_eq!(s2.next_fire_at, slot + 300); + let occ = h.occ(&id).await; + assert_eq!( + (occ.slot, occ.trigger.as_str(), occ.status.as_str()), + (slot, "scheduled", "queued") + ); + // Exactly one: clock advanced past the slot. + assert!(matches!( + h.sched + .store() + .claim_due_slot(&s.id, slot + 10) + .await + .unwrap(), + super::store::SlotOutcome::NotDue + )); + assert_eq!( + h.sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap() + .len(), + 1 + ); +} + +/// T3: 40 s late on a 300 s schedule keeps the original grid. +#[tokio::test] +async fn t03_late_tick_keeps_original_grid() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + h.sched + .store() + .claim_due_slot(&s.id, t0 + 340) + .await + .unwrap(); + let s2 = h.sched.store().get_by_id(&s.id).await.unwrap().unwrap(); + assert_eq!(s2.next_fire_at, t0 + 600); + assert_eq!(s2.last_fire_at, Some(t0 + 300)); +} + +/// T4: two-slot gap within explicit grace admits once, no backfill. +#[tokio::test] +async fn t04_gap_within_grace_admits_once_no_backfill() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let mut s = h.mk(&repo, "s", "w", 300, t0).await; + s = h + .sched + .store() + .update_schedule( + &s.id, + UpdateInput { + grace_secs: Some(Some(700)), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + assert_eq!(s.effective_grace_secs(), 700); + h.sched + .store() + .claim_due_slot(&s.id, t0 + 900) + .await + .unwrap(); + let rows = h + .sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].slot, t0 + 300); + let s2 = h.sched.store().get_by_id(&s.id).await.unwrap().unwrap(); + assert_eq!(s2.next_fire_at, t0 + 1200); +} + +/// T5: grace-boundary admission + overflow rejection. +#[tokio::test] +async fn t05_grace_boundaries_and_overflow_rejection() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + // Exactly at grace: admitted; one past: skipped. + let a = h.mk(&repo, "a", "w", 300, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&a.id, t0 + 600) + .await + .unwrap(); + assert!(matches!(out, super::store::SlotOutcome::Admitted { .. })); + let b = h.mk(&repo, "b", "w", 300, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&b.id, t0 + 601) + .await + .unwrap(); + assert!(matches!( + out, + super::store::SlotOutcome::SkippedMissed { .. } + )); + // Overflow rejected at create and update. + let bad = h + .sched + .store() + .create_schedule( + CreateInput { + name: "huge".into(), + repo_root: h.canon(&repo), + workflow_name: "w".into(), + params: None, + every_secs: i64::MAX, + grace_secs: None, + overlap: "skip".into(), + }, + t0, + ) + .await; + assert!(bad.is_err()); + let err = h + .sched + .store() + .update_schedule( + &a.id, + UpdateInput { + every_secs: Some(i64::MAX), + ..Default::default() + }, + t0, + ) + .await + .unwrap_err(); + assert!(err.contains("overflow")); +} + +/// T6: default grace + 2 h outage → `skipped_missed`. +#[tokio::test] +async fn t06_default_grace_outage_skips() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + assert_eq!(s.effective_grace_secs(), 300); + let out = h + .sched + .store() + .claim_due_slot(&s.id, t0 + 300 + 7200) + .await + .unwrap(); + let id = match out { + super::store::SlotOutcome::SkippedMissed { occurrence_id } => occurrence_id, + o => panic!("got {o:?}"), + }; + let occ = h.occ(&id).await; + assert!(occ.finished_at.is_some() && occ.launch_spec.is_none()); +} + +/// T7: explicit 4 h grace fires once after the same outage. +#[tokio::test] +async fn t07_explicit_grace_fires_once_after_outage() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + h.sched + .store() + .update_schedule( + &s.id, + UpdateInput { + grace_secs: Some(Some(14400)), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + let out = h + .sched + .store() + .claim_due_slot(&s.id, t0 + 300 + 7200) + .await + .unwrap(); + assert!(matches!(out, super::store::SlotOutcome::Admitted { .. })); + assert_eq!( + h.sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap() + .len(), + 1 + ); +} + +/// T8: grace=0 admits at the exact slot, skips any later. +#[tokio::test] +async fn t08_zero_grace_exact_slot_only() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let a = h.mk(&repo, "a", "w", 300, t0).await; + h.sched + .store() + .update_schedule( + &a.id, + UpdateInput { + grace_secs: Some(Some(0)), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + let out = h + .sched + .store() + .claim_due_slot(&a.id, t0 + 300) + .await + .unwrap(); + assert!(matches!(out, super::store::SlotOutcome::Admitted { .. })); + let b = h.mk(&repo, "b", "w", 300, t0).await; + h.sched + .store() + .update_schedule( + &b.id, + UpdateInput { + grace_secs: Some(Some(0)), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + let out = h + .sched + .store() + .claim_due_slot(&b.id, t0 + 301) + .await + .unwrap(); + assert!(matches!( + out, + super::store::SlotOutcome::SkippedMissed { .. } + )); +} + +/// T9: one open occurrence blocks three subsequent slots (skips accumulate). +#[tokio::test] +async fn t09_open_occurrence_blocks_three_slots() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 60, t0).await; + let slot = t0 + 60; + assert!(matches!( + h.sched.store().claim_due_slot(&s.id, slot).await.unwrap(), + super::store::SlotOutcome::Admitted { .. } + )); + for i in 1..=3 { + assert!( + matches!( + h.sched + .store() + .claim_due_slot(&s.id, slot + 60 * i) + .await + .unwrap(), + super::store::SlotOutcome::SkippedOverlap { .. } + ), + "slot {i} must skip on overlap" + ); + } + let rows = h + .sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap(); + assert_eq!(rows.iter().filter(|r| r.status == "queued").count(), 1); + assert_eq!( + rows.iter() + .filter(|r| r.status == "skipped_overlap") + .count(), + 3 + ); +} + +/// T10: queued, unlinked dispatching, and dispatched runs each block overlap. +#[tokio::test] +async fn t10_each_open_state_blocks_overlap() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + // Queued blocks. + let q = h.mk(&repo, "q", "w", 60, t0).await; + h.sched + .store() + .claim_due_slot(&q.id, t0 + 60) + .await + .unwrap(); + assert!(matches!( + h.sched + .store() + .claim_due_slot(&q.id, t0 + 120) + .await + .unwrap(), + super::store::SlotOutcome::SkippedOverlap { .. } + )); + // Unlinked dispatching blocks. + let d = h.mk(&repo, "d", "w", 60, t0).await; + h.sched + .store() + .claim_due_slot(&d.id, t0 + 60) + .await + .unwrap(); + h.sched.store().claim_queued(t0 + 60).await.unwrap(); + assert!(matches!( + h.sched + .store() + .claim_due_slot(&d.id, t0 + 120) + .await + .unwrap(), + super::store::SlotOutcome::SkippedOverlap { .. } + )); + // Dispatched (linked, e.g. paused) blocks. + let p = h.mk(&repo, "p", "w", 60, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&p.id, t0 + 60) + .await + .unwrap(); + let occ_id = match out { + super::store::SlotOutcome::Admitted { occurrence_id } => occurrence_id, + o => panic!("got {o:?}"), + }; + h.sched + .store() + .link_run(&occ_id, "bogus-run") + .await + .unwrap(); + assert!(matches!( + h.sched + .store() + .claim_due_slot(&p.id, t0 + 120) + .await + .unwrap(), + super::store::SlotOutcome::SkippedOverlap { .. } + )); +} + +/// T11: `overlap=allow` admits additional occurrences. +#[tokio::test] +async fn t11_overlap_allow_admits() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let mut s = h.mk(&repo, "s", "w", 60, t0).await; + s = h + .sched + .store() + .update_schedule( + &s.id, + UpdateInput { + overlap: Some("allow".into()), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + assert_eq!(s.overlap, "allow"); + for i in 0..3 { + assert!(matches!( + h.sched + .store() + .claim_due_slot(&s.id, t0 + 60 * (i + 1)) + .await + .unwrap(), + super::store::SlotOutcome::Admitted { .. } + )); + } +} + +/// T12: two concurrent queue claimers select one queued occurrence once. +#[tokio::test] +async fn t12_concurrent_queue_claim_single_winner() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut tasks = Vec::new(); + for _ in 0..2 { + let (store, barrier) = (h.sched.store().clone(), Arc::clone(&barrier)); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + store.claim_queued(t0).await.unwrap().len() + })); + } + let mut total = 0; + for t in tasks { + total += t.await.unwrap(); + } + assert_eq!(total, 1); +} + +/// T13: repeated manual fires in the same second get distinct IDs. +#[tokio::test] +async fn t13_manual_fires_same_second_distinct_ids() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let mut ids = Vec::new(); + for _ in 0..3 { + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + assert_eq!(r.status, "queued"); + ids.push(r.occurrence_id); + } + assert_eq!(ids.len(), 3); + let uniq: std::collections::HashSet<_> = ids.iter().collect(); + assert_eq!(uniq.len(), 3); +} + +/// T14: manual fire bypasses disabled/grace/overlap and preserves the clock. +#[tokio::test] +async fn t14_manual_bypasses_checks_preserves_clock() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + // Open occurrence (overlap) + disable + far past grace. + h.sched + .store() + .claim_due_slot(&s.id, t0 + 300) + .await + .unwrap(); + h.sched + .store() + .set_enabled(&s.id, false, t0 + 300) + .await + .unwrap(); + let before = h.sched.store().get_by_id(&s.id).await.unwrap().unwrap(); + let r = h + .sched + .store() + .enqueue_manual(&s.id, t0 + 10_000) + .await + .unwrap(); + let occ = h.occ(&r.occurrence_id).await; + assert_eq!( + (occ.trigger.as_str(), occ.status.as_str()), + ("manual", "queued") + ); + assert_eq!(occ.slot, t0 + 10_000); + assert!(occ.start_deadline_at.is_none()); + let after = h.sched.store().get_by_id(&s.id).await.unwrap().unwrap(); + assert_eq!( + (after.next_fire_at, after.last_fire_at), + (before.next_fire_at, before.last_fire_at) + ); +} + +/// T15: daemon-offline manual request survives delay+grace, executes at tick. +#[tokio::test] +async fn t15_offline_manual_survives_and_executes() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hello-t15"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + // Far past the 120 s recovery delay and the schedule grace. + let summary = h.tick(h.at(t0, 5000)).await; + assert!(summary.fired >= 1); + let occ = h.wait_occ(&r.occurrence_id).await; + assert_eq!(occ.status, "completed"); + assert!(occ.run_id.is_some()); +} diff --git a/src/schedule/tests_lifecycle.rs b/src/schedule/tests_lifecycle.rs new file mode 100644 index 000000000..deb4ca62b --- /dev/null +++ b/src/schedule/tests_lifecycle.rs @@ -0,0 +1,601 @@ +//! Scheduler execution-context/lifecycle tests (item #366 PR-B: T36–T44, +//! T49, T49b). Command workflows execute for real (no agent binary needed); +//! agent steps run against capturing mock sends. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use chrono::Utc; +use flare_workflow::json::PipelineData; +use flare_workflow::types::*; +use flare_workflow::{StateStore, WorkflowRunId}; + +use super::store::{CreateInput, UpdateInput}; +use super::tests::{Harness, capture_send, mock_send}; +use super::{OccurrenceRow, start_key_for}; + +/// T36: execution lands in the schedule's repo — command cwd and agent cwd. +#[tokio::test] +async fn t36_execution_runs_in_schedule_repo() { + let cwds = Arc::new(std::sync::Mutex::new(Vec::new())); + let counter = Arc::new(AtomicUsize::new(0)); + let h = Harness::with_send(capture_send(Arc::clone(&counter), cwds.clone())); + // No process-wide chdir anywhere: the daemon's ambient cwd is wherever + // the test runner sits, and execution must still land in repo_b. + let repo_b = h.repo("b"); + h.write_wf( + &repo_b, + "w", + r#"{"name":"w","steps":[{"name":"s","command":["pwd"]}]}"#, + ); + h.write_wf( + &repo_b, + "ag", + r#"{"name":"ag","steps":[{"name":"s","agent":"a","prompt":"hi"}]}"#, + ); + let t0 = Utc::now().timestamp(); + // Command step output is the schedule's repo, not the daemon's cwd. + let s = h.mk(&repo_b, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.tick(h.at(t0, 1)).await; + let occ = h.wait_occ(&r.occurrence_id).await; + assert_eq!(occ.status, "completed"); + let run_id: WorkflowRunId = occ.run_id.unwrap().parse().unwrap(); + let state = h.sched.engine.state_store().load(run_id).await.unwrap(); + let out = state.output.unwrap_or_default(); + assert_eq!( + std::fs::canonicalize(out.trim()).unwrap(), + std::fs::canonicalize(&repo_b).unwrap() + ); + // Agent steps receive the same cwd in the invocation. + let sa = h.mk(&repo_b, "sa", "ag", 300, t0).await; + let ra = h.sched.store().enqueue_manual(&sa.id, t0).await.unwrap(); + h.tick(h.at(t0, 2)).await; + h.wait_occ(&ra.occurrence_id).await; + let expected = std::fs::canonicalize(&repo_b).unwrap(); + let got = cwds.lock().unwrap().clone(); + assert!(!got.is_empty()); + assert!(got.iter().all(|c| c.as_ref() == Some(&expected))); + assert!(counter.load(Ordering::SeqCst) > 0); +} + +/// T37: same workflow name in two repos executes each repo's own file; +/// concurrent same-schedule starts share one definition registration. +#[tokio::test] +async fn t37_repos_stay_isolated_and_concurrent_starts_share_def() { + let h = Harness::new(); + let repo_a = h.repo("a"); + let repo_b = h.repo("b"); + h.echo_wf(&repo_a, "nightly", "OUT-A"); + h.echo_wf(&repo_b, "nightly", "OUT-B"); + let t0 = Utc::now().timestamp(); + let sa = h.mk(&repo_a, "sa", "nightly", 300, t0).await; + let sb = h.mk(&repo_b, "nightly", "nightly", 300, t0).await; + let ra = h.sched.store().enqueue_manual(&sa.id, t0).await.unwrap(); + let rb = h.sched.store().enqueue_manual(&sb.id, t0).await.unwrap(); + h.tick(h.at(t0, 1)).await; + let (oa, ob) = ( + h.wait_occ(&ra.occurrence_id).await, + h.wait_occ(&rb.occurrence_id).await, + ); + assert_eq!( + (oa.status.as_str(), ob.status.as_str()), + ("completed", "completed") + ); + let out = |o: &OccurrenceRow| { + let run: WorkflowRunId = o.run_id.as_ref().unwrap().parse().unwrap(); + run + }; + let sta = h.sched.engine.state_store().load(out(&oa)).await.unwrap(); + let stb = h.sched.engine.state_store().load(out(&ob)).await.unwrap(); + assert_eq!(sta.output.as_deref(), Some("OUT-A")); + assert_eq!(stb.output.as_deref(), Some("OUT-B")); + // Concurrent same-schedule starts: two occurrences, two runs, one output. + h.sched + .store() + .update_schedule( + &sa.id, + UpdateInput { + overlap: Some("allow".into()), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + let r1 = h.sched.store().enqueue_manual(&sa.id, t0).await.unwrap(); + let r2 = h.sched.store().enqueue_manual(&sa.id, t0).await.unwrap(); + h.tick(h.at(t0, 2)).await; + let (o1, o2) = ( + h.wait_occ(&r1.occurrence_id).await, + h.wait_occ(&r2.occurrence_id).await, + ); + assert_eq!( + (o1.status.as_str(), o2.status.as_str()), + ("completed", "completed") + ); + assert_ne!(o1.run_id, o2.run_id); +} + +/// T38: missing/invalid workflow at execution → `dispatch_failed`, enabled. +#[tokio::test] +async fn t38_bad_definition_at_execution_fails_open() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + std::fs::remove_file(repo.join(".agentflare").join("workflows").join("w.json")).unwrap(); + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.tick(h.at(t0, 1)).await; + let occ = h.wait_occ(&r.occurrence_id).await; + assert_eq!(occ.status, "dispatch_failed"); + assert!( + h.sched + .store() + .get_by_id(&s.id) + .await + .unwrap() + .unwrap() + .enabled + ); +} + +/// T39: params corruption freezes only that schedule; update clears it. +#[tokio::test] +async fn t39_corrupt_params_freeze_and_update_clears() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let bad = h.mk(&repo, "bad", "w", 300, t0).await; + let good = h.mk(&repo, "good", "w", 300, t0).await; + h.sched + .store() + .test_exec(format!( + "UPDATE schedules SET params = '{{oops' WHERE id = '{}';", + bad.id + )) + .await + .unwrap(); + let summary = h.tick(h.at(t0, 310)).await; + h.sched.drain_fires().await; + assert_eq!(summary.admitted, 1); + assert!(summary.errors.iter().any(|e| e.contains("frozen"))); + let bad_row = h.sched.store().get_by_id(&bad.id).await.unwrap().unwrap(); + assert!(bad_row.frozen_error.is_some()); + assert!( + h.sched + .store() + .occurrences_for_schedule(&bad.id, 10) + .await + .unwrap() + .is_empty() + ); + // Only that schedule froze. + assert_eq!( + h.sched + .store() + .occurrences_for_schedule(&good.id, 10) + .await + .unwrap() + .len(), + 1 + ); + // Update clears the freeze and admission resumes. + h.sched + .store() + .update_schedule( + &bad.id, + UpdateInput { + params: Some(serde_json::json!({})), + ..Default::default() + }, + t0 + 310, + ) + .await + .unwrap(); + let bad_row = h.sched.store().get_by_id(&bad.id).await.unwrap().unwrap(); + assert!(bad_row.frozen_error.is_none()); + let summary = h.tick(h.at(t0, 320)).await; + h.sched.drain_fires().await; + assert!(summary.admitted >= 1); + assert!( + !h.sched + .store() + .occurrences_for_schedule(&bad.id, 10) + .await + .unwrap() + .is_empty() + ); +} + +/// T40: retention keeps newest 100 + 100, spares live rows and receipts. +#[tokio::test] +async fn t40_retention_caps_with_separate_skip_budget() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 60, t0).await; + let mut batch = String::new(); + for i in 0..150 { + batch.push_str(&format!( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,created_at,finished_at) \ + VALUES ('t{i}','{}',{i},'scheduled','completed',{i},{i});", + s.id + )); + } + for i in 0..150 { + let slot = 1000 + i; + batch.push_str(&format!( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,created_at,finished_at) \ + VALUES ('k{i}','{}',{slot},'scheduled','skipped_overlap',{slot},{slot});", + s.id + )); + } + for i in 0..5 { + let slot = 2000 + i; + batch.push_str(&format!( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,created_at) \ + VALUES ('live{i}','{}',{slot},'manual','dispatched',{slot});", + s.id + )); + } + for i in 0..3 { + batch.push_str(&format!( + "INSERT INTO workflow_start_receipts (start_key, run_id, created_at) \ + VALUES ('rk{i}', '00000000-0000-7000-8000-00000000000{i}', {i});" + )); + } + h.sched.store().test_exec(batch).await.unwrap(); + let removed = h.sched.store().prune_schedule(&s.id).await.unwrap(); + assert_eq!(removed, 100); + let rows = h + .sched + .store() + .occurrences_for_schedule(&s.id, 500) + .await + .unwrap(); + assert_eq!(rows.iter().filter(|r| r.status == "completed").count(), 100); + assert_eq!( + rows.iter() + .filter(|r| r.status == "skipped_overlap") + .count(), + 100 + ); + assert_eq!(rows.iter().filter(|r| r.status == "dispatched").count(), 5); + for i in 0..3 { + assert!( + h.sched + .engine + .state_store() + .lookup_start(&format!("rk{i}")) + .await + .unwrap() + .is_some() + ); + } + // Blocked 60 s schedule: new skips still land under the same cap. + let run = h + .craft_run("paused-occ", WorkflowStatus::Paused, None) + .await; + h.sched + .store() + .test_exec(format!( + "INSERT INTO schedule_runs (id,schedule_id,slot,trigger,status,run_id,created_at) \ + VALUES ('paused-occ','{}',{t0},'scheduled','dispatched','{run}',{t0});", + s.id, + )) + .await + .unwrap(); + h.tick(h.at(t0, 61)).await; + h.sched.drain_fires().await; + let rows = h + .sched + .store() + .occurrences_for_schedule(&s.id, 500) + .await + .unwrap(); + assert_eq!( + rows.iter() + .filter(|r| r.status == "skipped_overlap") + .count(), + 100 + ); +} + +/// T41: migration upgrades a previous-list database without losing rows. +#[tokio::test] +async fn t41_migration_upgrades_without_losing_rows() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("old.db"); + let run = WorkflowRunId::new(); + let mut state = WorkflowState::new(run, WorkflowId::new("w"), PipelineData); + state.status = WorkflowStatus::Running; + state.input = "hello".into(); + let state_json = serde_json::to_string(&state).unwrap(); + let entry = serde_json::to_string(&JournalEntry::Input { + value: b"in".to_vec(), + }) + .unwrap(); + { + let conn = rusqlite::Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE workflow_runs (id TEXT PRIMARY KEY, workflow_id TEXT NOT NULL, \ + status TEXT NOT NULL, current_step TEXT, input TEXT, output TEXT, error TEXT, \ + state_json TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL); + CREATE TABLE journal (run_id TEXT NOT NULL, seq INTEGER NOT NULL, \ + entry_type TEXT NOT NULL, payload TEXT NOT NULL, PRIMARY KEY (run_id, seq)); + CREATE TABLE step_state (run_id TEXT NOT NULL, step_id TEXT NOT NULL, \ + status TEXT NOT NULL, attempt INTEGER, last_error TEXT, started_at TEXT, \ + completed_at TEXT, duration_ms INTEGER, input_tokens INTEGER, \ + output_tokens INTEGER, PRIMARY KEY (run_id, step_id)); + CREATE TABLE run_vars (run_id TEXT NOT NULL, key TEXT NOT NULL, value TEXT, \ + PRIMARY KEY (run_id, key)); + PRAGMA user_version = 5;", + ) + .unwrap(); + conn.execute( + "INSERT INTO workflow_runs (id, workflow_id, status, input, state_json, created_at, updated_at) \ + VALUES (?1, 'w', 'running', 'hello', ?2, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", + rusqlite::params![run.to_string(), state_json], + ) + .unwrap(); + conn.execute( + "INSERT INTO journal (run_id, seq, entry_type, payload) VALUES (?1, 1, 'input', ?2)", + rusqlite::params![run.to_string(), entry], + ) + .unwrap(); + } + // Opening migrates (schedule tables land) and old rows survive. + let sched_store = super::ScheduleStore::open(&path).unwrap(); + let wf_store = + flare_workflow::sqlite_store::SqliteStore::::open_file( + &path, + ) + .unwrap(); + let loaded = wf_store.load(run).await.unwrap(); + assert_eq!(loaded.input, "hello"); + assert_eq!(wf_store.journal(run).await.unwrap().len(), 1); + assert!(sched_store.due_schedules(0).await.unwrap().is_empty()); +} + +/// T42: removal cascades history/queue, keeps receipts, cancels nothing. +#[tokio::test] +async fn t42_removal_cascades_but_keeps_receipts_and_runs() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.tick(h.at(t0, 1)).await; + h.sched.drain_fires().await; + // Wait for linkage (dispatched) — driver may still be running. + let end = std::time::Instant::now() + std::time::Duration::from_secs(15); + let occ = loop { + let o = h.occ(&r.occurrence_id).await; + if o.run_id.is_some() { + break o; + } + assert!(std::time::Instant::now() < end); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + }; + let run: WorkflowRunId = occ.run_id.clone().unwrap().parse().unwrap(); + assert!(h.sched.store().remove_schedule(&s.id).await.unwrap()); + assert!( + h.sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + h.sched + .engine + .state_store() + .lookup_start(&start_key_for(&r.occurrence_id)) + .await + .unwrap(), + Some(run) + ); + let state = h.sched.engine.state_store().load(run).await.unwrap(); + assert!(!state.status.is_terminal() || state.status != WorkflowStatus::Cancelled); +} + +/// T43: updates preserve accepted snapshots; enable resets without catch-up. +#[tokio::test] +async fn t43_update_preserves_snapshots_enable_resets() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "v1", "OUT-1"); + h.echo_wf(&repo, "v2", "OUT-2"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "v1", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched + .store() + .update_schedule( + &s.id, + UpdateInput { + workflow_name: Some("v2".into()), + params: Some(serde_json::json!({"x": 1})), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + // Accepted occurrence still carries the v1 snapshot. + let spec = h.occ(&r.occurrence_id).await.parsed_launch_spec().unwrap(); + assert_eq!(spec.workflow_name, "v1"); + // Disable, then enable: next resets to now+every with no catch-up. + h.sched + .store() + .set_enabled(&s.id, false, t0 + 10) + .await + .unwrap(); + h.sched + .store() + .set_enabled(&s.id, true, t0 + 1000) + .await + .unwrap(); + let s2 = h.sched.store().get_by_id(&s.id).await.unwrap().unwrap(); + assert_eq!(s2.next_fire_at, t0 + 1300); + h.tick(h.at(t0, 1001)).await; + let rows = h + .sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap(); + assert!(rows.iter().all(|o| o.trigger != "scheduled")); +} + +/// T44: create/update validation matrix. +#[tokio::test] +async fn t44_validation_matrix() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let base = || CreateInput { + name: "ok".into(), + repo_root: h.canon(&repo), + workflow_name: "w".into(), + params: None, + every_secs: 300, + grace_secs: None, + overlap: "skip".into(), + }; + let bad = |mut f: Box| { + let mut c = base(); + f(&mut c); + c + }; + for c in [ + bad(Box::new(|c| c.name = "".into())), + bad(Box::new(|c| c.name = "a/b".into())), + bad(Box::new(|c| c.name = "has space".into())), + bad(Box::new(|c| c.name = "x".repeat(65))), + bad(Box::new(|c| c.every_secs = 59)), + bad(Box::new(|c| c.grace_secs = Some(-1))), + bad(Box::new(|c| c.overlap = "sometimes".into())), + bad(Box::new(|c| c.params = Some(serde_json::json!([1])))), + bad(Box::new(|c| { + c.params = Some(serde_json::json!({"schedule": 1})) + })), + bad(Box::new(|c| c.workflow_name = "missing".into())), + bad(Box::new(|c| c.repo_root = "/does/not/exist-xyz".into())), + ] { + assert!(h.sched.store().create_schedule(c, t0).await.is_err()); + } + let ok = h.sched.store().create_schedule(base(), t0).await.unwrap(); + assert_eq!(ok.next_fire_at, t0 + 300); + // Duplicate name in the same repo is rejected. + assert!(h.sched.store().create_schedule(base(), t0).await.is_err()); + // Bad updates rejected; good update applies. + assert!( + h.sched + .store() + .update_schedule( + &ok.id, + UpdateInput { + every_secs: Some(10), + ..Default::default() + }, + t0 + ) + .await + .is_err() + ); + let ok2 = h + .sched + .store() + .update_schedule( + &ok.id, + UpdateInput { + every_secs: Some(600), + ..Default::default() + }, + t0, + ) + .await + .unwrap(); + assert_eq!((ok2.every_secs, ok2.next_fire_at), (600, t0 + 600)); +} + +/// T49: scheduler instances are independent — never a shared static engine/db. +#[tokio::test] +async fn t49_scheduler_instances_are_independent() { + let a = Harness::new(); + let b = Harness::new(); + let repo = a.repo("r"); + a.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = a.mk(&repo, "s", "w", 300, t0).await; + assert!(b.sched.store().get_by_id(&s.id).await.unwrap().is_none()); + assert_ne!( + a.dir.path().join("workflows.db"), + b.dir.path().join("workflows.db") + ); +} + +/// T49b: the ticker fires schedules on its own, without work-item recovery. +#[tokio::test] +async fn t49b_ticker_runs_independently() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("workflows.db"); + let repo = dir.path().join("r"); + std::fs::create_dir_all(repo.join(".agentflare").join("workflows")).unwrap(); + std::fs::write( + repo.join(".agentflare").join("workflows").join("w.json"), + r#"{"name":"w","steps":[{"name":"s","command":["echo","tick"]}]}"#, + ) + .unwrap(); + let store = super::ScheduleStore::open(&db).unwrap(); + let t0 = Utc::now().timestamp(); + let canon = std::fs::canonicalize(&repo) + .unwrap() + .to_string_lossy() + .into_owned(); + let s = store + .create_schedule( + CreateInput { + name: "s".into(), + repo_root: canon, + workflow_name: "w".into(), + params: None, + every_secs: 300, + grace_secs: None, + overlap: "skip".into(), + }, + t0 - 290, + ) + .await + .unwrap(); + // Due ~10 s after open; 1 s test ticks keep this fast. + let counter = Arc::new(AtomicUsize::new(0)); + let handle = crate::dashboard::server::dispatch_boot::spawn_schedule_ticker_every( + db.clone(), + mock_send(counter), + 1, + ); + let end = std::time::Instant::now() + std::time::Duration::from_secs(20); + let occ_id = loop { + let rows = store.occurrences_for_schedule(&s.id, 10).await.unwrap(); + if let Some(o) = rows.iter().find(|o| o.is_terminal()) { + break o.id.clone(); + } + assert!( + std::time::Instant::now() < end, + "ticker never fired the schedule" + ); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + }; + handle.abort(); + let occ = store.get_occurrence(&occ_id).await.unwrap().unwrap(); + assert_eq!(occ.status, "completed"); +} diff --git a/src/schedule/tests_recovery.rs b/src/schedule/tests_recovery.rs new file mode 100644 index 000000000..d7ef4eb83 --- /dev/null +++ b/src/schedule/tests_recovery.rs @@ -0,0 +1,568 @@ +//! Scheduler recovery/reconciliation tests (item #366 PR-B: T18–T23, +//! T25–T26, T25b, T27–T32). Recovery paths drive `reconcile_*` directly for +//! determinism; the harness comes from the sibling `tests` module. + +use chrono::Utc; +use flare_workflow::StateStore; +use flare_workflow::types::*; + +use super::start_key_for; +use super::tests::Harness; + +/// T18 (scheduler half): old dispatch with a receipt links, never retries. +#[tokio::test] +async fn t18_old_dispatch_with_receipt_links_without_retry() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + // Fire held 500 s (past the 120 s delay) with a live receipt behind it. + h.backdate_dispatch(&r.occurrence_id, t0 - 500).await; + let run = h + .craft_run(&r.occurrence_id, WorkflowStatus::Running, Some(0)) + .await; + let occ = h.occ(&r.occurrence_id).await; + // Links (fresh lease → left running, so no state transition to count). + h.sched + .reconcile_dispatching(&occ, h.at(t0, 10)) + .await + .unwrap(); + let occ = h.occ(&r.occurrence_id).await; + assert_eq!(occ.status, "dispatched"); + assert_eq!(occ.run_id, Some(run.to_string())); + // No retry, no duplicate: exactly one run, nothing queued. + assert_eq!( + h.sched.engine.state_store().list_all().await.unwrap().len(), + 1 + ); + assert!( + h.sched + .store() + .list_open("queued") + .await + .unwrap() + .is_empty() + ); +} + +/// T19 (scheduler half): lost linkage recovered from the receipt, no restart. +#[tokio::test] +async fn t19_lost_linkage_recovers_from_receipt() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + // Run started + completed under the occurrence's key, never linked. + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run = h + .craft_run(&r.occurrence_id, WorkflowStatus::Completed, None) + .await; + let occ = h.occ(&r.occurrence_id).await; + let moved = h + .sched + .reconcile_dispatching(&occ, h.at(t0, 500)) + .await + .unwrap(); + assert!(moved); + let occ = h.occ(&r.occurrence_id).await; + assert_eq!(occ.status, "completed"); + assert_eq!(occ.run_id, Some(run.to_string())); + assert_eq!( + h.sched.engine.state_store().list_all().await.unwrap().len(), + 1 + ); +} + +/// T20: completed run evicted before linkage → `evicted`, receipt kept, idle. +#[tokio::test] +async fn t20_evicted_before_linkage_marks_evicted() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run = h + .craft_run(&r.occurrence_id, WorkflowStatus::Completed, None) + .await; + h.sched.engine.state_store().delete(run).await.unwrap(); + let occ = h.occ(&r.occurrence_id).await; + let moved = h + .sched + .reconcile_dispatching(&occ, h.at(t0, 500)) + .await + .unwrap(); + assert!(moved); + let occ = h.occ(&r.occurrence_id).await; + assert_eq!(occ.status, "evicted"); + assert_eq!( + h.sched + .engine + .state_store() + .lookup_start(&start_key_for(&r.occurrence_id)) + .await + .unwrap(), + Some(run) + ); + assert_eq!( + h.sched.engine.state_store().list_all().await.unwrap().len(), + 0 + ); +} + +/// T21 (scheduler half): commit-without-driver + stale lease → interrupted. +#[tokio::test] +async fn t21_undriven_stale_run_is_interrupted() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + h.backdate_dispatch(&r.occurrence_id, t0 - 500).await; + // Crashed after commit before driver launch: durable run, no driver. + h.craft_run(&r.occurrence_id, WorkflowStatus::Running, Some(3600)) + .await; + let occ = h.occ(&r.occurrence_id).await; + let moved = h + .sched + .reconcile_dispatching(&occ, h.at(t0, 10)) + .await + .unwrap(); + assert!(moved); + let occ = h.occ(&r.occurrence_id).await; + assert_eq!(occ.status, "interrupted"); + assert_eq!( + h.sched.engine.state_store().list_all().await.unwrap().len(), + 1 + ); +} + +/// T22: no receipt after a lost dispatch → same-occurrence retry in deadline, +/// `dispatch_failed` outside it. +#[tokio::test] +async fn t22_lost_dispatch_retries_within_deadline_only() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + // Within deadline: back to queued, same occurrence/slot/spec. + let a = h.mk(&repo, "a", "w", 300, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&a.id, t0 + 300) + .await + .unwrap(); + let aid = match out { + super::store::SlotOutcome::Admitted { occurrence_id } => occurrence_id, + o => panic!("got {o:?}"), + }; + h.sched.store().claim_queued(t0 + 300).await.unwrap(); + h.backdate_dispatch(&aid, t0 - 500).await; + let occ = h.occ(&aid).await; + assert!( + h.sched + .reconcile_dispatching(&occ, h.at(t0 + 300, 100)) + .await + .unwrap() + ); + let occ = h.occ(&aid).await; + assert_eq!(occ.status, "queued"); + assert!(occ.dispatch_started_at.is_none()); + // Same occurrence is what retries: re-claim selects it again. + let reclaimed = h.sched.store().claim_queued(t0 + 400).await.unwrap(); + assert_eq!( + reclaimed.iter().map(|o| o.id.clone()).collect::>(), + vec![aid.clone()] + ); + // Outside deadline: dispatch_failed with the reason kept. + let b = h.mk(&repo, "b", "w", 300, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&b.id, t0 + 300) + .await + .unwrap(); + let bid = match out { + super::store::SlotOutcome::Admitted { occurrence_id } => occurrence_id, + o => panic!("got {o:?}"), + }; + h.sched.store().claim_queued(t0 + 300).await.unwrap(); + h.backdate_dispatch(&bid, t0 - 500).await; + let occ = h.occ(&bid).await; + assert!( + h.sched + .reconcile_dispatching(&occ, h.at(t0 + 300, 301)) + .await + .unwrap() + ); + let occ = h.occ(&bid).await; + assert_eq!(occ.status, "dispatch_failed"); + assert!(occ.error.as_deref().unwrap_or("").contains("grace")); +} + +/// T23: manual lost dispatch retries without schedule grace. +#[tokio::test] +async fn t23_manual_lost_dispatch_retries_without_grace() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + h.backdate_dispatch(&r.occurrence_id, t0 - 100_000).await; + let occ = h.occ(&r.occurrence_id).await; + assert!( + h.sched + .reconcile_dispatching(&occ, h.at(t0, 200_000)) + .await + .unwrap() + ); + assert_eq!(h.occ(&r.occurrence_id).await.status, "queued"); +} + +/// T25: expired scheduled queue entry → `skipped_missed` at claim; a live +/// admitted attempt is never duplicated. +#[tokio::test] +async fn t25_expired_queue_entry_skips_live_attempt_singles() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + // Expired: admitted at slot, disabled so nothing else moves, claimed + // past its deadline. + let a = h.mk(&repo, "a", "w", 300, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&a.id, t0 + 300) + .await + .unwrap(); + let aid = match out { + super::store::SlotOutcome::Admitted { occurrence_id } => occurrence_id, + o => panic!("got {o:?}"), + }; + h.sched + .store() + .set_enabled(&a.id, false, t0 + 300) + .await + .unwrap(); + let summary = h.tick(h.at(t0 + 300, 350)).await; + assert_eq!(summary.fired, 0); + assert_eq!(h.occ(&aid).await.status, "skipped_missed"); + // Live: dispatching + receipt + fresh run links exactly once. + let b = h.mk(&repo, "b", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&b.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run = h + .craft_run(&r.occurrence_id, WorkflowStatus::Running, Some(0)) + .await; + let summary = h.tick(h.at(t0, 10)).await; + assert_eq!(summary.fired, 0); + let occ = h.occ(&r.occurrence_id).await; + assert_eq!(occ.status, "dispatched"); + assert_eq!(occ.run_id, Some(run.to_string())); + assert_eq!( + h.sched.engine.state_store().list_all().await.unwrap().len(), + 1 + ); +} + +/// T25b: eager write converges; tick fallback converges without it. +#[tokio::test] +async fn t25b_eager_and_fallback_converge() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + // Fallback: run completes before linkage; the tick copies the outcome. + let a = h.mk(&repo, "a", "w", 300, t0).await; + let ra = h.sched.store().enqueue_manual(&a.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run = h + .craft_run(&ra.occurrence_id, WorkflowStatus::Completed, None) + .await; + h.sched + .store() + .link_run(&ra.occurrence_id, &run.to_string()) + .await + .unwrap(); + h.tick(h.at(t0, 5)).await; + let occ = h.occ(&ra.occurrence_id).await; + assert_eq!(occ.status, "completed"); + assert!(occ.finished_at.is_some()); + // Eager: end-to-end fire settles the occurrence with no tick at all. + let b = h.mk(&repo, "b", "w", 300, t0).await; + let rb = h.sched.store().enqueue_manual(&b.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + // Drive the fire directly: claim + spawn happen here, then converge. + let claimed = h.sched.store().claim_queued(t0).await.unwrap(); + assert!(claimed.is_empty()); + // Requeue to simulate the queue claim path going through tick fires. + h.sched + .store() + .mark_queued(&rb.occurrence_id) + .await + .unwrap(); + h.tick(h.at(t0, 6)).await; + let occ = h.wait_occ(&rb.occurrence_id).await; + assert_eq!(occ.status, "completed"); +} + +/// T26: failed outcome + error copy to the occurrence. +#[tokio::test] +async fn t26_failed_outcome_and_error_copied() { + let h = Harness::new(); + let repo = h.repo("r"); + h.write_wf( + &repo, + "w", + r#"{"name":"w","steps":[{"name":"s","command":["false"]}]}"#, + ); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.tick(h.at(t0, 1)).await; + let occ = h.wait_occ(&r.occurrence_id).await; + assert_eq!(occ.status, "failed"); + assert!(occ.error.as_deref().unwrap_or("").contains("false")); +} + +/// T27: missing linked workflow → `evicted`, never retried. +#[tokio::test] +async fn t27_missing_linked_workflow_evicts() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run = h + .craft_run(&r.occurrence_id, WorkflowStatus::Running, Some(0)) + .await; + h.sched + .store() + .link_run(&r.occurrence_id, &run.to_string()) + .await + .unwrap(); + h.sched.engine.state_store().delete(run).await.unwrap(); + let occ = h.occ(&r.occurrence_id).await; + assert!( + h.sched + .reconcile_dispatched(&occ, h.at(t0, 5)) + .await + .unwrap() + ); + assert_eq!(h.occ(&r.occurrence_id).await.status, "evicted"); + assert_eq!( + h.sched.engine.state_store().list_all().await.unwrap().len(), + 0 + ); + assert!( + h.sched + .store() + .list_open("queued") + .await + .unwrap() + .is_empty() + ); +} + +/// T28: stale orphan cancels to `interrupted` before overlap admission. +#[tokio::test] +async fn t28_stale_orphan_interrupts_then_slot_admits() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 60, t0).await; + let out = h + .sched + .store() + .claim_due_slot(&s.id, t0 + 60) + .await + .unwrap(); + let old = match out { + super::store::SlotOutcome::Admitted { occurrence_id } => occurrence_id, + o => panic!("got {o:?}"), + }; + h.sched.store().claim_queued(t0 + 60).await.unwrap(); + let run = h.craft_run(&old, WorkflowStatus::Running, Some(3600)).await; + h.sched + .store() + .link_run(&old, &run.to_string()) + .await + .unwrap(); + // Next slot due (t0+120) while the stale run still blocks it. + let summary = h.tick(h.at(t0, 130)).await; + h.sched.drain_fires().await; + assert_eq!(h.occ(&old).await.status, "interrupted"); + assert_eq!(summary.admitted, 1); + let rows = h + .sched + .store() + .occurrences_for_schedule(&s.id, 10) + .await + .unwrap(); + // The new slot was admitted (it may already have fired through). + assert!(rows.iter().any(|r| r.slot == t0 + 120)); +} + +/// T29+T30: paused runs keep lease-loss immunity and keep blocking overlap. +#[tokio::test] +async fn t29_paused_run_survives_ticks_and_blocks() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 60, t0).await; + let r = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + // Paused, no lease: exactly what a settled pause leaves behind. + let run = h + .craft_run(&r.occurrence_id, WorkflowStatus::Paused, None) + .await; + h.sched + .store() + .link_run(&r.occurrence_id, &run.to_string()) + .await + .unwrap(); + for i in 1..=3 { + // Ticks land mid-slot (within grace) so each due slot skips on + // overlap instead of missing on lateness. + let summary = h.tick(h.at(t0, 90 + 60 * (i as i64 - 1))).await; + assert_eq!(summary.skipped_overlap, 1, "tick {i} must skip on overlap"); + assert_eq!(h.occ(&r.occurrence_id).await.status, "dispatched"); + } + assert_eq!( + h.sched.engine.state_store().load(run).await.unwrap().status, + WorkflowStatus::Paused + ); + assert_eq!( + h.sched + .store() + .occurrences_for_schedule(&s.id, 20) + .await + .unwrap() + .iter() + .filter(|o| o.status == "skipped_overlap") + .count(), + 3 + ); +} + +/// T31: rechecked state wins — completed-with-stale-lease copies out, +/// paused-after-stale stays. +#[tokio::test] +async fn t31_newer_state_wins_over_scan() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + // Completed despite the stale lease: real outcome, not interruption. + let r1 = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run1 = h + .craft_run(&r1.occurrence_id, WorkflowStatus::Running, Some(3600)) + .await; + h.sched + .store() + .link_run(&r1.occurrence_id, &run1.to_string()) + .await + .unwrap(); + h.sched + .engine + .state_store() + .update(run1, |st| st.status = WorkflowStatus::Completed) + .await + .unwrap(); + let occ = h.occ(&r1.occurrence_id).await; + assert!( + h.sched + .reconcile_dispatched(&occ, h.at(t0, 5)) + .await + .unwrap() + ); + assert_eq!(h.occ(&r1.occurrence_id).await.status, "completed"); + // Paused after the scan snapshot: untouched despite the stale lease. + let r2 = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run2 = h + .craft_run(&r2.occurrence_id, WorkflowStatus::Running, Some(3600)) + .await; + h.sched + .store() + .link_run(&r2.occurrence_id, &run2.to_string()) + .await + .unwrap(); + h.sched.engine.pause_workflow(run2).await.unwrap(); + let occ = h.occ(&r2.occurrence_id).await; + assert!( + !h.sched + .reconcile_dispatched(&occ, h.at(t0, 5)) + .await + .unwrap() + ); + assert_eq!(h.occ(&r2.occurrence_id).await.status, "dispatched"); +} + +/// T32: unreadable run rows stay open without aborting the sweep. +#[tokio::test] +async fn t32_corrupt_row_does_not_abort_reconciliation() { + let h = Harness::new(); + let repo = h.repo("r"); + h.echo_wf(&repo, "w", "hi"); + let t0 = Utc::now().timestamp(); + let s = h.mk(&repo, "s", "w", 300, t0).await; + // Corrupt: occurrence linked to a run row with garbage state_json, so + // even conditional cancellation fails and the row stays open. + let rc = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + h.sched + .store() + .test_exec( + "INSERT INTO workflow_runs (id, workflow_id, status, state_json, created_at, updated_at) \ + VALUES ('corrupt-run', 'w', 'running', 'not-json{{', 'x', 'x');".to_string(), + ) + .await + .unwrap(); + h.sched + .store() + .link_run(&rc.occurrence_id, "corrupt-run") + .await + .unwrap(); + // Healthy sibling settles in the same tick. + let rok = h.sched.store().enqueue_manual(&s.id, t0).await.unwrap(); + h.sched.store().claim_queued(t0).await.unwrap(); + let run = h + .craft_run(&rok.occurrence_id, WorkflowStatus::Failed, None) + .await; + h.sched + .engine + .state_store() + .update(run, |st| st.error = Some("boom".into())) + .await + .unwrap(); + h.sched + .store() + .link_run(&rok.occurrence_id, &run.to_string()) + .await + .unwrap(); + let summary = h.sched.tick_once(h.at(t0, 5)).await.unwrap(); + assert!(!summary.errors.is_empty()); + assert_eq!(h.occ(&rc.occurrence_id).await.status, "dispatched"); + let ok = h.occ(&rok.occurrence_id).await; + assert_eq!(ok.status, "failed"); + assert_eq!(ok.error.as_deref(), Some("boom")); +} diff --git a/src/schedule/tick.rs b/src/schedule/tick.rs new file mode 100644 index 000000000..2d5d0aaa7 --- /dev/null +++ b/src/schedule/tick.rs @@ -0,0 +1,428 @@ +//! Daemon-owned scheduling: one tick at a time — reconcile outstanding +//! dispatching/dispatched occurrences, claim due slots, claim the queue and +//! spawn daemon-owned fire tasks, prune touched histories. SQLite sections +//! are blocking and transactional; engine calls are async on the ambient +//! runtime; never a transaction across a filesystem read, an async wait, or +//! an agent launch. + +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; + +use chrono::{DateTime, Utc}; +use flare_workflow::json::{JsonWorkflow, PipelineData, SendMessage, compile_workflow_in}; +use flare_workflow::sqlite_store::SqliteStore; +use flare_workflow::types::WorkflowStatus; +use flare_workflow::{OrphanCancelDisposition, StateStore, WorkflowEngine, WorkflowRunId}; + +use super::store::SlotOutcome; +use super::{OccurrenceRow, START_RECOVERY_DELAY_SECS, Scheduler, TickSummary, start_key_for}; + +/// RAII in-flight tracking: a live attempt is never timed out into another +/// attempt or cancelled as an orphan merely because its initial lease aged. +struct InFlightGuard { + set: Arc>>, + id: String, +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + self.set.lock().expect("in-flight set").remove(&self.id); + } +} + +fn terminal_str(status: WorkflowStatus) -> &'static str { + match status { + WorkflowStatus::Completed => "completed", + WorkflowStatus::Failed => "failed", + WorkflowStatus::Cancelled => "cancelled", + _ => "interrupted", + } +} + +impl Scheduler { + /// One scheduler pass (§6.1 ordering). Database unavailability fails the + /// tick; one bad row never aborts the rest (recorded in `errors`). + pub async fn tick_once(&self, now: DateTime) -> Result { + let _tick = self.tick_lock.lock().await; + self.reap_fires(); + let mut summary = TickSummary::default(); + let mut touched: HashSet = HashSet::new(); + + for occ in self.store.list_open("dispatching").await? { + touched.insert(occ.schedule_id.clone()); + match self.reconcile_dispatching(&occ, now).await { + Ok(true) => summary.reconciled += 1, + Ok(false) => {} + Err(e) => summary + .errors + .push(format!("reconcile dispatching {}: {e}", occ.id)), + } + } + for occ in self.store.list_open("dispatched").await? { + touched.insert(occ.schedule_id.clone()); + match self.reconcile_dispatched(&occ, now).await { + Ok(true) => summary.reconciled += 1, + Ok(false) => {} + Err(e) => summary + .errors + .push(format!("reconcile dispatched {}: {e}", occ.id)), + } + } + for sched in self.store.due_schedules(now.timestamp()).await? { + touched.insert(sched.id.clone()); + match self.store.claim_due_slot(&sched.id, now.timestamp()).await { + Ok(SlotOutcome::Admitted { .. }) => summary.admitted += 1, + Ok(SlotOutcome::SkippedMissed { .. }) => summary.skipped_missed += 1, + Ok(SlotOutcome::SkippedOverlap { .. }) => summary.skipped_overlap += 1, + Ok(SlotOutcome::Frozen) => summary + .errors + .push(format!("schedule '{}' frozen, skipped", sched.name)), + Ok(SlotOutcome::NotDue) => {} + Err(e) => summary + .errors + .push(format!("claim slot for '{}': {e}", sched.name)), + } + } + match self.store.claim_queued(now.timestamp()).await { + Ok(claimed) => { + for occ in claimed { + touched.insert(occ.schedule_id.clone()); + self.spawn_fire(occ.id.clone()).await; + summary.fired += 1; + } + } + Err(e) => summary.errors.push(format!("claim queue: {e}")), + } + for sid in touched { + match self.store.prune_schedule(&sid).await { + Ok(n) => summary.pruned += n, + Err(e) => summary.errors.push(format!("prune {sid}: {e}")), + } + } + Ok(summary) + } + + /// Unlinked dispatch recovery (§6.4): a live in-flight attempt is left + /// alone; a receipt links without a retry; otherwise the delay decides + /// between waiting, requeueing, and `dispatch_failed`. + pub(super) async fn reconcile_dispatching( + &self, + occ: &OccurrenceRow, + now: DateTime, + ) -> Result { + if self.is_in_flight(&occ.id) { + return Ok(false); + } + let key = start_key_for(&occ.id); + match self.lookup_start(&key).await? { + Some(run_id) => { + self.store.link_run(&occ.id, &run_id.to_string()).await?; + self.reconcile_linked(&occ.id, run_id, now).await + } + None => { + let started = occ.dispatch_started_at.unwrap_or(occ.created_at); + if now.timestamp() - started < START_RECOVERY_DELAY_SECS { + return Ok(false); + } + if occ.trigger == "manual" { + // Manuals never expire. + return self.store.mark_queued(&occ.id).await; + } + match occ.start_deadline_at { + // Same occurrence, same slot/spec/key — only ever retried + // within its scheduled deadline. + Some(d) if now.timestamp() <= d => Ok(self.store.mark_queued(&occ.id).await?), + Some(_) => Ok(self + .store + .mark_dispatch_failed( + &occ.id, + "lost before start; outside grace", + now.timestamp(), + ) + .await?), + // Defensive: a scheduled row without a deadline must not + // lose work silently; requeue it. + None => Ok(self.store.mark_queued(&occ.id).await?), + } + } + } + } + + pub(super) async fn reconcile_dispatched( + &self, + occ: &OccurrenceRow, + now: DateTime, + ) -> Result { + let Some(run_id) = occ.run_id.as_deref() else { + return Err(format!("dispatched occurrence {} has no run_id", occ.id)); + }; + let run_id = run_id + .parse::() + .map_err(|e| format!("bad run_id on {}: {e}", occ.id))?; + self.reconcile_linked(&occ.id, run_id, now).await + } + + /// Linked-run reconciliation + tick fallback (§6.5): copy real terminal + /// outcomes, mark evicted rows, leave paused and fresh-lease runs alone, + /// and conditionally cancel only provably-stale orphans. + pub(super) async fn reconcile_linked( + &self, + occurrence_id: &str, + run_id: WorkflowRunId, + now: DateTime, + ) -> Result { + reconcile_linked_run(&self.store, &self.engine, occurrence_id, run_id, now).await + } + + async fn spawn_fire(&self, occurrence_id: String) { + // Resolve the owning schedule for its register-plus-start lock; + // the fire task re-reads the row and stops if it moved on. + let sched_id = match self.store.get_occurrence(&occurrence_id).await { + Ok(Some(occ)) if occ.status == "dispatching" => occ.schedule_id, + // Row moved on (linked, settled, requeued, or removed) between + // claim and spawn: nothing to fire. + _ => return, + }; + if !self.mark_in_flight(&occurrence_id) { + return; + } + let store = self.store.clone(); + let engine = self.engine.clone(); + let send = self.send.clone(); + let in_flight = Arc::clone(&self.in_flight); + let sched_lock = self.schedule_lock(&sched_id); + let handle = tokio::spawn(async move { + let _guard = InFlightGuard { + set: in_flight, + id: occurrence_id.clone(), + }; + fire_one(&store, &engine, &send, &sched_lock, occurrence_id).await; + }); + self.fires.lock().expect("fire handles").push(handle); + } + + async fn lookup_start(&self, key: &str) -> Result, String> { + self.engine + .state_store() + .lookup_start(key) + .await + .map_err(|e| e.to_string()) + } +} + +/// Shared linked-run reconciliation behind `Scheduler::reconcile_linked` +/// and the fire path: after every linkage, converge an already-terminal run +/// at once. This closes the race where a fast driver settles (and its eager +/// write finds the row still `dispatching`) before the fire task's link +/// lands — without it the occurrence would wait for the next tick. +async fn reconcile_linked_run( + store: &super::ScheduleStore, + engine: &WorkflowEngine>, + occurrence_id: &str, + run_id: WorkflowRunId, + now: DateTime, +) -> Result { + let now_ts = now.timestamp(); + let linked = match engine.state_store().load(run_id).await { + Ok(s) => Some(s), + Err(flare_workflow::WorkflowError::NotFound(_)) => None, + Err(e) => return Err(e.to_string()), + }; + match linked { + None => Ok(store + .settle_open(occurrence_id, "evicted", None, now_ts) + .await?), + Some(state) => { + if state.status.is_terminal() { + return store + .settle_open( + occurrence_id, + terminal_str(state.status), + state.error.as_deref(), + now_ts, + ) + .await; + } + if state.status == WorkflowStatus::Paused { + // Intentional inactivity, not a crash orphan — regardless + // of lease. Still blocks overlap. + return Ok(false); + } + let fresh = state + .lease + .as_ref() + .is_some_and(|l| l.is_fresh(now, engine.lease_ttl())); + if fresh { + return Ok(false); + } + match engine.cancel_if_orphaned(run_id, now).await { + Ok(OrphanCancelDisposition::Interrupted) => Ok(store + .settle_open(occurrence_id, "interrupted", None, now_ts) + .await?), + Ok(OrphanCancelDisposition::Terminal(status)) => { + // Rechecked state settled concurrently; copy the real + // outcome — newer state always wins. + let current = match engine.state_store().load(run_id).await { + Ok(s) if s.status == status => Some(s), + _ => None, + }; + Ok(store + .settle_open( + occurrence_id, + terminal_str(status), + current.as_ref().and_then(|s| s.error.as_deref()), + now_ts, + ) + .await?) + } + Ok(OrphanCancelDisposition::Missing) => Ok(store + .settle_open(occurrence_id, "evicted", None, now_ts) + .await?), + Ok(OrphanCancelDisposition::Unchanged) => Ok(false), + Err(e) => Err(format!("cancel_if_orphaned: {e}")), + } + } + } +} + +/// Daemon-owned fire (§6.3): receipt recheck, execution-time revalidation, +/// per-schedule register-plus-start serialization, idempotent linkage. Store +/// failures and uncertain starts leave `dispatching` for reconciliation; +/// `dispatch_failed` requires proving no receipt exists. +async fn fire_one( + store: &super::ScheduleStore, + engine: &WorkflowEngine>, + send: &SendMessage, + sched_lock: &Arc>, + occurrence_id: String, +) { + let fail = |msg: String| { + let (store, occurrence_id) = (store.clone(), occurrence_id.clone()); + async move { + if let Err(e) = store + .mark_dispatch_failed(&occurrence_id, &msg, Utc::now().timestamp()) + .await + { + eprintln!("agentflare-schedule: mark dispatch_failed failed: {e}"); + } + } + }; + let outcome = async { + let occ = store + .get_occurrence(&occurrence_id) + .await? + .ok_or_else(|| "occurrence gone".to_string())?; + if occ.status != "dispatching" { + return Ok::<_, String>(()); + } + let key = start_key_for(&occurrence_id); + // Before executing, check the receipt. Never restart. + if let Some(run_id) = engine + .state_store() + .lookup_start(&key) + .await + .map_err(|e| e.to_string())? + { + store.link_run(&occ.id, &run_id.to_string()).await?; + let _ = reconcile_linked_run(store, engine, &occ.id, run_id, Utc::now()).await; + return Ok(()); + } + let spec = occ.parsed_launch_spec()?; + if spec.params.as_object().is_none() { + store + .mark_dispatch_failed( + &occ.id, + "launch_spec params not an object", + Utc::now().timestamp(), + ) + .await?; + return Ok(()); + } + // Execution-time revalidation: current file contents, every fire. + let definition_json = crate::workflow::resolve_named_definition( + std::path::Path::new(&spec.repo_root), + &spec.workflow_name, + ) + .map_err(|e| format!("definition: {e}"))?; + let json: JsonWorkflow = + serde_json::from_str(&definition_json).map_err(|e| format!("definition: {e}"))?; + let repo_root = std::path::PathBuf::from(&spec.repo_root); + let mut definition = compile_workflow_in(&json, send.clone(), Some(repo_root)) + .map_err(|e| format!("definition: {e}"))?; + let definition_id = + flare_workflow::WorkflowId::new(format!("schedule:{}", occ.schedule_id)); + definition.id = definition_id.clone(); + let mut params = spec.params.clone(); + let params_obj = params + .as_object_mut() + .ok_or_else(|| "launch_spec params claim reserved key `schedule`".to_string())?; + if params_obj.contains_key("schedule") { + return Err("launch_spec params claim reserved key `schedule`".to_string()); + } + params_obj.insert( + "schedule".to_string(), + serde_json::json!({ + "schedule_id": occ.schedule_id, + "name": spec.schedule_name, + "occurrence_id": occ.id, + "slot": occ.slot, + "trigger": occ.trigger, + }), + ); + // Register-plus-start serialized per schedule; the lock releases + // after start returns, not after workflow completion. + let started = { + let _held = sched_lock.lock().await; + engine + .register_workflow(definition) + .map_err(|e| format!("definition: {e}"))?; + engine + .start_workflow_once_with_params( + &key, + definition_id, + PipelineData, + String::new(), + params, + ) + .await + .map_err(|e| e.to_string())? + }; + store + .link_run(&occ.id, &started.run_id().to_string()) + .await?; + // Converge a run that settled between start-return and link-commit + // (its eager write could not match the still-`dispatching` row). + let _ = reconcile_linked_run(store, engine, &occ.id, started.run_id(), Utc::now()).await; + Ok::<_, String>(()) + }; + match outcome.await { + Ok(()) => {} + Err(e) if e == "occurrence gone" => {} + Err(e) if e.starts_with("definition:") || e.contains("launch_spec") => { + fail(e).await; + } + Err(e) => { + // Uncertain (store failure, shutdown, post-commit ambiguity): + // prove no receipt exists before calling it a failure. + match engine + .state_store() + .lookup_start(&start_key_for(&occurrence_id)) + .await + { + Ok(Some(run_id)) => { + let _ = store.link_run(&occurrence_id, &run_id.to_string()).await; + } + Ok(None) => { + eprintln!( + "agentflare-schedule: fire {occurrence_id} uncertain, left dispatching: {e}" + ); + } + Err(le) => { + eprintln!( + "agentflare-schedule: fire {occurrence_id} uncertain and receipt unreadable ({le}); left dispatching: {e}" + ); + } + } + } + } +} diff --git a/src/workflow.rs b/src/workflow.rs index 99890bd3f..09d830cfb 100644 --- a/src/workflow.rs +++ b/src/workflow.rs @@ -68,7 +68,7 @@ pub(crate) fn agent_send_hook() -> SendMessage { args, hard_cap_secs, idle_timeout_secs, - cwd: _, + cwd, owner: _, persona, } = inv; @@ -82,12 +82,16 @@ pub(crate) fn agent_send_hook() -> SendMessage { } extra_args.extend(args); let request_json = pin_stream_json(&agent, &mut extra_args); - // No projected scratch dir here: the persona definition is - // whatever the ambient cwd's `.claude/agents/` holds. - let agents_dir = std::env::current_dir() - .unwrap_or_default() - .join(".claude") - .join("agents"); + // Persona definitions resolve against the step's own working + // directory (a schedule's repo root), not the daemon's ambient + // cwd — same reason the launch below honors `cwd`. + let agents_dir = match &cwd { + Some(dir) => dir.join(".claude").join("agents"), + None => std::env::current_dir() + .unwrap_or_default() + .join(".claude") + .join("agents"), + }; let prompt = apply_persona( &agent, persona.as_deref(), @@ -103,8 +107,13 @@ pub(crate) fn agent_send_hook() -> SendMessage { // before it, or the call already happened on whatever thread // invoked this closure before the returned future was ever // polled/awaited. - let outcome = tokio::task::spawn_blocking(move || { - crate::agent_launch::run_headless( + let outcome = tokio::task::spawn_blocking(move || match cwd { + // Explicit step cwd (e.g. a schedule's repo root): run the + // agent subprocess there instead of the daemon's ambient + // cwd — required for scheduler-owned runs, which never + // re-enter any dispatcher's chdir span. + Some(cwd) => crate::agent_launch::run_headless_in( + &cwd, agent_registry::REGISTRY, &agent, &prompt, @@ -112,7 +121,16 @@ pub(crate) fn agent_send_hook() -> SendMessage { idle_timeout, &extra_args, request_json, - ) + ), + None => crate::agent_launch::run_headless( + agent_registry::REGISTRY, + &agent, + &prompt, + hard_cap, + idle_timeout, + &extra_args, + request_json, + ), }) .await .map_err(|e| format!("agent task panicked: {e}"))?; From c028093aa2c78ccbfc60939496d90331bd0147cf Mon Sep 17 00:00:00 2001 From: AppzDev Date: Fri, 16 Oct 2026 22:25:03 +0530 Subject: [PATCH 2/2] fix(schedule): portable print-cwd in T36, correct cancel_if_orphaned doc (item #369 PR-B) T36 spawned `pwd` directly; on Windows it resolves to Git coreutils pwd.exe, whose POSIX output breaks canonicalize comparison. Use cmd /c cd on Windows, pwd elsewhere. Doc: terminal callback is NOT invoked by cancel_if_orphaned; caller maps disposition. Agentflare-Agent: opencode Agentflare-Branch: task/369-pr-b-durable-workflow-scheduling-migrati Agentflare-Item: 369 --- crates/flare-workflow/src/engine_start_once.rs | 3 ++- src/schedule/tests_lifecycle.rs | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/flare-workflow/src/engine_start_once.rs b/crates/flare-workflow/src/engine_start_once.rs index 61e783146..95f28d364 100644 --- a/crates/flare-workflow/src/engine_start_once.rs +++ b/crates/flare-workflow/src/engine_start_once.rs @@ -162,7 +162,8 @@ impl WorkflowEngine> { /// run whose executor lease is stale or missing at `now`): rechecks /// status and lease atomically under the store's mutation protection, /// persists the disposition consistently, and publishes a cancellation - /// event plus the terminal callback only when state actually changed. + /// event only when state actually changed. The terminal callback is NOT + /// invoked: the caller maps the returned disposition itself. /// Never use unconditional `cancel_workflow` here and assume /// interruption — a paused run or a fresh lease must survive. pub async fn cancel_if_orphaned( diff --git a/src/schedule/tests_lifecycle.rs b/src/schedule/tests_lifecycle.rs index deb4ca62b..9ee61eb73 100644 --- a/src/schedule/tests_lifecycle.rs +++ b/src/schedule/tests_lifecycle.rs @@ -23,6 +23,16 @@ async fn t36_execution_runs_in_schedule_repo() { // No process-wide chdir anywhere: the daemon's ambient cwd is wherever // the test runner sits, and execution must still land in repo_b. let repo_b = h.repo("b"); + // Portable print-cwd: `pwd` is not a native Windows binary (Git's + // coreutils shim prints a POSIX path `std::fs::canonicalize` cannot + // resolve), so ask the native shell on Windows instead. + #[cfg(windows)] + h.write_wf( + &repo_b, + "w", + r#"{"name":"w","steps":[{"name":"s","command":["cmd","/c","cd"]}]}"#, + ); + #[cfg(not(windows))] h.write_wf( &repo_b, "w",