From 46b41f349ce7b91488b366e6902999fe3146aed3 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:14:17 +0000 Subject: [PATCH 1/4] Add published-package local retry timing and replay checks --- scripts/acceptance/installed-native/README.md | 20 ++ tests/local_retry_timing_server.rs | 239 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 tests/local_retry_timing_server.rs diff --git a/scripts/acceptance/installed-native/README.md b/scripts/acceptance/installed-native/README.md index 924a49b..2b47bbe 100644 --- a/scripts/acceptance/installed-native/README.md +++ b/scripts/acceptance/installed-native/README.md @@ -66,6 +66,26 @@ Record the package checksum, Server digest, runner commit and cleanup alongside the result. Compiling the ignored test or using source dependencies is not published-artifact acceptance. +### Local retry-delay observation + +`tests/local_retry_timing_server.rs` covers shared #219 through the ordinary +published SDK Worker path. Copy it into a fresh exact-registry consumer with +Tokio and serde_json. Bind the selected qualified native tuple and set +`DURABLE_WORKFLOW_EXPECT_SERVER_VERSION` alongside the existing isolated +numeric-loopback URL and synthetic-token variables. Run only +`published_local_retries_repeat_last_and_replay_committed_results` with +`--ignored --exact --nocapture`. + +Five local attempts with `[1, 2]` must record four retry delays `[1, 2, 2, 2]` +and independently measured callback gaps no shorter than those delays. The +empty-list case records four explicit zeros. Each case binds the run, logical +activity and five attempts through public local ActivityRetryScheduled events. +A replacement Worker instance then replays the committed local result after a +signal, without another callback or changed history prefix. Empty-list gap +measurements are retained without promising zero execution or queue latency. +The caller owns exact package/build/producer/resource and cleanup evidence; +compilation or the ordinary synthetic observer control is not installed proof. + ### Replay-aware logging feature acceptance `tests/workflow_logging_server.rs` checks the logging feature separately from diff --git a/tests/local_retry_timing_server.rs b/tests/local_retry_timing_server.rs new file mode 100644 index 0000000..c198fb1 --- /dev/null +++ b/tests/local_retry_timing_server.rs @@ -0,0 +1,239 @@ +//! Published-package observation of local retry delays for shared issue #219. +//! The caller owns the isolated Server, exact package/producer proof and cleanup. +use durable_workflow::{ + json, ActivityError, ActivityRetryPolicy, Client, LocalActivityOptions, Value, Worker, + WorkflowResultOptions, +}; +use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +fn retry_policy(intervals: &[u64]) -> ActivityRetryPolicy { + ActivityRetryPolicy::new(5) + .backoff_intervals(intervals.iter().copied().map(Duration::from_secs)) +} + +fn worker( + client: &Client, + queue: &str, + kind: &str, + intervals: Vec, + observations: Arc>>, + replacement: bool, +) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(format!("{queue}-{replacement}")) + .local_activities(true) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow(kind, move |ctx, _| { + let policy = retry_policy(&intervals); + async move { + let value = ctx + .local_activity_with_options( + "retry-probe", + LocalActivityOptions::new() + .retry_policy(policy) + .start_to_close_timeout(Duration::from_secs(3)) + .schedule_to_close_timeout(Duration::from_secs(30)), + json!([]), + ) + .await?; + ctx.wait_signal("finish").await?; + Ok(value) + } + }); + worker.register_activity("retry-probe", move |ctx, _| { + observations + .lock() + .unwrap() + .push((ctx.attempt_number, Instant::now())); + assert!(!replacement, "committed local results must not call the handler"); + async move { + if ctx.attempt_number < 5 { + return Err(ActivityError::new("RetryProbe", "declared retry").into()); + } + Ok(json!([5])) + } + }); + worker +} + +fn activity_events(history: &Value) -> Vec<&Value> { + history["history_events"] + .as_array() + .unwrap() + .iter() + .filter(|event| { + matches!( + event["type"].as_str(), + Some("ActivityScheduled" | "ActivityStarted" | "ActivityRetryScheduled" | "ActivityCompleted") + ) + }) + .collect() +} + +fn verify_reports(history: &Value, workflow: &str, run: &str, expected: &[u64], terminal: bool) -> Vec { + assert_eq!(history["workflow"]["instance_id"], workflow); + assert_eq!(history["workflow"]["run_id"], run); + assert_eq!(history["workflow"]["details_state"], "retained"); + assert_eq!(history["history_complete"], terminal); + let events = activity_events(history); + let scheduled: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityScheduled").collect(); + let completed: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityCompleted").collect(); + assert_eq!(scheduled.len(), 1); + assert_eq!(completed.len(), 1); + let execution = scheduled[0]["payload"]["activity_execution_id"].as_str().unwrap(); + assert!(!execution.is_empty()); + for event in &events { + let payload = &event["payload"]; + assert_eq!(payload["sequence"], 1); + assert_eq!(payload["activity_execution_id"], execution); + assert_eq!(payload["execution_mode"], "local"); + assert_eq!(payload["local_activity"], true); + } + let starts: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityStarted").collect(); + assert_eq!(starts.len(), 5); + let mut attempts = BTreeSet::new(); + for (index, event) in starts.iter().enumerate() { + assert_eq!(event["payload"]["attempt_number"], index + 1); + let id = event["payload"]["activity_attempt_id"].as_str().unwrap(); + assert!(!id.is_empty() && attempts.insert(id)); + } + let retries: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityRetryScheduled").collect(); + assert_eq!(retries.len(), expected.len()); + assert_eq!(expected.len(), 4); + for (index, (event, delay)) in retries.iter().zip(expected).enumerate() { + let payload = &event["payload"]; + assert_eq!(payload["retry_after_attempt"], index + 1); + assert_eq!(payload["retry_after_attempt_id"], starts[index]["payload"]["activity_attempt_id"]); + assert_eq!(payload["retry_backoff_seconds"].as_u64(), Some(*delay)); + } + assert_eq!(completed[0]["payload"]["attempt_number"], 5); + retries.into_iter().map(|event| event["payload"].clone()).collect() +} + +fn verify_gaps(observations: &[(u64, Instant)], expected: &[u64]) -> Vec { + assert_eq!(observations.len(), 5); + assert_eq!(expected.len(), 4); + for (index, (attempt, _)) in observations.iter().enumerate() { + assert_eq!(*attempt, (index + 1) as u64); + } + observations + .windows(2) + .zip(expected) + .map(|(pair, seconds)| { + let gap = pair[1].1.duration_since(pair[0].1); + assert!(gap >= Duration::from_secs(*seconds), "retry ran before its declared delay"); + gap.as_millis() + }) + .collect() +} + +#[tokio::test] +#[ignore = "requires an exact published SDK and isolated qualified Server tuple"] +async fn published_local_retries_repeat_last_and_replay_committed_results() { + assert_eq!(std::env::var("DURABLE_WORKFLOW_LOCAL_ISOLATED").as_deref(), Ok("1")); + let origin = std::env::var("DURABLE_WORKFLOW_SERVER_URL").unwrap(); + assert!(origin.strip_prefix("http://127.0.0.1:").and_then(|port| port.parse::().ok()).is_some_and(|port| port > 0)); + let expected_version = std::env::var("DURABLE_WORKFLOW_EXPECT_SERVER_VERSION").unwrap(); + assert!(!expected_version.is_empty()); + let client = Client::builder(origin) + .namespace("default") + .timeout(Duration::from_secs(3)) + .token(Some(std::env::var("DURABLE_WORKFLOW_AUTH_TOKEN").unwrap())) + .build() + .unwrap(); + let cluster = client.cluster_info().await.unwrap(); + assert_eq!(cluster["implementation"], "rust"); + assert_eq!(cluster["version"], expected_version); + for (name, intervals, expected) in [ + ("short", vec![1, 2], vec![1, 2, 2, 2]), + ("empty", vec![], vec![0, 0, 0, 0]), + ] { + tokio::time::timeout(Duration::from_secs(35), async { + let id = format!("rust-retry-{name}-{}", durable_workflow::Uuid::new_v4()); + let kind = format!("qualification.local-retry.{id}"); + let observations = Arc::new(Mutex::new(Vec::new())); + let original = worker(&client, &id, &kind, intervals.clone(), observations.clone(), false); + original.register().await.unwrap(); + let handle = client.start_workflow(&kind, &id, &id, json!([])).await.unwrap(); + let run = handle.run_id.as_deref().unwrap(); + assert!(original.run_once().await.unwrap() > 0); + let committed = client.export_workflow_history(&id, run).await.unwrap(); + let reports = verify_reports(&committed, &id, run, &expected, false); + let gaps = verify_gaps(&observations.lock().unwrap(), &expected); + client.deregister_worker_registration(&format!("{id}-false")).await.unwrap(); + let replacement = worker(&client, &id, &kind, intervals, observations.clone(), true); + replacement.register().await.unwrap(); + handle.signal("finish", json!([])).await.unwrap(); + let mut terminal = false; + for _ in 0..20 { + replacement.run_once().await.unwrap(); + let description = handle.describe().await.unwrap(); + if description.is_terminal() { + assert_eq!(description.status.as_deref(), Some("completed")); + terminal = true; + break; + } + } + assert!(terminal, "replacement did not complete within its polling bound"); + assert_eq!(handle.result(WorkflowResultOptions::default()).await.unwrap(), json!([5])); + assert_eq!(observations.lock().unwrap().len(), 5); + let final_history = client.export_workflow_history(&id, run).await.unwrap(); + assert_eq!(verify_reports(&final_history, &id, run, &expected, true), reports); + let prefix = committed["history_events"].as_array().unwrap(); + assert_eq!(&final_history["history_events"].as_array().unwrap()[..prefix.len()], prefix); + client.deregister_worker_registration(&format!("{id}-true")).await.unwrap(); + let report = json!({"case":name,"workflow_id":id,"run_id":run, + "expected_backoff_seconds":expected,"callback_gap_ms":gaps,"retry_reports":reports, + "callback_attempts":5,"callbacks_after_replay":0,"history":final_history}); + let text = report.to_string(); + assert!(text.len() <= 512 * 1024); + println!("LOCAL_RETRY_TIMING {text}"); + }).await.expect("one local retry case exceeded its fixed bound"); + } +} + +#[test] +fn callback_observer_rejects_zero_tail_and_accepts_repeat_last() { + let start = Instant::now(); + let observed = |times: &[u64]| times.iter().enumerate().map(|(i, seconds)| ((i + 1) as u64, start + Duration::from_secs(*seconds))).collect::>(); + assert_eq!(verify_gaps(&observed(&[0, 1, 3, 5, 7]), &[1, 2, 2, 2]), vec![1000, 2000, 2000, 2000]); + assert!(std::panic::catch_unwind(|| verify_gaps(&observed(&[0, 1, 3, 3, 3]), &[1, 2, 2, 2])).is_err()); + assert_eq!(verify_gaps(&observed(&[0, 0, 0, 0, 0]), &[0; 4]), vec![0; 4]); +} + +#[test] +fn committed_reports_reject_zero_tail_and_remote_activity_mixups() { + let mut events = vec![json!({"type":"ActivityScheduled","payload":{ + "sequence":1,"activity_execution_id":"local-id","execution_mode":"local","local_activity":true}})]; + for number in 1..=5 { + let payload = json!({"sequence":1,"activity_execution_id":"local-id","execution_mode":"local", + "local_activity":true,"attempt_number":number,"activity_attempt_id":format!("attempt-{number}")}); + events.push(json!({"type":"ActivityStarted","payload":payload})); + if number < 5 { + let mut retry = events.last().unwrap()["payload"].clone(); + retry["retry_after_attempt"] = json!(number); + retry["retry_after_attempt_id"] = json!(format!("attempt-{number}")); + retry["retry_backoff_seconds"] = json!(if number == 1 {1} else {2}); + events.push(json!({"type":"ActivityRetryScheduled","payload":retry})); + } + } + let last = events.last().unwrap()["payload"].clone(); + events.push(json!({"type":"ActivityCompleted","payload":last})); + let accepted = json!({"workflow":{"instance_id":"workflow","run_id":"run","details_state":"retained"}, + "history_complete":true,"history_events":events}); + assert_eq!(verify_reports(&accepted, "workflow", "run", &[1, 2, 2, 2], true).len(), 4); + let mut zero_tail = accepted.clone(); + for event in zero_tail["history_events"].as_array_mut().unwrap() { + if event["type"] == "ActivityRetryScheduled" && event["payload"]["retry_after_attempt"].as_u64().unwrap() >= 3 { + event["payload"]["retry_backoff_seconds"] = json!(0); + } + } + assert!(std::panic::catch_unwind(|| verify_reports(&zero_tail, "workflow", "run", &[1, 2, 2, 2], true)).is_err()); + let mut remote = accepted; + remote["history_events"][0]["payload"]["execution_mode"] = json!("remote"); + assert!(std::panic::catch_unwind(|| verify_reports(&remote, "workflow", "run", &[1, 2, 2, 2], true)).is_err()); +} From 72b2aa1e6d86452b71cda2e74c156107e26ce486 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:18:07 +0000 Subject: [PATCH 2/4] Apply formatter output to the local retry observer --- tests/local_retry_timing_server.rs | 172 +++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 31 deletions(-) diff --git a/tests/local_retry_timing_server.rs b/tests/local_retry_timing_server.rs index c198fb1..4a65475 100644 --- a/tests/local_retry_timing_server.rs +++ b/tests/local_retry_timing_server.rs @@ -49,7 +49,10 @@ fn worker( .lock() .unwrap() .push((ctx.attempt_number, Instant::now())); - assert!(!replacement, "committed local results must not call the handler"); + assert!( + !replacement, + "committed local results must not call the handler" + ); async move { if ctx.attempt_number < 5 { return Err(ActivityError::new("RetryProbe", "declared retry").into()); @@ -68,23 +71,42 @@ fn activity_events(history: &Value) -> Vec<&Value> { .filter(|event| { matches!( event["type"].as_str(), - Some("ActivityScheduled" | "ActivityStarted" | "ActivityRetryScheduled" | "ActivityCompleted") + Some( + "ActivityScheduled" + | "ActivityStarted" + | "ActivityRetryScheduled" + | "ActivityCompleted" + ) ) }) .collect() } -fn verify_reports(history: &Value, workflow: &str, run: &str, expected: &[u64], terminal: bool) -> Vec { +fn verify_reports( + history: &Value, + workflow: &str, + run: &str, + expected: &[u64], + terminal: bool, +) -> Vec { assert_eq!(history["workflow"]["instance_id"], workflow); assert_eq!(history["workflow"]["run_id"], run); assert_eq!(history["workflow"]["details_state"], "retained"); assert_eq!(history["history_complete"], terminal); let events = activity_events(history); - let scheduled: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityScheduled").collect(); - let completed: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityCompleted").collect(); + let scheduled: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityScheduled") + .collect(); + let completed: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityCompleted") + .collect(); assert_eq!(scheduled.len(), 1); assert_eq!(completed.len(), 1); - let execution = scheduled[0]["payload"]["activity_execution_id"].as_str().unwrap(); + let execution = scheduled[0]["payload"]["activity_execution_id"] + .as_str() + .unwrap(); assert!(!execution.is_empty()); for event in &events { let payload = &event["payload"]; @@ -93,7 +115,10 @@ fn verify_reports(history: &Value, workflow: &str, run: &str, expected: &[u64], assert_eq!(payload["execution_mode"], "local"); assert_eq!(payload["local_activity"], true); } - let starts: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityStarted").collect(); + let starts: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityStarted") + .collect(); assert_eq!(starts.len(), 5); let mut attempts = BTreeSet::new(); for (index, event) in starts.iter().enumerate() { @@ -101,17 +126,26 @@ fn verify_reports(history: &Value, workflow: &str, run: &str, expected: &[u64], let id = event["payload"]["activity_attempt_id"].as_str().unwrap(); assert!(!id.is_empty() && attempts.insert(id)); } - let retries: Vec<_> = events.iter().filter(|event| event["type"] == "ActivityRetryScheduled").collect(); + let retries: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityRetryScheduled") + .collect(); assert_eq!(retries.len(), expected.len()); assert_eq!(expected.len(), 4); for (index, (event, delay)) in retries.iter().zip(expected).enumerate() { let payload = &event["payload"]; assert_eq!(payload["retry_after_attempt"], index + 1); - assert_eq!(payload["retry_after_attempt_id"], starts[index]["payload"]["activity_attempt_id"]); + assert_eq!( + payload["retry_after_attempt_id"], + starts[index]["payload"]["activity_attempt_id"] + ); assert_eq!(payload["retry_backoff_seconds"].as_u64(), Some(*delay)); } assert_eq!(completed[0]["payload"]["attempt_number"], 5); - retries.into_iter().map(|event| event["payload"].clone()).collect() + retries + .into_iter() + .map(|event| event["payload"].clone()) + .collect() } fn verify_gaps(observations: &[(u64, Instant)], expected: &[u64]) -> Vec { @@ -125,7 +159,10 @@ fn verify_gaps(observations: &[(u64, Instant)], expected: &[u64]) -> Vec { .zip(expected) .map(|(pair, seconds)| { let gap = pair[1].1.duration_since(pair[0].1); - assert!(gap >= Duration::from_secs(*seconds), "retry ran before its declared delay"); + assert!( + gap >= Duration::from_secs(*seconds), + "retry ran before its declared delay" + ); gap.as_millis() }) .collect() @@ -134,9 +171,15 @@ fn verify_gaps(observations: &[(u64, Instant)], expected: &[u64]) -> Vec { #[tokio::test] #[ignore = "requires an exact published SDK and isolated qualified Server tuple"] async fn published_local_retries_repeat_last_and_replay_committed_results() { - assert_eq!(std::env::var("DURABLE_WORKFLOW_LOCAL_ISOLATED").as_deref(), Ok("1")); + assert_eq!( + std::env::var("DURABLE_WORKFLOW_LOCAL_ISOLATED").as_deref(), + Ok("1") + ); let origin = std::env::var("DURABLE_WORKFLOW_SERVER_URL").unwrap(); - assert!(origin.strip_prefix("http://127.0.0.1:").and_then(|port| port.parse::().ok()).is_some_and(|port| port > 0)); + assert!(origin + .strip_prefix("http://127.0.0.1:") + .and_then(|port| port.parse::().ok()) + .is_some_and(|port| port > 0)); let expected_version = std::env::var("DURABLE_WORKFLOW_EXPECT_SERVER_VERSION").unwrap(); assert!(!expected_version.is_empty()); let client = Client::builder(origin) @@ -156,15 +199,28 @@ async fn published_local_retries_repeat_last_and_replay_committed_results() { let id = format!("rust-retry-{name}-{}", durable_workflow::Uuid::new_v4()); let kind = format!("qualification.local-retry.{id}"); let observations = Arc::new(Mutex::new(Vec::new())); - let original = worker(&client, &id, &kind, intervals.clone(), observations.clone(), false); + let original = worker( + &client, + &id, + &kind, + intervals.clone(), + observations.clone(), + false, + ); original.register().await.unwrap(); - let handle = client.start_workflow(&kind, &id, &id, json!([])).await.unwrap(); + let handle = client + .start_workflow(&kind, &id, &id, json!([])) + .await + .unwrap(); let run = handle.run_id.as_deref().unwrap(); assert!(original.run_once().await.unwrap() > 0); let committed = client.export_workflow_history(&id, run).await.unwrap(); let reports = verify_reports(&committed, &id, run, &expected, false); let gaps = verify_gaps(&observations.lock().unwrap(), &expected); - client.deregister_worker_registration(&format!("{id}-false")).await.unwrap(); + client + .deregister_worker_registration(&format!("{id}-false")) + .await + .unwrap(); let replacement = worker(&client, &id, &kind, intervals, observations.clone(), true); replacement.register().await.unwrap(); handle.signal("finish", json!([])).await.unwrap(); @@ -178,31 +234,66 @@ async fn published_local_retries_repeat_last_and_replay_committed_results() { break; } } - assert!(terminal, "replacement did not complete within its polling bound"); - assert_eq!(handle.result(WorkflowResultOptions::default()).await.unwrap(), json!([5])); + assert!( + terminal, + "replacement did not complete within its polling bound" + ); + assert_eq!( + handle + .result(WorkflowResultOptions::default()) + .await + .unwrap(), + json!([5]) + ); assert_eq!(observations.lock().unwrap().len(), 5); let final_history = client.export_workflow_history(&id, run).await.unwrap(); - assert_eq!(verify_reports(&final_history, &id, run, &expected, true), reports); + assert_eq!( + verify_reports(&final_history, &id, run, &expected, true), + reports + ); let prefix = committed["history_events"].as_array().unwrap(); - assert_eq!(&final_history["history_events"].as_array().unwrap()[..prefix.len()], prefix); - client.deregister_worker_registration(&format!("{id}-true")).await.unwrap(); + assert_eq!( + &final_history["history_events"].as_array().unwrap()[..prefix.len()], + prefix + ); + client + .deregister_worker_registration(&format!("{id}-true")) + .await + .unwrap(); let report = json!({"case":name,"workflow_id":id,"run_id":run, "expected_backoff_seconds":expected,"callback_gap_ms":gaps,"retry_reports":reports, "callback_attempts":5,"callbacks_after_replay":0,"history":final_history}); let text = report.to_string(); assert!(text.len() <= 512 * 1024); println!("LOCAL_RETRY_TIMING {text}"); - }).await.expect("one local retry case exceeded its fixed bound"); + }) + .await + .expect("one local retry case exceeded its fixed bound"); } } #[test] fn callback_observer_rejects_zero_tail_and_accepts_repeat_last() { let start = Instant::now(); - let observed = |times: &[u64]| times.iter().enumerate().map(|(i, seconds)| ((i + 1) as u64, start + Duration::from_secs(*seconds))).collect::>(); - assert_eq!(verify_gaps(&observed(&[0, 1, 3, 5, 7]), &[1, 2, 2, 2]), vec![1000, 2000, 2000, 2000]); - assert!(std::panic::catch_unwind(|| verify_gaps(&observed(&[0, 1, 3, 3, 3]), &[1, 2, 2, 2])).is_err()); - assert_eq!(verify_gaps(&observed(&[0, 0, 0, 0, 0]), &[0; 4]), vec![0; 4]); + let observed = |times: &[u64]| { + times + .iter() + .enumerate() + .map(|(i, seconds)| ((i + 1) as u64, start + Duration::from_secs(*seconds))) + .collect::>() + }; + assert_eq!( + verify_gaps(&observed(&[0, 1, 3, 5, 7]), &[1, 2, 2, 2]), + vec![1000, 2000, 2000, 2000] + ); + assert!( + std::panic::catch_unwind(|| verify_gaps(&observed(&[0, 1, 3, 3, 3]), &[1, 2, 2, 2])) + .is_err() + ); + assert_eq!( + verify_gaps(&observed(&[0, 0, 0, 0, 0]), &[0; 4]), + vec![0; 4] + ); } #[test] @@ -217,7 +308,7 @@ fn committed_reports_reject_zero_tail_and_remote_activity_mixups() { let mut retry = events.last().unwrap()["payload"].clone(); retry["retry_after_attempt"] = json!(number); retry["retry_after_attempt_id"] = json!(format!("attempt-{number}")); - retry["retry_backoff_seconds"] = json!(if number == 1 {1} else {2}); + retry["retry_backoff_seconds"] = json!(if number == 1 { 1 } else { 2 }); events.push(json!({"type":"ActivityRetryScheduled","payload":retry})); } } @@ -225,15 +316,34 @@ fn committed_reports_reject_zero_tail_and_remote_activity_mixups() { events.push(json!({"type":"ActivityCompleted","payload":last})); let accepted = json!({"workflow":{"instance_id":"workflow","run_id":"run","details_state":"retained"}, "history_complete":true,"history_events":events}); - assert_eq!(verify_reports(&accepted, "workflow", "run", &[1, 2, 2, 2], true).len(), 4); + assert_eq!( + verify_reports(&accepted, "workflow", "run", &[1, 2, 2, 2], true).len(), + 4 + ); let mut zero_tail = accepted.clone(); for event in zero_tail["history_events"].as_array_mut().unwrap() { - if event["type"] == "ActivityRetryScheduled" && event["payload"]["retry_after_attempt"].as_u64().unwrap() >= 3 { + if event["type"] == "ActivityRetryScheduled" + && event["payload"]["retry_after_attempt"].as_u64().unwrap() >= 3 + { event["payload"]["retry_backoff_seconds"] = json!(0); } } - assert!(std::panic::catch_unwind(|| verify_reports(&zero_tail, "workflow", "run", &[1, 2, 2, 2], true)).is_err()); + assert!(std::panic::catch_unwind(|| verify_reports( + &zero_tail, + "workflow", + "run", + &[1, 2, 2, 2], + true + )) + .is_err()); let mut remote = accepted; remote["history_events"][0]["payload"]["execution_mode"] = json!("remote"); - assert!(std::panic::catch_unwind(|| verify_reports(&remote, "workflow", "run", &[1, 2, 2, 2], true)).is_err()); + assert!(std::panic::catch_unwind(|| verify_reports( + &remote, + "workflow", + "run", + &[1, 2, 2, 2], + true + )) + .is_err()); } From 6de9927e70d1bc5ef08f086867286c76cf62ce4c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:21:18 +0000 Subject: [PATCH 3/4] Bind retry replay signaling and result reads to the selected run --- tests/local_retry_timing_server.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/local_retry_timing_server.rs b/tests/local_retry_timing_server.rs index 4a65475..d013e82 100644 --- a/tests/local_retry_timing_server.rs +++ b/tests/local_retry_timing_server.rs @@ -223,11 +223,11 @@ async fn published_local_retries_repeat_last_and_replay_committed_results() { .unwrap(); let replacement = worker(&client, &id, &kind, intervals, observations.clone(), true); replacement.register().await.unwrap(); - handle.signal("finish", json!([])).await.unwrap(); + handle.signal_selected_run("finish", json!([])).await.unwrap(); let mut terminal = false; for _ in 0..20 { replacement.run_once().await.unwrap(); - let description = handle.describe().await.unwrap(); + let description = handle.describe_selected_run().await.unwrap(); if description.is_terminal() { assert_eq!(description.status.as_deref(), Some("completed")); terminal = true; @@ -240,7 +240,7 @@ async fn published_local_retries_repeat_last_and_replay_committed_results() { ); assert_eq!( handle - .result(WorkflowResultOptions::default()) + .result_selected_run(WorkflowResultOptions::default()) .await .unwrap(), json!([5]) From 9c25dbc17af8d99f4be5633aa06d009526847976 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:24:19 +0000 Subject: [PATCH 4/4] Format the selected-run signal chain --- tests/local_retry_timing_server.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/local_retry_timing_server.rs b/tests/local_retry_timing_server.rs index d013e82..0607e21 100644 --- a/tests/local_retry_timing_server.rs +++ b/tests/local_retry_timing_server.rs @@ -223,7 +223,10 @@ async fn published_local_retries_repeat_last_and_replay_committed_results() { .unwrap(); let replacement = worker(&client, &id, &kind, intervals, observations.clone(), true); replacement.register().await.unwrap(); - handle.signal_selected_run("finish", json!([])).await.unwrap(); + handle + .signal_selected_run("finish", json!([])) + .await + .unwrap(); let mut terminal = false; for _ in 0..20 { replacement.run_once().await.unwrap();