diff --git a/scripts/acceptance/installed-native/README.md b/scripts/acceptance/installed-native/README.md index 924a49b..2b47bbe 100644 --- a/scripts/acceptance/installed-native/README.md +++ b/scripts/acceptance/installed-native/README.md @@ -66,6 +66,26 @@ Record the package checksum, Server digest, runner commit and cleanup alongside the result. Compiling the ignored test or using source dependencies is not published-artifact acceptance. +### Local retry-delay observation + +`tests/local_retry_timing_server.rs` covers shared #219 through the ordinary +published SDK Worker path. Copy it into a fresh exact-registry consumer with +Tokio and serde_json. Bind the selected qualified native tuple and set +`DURABLE_WORKFLOW_EXPECT_SERVER_VERSION` alongside the existing isolated +numeric-loopback URL and synthetic-token variables. Run only +`published_local_retries_repeat_last_and_replay_committed_results` with +`--ignored --exact --nocapture`. + +Five local attempts with `[1, 2]` must record four retry delays `[1, 2, 2, 2]` +and independently measured callback gaps no shorter than those delays. The +empty-list case records four explicit zeros. Each case binds the run, logical +activity and five attempts through public local ActivityRetryScheduled events. +A replacement Worker instance then replays the committed local result after a +signal, without another callback or changed history prefix. Empty-list gap +measurements are retained without promising zero execution or queue latency. +The caller owns exact package/build/producer/resource and cleanup evidence; +compilation or the ordinary synthetic observer control is not installed proof. + ### Replay-aware logging feature acceptance `tests/workflow_logging_server.rs` checks the logging feature separately from diff --git a/tests/local_retry_timing_server.rs b/tests/local_retry_timing_server.rs new file mode 100644 index 0000000..0607e21 --- /dev/null +++ b/tests/local_retry_timing_server.rs @@ -0,0 +1,352 @@ +//! Published-package observation of local retry delays for shared issue #219. +//! The caller owns the isolated Server, exact package/producer proof and cleanup. +use durable_workflow::{ + json, ActivityError, ActivityRetryPolicy, Client, LocalActivityOptions, Value, Worker, + WorkflowResultOptions, +}; +use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +fn retry_policy(intervals: &[u64]) -> ActivityRetryPolicy { + ActivityRetryPolicy::new(5) + .backoff_intervals(intervals.iter().copied().map(Duration::from_secs)) +} + +fn worker( + client: &Client, + queue: &str, + kind: &str, + intervals: Vec, + observations: Arc>>, + replacement: bool, +) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(format!("{queue}-{replacement}")) + .local_activities(true) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow(kind, move |ctx, _| { + let policy = retry_policy(&intervals); + async move { + let value = ctx + .local_activity_with_options( + "retry-probe", + LocalActivityOptions::new() + .retry_policy(policy) + .start_to_close_timeout(Duration::from_secs(3)) + .schedule_to_close_timeout(Duration::from_secs(30)), + json!([]), + ) + .await?; + ctx.wait_signal("finish").await?; + Ok(value) + } + }); + worker.register_activity("retry-probe", move |ctx, _| { + observations + .lock() + .unwrap() + .push((ctx.attempt_number, Instant::now())); + assert!( + !replacement, + "committed local results must not call the handler" + ); + async move { + if ctx.attempt_number < 5 { + return Err(ActivityError::new("RetryProbe", "declared retry").into()); + } + Ok(json!([5])) + } + }); + worker +} + +fn activity_events(history: &Value) -> Vec<&Value> { + history["history_events"] + .as_array() + .unwrap() + .iter() + .filter(|event| { + matches!( + event["type"].as_str(), + Some( + "ActivityScheduled" + | "ActivityStarted" + | "ActivityRetryScheduled" + | "ActivityCompleted" + ) + ) + }) + .collect() +} + +fn verify_reports( + history: &Value, + workflow: &str, + run: &str, + expected: &[u64], + terminal: bool, +) -> Vec { + assert_eq!(history["workflow"]["instance_id"], workflow); + assert_eq!(history["workflow"]["run_id"], run); + assert_eq!(history["workflow"]["details_state"], "retained"); + assert_eq!(history["history_complete"], terminal); + let events = activity_events(history); + let scheduled: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityScheduled") + .collect(); + let completed: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityCompleted") + .collect(); + assert_eq!(scheduled.len(), 1); + assert_eq!(completed.len(), 1); + let execution = scheduled[0]["payload"]["activity_execution_id"] + .as_str() + .unwrap(); + assert!(!execution.is_empty()); + for event in &events { + let payload = &event["payload"]; + assert_eq!(payload["sequence"], 1); + assert_eq!(payload["activity_execution_id"], execution); + assert_eq!(payload["execution_mode"], "local"); + assert_eq!(payload["local_activity"], true); + } + let starts: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityStarted") + .collect(); + assert_eq!(starts.len(), 5); + let mut attempts = BTreeSet::new(); + for (index, event) in starts.iter().enumerate() { + assert_eq!(event["payload"]["attempt_number"], index + 1); + let id = event["payload"]["activity_attempt_id"].as_str().unwrap(); + assert!(!id.is_empty() && attempts.insert(id)); + } + let retries: Vec<_> = events + .iter() + .filter(|event| event["type"] == "ActivityRetryScheduled") + .collect(); + assert_eq!(retries.len(), expected.len()); + assert_eq!(expected.len(), 4); + for (index, (event, delay)) in retries.iter().zip(expected).enumerate() { + let payload = &event["payload"]; + assert_eq!(payload["retry_after_attempt"], index + 1); + assert_eq!( + payload["retry_after_attempt_id"], + starts[index]["payload"]["activity_attempt_id"] + ); + assert_eq!(payload["retry_backoff_seconds"].as_u64(), Some(*delay)); + } + assert_eq!(completed[0]["payload"]["attempt_number"], 5); + retries + .into_iter() + .map(|event| event["payload"].clone()) + .collect() +} + +fn verify_gaps(observations: &[(u64, Instant)], expected: &[u64]) -> Vec { + assert_eq!(observations.len(), 5); + assert_eq!(expected.len(), 4); + for (index, (attempt, _)) in observations.iter().enumerate() { + assert_eq!(*attempt, (index + 1) as u64); + } + observations + .windows(2) + .zip(expected) + .map(|(pair, seconds)| { + let gap = pair[1].1.duration_since(pair[0].1); + assert!( + gap >= Duration::from_secs(*seconds), + "retry ran before its declared delay" + ); + gap.as_millis() + }) + .collect() +} + +#[tokio::test] +#[ignore = "requires an exact published SDK and isolated qualified Server tuple"] +async fn published_local_retries_repeat_last_and_replay_committed_results() { + assert_eq!( + std::env::var("DURABLE_WORKFLOW_LOCAL_ISOLATED").as_deref(), + Ok("1") + ); + let origin = std::env::var("DURABLE_WORKFLOW_SERVER_URL").unwrap(); + assert!(origin + .strip_prefix("http://127.0.0.1:") + .and_then(|port| port.parse::().ok()) + .is_some_and(|port| port > 0)); + let expected_version = std::env::var("DURABLE_WORKFLOW_EXPECT_SERVER_VERSION").unwrap(); + assert!(!expected_version.is_empty()); + let client = Client::builder(origin) + .namespace("default") + .timeout(Duration::from_secs(3)) + .token(Some(std::env::var("DURABLE_WORKFLOW_AUTH_TOKEN").unwrap())) + .build() + .unwrap(); + let cluster = client.cluster_info().await.unwrap(); + assert_eq!(cluster["implementation"], "rust"); + assert_eq!(cluster["version"], expected_version); + for (name, intervals, expected) in [ + ("short", vec![1, 2], vec![1, 2, 2, 2]), + ("empty", vec![], vec![0, 0, 0, 0]), + ] { + tokio::time::timeout(Duration::from_secs(35), async { + let id = format!("rust-retry-{name}-{}", durable_workflow::Uuid::new_v4()); + let kind = format!("qualification.local-retry.{id}"); + let observations = Arc::new(Mutex::new(Vec::new())); + let original = worker( + &client, + &id, + &kind, + intervals.clone(), + observations.clone(), + false, + ); + original.register().await.unwrap(); + let handle = client + .start_workflow(&kind, &id, &id, json!([])) + .await + .unwrap(); + let run = handle.run_id.as_deref().unwrap(); + assert!(original.run_once().await.unwrap() > 0); + let committed = client.export_workflow_history(&id, run).await.unwrap(); + let reports = verify_reports(&committed, &id, run, &expected, false); + let gaps = verify_gaps(&observations.lock().unwrap(), &expected); + client + .deregister_worker_registration(&format!("{id}-false")) + .await + .unwrap(); + let replacement = worker(&client, &id, &kind, intervals, observations.clone(), true); + replacement.register().await.unwrap(); + handle + .signal_selected_run("finish", json!([])) + .await + .unwrap(); + let mut terminal = false; + for _ in 0..20 { + replacement.run_once().await.unwrap(); + let description = handle.describe_selected_run().await.unwrap(); + if description.is_terminal() { + assert_eq!(description.status.as_deref(), Some("completed")); + terminal = true; + break; + } + } + assert!( + terminal, + "replacement did not complete within its polling bound" + ); + assert_eq!( + handle + .result_selected_run(WorkflowResultOptions::default()) + .await + .unwrap(), + json!([5]) + ); + assert_eq!(observations.lock().unwrap().len(), 5); + let final_history = client.export_workflow_history(&id, run).await.unwrap(); + assert_eq!( + verify_reports(&final_history, &id, run, &expected, true), + reports + ); + let prefix = committed["history_events"].as_array().unwrap(); + assert_eq!( + &final_history["history_events"].as_array().unwrap()[..prefix.len()], + prefix + ); + client + .deregister_worker_registration(&format!("{id}-true")) + .await + .unwrap(); + let report = json!({"case":name,"workflow_id":id,"run_id":run, + "expected_backoff_seconds":expected,"callback_gap_ms":gaps,"retry_reports":reports, + "callback_attempts":5,"callbacks_after_replay":0,"history":final_history}); + let text = report.to_string(); + assert!(text.len() <= 512 * 1024); + println!("LOCAL_RETRY_TIMING {text}"); + }) + .await + .expect("one local retry case exceeded its fixed bound"); + } +} + +#[test] +fn callback_observer_rejects_zero_tail_and_accepts_repeat_last() { + let start = Instant::now(); + let observed = |times: &[u64]| { + times + .iter() + .enumerate() + .map(|(i, seconds)| ((i + 1) as u64, start + Duration::from_secs(*seconds))) + .collect::>() + }; + assert_eq!( + verify_gaps(&observed(&[0, 1, 3, 5, 7]), &[1, 2, 2, 2]), + vec![1000, 2000, 2000, 2000] + ); + assert!( + std::panic::catch_unwind(|| verify_gaps(&observed(&[0, 1, 3, 3, 3]), &[1, 2, 2, 2])) + .is_err() + ); + assert_eq!( + verify_gaps(&observed(&[0, 0, 0, 0, 0]), &[0; 4]), + vec![0; 4] + ); +} + +#[test] +fn committed_reports_reject_zero_tail_and_remote_activity_mixups() { + let mut events = vec![json!({"type":"ActivityScheduled","payload":{ + "sequence":1,"activity_execution_id":"local-id","execution_mode":"local","local_activity":true}})]; + for number in 1..=5 { + let payload = json!({"sequence":1,"activity_execution_id":"local-id","execution_mode":"local", + "local_activity":true,"attempt_number":number,"activity_attempt_id":format!("attempt-{number}")}); + events.push(json!({"type":"ActivityStarted","payload":payload})); + if number < 5 { + let mut retry = events.last().unwrap()["payload"].clone(); + retry["retry_after_attempt"] = json!(number); + retry["retry_after_attempt_id"] = json!(format!("attempt-{number}")); + retry["retry_backoff_seconds"] = json!(if number == 1 { 1 } else { 2 }); + events.push(json!({"type":"ActivityRetryScheduled","payload":retry})); + } + } + let last = events.last().unwrap()["payload"].clone(); + events.push(json!({"type":"ActivityCompleted","payload":last})); + let accepted = json!({"workflow":{"instance_id":"workflow","run_id":"run","details_state":"retained"}, + "history_complete":true,"history_events":events}); + assert_eq!( + verify_reports(&accepted, "workflow", "run", &[1, 2, 2, 2], true).len(), + 4 + ); + let mut zero_tail = accepted.clone(); + for event in zero_tail["history_events"].as_array_mut().unwrap() { + if event["type"] == "ActivityRetryScheduled" + && event["payload"]["retry_after_attempt"].as_u64().unwrap() >= 3 + { + event["payload"]["retry_backoff_seconds"] = json!(0); + } + } + assert!(std::panic::catch_unwind(|| verify_reports( + &zero_tail, + "workflow", + "run", + &[1, 2, 2, 2], + true + )) + .is_err()); + let mut remote = accepted; + remote["history_events"][0]["payload"]["execution_mode"] = json!("remote"); + assert!(std::panic::catch_unwind(|| verify_reports( + &remote, + "workflow", + "run", + &[1, 2, 2, 2], + true + )) + .is_err()); +}