From 89ced9fc5f7149365c6a930119a12818430227a9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:45:19 +0000 Subject: [PATCH 1/4] Preserve fractional persisted replay clocks in redrive --- CHANGELOG.md | 4 + .../Support/DefaultWorkflowControlPlane.php | 2 +- src/V2/Support/FailedRunRedrivePlan.php | 12 +- src/V2/Support/RedriveReplayClock.php | 10 ++ .../V2EmbeddedReplayRegressionCorpusTest.php | 99 +++++++++++++++++ .../redrive-reused-fractional-clock.json | 104 ++++++++++++++++++ .../V2/TestRedriveFractionalClockWorkflow.php | 46 ++++++++ 7 files changed, 270 insertions(+), 7 deletions(-) create mode 100644 tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json create mode 100644 tests/Fixtures/V2/TestRedriveFractionalClockWorkflow.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 8706e54be..49935f328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Failed-run redrive preserves fractional seconds in replay clocks for the + original run start and reused steps. Repeated redrive keeps previously + recorded clock strings unchanged. + - Failed-run redrive accepts a verified linear activity retry chain. A retried completed prefix is reused and only the failed boundary step runs again. Substituted attempt/task identities refuse with diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index 3d9f160d9..c2ff14488 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -268,7 +268,7 @@ public function redrive(string $instanceId, string $failedRunId, array $options ]), 'replayed_started_at' => is_string($startedPayload['replayed_started_at'] ?? null) ? $startedPayload['replayed_started_at'] - : $source->started_at?->toIso8601String(), + : RedriveReplayClock::formatRecordedAt($source->started_at), 'workflow_link_id' => $link->id, ], ), null, $command); diff --git a/src/V2/Support/FailedRunRedrivePlan.php b/src/V2/Support/FailedRunRedrivePlan.php index 50ee2a3fc..140c05fa2 100644 --- a/src/V2/Support/FailedRunRedrivePlan.php +++ b/src/V2/Support/FailedRunRedrivePlan.php @@ -343,7 +343,7 @@ public static function forRun(WorkflowRun $run): array $reusedSteps[] = [ 'event_type' => $marker->event_type, 'payload' => $markerPayload, - 'recorded_at' => $marker->recorded_at?->toIso8601String(), + 'recorded_at' => RedriveReplayClock::formatRecordedAt($marker->recorded_at), ]; continue; @@ -462,7 +462,7 @@ public static function forRun(WorkflowRun $run): array $reusedSteps[] = [ 'event_type' => HistoryEventType::ActivityCompleted, 'payload' => $payload, - 'recorded_at' => $completion->recorded_at?->toIso8601String(), + 'recorded_at' => RedriveReplayClock::formatRecordedAt($completion->recorded_at), ]; } @@ -914,7 +914,7 @@ private static function updateBundles( $bundleEvents[] = [ 'event_type' => $event->event_type, 'payload' => $eventPayload, - 'recorded_at' => $event->recorded_at?->toIso8601String(), + 'recorded_at' => RedriveReplayClock::formatRecordedAt($event->recorded_at), ]; } @@ -1031,7 +1031,7 @@ private static function firedTimerStep(array $events): ?array return [ 'event_type' => HistoryEventType::TimerFired, 'payload' => $firedPayload, - 'recorded_at' => $fired->recorded_at?->toIso8601String(), + 'recorded_at' => RedriveReplayClock::formatRecordedAt($fired->recorded_at), 'scheduled_payload' => $scheduledPayload, ]; } @@ -1182,7 +1182,7 @@ private static function signalSteps(array $events, WorkflowRun $run, array $prov $signalEvents[] = [ 'event_type' => $event->event_type, 'payload' => $eventPayload, - 'recorded_at' => $event->recorded_at?->toIso8601String(), + 'recorded_at' => RedriveReplayClock::formatRecordedAt($event->recorded_at), ]; } @@ -1325,7 +1325,7 @@ private static function completedChildStep(array $events, WorkflowRun $run, arra $childEvents[] = [ 'event_type' => $event->event_type, 'payload' => $eventPayload, - 'recorded_at' => $event->recorded_at?->toIso8601String(), + 'recorded_at' => RedriveReplayClock::formatRecordedAt($event->recorded_at), ]; } diff --git a/src/V2/Support/RedriveReplayClock.php b/src/V2/Support/RedriveReplayClock.php index ef6cc5767..bffab553b 100644 --- a/src/V2/Support/RedriveReplayClock.php +++ b/src/V2/Support/RedriveReplayClock.php @@ -12,6 +12,16 @@ final class RedriveReplayClock { + public static function formatRecordedAt(?CarbonInterface $time): ?string + { + if ($time === null) { + return null; + } + return $time->format('u') === '000000' + ? $time->toIso8601String() + : $time->format('Y-m-d\TH:i:s.uP'); + } + public static function startedAt(WorkflowRun $run): ?CarbonInterface { if ($run->run_number <= 1) { diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 615b3faaf..64a603133 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -23,6 +23,7 @@ use Tests\Fixtures\V2\TestLocalActivityWorkflow; use Tests\Fixtures\V2\TestParallelGreetingChildrenWorkflow; use Tests\Fixtures\V2\TestQueryReplayGuardActivity; +use Tests\Fixtures\V2\TestRedriveFractionalClockWorkflow; use Tests\Fixtures\V2\TestRepeatedPatchClockWorkflow; use Tests\Fixtures\V2\TestSinglePatchReplayWorkflow; use Tests\Fixtures\V2\TestTimerWorkflow; @@ -1616,6 +1617,104 @@ public function testRedriveTypedLineageCorpusLinksBothRunsFromHistory(): void } } + public function testFractionalRedriveClocksSurviveRepeatedNativeAdmissionAndColdReplay(): void + { + $path = self::FIXTURE_DIR . '/redrive-reused-fractional-clock.json'; + if (! is_file($path)) { + $this->markTestSkipped('The fractional redrive clock evidence is absent.'); + } + $fixture = json_decode(file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + $scenario = $fixture['redrive_fractional_clock']; + self::stopWorkers(); + config([ + 'queue.default' => 'sync', + 'queue.connections.sync.driver' => 'sync', + 'workflows.v2.testing.redrive_failures_before_success' => $scenario['redrives'], + ]); + Cache::forget('test:redrive:first-calls'); + Cache::forget('test:redrive:second-calls'); + $this->clearWorkflowState(); + WorkflowStub::fake(); + Carbon::setTestNow(Carbon::parse($scenario['source_times']['started_at'])); + $stub = WorkflowStub::make($fixture['workflow']['type'], 'regression-corpus-fractional-clock'); + $stub->start(...$fixture['workflow']['arguments']); + Carbon::setTestNow(Carbon::parse($scenario['source_times']['timer_fired_at'])); + $timer = WorkflowTask::query() + ->where('workflow_run_id', $stub->runId()) + ->where('task_type', TaskType::Timer->value) + ->sole(); + $this->app->call([new RunTimerTask($timer->id), 'handle']); + $source = WorkflowRun::query()->findOrFail($stub->runId()); + $this->assertSame(RunStatus::Failed, $source->status); + // Give the consumed activity a distinct persisted instant. The real + // admission, timer and failure lifecycle ran before this clock mutation. + WorkflowHistoryEvent::query() + ->where('workflow_run_id', $source->id) + ->where('event_type', HistoryEventType::ActivityCompleted) + ->sole() + ->forceFill([ + 'recorded_at' => Carbon::parse($scenario['source_times']['activity_completed_at']), + ])->save(); + $originalHistory = $source->historyEvents()->get()->toArray(); + $this->assertSame( + $scenario['expected_times'], + (new QueryStateReplayer())->query($source->fresh(), 'recordedTimes', []), + ); + $predecessor = $source; + $firstSerializedClocks = null; + for ($attempt = 1; $attempt <= $scenario['redrives']; ++$attempt) { + Carbon::setTestNow(now()->addMinutes($scenario['redrive_after_minutes'])); + $result = app(WorkflowControlPlane::class)->redrive($source->workflow_instance_id, $predecessor->id); + $this->assertTrue($result['accepted'], (string) $result['reason']); + $successor = WorkflowRun::query()->findOrFail($result['workflow_run_id']); + $this->assertSame( + $attempt === $scenario['redrives'] ? RunStatus::Completed : RunStatus::Failed, + $successor->status, + ); + $this->assertSame( + $scenario['expected_times'], + (new QueryStateReplayer())->query($successor->fresh(), 'recordedTimes', []), + ); + $replayed = (new WorkflowReplayer())->replay($successor->fresh())->workflow; + $this->assertInstanceOf(TestRedriveFractionalClockWorkflow::class, $replayed); + $this->assertSame($scenario['expected_times'], $replayed->recordedTimes()); + $started = $successor->historyEvents() + ->where('event_type', HistoryEventType::WorkflowStarted) + ->sole(); + $activity = $successor->historyEvents() + ->where('event_type', HistoryEventType::ActivityCompleted) + ->orderBy('sequence') + ->firstOrFail(); + $fired = $successor->historyEvents() + ->where('event_type', HistoryEventType::TimerFired) + ->sole(); + $serializedClocks = [ + $started->payload['replayed_started_at'], + $activity->payload['reused_recorded_at'], + $fired->payload['reused_recorded_at'], + ]; + if ($firstSerializedClocks !== null) { + $this->assertSame($firstSerializedClocks, $serializedClocks); + } + $firstSerializedClocks ??= $serializedClocks; + $this->assertSame(0, WorkflowTimer::query() + ->where('workflow_run_id', $successor->id) + ->count()); + $this->assertSame(0, WorkflowTask::query() + ->where('workflow_run_id', $successor->id) + ->where('task_type', TaskType::Timer->value) + ->count()); + $this->assertSame($originalHistory, $source->historyEvents()->get()->toArray()); + $predecessor = $successor; + } + $output = $predecessor->workflowOutput(); + unset($output['result']); + $this->assertSame($scenario['expected_times'], $output); + $this->assertSame(1, (int) Cache::get('test:redrive:first-calls')); + $this->assertSame(3, (int) Cache::get('test:redrive:second-calls')); + Carbon::setTestNow(); + } + public function testRedriveReusedTimerCorpusDoesNotWaitAgain(): void { $path = self::FIXTURE_DIR . '/redrive-reused-timer-clock.json'; diff --git a/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json b/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json new file mode 100644 index 000000000..1ce4b4eaa --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "redrive-reused-fractional-clock", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRedriveFractionalClockWorkflow", + "arguments": [ + "Taylor", + 5 + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "recovery_kind": "redrive", + "resume_step_sequence": 3, + "replayed_started_at": "2026-11-03T00:00:00.123456+00:00" + }, + "recorded_at": "2026-11-03T00:05:00+00:00" + }, + { + "sequence": 2, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro", + "reused_from_run_id": "01JBREDRIVESOURCERUN000000", + "reused_recorded_at": "2026-11-03T00:00:00.234567+00:00" + }, + "recorded_at": "2026-11-03T00:05:00+00:00" + }, + { + "sequence": 3, + "event_type": "TimerScheduled", + "payload": { + "timer_id": "01JBREDRIVESUCCESSORTIMER0", + "sequence": 2, + "delay_seconds": 5, + "fire_at": "2026-11-03T00:00:05.000000Z" + }, + "recorded_at": "2026-11-03T00:05:00+00:00" + }, + { + "sequence": 4, + "event_type": "TimerFired", + "payload": { + "timer_id": "01JBREDRIVESUCCESSORTIMER0", + "sequence": 2, + "delay_seconds": 5, + "fired_at": "2026-11-03T00:00:06.000000Z", + "reused_from_run_id": "01JBREDRIVESOURCERUN000000", + "reused_recorded_at": "2026-11-03T00:00:06.345678+00:00" + }, + "recorded_at": "2026-11-03T00:05:00+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 3, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-03T00:05:01.456789+00:00" + } + ], + "expected": { + "completed": true, + "result": { + "result": "Hello, Taylor!", + "started_at": "1793664000.123456", + "activity_completed_at": "1793664000.234567", + "timer_fired_at": "1793664006.345678" + }, + "commands": [] + }, + "consumers": [ + "workflow-fiber-runner" + ], + "redrive_fractional_clock": { + "source_times": { + "started_at": "2026-11-03T00:00:00.123456+00:00", + "activity_completed_at": "2026-11-03T00:00:00.234567+00:00", + "timer_fired_at": "2026-11-03T00:00:06.345678+00:00" + }, + "redrive_after_minutes": 5, + "redrives": 2, + "expected_times": { + "started_at": "1793664000.123456", + "activity_completed_at": "1793664000.234567", + "timer_fired_at": "1793664006.345678" + } + } +} diff --git a/tests/Fixtures/V2/TestRedriveFractionalClockWorkflow.php b/tests/Fixtures/V2/TestRedriveFractionalClockWorkflow.php new file mode 100644 index 000000000..15ca11c1e --- /dev/null +++ b/tests/Fixtures/V2/TestRedriveFractionalClockWorkflow.php @@ -0,0 +1,46 @@ + + */ + private array $recordedTimes = []; + + /** + * @return array + */ + public function handle(string $name, int $seconds): array + { + $this->recordedTimes['started_at'] = now()->format('U.u'); + $greeting = activity(TestRedriveFirstActivity::class, $name); + $this->recordedTimes['activity_completed_at'] = now()->format('U.u'); + timer($seconds); + $this->recordedTimes['timer_fired_at'] = now()->format('U.u'); + return [ + 'result' => activity(TestRedriveSecondActivity::class, $greeting), + ...$this->recordedTimes, + ]; + } + + /** + * @return array + */ + #[QueryMethod] + public function recordedTimes(): array + { + return $this->recordedTimes; + } +} From d8d57ffd2f733cc474116739f3f3baa600249617 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:52:59 +0000 Subject: [PATCH 2/4] Keep whole-second corpus coverage beside fractional clock controls --- .../V2EmbeddedReplayRegressionCorpusTest.php | 168 ++++++++++-------- .../redrive-reused-fractional-clock.json | 6 +- 2 files changed, 96 insertions(+), 78 deletions(-) diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 64a603133..342afb50d 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -552,7 +552,10 @@ public function testRedriveReusedMarkersCorpusKeepsTheirStepsAndTime(): void ->firstOrFail(); $this->assertSame($step['sequence'], $reused->payload['sequence']); $this->assertSame($sourceRunId, $reused->payload['reused_from_run_id'] ?? null); - $this->assertSame($sourceTimes[$step['sequence']], $reused->payload['reused_recorded_at'] ?? null); + $this->assertSame( + $sourceTimes[$step['sequence']], + Carbon::parse($reused->payload['reused_recorded_at'])->toIso8601String(), + ); } $output = $successor->workflowOutput(); $this->assertSame($scenario['version'], $output['version']); @@ -613,7 +616,10 @@ public function testRedriveReusedUpsertsCorpusKeepsTheirStepsRowsAndTime(): void ->sole(); $this->assertSame($step['sequence'], $reused->payload['sequence']); $this->assertSame($sourceRunId, $reused->payload['reused_from_run_id'] ?? null); - $this->assertSame($sourceTimes[$step['sequence']], $reused->payload['reused_recorded_at'] ?? null); + $this->assertSame( + $sourceTimes[$step['sequence']], + Carbon::parse($reused->payload['reused_recorded_at'])->toIso8601String(), + ); } $this->assertEquals($scenario['memo'], $successor->typedMemos()); $this->assertEquals($scenario['search_attributes'], $successor->typedSearchAttributes()); @@ -681,7 +687,10 @@ public function testRedriveReusedSignalWaitCorpusKeepsItsValueAndTime(): void ); foreach ($reused as $event) { $this->assertSame($sourceRunId, $event->payload['reused_from_run_id'] ?? null); - $this->assertSame($sourceTimes[$event->event_type->value], $event->payload['reused_recorded_at'] ?? null); + $this->assertSame( + $sourceTimes[$event->event_type->value], + Carbon::parse($event->payload['reused_recorded_at'])->toIso8601String(), + ); } $output = $successor->workflowOutput(); $this->assertSame($scenario['signal_value'], $output['approved_by']); @@ -1635,84 +1644,89 @@ public function testFractionalRedriveClocksSurviveRepeatedNativeAdmissionAndCold Cache::forget('test:redrive:second-calls'); $this->clearWorkflowState(); WorkflowStub::fake(); - Carbon::setTestNow(Carbon::parse($scenario['source_times']['started_at'])); - $stub = WorkflowStub::make($fixture['workflow']['type'], 'regression-corpus-fractional-clock'); - $stub->start(...$fixture['workflow']['arguments']); - Carbon::setTestNow(Carbon::parse($scenario['source_times']['timer_fired_at'])); - $timer = WorkflowTask::query() - ->where('workflow_run_id', $stub->runId()) - ->where('task_type', TaskType::Timer->value) - ->sole(); - $this->app->call([new RunTimerTask($timer->id), 'handle']); - $source = WorkflowRun::query()->findOrFail($stub->runId()); - $this->assertSame(RunStatus::Failed, $source->status); - // Give the consumed activity a distinct persisted instant. The real - // admission, timer and failure lifecycle ran before this clock mutation. - WorkflowHistoryEvent::query() - ->where('workflow_run_id', $source->id) - ->where('event_type', HistoryEventType::ActivityCompleted) - ->sole() - ->forceFill([ - 'recorded_at' => Carbon::parse($scenario['source_times']['activity_completed_at']), - ])->save(); - $originalHistory = $source->historyEvents()->get()->toArray(); - $this->assertSame( - $scenario['expected_times'], - (new QueryStateReplayer())->query($source->fresh(), 'recordedTimes', []), - ); - $predecessor = $source; - $firstSerializedClocks = null; - for ($attempt = 1; $attempt <= $scenario['redrives']; ++$attempt) { - Carbon::setTestNow(now()->addMinutes($scenario['redrive_after_minutes'])); - $result = app(WorkflowControlPlane::class)->redrive($source->workflow_instance_id, $predecessor->id); - $this->assertTrue($result['accepted'], (string) $result['reason']); - $successor = WorkflowRun::query()->findOrFail($result['workflow_run_id']); - $this->assertSame( - $attempt === $scenario['redrives'] ? RunStatus::Completed : RunStatus::Failed, - $successor->status, - ); - $this->assertSame( - $scenario['expected_times'], - (new QueryStateReplayer())->query($successor->fresh(), 'recordedTimes', []), - ); - $replayed = (new WorkflowReplayer())->replay($successor->fresh())->workflow; - $this->assertInstanceOf(TestRedriveFractionalClockWorkflow::class, $replayed); - $this->assertSame($scenario['expected_times'], $replayed->recordedTimes()); - $started = $successor->historyEvents() - ->where('event_type', HistoryEventType::WorkflowStarted) + try { + Carbon::setTestNow(Carbon::parse($scenario['source_times']['started_at'])); + $stub = WorkflowStub::make($fixture['workflow']['type'], 'regression-corpus-fractional-clock'); + $stub->start(...$fixture['workflow']['arguments']); + Carbon::setTestNow(Carbon::parse($scenario['source_times']['timer_fired_at'])); + $timer = WorkflowTask::query() + ->where('workflow_run_id', $stub->runId()) + ->where('task_type', TaskType::Timer->value) ->sole(); - $activity = $successor->historyEvents() + $this->app->call([new RunTimerTask($timer->id), 'handle']); + $source = WorkflowRun::query()->findOrFail($stub->runId()); + $this->assertSame(RunStatus::Failed, $source->status); + // Give the consumed activity a distinct persisted instant. The real + // admission, timer and failure lifecycle ran before this clock mutation. + WorkflowHistoryEvent::query() + ->where('workflow_run_id', $source->id) ->where('event_type', HistoryEventType::ActivityCompleted) - ->orderBy('sequence') - ->firstOrFail(); - $fired = $successor->historyEvents() - ->where('event_type', HistoryEventType::TimerFired) - ->sole(); - $serializedClocks = [ - $started->payload['replayed_started_at'], - $activity->payload['reused_recorded_at'], - $fired->payload['reused_recorded_at'], - ]; - if ($firstSerializedClocks !== null) { - $this->assertSame($firstSerializedClocks, $serializedClocks); + ->sole() + ->forceFill([ + 'recorded_at' => Carbon::parse($scenario['source_times']['activity_completed_at']), + ])->save(); + $originalHistory = $source->historyEvents()->get()->toArray(); + $predecessor = $source; + $firstSerializedClocks = null; + for ($attempt = 1; $attempt <= $scenario['redrives']; ++$attempt) { + Carbon::setTestNow(now()->addMinutes($scenario['redrive_after_minutes'])); + $result = app(WorkflowControlPlane::class)->redrive($source->workflow_instance_id, $predecessor->id); + $this->assertTrue($result['accepted'], (string) $result['reason']); + $successor = WorkflowRun::query()->findOrFail($result['workflow_run_id']); + $this->assertSame( + $attempt === $scenario['redrives'] ? RunStatus::Completed : RunStatus::Failed, + $successor->status, + ); + if ($successor->status === RunStatus::Completed) { + $this->assertSame( + $scenario['expected_times'], + (new QueryStateReplayer())->query($successor->fresh(), 'recordedTimes', []), + ); + $replayed = (new WorkflowReplayer())->replay($successor->fresh())->workflow; + $this->assertInstanceOf(TestRedriveFractionalClockWorkflow::class, $replayed); + $this->assertSame($scenario['expected_times'], $replayed->recordedTimes()); + } + $started = $successor->historyEvents() + ->where('event_type', HistoryEventType::WorkflowStarted) + ->sole(); + $activity = $successor->historyEvents() + ->where('event_type', HistoryEventType::ActivityCompleted) + ->orderBy('sequence') + ->firstOrFail(); + $fired = $successor->historyEvents() + ->where('event_type', HistoryEventType::TimerFired) + ->sole(); + $serializedClocks = [ + $started->payload['replayed_started_at'], + $activity->payload['reused_recorded_at'], + $fired->payload['reused_recorded_at'], + ]; + $this->assertSame(array_values($scenario['expected_times']), array_map( + static fn (string $clock): string => Carbon::parse($clock)->format('U.u'), + $serializedClocks, + )); + if ($firstSerializedClocks !== null) { + $this->assertSame($firstSerializedClocks, $serializedClocks); + } + $firstSerializedClocks ??= $serializedClocks; + $this->assertSame(0, WorkflowTimer::query() + ->where('workflow_run_id', $successor->id) + ->count()); + $this->assertSame(0, WorkflowTask::query() + ->where('workflow_run_id', $successor->id) + ->where('task_type', TaskType::Timer->value) + ->count()); + $this->assertSame($originalHistory, $source->historyEvents()->get()->toArray()); + $predecessor = $successor; } - $firstSerializedClocks ??= $serializedClocks; - $this->assertSame(0, WorkflowTimer::query() - ->where('workflow_run_id', $successor->id) - ->count()); - $this->assertSame(0, WorkflowTask::query() - ->where('workflow_run_id', $successor->id) - ->where('task_type', TaskType::Timer->value) - ->count()); - $this->assertSame($originalHistory, $source->historyEvents()->get()->toArray()); - $predecessor = $successor; + $output = $predecessor->workflowOutput(); + unset($output['result']); + $this->assertSame($scenario['expected_times'], $output); + $this->assertSame(1, (int) Cache::get('test:redrive:first-calls')); + $this->assertSame(3, (int) Cache::get('test:redrive:second-calls')); + } finally { + Carbon::setTestNow(); } - $output = $predecessor->workflowOutput(); - unset($output['result']); - $this->assertSame($scenario['expected_times'], $output); - $this->assertSame(1, (int) Cache::get('test:redrive:first-calls')); - $this->assertSame(3, (int) Cache::get('test:redrive:second-calls')); - Carbon::setTestNow(); } public function testRedriveReusedTimerCorpusDoesNotWaitAgain(): void diff --git a/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json b/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json index 1ce4b4eaa..00f5b5f51 100644 --- a/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json +++ b/tests/Fixtures/V2/ReplayRegression/redrive-reused-fractional-clock.json @@ -82,7 +82,11 @@ "activity_completed_at": "1793664000.234567", "timer_fired_at": "1793664006.345678" }, - "commands": [] + "commands": [ + { + "type": "complete_workflow" + } + ] }, "consumers": [ "workflow-fiber-runner" From 3cdab091541a1327c6173b1c1e25df0a2670233e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:08:09 +0000 Subject: [PATCH 3/4] Assert full persisted instants in native redrive controls --- tests/Feature/V2/V2FailedRunRedriveTest.php | 41 +++++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tests/Feature/V2/V2FailedRunRedriveTest.php b/tests/Feature/V2/V2FailedRunRedriveTest.php index 50e44934b..c8c29b224 100644 --- a/tests/Feature/V2/V2FailedRunRedriveTest.php +++ b/tests/Feature/V2/V2FailedRunRedriveTest.php @@ -4,6 +4,8 @@ namespace Tests\Feature\V2; +use Carbon\CarbonInterface; +use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; @@ -553,8 +555,8 @@ public function testRedriveReusesSideEffectsAndVersionMarkersAtTheirRecordedTime $this->assertSame($sourceMarker->payload['version'], $reusedMarker->payload['version']); foreach ([[$sourceSideEffect, $reusedSideEffect], [$sourceMarker, $reusedMarker]] as [$original, $reused]) { $this->assertSame($sourceRunId, $reused->payload['reused_from_run_id'] ?? null); - $this->assertSame( - $original->recorded_at?->toIso8601String(), + $this->assertReplayClock( + $original->recorded_at, $reused->payload['reused_recorded_at'] ?? null, ); } @@ -675,7 +677,7 @@ public function testRedriveReusesWorkflowCodeMemoAndSearchAttributeUpserts(): vo ->sole(); $payload = $reused->payload; $this->assertSame($sourceRunId, $payload['reused_from_run_id'] ?? null); - $this->assertSame($original->recorded_at?->toIso8601String(), $payload['reused_recorded_at'] ?? null); + $this->assertReplayClock($original->recorded_at, $payload['reused_recorded_at'] ?? null); unset($payload['reused_from_run_id'], $payload['reused_recorded_at']); // The source's task and command are its recording context, not the step. $expected = $original->payload; @@ -722,7 +724,7 @@ public function testARedriveSuccessorWithUpsertsCanBeRedrivenAgain(): void ->where('workflow_run_id', $firstRunId) ->where('event_type', HistoryEventType::MemoUpserted) ->sole() - ->recorded_at?->toIso8601String(); + ->recorded_at; $second = app(WorkflowControlPlane::class)->redrive('redrive-upserts-chain', $firstRunId); $this->assertTrue($second['accepted'], (string) $second['reason']); @@ -748,8 +750,8 @@ public function testARedriveSuccessorWithUpsertsCanBeRedrivenAgain(): void ->sole(); // A step reused twice keeps the run it came from and its first time. $this->assertSame($secondRun->id, $reusedMemo->payload['reused_from_run_id']); - $this->assertSame($memoRecordedAt, $reusedMemo->payload['reused_recorded_at']); - $this->assertSame($memoRecordedAt, $thirdRun->workflowOutput()['time_after_memo']); + $this->assertReplayClock($memoRecordedAt, $reusedMemo->payload['reused_recorded_at']); + $this->assertSame($memoRecordedAt?->toIso8601String(), $thirdRun->workflowOutput()['time_after_memo']); } public function testRedriveReusesAnAppliedSignalWaitAtItsRecordedTime(): void @@ -800,8 +802,8 @@ public function testRedriveReusesAnAppliedSignalWaitAtItsRecordedTime(): void foreach ($reused as $event) { $original = $sourceEvents->firstWhere('event_type', $event->event_type); $this->assertSame($sourceRunId, $event->payload['reused_from_run_id'] ?? null); - $this->assertSame( - $original->recorded_at?->toIso8601String(), + $this->assertReplayClock( + $original->recorded_at, $event->payload['reused_recorded_at'] ?? null, ); } @@ -1328,8 +1330,8 @@ public function testRedriveReappliesAnUpdateTheSourceApplied(): void $original = $this->sourceEvent($source, $type); $reused = $this->sourceEvent($successor, $type); $this->assertSame($source->id, $reused->payload['reused_from_run_id'] ?? null); - $this->assertSame( - $original->recorded_at?->toIso8601String(), + $this->assertReplayClock( + $original->recorded_at, $reused->payload['reused_recorded_at'] ?? null ); } @@ -1378,8 +1380,8 @@ public function testAnUpdateAndSignalWaitRedriveAcrossAChain(): void // The immediate source, the original run and its first time. $this->assertSame($secondRun->id, $events[0]->payload['reused_from_run_id'], $type->value); $this->assertSame($first->id, $events[0]->payload['workflow_run_id'] ?? $first->id, $type->value); - $this->assertSame( - $original->recorded_at?->toIso8601String(), + $this->assertReplayClock( + $original->recorded_at, $events[0]->payload['reused_recorded_at'], $type->value, ); @@ -2010,8 +2012,8 @@ public function testRedriveReusesAFiredTimerAtItsRecordedTime(): void $this->assertSame($reusedScheduled->payload['timer_id'], $reusedFired->payload['timer_id']); $this->assertNotSame($sourceFired->payload['timer_id'], $reusedFired->payload['timer_id']); $this->assertSame($sourceRunId, $reusedFired->payload['reused_from_run_id'] ?? null); - $this->assertSame( - $sourceFired->recorded_at?->toIso8601String(), + $this->assertReplayClock( + $sourceFired->recorded_at, $reusedFired->payload['reused_recorded_at'] ?? null, ); @@ -2124,8 +2126,8 @@ public function testRedriveReusesACompletedChildWorkflowAtItsRecordedTime(): voi foreach ($reused as $index => $event) { $original = $sourceChildEvents[$index]; $this->assertSame($immediateSource, $event->payload['reused_from_run_id'] ?? null); - $this->assertSame( - $original->recorded_at?->toIso8601String(), + $this->assertReplayClock( + $original->recorded_at, $event->payload['reused_recorded_at'] ?? null, ); $this->assertSame($childRunId, $event->payload['child_workflow_run_id']); @@ -2679,4 +2681,11 @@ private function assertRedriveAction( $this->assertArrayHasKey('redrive_blocked_reason', $describe['actions']); $this->assertSame($reason, $describe['actions']['redrive_blocked_reason'], "describe reason for {$runId}"); } + private function assertReplayClock(?CarbonInterface $expected, mixed $actual, string $message = ''): void + { + $this->assertNotNull($expected, $message); + $this->assertIsString($actual, $message); + $this->assertSame($expected->format('U.u'), Carbon::parse($actual)->format('U.u'), $message); + } + } From a990a42e463cc07cf84f40393670d8412115ea23 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:23:49 +0000 Subject: [PATCH 4/4] Format fractional clock regression controls --- .../V2EmbeddedReplayRegressionCorpusTest.php | 11 ++++---- tests/Feature/V2/V2FailedRunRedriveTest.php | 27 +++++-------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 342afb50d..91272d64f 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -1665,7 +1665,9 @@ public function testFractionalRedriveClocksSurviveRepeatedNativeAdmissionAndCold ->forceFill([ 'recorded_at' => Carbon::parse($scenario['source_times']['activity_completed_at']), ])->save(); - $originalHistory = $source->historyEvents()->get()->toArray(); + $originalHistory = $source->historyEvents() + ->get() + ->toArray(); $predecessor = $source; $firstSerializedClocks = null; for ($attempt = 1; $attempt <= $scenario['redrives']; ++$attempt) { @@ -1682,7 +1684,8 @@ public function testFractionalRedriveClocksSurviveRepeatedNativeAdmissionAndCold $scenario['expected_times'], (new QueryStateReplayer())->query($successor->fresh(), 'recordedTimes', []), ); - $replayed = (new WorkflowReplayer())->replay($successor->fresh())->workflow; + $replayed = (new WorkflowReplayer())->replay($successor->fresh()) +->workflow; $this->assertInstanceOf(TestRedriveFractionalClockWorkflow::class, $replayed); $this->assertSame($scenario['expected_times'], $replayed->recordedTimes()); } @@ -1709,9 +1712,7 @@ public function testFractionalRedriveClocksSurviveRepeatedNativeAdmissionAndCold $this->assertSame($firstSerializedClocks, $serializedClocks); } $firstSerializedClocks ??= $serializedClocks; - $this->assertSame(0, WorkflowTimer::query() - ->where('workflow_run_id', $successor->id) - ->count()); + $this->assertSame(0, WorkflowTimer::query() ->where('workflow_run_id', $successor->id) ->count()); $this->assertSame(0, WorkflowTask::query() ->where('workflow_run_id', $successor->id) ->where('task_type', TaskType::Timer->value) diff --git a/tests/Feature/V2/V2FailedRunRedriveTest.php b/tests/Feature/V2/V2FailedRunRedriveTest.php index c8c29b224..ce01d5a39 100644 --- a/tests/Feature/V2/V2FailedRunRedriveTest.php +++ b/tests/Feature/V2/V2FailedRunRedriveTest.php @@ -555,10 +555,7 @@ public function testRedriveReusesSideEffectsAndVersionMarkersAtTheirRecordedTime $this->assertSame($sourceMarker->payload['version'], $reusedMarker->payload['version']); foreach ([[$sourceSideEffect, $reusedSideEffect], [$sourceMarker, $reusedMarker]] as [$original, $reused]) { $this->assertSame($sourceRunId, $reused->payload['reused_from_run_id'] ?? null); - $this->assertReplayClock( - $original->recorded_at, - $reused->payload['reused_recorded_at'] ?? null, - ); + $this->assertReplayClock($original->recorded_at, $reused->payload['reused_recorded_at'] ?? null); } $expectedTimes = [ @@ -802,10 +799,7 @@ public function testRedriveReusesAnAppliedSignalWaitAtItsRecordedTime(): void foreach ($reused as $event) { $original = $sourceEvents->firstWhere('event_type', $event->event_type); $this->assertSame($sourceRunId, $event->payload['reused_from_run_id'] ?? null); - $this->assertReplayClock( - $original->recorded_at, - $event->payload['reused_recorded_at'] ?? null, - ); + $this->assertReplayClock($original->recorded_at, $event->payload['reused_recorded_at'] ?? null); } $this->assertSame(0, WorkflowSignal::query()->where('workflow_run_id', $successor->id)->count()); // The consumed message stays consumed for the successor. @@ -1330,10 +1324,7 @@ public function testRedriveReappliesAnUpdateTheSourceApplied(): void $original = $this->sourceEvent($source, $type); $reused = $this->sourceEvent($successor, $type); $this->assertSame($source->id, $reused->payload['reused_from_run_id'] ?? null); - $this->assertReplayClock( - $original->recorded_at, - $reused->payload['reused_recorded_at'] ?? null - ); + $this->assertReplayClock($original->recorded_at, $reused->payload['reused_recorded_at'] ?? null); } } @@ -2012,10 +2003,7 @@ public function testRedriveReusesAFiredTimerAtItsRecordedTime(): void $this->assertSame($reusedScheduled->payload['timer_id'], $reusedFired->payload['timer_id']); $this->assertNotSame($sourceFired->payload['timer_id'], $reusedFired->payload['timer_id']); $this->assertSame($sourceRunId, $reusedFired->payload['reused_from_run_id'] ?? null); - $this->assertReplayClock( - $sourceFired->recorded_at, - $reusedFired->payload['reused_recorded_at'] ?? null, - ); + $this->assertReplayClock($sourceFired->recorded_at, $reusedFired->payload['reused_recorded_at'] ?? null); $expectedTime = $sourceFired->recorded_at?->toIso8601String(); $output = $successor->workflowOutput(); @@ -2126,10 +2114,7 @@ public function testRedriveReusesACompletedChildWorkflowAtItsRecordedTime(): voi foreach ($reused as $index => $event) { $original = $sourceChildEvents[$index]; $this->assertSame($immediateSource, $event->payload['reused_from_run_id'] ?? null); - $this->assertReplayClock( - $original->recorded_at, - $event->payload['reused_recorded_at'] ?? null, - ); + $this->assertReplayClock($original->recorded_at, $event->payload['reused_recorded_at'] ?? null); $this->assertSame($childRunId, $event->payload['child_workflow_run_id']); $this->assertSame($original->payload['workflow_link_id'], $event->payload['workflow_link_id']); $this->assertArrayNotHasKey('task', $event->payload); @@ -2681,11 +2666,11 @@ private function assertRedriveAction( $this->assertArrayHasKey('redrive_blocked_reason', $describe['actions']); $this->assertSame($reason, $describe['actions']['redrive_blocked_reason'], "describe reason for {$runId}"); } + private function assertReplayClock(?CarbonInterface $expected, mixed $actual, string $message = ''): void { $this->assertNotNull($expected, $message); $this->assertIsString($actual, $message); $this->assertSame($expected->format('U.u'), Carbon::parse($actual)->format('U.u'), $message); } - }