From cebeab3b15505a314f8dc8d522e2f0f45509a85a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:43:20 +0000 Subject: [PATCH 01/32] Allow redrive through validated activity retry histories --- CHANGELOG.md | 8 + regression-corpus-policy.json | 6 + src/V2/Support/ActivityRetryHistory.php | 85 ++++ src/V2/Support/FailedRunRedrivePlan.php | 42 +- tests/Feature/V2/V2FailedRunRedriveTest.php | 128 ++++- .../redrive-native-active-start-adoption.json | 229 +++++++++ ...etried-activity-after-completed-child.json | 441 ++++++++++++++++++ .../redrive-retried-completed-activity.json | 332 +++++++++++++ .../redrive-retried-failed-activity.json | 210 +++++++++ .../V2/TestRedriveRetryProbeActivity.php | 29 ++ .../V2/TestRedriveRetryProbeWorkflow.php | 29 ++ .../Fixtures/V2/TestRedriveRetryWorkflow.php | 20 + tests/Unit/V2/FailedRunRedrivePlanTest.php | 220 +++++++++ tests/Unit/V2/ReplayRegressionCorpusTest.php | 39 ++ 14 files changed, 1816 insertions(+), 2 deletions(-) create mode 100644 src/V2/Support/ActivityRetryHistory.php create mode 100644 tests/Fixtures/V2/ReplayRegression/redrive-native-active-start-adoption.json create mode 100644 tests/Fixtures/V2/ReplayRegression/redrive-retried-activity-after-completed-child.json create mode 100644 tests/Fixtures/V2/ReplayRegression/redrive-retried-completed-activity.json create mode 100644 tests/Fixtures/V2/ReplayRegression/redrive-retried-failed-activity.json create mode 100644 tests/Fixtures/V2/TestRedriveRetryProbeActivity.php create mode 100644 tests/Fixtures/V2/TestRedriveRetryProbeWorkflow.php create mode 100644 tests/Fixtures/V2/TestRedriveRetryWorkflow.php diff --git a/CHANGELOG.md b/CHANGELOG.md index ef222ece1..8706e54be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- Failed-run redrive accepts a verified linear activity retry chain. A retried + completed prefix is reused and only the failed boundary step runs again. + Substituted attempt/task identities refuse with + `invalid_activity_retry_history`; existing service-worker definition checks + still apply. An exactly bound accepted start/adopt receipt is harmless, while + rejected or mismatched starts remain unsupported. This core correction is + separate from the experimental Saga compensation scope foundation (#1063). + - **Run migrations.** Opt-in deadline-triggered cleanup (#863, experimental, embedded). A start can reserve the last part of its hard execution or run timeout for cleanup with `StartOptions::withTimeoutCleanup($seconds)` (or diff --git a/regression-corpus-policy.json b/regression-corpus-policy.json index 114529c98..2df39af3d 100644 --- a/regression-corpus-policy.json +++ b/regression-corpus-policy.json @@ -74,6 +74,12 @@ "history_sequence", "history_payload" ] + }, + { + "glob": "src/V2/Support/FailedRunRedrivePlan.php" + }, + { + "glob": "src/V2/Support/ActivityRetryHistory.php" } ] } diff --git a/src/V2/Support/ActivityRetryHistory.php b/src/V2/Support/ActivityRetryHistory.php new file mode 100644 index 000000000..dd9b84516 --- /dev/null +++ b/src/V2/Support/ActivityRetryHistory.php @@ -0,0 +1,85 @@ + $events + */ + public static function valid(array $events): bool + { + $hasRetries = false; + foreach ($events as $event) { + $hasRetries = $hasRetries || $event->event_type === HistoryEventType::ActivityRetryScheduled; + } + if (! $hasRetries) { + return true; + } + $execution = null; + $attempt = null; + $task = null; + $number = 0; + $nextTask = null; + $terminal = false; + $seenAttempts = []; + $seenTasks = []; + foreach ($events as $event) { + $payload = $event->payload; + $id = $payload['activity_execution_id'] ?? null; + if (! is_string($id) || $id === '' || ($execution !== null && $id !== $execution) || $terminal) { + return false; + } + $execution = $id; + if ($event->event_type === HistoryEventType::ActivityStarted) { + $currentAttempt = $payload['activity_attempt_id'] ?? null; + $currentTask = $event->workflow_task_id ?? ($payload['workflow_task_id'] ?? null); + if (isset($event->workflow_task_id, $payload['workflow_task_id']) && $event->workflow_task_id !== $payload['workflow_task_id']) { + return false; + } + if (! is_string($currentAttempt) || $currentAttempt === '' || isset($seenAttempts[$currentAttempt]) + || ! is_string($currentTask) || $currentTask === '' || isset($seenTasks[$currentTask]) + || ($payload['attempt_number'] ?? null) !== $number + 1 + || ($number > 0 && $nextTask !== $currentTask)) { + return false; + } + $attempt = $currentAttempt; + $task = $currentTask; + $seenAttempts[$attempt] = true; + $seenTasks[$task] = true; + ++$number; + $nextTask = null; + } elseif ($event->event_type === HistoryEventType::ActivityRetryScheduled) { + $retry = $payload['retry_task_id'] ?? null; + $maximum = $payload['max_attempts'] ?? null; + if ($attempt === null || $nextTask !== null + || ($payload['activity_attempt_id'] ?? null) !== $attempt + || ($payload['retry_after_attempt_id'] ?? null) !== $attempt + || ($payload['retry_after_attempt'] ?? null) !== $number + || ($payload['retry_of_task_id'] ?? null) !== $task + || ! is_string($retry) || $retry === '' || isset($seenTasks[$retry]) + || ($maximum !== null && (! is_int($maximum) || $maximum <= $number))) { + return false; + } + $nextTask = $retry; + } elseif (in_array( + $event->event_type, + [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed], + true + )) { + if ($nextTask !== null || $attempt === null || ($payload['activity_attempt_id'] ?? null) !== $attempt + || ($payload['attempt_number'] ?? null) !== $number) { + return false; + } + $terminal = true; + } + } + return $terminal; + } +} diff --git a/src/V2/Support/FailedRunRedrivePlan.php b/src/V2/Support/FailedRunRedrivePlan.php index ea39c75c8..50ee2a3fc 100644 --- a/src/V2/Support/FailedRunRedrivePlan.php +++ b/src/V2/Support/FailedRunRedrivePlan.php @@ -7,6 +7,9 @@ use Illuminate\Support\Str; use Throwable; use Workflow\Serializers\Serializer; +use Workflow\V2\Enums\CommandOutcome; +use Workflow\V2\Enums\CommandStatus; +use Workflow\V2\Enums\CommandType; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Models\WorkflowHistoryEvent; @@ -86,7 +89,7 @@ public static function forRun(WorkflowRun $run): array } if ($event->event_type === HistoryEventType::StartAccepted) { - if ($started !== null) { + if ($started !== null && ! self::isActiveStartAdoption($run, $event)) { return self::reject('unexpected_start'); } @@ -199,6 +202,7 @@ public static function forRun(WorkflowRun $run): array if (! in_array($event->event_type, [ HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted, + HistoryEventType::ActivityRetryScheduled, HistoryEventType::ActivityHeartbeatRecorded, HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, @@ -381,6 +385,10 @@ public static function forRun(WorkflowRun $run): array return self::reject('activity_sequence_gap'); } + if (! ActivityRetryHistory::valid($stepEvents)) { + return self::reject('invalid_activity_retry_history'); + } + $activityType = null; $completion = null; $failureCount = 0; @@ -612,6 +620,38 @@ public static function successorEvents(array $plan, string $sourceRunId): array return $events; } + /** + * Native start/adopt records a command receipt, not another workflow start or reusable step. + * Only accepted adoption proves the same resolved run and command identity. + * A rejected start is not adoption evidence and retains its existing refusal. + */ + private static function isActiveStartAdoption(WorkflowRun $run, WorkflowHistoryEvent $event): bool + { + $payload = $event->payload; + $command = $payload['command'] ?? null; + if (! is_array($command) || ($payload['outcome'] ?? null) !== CommandOutcome::ReturnedExistingActive->value + || ($payload['rejection_reason'] ?? null) !== null + || $event->workflow_run_id !== $run->id) { + return false; + } + foreach (['workflow_instance_id', 'workflow_run_id', 'workflow_class', 'workflow_type'] as $key) { + $expected = $key === 'workflow_run_id' ? $run->id : $run->getAttribute($key); + if (! is_string($expected) || $expected === '' || ($payload[$key] ?? null) !== $expected) { + return false; + } + } + $commandId = $payload['workflow_command_id'] ?? null; + return is_string($commandId) && $commandId !== '' + && $event->workflow_command_id === $commandId + && ($command['id'] ?? null) === $commandId + && ($command['type'] ?? null) === CommandType::Start->value + && ($command['target_scope'] ?? null) === 'instance' + && ($command['resolved_run_id'] ?? null) === $run->id + && ($command['status'] ?? null) === CommandStatus::Accepted->value + && ($command['outcome'] ?? null) === CommandOutcome::ReturnedExistingActive->value + && ($command['rejection_reason'] ?? null) === null; + } + /** * Who may have recorded a reused-step payload in this run's history: the * run itself, or, for a step it reused, its immediate redrive source while diff --git a/tests/Feature/V2/V2FailedRunRedriveTest.php b/tests/Feature/V2/V2FailedRunRedriveTest.php index 6edb98ef4..f1bed9890 100644 --- a/tests/Feature/V2/V2FailedRunRedriveTest.php +++ b/tests/Feature/V2/V2FailedRunRedriveTest.php @@ -7,6 +7,7 @@ use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use PHPUnit\Framework\Attributes\DataProvider; use Symfony\Component\Process\Process; use Tests\Fixtures\V2\TestGreetingWorkflow; use Tests\Fixtures\V2\TestParentFailingChildWorkflow; @@ -17,6 +18,7 @@ use Tests\Fixtures\V2\TestRedriveIncompatibleWorkflow; use Tests\Fixtures\V2\TestRedriveMarkersWorkflow; use Tests\Fixtures\V2\TestRedrivePatchedWorkflow; +use Tests\Fixtures\V2\TestRedriveRetryProbeWorkflow; use Tests\Fixtures\V2\TestRedriveSignalWorkflow; use Tests\Fixtures\V2\TestRedriveTimerWorkflow; use Tests\Fixtures\V2\TestRedriveTwoSignalsWorkflow; @@ -33,7 +35,9 @@ use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; +use Workflow\V2\Jobs\RunActivityTask; use Workflow\V2\Jobs\RunTimerTask; +use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; @@ -185,6 +189,64 @@ public function testFailedRunReusesCompletedActivityAndRetriesOnlyFailedActivity ); } + /** + * @return array + */ + public static function retriedActivityCases(): array + { + return [ + 'retried failed boundary' => [0], + 'retried completed prefix and failed boundary' => [1], + ]; + } + + #[DataProvider('retriedActivityCases')] + public function testOrdinaryRetriedActivitiesRedriveWithoutRepeatingTheCompletedPrefix(int $firstFailures): void + { + Queue::fake(); + config([ + 'workflows.v2.testing.redrive_retry_failures' => [ + 'first' => $firstFailures, + 'second' => 2, + ], + ]); + $counter = 'test:redrive:retry-probe:' . $firstFailures; + Cache::forget($counter . ':first'); + Cache::forget($counter . ':second'); + $workflow = WorkflowStub::make(TestRedriveRetryProbeWorkflow::class, 'redrive-retry-probe-' . $firstFailures); + $workflow->start($counter, new StartOptions(executionTimeoutSeconds: 600)); + $source = $this->drainRetryProbe($workflow->runId()); + $this->assertSame(RunStatus::Failed, $source->status); + $this->assertSame(1 + $firstFailures, (int) Cache::get($counter . ':first')); + $this->assertSame(2, (int) Cache::get($counter . ':second')); + $history = $source->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + + $control = app(WorkflowControlPlane::class); + $redrive = $control->redrive($source->workflow_instance_id, $source->id, [ + 'request_id' => 'retry-probe', + ]); + $this->assertTrue($redrive['accepted'], (string) $redrive['reason']); + $this->assertSame(2, $redrive['resume_step_sequence']); + $successor = $this->drainRetryProbe($redrive['workflow_run_id']); + $this->assertSame(RunStatus::Completed, $successor->status); + $this->assertSame('value:first:second', $successor->workflowOutput()); + $this->assertSame(1 + $firstFailures, (int) Cache::get($counter . ':first')); + $this->assertSame(3, (int) Cache::get($counter . ':second')); + $this->assertSame( + $source->execution_deadline_at->toISOString(), + $successor->execution_deadline_at->toISOString() + ); + $this->assertSame($history, $source->historyEvents()->orderBy('sequence')->get()->toArray()); + $repeated = $control->redrive($source->workflow_instance_id, $source->id, [ + 'request_id' => 'retry-probe', + ]); + $this->assertSame($successor->id, $repeated['workflow_run_id']); + $this->assertSame(3, (int) Cache::get($counter . ':second')); + } + public function testRedriveCarriesStartMemoAndSearchAttributesToTheSuccessor(): void { WorkflowStub::fake(); @@ -263,7 +325,19 @@ public function testCompletedRunIsNotEligibleForRedrive(): void $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', 'redrive-completed-1')->count()); } - public function testServiceRunRequiresMatchingRecordedWorkerDefinitionFingerprint(): void + /** + * @return array + */ + public static function serviceRetryHistories(): array + { + return [ + 'ordinary prefix' => [false], + 'retried completed prefix' => [true], + ]; + } + + #[DataProvider('serviceRetryHistories')] + public function testServiceRunRequiresMatchingRecordedWorkerDefinitionFingerprint(bool $retriedPrefix): void { Queue::fake(); @@ -292,12 +366,45 @@ public function testServiceRunRequiresMatchingRecordedWorkerDefinitionFingerprin 'activity_type' => 'remote.first', 'sequence' => 1, ]); + if ($retriedPrefix) { + WorkflowHistoryEvent::record($source, HistoryEventType::ActivityStarted, [ + 'activity_execution_id' => 'service-activity-1', + 'activity_type' => 'remote.first', + 'activity_attempt_id' => 'service-attempt-1', + 'workflow_task_id' => 'service-activity-task-1', + 'attempt_number' => 1, + 'sequence' => 1, + ]); + WorkflowHistoryEvent::record($source, HistoryEventType::ActivityRetryScheduled, [ + 'activity_execution_id' => 'service-activity-1', + 'activity_type' => 'remote.first', + 'activity_attempt_id' => 'service-attempt-1', + 'retry_after_attempt_id' => 'service-attempt-1', + 'retry_after_attempt' => 1, + 'retry_of_task_id' => 'service-activity-task-1', + 'retry_task_id' => 'service-activity-task-2', + 'max_attempts' => 2, + 'sequence' => 1, + ]); + WorkflowHistoryEvent::record($source, HistoryEventType::ActivityStarted, [ + 'activity_execution_id' => 'service-activity-1', + 'activity_type' => 'remote.first', + 'activity_attempt_id' => 'service-attempt-2', + 'workflow_task_id' => 'service-activity-task-2', + 'attempt_number' => 2, + 'sequence' => 1, + ]); + } WorkflowHistoryEvent::record($source, HistoryEventType::ActivityCompleted, [ 'activity_execution_id' => 'service-activity-1', 'activity_type' => 'remote.first', 'sequence' => 1, 'result' => Serializer::serializeWithCodec('avro', 'first-result'), 'payload_codec' => 'avro', + ...($retriedPrefix ? [ + 'activity_attempt_id' => 'service-attempt-2', + 'attempt_number' => 2, + ] : []), ]); WorkflowHistoryEvent::record($source, HistoryEventType::ActivityScheduled, [ 'activity_execution_id' => 'service-activity-2', @@ -2445,6 +2552,25 @@ private function failedSignalSource(string $instanceId): WorkflowRun return WorkflowRun::query()->findOrFail($workflow->runId()); } + private function drainRetryProbe(string $runId): WorkflowRun + { + for ($turn = 0; $turn < 20; ++$turn) { + $run = WorkflowRun::query()->findOrFail($runId); + if ($run->status->isTerminal()) { + return $run; + } + $task = WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('status', TaskStatus::Ready)->orderBy('available_at')->firstOrFail(); + if ($task->available_at->isFuture()) { + $this->travelTo($task->available_at); + } + $job = $task->task_type === TaskType::Activity + ? new RunActivityTask($task->id) : new RunWorkflowTask($task->id); + $this->app->call([$job, 'handle']); + } + $this->fail('The ordinary retry fixture did not converge within twenty task deliveries.'); + } + /** * Every engine table and read projection a refused redrive must leave * untouched, including the run's message cursor. diff --git a/tests/Fixtures/V2/ReplayRegression/redrive-native-active-start-adoption.json b/tests/Fixtures/V2/ReplayRegression/redrive-native-active-start-adoption.json new file mode 100644 index 000000000..c59213735 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/redrive-native-active-start-adoption.json @@ -0,0 +1,229 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "redrive-native-active-start-adoption", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "arguments": [ + "Taylor", + false + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "StartAccepted", + "recorded_at": "2026-11-12T00:00:01+00:00", + "workflow_run_id": "source-run", + "workflow_command_id": "adoption-command", + "payload": { + "workflow_instance_id": "source-instance", + "workflow_run_id": "source-run", + "workflow_class": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "workflow_type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "workflow_command_id": "adoption-command", + "outcome": "returned_existing_active" + } + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + } + ], + "redrive_plan": { + "source_history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "StartAccepted", + "recorded_at": "2026-11-12T00:00:01+00:00", + "workflow_run_id": "source-run", + "workflow_command_id": "adoption-command", + "payload": { + "workflow_instance_id": "source-instance", + "workflow_run_id": "source-run", + "workflow_class": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "workflow_type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "workflow_command_id": "adoption-command", + "outcome": "returned_existing_active", + "command": { + "id": "adoption-command", + "type": "start", + "target_scope": "instance", + "resolved_run_id": "source-run", + "status": "accepted", + "outcome": "returned_existing_active" + } + } + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2" + }, + "recorded_at": "2026-11-12T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityFailed", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "message": "provider unavailable", + "exception_class": "RuntimeException" + }, + "recorded_at": "2026-11-12T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "WorkflowFailed", + "payload": { + "failed_step_kind": "activity", + "failed_step_sequence": 2 + }, + "recorded_at": "2026-11-12T00:00:06+00:00" + } + ], + "expected": { + "eligible": true, + "resume_step_sequence": 2, + "completed_count": 1, + "reused_event_types": [ + "ActivityCompleted" + ] + }, + "refusals": [ + { + "event_index": 1, + "field": "outcome", + "value": "started_new", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "workflow_instance_id", + "value": "other-instance", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "workflow_run_id", + "value": "other-run", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "workflow_class", + "value": "OtherWorkflow", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "workflow_type", + "value": "other-type", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "workflow_command_id", + "value": "other-command", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "rejection_reason", + "value": "refused", + "reason": "unexpected_start" + }, + { + "event_index": 1, + "field": "command", + "value": null, + "reason": "unexpected_start" + } + ], + "source_run": { + "id": "source-run", + "workflow_instance_id": "source-instance", + "workflow_class": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "workflow_type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow" + } + }, + "expected": { + "completed": false, + "result": null, + "commands": [ + { + "type": "schedule_activity", + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "payload_codec": "avro" + } + ] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/redrive-retried-activity-after-completed-child.json b/tests/Fixtures/V2/ReplayRegression/redrive-retried-activity-after-completed-child.json new file mode 100644 index 000000000..7c0d4baae --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/redrive-retried-activity-after-completed-child.json @@ -0,0 +1,441 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "redrive-retried-activity-after-completed-child", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "arguments": [ + "Taylor", + true + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "01JCREDRIVECHILDLINK000000", + "child_call_id": "01JCREDRIVECHILDLINK000000", + "child_workflow_instance_id": "01jcredrivechildinstance00", + "child_workflow_run_id": "01jcredrivechildrun0000000", + "child_workflow_class": "Tests\\Fixtures\\V2\\TestRedriveGreetingChildWorkflow", + "child_workflow_type": "test-redrive-greeting-child-workflow", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon" + }, + "recorded_at": "2026-11-12T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "01JCREDRIVECHILDLINK000000", + "child_call_id": "01JCREDRIVECHILDLINK000000", + "child_workflow_instance_id": "01jcredrivechildinstance00", + "child_workflow_run_id": "01jcredrivechildrun0000000", + "child_workflow_class": "Tests\\Fixtures\\V2\\TestRedriveGreetingChildWorkflow", + "child_workflow_type": "test-redrive-greeting-child-workflow", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 1, + "workflow_link_id": "01JCREDRIVECHILDLINK000000", + "child_call_id": "01JCREDRIVECHILDLINK000000", + "child_workflow_instance_id": "01jcredrivechildinstance00", + "child_workflow_run_id": "01jcredrivechildrun0000000", + "child_workflow_class": "Tests\\Fixtures\\V2\\TestRedriveGreetingChildWorkflow", + "child_workflow_type": "test-redrive-greeting-child-workflow", + "child_run_number": 1, + "child_status": "completed", + "closed_reason": "completed", + "closed_at": "2026-11-12T00:00:01.000000Z", + "output": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2" + }, + "recorded_at": "2026-11-12T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "attempt_number": 1, + "workflow_task_id": "task-2-1" + }, + "recorded_at": "2026-11-12T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "retry_after_attempt_id": "attempt-2-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-2-1", + "retry_task_id": "task-2-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "workflow_task_id": "task-2-2" + }, + "recorded_at": "2026-11-12T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:08+00:00" + } + ], + "redrive_plan": { + "source_history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "01JCREDRIVECHILDLINK000000", + "child_call_id": "01JCREDRIVECHILDLINK000000", + "child_workflow_instance_id": "01jcredrivechildinstance00", + "child_workflow_run_id": "01jcredrivechildrun0000000", + "child_workflow_class": "Tests\\Fixtures\\V2\\TestRedriveGreetingChildWorkflow", + "child_workflow_type": "test-redrive-greeting-child-workflow", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon" + }, + "recorded_at": "2026-11-12T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "01JCREDRIVECHILDLINK000000", + "child_call_id": "01JCREDRIVECHILDLINK000000", + "child_workflow_instance_id": "01jcredrivechildinstance00", + "child_workflow_run_id": "01jcredrivechildrun0000000", + "child_workflow_class": "Tests\\Fixtures\\V2\\TestRedriveGreetingChildWorkflow", + "child_workflow_type": "test-redrive-greeting-child-workflow", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 1, + "workflow_link_id": "01JCREDRIVECHILDLINK000000", + "child_call_id": "01JCREDRIVECHILDLINK000000", + "child_workflow_instance_id": "01jcredrivechildinstance00", + "child_workflow_run_id": "01jcredrivechildrun0000000", + "child_workflow_class": "Tests\\Fixtures\\V2\\TestRedriveGreetingChildWorkflow", + "child_workflow_type": "test-redrive-greeting-child-workflow", + "child_run_number": 1, + "child_status": "completed", + "closed_reason": "completed", + "closed_at": "2026-11-12T00:00:01.000000Z", + "output": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2" + }, + "recorded_at": "2026-11-12T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "attempt_number": 1, + "workflow_task_id": "task-2-1" + }, + "recorded_at": "2026-11-12T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "retry_after_attempt_id": "attempt-2-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-2-1", + "retry_task_id": "task-2-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "workflow_task_id": "task-2-2" + }, + "recorded_at": "2026-11-12T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:08+00:00" + }, + { + "sequence": 10, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 3, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-3" + }, + "recorded_at": "2026-11-12T00:00:09+00:00" + }, + { + "sequence": 11, + "event_type": "ActivityStarted", + "payload": { + "sequence": 3, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-3", + "activity_attempt_id": "attempt-3-1", + "attempt_number": 1, + "workflow_task_id": "task-3-1" + }, + "recorded_at": "2026-11-12T00:00:10+00:00" + }, + { + "sequence": 12, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 3, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-3", + "activity_attempt_id": "attempt-3-1", + "retry_after_attempt_id": "attempt-3-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-3-1", + "retry_task_id": "task-3-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:11+00:00" + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "sequence": 3, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-3", + "activity_attempt_id": "attempt-3-2", + "attempt_number": 2, + "workflow_task_id": "task-3-2" + }, + "recorded_at": "2026-11-12T00:00:12+00:00" + }, + { + "sequence": 14, + "event_type": "ActivityFailed", + "payload": { + "sequence": 3, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-3", + "activity_attempt_id": "attempt-3-2", + "attempt_number": 2, + "message": "provider unavailable", + "exception_class": "RuntimeException" + }, + "recorded_at": "2026-11-12T00:00:13+00:00" + }, + { + "sequence": 15, + "event_type": "WorkflowFailed", + "payload": { + "failed_step_kind": "activity", + "failed_step_sequence": 3 + }, + "recorded_at": "2026-11-12T00:00:14+00:00" + } + ], + "expected": { + "eligible": true, + "resume_step_sequence": 3, + "completed_count": 1, + "reused_event_types": [ + "ChildRunCompleted", + "ActivityCompleted" + ] + }, + "refusals": [ + { + "event_index": 6, + "field": "activity_execution_id", + "value": "substituted-execution", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 6, + "field": "activity_attempt_id", + "value": "substituted-attempt", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 6, + "field": "retry_task_id", + "value": "substituted-task", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 6, + "field": "retry_after_attempt", + "value": 9, + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 11, + "field": "activity_execution_id", + "value": "substituted-execution", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 11, + "field": "activity_attempt_id", + "value": "substituted-attempt", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 11, + "field": "retry_task_id", + "value": "substituted-task", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 11, + "field": "retry_after_attempt", + "value": 9, + "reason": "invalid_activity_retry_history" + } + ] + }, + "expected": { + "completed": false, + "result": null, + "commands": [ + { + "type": "schedule_activity", + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "payload_codec": "avro" + } + ] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/redrive-retried-completed-activity.json b/tests/Fixtures/V2/ReplayRegression/redrive-retried-completed-activity.json new file mode 100644 index 000000000..a63c15a0a --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/redrive-retried-completed-activity.json @@ -0,0 +1,332 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "redrive-retried-completed-activity", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "arguments": [ + "Taylor", + false + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1" + }, + "recorded_at": "2026-11-12T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityStarted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-1", + "attempt_number": 1, + "workflow_task_id": "task-1-1" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-1", + "retry_after_attempt_id": "attempt-1-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-1-1", + "retry_task_id": "task-1-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityStarted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-2", + "attempt_number": 2, + "workflow_task_id": "task-1-2" + }, + "recorded_at": "2026-11-12T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-2", + "attempt_number": 2, + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:05+00:00" + } + ], + "redrive_plan": { + "source_history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1" + }, + "recorded_at": "2026-11-12T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityStarted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-1", + "attempt_number": 1, + "workflow_task_id": "task-1-1" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-1", + "retry_after_attempt_id": "attempt-1-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-1-1", + "retry_task_id": "task-1-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityStarted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-2", + "attempt_number": 2, + "workflow_task_id": "task-1-2" + }, + "recorded_at": "2026-11-12T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "activity_attempt_id": "attempt-1-2", + "attempt_number": 2, + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2" + }, + "recorded_at": "2026-11-12T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "attempt_number": 1, + "workflow_task_id": "task-2-1" + }, + "recorded_at": "2026-11-12T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "retry_after_attempt_id": "attempt-2-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-2-1", + "retry_task_id": "task-2-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:08+00:00" + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "workflow_task_id": "task-2-2" + }, + "recorded_at": "2026-11-12T00:00:09+00:00" + }, + { + "sequence": 11, + "event_type": "ActivityFailed", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "message": "provider unavailable", + "exception_class": "RuntimeException" + }, + "recorded_at": "2026-11-12T00:00:10+00:00" + }, + { + "sequence": 12, + "event_type": "WorkflowFailed", + "payload": { + "failed_step_kind": "activity", + "failed_step_sequence": 2 + }, + "recorded_at": "2026-11-12T00:00:11+00:00" + } + ], + "expected": { + "eligible": true, + "resume_step_sequence": 2, + "completed_count": 1, + "reused_event_types": [ + "ActivityCompleted" + ] + }, + "refusals": [ + { + "event_index": 3, + "field": "activity_execution_id", + "value": "substituted-execution", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 3, + "field": "activity_attempt_id", + "value": "substituted-attempt", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 3, + "field": "retry_task_id", + "value": "substituted-task", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 3, + "field": "retry_after_attempt", + "value": 9, + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 8, + "field": "activity_execution_id", + "value": "substituted-execution", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 8, + "field": "activity_attempt_id", + "value": "substituted-attempt", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 8, + "field": "retry_task_id", + "value": "substituted-task", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 8, + "field": "retry_after_attempt", + "value": 9, + "reason": "invalid_activity_retry_history" + } + ] + }, + "expected": { + "completed": false, + "result": null, + "commands": [ + { + "type": "schedule_activity", + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "payload_codec": "avro" + } + ] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/redrive-retried-failed-activity.json b/tests/Fixtures/V2/ReplayRegression/redrive-retried-failed-activity.json new file mode 100644 index 000000000..c448c16be --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/redrive-retried-failed-activity.json @@ -0,0 +1,210 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "redrive-retried-failed-activity", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRedriveRetryWorkflow", + "arguments": [ + "Taylor", + false + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1" + }, + "recorded_at": "2026-11-12T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + } + ], + "redrive_plan": { + "source_history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-11-12T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1" + }, + "recorded_at": "2026-11-12T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 1, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveFirstActivity", + "activity_execution_id": "execution-1", + "result": "wwHioz3/VYAiNwocSGVsbG8sIFRheWxvciE=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-12T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2" + }, + "recorded_at": "2026-11-12T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "attempt_number": 1, + "workflow_task_id": "task-2-1" + }, + "recorded_at": "2026-11-12T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityRetryScheduled", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-1", + "retry_after_attempt_id": "attempt-2-1", + "retry_after_attempt": 1, + "retry_of_task_id": "task-2-1", + "retry_task_id": "task-2-2", + "max_attempts": 2, + "retry_reason": "failure", + "retry_backoff_seconds": 0 + }, + "recorded_at": "2026-11-12T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "workflow_task_id": "task-2-2" + }, + "recorded_at": "2026-11-12T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityFailed", + "payload": { + "sequence": 2, + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "activity_execution_id": "execution-2", + "activity_attempt_id": "attempt-2-2", + "attempt_number": 2, + "message": "provider unavailable", + "exception_class": "RuntimeException" + }, + "recorded_at": "2026-11-12T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "WorkflowFailed", + "payload": { + "failed_step_kind": "activity", + "failed_step_sequence": 2 + }, + "recorded_at": "2026-11-12T00:00:08+00:00" + } + ], + "expected": { + "eligible": true, + "resume_step_sequence": 2, + "completed_count": 1, + "reused_event_types": [ + "ActivityCompleted" + ] + }, + "refusals": [ + { + "event_index": 5, + "field": "activity_execution_id", + "value": "substituted-execution", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 5, + "field": "activity_attempt_id", + "value": "substituted-attempt", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 5, + "field": "retry_task_id", + "value": "substituted-task", + "reason": "invalid_activity_retry_history" + }, + { + "event_index": 5, + "field": "retry_after_attempt", + "value": 9, + "reason": "invalid_activity_retry_history" + } + ] + }, + "expected": { + "completed": false, + "result": null, + "commands": [ + { + "type": "schedule_activity", + "activity_type": "Tests\\Fixtures\\V2\\TestRedriveSecondActivity", + "payload_codec": "avro" + } + ] + } +} diff --git a/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php b/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php new file mode 100644 index 000000000..09663f1a9 --- /dev/null +++ b/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php @@ -0,0 +1,29 @@ +heartbeat([ + 'site' => $site, + ]); + if ($calls <= (int) config('workflows.v2.testing.redrive_retry_failures.' . $site, 0)) { + throw new RuntimeException('Retryable fixture failure at ' . $site . '.'); + } + return $value . ':' . $site; + } +} diff --git a/tests/Fixtures/V2/TestRedriveRetryProbeWorkflow.php b/tests/Fixtures/V2/TestRedriveRetryProbeWorkflow.php new file mode 100644 index 000000000..73efb5ba9 --- /dev/null +++ b/tests/Fixtures/V2/TestRedriveRetryProbeWorkflow.php @@ -0,0 +1,29 @@ +runWithHistory($this->retryHistory())); + self::assertTrue($plan['eligible'], (string) $plan['reason']); + self::assertSame(2, $plan['resume_step_sequence']); + } + + public function testARetriedCompletedPrefixIsReusable(): void + { + $plan = FailedRunRedrivePlan::forRun($this->runWithHistory($this->completedRetryHistory())); + + self::assertTrue($plan['eligible'], (string) $plan['reason']); + self::assertSame(2, $plan['resume_step_sequence']); + self::assertCount(1, $plan['completed']); + self::assertSame(1, $plan['completed'][0]['sequence']); + } + + public function testHeartbeatsBetweenRetriedPrefixAttemptsDoNotPreventReuse(): void + { + $history = $this->completedRetryHistory(); + foreach ($history as $index => $event) { + if ($event['event_type'] === HistoryEventType::ActivityStarted + && ($event['payload']['sequence'] ?? null) === 1) { + array_splice($history, $index + 1, 0, [[ + 'event_type' => HistoryEventType::ActivityHeartbeatRecorded, + 'payload' => $event['payload'], + ]]); + break; + } + } + $plan = FailedRunRedrivePlan::forRun($this->runWithHistory($history)); + + self::assertTrue($plan['eligible'], (string) $plan['reason']); + self::assertSame(2, $plan['resume_step_sequence']); + self::assertCount(1, $plan['completed']); + } + + public function testARejectedStartDoesNotBecomeActiveAdoptionEvidence(): void + { + $run = $this->adoptionRun(); + $run->historyEvents[2]->event_type = HistoryEventType::StartRejected; + $plan = FailedRunRedrivePlan::forRun($run); + + self::assertFalse($plan['eligible']); + self::assertSame('unsupported_history', $plan['reason']); + } + + public function testRetryIdentitySubstitutionIsRefused(): void + { + foreach (['activity_execution_id', 'activity_attempt_id', 'retry_after_attempt', 'retry_task_id'] as $field) { + $history = $this->retryHistory(); + $history[6]['payload'][$field] = $field === 'retry_after_attempt' ? 9 : 'substituted'; + $plan = FailedRunRedrivePlan::forRun($this->runWithHistory($history)); + self::assertFalse($plan['eligible']); + self::assertSame('invalid_activity_retry_history', $plan['reason']); + } + } + + public function testNativeActiveAdoptionIsHarmlessOnlyWhenBoundToTheSameRunAndCommand(): void + { + $plan = FailedRunRedrivePlan::forRun($this->adoptionRun()); + self::assertTrue($plan['eligible'], (string) $plan['reason']); + self::assertSame(2, $plan['resume_step_sequence']); + } + + public function testSubstitutedActiveStartAdoptionCannotAuthorizeRedrive(): void + { + foreach ([ + ['outcome', 'started_new'], + ['workflow_instance_id', 'other-instance'], + ['workflow_run_id', 'other-run'], + ['workflow_class', 'OtherWorkflow'], + ['workflow_type', 'other.type'], + ['workflow_command_id', 'other-command'], + ['rejection_reason', 'refused'], + ['command', null], + ['command.id', 'other-command'], + ['command.type', 'signal'], + ['command.status', 'rejected'], + ['command.outcome', 'started_new'], + ['command.target_scope', 'run'], + ['command.resolved_run_id', 'other-run'], + ] as [$field, $replacement]) { + $run = $this->adoptionRun(); + $event = $run->historyEvents[2]; + $payload = $event->payload; + $keys = explode('.', $field); + if (count($keys) === 2) { + $payload[$keys[0]][$keys[1]] = $replacement; + } else { + $payload[$field] = $replacement; + } + $event->payload = $payload; + $plan = FailedRunRedrivePlan::forRun($run); + self::assertFalse($plan['eligible'], $field); + self::assertSame('unexpected_start', $plan['reason'], $field); + } + foreach (['workflow_run_id', 'workflow_command_id'] as $field) { + $run = $this->adoptionRun(); + $run->historyEvents[2]->setAttribute($field, 'substituted'); + self::assertSame('unexpected_start', FailedRunRedrivePlan::forRun($run)['reason']); + } + } + + private function adoptionRun(): WorkflowRun + { + $history = $this->linearFailureHistory(); + array_splice($history, 2, 0, [[ + 'event_type' => HistoryEventType::StartAccepted, + 'payload' => [ + 'workflow_instance_id' => 'instance-a', + 'workflow_run_id' => 'run-a', + 'workflow_class' => 'TestWorkflow', + 'workflow_type' => 'test.workflow', + 'workflow_command_id' => 'adoption-command', + 'outcome' => 'returned_existing_active', + 'command' => [ + 'id' => 'adoption-command', + 'type' => 'start', + 'target_scope' => 'instance', + 'resolved_run_id' => 'run-a', + 'status' => 'accepted', + 'outcome' => 'returned_existing_active', + ], + ], + ]]); + $run = $this->runWithHistory($history); + $run->forceFill([ + 'id' => 'run-a', + 'workflow_instance_id' => 'instance-a', + 'workflow_class' => 'TestWorkflow', + 'workflow_type' => 'test.workflow', + ]); + $run->historyEvents[2]->forceFill([ + 'workflow_run_id' => 'run-a', + 'workflow_command_id' => 'adoption-command', + ]); + return $run; + } + + private function retryHistory(): array + { + $history = $this->linearFailureHistory(); + array_splice($history, 4, 1, [ + [ + 'event_type' => HistoryEventType::ActivityScheduled, + 'payload' => [ + 'sequence' => 2, + 'activity_type' => 'second', + 'activity_execution_id' => 'execution-2', + ], + ], + [ + 'event_type' => HistoryEventType::ActivityStarted, + 'payload' => [ + 'sequence' => 2, + 'activity_type' => 'second', + 'activity_execution_id' => 'execution-2', + 'activity_attempt_id' => 'attempt-1', + 'attempt_number' => 1, + 'workflow_task_id' => 'task-1', + ], + ], + [ + 'event_type' => HistoryEventType::ActivityRetryScheduled, + 'payload' => [ + 'sequence' => 2, + 'activity_type' => 'second', + 'activity_execution_id' => 'execution-2', + 'activity_attempt_id' => 'attempt-1', + 'retry_after_attempt_id' => 'attempt-1', + 'retry_after_attempt' => 1, + 'retry_of_task_id' => 'task-1', + 'retry_task_id' => 'task-2', + 'max_attempts' => 2, + ], + ], + [ + 'event_type' => HistoryEventType::ActivityStarted, + 'payload' => [ + 'sequence' => 2, + 'activity_type' => 'second', + 'activity_execution_id' => 'execution-2', + 'activity_attempt_id' => 'attempt-2', + 'attempt_number' => 2, + 'workflow_task_id' => 'task-2', + ], + ], + [ + 'event_type' => HistoryEventType::ActivityFailed, + 'payload' => [ + 'sequence' => 2, + 'activity_type' => 'second', + 'activity_execution_id' => 'execution-2', + 'activity_attempt_id' => 'attempt-2', + 'attempt_number' => 2, + ], + ], + ]); + return $history; + } + + /** + * @return list}> + */ + private function completedRetryHistory(): array + { + $fixture = json_decode( + file_get_contents( + dirname(__DIR__, 2) . '/Fixtures/V2/ReplayRegression/redrive-retried-completed-activity.json' + ), + true, + flags: JSON_THROW_ON_ERROR, + ); + return array_map(static function (array $event): array { + $event['event_type'] = HistoryEventType::from($event['event_type']); + return $event; + }, $fixture['redrive_plan']['source_history']); + } + /** * @return list}> */ diff --git a/tests/Unit/V2/ReplayRegressionCorpusTest.php b/tests/Unit/V2/ReplayRegressionCorpusTest.php index 8f7d514fa..af16aabca 100644 --- a/tests/Unit/V2/ReplayRegressionCorpusTest.php +++ b/tests/Unit/V2/ReplayRegressionCorpusTest.php @@ -24,12 +24,14 @@ use Throwable; use Workflow\Serializers\Serializer; use Workflow\V2\Enums\HistoryEventType; +use Workflow\V2\Enums\RunStatus; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Support\ActivitySnapshot; use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\ExecutionTracing; +use Workflow\V2\Support\FailedRunRedrivePlan; use Workflow\V2\Support\HistoryEventPayloadContract; use Workflow\V2\Support\PortableLocalActivityPreparation; use Workflow\V2\Support\PreparedLocalRetryPolicy; @@ -81,6 +83,9 @@ protected function tearDown(): void public function testFixtureExecutesThroughColdReplayRunner(array $fixture): void { $this->assertHistoryPayloadContract($fixture); + if (isset($fixture['redrive_plan'])) { + $this->assertRetriedActivityRedrivePlan($fixture); + } if (in_array( $fixture['id'], @@ -409,6 +414,40 @@ public static function replayRegressionFixtures(): array return $fixtures; } + private function assertRetriedActivityRedrivePlan(array $fixture): void + { + $scenario = $fixture['redrive_plan']; + $runWithHistory = static function (array $history) use ($scenario): WorkflowRun { + $run = new WorkflowRun([ + ...($scenario['source_run'] ?? []), + 'status' => RunStatus::Failed, + 'payload_codec' => 'avro', + ]); + $run->setRelation('historyEvents', collect($history)->map( + static fn (array $event): WorkflowHistoryEvent => new WorkflowHistoryEvent($event) + )); + return $run; + }; + $plan = FailedRunRedrivePlan::forRun($runWithHistory($scenario['source_history'])); + $expected = $scenario['expected']; + $this->assertSame($expected['eligible'], $plan['eligible'], $fixture['id'] . ': ' . $plan['reason']); + $this->assertNull($plan['reason']); + $this->assertSame($expected['resume_step_sequence'], $plan['resume_step_sequence']); + $this->assertCount($expected['completed_count'], $plan['completed']); + $this->assertSame($expected['reused_event_types'], array_map( + static fn (array $step): string => $step['event_type']->value, + $plan['reused_steps'] + )); + foreach ($scenario['refusals'] as $case) { + $history = $scenario['source_history']; + $history[$case['event_index']]['payload'][$case['field']] = $case['value']; + $rejected = FailedRunRedrivePlan::forRun($runWithHistory($history)); + $this->assertFalse($rejected['eligible'], $fixture['id'] . ': ' . $case['field']); + $this->assertSame($case['reason'], $rejected['reason']); + $this->assertSame([], $rejected['reused_steps']); + } + } + private function assertPreparedV1RefusesTheVersionedPolicy(array $fixture): void { $policy = $fixture['unqualified_preparation_policy']; From c0f18b322e6f2b00c2559a9027474f7d296c8bd3 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:50:14 +0000 Subject: [PATCH 02/32] Exercise redrive retries with supported queue and heartbeat fixtures --- tests/Feature/V2/V2FailedRunRedriveTest.php | 2 ++ tests/Fixtures/V2/TestRedriveRetryProbeActivity.php | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Feature/V2/V2FailedRunRedriveTest.php b/tests/Feature/V2/V2FailedRunRedriveTest.php index f1bed9890..50e44934b 100644 --- a/tests/Feature/V2/V2FailedRunRedriveTest.php +++ b/tests/Feature/V2/V2FailedRunRedriveTest.php @@ -203,6 +203,8 @@ public static function retriedActivityCases(): array #[DataProvider('retriedActivityCases')] public function testOrdinaryRetriedActivitiesRedriveWithoutRepeatingTheCompletedPrefix(int $firstFailures): void { + config() + ->set('queue.default', 'redis'); Queue::fake(); config([ 'workflows.v2.testing.redrive_retry_failures' => [ diff --git a/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php b/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php index 09663f1a9..c55db17fc 100644 --- a/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php +++ b/tests/Fixtures/V2/TestRedriveRetryProbeActivity.php @@ -19,7 +19,7 @@ public function handle(string $site, string $value, string $counter): string { $calls = Cache::increment($counter . ':' . $site); $this->heartbeat([ - 'site' => $site, + 'message' => $site, ]); if ($calls <= (int) config('workflows.v2.testing.redrive_retry_failures.' . $site, 0)) { throw new RuntimeException('Retryable fixture failure at ' . $site . '.'); From e41807ade65f3dfd0a43ecbbd3cd5124942a42cf Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:05:28 +0000 Subject: [PATCH 03/32] Require closed attempt ownership grammar in retry histories --- src/V2/Support/ActivityRetryHistory.php | 93 ++++++++++++++++++++----- 1 file changed, 74 insertions(+), 19 deletions(-) diff --git a/src/V2/Support/ActivityRetryHistory.php b/src/V2/Support/ActivityRetryHistory.php index dd9b84516..c675b994a 100644 --- a/src/V2/Support/ActivityRetryHistory.php +++ b/src/V2/Support/ActivityRetryHistory.php @@ -22,31 +22,40 @@ public static function valid(array $events): bool if (! $hasRetries) { return true; } + $phase = 'schedule'; $execution = null; $attempt = null; $task = null; $number = 0; $nextTask = null; - $terminal = false; $seenAttempts = []; $seenTasks = []; foreach ($events as $event) { $payload = $event->payload; $id = $payload['activity_execution_id'] ?? null; - if (! is_string($id) || $id === '' || ($execution !== null && $id !== $execution) || $terminal) { + if ( + ! is_string($id) + || $id === '' + || ($execution !== null && $id !== $execution) + || $phase === 'terminal' + ) { return false; } $execution = $id; - if ($event->event_type === HistoryEventType::ActivityStarted) { - $currentAttempt = $payload['activity_attempt_id'] ?? null; - $currentTask = $event->workflow_task_id ?? ($payload['workflow_task_id'] ?? null); - if (isset($event->workflow_task_id, $payload['workflow_task_id']) && $event->workflow_task_id !== $payload['workflow_task_id']) { + if ($event->event_type === HistoryEventType::ActivityScheduled) { + if ($phase !== 'schedule') { return false; } - if (! is_string($currentAttempt) || $currentAttempt === '' || isset($seenAttempts[$currentAttempt]) + $phase = 'start'; + } elseif ($event->event_type === HistoryEventType::ActivityStarted) { + $currentAttempt = $payload['activity_attempt_id'] ?? null; + $currentTask = $event->workflow_task_id ?? ($payload['workflow_task_id'] ?? null); + if (! in_array($phase, ['start', 'retry'], true) + || ! is_string($currentAttempt) || $currentAttempt === '' || isset($seenAttempts[$currentAttempt]) || ! is_string($currentTask) || $currentTask === '' || isset($seenTasks[$currentTask]) || ($payload['attempt_number'] ?? null) !== $number + 1 - || ($number > 0 && $nextTask !== $currentTask)) { + || ($number > 0 && $nextTask !== $currentTask) + || ! self::matchesTask($event, $currentTask)) { return false; } $attempt = $currentAttempt; @@ -55,31 +64,77 @@ public static function valid(array $events): bool $seenTasks[$task] = true; ++$number; $nextTask = null; + $phase = 'active'; } elseif ($event->event_type === HistoryEventType::ActivityRetryScheduled) { $retry = $payload['retry_task_id'] ?? null; $maximum = $payload['max_attempts'] ?? null; - if ($attempt === null || $nextTask !== null + if ($phase !== 'active' || ($payload['activity_attempt_id'] ?? null) !== $attempt || ($payload['retry_after_attempt_id'] ?? null) !== $attempt || ($payload['retry_after_attempt'] ?? null) !== $number || ($payload['retry_of_task_id'] ?? null) !== $task || ! is_string($retry) || $retry === '' || isset($seenTasks[$retry]) - || ($maximum !== null && (! is_int($maximum) || $maximum <= $number))) { + || ($maximum !== null && (! is_int($maximum) || $maximum <= $number)) + || ! self::matchesTask($event, $task)) { return false; } $nextTask = $retry; - } elseif (in_array( - $event->event_type, - [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed], - true - )) { - if ($nextTask !== null || $attempt === null || ($payload['activity_attempt_id'] ?? null) !== $attempt - || ($payload['attempt_number'] ?? null) !== $number) { + $phase = 'retry'; + } elseif (in_array($event->event_type, [ + HistoryEventType::ActivityHeartbeatRecorded, + HistoryEventType::ActivityCompleted, + HistoryEventType::ActivityFailed, + ], true)) { + if ($phase !== 'active' || ($payload['activity_attempt_id'] ?? null) !== $attempt + || ($payload['attempt_number'] ?? null) !== $number + || ! self::matchesTask($event, $task)) { return false; } - $terminal = true; + if ($event->event_type !== HistoryEventType::ActivityHeartbeatRecorded) { + $phase = 'terminal'; + } + } else { + return false; + } + if (in_array($phase, ['active', 'retry', 'terminal'], true) + && ! self::matchesAttemptSnapshot($payload, $execution, $attempt, $number, $task)) { + return false; + } + } + return $phase === 'terminal'; + } + + private static function matchesTask(WorkflowHistoryEvent $event, string $task): bool + { + $payload = $event->payload; + return ($event->workflow_task_id === null || $event->workflow_task_id === $task) + && (! array_key_exists('workflow_task_id', $payload) || $payload['workflow_task_id'] === $task); + } + + private static function matchesAttemptSnapshot( + array $payload, + string $execution, + string $attempt, + int $number, + string $task, + ): bool { + if (! array_key_exists('activity_attempt', $payload)) { + return true; + } + $snapshot = $payload['activity_attempt']; + if (! is_array($snapshot)) { + return false; + } + foreach ([ + 'id' => $attempt, + 'activity_execution_id' => $execution, + 'attempt_number' => $number, + 'task_id' => $task, + ] as $field => $expected) { + if (array_key_exists($field, $snapshot) && $snapshot[$field] !== $expected) { + return false; } } - return $terminal; + return true; } } From 42dd4e3349ce79c26d201a813a4c37ee3d23df28 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:04:59 +0000 Subject: [PATCH 04/32] Reject malformed heartbeat and schedule retry histories --- tests/Unit/V2/ActivityRetryHistoryTest.php | 184 +++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 tests/Unit/V2/ActivityRetryHistoryTest.php diff --git a/tests/Unit/V2/ActivityRetryHistoryTest.php b/tests/Unit/V2/ActivityRetryHistoryTest.php new file mode 100644 index 000000000..16206b9cc --- /dev/null +++ b/tests/Unit/V2/ActivityRetryHistoryTest.php @@ -0,0 +1,184 @@ +previousContainer = Container::getInstance(); + $application = new Application(dirname(__DIR__, 3)); + $application->instance('config', new Repository([])); + } + + protected function tearDown(): void + { + Container::setInstance($this->previousContainer); + parent::tearDown(); + } + + public static function terminalKinds(): array + { + return [ + 'completed prefix' => [HistoryEventType::ActivityCompleted], + 'failed boundary' => [HistoryEventType::ActivityFailed], + ]; + } + + #[DataProvider('terminalKinds')] + public function testCanonicalRetryChainBindsEachHeartbeatToItsActiveAttempt(HistoryEventType $terminal): void + { + self::assertTrue(ActivityRetryHistory::valid(self::models(self::history($terminal)))); + } + + public static function malformedChains(): iterable + { + foreach (self::terminalKinds() as $kind => [$terminal]) { + foreach ([ + 'heartbeat wrong attempt', + 'heartbeat wrong number', + 'heartbeat wrong payload task', + 'heartbeat wrong row task', + 'heartbeat wrong snapshot task', + 'heartbeat before start', + 'heartbeat after retry', + 'duplicate schedule', + 'mid-chain schedule', + 'missing schedule', + ] as $mutation) { + yield $kind . ': ' . $mutation => [$terminal, $mutation]; + } + } + } + + #[DataProvider('malformedChains')] + public function testMalformedChainCannotAuthorizeRedrive(HistoryEventType $terminal, string $mutation): void + { + $history = self::history($terminal); + switch ($mutation) { + case 'heartbeat wrong attempt': + $history[2]['payload']['activity_attempt_id'] = 'other-attempt'; + break; + case 'heartbeat wrong number': + $history[2]['payload']['attempt_number'] = 2; + break; + case 'heartbeat wrong payload task': + $history[2]['payload']['workflow_task_id'] = 'other-task'; + break; + case 'heartbeat wrong row task': + $history[2]['workflow_task_id'] = 'other-task'; + break; + case 'heartbeat wrong snapshot task': + $history[2]['payload']['activity_attempt']['task_id'] = 'other-task'; + break; + case 'heartbeat before start': + $heartbeat = array_splice($history, 2, 1); + array_splice($history, 1, 0, $heartbeat); + break; + case 'heartbeat after retry': + $heartbeat = array_splice($history, 2, 1); + array_splice($history, 3, 0, $heartbeat); + break; + case 'duplicate schedule': + array_splice($history, 1, 0, [$history[0]]); + break; + case 'mid-chain schedule': + array_splice($history, 4, 0, [$history[0]]); + break; + case 'missing schedule': + array_shift($history); + break; + } + self::assertFalse(ActivityRetryHistory::valid(self::models($history)), $mutation); + } + + private static function history(HistoryEventType $terminal): array + { + $identity = [ + 'activity_execution_id' => 'execution-1', + 'activity_type' => 'test.activity', + 'sequence' => 1, + ]; + $attempt = static fn (int $number): array => [ + ...$identity, + 'activity_attempt_id' => 'attempt-' . $number, + 'attempt_number' => $number, + ]; + $heartbeat = static fn (int $number): array => [ + ...$attempt($number), + 'activity_attempt' => [ + 'id' => 'attempt-' . $number, + 'attempt_number' => $number, + 'task_id' => 'task-' . $number, + ], + ]; + return [ + [ + 'event_type' => HistoryEventType::ActivityScheduled, + 'payload' => $identity, + ], + [ + 'event_type' => HistoryEventType::ActivityStarted, + 'workflow_task_id' => 'task-1', + 'payload' => [ + ...$attempt(1), + 'workflow_task_id' => 'task-1', + ], + ], + [ + 'event_type' => HistoryEventType::ActivityHeartbeatRecorded, + 'workflow_task_id' => 'task-1', + 'payload' => $heartbeat(1), + ], + [ + 'event_type' => HistoryEventType::ActivityRetryScheduled, + 'workflow_task_id' => 'task-1', + 'payload' => [ + ...$attempt(1), + 'retry_after_attempt_id' => 'attempt-1', + 'retry_after_attempt' => 1, + 'retry_of_task_id' => 'task-1', + 'retry_task_id' => 'task-2', + 'max_attempts' => 2, + ], + ], + [ + 'event_type' => HistoryEventType::ActivityStarted, + 'workflow_task_id' => 'task-2', + 'payload' => [ + ...$attempt(2), + 'workflow_task_id' => 'task-2', + ], + ], + [ + 'event_type' => HistoryEventType::ActivityHeartbeatRecorded, + 'workflow_task_id' => 'task-2', + 'payload' => $heartbeat(2), + ], + [ + 'event_type' => $terminal, + 'workflow_task_id' => 'task-2', + 'payload' => $attempt(2), + ], + ]; + } + + private static function models(array $history): array + { + return array_map(static fn (array $event): WorkflowHistoryEvent => new WorkflowHistoryEvent($event), $history); + } +} From d0920b12e510bce8a52c1f7a6f6df3439370f69f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:20:49 +0000 Subject: [PATCH 05/32] Prepare maintainer compensation scope foundation and InnoDB lock controls --- docs/compensation-scopes.md | 279 +++++++++ .../CompensateObligationActivity.php | 85 +++ .../Activities/ForwardOperationActivity.php | 80 +++ .../Activities/PrepareRecoveryActivity.php | 34 ++ .../Activities/RegisterChildScopeActivity.php | 54 ++ .../Activities/RegisterObligationActivity.php | 35 ++ .../Activities/RequestScopeActivity.php | 36 ++ .../Activities/VerifyScopeActivity.php | 30 + src/V2/Compensation/Definition/Obligation.php | 58 ++ .../Definition/PortableDescriptor.php | 73 +++ .../Compensation/Definition/ScopeIdentity.php | 52 ++ src/V2/Compensation/DurableScope.php | 70 +++ .../Execution/EvidenceReferences.php | 34 ++ .../Compensation/Execution/ForwardResult.php | 43 ++ .../Execution/HandlerRegistry.php | 48 ++ .../Execution/OperationContext.php | 29 + src/V2/Compensation/Execution/Resolution.php | 45 ++ .../Execution/ScopedOperation.php | 22 + .../Compensation/Persistence/ChildScopes.php | 104 ++++ .../Persistence/ScopeProgress.php | 195 +++++++ .../Persistence/ScopeRepository.php | 209 +++++++ .../Compensation/Persistence/ScopeStorage.php | 74 +++ .../Compensation/Recovery/ChildResolution.php | 52 ++ .../Recovery/RecoveryWorkflow.php | 30 + .../Compensation/Recovery/ScopeRecovery.php | 121 ++++ .../Compensation/Recovery/ScopeRetention.php | 94 ++++ .../Security/ScopeAuthorization.php | 20 + .../Compensation/Security/ScopeBoundary.php | 97 ++++ ...1_11_000100_create_compensation_scopes.php | 91 +++ .../V2/V2CompensationForwardAdmissionTest.php | 236 ++++++++ .../V2CompensationInnoDbConcurrencyTest.php | 172 ++++++ .../V2/V2CompensationScopeConcurrencyTest.php | 236 ++++++++ .../V2/V2CompensationScopeRunBoundaryTest.php | 210 +++++++ tests/Feature/V2/V2CompensationScopeTest.php | 528 ++++++++++++++++++ .../V2/TestForwardAdmissionWorkflow.php | 21 + .../Fixtures/V2/TestRetainedScopeWorkflow.php | 35 ++ .../V2/TestScopeBoundaryChildWorkflow.php | 32 ++ .../V2/TestScopeRunBoundaryWorkflow.php | 51 ++ tests/Fixtures/V2/TestScopedOperation.php | 54 ++ .../Fixtures/V2/TestWaitingScopeWorkflow.php | 30 + .../CompensationScopeRecoveryProcessTest.php | 90 +++ tests/Unit/V2/Compensation/ChildScopeTest.php | 74 +++ .../V2/Compensation/ScopeDescriptorTest.php | 97 ++++ .../V2/Compensation/ScopeProgressTest.php | 104 ++++ .../V2/Compensation/ScopeRecoveryTest.php | 111 ++++ .../V2/Compensation/ScopeRepositoryTest.php | 151 +++++ 46 files changed, 4426 insertions(+) create mode 100644 docs/compensation-scopes.md create mode 100644 src/V2/Compensation/Activities/CompensateObligationActivity.php create mode 100644 src/V2/Compensation/Activities/ForwardOperationActivity.php create mode 100644 src/V2/Compensation/Activities/PrepareRecoveryActivity.php create mode 100644 src/V2/Compensation/Activities/RegisterChildScopeActivity.php create mode 100644 src/V2/Compensation/Activities/RegisterObligationActivity.php create mode 100644 src/V2/Compensation/Activities/RequestScopeActivity.php create mode 100644 src/V2/Compensation/Activities/VerifyScopeActivity.php create mode 100644 src/V2/Compensation/Definition/Obligation.php create mode 100644 src/V2/Compensation/Definition/PortableDescriptor.php create mode 100644 src/V2/Compensation/Definition/ScopeIdentity.php create mode 100644 src/V2/Compensation/DurableScope.php create mode 100644 src/V2/Compensation/Execution/EvidenceReferences.php create mode 100644 src/V2/Compensation/Execution/ForwardResult.php create mode 100644 src/V2/Compensation/Execution/HandlerRegistry.php create mode 100644 src/V2/Compensation/Execution/OperationContext.php create mode 100644 src/V2/Compensation/Execution/Resolution.php create mode 100644 src/V2/Compensation/Execution/ScopedOperation.php create mode 100644 src/V2/Compensation/Persistence/ChildScopes.php create mode 100644 src/V2/Compensation/Persistence/ScopeProgress.php create mode 100644 src/V2/Compensation/Persistence/ScopeRepository.php create mode 100644 src/V2/Compensation/Persistence/ScopeStorage.php create mode 100644 src/V2/Compensation/Recovery/ChildResolution.php create mode 100644 src/V2/Compensation/Recovery/RecoveryWorkflow.php create mode 100644 src/V2/Compensation/Recovery/ScopeRecovery.php create mode 100644 src/V2/Compensation/Recovery/ScopeRetention.php create mode 100644 src/V2/Compensation/Security/ScopeAuthorization.php create mode 100644 src/V2/Compensation/Security/ScopeBoundary.php create mode 100644 src/migrations/2026_11_11_000100_create_compensation_scopes.php create mode 100644 tests/Feature/V2/V2CompensationForwardAdmissionTest.php create mode 100644 tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php create mode 100644 tests/Feature/V2/V2CompensationScopeConcurrencyTest.php create mode 100644 tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php create mode 100644 tests/Feature/V2/V2CompensationScopeTest.php create mode 100644 tests/Fixtures/V2/TestForwardAdmissionWorkflow.php create mode 100644 tests/Fixtures/V2/TestRetainedScopeWorkflow.php create mode 100644 tests/Fixtures/V2/TestScopeBoundaryChildWorkflow.php create mode 100644 tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php create mode 100644 tests/Fixtures/V2/TestScopedOperation.php create mode 100644 tests/Fixtures/V2/TestWaitingScopeWorkflow.php create mode 100644 tests/Support/CompensationScopeRecoveryProcessTest.php create mode 100644 tests/Unit/V2/Compensation/ChildScopeTest.php create mode 100644 tests/Unit/V2/Compensation/ScopeDescriptorTest.php create mode 100644 tests/Unit/V2/Compensation/ScopeProgressTest.php create mode 100644 tests/Unit/V2/Compensation/ScopeRecoveryTest.php create mode 100644 tests/Unit/V2/Compensation/ScopeRepositoryTest.php diff --git a/docs/compensation-scopes.md b/docs/compensation-scopes.md new file mode 100644 index 000000000..d5ac4d132 --- /dev/null +++ b/docs/compensation-scopes.md @@ -0,0 +1,279 @@ +# Durable compensation scopes (experimental) + +Implementation of the embedded, sequential scope design in +[retained compensation proposal](https://github.com/durable-workflow/workflow/issues/1063), preparing the +nested-machine integration in [business state-machine proposal](https://github.com/durable-workflow/workflow/issues/1069). +This is not a released API. Existing run-local Saga and the four-method +`TransitionStep` contract remain unchanged. + +## Reviewable increments + +1. Retained identities, bounded portable descriptors and atomic registration + journal. Prove exact duplicate adoption, conflicting reuse refusal, namespace + and generation separation, and rollback of incomplete registration. +2. Content-bound cancellation requests and canonical recovery identity. Reuse + native start/adopt, task ownership, activity attempts and redrive. Do not add + another executor, lease, claim table, heartbeat or retry policy engine. + Prove duplicate routing, closure races, namespace/generation substitution and + refused identity reuse. +3. Recorded activity boundaries and native Saga recovery, including closed runs. + Prove cold replay, registration acknowledgement loss, handler unavailability, + authorization, cancellation and recovery after worker restart. +4. Sequential child-scope participants and verified child-first compensation. + Prove generation binding, missing child evidence, cycle refusal and bounded + participation. State-bound SM bindings and the two-integration example remain + the next business state-machine slice, after this retained-scope foundation. +5. Retention and compatibility review; run ECS, PHPStan, unit, coverage and feature + gates. Record the limits of local evidence separately from CI qualification. + +Each increment follows tests, implementation, review, fixes and repeated review. +The issue remains the work record; this document defines the technical contract. + +## Boundaries + +A scope is `(application, namespace, key, generation)`, not a workflow run. +The native default namespace remains `null`; it is never replaced by an invented +alias. The original run and a later recovery run have separate identities. One scope +contains at most 256 obligations. Stored descriptors contain a closed handler +identifier and version, stable operation key and portable data (16 KiB, depth 8, +512 values). They never contain executable closures or arbitrary class names. +Applications store protected references instead of credentials or customer data. + +The registration journal and obligation commit together. Workflow history records +activity boundary outcomes; a lost activity acknowledgement is adopted by exact +identity. Forward work must use the scope admission gate. Registering a descriptor +alone does not make an unrelated activity cancellation-safe. + +Live database, authorization and handler resolution happen at activity boundaries, +never during deterministic replay. Native activity retries and native Saga remain +responsible for execution. No independent timer, worker or retry engine is added. +Every scope derives one canonical recovery workflow instance ID. Native DW +start/adopt admits one run; native redrive admits one successor after an eligible +failure. A native returned-existing-active start receipt remains eligible for +redrive only when its run identity and accepted start-command snapshot match. +A technical redrive key includes the exact failed source run. One unchanged +retained request can therefore recover several failed successors while duplicate +requests for one source adopt the same successor. Scope progress must be written +under the current native activity attempt's authority, in the same database transaction as the ledger mutation. A status check +followed by an unguarded write is insufficient. No scope lease or separate owner +claim is introduced. Native fencing cannot recall an issued external request. +Handlers must reconcile unknown provider outcomes using stable operation keys. +A returned callback, timeout or expired lease is not verified compensation. + +Rows use the workflow storage connection. Scope rows serialize ledger mutations; +obligation and journal rows follow that lock. No external call runs under a database +lock. The ledger has no cascading foreign key to run history: normal history +pruning must not erase unresolved obligations or reuse their identities. + +## Evidence + +Checked before implementation: upstream main `46ca8f67` contains the native +ownership guarantees. Four isolated PostgreSQL probes (32 assertions) proved +concurrent canonical start, concurrent redrive, exclusive workflow-task claim and +the fact that `business_key` alone does not deduplicate different instance IDs. +Three upstream PostgreSQL cancellation concurrency tests (160 assertions) preserved +canonical request/root/budget and explicitly refused a conflicting root. + +The first ledger increment passed 11 targeted tests (23 assertions), including an +injected journal-insert failure. It does not yet prove workflow-boundary integration. +The previous draft's recovery lease tests described an unimplemented competing +mechanism; they are replaced by tests of native identity and authority integration. + + +## Runtime flow and responsibilities + +```mermaid +flowchart TD + A[DurableScope.execute] --> B[RegisterObligationActivity] + B --> C[Atomic descriptor and journal] + C --> D[Native Saga callback registered before effect] + D --> E[ForwardOperationActivity] + E --> F[ScopeBoundary: native attempt + authorization + gate] + F --> G[Host operation outside SQL locks] + G --> H[Guarded forward outcome] + R[ScopeRecovery.request] --> Q[Content-bound request closes gate] + Q --> I{Origin active?} + I -->|yes| J[Native cooperative cancellation and original Saga] + I -->|closed| K[Canonical RecoveryWorkflow: native start/adopt] + K --> L[Native Saga in reverse registration order] + J --> L + L --> M[CompensateObligationActivity] + M --> N{Ordinary operation or child scope?} + N -->|operation| O[Closed versioned handler registry] + N -->|child| P[Exact child request and verified resolution] + O --> V[Guarded authoritative evidence] + P --> V + V --> W{All obligations verified?} + W -->|yes| X[VerifyScopeActivity] + W -->|unknown or failed| Y[Native retries; retain unresolved ledger] + Y --> Z[Eligible native redrive reuses verified activity prefix] +``` + +`ScopeBoundary` uses the existing native activity status/authority checks while +retaining their SQL locks in the ledger transaction. Its lock order is native +attempt, execution, run and task, followed by the scope and its journal. It checks +authority again after the mutation. Provider calls run between guarded transactions, +never inside them. The journal records principal, authorization policy version, +origin/current run, activity and attempt references. Child participation records +the child authorization policy separately from the parent registration policy. +A missing or invalid policy version refuses participation before registration. +The host must bind +`ScopeAuthorization`; there is no permissive default. + +`HandlerRegistry` is supplied by trusted host code. Its entries implement +`ScopedOperation::assertInput()`, `execute()` and `compensate()`. These are provider +boundary handlers; they do not change the four `TransitionStep` methods. Forward +and compensation outcomes accept a list of at most 32 immutable evidence +references, each at most 128 bytes. This leaves room for native identity metadata inside the +existing 16 KiB, 512-value journal bound. Larger provider proofs should be stored +as protected evidence bundles and referenced by a stable identifier. An +uncertain forward result requires reconciliation against the stable operation ID +before an unsafe provider reissue. `Resolution::verified()` additionally requires +settled-forward evidence; neither a callback return nor a native terminal run proves +that business result. Handler unavailability and descriptor hash mismatch refuse +before both forward and compensation provider operations. Forward handler and input +validation run before dispatch admission: a missing pinned handler or incompatible +input retains its native nonretryable reason without creating provider uncertainty. +A refused retry preserves any earlier uncertain attempt. Exceptions after provider +dispatch remain unknown and require reconciliation, even if their PHP type is +nonretryable. + +`ChildScopes` registers compensation participants, not state-machine bindings. +The initial graph is a bounded tree with one nesting level, at most 16 children +per parent and at most 256 obligations per scope (at most 4,352 across that tree). +Children cannot be shared between parents. One application/namespace graph-integrity +row serializes edge changes before scope locks. It contains no executor, lease, +heartbeat or retry state. A missing or unverified child stops earlier parent work. +Long child cleanup can exhaust the ordinary native activity retry budget; the +parent remains unresolved and can resume through eligible native redrive. + +A scope stays attached to its original native run for forward execution. This +increment does not transfer forward ownership across continue-as-new or origin +redrive. The first accepted request pins the cleanup principal. Later authorized +requests can route recovery but cannot replace that principal. Recovery evaluates +current authorization again at every activity boundary. Revoked authority leaves +work unresolved rather than silently adopting another principal. + +The default scoped forward/compensation activities use three native attempts and +the native backoff. No scope retry loop is introduced. `DurableScope.execute()` +accepts native `ActivityOptions` for forward work. Immediate terminal cancellation +or termination can prevent cleanup; retained obligations remain for explicit +recovery. A request accepted on an active origin is durable routing evidence; if +that origin closes without resolving its obligations, repeat the same authorized +request to route it to canonical recovery. + +`ScopeRetention::release()` requires verified work and terminal native executions, +and refuses a child still retained by an unresolved parent. It removes retained +inputs and forward results while preserving scope/generation, operation and request +identities, fingerprints, outcome references and journal tombstones. Forward +observation references are recorded in the immutable journal before release. +The journal does not retain the forward business value or obligation input. +No automatic expiry is added. Ordinary run-detail pruning cannot cascade into this ledger. + +## Continue-as-new and originating-run redrive + +Scope identity and native child supervision are separate. When a workflow +continues as new, native DW can hand its still-open children to the continued +run. This does not transfer the parent's compensation scope or the children's +own scopes. Each retained scope keeps its `original_run_id`. A parent that must +compensate a child registers it with `participate()` before dispatch, regardless +of which run later supervises that child. + +New registration from a continued or redriven origin is refused by the retained +original-run binding. A redrive may replay an observed activity prefix, but it +cannot use that prefix to dispatch new work in the old scope. Resuming an +uncertain forward activity in the successor is also refused by `ScopeBoundary` +before provider dispatch. The original obligations and provider evidence remain +available. Hosts request `ScopeRecovery` for that exact identity and generation +to resolve them. This increment does not silently create replacement scopes, +move obligations to a new generation or infer compensation from run closure. + +`V2CompensationScopeRunBoundaryTest` exercises real continue-as-new with an open +scoped child, native supervision handoff, refused successor work and verified +child-first recovery. It also checks originating-run redrive after an unrelated +failed activity and after an uncertain scoped forward effect. Recovery preserves +the original terminal history in all cases. + +## Core redrive regression evidence + +The replay corpus policy explicitly guards `FailedRunRedrivePlan` and +`ActivityRetryHistory`, so changes to either require executable regression evidence. + +The planner changes are independent of compensation scopes. They accept native +`ActivityRetryScheduled` lineage only when execution, attempt, task and attempt +number references agree. A valid native active-start adoption receipt must also +match the run and accepted command snapshot. The planner keeps current main's +completed-child reuse behavior. + +The `redrive-retried-completed-activity`, `redrive-retried-failed-activity` and +`redrive-retried-activity-after-completed-child` regression-corpus fixtures retain +both a cold replay prefix and the failed source history. The official +`ReplayRegressionCorpusTest` consumer executes `FailedRunRedrivePlan` against the +source and its identity-substitution cases, then executes the cold workflow +replay. This provides planner evidence separately from the scope integration +tests. `FailedRunRedrivePlanTest` separately covers start-adoption receipt +substitution. The separate `redrive-native-active-start-adoption` corpus fixture +executes the same acceptance and substitution checks for recorded start receipts. +Its planner source retains the native audit-enriched command snapshot. Its cold +replay prefix contains the authored event payload, which is checked against the +strict event payload contract before native audit enrichment. +Existing completed-child corpus and feature regressions remain +applicable. These evidence paths allow the core planner change to be reviewed +or extracted separately from the experimental scope API. + +## Authoring and host setup + +The embedded host binds `ScopeAuthorization` to its authorization policy and +`HandlerRegistry` to a closed map of handler type/version pairs. Normal package +migrations create the ledger tables on the configured workflow storage connection. +No additional queue, worker or lease configuration is required. + +Inside a native workflow, create `DurableScope($this, $identity, $principal)` and +call `execute(new Obligation($key, $handlerType, $handlerVersion, $input))`. Register +all effects through this helper. On cleanup, call the existing +`$this->compensate()` and then `$scope->verifyCompensation()`. The helper does not +catch the application's business failures or decide its business state. + +`participate($key, $childIdentity)` registers a compensation participant. It does +not start a child, await its business result or establish a state-machine binding. +Register that participant before dispatching new child work. Each child registers +its own provider obligations before effects. A missing child remains unresolved +and cannot satisfy the parent barrier. + +After the origin closes, the host calls `ScopeRecovery::request($identity, +$requestKey, $principal, $reason)`. Obtain the principal from trusted application +authorization, not from untrusted request content. Repeating the same request +requires the same principal and reason. Inspecting or requesting retained work +must go through an authorized host boundary; the `@internal` persistence classes +are not an HTTP-facing data access API. + +## Evidence and review + +| Guarantee | Responsible component | Acceptance test | +| --- | --- | --- | +| Exact request and generation identity | ScopeIdentity, ScopeRepository, ScopeRecovery | ScopeRecoveryTest, V2CompensationScopeConcurrencyTest | +| Current native attempt owns ledger mutation | ScopeBoundary | ExpiredNativeAttemptCannotPublishVerifiedCompensation | +| No false success for late/unknown effects | ScopeProgress, CompensateObligationActivity | PossibleLateForwardEffectCannotBecomeVerifiedAfterRetryExhaustion | +| Closed-run recovery and original terminal history | RecoveryWorkflow, native Saga | ClosedOriginUsesCanonicalRecoveryAndKeepsOriginalTerminalHistory | +| Verified progress survives failure and redrive | Native redrive plus ledger | RetryExhaustionStaysUnresolvedAndNativeRedriveReusesVerifiedWork, OneRetainedRequestCanRedriveTwoFailuresWithoutRepeatingVerifiedWork | +| Native start adoption preserves eligible redrive | FailedRunRedrivePlan | NativeActiveAdoptionDoesNotPreventRecoveryRedriveAfterFailure, SubstitutedActiveStartAdoptionCannotAuthorizeRedrive | +| Forward evidence survives explicit release | ScopeProgress, ScopeRetention | ForwardEvidenceSurvivesReleaseWithoutRetainingBusinessInputsOrResults | +| Replay does no handler/provider resolution | DurableScope, recorded activities | ColdReplayDoesNotResolveMutableHandlersOrRepeatProviderEffects | +| Child-first sequential progress | ChildScopes, ChildResolution | CompletedChildScopeIsVerifiedBeforeEarlierParentCompensation, CancellationWhileChildWaitsResolvesItsSagaBeforeEarlierParentWork | +| Child authorization evidence | RegisterChildScopeActivity | ChildParticipationRecordsItsOwnAuthorizationPolicyVersion, InvalidChildParticipationPolicyCannotRegisterAChild | +| Retained descriptor integrity before effects | ForwardOperationActivity, CompensateObligationActivity | TamperedRegistrationCannotReachTheForwardProvider, TamperedRetainedDescriptorIsRefusedBeforeProviderCall | +| Live storage is forbidden during replay | ScopeRepository, ScopeStorage | OpeningLiveStorageFromWorkflowReplayIsRefusedBeforeWriting | +| Bounded retained work | ScopeRepository, ChildScopes | ObligationLimitCannotLeaveAPartialRegistration, ChildFanOutLimitCannotLeaveAPartialParticipant | +| No cycle under concurrent registration | ChildScopes integrity lock | CompetingEdgesCannotCreateACycle | +| Ordinary pruning preserves recovery | Independent retained ledger | OrdinaryRunPruningPreservesRetainedScopeRecovery | +| Explicit release cannot reuse identifiers | ScopeRetention | ReleaseRefusesUnresolvedWorkThenRetainsAnUnreusableTombstone | +| Fresh-process recovery | Native workflow and persisted ledger | RetainedRecoveryRunsInAFreshPhpProcess | + +Integration exposed a native redrive limitation: `ActivityRetryScheduled` was +refused as unsupported history. `ActivityRetryHistory` validates exact execution, +attempt counters and task lineage before the existing redrive planner accepts that +history. Substitution controls accompany the real retry-exhaustion regression. + +Local evidence does not establish a published package, CI matrix qualification, +service-mode/cross-SDK support or production-provider behavior. The full quality +cycle and final evidence counts are recorded after the last code review. diff --git a/src/V2/Compensation/Activities/CompensateObligationActivity.php b/src/V2/Compensation/Activities/CompensateObligationActivity.php new file mode 100644 index 000000000..b007b62ca --- /dev/null +++ b/src/V2/Compensation/Activities/CompensateObligationActivity.php @@ -0,0 +1,85 @@ +run($this, $scope, 'compensate', null, function (array $snapshot) use ( + $scope, + $id, + &$origin + ): ?array { + $origin = $snapshot['original_run_id']; + return app(ScopeProgress::class)->beginCompensation($scope, $id, $this->attemptId()); + }); + if ($row === null) { + return; + } + $obligation = Obligation::fromStored($row); + $context = new OperationContext( + $scope, + $obligation, + $id, + $origin, + $this->activityId(), + $this->attemptId(), + $row['forward_status'] !== 'observed', + $row['forward_activity_id'], + $row['forward_attempt_id'], + ); + $unavailable = false; + try { + if ($obligation->handlerType === ChildScopes::TYPE) { + $result = app(ChildResolution::class)->resolve($scope, $obligation, $id); + } else { + $handler = app(HandlerRegistry::class)->resolve($obligation); + $result = $row['forward_status'] === 'registered' + ? Resolution::verified(['never-dispatched:' . $id], true) + : $handler->compensate($context); + } + } catch (NonRetryableException) { + $unavailable = true; + $result = Resolution::unknown('handler-unavailable'); + } catch (Throwable) { + $result = Resolution::unknown('provider-outcome-unknown'); + } + $boundary->run( + $this, + $scope, + 'compensate', + null, + fn () => app(ScopeProgress::class)->resolve($scope, $id, $this->attemptId(), $result) + ); + if ($unavailable) { + throw new NonRetryableException('scope_handler_unavailable'); + } + if (! $result->verified) { + throw new \RuntimeException('scope_compensation_unknown'); + } + } +} diff --git a/src/V2/Compensation/Activities/ForwardOperationActivity.php b/src/V2/Compensation/Activities/ForwardOperationActivity.php new file mode 100644 index 000000000..26dd99b5a --- /dev/null +++ b/src/V2/Compensation/Activities/ForwardOperationActivity.php @@ -0,0 +1,80 @@ +run( + $this, + $scope, + 'forward', + $principal, + function () use ($scope, $id, &$handler): array { + return app(ScopeProgress::class)->admitForward( + $scope, + $id, + $this->attemptId(), + $this->activityId(), + static function (array $row) use (&$handler): void { + $handler = app(HandlerRegistry::class)->resolve(Obligation::fromStored($row)); + } + ); + } + ); + if ($admission['cached'] !== null) { + return $admission['cached']['value']; + } + if (! $handler instanceof ScopedOperation) { + throw new \LogicException('The forward handler was not validated before dispatch.'); + } + $obligation = Obligation::fromStored($admission['row']); + $context = new OperationContext( + $scope, + $obligation, + $id, + $this->runId(), + $this->activityId(), + $this->attemptId(), + $admission['uncertain'] + ); + try { + $result = $handler->execute($context); + } catch (Throwable) { + $result = ForwardResult::unknown('provider-outcome-unknown'); + } + $boundary->run( + $this, + $scope, + 'forward', + $principal, + fn () => app(ScopeProgress::class)->forwardResult($scope, $id, $this->attemptId(), $result) + ); + if (! $result->observed) { + throw new \RuntimeException('scope_forward_unknown'); + } + return $result->value; + } +} diff --git a/src/V2/Compensation/Activities/PrepareRecoveryActivity.php b/src/V2/Compensation/Activities/PrepareRecoveryActivity.php new file mode 100644 index 000000000..973138b14 --- /dev/null +++ b/src/V2/Compensation/Activities/PrepareRecoveryActivity.php @@ -0,0 +1,34 @@ +run( + $this, + $scope, + 'compensate', + null, + static function (array $snapshot): array { + if (! in_array($snapshot['status'], ['requested', 'verified'], true)) { + throw new \LogicException('The scope has no accepted compensation request.'); + } + return array_column($snapshot['obligations'], 'id'); + } + ); + } +} diff --git a/src/V2/Compensation/Activities/RegisterChildScopeActivity.php b/src/V2/Compensation/Activities/RegisterChildScopeActivity.php new file mode 100644 index 000000000..ac5fbbfe9 --- /dev/null +++ b/src/V2/Compensation/Activities/RegisterChildScopeActivity.php @@ -0,0 +1,54 @@ +run( + $this, + $scope, + 'register', + $principal, + function () use ($scope, $key, $child, $principal): string { + try { + $policy = app(ScopeAuthorization::class)->assertAllowed($child, $principal, 'participate'); + PortableDescriptor::identifier($policy, 'authorization policy version'); + } catch (Throwable) { + throw new NonRetryableException('scope_child_authorization_refused'); + } + $id = app(ChildScopes::class)->register($scope, $this->runId(), $key, $child); + ScopeStorage::append(ScopeStorage::locked($scope), 'child_authorized', [ + 'childScopeId' => $child->id(), + 'principal' => $principal, + 'operation' => 'participate', + 'policyVersion' => $policy, + 'runId' => $this->runId(), + 'activityId' => $this->activityId(), + 'attemptId' => $this->attemptId(), + ]); + return $id; + } + ); + } +} diff --git a/src/V2/Compensation/Activities/RegisterObligationActivity.php b/src/V2/Compensation/Activities/RegisterObligationActivity.php new file mode 100644 index 000000000..016372a62 --- /dev/null +++ b/src/V2/Compensation/Activities/RegisterObligationActivity.php @@ -0,0 +1,35 @@ +run($this, $scope, 'register', $principal, function () use ( + $scope, + $obligation + ): string { + app(HandlerRegistry::class)->resolve($obligation); + $repository = app(ScopeRepository::class); + $repository->open($scope, $this->runId()); + return $repository->register($scope, $obligation); + }); + } +} diff --git a/src/V2/Compensation/Activities/RequestScopeActivity.php b/src/V2/Compensation/Activities/RequestScopeActivity.php new file mode 100644 index 000000000..859361d11 --- /dev/null +++ b/src/V2/Compensation/Activities/RequestScopeActivity.php @@ -0,0 +1,36 @@ +run( + $this, + $scope, + 'request', + $principal, + static fn (array $snapshot, string $policyVersion) => app(ScopeRepository::class)->request( + $scope, + 'native-saga', + $principal, + 'workflow-cleanup', + $policyVersion + ) + ); + } +} diff --git a/src/V2/Compensation/Activities/VerifyScopeActivity.php b/src/V2/Compensation/Activities/VerifyScopeActivity.php new file mode 100644 index 000000000..4ba956e36 --- /dev/null +++ b/src/V2/Compensation/Activities/VerifyScopeActivity.php @@ -0,0 +1,30 @@ +run( + $this, + $scope, + 'compensate', + null, + static fn () => app(ScopeProgress::class)->finish($scope) + ); + } +} diff --git a/src/V2/Compensation/Definition/Obligation.php b/src/V2/Compensation/Definition/Obligation.php new file mode 100644 index 000000000..936c73331 --- /dev/null +++ b/src/V2/Compensation/Definition/Obligation.php @@ -0,0 +1,58 @@ +toArray()); + } + + public function fingerprint(): string + { + return hash('sha256', PortableDescriptor::encode($this->toArray())); + } + + public function toArray(): array + { + return get_object_vars($this); + } + + public static function fromStored(array $row): self + { + $value = self::fromArray(json_decode($row['descriptor'], true, flags: JSON_THROW_ON_ERROR)); + if (! hash_equals($row['fingerprint'], $value->fingerprint())) { + throw new \LogicException('The retained compensation descriptor content hash does not match.'); + } + return $value; + } + + public static function fromArray(array $value): self + { + $keys = array_keys($value); + sort($keys, SORT_STRING); + if ($keys !== ['handlerType', 'handlerVersion', 'input', 'key']) { + throw new InvalidArgumentException('Invalid compensation obligation shape.'); + } + return new self($value['key'], $value['handlerType'], $value['handlerVersion'], $value['input']); + } +} diff --git a/src/V2/Compensation/Definition/PortableDescriptor.php b/src/V2/Compensation/Definition/PortableDescriptor.php new file mode 100644 index 000000000..6dfe82a6c --- /dev/null +++ b/src/V2/Compensation/Definition/PortableDescriptor.php @@ -0,0 +1,73 @@ + self::MAX_BYTES) { + throw new InvalidArgumentException('Compensation data exceeds the byte limit.'); + } + return $json; + } + + public static function identifier(string $value, string $name): void + { + if ($value === '' || strlen($value) > 128 || ! preg_match('/\A[a-zA-Z0-9][a-zA-Z0-9._:\/-]*\z/D', $value)) { + throw new InvalidArgumentException("Invalid compensation {$name}."); + } + } + + private static function normalize(mixed $value, int $depth, int &$count): mixed + { + if (++$count > self::MAX_VALUES || $depth > self::MAX_DEPTH) { + throw new InvalidArgumentException('Compensation data exceeds the structural limit.'); + } + if (is_array($value)) { + if (! array_is_list($value)) { + ksort($value, SORT_STRING); + } + foreach ($value as $key => $child) { + $value[$key] = self::normalize($child, $depth + 1, $count); + } + return $value; + } + if ($value === null || is_bool($value) || is_int($value) || is_string($value) + || (is_float($value) && is_finite($value))) { + return $value; + } + throw new InvalidArgumentException( + 'Compensation data cannot contain objects, resources or non-finite numbers.' + ); + } +} diff --git a/src/V2/Compensation/Definition/ScopeIdentity.php b/src/V2/Compensation/Definition/ScopeIdentity.php new file mode 100644 index 000000000..f2ab19902 --- /dev/null +++ b/src/V2/Compensation/Definition/ScopeIdentity.php @@ -0,0 +1,52 @@ + $value) { + if ($name === 'namespace' && $value === null) { + continue; + } + PortableDescriptor::identifier($value, $name); + } + } + + public function id(): string + { + return hash('sha256', PortableDescriptor::encode($this->toArray())); + } + + public function recoveryInstanceId(): string + { + return 'dw-compensation-' . $this->id(); + } + + public function toArray(): array + { + return get_object_vars($this); + } + + public static function fromArray(array $value): self + { + if (array_keys($value) !== ['application', 'namespace', 'key', 'generation']) { + $keys = array_keys($value); + sort($keys, SORT_STRING); + if ($keys !== ['application', 'generation', 'key', 'namespace']) { + throw new \InvalidArgumentException('Invalid compensation scope identity shape.'); + } + } + return new self($value['application'], $value['namespace'], $value['key'], $value['generation']); + } +} diff --git a/src/V2/Compensation/DurableScope.php b/src/V2/Compensation/DurableScope.php new file mode 100644 index 000000000..6237a7f75 --- /dev/null +++ b/src/V2/Compensation/DurableScope.php @@ -0,0 +1,70 @@ +identity->toArray(); + $principal = $this->principal; + $id = Workflow::activity(RegisterObligationActivity::class, $scope, $obligation->toArray(), $principal); + // The callback is rebuilt by replay before dispatching the effect and never needs its return value. + $this->saga->addCompensation(static function () use ($scope, $principal, $id): void { + Workflow::activity(RequestScopeActivity::class, $scope, $principal); + Workflow::activity(CompensateObligationActivity::class, $scope, $id); + }); + return Workflow::activity( + ForwardOperationActivity::class, + $options ?? new ActivityOptions(), + $scope, + $id, + $principal + ); + } + + public function participate(string $key, ScopeIdentity $child): void + { + $scope = $this->identity->toArray(); + $principal = $this->principal; + $id = Workflow::activity(RegisterChildScopeActivity::class, $scope, $key, $child->toArray(), $principal); + $this->saga->addCompensation(static function () use ($scope, $principal, $id): void { + Workflow::activity(RequestScopeActivity::class, $scope, $principal); + Workflow::activity(CompensateObligationActivity::class, $scope, $id); + }); + } + + public function verifyCompensation(): array + { + return Workflow::activity(VerifyScopeActivity::class, $this->identity->toArray()); + } +} diff --git a/src/V2/Compensation/Execution/EvidenceReferences.php b/src/V2/Compensation/Execution/EvidenceReferences.php new file mode 100644 index 000000000..05b8273ad --- /dev/null +++ b/src/V2/Compensation/Execution/EvidenceReferences.php @@ -0,0 +1,34 @@ + self::MAX_REFERENCES) { + throw new InvalidArgumentException('Compensation data exceeds the evidence reference limit.'); + } + foreach ($references as $reference) { + if (! is_string($reference)) { + throw new InvalidArgumentException('Evidence requires immutable reference strings.'); + } + PortableDescriptor::identifier($reference, 'evidence reference'); + } + } +} diff --git a/src/V2/Compensation/Execution/ForwardResult.php b/src/V2/Compensation/Execution/ForwardResult.php new file mode 100644 index 000000000..dae97b3dd --- /dev/null +++ b/src/V2/Compensation/Execution/ForwardResult.php @@ -0,0 +1,43 @@ +toArray()); + } + + public static function observed(array $value, array $evidence): self + { + return new self(true, $value, $evidence, 'observed'); + } + + public static function unknown(string $reason): self + { + return new self(false, [], [], $reason); + } + + public function toArray(): array + { + return get_object_vars($this); + } +} diff --git a/src/V2/Compensation/Execution/HandlerRegistry.php b/src/V2/Compensation/Execution/HandlerRegistry.php new file mode 100644 index 000000000..b9cc28f43 --- /dev/null +++ b/src/V2/Compensation/Execution/HandlerRegistry.php @@ -0,0 +1,48 @@ +> $handlers + */ + public function __construct( + private array $handlers = [] + ) { + if (isset($handlers[\Workflow\V2\Compensation\Persistence\ChildScopes::TYPE])) { + throw new \InvalidArgumentException('The child-scope handler key is reserved.'); + } + foreach ($handlers as $versions) { + foreach ($versions as $handler) { + if (! $handler instanceof ScopedOperation) { + throw new \InvalidArgumentException('Scope handlers must implement ScopedOperation.'); + } + } + } + } + + public function resolve(Obligation $obligation): ScopedOperation + { + $handler = $this->handlers[$obligation->handlerType][$obligation->handlerVersion] ?? null; + if (! $handler instanceof ScopedOperation) { + throw new NonRetryableException('scope_handler_unavailable'); + } + try { + $handler->assertInput($obligation->input); + } catch (\Throwable) { + throw new NonRetryableException('scope_input_incompatible'); + } + return $handler; + } +} diff --git a/src/V2/Compensation/Execution/OperationContext.php b/src/V2/Compensation/Execution/OperationContext.php new file mode 100644 index 000000000..6ca3caac3 --- /dev/null +++ b/src/V2/Compensation/Execution/OperationContext.php @@ -0,0 +1,29 @@ +toArray()); + } + + public static function verified(array $evidence, bool $forwardSettled): self + { + return new self(true, 'verified', $evidence, $forwardSettled); + } + + public static function unknown(string $reason): self + { + return new self(false, $reason, [], false); + } + + public function toArray(): array + { + return get_object_vars($this); + } +} diff --git a/src/V2/Compensation/Execution/ScopedOperation.php b/src/V2/Compensation/Execution/ScopedOperation.php new file mode 100644 index 000000000..6a37762ab --- /dev/null +++ b/src/V2/Compensation/Execution/ScopedOperation.php @@ -0,0 +1,22 @@ +application !== $child->application || $parent->namespace !== $child->namespace || $parent->id() === $child->id()) { + throw new LogicException('A child scope must have another identity in the same application and namespace.'); + } + $database = ConfiguredV2Models::connection(); + $graph = hash('sha256', PortableDescriptor::encode([$parent->application, $parent->namespace])); + return SqliteTransaction::run(static function () use ( + $parent, + $originalRunId, + $key, + $child, + $database, + $graph + ): string { + // Serialize edge mutations before scope locks. This is a graph-integrity lock, not executor ownership. + $existingGraph = $database->table('workflow_compensation_graphs') + ->where('id', $graph) + ->lockForUpdate() + ->first(); + if ($existingGraph === null) { + $database->table('workflow_compensation_graphs') + ->insertOrIgnore([ + 'id' => $graph, + ]); + $database->table('workflow_compensation_graphs') + ->where('id', $graph) + ->lockForUpdate() + ->first(); + } + $existing = $database->table('workflow_compensation_children') + ->where('child_id', $child->id()) + ->lockForUpdate() + ->first(); + if (($existing !== null && $existing->parent_id !== $parent->id()) + || $database->table('workflow_compensation_children') + ->where('child_id', $parent->id()) + ->lockForUpdate() + ->first() !== null + || $database->table('workflow_compensation_children') + ->where('parent_id', $child->id()) + ->lockForUpdate() + ->first() !== null) { + throw new LogicException( + 'A child scope cannot be shared, cyclic, or nested beyond the supported depth.' + ); + } + if ($existing === null && $database->table('workflow_compensation_children')->where( + 'parent_id', + $parent->id() + )->lockForUpdate() + ->get() + ->count() >= self::MAX_CHILDREN) { + throw new LogicException('The child scope fan-out limit is exhausted.'); + } + $repository = new ScopeRepository(); + $repository->open($parent, $originalRunId); + $obligation = new Obligation($key, self::TYPE, self::VERSION, [ + 'child' => $child->toArray(), + ]); + $id = $repository->register($parent, $obligation); + if ($existing !== null && $existing->obligation_id !== $id) { + throw new LogicException('The child generation already belongs to another participant.'); + } + if ($existing === null) { + $database->table('workflow_compensation_children') + ->insert([ + 'child_id' => $child->id(), + 'parent_id' => $parent->id(), + 'obligation_id' => $id, + ]); + } + return $id; + }, connection: $database); + } +} diff --git a/src/V2/Compensation/Persistence/ScopeProgress.php b/src/V2/Compensation/Persistence/ScopeProgress.php new file mode 100644 index 000000000..ca61b03d2 --- /dev/null +++ b/src/V2/Compensation/Persistence/ScopeProgress.php @@ -0,0 +1,195 @@ +obligation($scope, $database, $id); + if ($scope['status'] !== 'open') { + throw new LogicException('The scope has closed forward admission.'); + } + if ($row['forward_status'] === 'observed') { + return [ + 'cached' => json_decode($row['forward_result'], true, flags: JSON_THROW_ON_ERROR), + 'uncertain' => false, + 'row' => $row, + ]; + } + $beforeDispatch?->__invoke($row); + $uncertain = $row['forward_status'] !== 'registered'; + $database->table(ScopeStorage::OBLIGATIONS)->where('id', $id)->update([ + 'forward_status' => 'in_flight', + 'forward_activity_id' => $activityId, + 'forward_attempt_id' => $attemptId, + ]); + ScopeStorage::append($scope, 'forward_admitted', [ + 'obligationId' => $id, + 'attemptId' => $attemptId, + 'uncertain' => $uncertain, + ]); + return [ + 'cached' => null, + 'uncertain' => $uncertain, + 'row' => $row, + ]; + }); + } + + public function forwardResult(ScopeIdentity $identity, string $id, string $attemptId, ForwardResult $result): void + { + ScopeStorage::mutate($identity, function (array $scope, Connection $database) use ( + $id, + $attemptId, + $result + ): void { + $row = $this->obligation($scope, $database, $id); + $payload = PortableDescriptor::encode($result->toArray()); + if ($row['forward_attempt_id'] !== $attemptId) { + throw new LogicException('The forward result belongs to another native attempt.'); + } + if ($row['forward_result'] === $payload && $row['forward_status'] === ($result->observed ? 'observed' : 'unknown')) { + return; + } + if ($row['forward_status'] === 'observed' || $row['compensation_status'] === 'verified') { + throw new LogicException('A terminal scope outcome cannot be replaced.'); + } + $database->table(ScopeStorage::OBLIGATIONS)->where('id', $id)->update([ + 'forward_status' => $result->observed ? 'observed' : 'unknown', + 'forward_result' => $payload, + ]); + ScopeStorage::append($scope, 'forward_outcome', [ + 'obligationId' => $id, + 'attemptId' => $attemptId, + 'observed' => $result->observed, + 'reason' => $result->reason, + 'evidence' => $result->evidence, + ]); + }); + } + + public function beginCompensation(ScopeIdentity $identity, string $id, string $attemptId): ?array + { + return ScopeStorage::mutate($identity, function (array $scope, Connection $database) use ( + $id, + $attemptId + ): ?array { + $row = $this->obligation($scope, $database, $id); + if (! in_array($scope['status'], ['requested', 'verified'], true)) { + throw new LogicException('Compensation requires an accepted request.'); + } + if ($row['compensation_status'] === 'verified') { + return null; + } + $next = $database->table(ScopeStorage::OBLIGATIONS)->where('scope_id', $scope['id']) + ->where('compensation_status', '<>', 'verified') + ->orderByDesc('position') + ->value('id'); + if ($next !== $id) { + throw new LogicException('Sequential compensation must resolve later obligations first.'); + } + $database->table(ScopeStorage::OBLIGATIONS)->where('id', $id)->update([ + 'compensation_status' => 'executing', + 'compensation_attempt_id' => $attemptId, + ]); + ScopeStorage::append($scope, 'compensation_started', [ + 'obligationId' => $id, + 'attemptId' => $attemptId, + ]); + return $row; + }); + } + + public function resolve(ScopeIdentity $identity, string $id, string $attemptId, Resolution $result): void + { + ScopeStorage::mutate($identity, function (array $scope, Connection $database) use ( + $id, + $attemptId, + $result + ): void { + $row = $this->obligation($scope, $database, $id); + $payload = PortableDescriptor::encode($result->toArray()); + if ($row['compensation_attempt_id'] !== $attemptId) { + throw new LogicException('The compensation outcome belongs to another native attempt.'); + } + if ($row['evidence'] === $payload && $row['compensation_status'] === ($result->verified ? 'verified' : 'unknown')) { + return; + } + if ($row['compensation_status'] === 'verified' || ($result->verified && ! $result->forwardSettled)) { + throw new LogicException('Compensation is not verified while a late forward effect is still possible.'); + } + $database->table(ScopeStorage::OBLIGATIONS)->where('id', $id)->update([ + 'compensation_status' => $result->verified ? 'verified' : 'unknown', + 'evidence' => $payload, + ]); + ScopeStorage::append($scope, 'compensation_outcome', [ + 'obligationId' => $id, + 'attemptId' => $attemptId, + 'verified' => $result->verified, + 'reason' => $result->reason, + 'evidence' => $result->evidence, + ]); + }); + } + + public function finish(ScopeIdentity $identity): array + { + return ScopeStorage::mutate($identity, static function (array $scope, Connection $database): array { + if (! in_array($scope['status'], ['requested', 'verified'], true) + || $database->table(ScopeStorage::OBLIGATIONS)->where('scope_id', $scope['id']) + ->where('compensation_status', '<>', 'verified') + ->exists()) { + throw new LogicException('The scope retains unresolved compensation obligations.'); + } + if ($scope['status'] !== 'verified') { + $database->table(ScopeStorage::SCOPES)->where('id', $scope['id'])->update([ + 'status' => 'verified', + ]); + ScopeStorage::append($scope, 'verified', []); + } + return [ + 'scopeId' => $scope['id'], + 'status' => 'verified', + ]; + }); + } + + private function obligation(array $scope, Connection $database, string $id): array + { + $row = $database->table(ScopeStorage::OBLIGATIONS)->where('scope_id', $scope['id'])->where('id', $id)->first(); + if ($row === null) { + throw new LogicException('The obligation is unavailable in the exact scope generation.'); + } + return (array) $row; + } +} diff --git a/src/V2/Compensation/Persistence/ScopeRepository.php b/src/V2/Compensation/Persistence/ScopeRepository.php new file mode 100644 index 000000000..56da024d7 --- /dev/null +++ b/src/V2/Compensation/Persistence/ScopeRepository.php @@ -0,0 +1,209 @@ +table(self::SCOPES)->where('id', $identity->id())->lockForUpdate()->first(); + $created = $existing === null ? $database->table(self::SCOPES)->insertOrIgnore([ + 'id' => $identity->id(), + 'identity' => PortableDescriptor::encode($identity->toArray()), + 'original_run_id' => $originalRunId, + 'status' => 'open', + ]) : 0; + $scope = ScopeStorage::locked($identity); + if ($scope['original_run_id'] !== $originalRunId) { + throw new LogicException('A compensation scope cannot change its original run.'); + } + if ($created === 1) { + ScopeStorage::append($scope, 'opened', [ + 'originalRunId' => $originalRunId, + ]); + } + }, connection: $database); + } + + public function register(ScopeIdentity $identity, Obligation $obligation): string + { + return ScopeStorage::mutate($identity, static function (array $scope, Connection $database) use ( + $identity, + $obligation + ): string { + if ($scope['status'] === 'released') { + throw new LogicException('A released compensation scope cannot be reused.'); + } + $id = hash('sha256', PortableDescriptor::encode([$identity->id(), $obligation->key])); + $existing = $database->table(self::OBLIGATIONS)->where('id', $id)->first(); + if ($existing !== null) { + if ($existing->fingerprint !== $obligation->fingerprint()) { + throw new LogicException('A compensation operation key was reused with different content.'); + } + return $id; + } + if ($scope['status'] !== 'open' || $scope['obligation_count'] >= self::MAX_OBLIGATIONS) { + throw new LogicException('The compensation scope does not admit new obligations.'); + } + $position = (int) $scope['obligation_count'] + 1; + $database->table(self::OBLIGATIONS)->insert([ + 'id' => $id, + 'scope_id' => $identity->id(), + 'position' => $position, + 'fingerprint' => $obligation->fingerprint(), + 'descriptor' => PortableDescriptor::encode($obligation->toArray()), + 'forward_status' => 'registered', + 'compensation_status' => 'pending', + ]); + $database->table(self::SCOPES)->where('id', $scope['id'])->update([ + 'obligation_count' => $position, + ]); + ScopeStorage::append($scope, 'registered', [ + 'obligationId' => $id, + 'fingerprint' => $obligation->fingerprint(), + ]); + return $id; + }); + } + + public function request( + ScopeIdentity $identity, + string $key, + string $principal, + string $reason, + ?string $policyVersion = null + ): void { + foreach ([ + 'key' => $key, + 'principal' => $principal, + 'reason' => $reason, + ] as $name => $value) { + PortableDescriptor::identifier($value, $name); + } + $descriptor = PortableDescriptor::encode([ + 'key' => $key, + 'principal' => $principal, + 'reason' => $reason, + ]); + $fingerprint = hash('sha256', $descriptor); + ScopeStorage::mutate($identity, static function (array $scope, Connection $database) use ( + $identity, + $key, + $descriptor, + $fingerprint, + $policyVersion + ): void { + $id = hash('sha256', PortableDescriptor::encode([$identity->id(), $key])); + $existing = $database->table('workflow_compensation_requests') + ->where('id', $id) + ->first(); + if ($existing !== null) { + if ($existing->fingerprint !== $fingerprint) { + throw new LogicException( + 'The compensation request identity was reused with different authority or content.' + ); + } + return; + } + if ($scope['status'] === 'released' || $scope['request_count'] >= 128) { + throw new LogicException('The compensation scope does not admit another request.'); + } + $position = (int) $scope['request_count'] + 1; + $database->table('workflow_compensation_requests') + ->insert([ + 'id' => $id, + 'scope_id' => $scope['id'], + 'position' => $position, + 'fingerprint' => $fingerprint, + 'descriptor' => $descriptor, + 'authorization_policy_version' => $policyVersion, + ]); + $database->table(self::SCOPES)->where('id', $scope['id'])->update([ + 'request_count' => $position, + 'status' => $scope['status'] === 'verified' ? 'verified' : 'requested', + ]); + ScopeStorage::append($scope, 'requested', [ + 'requestId' => $id, + 'fingerprint' => $fingerprint, + 'policyVersion' => $policyVersion, + ]); + }); + } + + public function primaryRequest(ScopeIdentity $identity): array + { + return ScopeStorage::mutate($identity, static function (array $scope, Connection $database): array { + $row = $database->table('workflow_compensation_requests') + ->where('scope_id', $scope['id'])->orderBy('position')->first(); + if ($row === null) { + throw new LogicException('The compensation scope has no authorized request.'); + } + return json_decode($row->descriptor, true, flags: JSON_THROW_ON_ERROR); + }); + } + + public function metadata(ScopeIdentity $identity): array + { + return ScopeStorage::mutate($identity, static fn (array $scope): array => $scope); + } + + public function inspect(ScopeIdentity $identity): array + { + return ScopeStorage::mutate($identity, static function (array $scope, Connection $database): array { + $scope['identity'] = json_decode($scope['identity'], true, flags: JSON_THROW_ON_ERROR); + $scope['obligations'] = $database->table(self::OBLIGATIONS)->where('scope_id', $scope['id']) + ->orderBy('position') + ->get() + ->map(static function (object $row): array { + $item = (array) $row; + $item['descriptor'] = json_decode($item['descriptor'], true, flags: JSON_THROW_ON_ERROR); + return $item; + })->all(); + return $scope; + }); + } + + public function journal(ScopeIdentity $identity): array + { + return ScopeStorage::mutate($identity, static fn (array $scope, Connection $database): array => + $database->table(self::JOURNAL)->where('scope_id', $scope['id'])->orderBy('sequence') + ->get() + ->map(static fn (object $row): array => (array) $row) + ->all()); + } +} diff --git a/src/V2/Compensation/Persistence/ScopeStorage.php b/src/V2/Compensation/Persistence/ScopeStorage.php new file mode 100644 index 000000000..3abea904b --- /dev/null +++ b/src/V2/Compensation/Persistence/ScopeStorage.php @@ -0,0 +1,74 @@ + $operation(self::locked($identity), $database), + admission: static function () use ($database, $identity): void { + $database->table(self::SCOPES)->where('id', $identity->id())->update([ + 'id' => $identity->id(), + ]); + }, + connection: $database + ); + } + + public static function locked(ScopeIdentity $identity): array + { + $row = ConfiguredV2Models::connection()->table(self::SCOPES)->where('id', $identity->id())->lockForUpdate() + ->first(); + if ($row === null || $row->identity !== PortableDescriptor::encode($identity->toArray())) { + throw new LogicException('The exact compensation scope generation is unavailable.'); + } + return (array) $row; + } + + public static function append(array $scope, string $kind, array $data): void + { + $database = ConfiguredV2Models::connection(); + $sequence = (int) $database->table(self::SCOPES)->where('id', $scope['id'])->value('event_sequence') + 1; + if ($sequence > 8192) { + throw new LogicException('The scope journal work limit is exhausted; unresolved work is retained.'); + } + $database->table(self::JOURNAL)->insert([ + 'scope_id' => $scope['id'], + 'sequence' => $sequence, + 'kind' => $kind, + 'data' => PortableDescriptor::encode($data), + 'recorded_at' => now('UTC'), + ]); + $database->table(self::SCOPES)->where('id', $scope['id'])->update([ + 'event_sequence' => $sequence, + ]); + } +} diff --git a/src/V2/Compensation/Recovery/ChildResolution.php b/src/V2/Compensation/Recovery/ChildResolution.php new file mode 100644 index 000000000..55fa1117c --- /dev/null +++ b/src/V2/Compensation/Recovery/ChildResolution.php @@ -0,0 +1,52 @@ +handlerVersion !== ChildScopes::VERSION || array_keys($obligation->input) !== [ + 'child', + ] || ! is_array($obligation->input['child'])) { + return Resolution::unknown('child-descriptor-incompatible'); + } + $child = ScopeIdentity::fromArray($obligation->input['child']); + if ($child->application !== $parent->application || $child->namespace !== $parent->namespace) { + return Resolution::unknown('child-owner-mismatch'); + } + $principal = $this->repository->primaryRequest($parent)['principal']; + try { + $this->authorization->assertAllowed($child, $principal, 'compensate'); + $this->recovery->request($child, 'parent-' . $operationId, $principal, 'parent-compensation'); + $snapshot = $this->repository->inspect($child); + } catch (Throwable) { + return Resolution::unknown('child-unresolved'); + } + return $snapshot['status'] === 'verified' + ? Resolution::verified(['scope-verified:' . $child->id()], true) + : Resolution::unknown('child-unresolved'); + } +} diff --git a/src/V2/Compensation/Recovery/RecoveryWorkflow.php b/src/V2/Compensation/Recovery/RecoveryWorkflow.php new file mode 100644 index 000000000..d9025e8ba --- /dev/null +++ b/src/V2/Compensation/Recovery/RecoveryWorkflow.php @@ -0,0 +1,30 @@ +addCompensation(static fn () => self::activity(CompensateObligationActivity::class, $scope, $id)); + } + $this->compensate(); + return self::activity(VerifyScopeActivity::class, $scope); + } +} diff --git a/src/V2/Compensation/Recovery/ScopeRecovery.php b/src/V2/Compensation/Recovery/ScopeRecovery.php new file mode 100644 index 000000000..3ace65c7a --- /dev/null +++ b/src/V2/Compensation/Recovery/ScopeRecovery.php @@ -0,0 +1,121 @@ +authorization->assertAllowed($scope, $principal, 'request'); + PortableDescriptor::identifier($policyVersion, 'authorization policy version'); + $snapshot = $this->repository->inspect($scope); + if ($snapshot['status'] === 'released') { + throw new LogicException('The retained compensation scope has been released.'); + } + $origin = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($snapshot['original_run_id']); + if (! $origin instanceof WorkflowRun || $origin->namespace !== $scope->namespace) { + throw new LogicException('The original compensation run is unavailable in the exact namespace.'); + } + $this->repository->request($scope, $requestKey, $principal, $reason, $policyVersion); + if ($snapshot['status'] === 'verified') { + return [ + 'route' => 'verified', + 'runId' => null, + ]; + } + if (! $origin->status->isTerminal()) { + $command = WorkflowStub::loadRun($origin->id)->attemptRequestCancellation($reason); + if ($command->accepted()) { + return [ + 'route' => 'origin', + 'runId' => $origin->id, + 'commandId' => $command->commandId(), + ]; + } + // Closure can win after the gate transaction. Route that same retained request below. + $origin->refresh(); + if (! $origin->status->isTerminal()) { + throw new LogicException('The originating run refused cooperative compensation.'); + } + } + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('workflow_instance_id', $scope->recoveryInstanceId()) + ->where('namespace', $scope->namespace) + ->orderByDesc('run_number') + ->first(); + if (! $run instanceof WorkflowRun) { + $result = WorkflowStub::make(RecoveryWorkflow::class, $scope->recoveryInstanceId(), $scope->namespace) + ->attemptStart(StartOptions::returnExistingActive(), $scope->toArray()); + $runId = $result->runId(); + if ($result->accepted() && is_string($runId)) { + return [ + 'route' => 'recovery', + 'runId' => $runId, + ]; + } + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('workflow_instance_id', $scope->recoveryInstanceId()) + ->where('namespace', $scope->namespace) + ->orderByDesc('run_number') + ->first(); + } + if (! $run instanceof WorkflowRun || $run->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class)) { + throw new LogicException('The canonical recovery identity is unavailable or reserved by another type.'); + } + if (in_array($run->status, [RunStatus::Pending, RunStatus::Running, RunStatus::Waiting], true)) { + return [ + 'route' => 'recovery', + 'runId' => $run->id, + ]; + } + if ($this->repository->inspect($scope)['status'] === 'verified') { + return [ + 'route' => 'verified', + 'runId' => $run->id, + ]; + } + if ($run->status !== RunStatus::Failed) { + throw new LogicException('The recovery is terminal without verified compensation.'); + } + $redrive = $this->runtime->redrive($scope->recoveryInstanceId(), $run->id, [ + 'namespace' => $scope->namespace, + 'request_id' => hash('sha256', PortableDescriptor::encode([$scope->id(), $requestKey, $run->id])), + ]); + if (($redrive['accepted'] ?? false) !== true) { + throw new LogicException( + 'Native DW redrive refused unresolved recovery: ' . ($redrive['reason'] ?? 'unknown') . '.' + ); + } + return [ + 'route' => 'recovery', + 'runId' => $redrive['workflow_run_id'], + ]; + } +} diff --git a/src/V2/Compensation/Recovery/ScopeRetention.php b/src/V2/Compensation/Recovery/ScopeRetention.php new file mode 100644 index 000000000..8715b9698 --- /dev/null +++ b/src/V2/Compensation/Recovery/ScopeRetention.php @@ -0,0 +1,94 @@ +authorization->assertAllowed($scope, $principal, 'release'); + PortableDescriptor::identifier($policy, 'authorization policy version'); + $snapshot = $this->repository->inspect($scope); + SqliteTransaction::run(static function () use ($scope, $principal, $policy, $snapshot): void { + $origin = ConfiguredV2Models::query('run_model', WorkflowRun::class)->whereKey( + $snapshot['original_run_id'] + )->lockForUpdate() + ->first(); + $recovery = ConfiguredV2Models::query('run_model', WorkflowRun::class)->where( + 'workflow_instance_id', + $scope->recoveryInstanceId() + ) + ->orderByDesc('run_number') + ->lockForUpdate() + ->first(); + if (! $origin instanceof WorkflowRun || $origin->namespace !== $scope->namespace || ! $origin->status->isTerminal() + || ($recovery instanceof WorkflowRun && ! $recovery->status->isTerminal())) { + throw new LogicException('Native execution still retains this compensation scope.'); + } + $database = ConfiguredV2Models::connection(); + $link = $database->table('workflow_compensation_children') + ->where('child_id', $scope->id()) + ->first(); + if ($link !== null) { + $parent = $database->table(ScopeStorage::SCOPES)->where('id', $link->parent_id)->lockForUpdate() + ->first(); + if ($parent === null || ! in_array($parent->status, ['verified', 'released'], true)) { + throw new LogicException('An unresolved parent retains this child scope.'); + } + } + ScopeStorage::mutate($scope, static function (array $locked) use ( + $scope, + $principal, + $policy, + $database + ): void { + if ($locked['status'] === 'released') { + return; + } + if ($locked['status'] !== 'verified') { + throw new LogicException('Unresolved compensation cannot be released.'); + } + foreach ($database->table(ScopeStorage::OBLIGATIONS)->where('scope_id', $scope->id())->get() as $row) { + $descriptor = json_decode($row->descriptor, true, flags: JSON_THROW_ON_ERROR); + unset($descriptor['input']); + $descriptor['released'] = true; + $database->table(ScopeStorage::OBLIGATIONS)->where('id', $row->id)->update([ + 'descriptor' => PortableDescriptor::encode($descriptor), + 'forward_result' => null, + ]); + } + $database->table(ScopeStorage::SCOPES)->where('id', $scope->id())->update([ + 'status' => 'released', + ]); + ScopeStorage::append($locked, 'released', [ + 'principal' => $principal, + 'policyVersion' => $policy, + 'reason' => 'explicit-release', + ]); + }); + }, connection: ConfiguredV2Models::connection()); + } +} diff --git a/src/V2/Compensation/Security/ScopeAuthorization.php b/src/V2/Compensation/Security/ScopeAuthorization.php new file mode 100644 index 000000000..31c0398e8 --- /dev/null +++ b/src/V2/Compensation/Security/ScopeAuthorization.php @@ -0,0 +1,20 @@ +taskId === null || $activity->attemptId() === null || $activity->run->namespace !== $scope->namespace) { + throw new NonRetryableException('scope_native_identity_mismatch'); + } + return SqliteTransaction::forTask($activity->taskId, function () use ( + $activity, + $scope, + $operation, + $principal, + $body + ): mixed { + $this->assertNativeAuthority($activity); + $snapshot = null; + if ($operation !== 'register') { + $snapshot = $this->repository->metadata($scope); + $origin = $snapshot['original_run_id'] === $activity->runId(); + $recovery = $activity->workflowId() === $scope->recoveryInstanceId() + && $activity->run->workflow_class === RecoveryWorkflow::class; + if ((! $origin && ! $recovery) || ($operation === 'forward' && ! $origin)) { + throw new NonRetryableException('scope_execution_owner_mismatch'); + } + if ($principal === null) { + $principal = $this->repository->primaryRequest($scope)['principal']; + } + } + if ($principal === null) { + throw new NonRetryableException('scope_principal_missing'); + } + try { + $policyVersion = $this->authorization->assertAllowed($scope, $principal, $operation); + PortableDescriptor::identifier($policyVersion, 'authorization policy version'); + } catch (\Throwable) { + throw new NonRetryableException('scope_authorization_refused'); + } + $snapshot = $operation === 'register' ? null : $this->repository->inspect($scope); + $result = $body($snapshot, $policyVersion); + // The callback can outlive its grant even while its SQL rows remain locked. + $this->assertNativeAuthority($activity); + ScopeStorage::append(ScopeStorage::locked($scope), 'authorized', [ + 'principal' => $principal, + 'operation' => $operation, + 'policyVersion' => $policyVersion, + 'runId' => $activity->runId(), + 'activityId' => $activity->activityId(), + 'attemptId' => $activity->attemptId(), + ]); + return $result; + }); + } + + private function assertNativeAuthority(Activity $activity): void + { + $status = app(DefaultActivityTaskBridge::class)->status($activity->attemptId()); + if (($status['can_continue'] ?? false) !== true + || ($status['workflow_run_id'] ?? null) !== $activity->runId() + || ($status['workflow_task_id'] ?? null) !== $activity->taskId + || ($status['activity_execution_id'] ?? null) !== $activity->activityId()) { + throw new NonRetryableException('scope_native_attempt_unavailable'); + } + } +} diff --git a/src/migrations/2026_11_11_000100_create_compensation_scopes.php b/src/migrations/2026_11_11_000100_create_compensation_scopes.php new file mode 100644 index 000000000..373f1d7aa --- /dev/null +++ b/src/migrations/2026_11_11_000100_create_compensation_scopes.php @@ -0,0 +1,91 @@ +string('id', 64) + ->primary(); + $table->text('identity'); + $table->string('original_run_id', 128) + ->index(); + $table->string('status', 32); + $table->unsignedInteger('obligation_count') + ->default(0); + $table->unsignedInteger('request_count') + ->default(0); + $table->unsignedInteger('event_sequence') + ->default(0); + }); + Schema::create('workflow_compensation_obligations', static function (Blueprint $table): void { + $table->string('id', 64) + ->primary(); + $table->string('scope_id', 64) + ->index(); + $table->unsignedInteger('position'); + $table->string('fingerprint', 64); + $table->text('descriptor'); + $table->string('forward_status', 32); + $table->string('compensation_status', 32); + $table->string('forward_activity_id', 26) + ->nullable(); + $table->string('forward_attempt_id', 26) + ->nullable(); + $table->string('compensation_attempt_id', 26) + ->nullable(); + $table->text('forward_result') + ->nullable(); + $table->text('evidence') + ->nullable(); + $table->unique(['scope_id', 'position'], 'compensation_obligation_position'); + }); + Schema::create('workflow_compensation_graphs', static function (Blueprint $table): void { + $table->string('id', 64) + ->primary(); + }); + Schema::create('workflow_compensation_children', static function (Blueprint $table): void { + $table->string('child_id', 64) + ->primary(); + $table->string('parent_id', 64) + ->index(); + $table->string('obligation_id', 64) + ->unique(); + }); + Schema::create('workflow_compensation_requests', static function (Blueprint $table): void { + $table->string('id', 64) + ->primary(); + $table->string('scope_id', 64) + ->index(); + $table->unsignedInteger('position'); + $table->string('fingerprint', 64); + $table->text('descriptor'); + $table->string('authorization_policy_version', 128) + ->nullable(); + $table->unique(['scope_id', 'position'], 'compensation_request_position'); + }); + Schema::create('workflow_compensation_journal', static function (Blueprint $table): void { + $table->string('scope_id', 64); + $table->unsignedInteger('sequence'); + $table->string('kind', 32); + $table->text('data'); + $table->timestamp('recorded_at', 6); + $table->primary(['scope_id', 'sequence'], 'compensation_journal_sequence'); + }); + } + + public function down(): void + { + Schema::dropIfExists('workflow_compensation_journal'); + Schema::dropIfExists('workflow_compensation_obligations'); + Schema::dropIfExists('workflow_compensation_requests'); + Schema::dropIfExists('workflow_compensation_children'); + Schema::dropIfExists('workflow_compensation_graphs'); + Schema::dropIfExists('workflow_compensation_scopes'); + } +}; diff --git a/tests/Feature/V2/V2CompensationForwardAdmissionTest.php b/tests/Feature/V2/V2CompensationForwardAdmissionTest.php new file mode 100644 index 000000000..d9d2e4c28 --- /dev/null +++ b/tests/Feature/V2/V2CompensationForwardAdmissionTest.php @@ -0,0 +1,236 @@ + 'sync', + 'queue.connections.sync.driver' => 'sync', + ]); + $this->registry($this->handler()); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { + throw new \LogicException('Scope access refused.'); + } + return 'test-policy-1'; + } + }); + } + + protected function tearDown(): void + { + \Illuminate\Support\Carbon::setTestNow(); + parent::tearDown(); + } + + public function testMissingPinnedForwardHandlerIsNonRetryableBeforeDispatch(): void + { + $this->replaceAfterRegistration(new HandlerRegistry([])); + [$scope, $workflow] = $this->origin(); + $this->assertRefusedBeforeDispatch($scope, $workflow, 'scope_handler_unavailable'); + } + + public function testIncompatibleForwardInputIsNonRetryableBeforeDispatch(): void + { + $this->replaceAfterRegistration(new HandlerRegistry([ + 'test.operation' => [ + '1' => $this->handler(false), + ], + ])); + [$scope, $workflow] = $this->origin(); + $this->assertRefusedBeforeDispatch($scope, $workflow, 'scope_input_incompatible'); + } + + public function testForwardRetryRefusalPreservesEarlierUnknownEffect(): void + { + [$scope, $workflow] = $this->origin('lost-ack'); + $before = (new ScopeRepository())->inspect($scope)['obligations'][0]; + $this->assertSame('unknown', $before['forward_status']); + $this->assertSame(1, (int) Cache::get('test:scope:provider-call-count')); + $this->registry(null); + $this->driveNativeRetries($workflow->runId()); + $this->assertForwardFailure($workflow, 'scope_handler_unavailable', 2); + $after = (new ScopeRepository())->inspect($scope)['obligations'][0]; + $this->assertSame('unknown', $after['forward_status']); + $this->assertSame($before['forward_attempt_id'], $after['forward_attempt_id']); + $this->assertSame($before['forward_result'], $after['forward_result']); + $this->assertSame('pending', $after['compensation_status']); + $this->assertSame(1, (int) Cache::get('test:scope:provider-call-count')); + $this->registry($this->handler()); + app(ScopeRecovery::class)->request($scope, 'recover', 'operator-a', 'cancel'); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(1, (int) Cache::get('test:scope:compensation-call-count')); + } + + public function testNonRetryableProviderExceptionAfterDispatchRemainsUnknownAndRetryable(): void + { + $this->registry($this->handler(throwAfterEffect: true)); + [$scope, $workflow] = $this->origin(); + $this->driveNativeRetries($workflow->runId()); + $this->assertForwardFailure($workflow, 'scope_forward_unknown', 3, false); + $row = (new ScopeRepository())->inspect($scope)['obligations'][0]; + $this->assertSame('unknown', $row['forward_status']); + $this->assertSame( + 'provider-outcome-unknown', + json_decode($row['forward_result'], true, flags: JSON_THROW_ON_ERROR)['reason'] + ); + $this->assertSame('pending', $row['compensation_status']); + $this->assertSame(3, (int) Cache::get('test:scope:provider-call-count')); + $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); + } + + private function assertRefusedBeforeDispatch(ScopeIdentity $scope, WorkflowStub $workflow, string $reason): void + { + $this->assertForwardFailure($workflow, $reason, 1); + $snapshot = (new ScopeRepository())->inspect($scope); + $row = $snapshot['obligations'][0]; + $this->assertSame('open', $snapshot['status']); + $this->assertSame('registered', $row['forward_status']); + $this->assertNull($row['forward_activity_id']); + $this->assertNull($row['forward_attempt_id']); + $this->assertNull($row['forward_result']); + $this->assertSame('pending', $row['compensation_status']); + $this->assertNull(Cache::get('test:scope:provider-call-count')); + $this->assertNull(Cache::get('test:scope:effect-count')); + $this->registry($this->handler()); + app(ScopeRecovery::class)->request($scope, 'recover', 'operator-a', 'cancel'); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertNull(Cache::get('test:scope:compensation-call-count')); + } + + private function assertForwardFailure( + WorkflowStub $workflow, + string $reason, + int $attempts, + bool $nonRetryable = true + ): void { + $this->assertSame('failed', $workflow->refresh()->status()); + $execution = ActivityExecution::query()->where('workflow_run_id', $workflow->runId()) + ->where('activity_class', ForwardOperationActivity::class)->firstOrFail(); + $this->assertSame($attempts, $execution->attempts()->count()); + $failure = WorkflowHistoryEvent::query()->where('workflow_run_id', $workflow->runId()) + ->where('event_type', HistoryEventType::ActivityFailed)->firstOrFail(); + $this->assertSame($reason, $failure->payload['exception']['message']); + $this->assertSame($nonRetryable, $failure->payload['non_retryable']); + } + + private function replaceAfterRegistration(HandlerRegistry $replacement): void + { + $replaced = false; + ConfiguredV2Models::connection()->listen(function (QueryExecuted $query) use ($replacement, &$replaced): void { + if (! $replaced && str_starts_with($query->sql, 'insert') + && str_contains($query->sql, 'workflow_compensation_obligations')) { + $replaced = true; + $this->app->instance(HandlerRegistry::class, $replacement); + } + }); + } + + private function registry(?ScopedOperation $handler): void + { + $this->app->instance(HandlerRegistry::class, new HandlerRegistry( + $handler === null ? [] : [ + 'test.operation' => [ + '1' => $handler, + ], + ] + )); + } + + private function handler(bool $compatible = true, bool $throwAfterEffect = false): ScopedOperation + { + return new class($compatible, $throwAfterEffect) implements ScopedOperation { + public function __construct( + private bool $compatible, + private bool $throwAfterEffect + ) { + } + + public function assertInput(array $input): void + { + if (! $this->compatible) { + throw new \InvalidArgumentException('The input schema changed after registration.'); + } + } + + public function execute(OperationContext $context): ForwardResult + { + Cache::increment('test:scope:provider-call-count'); + $result = (new TestScopedOperation())->execute($context); + if ($this->throwAfterEffect) { + throw new NonRetryableException('provider-acknowledgement-lost'); + } + return $result; + } + + public function compensate(OperationContext $context): Resolution + { + Cache::increment('test:scope:compensation-call-count'); + return (new TestScopedOperation())->compensate($context); + } + }; + } + + private function origin(string $mode = 'complete'): array + { + WorkflowStub::fake(); + $scope = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + $workflow = WorkflowStub::make(TestForwardAdmissionWorkflow::class, 'forward-admission-origin', 'client-a'); + $workflow->start($scope->toArray(), $mode); + return [$scope, $workflow]; + } + + private function driveNativeRetries(string $runId): void + { + for ($step = 0; $step < 20; $step++) { + if (WorkflowRun::query()->findOrFail($runId)->status->isTerminal()) { + return; + } + $task = WorkflowTask::query()->where('workflow_run_id', $runId)->where('status', 'ready') + ->orderBy('available_at') + ->first(); + $this->assertNotNull($task, 'A nonterminal forward run must retain a native retry task.'); + if ($task->available_at !== null && $task->available_at->isFuture()) { + \Illuminate\Support\Carbon::setTestNow($task->available_at->copy()->addMicrosecond()); + } + if ($task->task_type->value === 'activity') { + (new \Workflow\V2\Jobs\RunActivityTask($task->id))->handle(); + } else { + (new \Workflow\V2\Jobs\RunWorkflowTask($task->id))->handle(); + } + } + $this->fail('Native forward retries did not terminate.'); + } +} diff --git a/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php b/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php new file mode 100644 index 000000000..653cd05f7 --- /dev/null +++ b/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php @@ -0,0 +1,172 @@ +getDriverName(), ['mysql', 'mariadb'], true) + || ! function_exists('pcntl_fork')) { + $this->markTestSkipped('Separate processes and an InnoDB backend are required.'); + } + self::stopWorkers(); + Queue::fake(); + } + + public static function sharedRows(): array + { + return [ + 'namespace graph' => ['graph'], + 'scope adoption' => ['scope'], + ]; + } + + #[DataProvider('sharedRows')] + public function testConcurrentAdoptersDoNotUpgradeSharedDuplicateKeyLocks(string $kind): void + { + if ($kind === 'graph') { + (new ChildScopes())->register($this->identity('seed'), 'seed-run', 'child', $this->identity('seed-child')); + $operation = fn (int $index): string => (new ChildScopes())->register( + $this->identity('parent-' . $index), + 'run-' . $index, + 'child', + $this->identity('child-' . $index), + ); + $table = 'workflow_compensation_graphs'; + } else { + $scope = $this->identity('shared'); + (new ScopeRepository())->open($scope, 'original-run'); + $operation = static function () use ($scope): string { + (new ScopeRepository())->open($scope, 'original-run'); + return $scope->id(); + }; + $table = 'workflow_compensation_scopes'; + } + + $results = $this->raceAtExclusiveLookup($table, $operation); + $this->assertCount(2, $results); + if ($kind === 'graph') { + $this->assertNotSame($results[0], $results[1]); + $this->assertSame(1, DB::table($table)->count()); + $this->assertSame(3, DB::table('workflow_compensation_children')->count()); + } else { + $this->assertSame($results[0], $results[1]); + $this->assertSame(1, DB::table($table)->count()); + $this->assertSame(1, DB::table('workflow_compensation_journal')->where('kind', 'opened')->count()); + } + } + + /** + * Both callers reach the first exclusive lookup before either executes it. + * The predecessor has already acquired duplicate-key shared locks here; + * the corrected path reaches this barrier before any INSERT IGNORE. + * + * @return list + */ + private function raceAtExclusiveLookup(string $table, Closure $operation): array + { + $root = sys_get_temp_dir() . '/dw-innodb-scope-' . bin2hex(random_bytes(8)); + mkdir($root, 0700); + DB::purge(); + $children = []; + $results = []; + try { + for ($index = 0; $index < 2; ++$index) { + $pid = pcntl_fork(); + if ($pid === -1) { + throw new RuntimeException('Cannot fork the scope contention control.'); + } + if ($pid === 0) { + try { + DB::reconnect(); + DB::statement('SET SESSION innodb_lock_wait_timeout = 10'); + $waiting = false; + DB::connection()->beforeExecuting(static function (string $sql) use ( + $table, + $root, + $index, + &$waiting, + ): void { + $sql = strtolower($sql); + if (! $waiting && str_starts_with($sql, 'select') + && str_contains($sql, $table) && str_contains($sql, 'for update')) { + $waiting = true; + touch($root . '/ready-' . $index); + self::waitFile($root . '/release'); + } + }); + DB::beginTransaction(); + $result = $operation($index); + DB::commit(); + file_put_contents($root . '/' . $index . '.json', json_encode([ + 'result' => $result, + ], JSON_THROW_ON_ERROR)); + DB::disconnect(); + exit(0); + } catch (Throwable $error) { + file_put_contents($root . '/' . $index . '.json', json_encode([ + 'error' => $error::class . ': ' . $error->getMessage(), + ], JSON_THROW_ON_ERROR)); + DB::disconnect(); + exit(1); + } + } + $children[] = $pid; + } + self::waitFile($root . '/ready-0'); + self::waitFile($root . '/ready-1'); + touch($root . '/release'); + foreach ($children as $index => $pid) { + pcntl_waitpid($pid, $status); + $payload = json_decode(file_get_contents($root . '/' . $index . '.json'), true, flags: JSON_THROW_ON_ERROR); + $this->assertSame(0, pcntl_wexitstatus($status), $payload['error'] ?? 'Scope caller failed.'); + $results[] = $payload['result']; + } + DB::reconnect(); + return $results; + } finally { + touch($root . '/release'); + foreach ($children as $pid) { + if (pcntl_waitpid($pid, $status, WNOHANG) === 0) { + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status); + } + } + foreach (glob($root . '/*') as $file) { + unlink($file); + } + rmdir($root); + } + } + + private function identity(string $key): ScopeIdentity + { + return new ScopeIdentity('shop', 'client-a', $key, '1'); + } + + private static function waitFile(string $path): void + { + $deadline = microtime(true) + 10; + while (! is_file($path)) { + if (microtime(true) >= $deadline) { + throw new RuntimeException('Scope contention barrier timed out.'); + } + usleep(10_000); + } + } +} diff --git a/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php b/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php new file mode 100644 index 000000000..29e38524a --- /dev/null +++ b/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php @@ -0,0 +1,236 @@ +getDriverName() !== 'pgsql' || ! function_exists('pcntl_fork')) { + $this->markTestSkipped('Separate processes and PostgreSQL row-lock observation are required.'); + } + self::stopWorkers(); + Queue::fake(); + config([ + 'workflows.v2.task_dispatch_mode' => 'poll', + ]); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { + throw new RuntimeException('Scope access refused.'); + } + return 'test-policy-1'; + } + }); + } + + public function testConcurrentRequestsStartOneCanonicalRecovery(): void + { + $scope = $this->openOrigin(); + $results = $this->race( + static fn (int $index): array => app( + ScopeRecovery::class + )->request($scope, 'request-' . $index, 'operator-a', 'cancel') + ); + $this->assertSame($results[0]['runId'], $results[1]['runId']); + $this->assertSame( + 1, + WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count() + ); + $this->assertSame( + 2, + ConfiguredV2Models::connection()->table('workflow_compensation_requests')->where( + 'scope_id', + $scope->id() + )->count() + ); + } + + public function testRequestSerializesWithRegistrationAndClosesNewAdmission(): void + { + $scope = $this->openOrigin(); + $this->race(static function (int $index) use ($scope): array { + if ($index === 0) { + return [ + 'id' => (new ScopeRepository())->register( + $scope, + new Obligation('book', 'carrier.cancel', '1', []) + ), + ]; + } + return app(ScopeRecovery::class)->request($scope, 'request', 'operator-a', 'cancel'); + }); + $this->assertCount(1, (new ScopeRepository())->inspect($scope)['obligations']); + $this->expectException(\LogicException::class); + (new ScopeRepository())->register($scope, new Obligation('later', 'carrier.cancel', '1', [])); + } + + public function testCompetingEdgesCannotCreateACycle(): void + { + $a = new ScopeIdentity('shop', 'client-a', 'a', '1'); + $b = new ScopeIdentity('shop', 'client-a', 'b', '1'); + $results = $this->race(static function (int $index) use ($a, $b): array { + try { + $id = (new ChildScopes())->register( + $index === 0 ? $a : $b, + $index === 0 ? 'run-a' : 'run-b', + 'child', + $index === 0 ? $b : $a + ); + return [ + 'accepted' => true, + 'id' => $id, + ]; + } catch (\LogicException) { + return [ + 'accepted' => false, + ]; + } + }); + $this->assertTrue($results[0]['accepted']); + $this->assertFalse($results[1]['accepted']); + $this->assertSame(1, DB::table('workflow_compensation_children')->count()); + } + + private function openOrigin(): ScopeIdentity + { + $scope = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + $origin = WorkflowStub::make(TestSignalWorkflow::class, 'origin', 'client-a'); + $origin->start(); + WorkflowRun::query()->findOrFail($origin->runId())->forceFill([ + 'status' => 'completed', + 'closed_at' => now(), + ])->save(); + (new ScopeRepository())->open($scope, $origin->runId()); + return $scope; + } + + private function race(Closure $operation, bool $expectLock = true): array + { + $root = sys_get_temp_dir() . '/dw-conflict-' . bin2hex(random_bytes(6)); + mkdir($root); + DB::purge(); + $children = []; + $results = []; + try { + for ($index = 0; $index < 2; $index++) { + $pid = pcntl_fork(); + if ($pid === -1) { + throw new RuntimeException('Cannot fork the conflict probe.'); + } + if ($pid === 0) { + try { + DB::reconnect(); + DB::statement("SET application_name = 'dw-conflict-probe-{$index}'"); + DB::statement("SET lock_timeout = '15s'"); + if ($index === 0) { + DB::beginTransaction(); + $result = $operation($index); + touch($root . '/holding'); + self::waitFile($root . '/release'); + DB::commit(); + } else { + self::waitFile($root . '/holding'); + $result = $operation($index); + } + file_put_contents($root . '/' . $index . '.json', json_encode([ + 'result' => $result, + ], JSON_THROW_ON_ERROR)); + DB::disconnect(); + exit(0); + } catch (Throwable $error) { + file_put_contents($root . '/' . $index . '.json', json_encode([ + 'error' => $error::class . ': ' . $error->getMessage(), + ])); + exit(1); + } + } + $children[] = $pid; + } + DB::reconnect(); + self::waitFile($root . '/holding'); + if ($expectLock) { + $deadline = microtime(true) + 10; + do { + $state = DB::selectOne( + "SELECT wait_event_type FROM pg_stat_activity WHERE application_name = 'dw-conflict-probe-1'" + ); + if (($state->wait_event_type ?? null) === 'Lock') { + break; + } + if (is_file($root . '/1.json')) { + throw new RuntimeException( + 'Second caller finished without reaching the expected row lock: ' . file_get_contents( + $root . '/1.json' + ) + ); + } + usleep(10000); + } while (microtime(true) < $deadline); + self::assertSame( + 'Lock', + $state->wait_event_type ?? null, + 'The calls must overlap at the real PostgreSQL lock.' + ); + } else { + self::waitFile($root . '/1.json'); + } + touch($root . '/release'); + foreach ($children as $index => $pid) { + pcntl_waitpid($pid, $status); + $payload = json_decode( + file_get_contents($root . '/' . $index . '.json'), + true, + flags: JSON_THROW_ON_ERROR + ); + self::assertSame(0, pcntl_wexitstatus($status), $payload['error'] ?? 'Child failed.'); + $results[] = $payload['result']; + } + return $results; + } finally { + touch($root . '/release'); + foreach ($children as $pid) { + if (pcntl_waitpid($pid, $status, WNOHANG) === 0) { + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status); + } + } + foreach (glob($root . '/*') as $file) { + unlink($file); + } + rmdir($root); + } + } + + private static function waitFile(string $file): void + { + $deadline = microtime(true) + 10; + while (! is_file($file)) { + if (microtime(true) >= $deadline) { + throw new RuntimeException('Probe barrier timed out.'); + } + usleep(10000); + } + } +} diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php new file mode 100644 index 000000000..3e801809a --- /dev/null +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -0,0 +1,210 @@ + 'redis', + 'queue.connections.redis.driver' => 'redis', + ]); + Queue::fake(); + $this->app->instance(HandlerRegistry::class, new HandlerRegistry([ + 'test.operation' => [ + '1' => new TestScopedOperation(), + ], + ])); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { + throw new \LogicException('Scope access refused.'); + } + return 'boundary-policy-1'; + } + }); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testContinueAsNewHandsOffTheOpenChildWithoutTransferringScopeOwnership(): void + { + $scope = new ScopeIdentity('shop', 'client-a', 'continued-order', '1'); + $child = new ScopeIdentity('shop', 'client-a', 'continued-shipment', '1'); + $stub = WorkflowStub::make(TestScopeRunBoundaryWorkflow::class, 'scope-continue', 'client-a'); + $stub->start($scope->toArray(), 'continue', 0, $child->toArray()); + $this->drainTasks(); + $runs = WorkflowRun::query()->where('workflow_instance_id', 'scope-continue')->orderBy('run_number')->get(); + $this->assertCount(2, $runs); + [$origin, $continued] = [$runs[0], $runs[1]]; + $this->assertSame('continued', $origin->closed_reason); + $this->assertSame(RunStatus::Waiting, $continued->status); + $originHistory = $this->history($origin->id); + $this->assertSame(HistoryEventType::WorkflowContinuedAsNew->value, end($originHistory)['event_type']); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $origin->id)->where( + 'link_type', + 'child_workflow' + )->sole(); + $handoff = WorkflowChildSupervision::query()->where('supervising_workflow_run_id', $continued->id)->sole(); + $this->assertSame($link->id, $handoff->workflow_link_id); + $this->assertSame($origin->id, $handoff->parent_workflow_run_id); + $this->assertSame($link->child_workflow_run_id, $handoff->child_workflow_run_id); + $this->assertSame(RunStatus::Waiting, WorkflowRun::query()->findOrFail($link->child_workflow_run_id)->status); + $this->assertSame($origin->id, (new ScopeRepository())->inspect($scope)['original_run_id']); + $stub->signal('resume'); + $this->drainTasks(); + $this->assertSame(RunStatus::Failed, $continued->refresh()->status); + $this->assertForwardRefused($continued->id, $scope, 2); + $result = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->drainTasks(); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame('verified', (new ScopeRepository())->inspect($child)['status']); + $this->assertSame(RunStatus::Completed, WorkflowRun::query()->findOrFail($result['runId'])->status); + $this->assertSame( + ['continued-shipment/child', 'continued-order/first'], + Cache::get('test:scope:undo-scoped-log') + ); + $this->assertSame($originHistory, $this->history($origin->id)); + $effects = Cache::get('test:scope:effect-count'); + app(WorkflowReplayer::class)->replay($origin->fresh()); + $this->assertSame($effects, Cache::get('test:scope:effect-count')); + } + + public function testOriginRedriveRefusesNewForwardWorkAndRetainsTheOriginalRecovery(): void + { + $scope = new ScopeIdentity('shop', 'client-a', 'redriven-order', '1'); + $stub = WorkflowStub::make(TestScopeRunBoundaryWorkflow::class, 'scope-redrive', 'client-a'); + $stub->start($scope->toArray(), 'redrive'); + $this->drainTasks(); + $origin = WorkflowRun::query()->findOrFail($stub->runId()); + $this->assertSame(RunStatus::Failed, $origin->status); + $originHistory = $this->history($origin->id); + $result = app(WorkflowControlPlane::class)->redrive('scope-redrive', $origin->id, [ + 'namespace' => 'client-a', + 'request_id' => 'origin-redrive', + ]); + $this->assertTrue($result['accepted'], (string) $result['reason']); + $this->drainTasks(); + $successor = WorkflowRun::query()->findOrFail($result['workflow_run_id']); + $this->assertSame(RunStatus::Failed, $successor->status); + $this->assertForwardRefused($successor->id, $scope, 1); + $this->assertSame(2, (int) Cache::get('test:redrive:second-calls')); + $recovery = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->drainTasks(); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(RunStatus::Completed, WorkflowRun::query()->findOrFail($recovery['runId'])->status); + $this->assertSame(['redriven-order/first'], Cache::get('test:scope:undo-scoped-log')); + $this->assertSame($origin->id, (new ScopeRepository())->inspect($scope)['original_run_id']); + $this->assertSame($originHistory, $this->history($origin->id)); + } + + public function testOriginRedriveCannotReissueAnUncertainForwardEffectFromTheOriginalScope(): void + { + $scope = new ScopeIdentity('shop', 'client-a', 'unknown-redriven-order', '1'); + $stub = WorkflowStub::make(TestScopeRunBoundaryWorkflow::class, 'scope-unknown-redrive', 'client-a'); + $stub->start($scope->toArray(), 'unknown'); + $this->drainTasks(); + $origin = WorkflowRun::query()->findOrFail($stub->runId()); + $this->assertSame(RunStatus::Failed, $origin->status); + $originHistory = $this->history($origin->id); + $before = (new ScopeRepository())->inspect($scope); + $this->assertSame('unknown', $before['obligations'][0]['forward_status']); + $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); + $result = app(WorkflowControlPlane::class)->redrive('scope-unknown-redrive', $origin->id, [ + 'namespace' => 'client-a', + 'request_id' => 'origin-redrive', + ]); + $this->assertTrue($result['accepted'], (string) $result['reason']); + $this->drainTasks(); + $successor = WorkflowRun::query()->findOrFail($result['workflow_run_id']); + $this->assertSame(RunStatus::Failed, $successor->status); + $messages = WorkflowFailure::query()->where('workflow_run_id', $successor->id)->pluck('message')->all(); + $this->assertStringContainsString('scope_execution_owner_mismatch', implode('\n', $messages)); + $this->assertSame($before, (new ScopeRepository())->inspect($scope)); + $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); + $recovery = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->drainTasks(); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(RunStatus::Completed, WorkflowRun::query()->findOrFail($recovery['runId'])->status); + $this->assertSame(['unknown-redriven-order/first'], Cache::get('test:scope:undo-scoped-log')); + $this->assertSame($originHistory, $this->history($origin->id)); + } + + private function assertForwardRefused(string $runId, ScopeIdentity $scope, int $effectCount): void + { + $messages = WorkflowFailure::query()->where('workflow_run_id', $runId)->pluck('message')->all(); + $this->assertStringContainsString('cannot change its original run', implode('\n', $messages)); + $snapshot = (new ScopeRepository())->inspect($scope); + $this->assertSame('open', $snapshot['status']); + $this->assertNotContains('second', array_column(array_column($snapshot['obligations'], 'descriptor'), 'key')); + $this->assertSame($effectCount, (int) Cache::get('test:scope:effect-count')); + $this->assertNull(Cache::get('test:scope:undo-log')); + } + + private function history(string $runId): array + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->orderBy('sequence')->get() + ->map(static fn (WorkflowHistoryEvent $event): array => [ + 'sequence' => $event->sequence, + 'event_type' => $event->event_type->value, + 'payload' => $event->payload, + ])->all(); + } + + private function drainTasks(): void + { + for ($count = 0; $count < 300; ++$count) { + $task = WorkflowTask::query()->where('status', TaskStatus::Ready->value)->orderBy('created_at')->first(); + if ($task === null) { + return; + } + if ($task->available_at !== null && $task->available_at->isFuture()) { + Carbon::setTestNow($task->available_at->copy()->addSecond()); + } + $job = match ($task->task_type) { + TaskType::Workflow => new RunWorkflowTask($task->id), + TaskType::Activity => new RunActivityTask($task->id), + TaskType::Timer => new RunTimerTask($task->id), + }; + $this->app->call([$job, 'handle']); + } + $this->fail('Ready tasks did not settle within the bounded drain.'); + } +} diff --git a/tests/Feature/V2/V2CompensationScopeTest.php b/tests/Feature/V2/V2CompensationScopeTest.php new file mode 100644 index 000000000..a08833904 --- /dev/null +++ b/tests/Feature/V2/V2CompensationScopeTest.php @@ -0,0 +1,528 @@ + 'sync', + 'queue.connections.sync.driver' => 'sync', + ]); + $this->app->instance(HandlerRegistry::class, new HandlerRegistry([ + 'test.operation' => [ + '1' => new TestScopedOperation(), + ], + ])); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { + throw new \LogicException('Scope access refused.'); + } + return 'test-policy-1'; + } + }); + } + + protected function tearDown(): void + { + \Illuminate\Support\Carbon::setTestNow(); + parent::tearDown(); + } + + public function testClosedOriginUsesCanonicalRecoveryAndKeepsOriginalTerminalHistory(): void + { + [$scope, $origin] = $this->completedOrigin(); + $before = WorkflowHistoryEvent::query()->where('workflow_run_id', $origin->runId())->count(); + $result = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame('completed', WorkflowRun::query()->findOrFail($result['runId'])->status->value); + $this->assertSame('completed', $origin->refresh()->status()); + $this->assertSame($before, WorkflowHistoryEvent::query()->where('workflow_run_id', $origin->runId())->count()); + $this->assertSame(['second', 'first'], Cache::get('test:scope:undo-log')); + } + + public function testNativeSagaCompensatesAnEffectWhoseForwardResultWasLost(): void + { + WorkflowStub::fake(); + $scope = $this->identity(); + $workflow = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'lost-ack-origin', 'client-a'); + $workflow->start($scope->toArray(), 'lost-ack'); + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(['first'], Cache::get('test:scope:undo-log')); + $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); + } + + public function testRetryExhaustionStaysUnresolvedAndNativeRedriveReusesVerifiedWork(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:unknown-first', true); + $first = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->driveNativeRetries($first['runId']); + $this->assertSame('failed', WorkflowRun::query()->findOrFail($first['runId'])->status->value); + $this->assertSame('requested', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame( + 1, + count(array_filter(Cache::get('test:scope:undo-log'), static fn (string $key): bool => $key === 'second')) + ); + Cache::forget('test:scope:unknown-first'); + $again = app(ScopeRecovery::class)->request($scope, 'cancel-2', 'operator-a', 'order-cancelled'); + $this->assertNotSame($first['runId'], $again['runId']); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame( + 1, + count(array_filter(Cache::get('test:scope:undo-log'), static fn (string $key): bool => $key === 'second')) + ); + $this->assertSame( + 2, + WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count() + ); + } + + public function testNativeActiveAdoptionDoesNotPreventRecoveryRedriveAfterFailure(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:unknown-first', true); + $first = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'cancel'); + $adopted = WorkflowStub::make(RecoveryWorkflow::class, $scope->recoveryInstanceId(), $scope->namespace) + ->attemptStart(StartOptions::returnExistingActive(), $scope->toArray()); + $this->assertTrue($adopted->returnedExistingActive()); + $this->assertSame($first['runId'], $adopted->runId()); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId']) + ->where('event_type', \Workflow\V2\Enums\HistoryEventType::WorkflowStarted)->firstOrFail(); + $adoption = WorkflowHistoryEvent::query()->where('workflow_command_id', $adopted->commandId())->firstOrFail(); + $this->assertGreaterThan($started->sequence, $adoption->sequence); + $this->driveNativeRetries($first['runId']); + $this->assertSame('failed', WorkflowRun::query()->findOrFail($first['runId'])->status->value); + Cache::forget('test:scope:unknown-first'); + $redrive = app(ScopeRecovery::class)->request($scope, 'cancel-2', 'operator-a', 'cancel'); + $this->assertNotSame($first['runId'], $redrive['runId']); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(1, count(array_filter( + Cache::get('test:scope:undo-log'), + static fn (string $key): bool => $key === 'second' + ))); + } + + public function testOneRetainedRequestCanRedriveTwoFailuresWithoutRepeatingVerifiedWork(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:unknown-first', true); + $first = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->driveNativeRetries($first['runId']); + $second = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->assertNotSame($first['runId'], $second['runId']); + $duplicate = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->assertSame($second['runId'], $duplicate['runId']); + $this->driveNativeRetries($second['runId']); + $this->assertSame('failed', WorkflowRun::query()->findOrFail($second['runId'])->status->value); + Cache::forget('test:scope:unknown-first'); + $third = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->assertNotSame($second['runId'], $third['runId']); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame( + 3, + WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count() + ); + $this->assertSame(1, \Workflow\V2\Support\ConfiguredV2Models::connection() + ->table('workflow_compensation_requests') + ->where('scope_id', $scope->id()) + ->count()); + $this->assertSame(1, count(array_filter( + Cache::get('test:scope:undo-log'), + static fn (string $key): bool => $key === 'second' + ))); + } + + public function testForwardEvidenceSurvivesReleaseWithoutRetainingBusinessInputsOrResults(): void + { + [$scope] = $this->completedOrigin(); + $repository = new ScopeRepository(); + $observations = array_values(array_filter( + $repository->journal($scope), + static fn (array $row): bool => $row['kind'] === 'forward_outcome' + )); + $this->assertCount(2, $observations); + foreach ($observations as $row) { + $record = json_decode($row['data'], true, flags: JSON_THROW_ON_ERROR); + $this->assertArrayHasKey('evidence', $record); + $this->assertSame(['provider:' . $record['obligationId']], $record['evidence']); + $this->assertArrayNotHasKey('value', $record); + $this->assertArrayNotHasKey('input', $record); + } + app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + app(\Workflow\V2\Compensation\Recovery\ScopeRetention::class)->release($scope, 'operator-a'); + $retained = array_values(array_filter( + $repository->journal($scope), + static fn (array $row): bool => $row['kind'] === 'forward_outcome' + )); + $this->assertSame($observations, $retained); + foreach ($repository->inspect($scope)['obligations'] as $row) { + $this->assertNull($row['forward_result']); + $this->assertArrayNotHasKey('input', $row['descriptor']); + } + } + + public function testColdReplayDoesNotResolveMutableHandlersOrRepeatProviderEffects(): void + { + [$scope, $origin] = $this->completedOrigin(); + $effects = Cache::get('test:scope:effect-count'); + $this->app->instance(HandlerRegistry::class, new HandlerRegistry([])); + app(WorkflowReplayer::class)->replay(WorkflowRun::query()->findOrFail($origin->runId())); + $this->assertSame($effects, Cache::get('test:scope:effect-count')); + $this->assertNull(Cache::get('test:scope:undo-log')); + $this->assertSame('open', (new ScopeRepository())->inspect($scope)['status']); + } + + public function testUnavailablePinnedHandlerFailsBeforeCompensationEffect(): void + { + [$scope] = $this->completedOrigin(); + $this->app->instance(HandlerRegistry::class, new HandlerRegistry([])); + app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->assertSame('requested', (new ScopeRepository())->inspect($scope)['status']); + $this->assertNull(Cache::get('test:scope:undo-log')); + } + + public function testExpiredNativeAttemptCannotPublishVerifiedCompensation(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:expire-native', true); + app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $snapshot = (new ScopeRepository())->inspect($scope); + $this->assertSame('requested', $snapshot['status']); + $this->assertNotContains('verified', array_column($snapshot['obligations'], 'compensation_status')); + } + + public function testPossibleLateForwardEffectCannotBecomeVerifiedAfterRetryExhaustion(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:late-forward-possible', true); + $recovery = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->driveNativeRetries($recovery['runId']); + $snapshot = (new ScopeRepository())->inspect($scope); + $this->assertSame('requested', $snapshot['status']); + $this->assertNotContains('verified', array_column($snapshot['obligations'], 'compensation_status')); + $this->assertSame('failed', WorkflowRun::query()->findOrFail($recovery['runId'])->status->value); + } + + public function testRecordedAuthorizationIncludesNativeAttemptAndPolicyVersion(): void + { + [$scope] = $this->completedOrigin(); + app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $events = array_values( + array_filter((new ScopeRepository())->journal( + $scope + ), static fn (array $row): bool => $row['kind'] === 'authorized') + ); + $this->assertNotEmpty($events); + foreach ($events as $row) { + $record = json_decode($row['data'], true, flags: JSON_THROW_ON_ERROR); + $this->assertSame('test-policy-1', $record['policyVersion']); + $this->assertNotEmpty($record['attemptId']); + $this->assertNotEmpty($record['runId']); + } + } + + public function testCompletedChildScopeIsVerifiedBeforeEarlierParentCompensation(): void + { + WorkflowStub::fake(); + $child = new ScopeIdentity('shop', 'client-a', 'shipment-42', '1'); + $childOrigin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'child-origin', 'client-a'); + $childOrigin->start($child->toArray(), 'complete'); + $this->assertTrue($childOrigin->refresh()->completed()); + $parent = $this->identity(); + $parentOrigin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'parent-origin', 'client-a'); + $parentOrigin->start($parent->toArray(), 'with-child', $child->toArray()); + $this->assertTrue($parentOrigin->refresh()->completed()); + app(ScopeRecovery::class)->request($parent, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->assertSame('verified', (new ScopeRepository())->inspect($parent)['status']); + $this->assertSame('verified', (new ScopeRepository())->inspect($child)['status']); + $this->assertSame( + ['order-42/second', 'shipment-42/second', 'shipment-42/first', 'order-42/first'], + Cache::get('test:scope:undo-scoped-log') + ); + } + + public function testChildParticipationRecordsItsOwnAuthorizationPolicyVersion(): void + { + WorkflowStub::fake(); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + return $operation === 'participate' ? 'child-policy-7' : 'parent-policy-2'; + } + }); + $parent = $this->identity(); + $child = new ScopeIdentity('shop', 'client-a', 'shipment-42', '1'); + $origin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'parent-origin', 'client-a'); + $origin->start($parent->toArray(), 'with-child', $child->toArray()); + $this->assertTrue($origin->refresh()->completed()); + $records = array_values(array_filter( + (new ScopeRepository())->journal($parent), + static fn (array $row): bool => $row['kind'] === 'child_authorized' + )); + $this->assertCount(1, $records); + $record = json_decode($records[0]['data'], true, flags: JSON_THROW_ON_ERROR); + $this->assertSame('child-policy-7', $record['policyVersion']); + $this->assertSame($child->id(), $record['childScopeId']); + $this->assertSame('operator-a', $record['principal']); + $this->assertNotEmpty($record['attemptId']); + } + + public function testInvalidChildParticipationPolicyCannotRegisterAChild(): void + { + WorkflowStub::fake(); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + return $operation === 'participate' ? '' : 'parent-policy-2'; + } + }); + $parent = $this->identity(); + $child = new ScopeIdentity('shop', 'client-a', 'shipment-42', '1'); + $origin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'parent-origin', 'client-a'); + $origin->start($parent->toArray(), 'with-child', $child->toArray()); + $this->assertCount(1, (new ScopeRepository())->inspect($parent)['obligations']); + $this->assertSame(0, \Workflow\V2\Support\ConfiguredV2Models::connection() + ->table('workflow_compensation_children') + ->count()); + $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); + } + + public function testUnavailableChildCannotAdvanceEarlierParentCompensation(): void + { + WorkflowStub::fake(); + $parent = $this->identity(); + $child = new ScopeIdentity('shop', 'client-a', 'missing-shipment', '1'); + $origin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'parent-origin', 'client-a'); + $origin->start($parent->toArray(), 'with-child', $child->toArray()); + $this->assertTrue($origin->refresh()->completed()); + $result = app(ScopeRecovery::class)->request($parent, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->driveNativeRetries($result['runId']); + $this->assertSame('requested', (new ScopeRepository())->inspect($parent)['status']); + $this->assertSame(['second'], Cache::get('test:scope:undo-log')); + } + + public function testOrdinaryRunPruningPreservesRetainedScopeRecovery(): void + { + [$scope, $origin] = $this->completedOrigin(); + \Workflow\V2\Support\WorkflowRunRetentionCleanup::pruneRun($origin->runId()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $origin->runId())->count()); + app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(['second', 'first'], Cache::get('test:scope:undo-log')); + } + + public function testTamperedRetainedDescriptorIsRefusedBeforeProviderCall(): void + { + [$scope] = $this->completedOrigin(); + $row = (new ScopeRepository())->inspect($scope)['obligations'][1]; + $descriptor = $row['descriptor']; + $descriptor['input']['changed'] = true; + \Workflow\V2\Support\ConfiguredV2Models::connection()->table('workflow_compensation_obligations')->where( + 'id', + $row['id'] + )->update([ + 'descriptor' => json_encode($descriptor, JSON_THROW_ON_ERROR), + ]); + app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->assertSame('requested', (new ScopeRepository())->inspect($scope)['status']); + $this->assertNull(Cache::get('test:scope:undo-log')); + } + + public function testTamperedRegistrationCannotReachTheForwardProvider(): void + { + WorkflowStub::fake(); + $scope = $this->identity(); + $tampered = false; + \Workflow\V2\Support\ConfiguredV2Models::connection()->listen( + static function (QueryExecuted $query) use ($scope, &$tampered): void { + if ($tampered || ! str_starts_with($query->sql, 'insert') + || ! str_contains($query->sql, 'workflow_compensation_obligations')) { + return; + } + $tampered = true; + $row = $query->connection->table('workflow_compensation_obligations') + ->where('scope_id', $scope->id()) + ->first(); + $descriptor = json_decode($row->descriptor, true, flags: JSON_THROW_ON_ERROR); + $descriptor['input']['changed'] = true; + $query->connection->table('workflow_compensation_obligations') + ->where('id', $row->id) + ->update([ + 'descriptor' => json_encode($descriptor, JSON_THROW_ON_ERROR), + ]); + } + ); + $origin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'tampered-forward-origin', 'client-a'); + $origin->start($scope->toArray(), 'complete'); + $this->assertTrue($tampered); + $this->assertNull(Cache::get('test:scope:effect-count')); + $this->assertNotContains( + 'observed', + array_column((new ScopeRepository())->inspect($scope)['obligations'], 'forward_status') + ); + } + + public function testReleaseRefusesUnresolvedWorkThenRetainsAnUnreusableTombstone(): void + { + [$scope] = $this->completedOrigin(); + try { + app(\Workflow\V2\Compensation\Recovery\ScopeRetention::class)->release($scope, 'operator-a'); + $this->fail('Unresolved obligations were released.'); + } catch (\LogicException) { + $this->assertSame('open', (new ScopeRepository())->inspect($scope)['status']); + } + app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'order-cancelled'); + $before = (new ScopeRepository())->inspect($scope); + app(\Workflow\V2\Compensation\Recovery\ScopeRetention::class)->release($scope, 'operator-a'); + $after = (new ScopeRepository())->inspect($scope); + $this->assertSame('released', $after['status']); + $this->assertSame( + array_column($before['obligations'], 'fingerprint'), + array_column($after['obligations'], 'fingerprint') + ); + $this->assertArrayNotHasKey('input', $after['obligations'][0]['descriptor']); + $this->expectException(\LogicException::class); + app(ScopeRecovery::class)->request($scope, 'cancel-2', 'operator-a', 'order-cancelled'); + } + + public function testDefaultEmbeddedNamespaceIsPreservedWithoutInventingAnAlias(): void + { + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== null || $principal !== 'operator-a') { + throw new \LogicException('Default namespace access refused.'); + } + return 'default-policy-1'; + } + }); + WorkflowStub::fake(); + $scope = new ScopeIdentity('shop', null, 'order-42', '1'); + $origin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'default-origin'); + $origin->start($scope->toArray(), 'complete'); + $this->assertTrue($origin->refresh()->completed()); + $result = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'cancel'); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertNull(WorkflowRun::query()->findOrFail($result['runId'])->namespace); + } + + public function testRetainedRecoveryRunsInAFreshPhpProcess(): void + { + [$scope] = $this->completedOrigin(); + $input = tempnam(sys_get_temp_dir(), 'scope-recovery-input-'); + $report = tempnam(sys_get_temp_dir(), 'scope-recovery-report-'); + chmod($input, 0600); + try { + file_put_contents($input, json_encode([ + 'scope' => $scope->toArray(), + 'database' => config('database.connections.' . config('database.default')), + 'redis' => config('database.redis'), + 'cache' => config('cache.stores.redis'), + 'cache_prefix' => config('cache.prefix'), + 'report' => $report, + ], JSON_THROW_ON_ERROR)); + $process = new \Symfony\Component\Process\Process([ + PHP_BINARY, 'vendor/bin/phpunit', 'tests/Support/CompensationScopeRecoveryProcessTest.php', + ], dirname(__DIR__, 3), [ + 'DWF_SCOPE_RECOVERY_INPUT' => $input, + ], timeout: 30); + $process->mustRun(); + $result = json_decode(file_get_contents($report), true, flags: JSON_THROW_ON_ERROR); + $this->assertNotSame(getmypid(), $result['pid']); + $this->assertSame('verified', $result['status']); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(['second', 'first'], Cache::get('test:scope:undo-log')); + } finally { + unlink($input); + unlink($report); + } + } + + public function testCancellationWhileChildWaitsResolvesItsSagaBeforeEarlierParentWork(): void + { + WorkflowStub::fake(); + $child = new ScopeIdentity('shop', 'client-a', 'shipment-42', '1'); + $childOrigin = WorkflowStub::make( + \Tests\Fixtures\V2\TestWaitingScopeWorkflow::class, + 'waiting-child', + 'client-a' + ); + $childOrigin->start($child->toArray()); + $this->assertSame('waiting', $childOrigin->refresh()->status()); + $parent = $this->identity(); + $parentOrigin = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'parent-origin', 'client-a'); + $parentOrigin->start($parent->toArray(), 'with-child', $child->toArray()); + app(ScopeRecovery::class)->request($parent, 'cancel-1', 'operator-a', 'order-cancelled'); + $this->assertSame('verified', (new ScopeRepository())->inspect($child)['status']); + $this->assertSame('verified', (new ScopeRepository())->inspect($parent)['status']); + $this->assertSame( + ['order-42/second', 'shipment-42/second', 'shipment-42/first', 'order-42/first'], + Cache::get('test:scope:undo-scoped-log') + ); + } + + private function driveNativeRetries(string $runId): void + { + for ($step = 0; $step < 20; $step++) { + $run = WorkflowRun::query()->findOrFail($runId); + if ($run->status->isTerminal()) { + return; + } + $task = \Workflow\V2\Models\WorkflowTask::query()->where('workflow_run_id', $runId)->where( + 'status', + 'ready' + )->orderBy('available_at') + ->first(); + $this->assertNotNull($task, 'A nonterminal recovery must retain a native retry task.'); + if ($task->available_at !== null && $task->available_at->isFuture()) { + \Illuminate\Support\Carbon::setTestNow($task->available_at->copy()->addMicrosecond()); + } + if ($task->task_type->value === 'activity') { + (new \Workflow\V2\Jobs\RunActivityTask($task->id))->handle(); + } else { + (new \Workflow\V2\Jobs\RunWorkflowTask($task->id))->handle(); + } + } + $this->fail('Native retry budget did not terminate.'); + } + + private function completedOrigin(): array + { + WorkflowStub::fake(); + $scope = $this->identity(); + $workflow = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'retained-origin', 'client-a'); + $workflow->start($scope->toArray(), 'complete'); + $this->assertTrue($workflow->refresh()->completed()); + return [$scope, $workflow]; + } + + private function identity(): ScopeIdentity + { + return new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + } +} diff --git a/tests/Fixtures/V2/TestForwardAdmissionWorkflow.php b/tests/Fixtures/V2/TestForwardAdmissionWorkflow.php new file mode 100644 index 000000000..f371be17a --- /dev/null +++ b/tests/Fixtures/V2/TestForwardAdmissionWorkflow.php @@ -0,0 +1,21 @@ +execute(new Obligation('first', 'test.operation', '1', [ + 'mode' => $mode, + ])); + } +} diff --git a/tests/Fixtures/V2/TestRetainedScopeWorkflow.php b/tests/Fixtures/V2/TestRetainedScopeWorkflow.php new file mode 100644 index 000000000..4a86b4ab2 --- /dev/null +++ b/tests/Fixtures/V2/TestRetainedScopeWorkflow.php @@ -0,0 +1,35 @@ +execute(new Obligation('first', 'test.operation', '1', [ + 'mode' => $mode, + ]), new ActivityOptions(maxAttempts: 1)); + if ($child !== null) { + $scope->participate('delivery', ScopeIdentity::fromArray($child)); + } + $scope->execute(new Obligation('second', 'test.operation', '1', [])); + return [ + 'status' => 'done', + ]; + } catch (Throwable) { + $this->compensate(); + return $scope->verifyCompensation(); + } + } +} diff --git a/tests/Fixtures/V2/TestScopeBoundaryChildWorkflow.php b/tests/Fixtures/V2/TestScopeBoundaryChildWorkflow.php new file mode 100644 index 000000000..22da1a69e --- /dev/null +++ b/tests/Fixtures/V2/TestScopeBoundaryChildWorkflow.php @@ -0,0 +1,32 @@ +execute(new Obligation('child', 'test.operation', '1', [])); + signal('finish'); + return [ + 'status' => 'done', + ]; + } catch (Throwable) { + $this->compensate(); + return $scope->verifyCompensation(); + } + } +} diff --git a/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php b/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php new file mode 100644 index 000000000..a74b22812 --- /dev/null +++ b/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php @@ -0,0 +1,51 @@ +execute(new Obligation('first', 'test.operation', '1', [ + 'mode' => $mode === 'unknown' ? 'lost-ack' : 'complete', + ]), new ActivityOptions(maxAttempts: 1)); + if ($mode === 'continue') { + if ($childIdentity !== null) { + $scope->participate('delivery', ScopeIdentity::fromArray($childIdentity)); + self::select([ + 'child' => static fn () => self::child( + TestScopeBoundaryChildWorkflow::class, + new ChildWorkflowOptions(parentClosePolicy: ParentClosePolicy::Abandon), + $childIdentity + ), + 'ready' => static fn () => self::timer(0), + ]); + } + return self::continueAsNew($identity, $mode, 1, $childIdentity); + } + self::activity(TestRedriveSecondActivity::class, new ActivityOptions(maxAttempts: 1), 'boundary'); + } else { + signal('resume'); + } + // This is new work, so redrive's reused activity prefix cannot authorize it in the old scope. + $scope->execute(new Obligation('second', 'test.operation', '1', [])); + return [ + 'status' => 'done', + ]; + } +} diff --git a/tests/Fixtures/V2/TestScopedOperation.php b/tests/Fixtures/V2/TestScopedOperation.php new file mode 100644 index 000000000..ef5c94a0f --- /dev/null +++ b/tests/Fixtures/V2/TestScopedOperation.php @@ -0,0 +1,54 @@ +operationId; + if (! Cache::has($key)) { + Cache::forever($key, $context->obligation->key); + Cache::increment('test:scope:effect-count'); + } + if (($context->obligation->input['mode'] ?? null) === 'lost-ack') { + return ForwardResult::unknown('ack-lost'); + } + return ForwardResult::observed([ + 'reference' => $context->operationId, + ], ['provider:' . $context->operationId]); + } + + public function compensate(OperationContext $context): Resolution + { + $scopedLog = Cache::get('test:scope:undo-scoped-log', []); + $scopedLog[] = $context->scope->key . '/' . $context->obligation->key; + Cache::forever('test:scope:undo-scoped-log', $scopedLog); + $log = Cache::get('test:scope:undo-log', []); + $log[] = $context->obligation->key; + Cache::forever('test:scope:undo-log', $log); + if ($context->obligation->key === 'first' && Cache::get('test:scope:unknown-first', false)) { + return Resolution::unknown('provider-unavailable'); + } + if (Cache::pull('test:scope:expire-native', false)) { + \Illuminate\Support\Carbon::setTestNow(now()->addMinutes(10)); + } + Cache::forget('test:scope:effect:' . $context->operationId); + return Resolution::verified( + ['provider-undo:' . $context->operationId], + ! Cache::get('test:scope:late-forward-possible', false) + ); + } +} diff --git a/tests/Fixtures/V2/TestWaitingScopeWorkflow.php b/tests/Fixtures/V2/TestWaitingScopeWorkflow.php new file mode 100644 index 000000000..b2eee45ea --- /dev/null +++ b/tests/Fixtures/V2/TestWaitingScopeWorkflow.php @@ -0,0 +1,30 @@ +execute(new Obligation('first', 'test.operation', '1', [])); + $scope->execute(new Obligation('second', 'test.operation', '1', [])); + self::await(static fn (): bool => false, conditionKey: 'business-result'); + return [ + 'status' => 'unexpected-completion', + ]; + } catch (Throwable) { + $this->compensate(); + return $scope->verifyCompensation(); + } + } +} diff --git a/tests/Support/CompensationScopeRecoveryProcessTest.php b/tests/Support/CompensationScopeRecoveryProcessTest.php new file mode 100644 index 000000000..54d27a4cd --- /dev/null +++ b/tests/Support/CompensationScopeRecoveryProcessTest.php @@ -0,0 +1,90 @@ +input(); + $this->app->instance(HandlerRegistry::class, new HandlerRegistry([ + 'test.operation' => [ + '1' => new TestScopedOperation(), + ], + ])); + $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { + throw new \LogicException('Cold scope access refused.'); + } + return 'cold-policy-1'; + } + }); + $scope = ScopeIdentity::fromArray($input['scope']); + app(ScopeRecovery::class)->request($scope, 'cold-request', 'operator-a', 'cancel'); + for ($step = 0; $step < 20 && app(ScopeRepository::class)->inspect($scope)['status'] !== 'verified'; $step++) { + $this->artisan('queue:work', [ + '--once' => true, + '--queue' => 'scope-recovery-test', + '--sleep' => 0, + '--tries' => 1, + ])->assertExitCode(0); + } + $status = app(ScopeRepository::class)->inspect($scope)['status']; + $this->assertSame('verified', $status); + file_put_contents($input['report'], json_encode([ + 'pid' => getmypid(), + 'status' => $status, + ], JSON_THROW_ON_ERROR)); + } + + protected function getPackageProviders($app): array + { + return [WorkflowServiceProvider::class]; + } + + protected function defineEnvironment($app): void + { + $input = $this->input(); + $app['config']->set([ + 'database.default' => 'scope_process', + 'database.connections.scope_process' => $input['database'], + 'database.redis' => $input['redis'], + 'workflows.storage.connection' => 'scope_process', + 'queue.default' => 'database', + 'queue.connections.database' => [ + 'driver' => 'database', + 'connection' => 'scope_process', + 'table' => 'jobs', + 'queue' => 'scope-recovery-test', + 'retry_after' => 90, + ], + 'cache.default' => 'redis', + 'cache.stores.redis' => $input['cache'], + 'cache.prefix' => $input['cache_prefix'], + 'logging.default' => 'null', + ]); + } + + private function input(): array + { + $path = getenv('DWF_SCOPE_RECOVERY_INPUT'); + if (! is_string($path) || $path === '') { + $this->markTestSkipped('Invoked only by the retained-scope integration control.'); + } + return json_decode(file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + } +} diff --git a/tests/Unit/V2/Compensation/ChildScopeTest.php b/tests/Unit/V2/Compensation/ChildScopeTest.php new file mode 100644 index 000000000..35b28eb06 --- /dev/null +++ b/tests/Unit/V2/Compensation/ChildScopeTest.php @@ -0,0 +1,74 @@ +scope('order'); + $child = $this->scope('shipment'); + $children = new ChildScopes(); + $id = $children->register($parent, 'origin', 'delivery', $child); + $this->assertSame($id, $children->register($parent, 'origin', 'delivery', $child)); + $this->expectException(LogicException::class); + $children->register($parent, 'origin', 'delivery', new ScopeIdentity('shop', 'client-a', 'shipment', '2')); + } + + public function testCycleAndSecondNestingLevelAreRefused(): void + { + $children = new ChildScopes(); + $children->register($this->scope('order'), 'order-run', 'delivery', $this->scope('shipment')); + foreach ([[$this->scope('shipment'), $this->scope('order')], + [$this->scope('shipment'), $this->scope('return')]] as [$parent, $child]) { + try { + $children->register($parent, 'shipment-run', 'child', $child); + $this->fail('Unsupported or cyclic scope nesting was admitted.'); + } catch (LogicException) { + $this->assertSame(1, count((new ScopeRepository())->inspect($this->scope('order'))['obligations'])); + } + } + } + + public function testForeignNamespaceCannotBeAChild(): void + { + $this->expectException(LogicException::class); + (new ChildScopes())->register( + $this->scope('order'), + 'order-run', + 'delivery', + new ScopeIdentity('shop', 'client-b', 'shipment', '1') + ); + } + + public function testChildFanOutLimitCannotLeaveAPartialParticipant(): void + { + $children = new ChildScopes(); + $parent = $this->scope('order'); + for ($index = 0; $index < ChildScopes::MAX_CHILDREN; ++$index) { + $children->register($parent, 'order-run', 'delivery-' . $index, $this->scope('shipment-' . $index)); + } + try { + $children->register($parent, 'order-run', 'overflow', $this->scope('overflow')); + $this->fail('The child fan-out limit was exceeded.'); + } catch (LogicException) { + $this->assertCount(ChildScopes::MAX_CHILDREN, (new ScopeRepository())->inspect($parent)['obligations']); + $this->assertSame(ChildScopes::MAX_CHILDREN, \Workflow\V2\Support\ConfiguredV2Models::connection() + ->table('workflow_compensation_children') + ->count()); + } + } + + private function scope(string $key): ScopeIdentity + { + return new ScopeIdentity('shop', 'client-a', $key, '1'); + } +} diff --git a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php new file mode 100644 index 000000000..1ae31a2a6 --- /dev/null +++ b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php @@ -0,0 +1,97 @@ +assertSame($original->id(), (new ScopeIdentity('shop', 'client-a', 'order-42', '1'))->id()); + foreach ([['other', 'client-a', 'order-42', '1'], + ['shop', 'client-b', 'order-42', '1'], + ['shop', 'client-a', 'order-42', '2'], + ] as $parts) { + $this->assertNotSame($original->id(), (new ScopeIdentity(...$parts))->id()); + } + } + + public function testDescriptorCanonicalizesMapsWithoutReorderingLists(): void + { + $a = new Obligation('book', 'carrier.cancel', '1', [ + 'b' => 2, + 'a' => [ + '01' => 1, + '001' => 2, + ], + ]); + $b = new Obligation('book', 'carrier.cancel', '1', [ + 'a' => [ + '001' => 2, + '01' => 1, + ], + 'b' => 2, + ]); + $this->assertSame($a->fingerprint(), $b->fingerprint()); + $this->assertNotSame( + (new Obligation('book', 'carrier.cancel', '1', [1, 2]))->fingerprint(), + (new Obligation('book', 'carrier.cancel', '1', [2, 1]))->fingerprint() + ); + $this->assertSame($a->fingerprint(), Obligation::fromArray($a->toArray())->fingerprint()); + } + + public function testExecutableClassNameCannotBeAHandlerIdentifier(): void + { + $this->expectException(InvalidArgumentException::class); + new Obligation('book', 'App\\DeleteEverything', '1', []); + } + + public function testMutableObjectsCannotEnterTheDescriptor(): void + { + $this->expectException(InvalidArgumentException::class); + new Obligation('book', 'carrier.cancel', '1', [ + 'input' => new \stdClass(), + ]); + } + + public function testEvidenceReferenceLimitIsValidatedBeforeAcceptingProviderResults(): void + { + $references = array_fill(0, 33, 'proof'); + foreach ([ + static fn () => ForwardResult::observed([], $references), + static fn () => Resolution::verified($references, true), + ] as $result) { + try { + $result(); + $this->fail('The result admitted more evidence than its bounded journal envelope supports.'); + } catch (InvalidArgumentException $error) { + $this->assertStringContainsString('evidence reference limit', $error->getMessage()); + } + } + } + + public function testEvidenceReferencesCannotHideUnboundedMetadataInMapKeys(): void + { + $this->expectException(InvalidArgumentException::class); + ForwardResult::observed([], [ + 'provider-metadata' => 'proof', + ]); + } + + public function testOversizedDescriptorIsRejectedBeforeStorage(): void + { + $this->expectException(InvalidArgumentException::class); + new Obligation('book', 'carrier.cancel', '1', [ + 'input' => str_repeat('x', 16385), + ]); + } +} diff --git a/tests/Unit/V2/Compensation/ScopeProgressTest.php b/tests/Unit/V2/Compensation/ScopeProgressTest.php new file mode 100644 index 000000000..6acf7846e --- /dev/null +++ b/tests/Unit/V2/Compensation/ScopeProgressTest.php @@ -0,0 +1,104 @@ +registered(); + $progress = new ScopeProgress(); + $this->assertFalse($progress->admitForward($scope, $id, 'native-attempt-1')['uncertain']); + $progress->forwardResult($scope, $id, 'native-attempt-1', ForwardResult::unknown('ack-missing')); + $this->assertTrue($progress->admitForward($scope, $id, 'native-attempt-2')['uncertain']); + (new ScopeRepository())->request($scope, 'request', 'operator', 'cancel'); + $this->expectException(LogicException::class); + $progress->admitForward($scope, $id, 'native-attempt-3'); + } + + public function testReverseProgressStopsAtUnknownAndDoesNotRepeatVerifiedWork(): void + { + [$scope, $first] = $this->registered(); + $repository = new ScopeRepository(); + $second = $repository->register($scope, new Obligation('second', 'carrier.cancel', '1', [])); + $repository->request($scope, 'request', 'operator', 'cancel'); + $progress = new ScopeProgress(); + $progress->beginCompensation($scope, $second, 'native-undo-1'); + $progress->resolve($scope, $second, 'native-undo-1', Resolution::unknown('provider-unavailable')); + try { + $progress->beginCompensation($scope, $first, 'native-undo-2'); + $this->fail('Earlier compensation advanced past unresolved work.'); + } catch (LogicException) { + $this->assertSame('requested', $repository->inspect($scope)['status']); + } + $progress->beginCompensation($scope, $second, 'native-undo-3'); + $progress->resolve($scope, $second, 'native-undo-3', Resolution::verified(['proof-second'], true)); + $this->assertNull($progress->beginCompensation($scope, $second, 'native-undo-4')); + $progress->beginCompensation($scope, $first, 'native-undo-5'); + $progress->resolve($scope, $first, 'native-undo-5', Resolution::verified(['proof-first'], true)); + $this->assertSame('verified', $progress->finish($scope)['status']); + } + + public function testUnknownForwardCannotBecomeVerifiedWithoutSettledEvidence(): void + { + [$scope, $id] = $this->registered(); + $progress = new ScopeProgress(); + $progress->admitForward($scope, $id, 'native-forward'); + (new ScopeRepository())->request($scope, 'request', 'operator', 'cancel'); + $progress->beginCompensation($scope, $id, 'native-undo'); + $this->expectException(LogicException::class); + $progress->resolve($scope, $id, 'native-undo', Resolution::verified(['undo-ack'], false)); + } + + public function testCompensationCannotOverwriteAnotherNativeAttempt(): void + { + [$scope, $id] = $this->registered(); + (new ScopeRepository())->request($scope, 'request', 'operator', 'cancel'); + $progress = new ScopeProgress(); + $progress->beginCompensation($scope, $id, 'native-undo-1'); + $progress->beginCompensation($scope, $id, 'native-undo-2'); + $this->expectException(LogicException::class); + $progress->resolve($scope, $id, 'native-undo-1', Resolution::verified(['proof'], true)); + } + + public function testMaximumEvidenceReferencesFitForwardAndCompensationJournals(): void + { + [$scope, $id] = $this->registered(); + $references = array_fill(0, 32, 'p' . str_repeat('/', 127)); + $progress = new ScopeProgress(); + $repository = new ScopeRepository(); + $progress->admitForward($scope, $id, str_repeat('f', 26)); + $progress->forwardResult($scope, $id, str_repeat('f', 26), ForwardResult::observed([], $references)); + $repository->request($scope, 'request', 'operator', 'cancel'); + $progress->beginCompensation($scope, $id, str_repeat('c', 26)); + $progress->resolve($scope, $id, str_repeat('c', 26), Resolution::verified($references, true)); + $records = array_values(array_filter( + $repository->journal($scope), + static fn (array $row): bool => in_array($row['kind'], ['forward_outcome', 'compensation_outcome'], true) + )); + $this->assertCount(2, $records); + foreach ($records as $row) { + $this->assertSame($references, json_decode($row['data'], true, flags: JSON_THROW_ON_ERROR)['evidence']); + } + $this->assertSame('verified', $progress->finish($scope)['status']); + } + + private function registered(): array + { + $scope = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + $repository = new ScopeRepository(); + $repository->open($scope, 'original-run'); + return [$scope, $repository->register($scope, new Obligation('book', 'carrier.cancel', '1', []))]; + } +} diff --git a/tests/Unit/V2/Compensation/ScopeRecoveryTest.php b/tests/Unit/V2/Compensation/ScopeRecoveryTest.php new file mode 100644 index 000000000..40dedc801 --- /dev/null +++ b/tests/Unit/V2/Compensation/ScopeRecoveryTest.php @@ -0,0 +1,111 @@ +assertSame($a->recoveryInstanceId(), ScopeIdentity::fromArray($a->toArray())->recoveryInstanceId()); + $this->assertNotSame( + $a->recoveryInstanceId(), + (new ScopeIdentity('shop', 'client-b', 'order-42', '1'))->recoveryInstanceId() + ); + $this->assertNotSame( + $a->recoveryInstanceId(), + (new ScopeIdentity('shop', 'client-a', 'order-42', '2'))->recoveryInstanceId() + ); + } + + public function testRequestIsBoundToItsOriginalCallerAndContent(): void + { + [$scope, $router] = $this->open(); + $first = $router->request($scope, 'request-1', 'operator-a', 'cancel-order'); + $again = $router->request($scope, 'request-1', 'operator-a', 'cancel-order'); + $this->assertSame($first, $again); + $this->expectException(\LogicException::class); + $router->request($scope, 'request-1', 'operator-b', 'cancel-order'); + } + + public function testAnActiveOriginUsesNativeCooperativeCancellation(): void + { + [$scope, $router, $runId] = $this->open(); + $result = $router->request($scope, 'request-1', 'operator-a', 'cancel-order'); + $this->assertSame('origin', $result['route']); + $this->assertSame($runId, $result['runId']); + $this->assertSame('requested', (new ScopeRepository())->inspect($scope)['status']); + $this->assertNull(\Workflow\V2\Models\WorkflowInstance::query()->find($scope->recoveryInstanceId())); + } + + public function testClosedOriginStartsOneCanonicalRecoveryForDifferentRequests(): void + { + [$scope, $router, $runId] = $this->open(); + \Workflow\V2\Models\WorkflowRun::query()->findOrFail($runId)->forceFill([ + 'status' => 'completed', + 'closed_at' => now(), + ])->save(); + $first = $router->request($scope, 'request-1', 'operator-a', 'cancel-order'); + $second = $router->request($scope, 'request-2', 'operator-a', 'cancel-order'); + $this->assertSame('recovery', $first['route']); + $this->assertSame($first['runId'], $second['runId']); + $this->assertSame( + 1, + \Workflow\V2\Models\WorkflowRun::query()->where( + 'workflow_instance_id', + $scope->recoveryInstanceId() + )->count() + ); + } + + public function testDeniedRequestCannotCloseTheScopeOrCreateRecovery(): void + { + [$scope, $router] = $this->open(); + try { + $router->request($scope, 'request-1', 'intruder', 'cancel-order'); + $this->fail('An unauthorized request was accepted.'); + } catch (\LogicException) { + $this->assertSame('open', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame( + 0, + \Workflow\V2\Support\ConfiguredV2Models::connection()->table('workflow_compensation_requests')->count() + ); + } + } + + private function open(): array + { + config([ + 'workflows.v2.task_dispatch_mode' => 'poll', + ]); + $scope = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + $origin = WorkflowStub::make(TestSignalWorkflow::class, 'origin-workflow', 'client-a'); + $origin->start(); + (new ScopeRepository())->open($scope, $origin->runId()); + $authorization = new class() implements ScopeAuthorization { + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if (! in_array($principal, ['operator-a', 'operator-b'], true) || $scope->namespace !== 'client-a') { + throw new \LogicException('Not authorized for this scope.'); + } + return 'test-policy-1'; + } + }; + return [ + $scope, + new ScopeRecovery(new ScopeRepository(), $authorization, app(WorkflowControlPlane::class)), + $origin->runId(), + ]; + } +} diff --git a/tests/Unit/V2/Compensation/ScopeRepositoryTest.php b/tests/Unit/V2/Compensation/ScopeRepositoryTest.php new file mode 100644 index 000000000..f30e3e801 --- /dev/null +++ b/tests/Unit/V2/Compensation/ScopeRepositoryTest.php @@ -0,0 +1,151 @@ +scope(); + $repository = new ScopeRepository(); + $repository->open($scope, 'original-run'); + $first = $repository->register($scope, new Obligation('book', 'carrier.cancel', '1', [ + 'reference' => 'booking-1', + ])); + $again = (new ScopeRepository())->register($scope, new Obligation('book', 'carrier.cancel', '1', [ + 'reference' => 'booking-1', + ])); + $this->assertSame($first, $again); + $this->assertCount(1, $repository->inspect($scope)['obligations']); + $this->assertSame(['opened', 'registered'], array_column($repository->journal($scope), 'kind')); + } + + public function testConflictingDuplicateDoesNotReplaceOriginalDescriptor(): void + { + $repository = new ScopeRepository(); + $repository->open($this->scope(), 'original-run'); + $repository->register($this->scope(), new Obligation('book', 'carrier.cancel', '1', [])); + try { + $repository->register($this->scope(), new Obligation('book', 'carrier.cancel', '2', [])); + $this->fail('Conflicting descriptor was adopted.'); + } catch (LogicException) { + $this->assertSame( + '1', + $repository->inspect($this->scope())['obligations'][0]['descriptor']['handlerVersion'] + ); + } + } + + public function testScopeGenerationAndNamespaceCannotShareObligations(): void + { + $repository = new ScopeRepository(); + $repository->open($this->scope(), 'run-a'); + $repository->register($this->scope(), new Obligation('book', 'carrier.cancel', '1', [])); + foreach ([ + new ScopeIdentity('shop', 'client-b', 'order-42', '1'), + new ScopeIdentity('shop', 'client-a', 'order-42', '2'), + ] as $other) { + $repository->open($other, 'run-b'); + $this->assertSame([], $repository->inspect($other)['obligations']); + } + } + + public function testRegistrationAndItsJournalRollBackTogether(): void + { + $repository = new ScopeRepository(); + $repository->open($this->scope(), 'original-run'); + $connection = ConfiguredV2Models::connection(); + $connection->beginTransaction(); + $repository->register($this->scope(), new Obligation('book', 'carrier.cancel', '1', [])); + $connection->rollBack(); + $this->assertSame([], $repository->inspect($this->scope())['obligations']); + $this->assertSame(['opened'], array_column($repository->journal($this->scope()), 'kind')); + } + + public function testOriginalRunCannotBeSilentlyReplaced(): void + { + $repository = new ScopeRepository(); + $repository->open($this->scope(), 'original-run'); + $this->expectException(LogicException::class); + $repository->open($this->scope(), 'replacement-run'); + } + + public function testJournalWriteFailureCannotLeaveAnUnjournaledObligation(): void + { + $repository = new ScopeRepository(); + $repository->open($this->scope(), 'original-run'); + $fail = true; + ConfiguredV2Models::connection()->beforeExecuting(static function (string $sql) use (&$fail): void { + if ($fail && str_starts_with($sql, 'insert') && str_contains($sql, 'workflow_compensation_journal')) { + throw new \RuntimeException('injected journal failure'); + } + }); + try { + $repository->register($this->scope(), new Obligation('book', 'carrier.cancel', '1', [])); + $this->fail('Expected the injected failure.'); + } catch (\RuntimeException $error) { + $this->assertSame('injected journal failure', $error->getMessage()); + } finally { + $fail = false; + } + $this->assertSame([], $repository->inspect($this->scope())['obligations']); + $this->assertCount(1, $repository->journal($this->scope())); + $repository->register($this->scope(), new Obligation('book', 'carrier.cancel', '1', [])); + $this->assertSame(1, $repository->inspect($this->scope())['obligations'][0]['position']); + } + + public function testOpeningLiveStorageFromWorkflowReplayIsRefusedBeforeWriting(): void + { + $scope = $this->scope(); + $fiber = new Fiber(static function () use ($scope): void { + WorkflowFiberContext::enter(); + try { + (new ScopeRepository())->open($scope, 'original-run'); + } finally { + WorkflowFiberContext::leave(); + } + }); + try { + $fiber->start(); + $this->fail('Workflow replay opened live compensation storage.'); + } catch (LogicException $error) { + $this->assertStringContainsString('never in workflow replay', $error->getMessage()); + } + $this->assertSame(0, ConfiguredV2Models::connection()->table('workflow_compensation_scopes')->count()); + $this->assertSame(0, ConfiguredV2Models::connection()->table('workflow_compensation_journal')->count()); + } + + public function testObligationLimitCannotLeaveAPartialRegistration(): void + { + $repository = new ScopeRepository(); + $scope = $this->scope(); + $repository->open($scope, 'original-run'); + for ($index = 0; $index < ScopeRepository::MAX_OBLIGATIONS; ++$index) { + $repository->register($scope, new Obligation('work-' . $index, 'carrier.cancel', '1', [])); + } + $journal = $repository->journal($scope); + try { + $repository->register($scope, new Obligation('overflow', 'carrier.cancel', '1', [])); + $this->fail('The obligation limit was exceeded.'); + } catch (LogicException) { + $this->assertCount(ScopeRepository::MAX_OBLIGATIONS, $repository->inspect($scope)['obligations']); + $this->assertSame($journal, $repository->journal($scope)); + } + } + + private function scope(): ScopeIdentity + { + return new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + } +} From 0f07ba9796d942f8ca1db1798b34afa1d7b4442c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:47:02 +0000 Subject: [PATCH 06/32] Prepare unlocked scope authorization and versioned compensation contracts --- CHANGELOG.md | 5 + docs/api-stability.md | 21 ++- docs/compensation-scopes.md | 120 +++--------------- .../CompensateObligationActivity.php | 4 + .../Activities/ForwardOperationActivity.php | 22 +++- .../Activities/PrepareRecoveryActivity.php | 4 + .../Activities/RegisterChildScopeActivity.php | 23 ++-- .../Activities/RegisterObligationActivity.php | 7 +- .../Activities/RequestScopeActivity.php | 4 + .../Activities/VerifyScopeActivity.php | 4 + src/V2/Compensation/Definition/Obligation.php | 12 +- .../Definition/PortableDescriptor.php | 2 + .../Compensation/Definition/ScopeIdentity.php | 27 ++-- src/V2/Compensation/DurableScope.php | 1 + .../Execution/EvidenceReferences.php | 2 + .../Compensation/Execution/ForwardResult.php | 18 ++- src/V2/Compensation/Execution/Resolution.php | 18 ++- .../Persistence/ScopeProgress.php | 17 +-- .../Persistence/ScopeRepository.php | 14 +- .../Compensation/Recovery/ChildResolution.php | 2 + .../Recovery/RecoveryWorkflow.php | 3 + .../Compensation/Security/ScopeBoundary.php | 69 +++++++--- .../V2/V2CompensationForwardAdmissionTest.php | 39 ++++++ .../V2/V2CompensationScopeRunBoundaryTest.php | 2 +- tests/Feature/V2/V2CompensationScopeTest.php | 10 ++ .../V2/Compensation/ScopeDescriptorTest.php | 78 ++++++++++++ .../V2/Compensation/ScopeRepositoryTest.php | 4 +- 27 files changed, 374 insertions(+), 158 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8706e54be..a1433fe90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Experimental embedded compensation scopes (#1063): retained versioned + obligations, native Saga recovery, sequential child participation and explicit + release. Each scope keeps its originating run; continue-as-new, origin redrive + and child retry do not transfer forward ownership. + - Failed-run redrive accepts a verified linear activity retry chain. A retried completed prefix is reused and only the failed boundary step runs again. Substituted attempt/task identities refuse with diff --git a/docs/api-stability.md b/docs/api-stability.md index e9618b03f..292b5c849 100644 --- a/docs/api-stability.md +++ b/docs/api-stability.md @@ -114,7 +114,26 @@ The workflow package is consumed by three distinct audiences: into the workflow package as a library but do not author workflows. The semver guarantee below covers all three surfaces. Breaking changes to -anything marked `@api` require a major version bump. +anything marked `@api` require a major version bump, except surfaces explicitly +marked `@experimental` as described below. + +## Experimental compensation scopes + +The `Workflow\V2\Compensation` authoring helpers and host handler/authorization +contracts are experimental. Their API and persisted storage contract may change +in a minor release while the first increment is evaluated. The scope, obligation, +forward-result and resolution envelopes carry an explicit format version; unknown +versions refuse interpretation. Activity types and activity-sequence markers are +recorded identities and must be changed through explicit deterministic versioning. +`@internal` activities, format utilities and engine classes remain implementation +details rather than supported host entry points. + +This increment supports embedded Laravel, sequential compensation, one nesting +level and explicit release. It does not transfer forward scope ownership across +continue-as-new, originating-run redrive or child retry. Use a distinct generation +for each originating run. Retained obligations stay attached to that run and remain +recoverable through `ScopeRecovery`. See [durable compensation scopes](compensation-scopes.md). +Existing run-local Saga and its stability contract remain unchanged. ## What is covered diff --git a/docs/compensation-scopes.md b/docs/compensation-scopes.md index d5ac4d132..dfc6e64d2 100644 --- a/docs/compensation-scopes.md +++ b/docs/compensation-scopes.md @@ -6,38 +6,22 @@ nested-machine integration in [business state-machine proposal](https://github.c This is not a released API. Existing run-local Saga and the four-method `TransitionStep` contract remain unchanged. -## Reviewable increments - -1. Retained identities, bounded portable descriptors and atomic registration - journal. Prove exact duplicate adoption, conflicting reuse refusal, namespace - and generation separation, and rollback of incomplete registration. -2. Content-bound cancellation requests and canonical recovery identity. Reuse - native start/adopt, task ownership, activity attempts and redrive. Do not add - another executor, lease, claim table, heartbeat or retry policy engine. - Prove duplicate routing, closure races, namespace/generation substitution and - refused identity reuse. -3. Recorded activity boundaries and native Saga recovery, including closed runs. - Prove cold replay, registration acknowledgement loss, handler unavailability, - authorization, cancellation and recovery after worker restart. -4. Sequential child-scope participants and verified child-first compensation. - Prove generation binding, missing child evidence, cycle refusal and bounded - participation. State-bound SM bindings and the two-integration example remain - the next business state-machine slice, after this retained-scope foundation. -5. Retention and compatibility review; run ECS, PHPStan, unit, coverage and feature - gates. Record the limits of local evidence separately from CI qualification. - -Each increment follows tests, implementation, review, fixes and repeated review. -The issue remains the work record; this document defines the technical contract. - ## Boundaries -A scope is `(application, namespace, key, generation)`, not a workflow run. +A scope is `(application, namespace, key, generation)`. Its forward work belongs +to one originating run, so choose a generation unique to that run. Continuing, +redriving or retrying a child with an existing generation does not transfer its +ownership; new registration refuses with nonretryable `scope_execution_owner_mismatch`. The native default namespace remains `null`; it is never replaced by an invented alias. The original run and a later recovery run have separate identities. One scope contains at most 256 obligations. Stored descriptors contain a closed handler identifier and version, stable operation key and portable data (16 KiB, depth 8, 512 values). They never contain executable closures or arbitrary class names. Applications store protected references instead of credentials or customer data. +Scope identities, obligations, forward results and compensation resolutions use +closed version-1 envelopes. Their canonical arrays are explicit rather than PHP +object layouts. Unsupported versions refuse interpretation. Recorded activity +types and activity-sequence version markers are durable identities. The registration journal and obligation commit together. Workflow history records activity boundary outcomes; a lost activity acknowledgement is adopted by exact @@ -65,21 +49,6 @@ obligation and journal rows follow that lock. No external call runs under a data lock. The ledger has no cascading foreign key to run history: normal history pruning must not erase unresolved obligations or reuse their identities. -## Evidence - -Checked before implementation: upstream main `46ca8f67` contains the native -ownership guarantees. Four isolated PostgreSQL probes (32 assertions) proved -concurrent canonical start, concurrent redrive, exclusive workflow-task claim and -the fact that `business_key` alone does not deduplicate different instance IDs. -Three upstream PostgreSQL cancellation concurrency tests (160 assertions) preserved -canonical request/root/budget and explicitly refused a conflicting root. - -The first ledger increment passed 11 targeted tests (23 assertions), including an -injected journal-insert failure. It does not yet prove workflow-boundary integration. -The previous draft's recovery lease tests described an unimplemented competing -mechanism; they are replaced by tests of native identity and authority integration. - - ## Runtime flow and responsibilities ```mermaid @@ -112,7 +81,10 @@ flowchart TD `ScopeBoundary` uses the existing native activity status/authority checks while retaining their SQL locks in the ledger transaction. Its lock order is native attempt, execution, run and task, followed by the scope and its journal. It checks -authority again after the mutation. Provider calls run between guarded transactions, +authority again after the mutation. Host authorization and handler input +validation run before the transaction, with no native or scope locks held. The +transaction rechecks native authority, request identity and the validated forward +descriptor before admitting work. Provider calls run between guarded transactions, never inside them. The journal records principal, authorization policy version, origin/current run, activity and attempt references. Child participation records the child authorization policy separately from the parent registration policy. @@ -152,7 +124,9 @@ increment does not transfer forward ownership across continue-as-new or origin redrive. The first accepted request pins the cleanup principal. Later authorized requests can route recovery but cannot replace that principal. Recovery evaluates current authorization again at every activity boundary. Revoked authority leaves -work unresolved rather than silently adopting another principal. +work unresolved rather than silently adopting another principal. Requesting cleanup +on an active origin requests cancellation of the entire native run: all its Saga +callbacks can run, including other scopes and ordinary non-scope compensations. The default scoped forward/compensation activities use three native attempts and the native backoff. No scope retry loop is introduced. `DurableScope.execute()` @@ -194,33 +168,6 @@ child-first recovery. It also checks originating-run redrive after an unrelated failed activity and after an uncertain scoped forward effect. Recovery preserves the original terminal history in all cases. -## Core redrive regression evidence - -The replay corpus policy explicitly guards `FailedRunRedrivePlan` and -`ActivityRetryHistory`, so changes to either require executable regression evidence. - -The planner changes are independent of compensation scopes. They accept native -`ActivityRetryScheduled` lineage only when execution, attempt, task and attempt -number references agree. A valid native active-start adoption receipt must also -match the run and accepted command snapshot. The planner keeps current main's -completed-child reuse behavior. - -The `redrive-retried-completed-activity`, `redrive-retried-failed-activity` and -`redrive-retried-activity-after-completed-child` regression-corpus fixtures retain -both a cold replay prefix and the failed source history. The official -`ReplayRegressionCorpusTest` consumer executes `FailedRunRedrivePlan` against the -source and its identity-substitution cases, then executes the cold workflow -replay. This provides planner evidence separately from the scope integration -tests. `FailedRunRedrivePlanTest` separately covers start-adoption receipt -substitution. The separate `redrive-native-active-start-adoption` corpus fixture -executes the same acceptance and substitution checks for recorded start receipts. -Its planner source retains the native audit-enriched command snapshot. Its cold -replay prefix contains the authored event payload, which is checked against the -strict event payload contract before native audit enrichment. -Existing completed-child corpus and feature regressions remain -applicable. These evidence paths allow the core planner change to be reviewed -or extracted separately from the experimental scope API. - ## Authoring and host setup The embedded host binds `ScopeAuthorization` to its authorization policy and @@ -228,6 +175,12 @@ The embedded host binds `ScopeAuthorization` to its authorization policy and migrations create the ledger tables on the configured workflow storage connection. No additional queue, worker or lease configuration is required. +Derive `$principal` from trusted application policy code; never accept its identity +from an HTTP/webhook payload or workflow-start argument. The authorization binding +must verify that principal against the host's current trusted security context. +Stored SHA-256 content hashes detect accidental corruption; database write access +remains inside the host's trust boundary. + Inside a native workflow, create `DurableScope($this, $identity, $principal)` and call `execute(new Obligation($key, $handlerType, $handlerVersion, $input))`. Register all effects through this helper. On cleanup, call the existing @@ -246,34 +199,3 @@ authorization, not from untrusted request content. Repeating the same request requires the same principal and reason. Inspecting or requesting retained work must go through an authorized host boundary; the `@internal` persistence classes are not an HTTP-facing data access API. - -## Evidence and review - -| Guarantee | Responsible component | Acceptance test | -| --- | --- | --- | -| Exact request and generation identity | ScopeIdentity, ScopeRepository, ScopeRecovery | ScopeRecoveryTest, V2CompensationScopeConcurrencyTest | -| Current native attempt owns ledger mutation | ScopeBoundary | ExpiredNativeAttemptCannotPublishVerifiedCompensation | -| No false success for late/unknown effects | ScopeProgress, CompensateObligationActivity | PossibleLateForwardEffectCannotBecomeVerifiedAfterRetryExhaustion | -| Closed-run recovery and original terminal history | RecoveryWorkflow, native Saga | ClosedOriginUsesCanonicalRecoveryAndKeepsOriginalTerminalHistory | -| Verified progress survives failure and redrive | Native redrive plus ledger | RetryExhaustionStaysUnresolvedAndNativeRedriveReusesVerifiedWork, OneRetainedRequestCanRedriveTwoFailuresWithoutRepeatingVerifiedWork | -| Native start adoption preserves eligible redrive | FailedRunRedrivePlan | NativeActiveAdoptionDoesNotPreventRecoveryRedriveAfterFailure, SubstitutedActiveStartAdoptionCannotAuthorizeRedrive | -| Forward evidence survives explicit release | ScopeProgress, ScopeRetention | ForwardEvidenceSurvivesReleaseWithoutRetainingBusinessInputsOrResults | -| Replay does no handler/provider resolution | DurableScope, recorded activities | ColdReplayDoesNotResolveMutableHandlersOrRepeatProviderEffects | -| Child-first sequential progress | ChildScopes, ChildResolution | CompletedChildScopeIsVerifiedBeforeEarlierParentCompensation, CancellationWhileChildWaitsResolvesItsSagaBeforeEarlierParentWork | -| Child authorization evidence | RegisterChildScopeActivity | ChildParticipationRecordsItsOwnAuthorizationPolicyVersion, InvalidChildParticipationPolicyCannotRegisterAChild | -| Retained descriptor integrity before effects | ForwardOperationActivity, CompensateObligationActivity | TamperedRegistrationCannotReachTheForwardProvider, TamperedRetainedDescriptorIsRefusedBeforeProviderCall | -| Live storage is forbidden during replay | ScopeRepository, ScopeStorage | OpeningLiveStorageFromWorkflowReplayIsRefusedBeforeWriting | -| Bounded retained work | ScopeRepository, ChildScopes | ObligationLimitCannotLeaveAPartialRegistration, ChildFanOutLimitCannotLeaveAPartialParticipant | -| No cycle under concurrent registration | ChildScopes integrity lock | CompetingEdgesCannotCreateACycle | -| Ordinary pruning preserves recovery | Independent retained ledger | OrdinaryRunPruningPreservesRetainedScopeRecovery | -| Explicit release cannot reuse identifiers | ScopeRetention | ReleaseRefusesUnresolvedWorkThenRetainsAnUnreusableTombstone | -| Fresh-process recovery | Native workflow and persisted ledger | RetainedRecoveryRunsInAFreshPhpProcess | - -Integration exposed a native redrive limitation: `ActivityRetryScheduled` was -refused as unsupported history. `ActivityRetryHistory` validates exact execution, -attempt counters and task lineage before the existing redrive planner accepts that -history. Substitution controls accompany the real retry-exhaustion regression. - -Local evidence does not establish a published package, CI matrix qualification, -service-mode/cross-SDK support or production-provider behavior. The full quality -cycle and final evidence counts are recorded after the last code review. diff --git a/src/V2/Compensation/Activities/CompensateObligationActivity.php b/src/V2/Compensation/Activities/CompensateObligationActivity.php index b007b62ca..62dae941b 100644 --- a/src/V2/Compensation/Activities/CompensateObligationActivity.php +++ b/src/V2/Compensation/Activities/CompensateObligationActivity.php @@ -7,6 +7,7 @@ use Throwable; use Workflow\Exceptions\NonRetryableException; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\Obligation; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Execution\HandlerRegistry; @@ -18,8 +19,11 @@ use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.compensate-obligation.v1')] final class CompensateObligationActivity extends Activity { public int $tries = 3; diff --git a/src/V2/Compensation/Activities/ForwardOperationActivity.php b/src/V2/Compensation/Activities/ForwardOperationActivity.php index 26dd99b5a..5b66aee8b 100644 --- a/src/V2/Compensation/Activities/ForwardOperationActivity.php +++ b/src/V2/Compensation/Activities/ForwardOperationActivity.php @@ -6,6 +6,7 @@ use Throwable; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\Obligation; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Execution\ForwardResult; @@ -13,11 +14,15 @@ use Workflow\V2\Compensation\Execution\OperationContext; use Workflow\V2\Compensation\Execution\ScopedOperation; use Workflow\V2\Compensation\Persistence\ScopeProgress; +use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.forward-operation.v1')] final class ForwardOperationActivity extends Activity { public int $tries = 3; @@ -27,22 +32,29 @@ public function handle(array $identity, string $id, string $principal): array $scope = ScopeIdentity::fromArray($identity); $boundary = app(ScopeBoundary::class); $handler = null; + $validatedFingerprint = null; $admission = $boundary->run( $this, $scope, 'forward', $principal, - function () use ($scope, $id, &$handler): array { + function () use ($scope, $id, &$validatedFingerprint): array { return app(ScopeProgress::class)->admitForward( $scope, $id, $this->attemptId(), $this->activityId(), - static function (array $row) use (&$handler): void { - $handler = app(HandlerRegistry::class)->resolve(Obligation::fromStored($row)); - } + $validatedFingerprint, ); - } + }, + beforeTransaction: static function () use ($scope, $id, &$handler, &$validatedFingerprint): void { + $row = app(ScopeRepository::class)->obligation($scope, $id); + $validatedFingerprint = $row['fingerprint']; + // Cached forward results require no handler lookup or host I/O. + if ($row['forward_status'] !== 'observed') { + $handler = app(HandlerRegistry::class)->resolve(Obligation::fromStored($row)); + } + }, ); if ($admission['cached'] !== null) { return $admission['cached']['value']; diff --git a/src/V2/Compensation/Activities/PrepareRecoveryActivity.php b/src/V2/Compensation/Activities/PrepareRecoveryActivity.php index 973138b14..6f5589da8 100644 --- a/src/V2/Compensation/Activities/PrepareRecoveryActivity.php +++ b/src/V2/Compensation/Activities/PrepareRecoveryActivity.php @@ -5,12 +5,16 @@ namespace Workflow\V2\Compensation\Activities; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.prepare-recovery.v1')] final class PrepareRecoveryActivity extends Activity { public int $tries = 3; diff --git a/src/V2/Compensation/Activities/RegisterChildScopeActivity.php b/src/V2/Compensation/Activities/RegisterChildScopeActivity.php index ac5fbbfe9..a66a2024e 100644 --- a/src/V2/Compensation/Activities/RegisterChildScopeActivity.php +++ b/src/V2/Compensation/Activities/RegisterChildScopeActivity.php @@ -7,6 +7,7 @@ use Throwable; use Workflow\Exceptions\NonRetryableException; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\PortableDescriptor; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Persistence\ChildScopes; @@ -15,8 +16,11 @@ use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.register-child-scope.v1')] final class RegisterChildScopeActivity extends Activity { public int $tries = 3; @@ -25,18 +29,13 @@ public function handle(array $identity, string $key, array $childIdentity, strin { $scope = ScopeIdentity::fromArray($identity); $child = ScopeIdentity::fromArray($childIdentity); + $policy = null; return app(ScopeBoundary::class)->run( $this, $scope, 'register', $principal, - function () use ($scope, $key, $child, $principal): string { - try { - $policy = app(ScopeAuthorization::class)->assertAllowed($child, $principal, 'participate'); - PortableDescriptor::identifier($policy, 'authorization policy version'); - } catch (Throwable) { - throw new NonRetryableException('scope_child_authorization_refused'); - } + function () use ($scope, $key, $child, $principal, &$policy): string { $id = app(ChildScopes::class)->register($scope, $this->runId(), $key, $child); ScopeStorage::append(ScopeStorage::locked($scope), 'child_authorized', [ 'childScopeId' => $child->id(), @@ -48,7 +47,15 @@ function () use ($scope, $key, $child, $principal): string { 'attemptId' => $this->attemptId(), ]); return $id; - } + }, + beforeTransaction: static function () use ($child, $principal, &$policy): void { + try { + $policy = app(ScopeAuthorization::class)->assertAllowed($child, $principal, 'participate'); + PortableDescriptor::identifier($policy, 'authorization policy version'); + } catch (Throwable) { + throw new NonRetryableException('scope_child_authorization_refused'); + } + }, ); } } diff --git a/src/V2/Compensation/Activities/RegisterObligationActivity.php b/src/V2/Compensation/Activities/RegisterObligationActivity.php index 016372a62..a7593d03a 100644 --- a/src/V2/Compensation/Activities/RegisterObligationActivity.php +++ b/src/V2/Compensation/Activities/RegisterObligationActivity.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Compensation\Activities; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\Obligation; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Execution\HandlerRegistry; @@ -12,8 +13,11 @@ use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.register-obligation.v1')] final class RegisterObligationActivity extends Activity { public int $tries = 3; @@ -26,10 +30,9 @@ public function handle(array $identity, array $descriptor, string $principal): s $scope, $obligation ): string { - app(HandlerRegistry::class)->resolve($obligation); $repository = app(ScopeRepository::class); $repository->open($scope, $this->runId()); return $repository->register($scope, $obligation); - }); + }, beforeTransaction: static fn () => app(HandlerRegistry::class)->resolve($obligation)); } } diff --git a/src/V2/Compensation/Activities/RequestScopeActivity.php b/src/V2/Compensation/Activities/RequestScopeActivity.php index 859361d11..f357c89d2 100644 --- a/src/V2/Compensation/Activities/RequestScopeActivity.php +++ b/src/V2/Compensation/Activities/RequestScopeActivity.php @@ -5,13 +5,17 @@ namespace Workflow\V2\Compensation\Activities; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.request-scope.v1')] final class RequestScopeActivity extends Activity { public int $tries = 3; diff --git a/src/V2/Compensation/Activities/VerifyScopeActivity.php b/src/V2/Compensation/Activities/VerifyScopeActivity.php index 4ba956e36..4683b994b 100644 --- a/src/V2/Compensation/Activities/VerifyScopeActivity.php +++ b/src/V2/Compensation/Activities/VerifyScopeActivity.php @@ -5,13 +5,17 @@ namespace Workflow\V2\Compensation\Activities; use Workflow\V2\Activity; +use Workflow\V2\Attributes\Type; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Persistence\ScopeProgress; use Workflow\V2\Compensation\Security\ScopeBoundary; /** + * @internal Recorded compensation engine boundary. + * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ +#[Type('dw.compensation.verify-scope.v1')] final class VerifyScopeActivity extends Activity { public int $tries = 3; diff --git a/src/V2/Compensation/Definition/Obligation.php b/src/V2/Compensation/Definition/Obligation.php index 936c73331..d0630a337 100644 --- a/src/V2/Compensation/Definition/Obligation.php +++ b/src/V2/Compensation/Definition/Obligation.php @@ -34,7 +34,13 @@ public function fingerprint(): string public function toArray(): array { - return get_object_vars($this); + return [ + 'version' => 1, + 'key' => $this->key, + 'handlerType' => $this->handlerType, + 'handlerVersion' => $this->handlerVersion, + 'input' => $this->input, + ]; } public static function fromStored(array $row): self @@ -50,8 +56,8 @@ public static function fromArray(array $value): self { $keys = array_keys($value); sort($keys, SORT_STRING); - if ($keys !== ['handlerType', 'handlerVersion', 'input', 'key']) { - throw new InvalidArgumentException('Invalid compensation obligation shape.'); + if ($keys !== ['handlerType', 'handlerVersion', 'input', 'key', 'version'] || ($value['version'] ?? null) !== 1) { + throw new InvalidArgumentException('Unsupported compensation obligation format.'); } return new self($value['key'], $value['handlerType'], $value['handlerVersion'], $value['input']); } diff --git a/src/V2/Compensation/Definition/PortableDescriptor.php b/src/V2/Compensation/Definition/PortableDescriptor.php index 6dfe82a6c..83f28d91b 100644 --- a/src/V2/Compensation/Definition/PortableDescriptor.php +++ b/src/V2/Compensation/Definition/PortableDescriptor.php @@ -8,6 +8,8 @@ use JsonException; /** + * @internal Compensation format or engine implementation. + * * Canonical, bounded data only. This format never resolves classes or callables. * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. diff --git a/src/V2/Compensation/Definition/ScopeIdentity.php b/src/V2/Compensation/Definition/ScopeIdentity.php index f2ab19902..5eedbdbe1 100644 --- a/src/V2/Compensation/Definition/ScopeIdentity.php +++ b/src/V2/Compensation/Definition/ScopeIdentity.php @@ -4,6 +4,8 @@ namespace Workflow\V2\Compensation\Definition; +use InvalidArgumentException; + /** * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. */ @@ -15,7 +17,12 @@ public function __construct( public string $key, public string $generation, ) { - foreach (get_object_vars($this) as $name => $value) { + foreach ([ + 'application' => $application, + 'namespace' => $namespace, + 'key' => $key, + 'generation' => $generation, + ] as $name => $value) { if ($name === 'namespace' && $value === null) { continue; } @@ -35,17 +42,21 @@ public function recoveryInstanceId(): string public function toArray(): array { - return get_object_vars($this); + return [ + 'version' => 1, + 'application' => $this->application, + 'namespace' => $this->namespace, + 'key' => $this->key, + 'generation' => $this->generation, + ]; } public static function fromArray(array $value): self { - if (array_keys($value) !== ['application', 'namespace', 'key', 'generation']) { - $keys = array_keys($value); - sort($keys, SORT_STRING); - if ($keys !== ['application', 'generation', 'key', 'namespace']) { - throw new \InvalidArgumentException('Invalid compensation scope identity shape.'); - } + $keys = array_keys($value); + sort($keys, SORT_STRING); + if ($keys !== ['application', 'generation', 'key', 'namespace', 'version'] || ($value['version'] ?? null) !== 1) { + throw new InvalidArgumentException('Unsupported compensation scope identity format.'); } return new self($value['application'], $value['namespace'], $value['key'], $value['generation']); } diff --git a/src/V2/Compensation/DurableScope.php b/src/V2/Compensation/DurableScope.php index 6237a7f75..a4b2faba3 100644 --- a/src/V2/Compensation/DurableScope.php +++ b/src/V2/Compensation/DurableScope.php @@ -31,6 +31,7 @@ public function __construct( if (! WorkflowFiberContext::active()) { throw new \LogicException('DurableScope must be authored inside a native workflow.'); } + Workflow::getVersion('dw.compensation.scope-activities', 1, 1); } public function execute(Obligation $obligation, ?ActivityOptions $options = null): array diff --git a/src/V2/Compensation/Execution/EvidenceReferences.php b/src/V2/Compensation/Execution/EvidenceReferences.php index 05b8273ad..97344026a 100644 --- a/src/V2/Compensation/Execution/EvidenceReferences.php +++ b/src/V2/Compensation/Execution/EvidenceReferences.php @@ -8,6 +8,8 @@ use Workflow\V2\Compensation\Definition\PortableDescriptor; /** + * @internal Compensation format or engine implementation. + * * Leaves room for native identity metadata inside the unchanged bounded journal envelope. * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. diff --git a/src/V2/Compensation/Execution/ForwardResult.php b/src/V2/Compensation/Execution/ForwardResult.php index dae97b3dd..8de5d8327 100644 --- a/src/V2/Compensation/Execution/ForwardResult.php +++ b/src/V2/Compensation/Execution/ForwardResult.php @@ -38,6 +38,22 @@ public static function unknown(string $reason): self public function toArray(): array { - return get_object_vars($this); + return [ + 'version' => 1, + 'observed' => $this->observed, + 'value' => $this->value, + 'evidence' => $this->evidence, + 'reason' => $this->reason, + ]; + } + + public static function fromArray(array $value): self + { + $keys = array_keys($value); + sort($keys, SORT_STRING); + if ($keys !== ['evidence', 'observed', 'reason', 'value', 'version'] || ($value['version'] ?? null) !== 1) { + throw new InvalidArgumentException('Unsupported compensation forward-result format.'); + } + return new self($value['observed'], $value['value'], $value['evidence'], $value['reason']); } } diff --git a/src/V2/Compensation/Execution/Resolution.php b/src/V2/Compensation/Execution/Resolution.php index 9047faedd..7e8981acb 100644 --- a/src/V2/Compensation/Execution/Resolution.php +++ b/src/V2/Compensation/Execution/Resolution.php @@ -40,6 +40,22 @@ public static function unknown(string $reason): self public function toArray(): array { - return get_object_vars($this); + return [ + 'version' => 1, + 'verified' => $this->verified, + 'reason' => $this->reason, + 'evidence' => $this->evidence, + 'forwardSettled' => $this->forwardSettled, + ]; + } + + public static function fromArray(array $value): self + { + $keys = array_keys($value); + sort($keys, SORT_STRING); + if ($keys !== ['evidence', 'forwardSettled', 'reason', 'verified', 'version'] || ($value['version'] ?? null) !== 1) { + throw new InvalidArgumentException('Unsupported compensation resolution format.'); + } + return new self($value['verified'], $value['reason'], $value['evidence'], $value['forwardSettled']); } } diff --git a/src/V2/Compensation/Persistence/ScopeProgress.php b/src/V2/Compensation/Persistence/ScopeProgress.php index ca61b03d2..79155bb58 100644 --- a/src/V2/Compensation/Persistence/ScopeProgress.php +++ b/src/V2/Compensation/Persistence/ScopeProgress.php @@ -4,9 +4,9 @@ namespace Workflow\V2\Compensation\Persistence; -use Closure; use Illuminate\Database\Connection; use LogicException; +use Workflow\Exceptions\NonRetryableException; use Workflow\V2\Compensation\Definition\PortableDescriptor; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Execution\ForwardResult; @@ -19,34 +19,35 @@ */ final class ScopeProgress { - /** - * @param (Closure(array): void)|null $beforeDispatch Pure validation before recording dispatch uncertainty. - */ public function admitForward( ScopeIdentity $identity, string $id, string $attemptId, ?string $activityId = null, - ?Closure $beforeDispatch = null + ?string $validatedFingerprint = null ): array { return ScopeStorage::mutate($identity, function (array $scope, Connection $database) use ( $id, $attemptId, $activityId, - $beforeDispatch + $validatedFingerprint ): array { $row = $this->obligation($scope, $database, $id); if ($scope['status'] !== 'open') { throw new LogicException('The scope has closed forward admission.'); } + if ($validatedFingerprint !== null && ! hash_equals($row['fingerprint'], $validatedFingerprint)) { + throw new NonRetryableException('scope_forward_descriptor_changed'); + } if ($row['forward_status'] === 'observed') { return [ - 'cached' => json_decode($row['forward_result'], true, flags: JSON_THROW_ON_ERROR), + 'cached' => ForwardResult::fromArray( + json_decode($row['forward_result'], true, flags: JSON_THROW_ON_ERROR), + )->toArray(), 'uncertain' => false, 'row' => $row, ]; } - $beforeDispatch?->__invoke($row); $uncertain = $row['forward_status'] !== 'registered'; $database->table(ScopeStorage::OBLIGATIONS)->where('id', $id)->update([ 'forward_status' => 'in_flight', diff --git a/src/V2/Compensation/Persistence/ScopeRepository.php b/src/V2/Compensation/Persistence/ScopeRepository.php index 56da024d7..b7f02c61e 100644 --- a/src/V2/Compensation/Persistence/ScopeRepository.php +++ b/src/V2/Compensation/Persistence/ScopeRepository.php @@ -6,6 +6,7 @@ use Illuminate\Database\Connection; use LogicException; +use Workflow\Exceptions\NonRetryableException; use Workflow\V2\Compensation\Definition\Obligation; use Workflow\V2\Compensation\Definition\PortableDescriptor; use Workflow\V2\Compensation\Definition\ScopeIdentity; @@ -50,7 +51,7 @@ public function open(ScopeIdentity $identity, string $originalRunId): void ]) : 0; $scope = ScopeStorage::locked($identity); if ($scope['original_run_id'] !== $originalRunId) { - throw new LogicException('A compensation scope cannot change its original run.'); + throw new NonRetryableException('scope_execution_owner_mismatch'); } if ($created === 1) { ScopeStorage::append($scope, 'opened', [ @@ -182,6 +183,17 @@ public function metadata(ScopeIdentity $identity): array return ScopeStorage::mutate($identity, static fn (array $scope): array => $scope); } + public function obligation(ScopeIdentity $identity, string $id): array + { + return ScopeStorage::mutate($identity, static function (array $scope, Connection $database) use ($id): array { + $row = $database->table(self::OBLIGATIONS)->where('scope_id', $scope['id'])->where('id', $id)->first(); + if ($row === null) { + throw new NonRetryableException('scope_obligation_unavailable'); + } + return (array) $row; + }); + } + public function inspect(ScopeIdentity $identity): array { return ScopeStorage::mutate($identity, static function (array $scope, Connection $database): array { diff --git a/src/V2/Compensation/Recovery/ChildResolution.php b/src/V2/Compensation/Recovery/ChildResolution.php index 55fa1117c..b38d4c393 100644 --- a/src/V2/Compensation/Recovery/ChildResolution.php +++ b/src/V2/Compensation/Recovery/ChildResolution.php @@ -13,6 +13,8 @@ use Workflow\V2\Compensation\Security\ScopeAuthorization; /** + * @internal Compensation format or engine implementation. + * * Child request acknowledgement never substitutes for verified business compensation. * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. diff --git a/src/V2/Compensation/Recovery/RecoveryWorkflow.php b/src/V2/Compensation/Recovery/RecoveryWorkflow.php index d9025e8ba..4c02dd6cb 100644 --- a/src/V2/Compensation/Recovery/RecoveryWorkflow.php +++ b/src/V2/Compensation/Recovery/RecoveryWorkflow.php @@ -11,6 +11,8 @@ use Workflow\V2\Workflow; /** + * @internal Compensation format or engine implementation. + * * A normal native workflow with native Saga ordering and native activity retries. * * @experimental The Compensation API is not covered by stability guarantees and may change in a minor release. @@ -20,6 +22,7 @@ final class RecoveryWorkflow extends Workflow { public function handle(array $scope): array { + self::getVersion('dw.compensation.recovery-activities', 1, 1); $obligations = self::activity(PrepareRecoveryActivity::class, $scope); foreach ($obligations as $id) { $this->addCompensation(static fn () => self::activity(CompensateObligationActivity::class, $scope, $id)); diff --git a/src/V2/Compensation/Security/ScopeBoundary.php b/src/V2/Compensation/Security/ScopeBoundary.php index a9fddfd79..2605eedb3 100644 --- a/src/V2/Compensation/Security/ScopeBoundary.php +++ b/src/V2/Compensation/Security/ScopeBoundary.php @@ -12,6 +12,7 @@ use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Persistence\ScopeStorage; use Workflow\V2\Compensation\Recovery\RecoveryWorkflow; +use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\DefaultActivityTaskBridge; use Workflow\V2\Support\SqliteTransaction; @@ -33,40 +34,53 @@ public function run( ScopeIdentity $scope, string $operation, ?string $principal, - Closure $body + Closure $body, + ?Closure $beforeTransaction = null, ): mixed { if ($activity->taskId === null || $activity->attemptId() === null || $activity->run->namespace !== $scope->namespace) { throw new NonRetryableException('scope_native_identity_mismatch'); } + $request = null; + if ($operation !== 'register') { + $snapshot = $this->repository->metadata($scope); + $this->assertScopeOwner($activity, $scope, $operation, $snapshot); + if ($principal === null) { + $request = $this->repository->primaryRequest($scope); + $principal = $request['principal']; + } + } + if ($principal === null) { + throw new NonRetryableException('scope_principal_missing'); + } + $this->assertUnlockedHostContext(); + // Host authorization and input validation may perform I/O. Obtain the + // decision before the native attempt and ledger rows are locked. + try { + $policyVersion = $this->authorization->assertAllowed($scope, $principal, $operation); + PortableDescriptor::identifier($policyVersion, 'authorization policy version'); + } catch (\Throwable) { + throw new NonRetryableException('scope_authorization_refused'); + } + $this->assertUnlockedHostContext(); + $beforeTransaction?->__invoke(); + $this->assertUnlockedHostContext(); return SqliteTransaction::forTask($activity->taskId, function () use ( $activity, $scope, $operation, $principal, - $body + $body, + $request, + $policyVersion, ): mixed { $this->assertNativeAuthority($activity); $snapshot = null; if ($operation !== 'register') { $snapshot = $this->repository->metadata($scope); - $origin = $snapshot['original_run_id'] === $activity->runId(); - $recovery = $activity->workflowId() === $scope->recoveryInstanceId() - && $activity->run->workflow_class === RecoveryWorkflow::class; - if ((! $origin && ! $recovery) || ($operation === 'forward' && ! $origin)) { - throw new NonRetryableException('scope_execution_owner_mismatch'); + $this->assertScopeOwner($activity, $scope, $operation, $snapshot); + if ($request !== null && $request !== $this->repository->primaryRequest($scope)) { + throw new NonRetryableException('scope_authorization_context_changed'); } - if ($principal === null) { - $principal = $this->repository->primaryRequest($scope)['principal']; - } - } - if ($principal === null) { - throw new NonRetryableException('scope_principal_missing'); - } - try { - $policyVersion = $this->authorization->assertAllowed($scope, $principal, $operation); - PortableDescriptor::identifier($policyVersion, 'authorization policy version'); - } catch (\Throwable) { - throw new NonRetryableException('scope_authorization_refused'); } $snapshot = $operation === 'register' ? null : $this->repository->inspect($scope); $result = $body($snapshot, $policyVersion); @@ -84,6 +98,23 @@ public function run( }); } + private function assertScopeOwner(Activity $activity, ScopeIdentity $scope, string $operation, array $snapshot): void + { + $origin = $snapshot['original_run_id'] === $activity->runId(); + $recovery = $activity->workflowId() === $scope->recoveryInstanceId() + && $activity->run->workflow_class === RecoveryWorkflow::class; + if ((! $origin && ! $recovery) || ($operation === 'forward' && ! $origin)) { + throw new NonRetryableException('scope_execution_owner_mismatch'); + } + } + + private function assertUnlockedHostContext(): void + { + if (ConfiguredV2Models::connection()->transactionLevel() !== 0) { + throw new NonRetryableException('scope_host_preparation_requires_unlocked_context'); + } + } + private function assertNativeAuthority(Activity $activity): void { $status = app(DefaultActivityTaskBridge::class)->status($activity->attemptId()); diff --git a/tests/Feature/V2/V2CompensationForwardAdmissionTest.php b/tests/Feature/V2/V2CompensationForwardAdmissionTest.php index d9d2e4c28..d7f6cab8b 100644 --- a/tests/Feature/V2/V2CompensationForwardAdmissionTest.php +++ b/tests/Feature/V2/V2CompensationForwardAdmissionTest.php @@ -6,6 +6,8 @@ use Illuminate\Database\Events\QueryExecuted; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\DB; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\V2\TestForwardAdmissionWorkflow; use Tests\Fixtures\V2\TestScopedOperation; use Tests\TestCase; @@ -41,6 +43,14 @@ protected function setUp(): void $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string { + if (config('workflows.v2.testing.scope_host_probe') === 'authorization') { + $depths = Cache::get('test:scope:host-depths', []); + $depths[] = DB::transactionLevel(); + Cache::forever('test:scope:host-depths', $depths); + if (DB::transactionLevel() !== 0) { + throw new \LogicException('Host authorization was called under native SQL locks.'); + } + } if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { throw new \LogicException('Scope access refused.'); } @@ -55,6 +65,27 @@ protected function tearDown(): void parent::tearDown(); } + public static function hostPreparationKinds(): array + { + return [ + 'authorization' => ['authorization'], + 'input validation' => ['input'], + ]; + } + + #[DataProvider('hostPreparationKinds')] + public function testHostPreparationRunsWithoutNativeOrScopeLocks(string $kind): void + { + config()->set('workflows.v2.testing.scope_host_probe', $kind); + [$scope, $workflow] = $this->origin(); + $this->assertTrue($workflow->refresh()->completed()); + $depths = Cache::get('test:scope:host-depths', []); + $this->assertNotEmpty($depths); + $this->assertSame([0], array_values(array_unique($depths))); + $this->assertSame('observed', (new ScopeRepository())->inspect($scope)['obligations'][0]['forward_status']); + $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); + } + public function testMissingPinnedForwardHandlerIsNonRetryableBeforeDispatch(): void { $this->replaceAfterRegistration(new HandlerRegistry([])); @@ -180,6 +211,14 @@ public function __construct( public function assertInput(array $input): void { + if (config('workflows.v2.testing.scope_host_probe') === 'input') { + $depths = Cache::get('test:scope:host-depths', []); + $depths[] = DB::transactionLevel(); + Cache::forever('test:scope:host-depths', $depths); + if (DB::transactionLevel() !== 0) { + throw new \LogicException('Host input validation was called under native SQL locks.'); + } + } if (! $this->compatible) { throw new \InvalidArgumentException('The input schema changed after registration.'); } diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 3e801809a..08fbbef20 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -170,7 +170,7 @@ public function testOriginRedriveCannotReissueAnUncertainForwardEffectFromTheOri private function assertForwardRefused(string $runId, ScopeIdentity $scope, int $effectCount): void { $messages = WorkflowFailure::query()->where('workflow_run_id', $runId)->pluck('message')->all(); - $this->assertStringContainsString('cannot change its original run', implode('\n', $messages)); + $this->assertStringContainsString('scope_execution_owner_mismatch', implode('\n', $messages)); $snapshot = (new ScopeRepository())->inspect($scope); $this->assertSame('open', $snapshot['status']); $this->assertNotContains('second', array_column(array_column($snapshot['obligations'], 'descriptor'), 'key')); diff --git a/tests/Feature/V2/V2CompensationScopeTest.php b/tests/Feature/V2/V2CompensationScopeTest.php index a08833904..8477a47c4 100644 --- a/tests/Feature/V2/V2CompensationScopeTest.php +++ b/tests/Feature/V2/V2CompensationScopeTest.php @@ -62,6 +62,16 @@ public function testClosedOriginUsesCanonicalRecoveryAndKeepsOriginalTerminalHis $this->assertSame('completed', $origin->refresh()->status()); $this->assertSame($before, WorkflowHistoryEvent::query()->where('workflow_run_id', $origin->runId())->count()); $this->assertSame(['second', 'first'], Cache::get('test:scope:undo-log')); + foreach ([ + $origin->runId() => 'dw.compensation.scope-activities', + $result['runId'] => 'dw.compensation.recovery-activities', + ] as $runId => $changeId) { + $markers = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', \Workflow\V2\Enums\HistoryEventType::VersionMarkerRecorded)->get(); + $this->assertContains([$changeId, 1], $markers->map(static fn (WorkflowHistoryEvent $event): array => [ + $event->payload['change_id'], $event->payload['version'], + ])->all()); + } } public function testNativeSagaCompensatesAnEffectWhoseForwardResultWasLost(): void diff --git a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php index 1ae31a2a6..b4edac25b 100644 --- a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php +++ b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php @@ -5,14 +5,92 @@ namespace Tests\Unit\V2\Compensation; use InvalidArgumentException; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use ReflectionClass; +use Workflow\V2\Attributes\Type; +use Workflow\V2\Compensation\Activities\CompensateObligationActivity; +use Workflow\V2\Compensation\Activities\ForwardOperationActivity; +use Workflow\V2\Compensation\Activities\PrepareRecoveryActivity; +use Workflow\V2\Compensation\Activities\RegisterChildScopeActivity; +use Workflow\V2\Compensation\Activities\RegisterObligationActivity; +use Workflow\V2\Compensation\Activities\RequestScopeActivity; +use Workflow\V2\Compensation\Activities\VerifyScopeActivity; use Workflow\V2\Compensation\Definition\Obligation; +use Workflow\V2\Compensation\Definition\PortableDescriptor; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Execution\ForwardResult; use Workflow\V2\Compensation\Execution\Resolution; final class ScopeDescriptorTest extends TestCase { + public function testRecordedActivityTypesAreIndependentOfPhpClassNames(): void + { + foreach ([ + CompensateObligationActivity::class => 'dw.compensation.compensate-obligation.v1', + ForwardOperationActivity::class => 'dw.compensation.forward-operation.v1', + PrepareRecoveryActivity::class => 'dw.compensation.prepare-recovery.v1', + RegisterChildScopeActivity::class => 'dw.compensation.register-child-scope.v1', + RegisterObligationActivity::class => 'dw.compensation.register-obligation.v1', + RequestScopeActivity::class => 'dw.compensation.request-scope.v1', + VerifyScopeActivity::class => 'dw.compensation.verify-scope.v1', + ] as $class => $expected) { + $attributes = (new ReflectionClass($class))->getAttributes(Type::class); + $this->assertCount(1, $attributes); + $this->assertSame($expected, $attributes[0]->newInstance()->key); + } + } + + public function testVersionOnePersistedFormatsHaveStableCanonicalIdentities(): void + { + $scope = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + $obligation = new Obligation('book', 'carrier.cancel', '1', [ + 'reference' => 'booking-1', + ]); + $this->assertSame('ceb73af611903fb425f8bcdb3b415d52be6f619cc5f1f8f6c3c3996aea4c5a8d', $scope->id()); + $this->assertSame('43a5906816309c817fafc95d183ade5b5786123115a765b85b13b12440bcfa13', $obligation->fingerprint()); + $this->assertSame($scope->id(), ScopeIdentity::fromArray($scope->toArray())->id()); + $this->assertSame($obligation->fingerprint(), Obligation::fromArray($obligation->toArray())->fingerprint()); + $this->assertSame('df14bcf01709ef85b55c7fc5770dcd7f4f0690c6a1ed4dcae092ee91df669396', hash( + 'sha256', + PortableDescriptor::encode(ForwardResult::observed([ + 'reference' => 'booking-1', + ], ['provider:proof'])->toArray()), + )); + $this->assertSame('8e9b39641e72e2dd9b9f16c457c2d988f619e4fe17fae0586ed72f2d1ec92790', hash( + 'sha256', + PortableDescriptor::encode(Resolution::verified(['provider:proof'], true)->toArray()), + )); + } + + public static function unsupportedVersions(): iterable + { + foreach ([null, '1', true, 1.0, 2] as $index => $version) { + foreach (['scope', 'obligation', 'forward', 'resolution'] as $kind) { + yield $kind . '-' . $index => [$kind, $version]; + } + } + } + + #[DataProvider('unsupportedVersions')] + public function testUnsupportedPersistedVersionsRefuseInterpretation(string $kind, mixed $version): void + { + $payload = match ($kind) { + 'scope' => (new ScopeIdentity('shop', 'client-a', 'order-42', '1'))->toArray(), + 'obligation' => (new Obligation('book', 'carrier.cancel', '1', []))->toArray(), + 'forward' => ForwardResult::unknown('unknown')->toArray(), + 'resolution' => Resolution::unknown('unknown')->toArray(), + }; + $payload['version'] = $version; + $this->expectException(InvalidArgumentException::class); + match ($kind) { + 'scope' => ScopeIdentity::fromArray($payload), + 'obligation' => Obligation::fromArray($payload), + 'forward' => ForwardResult::fromArray($payload), + 'resolution' => Resolution::fromArray($payload), + }; + } + public function testScopeIdentityIncludesApplicationNamespaceAndGeneration(): void { $original = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); diff --git a/tests/Unit/V2/Compensation/ScopeRepositoryTest.php b/tests/Unit/V2/Compensation/ScopeRepositoryTest.php index f30e3e801..8ccf53ce4 100644 --- a/tests/Unit/V2/Compensation/ScopeRepositoryTest.php +++ b/tests/Unit/V2/Compensation/ScopeRepositoryTest.php @@ -7,6 +7,7 @@ use Fiber; use LogicException; use Tests\TestCase; +use Workflow\Exceptions\NonRetryableException; use Workflow\V2\Compensation\Definition\Obligation; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Persistence\ScopeRepository; @@ -77,7 +78,8 @@ public function testOriginalRunCannotBeSilentlyReplaced(): void { $repository = new ScopeRepository(); $repository->open($this->scope(), 'original-run'); - $this->expectException(LogicException::class); + $this->expectException(NonRetryableException::class); + $this->expectExceptionMessage('scope_execution_owner_mismatch'); $repository->open($this->scope(), 'replacement-run'); } From bc1d9fda08186e19545ea68099385978317d5ebe Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:50:39 +0000 Subject: [PATCH 07/32] Cover originating scope ownership across a native child retry --- .../V2/V2CompensationScopeRunBoundaryTest.php | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 08fbbef20..076046ba6 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -10,6 +10,7 @@ use Tests\Fixtures\V2\TestScopedOperation; use Tests\Fixtures\V2\TestScopeRunBoundaryWorkflow; use Tests\TestCase; +use Workflow\Serializers\Serializer; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Execution\HandlerRegistry; use Workflow\V2\Compensation\Persistence\ScopeRepository; @@ -29,6 +30,7 @@ use Workflow\V2\Models\WorkflowLink; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\WorkflowReplayer; use Workflow\V2\WorkflowStub; @@ -167,6 +169,55 @@ public function testOriginRedriveCannotReissueAnUncertainForwardEffectFromTheOri $this->assertSame($originHistory, $this->history($origin->id)); } + public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): void + { + config()->set('workflows.v2.types.workflows', [ + 'test.scope-run-boundary' => TestScopeRunBoundaryWorkflow::class, + ]); + $scope = new ScopeIdentity('shop', 'client-a', 'retried-child-order', '1'); + $parent = app(WorkflowControlPlane::class)->start('test.remote-parent', 'scope-retry-parent', [ + 'namespace' => 'client-a', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + ]); + $this->assertTrue($parent['started']); + $parentRunId = $parent['workflow_run_id']; + $task = WorkflowTask::query()->where('workflow_run_id', $parentRunId)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'scope-parent-worker', + 'lease_expires_at' => now()->addMinute(), + ])->save(); + $result = app(DefaultWorkflowTaskBridge::class)->complete($task->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'test.scope-run-boundary', + 'arguments' => Serializer::serializeWithCodec('avro', [$scope->toArray(), 'redrive']), + 'payload_codec' => 'avro', + 'retry_policy' => [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ], + ]]); + $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $this->drainTasks($parentRunId); + $links = WorkflowLink::query()->where('parent_workflow_run_id', $parentRunId) + ->where('link_type', 'child_workflow')->orderBy('created_at')->get(); + $this->assertCount(2, $links); + $origin = WorkflowRun::query()->findOrFail($links[0]->child_workflow_run_id); + $retry = WorkflowRun::query()->findOrFail($links[1]->child_workflow_run_id); + $this->assertSame($origin->workflow_instance_id, $retry->workflow_instance_id); + $this->assertSame(2, $retry->run_number); + $this->assertSame(RunStatus::Failed, $retry->status); + $this->assertForwardRefused($retry->id, $scope, 1); + $this->assertSame($origin->id, (new ScopeRepository())->inspect($scope)['original_run_id']); + $history = $this->history($origin->id); + app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $this->drainTasks($parentRunId); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(['retried-child-order/first'], Cache::get('test:scope:undo-scoped-log')); + $this->assertSame($history, $this->history($origin->id)); + } + private function assertForwardRefused(string $runId, ScopeIdentity $scope, int $effectCount): void { $messages = WorkflowFailure::query()->where('workflow_run_id', $runId)->pluck('message')->all(); @@ -188,10 +239,14 @@ private function history(string $runId): array ])->all(); } - private function drainTasks(): void + private function drainTasks(?string $excludedRunId = null): void { for ($count = 0; $count < 300; ++$count) { - $task = WorkflowTask::query()->where('status', TaskStatus::Ready->value)->orderBy('created_at')->first(); + $tasks = WorkflowTask::query()->where('status', TaskStatus::Ready->value)->orderBy('created_at'); + if ($excludedRunId !== null) { + $tasks->where('workflow_run_id', '!=', $excludedRunId); + } + $task = $tasks->first(); if ($task === null) { return; } From d55e269217c24cbaad21fc539f3dbf5b8f3b4d51 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:58:35 +0000 Subject: [PATCH 08/32] Declare compensation fixture namespaces and use injected job dependencies --- .../V2/V2CompensationForwardAdmissionTest.php | 15 ++++++++++++--- .../V2/V2CompensationScopeConcurrencyTest.php | 1 + .../V2/V2CompensationScopeRunBoundaryTest.php | 11 ++++++++++- tests/Feature/V2/V2CompensationScopeTest.php | 1 + .../CompensationScopeRecoveryProcessTest.php | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/Feature/V2/V2CompensationForwardAdmissionTest.php b/tests/Feature/V2/V2CompensationForwardAdmissionTest.php index d7f6cab8b..e933991fe 100644 --- a/tests/Feature/V2/V2CompensationForwardAdmissionTest.php +++ b/tests/Feature/V2/V2CompensationForwardAdmissionTest.php @@ -36,6 +36,7 @@ protected function setUp(): void { parent::setUp(); config([ + 'workflows.v2.namespaces' => ['client-a'], 'queue.default' => 'sync', 'queue.connections.sync.driver' => 'sync', ]); @@ -78,7 +79,15 @@ public function testHostPreparationRunsWithoutNativeOrScopeLocks(string $kind): { config()->set('workflows.v2.testing.scope_host_probe', $kind); [$scope, $workflow] = $this->origin(); - $this->assertTrue($workflow->refresh()->completed()); + $this->assertTrue($workflow->refresh()->completed(), json_encode( + WorkflowTask::query()->where('workflow_run_id', $workflow->runId())->limit(3)->get() + ->map(static fn (WorkflowTask $task): array => [ + 'status' => $task->status->value, + 'error' => $task->last_error, + 'claim_error' => $task->last_claim_error, + ])->all(), + JSON_THROW_ON_ERROR, + )); $depths = Cache::get('test:scope:host-depths', []); $this->assertNotEmpty($depths); $this->assertSame([0], array_values(array_unique($depths))); @@ -265,9 +274,9 @@ private function driveNativeRetries(string $runId): void \Illuminate\Support\Carbon::setTestNow($task->available_at->copy()->addMicrosecond()); } if ($task->task_type->value === 'activity') { - (new \Workflow\V2\Jobs\RunActivityTask($task->id))->handle(); + $this->app->call([new \Workflow\V2\Jobs\RunActivityTask($task->id), 'handle']); } else { - (new \Workflow\V2\Jobs\RunWorkflowTask($task->id))->handle(); + $this->app->call([new \Workflow\V2\Jobs\RunWorkflowTask($task->id), 'handle']); } } $this->fail('Native forward retries did not terminate.'); diff --git a/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php b/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php index 29e38524a..cae05c4d4 100644 --- a/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php +++ b/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php @@ -32,6 +32,7 @@ protected function setUp(): void self::stopWorkers(); Queue::fake(); config([ + 'workflows.v2.namespaces' => ['client-a'], 'workflows.v2.task_dispatch_mode' => 'poll', ]); $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 076046ba6..47993ada3 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -40,6 +40,7 @@ protected function setUp(): void { parent::setUp(); config([ + 'workflows.v2.namespaces' => ['client-a'], 'queue.default' => 'redis', 'queue.connections.redis.driver' => 'redis', ]); @@ -260,6 +261,14 @@ private function drainTasks(?string $excludedRunId = null): void }; $this->app->call([$job, 'handle']); } - $this->fail('Ready tasks did not settle within the bounded drain.'); + $this->fail('Ready tasks did not settle within the bounded drain: ' . json_encode( + WorkflowTask::query()->where('status', TaskStatus::Ready)->limit(3)->get() + ->map(static fn (WorkflowTask $task): array => [ + 'status' => $task->status->value, + 'error' => $task->last_error, + 'claim_error' => $task->last_claim_error, + ])->all(), + JSON_THROW_ON_ERROR, + )); } } diff --git a/tests/Feature/V2/V2CompensationScopeTest.php b/tests/Feature/V2/V2CompensationScopeTest.php index 8477a47c4..cedb73be1 100644 --- a/tests/Feature/V2/V2CompensationScopeTest.php +++ b/tests/Feature/V2/V2CompensationScopeTest.php @@ -27,6 +27,7 @@ protected function setUp(): void { parent::setUp(); config([ + 'workflows.v2.namespaces' => ['client-a'], 'queue.default' => 'sync', 'queue.connections.sync.driver' => 'sync', ]); diff --git a/tests/Support/CompensationScopeRecoveryProcessTest.php b/tests/Support/CompensationScopeRecoveryProcessTest.php index 54d27a4cd..eb51d0a86 100644 --- a/tests/Support/CompensationScopeRecoveryProcessTest.php +++ b/tests/Support/CompensationScopeRecoveryProcessTest.php @@ -60,6 +60,7 @@ protected function defineEnvironment($app): void { $input = $this->input(); $app['config']->set([ + 'workflows.v2.namespaces' => ['client-a'], 'database.default' => 'scope_process', 'database.connections.scope_process' => $input['database'], 'database.redis' => $input['redis'], From e357b89eae0444d0a49f8addd09cd5fbc24c821f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:18:29 +0000 Subject: [PATCH 09/32] Fix Saga retry and default namespace fixtures --- src/V2/Compensation/Definition/Obligation.php | 8 +++++++- src/V2/Compensation/Definition/ScopeIdentity.php | 8 +++++++- src/V2/Compensation/Execution/Resolution.php | 8 +++++++- src/V2/Compensation/Security/ScopeBoundary.php | 8 ++++++-- tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php | 6 +++++- tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php | 9 ++++++--- tests/Feature/V2/V2CompensationScopeTest.php | 1 + tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php | 3 +++ tests/Unit/V2/Compensation/ScopeDescriptorTest.php | 5 ++++- 9 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/V2/Compensation/Definition/Obligation.php b/src/V2/Compensation/Definition/Obligation.php index d0630a337..75e5915ad 100644 --- a/src/V2/Compensation/Definition/Obligation.php +++ b/src/V2/Compensation/Definition/Obligation.php @@ -56,7 +56,13 @@ public static function fromArray(array $value): self { $keys = array_keys($value); sort($keys, SORT_STRING); - if ($keys !== ['handlerType', 'handlerVersion', 'input', 'key', 'version'] || ($value['version'] ?? null) !== 1) { + if ($keys !== [ + 'handlerType', + 'handlerVersion', + 'input', + 'key', + 'version', + ] || ($value['version'] ?? null) !== 1) { throw new InvalidArgumentException('Unsupported compensation obligation format.'); } return new self($value['key'], $value['handlerType'], $value['handlerVersion'], $value['input']); diff --git a/src/V2/Compensation/Definition/ScopeIdentity.php b/src/V2/Compensation/Definition/ScopeIdentity.php index 5eedbdbe1..9f1ef4ba9 100644 --- a/src/V2/Compensation/Definition/ScopeIdentity.php +++ b/src/V2/Compensation/Definition/ScopeIdentity.php @@ -55,7 +55,13 @@ public static function fromArray(array $value): self { $keys = array_keys($value); sort($keys, SORT_STRING); - if ($keys !== ['application', 'generation', 'key', 'namespace', 'version'] || ($value['version'] ?? null) !== 1) { + if ($keys !== [ + 'application', + 'generation', + 'key', + 'namespace', + 'version', + ] || ($value['version'] ?? null) !== 1) { throw new InvalidArgumentException('Unsupported compensation scope identity format.'); } return new self($value['application'], $value['namespace'], $value['key'], $value['generation']); diff --git a/src/V2/Compensation/Execution/Resolution.php b/src/V2/Compensation/Execution/Resolution.php index 7e8981acb..35f8ba262 100644 --- a/src/V2/Compensation/Execution/Resolution.php +++ b/src/V2/Compensation/Execution/Resolution.php @@ -53,7 +53,13 @@ public static function fromArray(array $value): self { $keys = array_keys($value); sort($keys, SORT_STRING); - if ($keys !== ['evidence', 'forwardSettled', 'reason', 'verified', 'version'] || ($value['version'] ?? null) !== 1) { + if ($keys !== [ + 'evidence', + 'forwardSettled', + 'reason', + 'verified', + 'version', + ] || ($value['version'] ?? null) !== 1) { throw new InvalidArgumentException('Unsupported compensation resolution format.'); } return new self($value['verified'], $value['reason'], $value['evidence'], $value['forwardSettled']); diff --git a/src/V2/Compensation/Security/ScopeBoundary.php b/src/V2/Compensation/Security/ScopeBoundary.php index 2605eedb3..2ca64a7ea 100644 --- a/src/V2/Compensation/Security/ScopeBoundary.php +++ b/src/V2/Compensation/Security/ScopeBoundary.php @@ -98,8 +98,12 @@ public function run( }); } - private function assertScopeOwner(Activity $activity, ScopeIdentity $scope, string $operation, array $snapshot): void - { + private function assertScopeOwner( + Activity $activity, + ScopeIdentity $scope, + string $operation, + array $snapshot + ): void { $origin = $snapshot['original_run_id'] === $activity->runId(); $recovery = $activity->workflowId() === $scope->recoveryInstanceId() && $activity->run->workflow_class === RecoveryWorkflow::class; diff --git a/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php b/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php index 653cd05f7..9566ab718 100644 --- a/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php +++ b/tests/Feature/V2/V2CompensationInnoDbConcurrencyTest.php @@ -133,7 +133,11 @@ private function raceAtExclusiveLookup(string $table, Closure $operation): array touch($root . '/release'); foreach ($children as $index => $pid) { pcntl_waitpid($pid, $status); - $payload = json_decode(file_get_contents($root . '/' . $index . '.json'), true, flags: JSON_THROW_ON_ERROR); + $payload = json_decode( + file_get_contents($root . '/' . $index . '.json'), + true, + flags: JSON_THROW_ON_ERROR + ); $this->assertSame(0, pcntl_wexitstatus($status), $payload['error'] ?? 'Scope caller failed.'); $results[] = $payload['result']; } diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 47993ada3..6a11f2942 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -187,12 +187,13 @@ public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): $task->forceFill([ 'status' => TaskStatus::Leased, 'lease_owner' => 'scope-parent-worker', - 'lease_expires_at' => now()->addMinute(), + 'lease_expires_at' => now() + ->addMinute(), ])->save(); $result = app(DefaultWorkflowTaskBridge::class)->complete($task->id, [[ 'type' => 'start_child_workflow', 'workflow_type' => 'test.scope-run-boundary', - 'arguments' => Serializer::serializeWithCodec('avro', [$scope->toArray(), 'redrive']), + 'arguments' => Serializer::serializeWithCodec('avro', [$scope->toArray(), 'child-retry']), 'payload_codec' => 'avro', 'retry_policy' => [ 'max_attempts' => 2, @@ -202,7 +203,9 @@ public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); $this->drainTasks($parentRunId); $links = WorkflowLink::query()->where('parent_workflow_run_id', $parentRunId) - ->where('link_type', 'child_workflow')->orderBy('created_at')->get(); + ->where('link_type', 'child_workflow') + ->orderBy('created_at') + ->get(); $this->assertCount(2, $links); $origin = WorkflowRun::query()->findOrFail($links[0]->child_workflow_run_id); $retry = WorkflowRun::query()->findOrFail($links[1]->child_workflow_run_id); diff --git a/tests/Feature/V2/V2CompensationScopeTest.php b/tests/Feature/V2/V2CompensationScopeTest.php index cedb73be1..f1ac5c88f 100644 --- a/tests/Feature/V2/V2CompensationScopeTest.php +++ b/tests/Feature/V2/V2CompensationScopeTest.php @@ -423,6 +423,7 @@ public function testReleaseRefusesUnresolvedWorkThenRetainsAnUnreusableTombstone public function testDefaultEmbeddedNamespaceIsPreservedWithoutInventingAnAlias(): void { + config()->set('workflows.v2.namespaces', [null]); $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string { diff --git a/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php b/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php index a74b22812..7eaee0f71 100644 --- a/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php +++ b/tests/Fixtures/V2/TestScopeRunBoundaryWorkflow.php @@ -38,6 +38,9 @@ public function handle(array $identity, string $mode, int $generation = 0, ?arra } return self::continueAsNew($identity, $mode, 1, $childIdentity); } + if ($mode === 'child-retry') { + throw new \RuntimeException('Retry the child after its first forward effect.'); + } self::activity(TestRedriveSecondActivity::class, new ActivityOptions(maxAttempts: 1), 'boundary'); } else { signal('resume'); diff --git a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php index b4edac25b..980d4350c 100644 --- a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php +++ b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php @@ -48,7 +48,10 @@ public function testVersionOnePersistedFormatsHaveStableCanonicalIdentities(): v 'reference' => 'booking-1', ]); $this->assertSame('ceb73af611903fb425f8bcdb3b415d52be6f619cc5f1f8f6c3c3996aea4c5a8d', $scope->id()); - $this->assertSame('43a5906816309c817fafc95d183ade5b5786123115a765b85b13b12440bcfa13', $obligation->fingerprint()); + $this->assertSame( + '43a5906816309c817fafc95d183ade5b5786123115a765b85b13b12440bcfa13', + $obligation->fingerprint() + ); $this->assertSame($scope->id(), ScopeIdentity::fromArray($scope->toArray())->id()); $this->assertSame($obligation->fingerprint(), Obligation::fromArray($obligation->toArray())->fingerprint()); $this->assertSame('df14bcf01709ef85b55c7fc5770dcd7f4f0690c6a1ed4dcae092ee91df669396', hash( From fb53a83cbc186e97fd8b654c8cc3e51b077ae221 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:26:08 +0000 Subject: [PATCH 10/32] Add fenced admission for stopped compensation recovery --- .../Recovery/RecoveryRunAdmission.php | 307 +++++++++++++ .../Support/DefaultWorkflowControlPlane.php | 8 + .../Support/HistoryEventPayloadContract.php | 1 + src/V2/WorkflowStub.php | 176 ++++++-- tests/Unit/V2/RecoveryRunAdmissionTest.php | 414 ++++++++++++++++++ 5 files changed, 875 insertions(+), 31 deletions(-) create mode 100644 src/V2/Compensation/Recovery/RecoveryRunAdmission.php create mode 100644 tests/Unit/V2/RecoveryRunAdmissionTest.php diff --git a/src/V2/Compensation/Recovery/RecoveryRunAdmission.php b/src/V2/Compensation/Recovery/RecoveryRunAdmission.php new file mode 100644 index 000000000..57b7fbb40 --- /dev/null +++ b/src/V2/Compensation/Recovery/RecoveryRunAdmission.php @@ -0,0 +1,307 @@ +profile = $profile ?? StartOptions::returnExistingActive(); + } + + public function token(): string + { + // The predecessor is receipt content, never part of request identity. + return hash('sha256', PortableDescriptor::encode([ + 'compensation_recovery_v1', $this->scope->application, $this->scope->namespace, + $this->scope->recoveryInstanceId(), $this->requestKey, + ])); + } + + public function intent( + WorkflowInstance $instance, + string $workflowClass, + ?string $connection, + ?string $queue + ): array { + if ($instance->id !== $this->scope->recoveryInstanceId() + || $instance->namespace !== $this->scope->namespace + || $workflowClass !== RecoveryWorkflow::class + || $instance->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class)) { + throw new LogicException('Recovery admission requires the exact canonical scope and registered type.'); + } + $origin = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($this->originalRunId); + if (! $origin instanceof WorkflowRun || $origin->namespace !== $instance->namespace || ! $origin->status->isTerminal()) { + throw new LogicException('Recovery admission requires a terminal original run in the exact namespace.'); + } + $definition = WorkflowDefinition::fingerprint($workflowClass); + if ($definition === null) { + throw new LogicException('Recovery admission requires a registered definition fingerprint.'); + } + $profile = [ + 'connection' => $connection, + 'queue' => $queue, + 'compatibility' => WorkerCompatibility::current(), + 'business_key' => $this->profile->businessKey, + 'labels' => $this->profile->labels, + 'memo' => $this->profile->memo, + 'search_attributes' => $this->profile->searchAttributes, + 'execution_timeout_seconds' => $this->profile->executionTimeoutSeconds, + 'run_timeout_seconds' => $this->profile->runTimeoutSeconds, + 'timeout_cleanup_seconds' => $this->profile->timeoutCleanupSeconds, + 'priority' => $this->profile->priority, + 'fairness_key' => $this->profile->fairnessKey, + 'fairness_weight' => $this->profile->fairnessWeight, + ]; + $binding = [ + 'scope' => $this->scope->toArray(), + 'original_run_id' => $this->originalRunId, + 'workflow_class' => $workflowClass, + 'workflow_type' => $instance->workflow_type, + 'definition' => $definition, + 'arguments' => [$this->scope->toArray()], + 'profile' => $profile, + ]; + return [ + 'version' => 1, + 'binding' => $binding, + 'binding_fingerprint' => hash('sha256', PortableDescriptor::encode($binding)), + 'expected_predecessor_run_id' => $this->expectedPredecessorRunId, + ]; + } + + public function retained(WorkflowInstance $instance, array $intent): ?WorkflowCommand + { + $command = ConfiguredV2Models::query('command_model', WorkflowCommand::class) + ->where('workflow_instance_id', $instance->id) + ->where('request_id', $this->token()) + ->lockForUpdate() + ->first(); + if (! $command instanceof WorkflowCommand) { + return null; + } + $receipt = $command->context['compensation_recovery'] ?? null; + if ($command->command_type !== CommandType::Start || $command->status !== CommandStatus::Accepted + || ! is_array($receipt) || ! is_array($receipt['intent'] ?? null) + || PortableDescriptor::encode($receipt['intent']) !== PortableDescriptor::encode($intent)) { + throw new LogicException('Recovery request identity was reused with different admission content.'); + } + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($command->workflow_run_id); + $this->assertRun($instance, $run, $receipt); + return $command; + } + + /** + * Return the creation receipt only when the exact current run can be adopted + * or succeeded. No native rows are changed by a refusal. + */ + public function predecessor(WorkflowInstance $instance, ?WorkflowRun $run, array $intent): ?array + { + if ($run === null) { + if ($this->expectedPredecessorRunId !== null) { + throw new LogicException('Recovery predecessor is no longer current.'); + } + return null; + } + $receipt = self::creationReceipt($run); + if (! is_array($receipt) + || ($receipt['intent']['binding_fingerprint'] ?? null) !== $intent['binding_fingerprint']) { + throw new LogicException('Recovery run has no matching native admission lineage.'); + } + $this->assertRun($instance, $run, $receipt); + if (! $run->status->isTerminal()) { + if (PortableDescriptor::encode($receipt['intent']) !== PortableDescriptor::encode($intent)) { + throw new LogicException('Active recovery is not the successor of the requested predecessor.'); + } + return $receipt; + } + if ($run->id !== $this->expectedPredecessorRunId + || ! in_array($run->status, [RunStatus::Cancelled, RunStatus::Terminated], true)) { + throw new LogicException('Only the exact current cancelled or terminated recovery may restart.'); + } + foreach ([$run->execution_deadline_at, $run->run_deadline_at] as $deadline) { + if ($deadline !== null && $deadline->lessThanOrEqualTo(now())) { + throw new LogicException('Recovery restart cannot renew an exhausted deadline.'); + } + } + return $receipt; + } + + public function context( + CommandContext $context, + array $intent, + ?WorkflowRun $predecessor, + ?array $creationReceipt = null + ): CommandContext { + return $context->with([ + 'compensation_recovery' => [ + 'version' => 1, + 'operation' => 'start', + 'lineage_operation' => $creationReceipt['lineage_operation'] ?? 'start', + 'request_key' => $this->requestKey, + 'intent' => $intent, + 'execution_deadline_at' => $predecessor?->execution_deadline_at?->format('Y-m-d H:i:s.u'), + 'run_deadline_at' => $predecessor?->run_deadline_at?->format('Y-m-d H:i:s.u'), + ], + ]); + } + + public function bind(WorkflowCommand $command, WorkflowRun $run): void + { + $context = $command->context; + $context['compensation_recovery']['execution_deadline_at'] = $run->execution_deadline_at?->format( + 'Y-m-d H:i:s.u' + ); + $context['compensation_recovery']['run_deadline_at'] = $run->run_deadline_at?->format('Y-m-d H:i:s.u'); + $context['compensation_recovery']['timeout_cleanup_work_deadline_at'] = $run->timeout_cleanup_work_deadline_at?->format( + 'Y-m-d H:i:s.u' + ); + $command->forceFill([ + 'request_id' => $this->token(), + 'context' => $context, + ])->save(); + } + + public function link(WorkflowRun $predecessor, WorkflowRun $successor): WorkflowLink + { + return ConfiguredV2Models::query('link_model', WorkflowLink::class)->create([ + 'link_type' => 'compensation_recovery_restart', + 'parent_workflow_run_id' => $predecessor->id, + 'parent_workflow_instance_id' => $predecessor->workflow_instance_id, + 'child_workflow_run_id' => $successor->id, + 'child_workflow_instance_id' => $successor->workflow_instance_id, + ]); + } + + /** + * Carry the binding through native failed-run redrive. Its recorded technical + * allowance is not renewed by a later cancelled/terminated restart. + */ + public static function carryToRedrive( + WorkflowInstance $instance, + WorkflowRun $source, + WorkflowRun $successor, + WorkflowCommand $command, + ): ?array { + $receipt = self::creationReceipt($source); + if ($receipt === null) { + return null; + } + self::assertRun($instance, $source, $receipt); + $receipt['operation'] = 'redrive'; + $receipt['lineage_operation'] = 'redrive'; + $receipt['request_key'] = $command->request_id ?? $command->id; + $receipt['intent']['expected_predecessor_run_id'] = $source->id; + $receipt['execution_deadline_at'] = $successor->execution_deadline_at?->format('Y-m-d H:i:s.u'); + $receipt['run_deadline_at'] = $successor->run_deadline_at?->format('Y-m-d H:i:s.u'); + $receipt['timeout_cleanup_work_deadline_at'] = $successor->timeout_cleanup_work_deadline_at?->format( + 'Y-m-d H:i:s.u' + ); + $context = $command->context; + $context['compensation_recovery'] = $receipt; + $command->forceFill([ + 'context' => $context, + ])->save(); + return $receipt; + } + + private static function creationReceipt(WorkflowRun $run): ?array + { + $creation = ConfiguredV2Models::query('command_model', WorkflowCommand::class) + ->where('workflow_run_id', $run->id) + ->where('status', CommandStatus::Accepted->value) + ->whereIn('command_type', [CommandType::Start->value, CommandType::Redrive->value]) + ->whereIn('outcome', [CommandOutcome::StartedNew->value, CommandOutcome::Redriven->value]) + ->first(); + $receipt = $creation?->context['compensation_recovery'] ?? null; + return is_array($receipt) ? $receipt : null; + } + + private static function assertRun(WorkflowInstance $instance, mixed $run, array $receipt): void + { + $binding = $receipt['intent']['binding'] ?? null; + $fingerprint = $receipt['intent']['binding_fingerprint'] ?? null; + if (! is_array($binding) || ! is_string($fingerprint) + || ($receipt['version'] ?? null) !== 1 || ($receipt['intent']['version'] ?? null) !== 1 + || ! in_array($receipt['lineage_operation'] ?? null, ['start', 'redrive'], true) + || ! hash_equals(hash('sha256', PortableDescriptor::encode($binding)), $fingerprint)) { + throw new LogicException('Recovery receipt binding is invalid or has an unsupported version.'); + } + $profile = $binding['profile'] ?? []; + if (! $run instanceof WorkflowRun || ! is_array($binding) + || $run->workflow_instance_id !== $instance->id || $run->namespace !== $instance->namespace + || $run->workflow_class !== ($binding['workflow_class'] ?? null) + || $run->workflow_type !== ($binding['workflow_type'] ?? null) + || PortableDescriptor::encode($run->workflowArguments()) !== PortableDescriptor::encode( + $binding['arguments'] ?? [] + ) + || $run->connection !== ($profile['connection'] ?? null) || $run->queue !== ($profile['queue'] ?? null) + || $run->compatibility !== ($profile['compatibility'] ?? null) + || $run->run_timeout_seconds !== ($profile['run_timeout_seconds'] ?? null) + || TaskSchedulingFields::forRun($run) !== [ + 'priority' => $profile['priority'] ?? null, + 'fairness_key' => $profile['fairness_key'] ?? null, + 'fairness_weight' => $profile['fairness_weight'] ?? null, + ] + || $run->execution_deadline_at?->format('Y-m-d H:i:s.u') !== ($receipt['execution_deadline_at'] ?? null) + || $run->run_deadline_at?->format('Y-m-d H:i:s.u') !== ($receipt['run_deadline_at'] ?? null) + || $run->timeout_cleanup_work_deadline_at?->format( + 'Y-m-d H:i:s.u' + ) !== ($receipt['timeout_cleanup_work_deadline_at'] ?? null)) { + throw new LogicException('Recovery receipt no longer matches its native run.'); + } + $predecessor = $receipt['intent']['expected_predecessor_run_id'] ?? null; + if ($predecessor !== null && ! ConfiguredV2Models::query('link_model', WorkflowLink::class) + ->where( + 'link_type', + ($receipt['lineage_operation'] ?? null) === 'redrive' ? 'redrive' : 'compensation_recovery_restart' + ) + ->where('parent_workflow_run_id', $predecessor) + ->where('child_workflow_run_id', $run->id) + ->where('parent_workflow_instance_id', $instance->id) + ->where('child_workflow_instance_id', $instance->id) + ->exists()) { + throw new LogicException('Recovery successor is missing its predecessor lineage.'); + } + } +} diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index 3d9f160d9..87ea1dcb5 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -11,6 +11,7 @@ use Workflow\Serializers\CodecRegistry; use Workflow\Serializers\Serializer; use Workflow\V2\CommandContext; +use Workflow\V2\Compensation\Recovery\RecoveryRunAdmission; use Workflow\V2\Contracts\HistoryProjectionRole; use Workflow\V2\Contracts\RuntimeSignalControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; @@ -224,6 +225,10 @@ public function redrive(string $instanceId, string $failedRunId, array $options ], )); + $recoveryReceipt = isset($startedPayload['compensation_recovery']) + ? RecoveryRunAdmission::carryToRedrive($instance, $source, $continuedRun, $command) + : null; + $link = WorkflowLink::query()->create([ 'link_type' => 'redrive', 'sequence' => $plan['resume_step_sequence'], @@ -270,6 +275,9 @@ public function redrive(string $instanceId, string $failedRunId, array $options ? $startedPayload['replayed_started_at'] : $source->started_at?->toIso8601String(), 'workflow_link_id' => $link->id, + ...($recoveryReceipt === null ? [] : [ + 'compensation_recovery' => $recoveryReceipt, + ]), ], ), null, $command); diff --git a/src/V2/Support/HistoryEventPayloadContract.php b/src/V2/Support/HistoryEventPayloadContract.php index c33b856ae..d1ad94da4 100644 --- a/src/V2/Support/HistoryEventPayloadContract.php +++ b/src/V2/Support/HistoryEventPayloadContract.php @@ -123,6 +123,7 @@ final class HistoryEventPayloadContract 'continued_from_run_id', 'resume_step_sequence', 'recovery_kind', + 'compensation_recovery', 'definition_changed_from', 'replayed_started_at', 'retry_attempt', diff --git a/src/V2/WorkflowStub.php b/src/V2/WorkflowStub.php index 31e912d85..54a2a3790 100644 --- a/src/V2/WorkflowStub.php +++ b/src/V2/WorkflowStub.php @@ -17,6 +17,7 @@ use Workflow\Serializers\AvroValueJsonProjection; use Workflow\Serializers\CodecRegistry; use Workflow\Serializers\Serializer; +use Workflow\V2\Compensation\Recovery\RecoveryRunAdmission; use Workflow\V2\Contracts\BufferedSignalProjectionRole; use Workflow\V2\Contracts\HistoryExportRedactor; use Workflow\V2\Contracts\HistoryProjectionRole; @@ -1012,6 +1013,27 @@ public function attemptStart(...$arguments): StartResult ); } + /** + * The scope service must authorize and retain the request's first predecessor + * before calling this data-only native boundary. Ordinary start is unchanged. + * + * @internal Experimental compensation recovery admission. + */ + public function attemptRecoveryStart(RecoveryRunAdmission $admission): StartResult + { + if ($this->runTargeted) { + throw new LogicException('Recovery admission requires a canonical instance-targeted stub.'); + } + TimeoutCleanup::assertEmbeddedAdmission($admission->profile->timeoutCleanupSeconds); + + return $this->admitStart( + WorkflowMetadata::fromStartArguments([$admission->scope->toArray()]), + $admission->profile, + null, + $admission, + ); + } + public function signalWithStart( string $name, array $signalArguments = [], @@ -1729,6 +1751,7 @@ private function admitStart( WorkflowMetadata $metadata, StartOptions $startOptions, ?array $observabilityContext, + ?RecoveryRunAdmission $recoveryAdmission = null, ): StartResult { /** @var WorkflowCommand|null $command */ $command = null; @@ -1739,6 +1762,7 @@ private function admitStart( $metadata, $startOptions, $observabilityContext, + $recoveryAdmission, &$task, &$command, &$startedRun, @@ -1752,9 +1776,52 @@ private function admitStart( ->lockForUpdate() ->findOrFail($this->instance->id); - $run = $this->currentRunForInstance($instance, true); + $recoveryIntent = null; + if ($recoveryAdmission !== null) { + $workflowClass = TypeRegistry::resolveWorkflowClass( + $instance->workflow_class, + $instance->workflow_type + ); + $recoveryIntent = $recoveryAdmission->intent( + $instance, + $workflowClass, + RoutingResolver::workflowConnection($workflowClass, $metadata), + RoutingResolver::workflowQueue($workflowClass, $metadata), + ); + $command = $recoveryAdmission->retained($instance, $recoveryIntent); + if ($command !== null) { + return; + } + } + + // Refused recovery requests must not repair pointers or append to a + // stopped predecessor. The instance writer still serializes admission. + $run = $recoveryAdmission === null + ? $this->currentRunForInstance($instance, true) + : CurrentRunResolver::forInstance($instance, lockForUpdate: true); + $predecessor = null; + if ($recoveryAdmission !== null && $recoveryIntent !== null) { + $creationReceipt = $recoveryAdmission->predecessor($instance, $run, $recoveryIntent); + if ($run !== null && ! $run->status->isTerminal()) { + $command = $this->recordExistingStart( + $instance, + $run, + $metadata, + StartOptions::returnExistingActive(), + $recoveryAdmission->context( + CommandContext::system('compensation_recovery'), + $recoveryIntent, + $run, + $creationReceipt, + ), + ); + $recoveryAdmission->bind($command, $run); + return; + } + $predecessor = $run; + } - if ($run instanceof WorkflowRun) { + if ($recoveryAdmission === null && $run instanceof WorkflowRun) { $command = $this->recordExistingStart($instance, $run, $metadata, $startOptions); return; @@ -1766,6 +1833,12 @@ private function admitStart( $instance, $startOptions, ); + if ($recoveryAdmission !== null) { + $businessKey = $startOptions->businessKey; + $visibilityLabels = $startOptions->labels ?: null; + $memo = $startOptions->memo ?: null; + $searchAttributes = $startOptions->searchAttributes ?: null; + } $executionTimeoutSeconds = $startOptions->executionTimeoutSeconds; $runTimeoutSeconds = $startOptions->runTimeoutSeconds; $connection = RoutingResolver::workflowConnection($workflowClass, $metadata); @@ -1790,6 +1863,10 @@ private function admitStart( ? $startedAt->copy() ->addSeconds($runTimeoutSeconds) : null; + if ($predecessor !== null) { + $executionDeadlineAt = $predecessor->execution_deadline_at; + $runDeadlineAt = $predecessor->run_deadline_at; + } $payloadCodec = CodecRegistry::defaultCodec(); $serializedArguments = ExternalPayloads::externalizeForNamespace( Serializer::serializeWithCodec($payloadCodec, $metadata->arguments), @@ -1837,7 +1914,24 @@ private function admitStart( $this->seedTypedVisibilityMetadata($run, $memo, $searchAttributes); - $command = $this->recordNewStartCommand($instance, $run, $payloadCodec, $serializedArguments); + $command = $this->recordNewStartCommand( + $instance, + $run, + $payloadCodec, + $serializedArguments, + $recoveryAdmission !== null && $recoveryIntent !== null + ? $recoveryAdmission->context( + CommandContext::system('compensation_recovery'), + $recoveryIntent, + $run + ) + : null, + ); + $recoveryAdmission?->bind($command, $run); + $recoveryLink = null; + if ($recoveryAdmission !== null && $predecessor !== null) { + $recoveryLink = $recoveryAdmission->link($predecessor, $run); + } $instance->forceFill([ 'current_run_id' => $run->id, @@ -1845,7 +1939,7 @@ private function admitStart( 'visibility_labels' => $visibilityLabels, 'memo' => $memo, 'execution_timeout_seconds' => $executionTimeoutSeconds, - 'started_at' => $startedAt, + 'started_at' => $predecessor === null ? $startedAt : $instance->started_at, 'run_count' => $run->run_number, ])->save(); @@ -1860,6 +1954,14 @@ private function admitStart( 'run_timeout_seconds' => $runTimeoutSeconds, 'execution_deadline_at' => $executionDeadlineAt, 'run_deadline_at' => $runDeadlineAt, + ...($recoveryAdmission === null ? [] : [ + 'compensation_recovery' => $command->context['compensation_recovery'], + ]), + ...($recoveryLink === null ? [] : [ + 'continued_from_run_id' => $predecessor->id, + 'workflow_link_id' => $recoveryLink->id, + 'recovery_kind' => 'compensation_recovery_restart', + ]), TimeoutCleanup::STARTED_KEY => TimeoutCleanup::startedPayload( $startOptions->timeoutCleanupSeconds, $executionDeadlineAt, @@ -1901,37 +2003,41 @@ private function recordExistingStart( WorkflowRun $run, WorkflowMetadata $metadata, StartOptions $startOptions, + ?CommandContext $commandContext = null, ): WorkflowCommand { $canReturnExisting = $startOptions->duplicateStartPolicy === DuplicateStartPolicy::ReturnExistingActive && in_array($run->status, [RunStatus::Pending, RunStatus::Running, RunStatus::Waiting], true); - $command = WorkflowCommand::record($instance, $run, $this->commandAttributes([ - 'command_type' => CommandType::Start->value, - 'target_scope' => 'instance', - 'status' => $canReturnExisting - ? CommandStatus::Accepted->value - : CommandStatus::Rejected->value, - 'outcome' => $canReturnExisting - ? CommandOutcome::ReturnedExistingActive->value - : CommandOutcome::RejectedDuplicate->value, - 'payload_codec' => $run->payload_codec ?? CodecRegistry::defaultCodec(), - 'payload' => Serializer::serializeWithCodec( - $run->payload_codec ?? CodecRegistry::defaultCodec(), - $metadata->arguments - ), - 'rejection_reason' => $canReturnExisting - ? null - : 'instance_already_started', - 'accepted_at' => $canReturnExisting - ? now() - : null, - 'applied_at' => $canReturnExisting - ? now() - : null, - 'rejected_at' => $canReturnExisting - ? null - : now(), - ])); + $command = WorkflowCommand::record($instance, $run, $this->commandAttributesForContext( + $commandContext ?? $this->resolvedCommandContext(), + [ + 'command_type' => CommandType::Start->value, + 'target_scope' => 'instance', + 'status' => $canReturnExisting + ? CommandStatus::Accepted->value + : CommandStatus::Rejected->value, + 'outcome' => $canReturnExisting + ? CommandOutcome::ReturnedExistingActive->value + : CommandOutcome::RejectedDuplicate->value, + 'payload_codec' => $run->payload_codec ?? CodecRegistry::defaultCodec(), + 'payload' => Serializer::serializeWithCodec( + $run->payload_codec ?? CodecRegistry::defaultCodec(), + $metadata->arguments + ), + 'rejection_reason' => $canReturnExisting + ? null + : 'instance_already_started', + 'accepted_at' => $canReturnExisting + ? now() + : null, + 'applied_at' => $canReturnExisting + ? now() + : null, + 'rejected_at' => $canReturnExisting + ? null + : now(), + ] + )); WorkflowHistoryEvent::record($run, $canReturnExisting ? HistoryEventType::StartAccepted @@ -2101,6 +2207,14 @@ private function recordWorkflowStartedHistory( 'run_timeout_seconds' => $startMetadata['run_timeout_seconds'], 'execution_deadline_at' => $startMetadata['execution_deadline_at']?->toIso8601String(), 'run_deadline_at' => $startMetadata['run_deadline_at']?->toIso8601String(), + ...(isset($startMetadata['compensation_recovery']) ? [ + 'compensation_recovery' => $startMetadata['compensation_recovery'], + ] : []), + ...(isset($startMetadata['continued_from_run_id']) ? [ + 'continued_from_run_id' => $startMetadata['continued_from_run_id'], + 'workflow_link_id' => $startMetadata['workflow_link_id'], + 'recovery_kind' => $startMetadata['recovery_kind'], + ] : []), ...(isset($startMetadata[TimeoutCleanup::STARTED_KEY]) ? [ TimeoutCleanup::STARTED_KEY => $startMetadata[TimeoutCleanup::STARTED_KEY], ] : []), diff --git a/tests/Unit/V2/RecoveryRunAdmissionTest.php b/tests/Unit/V2/RecoveryRunAdmissionTest.php new file mode 100644 index 000000000..e3359b192 --- /dev/null +++ b/tests/Unit/V2/RecoveryRunAdmissionTest.php @@ -0,0 +1,414 @@ +set('queue.default', 'database'); + Queue::fake(); + } + + protected function tearDown(): void + { + parent::tearDown(); + foreach (['', '-wal', '-shm'] as $suffix) { + if (isset($this->databasePath) && is_file($this->databasePath . $suffix)) { + unlink($this->databasePath . $suffix); + } + } + } + + #[DataProvider('stops')] + public function testRestartAndLostAcknowledgementPreserveReceiptsHistoryAndBudgets(string $stop): void + { + [$scope, $origin, $stub] = $this->scope(); + $profile = (new StartOptions())->withExecutionTimeout(600) + ->withRunTimeout(300); + $initial = new RecoveryRunAdmission($scope, $origin, 'initial', null, $profile); + $first = $stub->attemptRecoveryStart($initial); + $firstRun = WorkflowRun::query()->findOrFail($first->runId()); + $oldTask = WorkflowTask::query()->where('workflow_run_id', $firstRun->id)->firstOrFail(); + $this->stop($stub, $stop); + $oldHistory = $this->history($firstRun->id); + $logicalStart = WorkflowInstance::query()->findOrFail($stub->id())->started_at; + $this->travel(30) + ->seconds(); + + $request = new RecoveryRunAdmission($scope, $origin, 'restart', $firstRun->id, $profile); + $successor = $stub->attemptRecoveryStart($request); + $next = WorkflowRun::query()->findOrFail($successor->runId()); + self::assertNotSame($firstRun->id, $next->id); + self::assertSame($firstRun->workflow_instance_id, $next->workflow_instance_id); + self::assertSame($firstRun->execution_deadline_at->format('c'), $next->execution_deadline_at->format('c')); + self::assertSame($firstRun->run_deadline_at->format('c'), $next->run_deadline_at->format('c')); + self::assertSame($logicalStart, WorkflowInstance::query()->findOrFail($stub->id())->started_at); + self::assertSame($oldHistory, $this->history($firstRun->id)); + self::assertSame($origin, WorkflowCommand::query()->findOrFail($successor->commandId()) + ->context['compensation_recovery']['intent']['binding']['original_run_id']); + self::assertSame(1, WorkflowLink::query()->where('link_type', 'compensation_recovery_restart')->count()); + self::assertSame(1, WorkflowTask::query()->where('workflow_run_id', $next->id)->count()); + $snapshot = $this->snapshot(); + self::assertSame($successor->commandId(), $stub->attemptRecoveryStart($request)->commandId()); + self::assertSame($snapshot, $this->snapshot()); + + // A late dispatch from the stopped run cannot make new progress. + (new RunWorkflowTask($oldTask->id))->handle(app(WorkflowExecutor::class)); + self::assertSame($oldHistory, $this->history($firstRun->id)); + $this->stop($stub, $stop); + $snapshot = $this->snapshot(); + self::assertSame($firstRun->id, $stub->attemptRecoveryStart($initial)->runId()); + self::assertSame($next->id, $stub->attemptRecoveryStart($request)->runId()); + self::assertSame($snapshot, $this->snapshot()); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'restart', $next->id, $profile) + )); + self::assertNotSame($next->id, $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'new-authorization', $next->id, $profile) + )->runId()); + } + + public function testCompetingRequestsAdoptOnlyTheSameContentBoundSuccessor(): void + { + [$scope, $origin, $stub] = $this->scope(); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null)); + $initialAdoption = new RecoveryRunAdmission($scope, $origin, 'initial-adopter', null); + self::assertSame($first->runId(), $stub->attemptRecoveryStart($initialAdoption)->runId()); + $this->stop($stub, 'terminate'); + $next = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'restart-a', $first->runId())); + $other = new RecoveryRunAdmission($scope, $origin, 'restart-b', $first->runId()); + $adopted = WorkflowStub::load($scope->recoveryInstanceId(), $scope->namespace)->attemptRecoveryStart($other); + self::assertTrue($adopted->returnedExistingActive()); + self::assertSame($next->runId(), $adopted->runId()); + self::assertSame(2, WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count()); + self::assertSame(1, WorkflowTask::query()->where('workflow_run_id', $next->runId())->count()); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'wrong-predecessor', null) + )); + $this->stop($stub, 'cancel'); + $snapshot = $this->snapshot(); + self::assertSame($next->runId(), $stub->attemptRecoveryStart($other)->runId()); + self::assertSame($first->runId(), $stub->attemptRecoveryStart($initialAdoption)->runId()); + self::assertSame($snapshot, $this->snapshot()); + } + + public function testChangedOwnerProfileScopeOrMissingLineageIsRefused(): void + { + [$scope, $origin, $stub] = $this->scope(); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null)); + $otherOrigin = WorkflowStub::make(TestSignalWorkflow::class, 'other-origin'); + $otherOrigin->start(); + $otherOrigin->terminate(); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $otherOrigin->runId(), 'initial', null) + )); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'initial', null, (new StartOptions())->withRunTimeout(60)) + )); + $wrongScope = new ScopeIdentity($scope->application, $scope->namespace, $scope->key, 'next-generation'); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($wrongScope, $origin, 'initial', null) + )); + $this->stop($stub, 'terminate'); + $next = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'restart', $first->runId())); + WorkflowLink::query()->where('child_workflow_run_id', $next->runId())->delete(); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'adopt', $first->runId()) + )); + } + + public function testRestartCannotRenewAnExpiredRecoveryBudget(): void + { + [$scope, $origin, $stub] = $this->scope(); + $profile = (new StartOptions())->withExecutionTimeout(20) + ->withRunTimeout(10); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null, $profile)); + $this->stop($stub, 'terminate'); + $this->travel(11) + ->seconds(); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'restart', $first->runId(), $profile) + )); + } + + public function testAnOrdinaryStartCannotBeAdoptedAsARecoveryReceipt(): void + { + [$scope, $origin, $stub] = $this->scope(); + $ordinary = $stub->start($scope->toArray()); + $this->stop($stub, 'terminate'); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'restart', $ordinary->runId()) + )); + self::assertTrue( + $stub->attemptStart(StartOptions::returnExistingActive(), $scope->toArray())->rejectedDuplicate() + ); + } + + public function testProjectionFailureRollsBackSuccessorReceiptLineageAndTaskTogether(): void + { + [$scope, $origin, $stub] = $this->scope(); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null)); + $this->stop($stub, 'terminate'); + $request = new RecoveryRunAdmission($scope, $origin, 'restart', $first->runId()); + $projector = app(HistoryProjectionRole::class); + $failedProjection = \Mockery::mock(HistoryProjectionRole::class); + $failedProjection->shouldReceive('projectRun') + ->once() + ->andThrow(new LogicException('Recovery projection failure.')); + app() + ->instance(HistoryProjectionRole::class, $failedProjection); + try { + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart($request)); + } finally { + app()->instance(HistoryProjectionRole::class, $projector); + } + $result = $stub->attemptRecoveryStart($request); + self::assertTrue($result->startedNew()); + self::assertSame(1, WorkflowCommand::query()->where('request_id', $request->token())->count()); + self::assertSame(1, WorkflowLink::query()->where('child_workflow_run_id', $result->runId())->count()); + } + + public function testInitialRecoveryPolicyIsSeparateFromTheClosedForwardDeadline(): void + { + $origin = WorkflowStub::make(TestSignalWorkflow::class, 'expired-forward'); + $origin->start((new StartOptions())->withExecutionTimeout(1)); + $origin->terminate(); + $this->travel(2) + ->seconds(); + $originRun = WorkflowRun::query()->findOrFail($origin->runId()); + self::assertTrue($originRun->execution_deadline_at->isPast()); + $scope = new ScopeIdentity('test-app', $originRun->namespace, 'order', 'generation-1'); + $stub = WorkflowStub::make(RecoveryWorkflow::class, $scope->recoveryInstanceId(), $scope->namespace); + $admitted = $stub->attemptRecoveryStart(new RecoveryRunAdmission( + $scope, + $originRun->id, + 'authorized-initial-recovery', + null, + (new StartOptions())->withExecutionTimeout(60), + )); + $recovery = WorkflowRun::query()->findOrFail($admitted->runId()); + self::assertTrue($recovery->execution_deadline_at->isFuture()); + self::assertSame( + $originRun->execution_deadline_at->format('c'), + $originRun->fresh() + ->execution_deadline_at->format('c') + ); + } + + public function testAStoppedNativeRedriveSuccessorKeepsItsOriginalRecoveryBinding(): void + { + [$scope, $origin, $stub] = $this->scope(); + WorkflowStub::fake(); + WorkflowStub::mock(PrepareRecoveryActivity::class, static function (): never { + throw new NonRetryableException('Synthetic preparation refusal.'); + }); + $profile = (new StartOptions())->withExecutionTimeout(600) + ->withRunTimeout(120); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null, $profile)); + self::assertSame(RunStatus::Failed, WorkflowRun::query()->findOrFail($first->runId())->status); + // The public activity fake executes inline. Keep that real failed + // history, then restore queued dispatch so the redrive can be stopped. + app() + ->forgetInstance('workflow.v2.mocks'); + app() + ->forgetInstance('workflow.v2.dispatched'); + self::assertFalse(WorkflowStub::faked()); + $redrive = app(WorkflowControlPlane::class)->redrive($scope->recoveryInstanceId(), $first->runId(), [ + 'namespace' => $scope->namespace, + 'request_id' => 'technical-redrive-request', + ]); + self::assertTrue($redrive['accepted'], (string) $redrive['reason']); + $redriven = WorkflowRun::query()->findOrFail($redrive['workflow_run_id']); + $adoption = new RecoveryRunAdmission($scope, $origin, 'adopt-redrive', $first->runId(), $profile); + $adopted = $stub->attemptRecoveryStart($adoption); + self::assertSame($redriven->id, $adopted->runId()); + self::assertSame($adopted->commandId(), $stub->attemptRecoveryStart($adoption)->commandId()); + $stub->refresh() + ->terminate(); + $history = $this->history($redriven->id); + $this->travel(10) + ->seconds(); + $next = $stub->attemptRecoveryStart(new RecoveryRunAdmission( + $scope, + $origin, + 'restart-redriven-recovery', + $redriven->id, + $profile, + )); + $successor = WorkflowRun::query()->findOrFail($next->runId()); + self::assertSame($successor->id, $stub->refresh()->runId()); + self::assertSame($redriven->execution_deadline_at->format('c'), $successor->execution_deadline_at->format('c')); + self::assertSame($redriven->run_deadline_at->format('c'), $successor->run_deadline_at->format('c')); + self::assertSame($history, $this->history($redriven->id)); + self::assertSame($redriven->id, $stub->attemptRecoveryStart($adoption)->runId()); + } + + public function testCanonicalSqliteWriterIsHeldBeforeReceiptOrPredecessorReads(): void + { + if (DB::connection()->getDriverName() !== 'sqlite') { + $this->markTestSkipped('This control observes the SQLite writer lock.'); + } + [$scope, $origin, $stub] = $this->scope(); + DB::statement('CREATE TABLE recovery_writer_probe (id INTEGER PRIMARY KEY, value INTEGER NOT NULL)'); + DB::insert('INSERT INTO recovery_writer_probe VALUES (1, 0)'); + $writer = new PDO('sqlite:' . DB::connection()->getDatabaseName()); + $writer->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + $writer->exec('PRAGMA busy_timeout=0'); + $observed = 0; + $blocked = false; + DB::listen(static function (QueryExecuted $query) use ($writer, &$observed, &$blocked): void { + if ($observed !== 0 || ! str_starts_with(strtolower(ltrim($query->sql)), 'select') + || ! str_contains($query->sql, 'workflow_instances')) { + return; + } + ++$observed; + try { + $writer->exec('UPDATE recovery_writer_probe SET value=value+1 WHERE id=1'); + } catch (PDOException $error) { + if (! str_contains(strtolower($error->getMessage()), 'locked')) { + throw $error; + } + $blocked = true; + } + }); + $result = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null)); + self::assertTrue($result->startedNew()); + self::assertSame(1, $observed); + self::assertTrue($blocked); + self::assertSame(0, (int) DB::selectOne('SELECT value FROM recovery_writer_probe WHERE id=1')->value); + self::assertSame(1, $writer->exec('UPDATE recovery_writer_probe SET value=value+1 WHERE id=1')); + } + + public function testAChangedCreatorBindingCannotBeAuthorizedByItsOldFingerprint(): void + { + [$scope, $origin, $stub] = $this->scope(); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null)); + $this->stop($stub, 'terminate'); + $command = WorkflowCommand::query()->findOrFail($first->commandId()); + $context = $command->context; + $context['compensation_recovery']['intent']['binding']['scope']['generation'] = 'different-generation'; + $command->forceFill([ + 'context' => $context, + ])->save(); + $this->assertRefusesWithoutWrites(static fn () => $stub->attemptRecoveryStart( + new RecoveryRunAdmission($scope, $origin, 'restart', $first->runId()) + )); + } + + public static function stops(): array + { + return [['cancel'], ['terminate']]; + } + + protected function getPackageProviders($app): array + { + return [WorkflowServiceProvider::class]; + } + + protected function defineEnvironment($app): void + { + $this->databasePath = tempnam(sys_get_temp_dir(), 'recovery-admission-'); + $app['config']->set('database.default', 'testing'); + $app['config']->set('database.connections.testing', [ + 'driver' => 'sqlite', + 'database' => $this->databasePath, + 'prefix' => '', + 'foreign_key_constraints' => true, + 'journal_mode' => 'WAL', + 'busy_timeout' => 20, + ]); + $app['config']->set('queue.default', 'database'); + $app['config']->set('cache.default', 'array'); + $app['config']->set('workflows.v1.enabled', false); + } + + protected function defineDatabaseMigrations(): void + { + $this->artisan('migrate', [ + '--database' => 'testing', + '--force' => true, + ])->assertExitCode(0); + } + + private function scope(): array + { + $origin = WorkflowStub::make(TestSignalWorkflow::class, 'forward-origin'); + $origin->start(); + $origin->terminate(); + $scope = new ScopeIdentity('test-app', WorkflowRun::query()->findOrFail( + $origin->runId() + )->namespace, 'order', 'generation-1'); + return [ + $scope, + $origin->runId(), + WorkflowStub::make(RecoveryWorkflow::class, $scope->recoveryInstanceId(), $scope->namespace), + ]; + } + + private function stop(WorkflowStub $stub, string $stop): void + { + $stop === 'cancel' ? $stub->cancel() : $stub->terminate(); + } + + private function history(string $runId): array + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->orderBy('sequence')->get()->toArray(); + } + + private function snapshot(): array + { + return array_map(static fn (string $model): array => $model::query()->orderBy('id')->get()->toArray(), [ + WorkflowInstance::class, WorkflowRun::class, WorkflowCommand::class, WorkflowHistoryEvent::class, + WorkflowTask::class, WorkflowLink::class, WorkflowRunSummary::class, + ]); + } + + private function assertRefusesWithoutWrites(callable $call): void + { + $before = $this->snapshot(); + try { + $call(); + self::fail('Expected recovery admission refusal.'); + } catch (LogicException $error) { + self::assertStringContainsString('recovery', strtolower($error->getMessage())); + } + self::assertSame($before, $this->snapshot()); + } +} From 1b329bf52f2bfa141eeeba2051a130231da7c581 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:57:56 +0000 Subject: [PATCH 11/32] Keep recovery redrive inside its immutable receipt binding --- .../Recovery/RecoveryRunAdmission.php | 45 +++++-- .../Support/DefaultWorkflowControlPlane.php | 22 +++- tests/Unit/V2/RecoveryRunAdmissionTest.php | 118 ++++++++++++++++++ 3 files changed, 175 insertions(+), 10 deletions(-) diff --git a/src/V2/Compensation/Recovery/RecoveryRunAdmission.php b/src/V2/Compensation/Recovery/RecoveryRunAdmission.php index 57b7fbb40..0f4d88732 100644 --- a/src/V2/Compensation/Recovery/RecoveryRunAdmission.php +++ b/src/V2/Compensation/Recovery/RecoveryRunAdmission.php @@ -11,8 +11,10 @@ use Workflow\V2\Enums\CommandOutcome; use Workflow\V2\Enums\CommandStatus; use Workflow\V2\Enums\CommandType; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Models\WorkflowCommand; +use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowLink; use Workflow\V2\Models\WorkflowRun; @@ -127,7 +129,10 @@ public function retained(WorkflowInstance $instance, array $intent): ?WorkflowCo throw new LogicException('Recovery request identity was reused with different admission content.'); } $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($command->workflow_run_id); - $this->assertRun($instance, $run, $receipt); + self::assertRun($instance, $run, $receipt); + if (self::sourceReceipt($instance, $run) === null) { + throw new LogicException('Recovery receipt has no matching native creation history.'); + } return $command; } @@ -143,12 +148,12 @@ public function predecessor(WorkflowInstance $instance, ?WorkflowRun $run, array } return null; } - $receipt = self::creationReceipt($run); + $receipt = self::sourceReceipt($instance, $run); if (! is_array($receipt) || ($receipt['intent']['binding_fingerprint'] ?? null) !== $intent['binding_fingerprint']) { throw new LogicException('Recovery run has no matching native admission lineage.'); } - $this->assertRun($instance, $run, $receipt); + self::assertRun($instance, $run, $receipt); if (! $run->status->isTerminal()) { if (PortableDescriptor::encode($receipt['intent']) !== PortableDescriptor::encode($intent)) { throw new LogicException('Active recovery is not the successor of the requested predecessor.'); @@ -222,8 +227,8 @@ public static function carryToRedrive( WorkflowRun $source, WorkflowRun $successor, WorkflowCommand $command, + ?array $receipt, ): ?array { - $receipt = self::creationReceipt($source); if ($receipt === null) { return null; } @@ -245,7 +250,11 @@ public static function carryToRedrive( return $receipt; } - private static function creationReceipt(WorkflowRun $run): ?array + /** + * Either atomic copy identifies a recovery. Missing, malformed or divergent + * copies must never turn it into an ordinary unbound start/redrive. + */ + public static function sourceReceipt(WorkflowInstance $instance, WorkflowRun $run): ?array { $creation = ConfiguredV2Models::query('command_model', WorkflowCommand::class) ->where('workflow_run_id', $run->id) @@ -253,8 +262,30 @@ private static function creationReceipt(WorkflowRun $run): ?array ->whereIn('command_type', [CommandType::Start->value, CommandType::Redrive->value]) ->whereIn('outcome', [CommandOutcome::StartedNew->value, CommandOutcome::Redriven->value]) ->first(); - $receipt = $creation?->context['compensation_recovery'] ?? null; - return is_array($receipt) ? $receipt : null; + $started = $run->relationLoaded('historyEvents') + ? $run->historyEvents->firstWhere('event_type', HistoryEventType::WorkflowStarted) + : ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->where('workflow_run_id', $run->id) + ->where('event_type', HistoryEventType::WorkflowStarted->value) + ->orderBy('sequence') + ->first(); + $payload = is_array($started?->payload) ? $started->payload : []; + $context = is_array($creation?->context) ? $creation->context : []; + $commandHasReceipt = array_key_exists('compensation_recovery', $context); + $historyHasReceipt = array_key_exists('compensation_recovery', $payload); + if (! $commandHasReceipt && ! $historyHasReceipt) { + return null; + } + $receipt = $context['compensation_recovery'] ?? null; + $marker = $payload['compensation_recovery'] ?? null; + if (! $commandHasReceipt || ! $historyHasReceipt || ! is_array($receipt) || ! is_array($marker) + || ($payload['workflow_command_id'] ?? null) !== $creation?->id + || PortableDescriptor::encode($receipt) !== PortableDescriptor::encode($marker) + || ($receipt['intent']['binding']['definition'] ?? null) !== ($payload['workflow_definition_fingerprint'] ?? null)) { + throw new LogicException('Recovery redrive requires matching native command and history receipts.'); + } + self::assertRun($instance, $run, $receipt); + return $receipt; } private static function assertRun(WorkflowInstance $instance, mixed $run, array $receipt): void diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index 87ea1dcb5..3ca145d2f 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -100,6 +100,10 @@ public function redrive(string $instanceId, string $failedRunId, array $options return; } + // Check both immutable copies even for retained redrive lookup. A + // missing history marker must not hide a command-bound recovery. + $recoveryReceipt = RecoveryRunAdmission::sourceReceipt($instance, $source); + /** @var WorkflowLink|null $existingLink */ $existingLink = WorkflowLink::query() ->where('parent_workflow_run_id', $source->id) @@ -120,6 +124,11 @@ public function redrive(string $instanceId, string $failedRunId, array $options $currentRun = CurrentRunResolver::forInstance($instance, lockForUpdate: true); + if ($recoveryReceipt !== null && WorkflowDefinitionFingerprint::currentForRun($source) + !== $recoveryReceipt['intent']['binding']['definition']) { + throw new LogicException('Recovery redrive cannot change its authorized workflow definition.'); + } + $externalFingerprint = $options['external_workflow_definition_fingerprint'] ?? null; $eligibility = FailedRunRedriveEligibility::evaluate( $source, @@ -142,6 +151,9 @@ public function redrive(string $instanceId, string $failedRunId, array $options $plan = $eligibility['plan']; $startedPayload = $eligibility['started_payload']; $definitionChange = $eligibility['definition_change'] ?? null; + if ($recoveryReceipt !== null && $definitionChange !== null) { + throw new LogicException('Recovery redrive cannot change its authorized workflow definition.'); + } $successorClass = $definitionChange['workflow_class'] ?? $source->workflow_class; // Change provenance describes one transition. A successor that keeps // its source's definition records none, even if the source changed. @@ -225,9 +237,13 @@ public function redrive(string $instanceId, string $failedRunId, array $options ], )); - $recoveryReceipt = isset($startedPayload['compensation_recovery']) - ? RecoveryRunAdmission::carryToRedrive($instance, $source, $continuedRun, $command) - : null; + $recoveryReceipt = RecoveryRunAdmission::carryToRedrive( + $instance, + $source, + $continuedRun, + $command, + $recoveryReceipt, + ); $link = WorkflowLink::query()->create([ 'link_type' => 'redrive', diff --git a/tests/Unit/V2/RecoveryRunAdmissionTest.php b/tests/Unit/V2/RecoveryRunAdmissionTest.php index e3359b192..3ab5edf27 100644 --- a/tests/Unit/V2/RecoveryRunAdmissionTest.php +++ b/tests/Unit/V2/RecoveryRunAdmissionTest.php @@ -16,11 +16,13 @@ use Workflow\Exceptions\NonRetryableException; use Workflow\Providers\WorkflowServiceProvider; use Workflow\V2\Compensation\Activities\PrepareRecoveryActivity; +use Workflow\V2\Compensation\Definition\PortableDescriptor; use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Recovery\RecoveryRunAdmission; use Workflow\V2\Compensation\Recovery\RecoveryWorkflow; use Workflow\V2\Contracts\HistoryProjectionRole; use Workflow\V2\Contracts\WorkflowControlPlane; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowCommand; @@ -31,6 +33,7 @@ use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\StartOptions; +use Workflow\V2\Support\FailedRunRedriveEligibility; use Workflow\V2\Support\WorkflowExecutor; use Workflow\V2\WorkflowStub; @@ -332,6 +335,104 @@ public function testAChangedCreatorBindingCannotBeAuthorizedByItsOldFingerprint( )); } + #[DataProvider('brokenRedriveReceipts')] + public function testRecoveryRedriveRequiresBothMatchingAtomicReceiptCopies(string $mutation): void + { + [$scope, $stub, $first] = $this->failedRecovery(); + $command = WorkflowCommand::query()->findOrFail($first->commandId()); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $first->runId()) + ->where('event_type', HistoryEventType::WorkflowStarted->value)->firstOrFail(); + $context = $command->context; + $payload = $started->payload; + if ($mutation === 'command') { + unset($context['compensation_recovery']); + } elseif ($mutation === 'history') { + unset($payload['compensation_recovery']); + } else { + $payload['compensation_recovery']['request_key'] = 'different-request'; + } + $command->forceFill([ + 'context' => $context, + ])->save(); + $started->forceFill([ + 'payload' => $payload, + ])->save(); + $this->assertRefusesWithoutWrites(static fn () => app(WorkflowControlPlane::class)->redrive( + $stub->id(), + $first->runId(), + [ + 'namespace' => $scope->namespace, + 'request_id' => 'redrive', + ] + )); + } + + public function testRecoveryRefusesAnOtherwiseReplayCompatibleDefinitionChange(): void + { + [$scope, $stub, $first] = $this->failedRecovery(); + $command = WorkflowCommand::query()->findOrFail($first->commandId()); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $first->runId()) + ->where('event_type', HistoryEventType::WorkflowStarted->value)->firstOrFail(); + // Represent an earlier, replay-compatible definition whose receipt and + // immutable history agree. The currently registered code has a new hash. + $context = $command->context; + $receipt = $context['compensation_recovery']; + $receipt['intent']['binding']['definition'] = str_repeat('a', 64); + $receipt['intent']['binding_fingerprint'] = hash( + 'sha256', + PortableDescriptor::encode($receipt['intent']['binding']) + ); + $context['compensation_recovery'] = $receipt; + $payload = $started->payload; + $payload['compensation_recovery'] = $receipt; + $payload['workflow_definition_fingerprint'] = str_repeat('a', 64); + $command->forceFill([ + 'context' => $context, + ])->save(); + $started->forceFill([ + 'payload' => $payload, + ])->save(); + $source = WorkflowRun::query()->findOrFail($first->runId()); + $eligibility = FailedRunRedriveEligibility::evaluate($source, $source, allowDefinitionChange: true); + self::assertNull($eligibility['reason']); + self::assertNotNull($eligibility['definition_change']); + $this->assertRefusesWithoutWrites(static fn () => app(WorkflowControlPlane::class)->redrive( + $stub->id(), + $first->runId(), + [ + 'namespace' => $scope->namespace, + 'request_id' => 'redrive', + 'allow_definition_change' => true, + ] + )); + } + + public function testRetainedRedriveAlsoRequiresItsSourceReceiptPair(): void + { + [$scope, $stub, $first] = $this->failedRecovery(); + $options = [ + 'namespace' => $scope->namespace, + 'request_id' => 'redrive', + ]; + self::assertTrue(app(WorkflowControlPlane::class)->redrive($stub->id(), $first->runId(), $options)['accepted']); + $command = WorkflowCommand::query()->findOrFail($first->commandId()); + $context = $command->context; + unset($context['compensation_recovery']); + $command->forceFill([ + 'context' => $context, + ])->save(); + $this->assertRefusesWithoutWrites(static fn () => app(WorkflowControlPlane::class)->redrive( + $stub->id(), + $first->runId(), + $options + )); + } + + public static function brokenRedriveReceipts(): array + { + return [['command'], ['history'], ['divergent']]; + } + public static function stops(): array { return [['cancel'], ['terminate']]; @@ -382,6 +483,23 @@ private function scope(): array ]; } + private function failedRecovery(): array + { + [$scope, $origin, $stub] = $this->scope(); + WorkflowStub::fake(); + WorkflowStub::mock(PrepareRecoveryActivity::class, static function (): never { + throw new NonRetryableException('Synthetic preparation refusal.'); + }); + $first = $stub->attemptRecoveryStart(new RecoveryRunAdmission($scope, $origin, 'initial', null)); + self::assertSame(RunStatus::Failed, WorkflowRun::query()->findOrFail($first->runId())->status); + app() + ->forgetInstance('workflow.v2.mocks'); + app() + ->forgetInstance('workflow.v2.dispatched'); + self::assertFalse(WorkflowStub::faked()); + return [$scope, $stub, $first]; + } + private function stop(WorkflowStub $stub, string $stop): void { $stop === 'cancel' ? $stub->cancel() : $stub->terminate(); From 444aa688d5b00db4edbc0f3147c82be5db7a9137 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:06:13 +0000 Subject: [PATCH 12/32] Pin compensation version-one cold replay histories --- regression-corpus-policy.json | 15 ++ ...tion-recovery-version-one-cold-replay.json | 227 ++++++++++++++++++ ...pensation-scope-future-format-refused.json | 218 +++++++++++++++++ ...pensation-scope-future-marker-refused.json | 218 +++++++++++++++++ ...nsation-scope-version-one-cold-replay.json | 227 ++++++++++++++++++ 5 files changed, 905 insertions(+) create mode 100644 tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json create mode 100644 tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json create mode 100644 tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json create mode 100644 tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json diff --git a/regression-corpus-policy.json b/regression-corpus-policy.json index 2df39af3d..2ee2ef9f7 100644 --- a/regression-corpus-policy.json +++ b/regression-corpus-policy.json @@ -80,6 +80,21 @@ }, { "glob": "src/V2/Support/ActivityRetryHistory.php" + }, + { + "glob": "src/V2/Compensation/DurableScope.php" + }, + { + "glob": "src/V2/Compensation/Recovery/RecoveryWorkflow.php" + }, + { + "glob": "src/V2/Compensation/Definition/*.php" + }, + { + "glob": "src/V2/Compensation/Execution/ForwardResult.php" + }, + { + "glob": "src/V2/Compensation/Execution/Resolution.php" } ] } diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json new file mode 100644 index 000000000..4d5d43c79 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json @@ -0,0 +1,227 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "compensation-recovery-version-one-cold-replay", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "workflow": { + "type": "Workflow\\V2\\Compensation\\Recovery\\RecoveryWorkflow", + "arguments": [ + { + "version": 1, + "application": "shop", + "namespace": "client-a", + "key": "retained-corpus-order", + "generation": "1" + } + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_type": "dw.compensation.recovery.v1", + "namespace": "client-a" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "VersionMarkerRecorded", + "payload": { + "sequence": 1, + "change_id": "dw.compensation.recovery-activities", + "version": 1, + "min_supported": 1, + "max_supported": 1 + }, + "recorded_at": "2026-11-13T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.prepare-recovery.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\PrepareRecoveryActivity", + "activity_execution_id": "corpus-execution-2", + "arguments": "wwHioz3/VYAiNwwCDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAA", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.prepare-recovery.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\PrepareRecoveryActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "workflow_task_id": "corpus-task-2" + }, + "recorded_at": "2026-11-13T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.prepare-recovery.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\PrepareRecoveryActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwwECoABOThhN2UwN2M0MzZhYzJlYWUwNzM4Yzg3ZTY5MWIxYjA5Y2VhMTU5MmE1NmI4NjAwMmUwMGZlY2I1MzcyYTZiOAqAAWI5NjYwZjFlZGZmOWYzYWUwMWRkYzYyNzQwNDNjYjdhYTM2ZmU4NzQyNTVmNTIzNzFiY2M0NTdlNzkxOGE0MWYA", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.compensate-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "activity_execution_id": "corpus-execution-3", + "arguments": "wwHioz3/VYAiNwwEDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.compensate-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "workflow_task_id": "corpus-task-3" + }, + "recorded_at": "2026-11-13T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.compensate-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwA=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.compensate-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "activity_execution_id": "corpus-execution-4", + "arguments": "wwHioz3/VYAiNwwEDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4AA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:08+00:00" + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.compensate-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "workflow_task_id": "corpus-task-4" + }, + "recorded_at": "2026-11-13T00:00:09+00:00" + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.compensate-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwA=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:10+00:00" + }, + { + "sequence": 12, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.verify-scope.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\VerifyScopeActivity", + "activity_execution_id": "corpus-execution-5", + "arguments": "wwHioz3/VYAiNwwCDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAA", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:11+00:00" + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.verify-scope.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\VerifyScopeActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "workflow_task_id": "corpus-task-5" + }, + "recorded_at": "2026-11-13T00:00:12+00:00" + }, + { + "sequence": 14, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.verify-scope.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\VerifyScopeActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4EDnNjb3BlSWQKgAE0MjhiNWEwMmEzZmM3MTUwMDU5YmUxZDI1ODRhYzAxOTU4N2JhYTg4N2ZkODA4OWNjNjM0MDUyMzg5ODRiZjcxDHN0YXR1cwoQdmVyaWZpZWQA", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:13+00:00" + } + ], + "expected": { + "completed": true, + "result": { + "scopeId": "428b5a02a3fc7150059be1d2584ac019587baa887fd8089cc63405238984bf71", + "status": "verified" + }, + "commands": [ + { + "type": "complete_workflow", + "result": "wwHioz3/VYAiNw4EDnNjb3BlSWQKgAE0MjhiNWEwMmEzZmM3MTUwMDU5YmUxZDI1ODRhYzAxOTU4N2JhYTg4N2ZkODA4OWNjNjM0MDUyMzg5ODRiZjcxDHN0YXR1cwoQdmVyaWZpZWQA", + "payload_codec": "avro" + } + ] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json new file mode 100644 index 000000000..b0d1edce9 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "compensation-scope-future-format-refused", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", + "arguments": [ + { + "version": 2, + "application": "shop", + "namespace": "client-a", + "key": "retained-corpus-order", + "generation": "1" + }, + "complete" + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", + "namespace": "client-a" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "VersionMarkerRecorded", + "payload": { + "sequence": 1, + "change_id": "dw.compensation.scope-activities", + "version": 1, + "min_supported": 1, + "max_supported": 1 + }, + "recorded_at": "2026-11-13T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "workflow_task_id": "corpus-task-2" + }, + "recorded_at": "2026-11-13T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwqAATk4YTdlMDdjNDM2YWMyZWFlMDczOGM4N2U2OTFiMWIwOWNlYTE1OTJhNTZiODYwMDJlMDBmZWNiNTM3MmE2Yjg=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "workflow_task_id": "corpus-task-3" + }, + "recorded_at": "2026-11-13T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4CEnJlZmVyZW5jZQoSYm9va2luZy0xAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:08+00:00" + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "workflow_task_id": "corpus-task-4" + }, + "recorded_at": "2026-11-13T00:00:09+00:00" + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwqAAWI5NjYwZjFlZGZmOWYzYWUwMWRkYzYyNzQwNDNjYjdhYTM2ZmU4NzQyNTVmNTIzNzFiY2M0NTdlNzkxOGE0MWY=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:10+00:00" + }, + { + "sequence": 12, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:11+00:00" + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "workflow_task_id": "corpus-task-5" + }, + "recorded_at": "2026-11-13T00:00:12+00:00" + }, + { + "sequence": 14, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4CEnJlZmVyZW5jZQoSYm9va2luZy0yAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:13+00:00" + } + ], + "expected_failure": { + "type": "compensation_identity_format_unsupported", + "exception": "InvalidArgumentException" + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json new file mode 100644 index 000000000..10c6b24d0 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "compensation-scope-future-marker-refused", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", + "arguments": [ + { + "version": 1, + "application": "shop", + "namespace": "client-a", + "key": "retained-corpus-order", + "generation": "1" + }, + "complete" + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", + "namespace": "client-a" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "VersionMarkerRecorded", + "payload": { + "sequence": 1, + "change_id": "dw.compensation.scope-activities", + "version": 2, + "min_supported": 1, + "max_supported": 2 + }, + "recorded_at": "2026-11-13T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "workflow_task_id": "corpus-task-2" + }, + "recorded_at": "2026-11-13T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwqAATk4YTdlMDdjNDM2YWMyZWFlMDczOGM4N2U2OTFiMWIwOWNlYTE1OTJhNTZiODYwMDJlMDBmZWNiNTM3MmE2Yjg=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "workflow_task_id": "corpus-task-3" + }, + "recorded_at": "2026-11-13T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4CEnJlZmVyZW5jZQoSYm9va2luZy0xAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:08+00:00" + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "workflow_task_id": "corpus-task-4" + }, + "recorded_at": "2026-11-13T00:00:09+00:00" + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwqAAWI5NjYwZjFlZGZmOWYzYWUwMWRkYzYyNzQwNDNjYjdhYTM2ZmU4NzQyNTVmNTIzNzFiY2M0NTdlNzkxOGE0MWY=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:10+00:00" + }, + { + "sequence": 12, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:11+00:00" + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "workflow_task_id": "corpus-task-5" + }, + "recorded_at": "2026-11-13T00:00:12+00:00" + }, + { + "sequence": 14, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4CEnJlZmVyZW5jZQoSYm9va2luZy0yAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:13+00:00" + } + ], + "expected_failure": { + "type": "compensation_marker_incompatible_range", + "exception": "Workflow\\V2\\Exceptions\\VersionRangeMismatchException" + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json new file mode 100644 index 000000000..0898eeb56 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json @@ -0,0 +1,227 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "compensation-scope-version-one-cold-replay", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", + "arguments": [ + { + "version": 1, + "application": "shop", + "namespace": "client-a", + "key": "retained-corpus-order", + "generation": "1" + }, + "complete" + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", + "namespace": "client-a" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "VersionMarkerRecorded", + "payload": { + "sequence": 1, + "change_id": "dw.compensation.scope-activities", + "version": 1, + "min_supported": 1, + "max_supported": 1 + }, + "recorded_at": "2026-11-13T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "workflow_task_id": "corpus-task-2" + }, + "recorded_at": "2026-11-13T00:00:03+00:00" + }, + { + "sequence": 5, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 2, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-2", + "activity_attempt_id": "corpus-attempt-2", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwqAATk4YTdlMDdjNDM2YWMyZWFlMDczOGM4N2U2OTFiMWIwOWNlYTE1OTJhNTZiODYwMDJlMDBmZWNiNTM3MmE2Yjg=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:04+00:00" + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:05+00:00" + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "workflow_task_id": "corpus-task-3" + }, + "recorded_at": "2026-11-13T00:00:06+00:00" + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 3, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-3", + "activity_attempt_id": "corpus-attempt-3", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4CEnJlZmVyZW5jZQoSYm9va2luZy0xAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:07+00:00" + }, + { + "sequence": 9, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:08+00:00" + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "workflow_task_id": "corpus-task-4" + }, + "recorded_at": "2026-11-13T00:00:09+00:00" + }, + { + "sequence": 11, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 4, + "activity_type": "dw.compensation.register-obligation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "activity_execution_id": "corpus-execution-4", + "activity_attempt_id": "corpus-attempt-4", + "attempt_number": 1, + "result": "wwHioz3/VYAiNwqAAWI5NjYwZjFlZGZmOWYzYWUwMWRkYzYyNzQwNDNjYjdhYTM2ZmU4NzQyNTVmNTIzNzFiY2M0NTdlNzkxOGE0MWY=", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:10+00:00" + }, + { + "sequence": 12, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:11+00:00" + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "workflow_task_id": "corpus-task-5" + }, + "recorded_at": "2026-11-13T00:00:12+00:00" + }, + { + "sequence": 14, + "event_type": "ActivityCompleted", + "payload": { + "sequence": 5, + "activity_type": "dw.compensation.forward-operation.v1", + "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "activity_execution_id": "corpus-execution-5", + "activity_attempt_id": "corpus-attempt-5", + "attempt_number": 1, + "result": "wwHioz3/VYAiNw4CEnJlZmVyZW5jZQoSYm9va2luZy0yAA==", + "payload_codec": "avro" + }, + "recorded_at": "2026-11-13T00:00:13+00:00" + } + ], + "expected": { + "completed": true, + "result": { + "status": "done" + }, + "commands": [ + { + "type": "complete_workflow", + "result": "wwHioz3/VYAiNw4CDHN0YXR1cwoIZG9uZQA=", + "payload_codec": "avro" + } + ] + } +} From 6fadccf8c816178b20ab044ea1bd11f9a8c2e136 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:06:13 +0000 Subject: [PATCH 13/32] Bind recovery requests to their original native admission --- docs/compensation-scopes.md | 32 +++- .../Persistence/ScopeRepository.php | 56 ++++++ .../Compensation/Recovery/ScopeRecovery.php | 170 +++++++++++++----- ...1_11_000100_create_compensation_scopes.php | 2 + tests/Feature/V2/V2CompensationScopeTest.php | 166 ++++++++++++++++- 5 files changed, 371 insertions(+), 55 deletions(-) diff --git a/docs/compensation-scopes.md b/docs/compensation-scopes.md index dfc6e64d2..bf9c183f1 100644 --- a/docs/compensation-scopes.md +++ b/docs/compensation-scopes.md @@ -35,9 +35,11 @@ Every scope derives one canonical recovery workflow instance ID. Native DW start/adopt admits one run; native redrive admits one successor after an eligible failure. A native returned-existing-active start receipt remains eligible for redrive only when its run identity and accepted start-command snapshot match. -A technical redrive key includes the exact failed source run. One unchanged -retained request can therefore recover several failed successors while duplicate -requests for one source adopt the same successor. Scope progress must be written +A retained request pins its first native route, predecessor and initial profile. +Repeating that key returns its original admission receipt, including after that +run fails or an operator cancels or terminates it. Each additional recovery run +requires a fresh authorized scope request key. Native activity attempts and +backoff within the same run keep their existing identities. Scope progress must be written under the current native activity attempt's authority, in the same database transaction as the ledger mutation. A status check followed by an unguarded write is insufficient. No scope lease or separate owner claim is introduced. Native fencing cannot recall an issued external request. @@ -117,7 +119,10 @@ Children cannot be shared between parents. One application/namespace graph-integ row serializes edge changes before scope locks. It contains no executor, lease, heartbeat or retry state. A missing or unverified child stops earlier parent work. Long child cleanup can exhaust the ordinary native activity retry budget; the -parent remains unresolved and can resume through eligible native redrive. +parent remains unresolved. A failed child requires a fresh authorized child +request. The stable parent-operation key never creates that request implicitly. +After the child ledger becomes verified, a fresh authorized parent request can +redrive the parent and observe that result before earlier parent work resumes. A scope stays attached to its original native run for forward execution. This increment does not transfer forward ownership across continue-as-new or origin @@ -136,6 +141,14 @@ recovery. A request accepted on an active origin is durable routing evidence; if that origin closes without resolving its obligations, repeat the same authorized request to route it to canonical recovery. +Cancelled or terminated canonical recovery can restart on that same instance +only through a fresh authorized request naming its exact current predecessor. +The origin and retained obligations stay fixed; old terminal histories stay +immutable. The initial recovery profile is independent of the closed forward +run. Restarts inherit the canonical recovery's absolute execution and run +deadlines and routing, with no renewed technical allowance. Exhausted deadlines +refuse another restart and leave the obligations unresolved. + `ScopeRetention::release()` requires verified work and terminal native executions, and refuses a child still retained by an unresolved parent. It removes retained inputs and forward results while preserving scope/generation, operation and request @@ -194,8 +207,15 @@ its own provider obligations before effects. A missing child remains unresolved and cannot satisfy the parent barrier. After the origin closes, the host calls `ScopeRecovery::request($identity, -$requestKey, $principal, $reason)`. Obtain the principal from trusted application +$requestKey, $principal, $reason, $initialProfile)`. The optional `StartOptions` +profile sets the initial recovery budget; retain that same profile on later +requests. Obtain the principal from trusted application authorization, not from untrusted request content. Repeating the same request -requires the same principal and reason. Inspecting or requesting retained work +requires the same principal, reason and profile. A returned failed/stopped +receipt means recovery remains unresolved; inspect it and submit a fresh +authorized key to request another eligible run. Results report `recoveryStatus`, +`scopeStatus` and `nextAction`; `submit_fresh_authorized_request` identifies that +boundary. Only the verified scope ledger clears the unresolved obligation. +Inspecting or requesting retained work must go through an authorized host boundary; the `@internal` persistence classes are not an HTTP-facing data access API. diff --git a/src/V2/Compensation/Persistence/ScopeRepository.php b/src/V2/Compensation/Persistence/ScopeRepository.php index b7f02c61e..c9aaa50c1 100644 --- a/src/V2/Compensation/Persistence/ScopeRepository.php +++ b/src/V2/Compensation/Persistence/ScopeRepository.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Compensation\Persistence; +use Closure; use Illuminate\Database\Connection; use LogicException; use Workflow\Exceptions\NonRetryableException; @@ -178,6 +179,61 @@ public function primaryRequest(ScopeIdentity $identity): array }); } + /** Bind the first native recovery target before an admission can lose its response. */ + public function recoveryRoute( + ScopeIdentity $identity, + string $requestKey, + string $profileFingerprint, + ?Closure $select = null, + ): ?array { + return ScopeStorage::mutate($identity, static function (array $scope, Connection $database) use ( + $identity, + $requestKey, + $profileFingerprint, + $select, + ): ?array { + $id = hash('sha256', PortableDescriptor::encode([$identity->id(), $requestKey])); + $row = $database->table('workflow_compensation_requests')->where('scope_id', $scope['id']) + ->where('id', $id)->lockForUpdate()->first(); + if ($row === null) { + throw new LogicException('Recovery admission requires an authorized retained request.'); + } + if ($row->recovery_route === null) { + if ($select === null) { + return null; + } + $route = [ + 'version' => 1, + 'profileFingerprint' => $profileFingerprint, + ...$select(), + ]; + } else { + $route = json_decode($row->recovery_route, true, flags: JSON_THROW_ON_ERROR); + } + $keys = is_array($route) ? array_keys($route) : []; + sort($keys, SORT_STRING); + if ($keys !== ['kind', 'predecessorRunId', 'profileFingerprint', 'version'] + || $route['version'] !== 1 + || ! in_array($route['kind'], ['start', 'redrive'], true) + || $route['profileFingerprint'] !== $profileFingerprint + || ($route['predecessorRunId'] !== null && (! is_string($route['predecessorRunId']) + || $route['predecessorRunId'] === '')) + || ($route['kind'] === 'redrive' && $route['predecessorRunId'] === null)) { + throw new LogicException('The retained recovery route is invalid or its request profile changed.'); + } + if ($row->recovery_route === null) { + $database->table('workflow_compensation_requests')->where('id', $id)->update([ + 'recovery_route' => PortableDescriptor::encode($route), + ]); + ScopeStorage::append($scope, 'recovery_route_bound', [ + 'requestId' => $id, + 'route' => $route, + ]); + } + return $route; + }); + } + public function metadata(ScopeIdentity $identity): array { return ScopeStorage::mutate($identity, static fn (array $scope): array => $scope); diff --git a/src/V2/Compensation/Recovery/ScopeRecovery.php b/src/V2/Compensation/Recovery/ScopeRecovery.php index 3ace65c7a..516477894 100644 --- a/src/V2/Compensation/Recovery/ScopeRecovery.php +++ b/src/V2/Compensation/Recovery/ScopeRecovery.php @@ -11,10 +11,14 @@ use Workflow\V2\Compensation\Security\ScopeAuthorization; use Workflow\V2\Contracts\WorkflowControlPlane; use Workflow\V2\Enums\RunStatus; +use Workflow\V2\Enums\HistoryEventType; +use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\StartOptions; use Workflow\V2\Support\ConfiguredV2Models; +use Workflow\V2\Support\CurrentRunResolver; use Workflow\V2\Support\TypeRegistry; +use Workflow\V2\Support\WorkflowFiberContext; use Workflow\V2\WorkflowStub; /** @@ -31,9 +35,16 @@ public function __construct( ) { } - public function request(ScopeIdentity $scope, string $requestKey, string $principal, string $reason): array - { + public function request( + ScopeIdentity $scope, + string $requestKey, + string $principal, + string $reason, + ?StartOptions $profile = null, + ): array { + $this->assertHostContext(); $policyVersion = $this->authorization->assertAllowed($scope, $principal, 'request'); + $this->assertHostContext(); PortableDescriptor::identifier($policyVersion, 'authorization policy version'); $snapshot = $this->repository->inspect($scope); if ($snapshot['status'] === 'released') { @@ -44,13 +55,16 @@ public function request(ScopeIdentity $scope, string $requestKey, string $princi throw new LogicException('The original compensation run is unavailable in the exact namespace.'); } $this->repository->request($scope, $requestKey, $principal, $reason, $policyVersion); - if ($snapshot['status'] === 'verified') { + $profile ??= StartOptions::returnExistingActive(); + $profileFingerprint = self::profileFingerprint($profile); + $boundRoute = $this->repository->recoveryRoute($scope, $requestKey, $profileFingerprint); + if ($boundRoute === null && $snapshot['status'] === 'verified') { return [ 'route' => 'verified', 'runId' => null, ]; } - if (! $origin->status->isTerminal()) { + if ($boundRoute === null && ! $origin->status->isTerminal()) { $command = WorkflowStub::loadRun($origin->id)->attemptRequestCancellation($reason); if ($command->accepted()) { return [ @@ -65,57 +79,127 @@ public function request(ScopeIdentity $scope, string $requestKey, string $princi throw new LogicException('The originating run refused cooperative compensation.'); } } - $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->where('workflow_instance_id', $scope->recoveryInstanceId()) - ->where('namespace', $scope->namespace) - ->orderByDesc('run_number') - ->first(); - if (! $run instanceof WorkflowRun) { + $route = $boundRoute ?? $this->repository->recoveryRoute( + $scope, + $requestKey, + $profileFingerprint, + fn (): array => $this->selectRecoveryRoute($scope, $profile), + ); + if ($route === null) { + throw new LogicException('The retained recovery route was not bound.'); + } + $admission = new RecoveryRunAdmission( + $scope, + $origin->id, + $requestKey, + $route['predecessorRunId'], + $profile, + ); + if ($route['kind'] === 'start') { $result = WorkflowStub::make(RecoveryWorkflow::class, $scope->recoveryInstanceId(), $scope->namespace) - ->attemptStart(StartOptions::returnExistingActive(), $scope->toArray()); - $runId = $result->runId(); - if ($result->accepted() && is_string($runId)) { - return [ - 'route' => 'recovery', - 'runId' => $runId, - ]; + ->attemptRecoveryStart($admission); + if (! $result->accepted() || ! is_string($result->runId())) { + throw new LogicException('Native recovery admission refused the retained request.'); } - $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->where('workflow_instance_id', $scope->recoveryInstanceId()) - ->where('namespace', $scope->namespace) - ->orderByDesc('run_number') - ->first(); - } - if (! $run instanceof WorkflowRun || $run->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class)) { - throw new LogicException('The canonical recovery identity is unavailable or reserved by another type.'); + return $this->recoveryResult($scope, $result->runId(), $result->commandId()); } - if (in_array($run->status, [RunStatus::Pending, RunStatus::Running, RunStatus::Waiting], true)) { - return [ - 'route' => 'recovery', - 'runId' => $run->id, - ]; - } - if ($this->repository->inspect($scope)['status'] === 'verified') { - return [ - 'route' => 'verified', - 'runId' => $run->id, - ]; - } - if ($run->status !== RunStatus::Failed) { - throw new LogicException('The recovery is terminal without verified compensation.'); - } - $redrive = $this->runtime->redrive($scope->recoveryInstanceId(), $run->id, [ + $redrive = $this->runtime->redrive($scope->recoveryInstanceId(), $route['predecessorRunId'], [ 'namespace' => $scope->namespace, - 'request_id' => hash('sha256', PortableDescriptor::encode([$scope->id(), $requestKey, $run->id])), + 'request_id' => $admission->token(), ]); if (($redrive['accepted'] ?? false) !== true) { throw new LogicException( 'Native DW redrive refused unresolved recovery: ' . ($redrive['reason'] ?? 'unknown') . '.' ); } + return $this->recoveryResult($scope, $redrive['workflow_run_id']); + } + + private function recoveryResult(ScopeIdentity $scope, string $runId, ?string $commandId = null): array + { + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('namespace', $scope->namespace)->findOrFail($runId); + $scopeStatus = $this->repository->metadata($scope)['status']; + $nextAction = $scopeStatus === 'verified' ? 'none' : match ($run->status) { + RunStatus::Failed, RunStatus::Cancelled, RunStatus::Terminated => 'submit_fresh_authorized_request', + RunStatus::Completed => 'inspect_unresolved_obligations', + default => 'observe_recovery', + }; return [ 'route' => 'recovery', - 'runId' => $redrive['workflow_run_id'], + 'runId' => $runId, + ...($commandId === null ? [] : ['commandId' => $commandId]), + 'recoveryStatus' => $run->status->value, + 'scopeStatus' => $scopeStatus, + 'nextAction' => $nextAction, + ]; + } + + private function selectRecoveryRoute(ScopeIdentity $scope, StartOptions $profile): array + { + $instance = ConfiguredV2Models::query('instance_model', WorkflowInstance::class) + ->where('namespace', $scope->namespace)->find($scope->recoveryInstanceId()); + $run = $instance instanceof WorkflowInstance ? CurrentRunResolver::forInstance($instance) : null; + if ($instance !== null && $instance->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class)) { + throw new LogicException('The canonical recovery identity is reserved by another type.'); + } + $predecessor = $run?->id; + $intent = null; + if ($run !== null) { + $started = $run->historyEvents()->where('event_type', HistoryEventType::WorkflowStarted->value)->sole(); + $intent = $started->payload['compensation_recovery']['intent'] ?? null; + if (! is_array($intent) || ($intent['version'] ?? null) !== 1 + || ! array_key_exists('expected_predecessor_run_id', $intent) + || ! is_array($intent['binding']['profile'] ?? null)) { + throw new LogicException('Recovery has no retained native predecessor and profile binding.'); + } + $expected = self::profileDescriptor($profile); + $recorded = ['version' => 1, ...array_intersect_key($intent['binding']['profile'], $expected)]; + if (PortableDescriptor::encode($expected) !== PortableDescriptor::encode($recorded)) { + throw new LogicException('Recovery requests must retain the initial recovery profile.'); + } + } + if ($run !== null && ! $run->status->isTerminal()) { + $predecessor = $intent['expected_predecessor_run_id']; + } elseif ($run !== null && ! in_array( + $run->status, + [RunStatus::Failed, RunStatus::Cancelled, RunStatus::Terminated], + true, + )) { + throw new LogicException('The recovery is terminal without verified compensation.'); + } + return [ + 'kind' => $run?->status === RunStatus::Failed ? 'redrive' : 'start', + 'predecessorRunId' => $predecessor, + ]; + } + + private function assertHostContext(): void + { + if (WorkflowFiberContext::active() || ConfiguredV2Models::connection()->transactionLevel() !== 0) { + throw new LogicException('Recovery authorization requires an unlocked host context outside workflow replay.'); + } + } + + private static function profileFingerprint(StartOptions $profile): string + { + return hash('sha256', PortableDescriptor::encode(self::profileDescriptor($profile))); + } + + private static function profileDescriptor(StartOptions $profile): array + { + return [ + 'version' => 1, + 'business_key' => $profile->businessKey, + 'labels' => $profile->labels, + 'memo' => $profile->memo, + 'search_attributes' => $profile->searchAttributes, + 'execution_timeout_seconds' => $profile->executionTimeoutSeconds, + 'run_timeout_seconds' => $profile->runTimeoutSeconds, + 'timeout_cleanup_seconds' => $profile->timeoutCleanupSeconds, + 'priority' => $profile->priority, + 'fairness_key' => $profile->fairnessKey, + 'fairness_weight' => $profile->fairnessWeight, ]; } } diff --git a/src/migrations/2026_11_11_000100_create_compensation_scopes.php b/src/migrations/2026_11_11_000100_create_compensation_scopes.php index 373f1d7aa..87375e3e6 100644 --- a/src/migrations/2026_11_11_000100_create_compensation_scopes.php +++ b/src/migrations/2026_11_11_000100_create_compensation_scopes.php @@ -67,6 +67,8 @@ public function up(): void $table->text('descriptor'); $table->string('authorization_policy_version', 128) ->nullable(); + $table->text('recovery_route') + ->nullable(); $table->unique(['scope_id', 'position'], 'compensation_request_position'); }); Schema::create('workflow_compensation_journal', static function (Blueprint $table): void { diff --git a/tests/Feature/V2/V2CompensationScopeTest.php b/tests/Feature/V2/V2CompensationScopeTest.php index f1ac5c88f..bbbdcfd9a 100644 --- a/tests/Feature/V2/V2CompensationScopeTest.php +++ b/tests/Feature/V2/V2CompensationScopeTest.php @@ -6,6 +6,7 @@ use Illuminate\Database\Events\QueryExecuted; use Illuminate\Support\Facades\Cache; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\V2\TestRetainedScopeWorkflow; use Tests\Fixtures\V2\TestScopedOperation; use Tests\TestCase; @@ -19,6 +20,7 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\StartOptions; use Workflow\V2\Support\WorkflowReplayer; +use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\WorkflowStub; final class V2CompensationScopeTest extends TestCase @@ -87,6 +89,115 @@ public function testNativeSagaCompensatesAnEffectWhoseForwardResultWasLost(): vo $this->assertSame(1, (int) Cache::get('test:scope:effect-count')); } + public static function operatorStops(): array + { + return [ + 'cancel' => ['cancel'], + 'terminate' => ['terminate'], + ]; + } + + #[DataProvider('operatorStops')] + public function testFreshRequestsResumeStoppedRecoveryWithoutRebindingOldRequests(string $stop): void + { + [$scope, $origin] = $this->completedOrigin(); + Cache::put('test:scope:unknown-first', true); + $first = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $firstStub = WorkflowStub::loadRun($first['runId']); + $this->assertFalse($firstStub->completed()); + $stopped = $stop === 'cancel' ? $firstStub->cancel('operator-stop') : $firstStub->terminate('operator-stop'); + $this->assertTrue($stopped->accepted()); + $firstHistory = WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId']) + ->orderBy('sequence')->get()->toArray(); + $second = app(ScopeRecovery::class)->request($scope, 'recovery-2', 'operator-a', 'cancel'); + $this->assertNotSame($first['runId'], $second['runId']); + $secondStub = WorkflowStub::loadRun($second['runId']); + $stopped = $stop === 'cancel' ? $secondStub->cancel('operator-stop') : $secondStub->terminate('operator-stop'); + $this->assertTrue($stopped->accepted()); + $secondHistory = WorkflowHistoryEvent::query()->where('workflow_run_id', $second['runId']) + ->orderBy('sequence')->get()->toArray(); + + foreach ([['recovery-1', $first], ['recovery-2', $second]] as [$key, $receipt]) { + $again = app(ScopeRecovery::class)->request($scope, $key, 'operator-a', 'cancel'); + $this->assertSame($receipt['runId'], $again['runId']); + $this->assertSame($receipt['commandId'], $again['commandId']); + $this->assertSame($stop === 'cancel' ? 'cancelled' : 'terminated', $again['recoveryStatus']); + $this->assertSame('submit_fresh_authorized_request', $again['nextAction']); + } + $this->assertSame(2, WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count()); + Cache::forget('test:scope:unknown-first'); + $third = app(ScopeRecovery::class)->request($scope, 'recovery-3', 'operator-a', 'cancel'); + $this->assertNotContains($third['runId'], [$first['runId'], $second['runId']]); + $this->assertTrue(WorkflowStub::loadRun($third['runId'])->completed()); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + $this->assertSame(1, count(array_filter( + Cache::get('test:scope:undo-log'), + static fn (string $key): bool => $key === 'second', + ))); + $this->assertSame($firstHistory, WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId']) + ->orderBy('sequence')->get()->toArray()); + $this->assertSame($secondHistory, WorkflowHistoryEvent::query()->where('workflow_run_id', $second['runId']) + ->orderBy('sequence')->get()->toArray()); + $retained = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $this->assertSame($first['runId'], $retained['runId']); + $this->assertSame($first['commandId'], $retained['commandId']); + $this->assertSame('verified', $retained['scopeStatus']); + $this->assertSame('none', $retained['nextAction']); + $this->assertSame($origin->runId(), (new ScopeRepository())->inspect($scope)['original_run_id']); + } + + public function testRetainedRecoveryRequestCannotChangeItsInitialBudget(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:unknown-first', true); + $profile = StartOptions::returnExistingActive()->withExecutionTimeout(600)->withRunTimeout(300); + $first = app(ScopeRecovery::class)->request($scope, 'bounded-recovery', 'operator-a', 'cancel', $profile); + $this->assertTrue(WorkflowStub::loadRun($first['runId'])->terminate('operator-stop')->accepted()); + $history = WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId'])->count(); + try { + app(ScopeRecovery::class)->request( + $scope, + 'bounded-recovery', + 'operator-a', + 'cancel', + $profile->withExecutionTimeout(601), + ); + $this->fail('The same recovery request admitted a different initial budget.'); + } catch (\LogicException $error) { + $this->assertStringContainsString('profile changed', $error->getMessage()); + } + $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count()); + $this->assertSame($history, WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId'])->count()); + } + + public function testRecoveryRefusesCallerTransactionsBeforeHostAuthorization(): void + { + [$scope] = $this->completedOrigin(); + $calls = 0; + $this->app->instance(ScopeAuthorization::class, new class($calls) implements ScopeAuthorization { + public function __construct(private int &$calls) + { + } + + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + ++$this->calls; + return 'policy'; + } + }); + $database = ConfiguredV2Models::connection(); + $database->beginTransaction(); + try { + app(ScopeRecovery::class)->request($scope, 'inside-transaction', 'operator-a', 'cancel'); + $this->fail('Recovery ran host authorization under a caller transaction.'); + } catch (\LogicException $error) { + $this->assertStringContainsString('unlocked host context', $error->getMessage()); + $this->assertSame(0, $calls); + } finally { + $database->rollBack(); + } + } + public function testRetryExhaustionStaysUnresolvedAndNativeRedriveReusesVerifiedWork(): void { [$scope] = $this->completedOrigin(); @@ -138,27 +249,31 @@ public function testNativeActiveAdoptionDoesNotPreventRecoveryRedriveAfterFailur ))); } - public function testOneRetainedRequestCanRedriveTwoFailuresWithoutRepeatingVerifiedWork(): void + public function testFreshRequestsCanRedriveTwoFailuresWithoutRepeatingVerifiedWork(): void { [$scope] = $this->completedOrigin(); Cache::put('test:scope:unknown-first', true); - $first = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $first = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'cancel'); $this->driveNativeRetries($first['runId']); - $second = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $retained = app(ScopeRecovery::class)->request($scope, 'cancel-1', 'operator-a', 'cancel'); + $this->assertSame($first['runId'], $retained['runId']); + $this->assertSame($first['commandId'], $retained['commandId']); + $this->assertSame('submit_fresh_authorized_request', $retained['nextAction']); + $second = app(ScopeRecovery::class)->request($scope, 'cancel-2', 'operator-a', 'cancel'); $this->assertNotSame($first['runId'], $second['runId']); - $duplicate = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $duplicate = app(ScopeRecovery::class)->request($scope, 'cancel-2', 'operator-a', 'cancel'); $this->assertSame($second['runId'], $duplicate['runId']); $this->driveNativeRetries($second['runId']); $this->assertSame('failed', WorkflowRun::query()->findOrFail($second['runId'])->status->value); Cache::forget('test:scope:unknown-first'); - $third = app(ScopeRecovery::class)->request($scope, 'cancel', 'operator-a', 'cancel'); + $third = app(ScopeRecovery::class)->request($scope, 'cancel-3', 'operator-a', 'cancel'); $this->assertNotSame($second['runId'], $third['runId']); $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); $this->assertSame( 3, WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count() ); - $this->assertSame(1, \Workflow\V2\Support\ConfiguredV2Models::connection() + $this->assertSame(3, \Workflow\V2\Support\ConfiguredV2Models::connection() ->table('workflow_compensation_requests') ->where('scope_id', $scope->id()) ->count()); @@ -277,6 +392,45 @@ public function testCompletedChildScopeIsVerifiedBeforeEarlierParentCompensation ); } + public function testFailedChildNeedsAnExplicitFreshRequestBeforeParentRecoveryCanAdvance(): void + { + WorkflowStub::fake(); + $child = new ScopeIdentity('shop', 'client-a', 'failed-child', '1'); + WorkflowStub::make(TestRetainedScopeWorkflow::class, 'failed-child-origin', 'client-a') + ->start($child->toArray(), 'complete'); + $parent = new ScopeIdentity('shop', 'client-a', 'waiting-parent', '1'); + WorkflowStub::make(TestRetainedScopeWorkflow::class, 'waiting-parent-origin', 'client-a') + ->start($parent->toArray(), 'with-child', $child->toArray()); + Cache::put('test:scope:unknown-first', true); + $parentRecovery = app(ScopeRecovery::class)->request($parent, 'parent-1', 'operator-a', 'cancel'); + $childRun = WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->sole(); + $this->driveNativeRetries($childRun->id); + $this->assertSame('failed', $childRun->refresh()->status->value); + $this->driveNativeRetries($parentRecovery['runId']); + $this->assertSame('failed', WorkflowRun::query()->findOrFail($parentRecovery['runId'])->status->value); + $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->count()); + $this->assertNotContains('waiting-parent/first', Cache::get('test:scope:undo-scoped-log')); + Cache::forget('test:scope:unknown-first'); + $retained = app(ScopeRecovery::class)->request($parent, 'parent-1', 'operator-a', 'cancel'); + $this->assertSame($parentRecovery['runId'], $retained['runId']); + $this->assertSame('submit_fresh_authorized_request', $retained['nextAction']); + $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->count()); + + app(ScopeRecovery::class)->request($child, 'explicit-child-2', 'operator-a', 'child-recovery'); + $this->assertSame('verified', (new ScopeRepository())->inspect($child)['status']); + $this->assertNotContains('waiting-parent/first', Cache::get('test:scope:undo-scoped-log')); + $resumed = app(ScopeRecovery::class)->request($parent, 'parent-2', 'operator-a', 'cancel'); + $this->assertTrue(WorkflowStub::loadRun($resumed['runId'])->completed()); + $this->assertSame('verified', (new ScopeRepository())->inspect($parent)['status']); + $log = Cache::get('test:scope:undo-scoped-log'); + $parentFirst = array_search('waiting-parent/first', $log, true); + $childFirst = array_keys($log, 'failed-child/first', true); + $this->assertGreaterThan(max($childFirst), $parentFirst); + foreach (['waiting-parent/second', 'failed-child/second', 'waiting-parent/first'] as $operation) { + $this->assertSame(1, count(array_keys($log, $operation, true))); + } + } + public function testChildParticipationRecordsItsOwnAuthorizationPolicyVersion(): void { WorkflowStub::fake(); From 5252b5033c994c906c7d28b625f34816d0117ba6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:06:13 +0000 Subject: [PATCH 14/32] Exercise an embedded target in the child retry scope control --- .../V2/V2CompensationScopeRunBoundaryTest.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 6a11f2942..b5ecc66fd 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -39,6 +39,7 @@ final class V2CompensationScopeRunBoundaryTest extends TestCase protected function setUp(): void { parent::setUp(); + self::stopWorkers(); config([ 'workflows.v2.namespaces' => ['client-a'], 'queue.default' => 'redis', @@ -172,9 +173,6 @@ public function testOriginRedriveCannotReissueAnUncertainForwardEffectFromTheOri public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): void { - config()->set('workflows.v2.types.workflows', [ - 'test.scope-run-boundary' => TestScopeRunBoundaryWorkflow::class, - ]); $scope = new ScopeIdentity('shop', 'client-a', 'retried-child-order', '1'); $parent = app(WorkflowControlPlane::class)->start('test.remote-parent', 'scope-retry-parent', [ 'namespace' => 'client-a', @@ -192,7 +190,7 @@ public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): ])->save(); $result = app(DefaultWorkflowTaskBridge::class)->complete($task->id, [[ 'type' => 'start_child_workflow', - 'workflow_type' => 'test.scope-run-boundary', + 'workflow_type' => TestScopeRunBoundaryWorkflow::class, 'arguments' => Serializer::serializeWithCodec('avro', [$scope->toArray(), 'child-retry']), 'payload_codec' => 'avro', 'retry_policy' => [ @@ -206,7 +204,12 @@ public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): ->where('link_type', 'child_workflow') ->orderBy('created_at') ->get(); - $this->assertCount(2, $links); + $this->assertCount(2, $links, json_encode([ + 'runs' => WorkflowRun::query()->whereIn('id', $links->pluck('child_workflow_run_id')) + ->get(['id', 'status'])->toArray(), + 'failures' => WorkflowFailure::query()->whereIn('workflow_run_id', $links->pluck('child_workflow_run_id')) + ->pluck('message')->all(), + ], JSON_THROW_ON_ERROR)); $origin = WorkflowRun::query()->findOrFail($links[0]->child_workflow_run_id); $retry = WorkflowRun::query()->findOrFail($links[1]->child_workflow_run_id); $this->assertSame($origin->workflow_instance_id, $retry->workflow_instance_id); From d06676e80a6c58e7f275351f63b7d5c25a4b19d9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:11:34 +0000 Subject: [PATCH 15/32] Declare compensation format refusal in replay corpus validation --- scripts/ci/validate-regression-corpus.py | 2 ++ .../compensation-scope-future-format-refused.json | 2 +- .../compensation-scope-future-marker-refused.json | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ci/validate-regression-corpus.py b/scripts/ci/validate-regression-corpus.py index b905c5918..9d4e4ec76 100644 --- a/scripts/ci/validate-regression-corpus.py +++ b/scripts/ci/validate-regression-corpus.py @@ -33,6 +33,7 @@ WORKFLOW_HISTORY_SHAPE_MISMATCH = "workflow_history_shape_mismatch" VERSION_MARKER_INCOMPATIBLE_RANGE = "version_marker_incompatible_range" VERSION_MARKER_HISTORY_RANGE_INVALID = "version_marker_history_range_invalid" +UNSUPPORTED_COMPENSATION_FORMAT = "unsupported_compensation_format" UNSUPPORTED_PAYLOAD_CODEC = "unsupported_payload_codec" PHP_GOLDEN_REPLAY_WORKFLOW = "Tests\\Fixtures\\V2\\TestGoldenReplayWorkflow" PHP_REPLAY_CONSUMERS = { @@ -1291,6 +1292,7 @@ def _replay_expected_failure(value: Any, context: str) -> Mapping[str, str]: WORKFLOW_HISTORY_SHAPE_MISMATCH, VERSION_MARKER_INCOMPATIBLE_RANGE, VERSION_MARKER_HISTORY_RANGE_INVALID, + UNSUPPORTED_COMPENSATION_FORMAT, "activity_retry_policy_mismatch", "workflow_entry_arguments_mismatch", UNSUPPORTED_PAYLOAD_CODEC, diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json index b0d1edce9..22c97d69d 100644 --- a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json @@ -212,7 +212,7 @@ } ], "expected_failure": { - "type": "compensation_identity_format_unsupported", + "type": "unsupported_compensation_format", "exception": "InvalidArgumentException" } } diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json index 10c6b24d0..08822fc0d 100644 --- a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json @@ -212,7 +212,7 @@ } ], "expected_failure": { - "type": "compensation_marker_incompatible_range", + "type": "version_marker_incompatible_range", "exception": "Workflow\\V2\\Exceptions\\VersionRangeMismatchException" } } From 12f96cc231af1a8beda75721d605f17eda750659 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:39:25 +0000 Subject: [PATCH 16/32] Validate retained recovery redrive targets before returning receipts --- .../Support/DefaultWorkflowControlPlane.php | 22 +++++++++ tests/Unit/V2/RecoveryRunAdmissionTest.php | 46 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/V2/Support/DefaultWorkflowControlPlane.php b/src/V2/Support/DefaultWorkflowControlPlane.php index 3ca145d2f..3950cd752 100644 --- a/src/V2/Support/DefaultWorkflowControlPlane.php +++ b/src/V2/Support/DefaultWorkflowControlPlane.php @@ -111,6 +111,28 @@ public function redrive(string $instanceId, string $failedRunId, array $options ->first(); if ($existingLink instanceof WorkflowLink) { + if ($recoveryReceipt !== null) { + /** @var WorkflowRun|null $target */ + $target = $this->runQuery() + ->where('workflow_instance_id', $instance->id) + ->where('namespace', $instance->namespace) + ->lockForUpdate() + ->find($existingLink->child_workflow_run_id); + if (! $target instanceof WorkflowRun + || $existingLink->parent_workflow_instance_id !== $instance->id + || $existingLink->child_workflow_instance_id !== $instance->id) { + throw new LogicException('Recovery redrive successor is outside its canonical instance.'); + } + $targetReceipt = RecoveryRunAdmission::sourceReceipt($instance, $target); + if ($targetReceipt === null + || ($targetReceipt['operation'] ?? null) !== 'redrive' + || ($targetReceipt['intent']['binding_fingerprint'] ?? null) + !== $recoveryReceipt['intent']['binding_fingerprint'] + || ($targetReceipt['intent']['expected_predecessor_run_id'] ?? null) !== $source->id + || ($targetReceipt['execution_deadline_at'] ?? null) !== $recoveryReceipt['execution_deadline_at']) { + throw new LogicException('Recovery redrive successor does not match its retained admission.'); + } + } $result = array_merge($result, [ 'accepted' => true, 'workflow_run_id' => $existingLink->child_workflow_run_id, diff --git a/tests/Unit/V2/RecoveryRunAdmissionTest.php b/tests/Unit/V2/RecoveryRunAdmissionTest.php index 3ab5edf27..3f75168e7 100644 --- a/tests/Unit/V2/RecoveryRunAdmissionTest.php +++ b/tests/Unit/V2/RecoveryRunAdmissionTest.php @@ -428,6 +428,52 @@ public function testRetainedRedriveAlsoRequiresItsSourceReceiptPair(): void )); } + #[DataProvider('substitutedSuccessors')] + public function testRetainedRecoveryRedriveRefusesAnotherAdmission(string $substitution): void + { + [$scope, $stub, $first] = $this->failedRecovery(); + $options = [ + 'namespace' => $scope->namespace, + 'request_id' => 'redrive', + ]; + $redrive = app(WorkflowControlPlane::class)->redrive($stub->id(), $first->runId(), $options); + self::assertTrue($redrive['accepted']); + $owner = WorkflowCommand::query()->findOrFail($first->commandId())->context['compensation_recovery']['intent']['binding']['original_run_id']; + $targetScope = $substitution === 'foreign' + ? new ScopeIdentity($scope->application, $scope->namespace, 'different-order', $scope->generation) + : $scope; + if ($substitution === 'later') { + WorkflowStub::loadRun($redrive['workflow_run_id'])->terminate(); + } + $target = WorkflowStub::make( + RecoveryWorkflow::class, + $targetScope->recoveryInstanceId(), + $targetScope->namespace + ) + ->attemptRecoveryStart(new RecoveryRunAdmission( + $targetScope, + $owner, + 'different-admission', + $substitution === 'later' ? $redrive['workflow_run_id'] : null, + )); + WorkflowLink::query()->where('parent_workflow_run_id', $first->runId())->where('link_type', 'redrive') + ->firstOrFail() + ->forceFill([ + 'child_workflow_instance_id' => $targetScope->recoveryInstanceId(), + 'child_workflow_run_id' => $target->runId(), + ])->save(); + $this->assertRefusesWithoutWrites(static fn () => app(WorkflowControlPlane::class)->redrive( + $stub->id(), + $first->runId(), + $options + )); + } + + public static function substitutedSuccessors(): array + { + return [['foreign'], ['later']]; + } + public static function brokenRedriveReceipts(): array { return [['command'], ['history'], ['divergent']]; From d04e2184525244ffe8b8f4cded3744014a2847eb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:07:43 +0000 Subject: [PATCH 17/32] Retain canonical recovery receipts until explicit scope release --- docs/compensation-scopes.md | 6 + .../Persistence/ScopeRepository.php | 20 +- .../Compensation/Recovery/ScopeRecovery.php | 62 ++++- .../Compensation/Recovery/ScopeRetention.php | 12 + .../Support/WorkflowRunRetentionCleanup.php | 12 + ...1_11_000100_create_compensation_scopes.php | 2 + .../V2/V2CompensationScopeRunBoundaryTest.php | 6 +- tests/Feature/V2/V2CompensationScopeTest.php | 223 +++++++++++++++++- ...tion-recovery-version-one-cold-replay.json | 59 ++++- ...pensation-scope-future-format-refused.json | 59 ++++- ...pensation-scope-future-marker-refused.json | 59 ++++- ...nsation-scope-version-one-cold-replay.json | 59 ++++- .../V2/Compensation/ScopeDescriptorTest.php | 4 + tests/Unit/V2/RecoveryRunAdmissionTest.php | 4 +- 14 files changed, 514 insertions(+), 73 deletions(-) diff --git a/docs/compensation-scopes.md b/docs/compensation-scopes.md index bf9c183f1..a45a5062e 100644 --- a/docs/compensation-scopes.md +++ b/docs/compensation-scopes.md @@ -156,6 +156,12 @@ identities, fingerprints, outcome references and journal tombstones. Forward observation references are recorded in the immutable journal before release. The journal does not retain the forward business value or obligation input. No automatic expiry is added. Ordinary run-detail pruning cannot cascade into this ledger. +Canonical recovery details, including both native receipt copies and redrive +links, remain pinned while the scope is retained. Verification alone does not +remove this hold because old authorized request keys still return their original +runs. Explicit release removes the hold; ordinary forward-origin details can be +pruned independently. Release authorization runs outside replay and SQL +transactions, before the atomic retirement mutation. ## Continue-as-new and originating-run redrive diff --git a/src/V2/Compensation/Persistence/ScopeRepository.php b/src/V2/Compensation/Persistence/ScopeRepository.php index c9aaa50c1..c0d2d9997 100644 --- a/src/V2/Compensation/Persistence/ScopeRepository.php +++ b/src/V2/Compensation/Persistence/ScopeRepository.php @@ -47,6 +47,7 @@ public function open(ScopeIdentity $identity, string $originalRunId): void $created = $existing === null ? $database->table(self::SCOPES)->insertOrIgnore([ 'id' => $identity->id(), 'identity' => PortableDescriptor::encode($identity->toArray()), + 'recovery_instance_id' => $identity->recoveryInstanceId(), 'original_run_id' => $originalRunId, 'status' => 'open', ]) : 0; @@ -179,7 +180,9 @@ public function primaryRequest(ScopeIdentity $identity): array }); } - /** Bind the first native recovery target before an admission can lose its response. */ + /** + * Bind the first native recovery target before an admission can lose its response. + */ public function recoveryRoute( ScopeIdentity $identity, string $requestKey, @@ -193,8 +196,11 @@ public function recoveryRoute( $select, ): ?array { $id = hash('sha256', PortableDescriptor::encode([$identity->id(), $requestKey])); - $row = $database->table('workflow_compensation_requests')->where('scope_id', $scope['id']) - ->where('id', $id)->lockForUpdate()->first(); + $row = $database->table('workflow_compensation_requests') + ->where('scope_id', $scope['id']) + ->where('id', $id) + ->lockForUpdate() + ->first(); if ($row === null) { throw new LogicException('Recovery admission requires an authorized retained request.'); } @@ -222,9 +228,11 @@ public function recoveryRoute( throw new LogicException('The retained recovery route is invalid or its request profile changed.'); } if ($row->recovery_route === null) { - $database->table('workflow_compensation_requests')->where('id', $id)->update([ - 'recovery_route' => PortableDescriptor::encode($route), - ]); + $database->table('workflow_compensation_requests') + ->where('id', $id) + ->update([ + 'recovery_route' => PortableDescriptor::encode($route), + ]); ScopeStorage::append($scope, 'recovery_route_bound', [ 'requestId' => $id, 'route' => $route, diff --git a/src/V2/Compensation/Recovery/ScopeRecovery.php b/src/V2/Compensation/Recovery/ScopeRecovery.php index 516477894..6835fca34 100644 --- a/src/V2/Compensation/Recovery/ScopeRecovery.php +++ b/src/V2/Compensation/Recovery/ScopeRecovery.php @@ -10,8 +10,8 @@ use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Security\ScopeAuthorization; use Workflow\V2\Contracts\WorkflowControlPlane; -use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\HistoryEventType; +use Workflow\V2\Enums\RunStatus; use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\StartOptions; @@ -101,25 +101,52 @@ public function request( if (! $result->accepted() || ! is_string($result->runId())) { throw new LogicException('Native recovery admission refused the retained request.'); } - return $this->recoveryResult($scope, $result->runId(), $result->commandId()); + return $this->recoveryResult($scope, $result->runId(), $route['predecessorRunId'], $result->commandId()); } $redrive = $this->runtime->redrive($scope->recoveryInstanceId(), $route['predecessorRunId'], [ 'namespace' => $scope->namespace, 'request_id' => $admission->token(), ]); - if (($redrive['accepted'] ?? false) !== true) { + if (($redrive['accepted'] ?? false) !== true || ! is_string($redrive['workflow_run_id'] ?? null)) { throw new LogicException( 'Native DW redrive refused unresolved recovery: ' . ($redrive['reason'] ?? 'unknown') . '.' ); } - return $this->recoveryResult($scope, $redrive['workflow_run_id']); + return $this->recoveryResult($scope, $redrive['workflow_run_id'], $route['predecessorRunId']); } - private function recoveryResult(ScopeIdentity $scope, string $runId, ?string $commandId = null): array - { + private function recoveryResult( + ScopeIdentity $scope, + string $runId, + ?string $expectedPredecessor, + ?string $commandId = null, + ): array { $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->where('namespace', $scope->namespace)->findOrFail($runId); - $scopeStatus = $this->repository->metadata($scope)['status']; + ->where('namespace', $scope->namespace) + ->find($runId); + $instance = ConfiguredV2Models::query('instance_model', WorkflowInstance::class) + ->where('namespace', $scope->namespace) + ->find($scope->recoveryInstanceId()); + if (! $run instanceof WorkflowRun || ! $instance instanceof WorkflowInstance + || $run->workflow_instance_id !== $instance->id + || $instance->workflow_class !== RecoveryWorkflow::class + || $run->workflow_class !== RecoveryWorkflow::class + || $instance->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class) + || $run->workflow_type !== $instance->workflow_type + || PortableDescriptor::encode($run->workflowArguments()) + !== PortableDescriptor::encode([$scope->toArray()])) { + throw new LogicException('Native recovery result does not belong to the exact canonical scope.'); + } + $snapshot = $this->repository->metadata($scope); + $receipt = RecoveryRunAdmission::sourceReceipt($instance, $run); + if ($receipt === null + || ($receipt['intent']['expected_predecessor_run_id'] ?? null) !== $expectedPredecessor + || ($receipt['intent']['binding']['original_run_id'] ?? null) !== $snapshot['original_run_id'] + || PortableDescriptor::encode($receipt['intent']['binding']['scope'] ?? []) + !== PortableDescriptor::encode($scope->toArray())) { + throw new LogicException('Native recovery result does not match the retained predecessor and owner.'); + } + $scopeStatus = $snapshot['status']; $nextAction = $scopeStatus === 'verified' ? 'none' : match ($run->status) { RunStatus::Failed, RunStatus::Cancelled, RunStatus::Terminated => 'submit_fresh_authorized_request', RunStatus::Completed => 'inspect_unresolved_obligations', @@ -128,7 +155,9 @@ private function recoveryResult(ScopeIdentity $scope, string $runId, ?string $co return [ 'route' => 'recovery', 'runId' => $runId, - ...($commandId === null ? [] : ['commandId' => $commandId]), + ...($commandId === null ? [] : [ + 'commandId' => $commandId, + ]), 'recoveryStatus' => $run->status->value, 'scopeStatus' => $scopeStatus, 'nextAction' => $nextAction, @@ -138,7 +167,8 @@ private function recoveryResult(ScopeIdentity $scope, string $runId, ?string $co private function selectRecoveryRoute(ScopeIdentity $scope, StartOptions $profile): array { $instance = ConfiguredV2Models::query('instance_model', WorkflowInstance::class) - ->where('namespace', $scope->namespace)->find($scope->recoveryInstanceId()); + ->where('namespace', $scope->namespace) + ->find($scope->recoveryInstanceId()); $run = $instance instanceof WorkflowInstance ? CurrentRunResolver::forInstance($instance) : null; if ($instance !== null && $instance->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class)) { throw new LogicException('The canonical recovery identity is reserved by another type.'); @@ -146,7 +176,8 @@ private function selectRecoveryRoute(ScopeIdentity $scope, StartOptions $profile $predecessor = $run?->id; $intent = null; if ($run !== null) { - $started = $run->historyEvents()->where('event_type', HistoryEventType::WorkflowStarted->value)->sole(); + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::WorkflowStarted->value)->sole(); $intent = $started->payload['compensation_recovery']['intent'] ?? null; if (! is_array($intent) || ($intent['version'] ?? null) !== 1 || ! array_key_exists('expected_predecessor_run_id', $intent) @@ -154,7 +185,10 @@ private function selectRecoveryRoute(ScopeIdentity $scope, StartOptions $profile throw new LogicException('Recovery has no retained native predecessor and profile binding.'); } $expected = self::profileDescriptor($profile); - $recorded = ['version' => 1, ...array_intersect_key($intent['binding']['profile'], $expected)]; + $recorded = [ + 'version' => 1, + ...array_intersect_key($intent['binding']['profile'], $expected), + ]; if (PortableDescriptor::encode($expected) !== PortableDescriptor::encode($recorded)) { throw new LogicException('Recovery requests must retain the initial recovery profile.'); } @@ -177,7 +211,9 @@ private function selectRecoveryRoute(ScopeIdentity $scope, StartOptions $profile private function assertHostContext(): void { if (WorkflowFiberContext::active() || ConfiguredV2Models::connection()->transactionLevel() !== 0) { - throw new LogicException('Recovery authorization requires an unlocked host context outside workflow replay.'); + throw new LogicException( + 'Recovery authorization requires an unlocked host context outside workflow replay.' + ); } } diff --git a/src/V2/Compensation/Recovery/ScopeRetention.php b/src/V2/Compensation/Recovery/ScopeRetention.php index 8715b9698..6c830f540 100644 --- a/src/V2/Compensation/Recovery/ScopeRetention.php +++ b/src/V2/Compensation/Recovery/ScopeRetention.php @@ -13,6 +13,7 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\SqliteTransaction; +use Workflow\V2\Support\WorkflowFiberContext; /** * Explicit retirement. Normal run-history pruning never removes retained obligations. @@ -29,8 +30,10 @@ public function __construct( public function release(ScopeIdentity $scope, string $principal): void { + $this->assertHostContext(); $policy = $this->authorization->assertAllowed($scope, $principal, 'release'); PortableDescriptor::identifier($policy, 'authorization policy version'); + $this->assertHostContext(); $snapshot = $this->repository->inspect($scope); SqliteTransaction::run(static function () use ($scope, $principal, $policy, $snapshot): void { $origin = ConfiguredV2Models::query('run_model', WorkflowRun::class)->whereKey( @@ -91,4 +94,13 @@ public function release(ScopeIdentity $scope, string $principal): void }); }, connection: ConfiguredV2Models::connection()); } + + private function assertHostContext(): void + { + if (WorkflowFiberContext::active() || ConfiguredV2Models::connection()->transactionLevel() !== 0) { + throw new LogicException( + 'Scope release authorization requires an unlocked host context outside workflow replay.' + ); + } + } } diff --git a/src/V2/Support/WorkflowRunRetentionCleanup.php b/src/V2/Support/WorkflowRunRetentionCleanup.php index 38e02fb08..f03001ddc 100644 --- a/src/V2/Support/WorkflowRunRetentionCleanup.php +++ b/src/V2/Support/WorkflowRunRetentionCleanup.php @@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use LogicException; +use Workflow\V2\Compensation\Persistence\ScopeStorage; use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; @@ -197,6 +198,17 @@ public static function retentionHoldReason(WorkflowRun|string $run): ?string { $run = $run instanceof WorkflowRun ? $run : ConfiguredV2Models::query('run_model', WorkflowRun::class)->findOrFail($run); + $database = $run->getConnection(); + // Retained request keys still need both native receipt copies and + // redrive links, even after verification. Only explicit release ends + // this hold. A concurrent release can at most defer pruning one pass. + if ($database->getSchemaBuilder()->hasTable(ScopeStorage::SCOPES) + && $database->table(ScopeStorage::SCOPES) + ->where('recovery_instance_id', $run->workflow_instance_id) + ->where('status', '!=', 'released') + ->exists()) { + return 'retained_compensation_scope'; + } $schedules = $run->historyEvents() ->where('event_type', HistoryEventType::ActivityScheduled->value) ->where('payload->activity->cancellation_policy', CancellationPolicy::Abandon->value)->get(); diff --git a/src/migrations/2026_11_11_000100_create_compensation_scopes.php b/src/migrations/2026_11_11_000100_create_compensation_scopes.php index 87375e3e6..f675ec1d7 100644 --- a/src/migrations/2026_11_11_000100_create_compensation_scopes.php +++ b/src/migrations/2026_11_11_000100_create_compensation_scopes.php @@ -13,6 +13,8 @@ public function up(): void $table->string('id', 64) ->primary(); $table->text('identity'); + $table->string('recovery_instance_id', 128) + ->unique(); $table->string('original_run_id', 128) ->index(); $table->string('status', 32); diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index b5ecc66fd..1d04fc009 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -208,7 +208,11 @@ public function testNativeChildRetryCannotTakeOverTheOriginalScopeGeneration(): 'runs' => WorkflowRun::query()->whereIn('id', $links->pluck('child_workflow_run_id')) ->get(['id', 'status'])->toArray(), 'failures' => WorkflowFailure::query()->whereIn('workflow_run_id', $links->pluck('child_workflow_run_id')) - ->pluck('message')->all(), + ->pluck('message') + ->all(), + 'tasks' => WorkflowTask::query()->whereIn('workflow_run_id', $links->pluck('child_workflow_run_id')) + ->get(['task_type', 'status', 'attempt_count', 'last_error', 'last_claim_error']) + ->toArray(), ], JSON_THROW_ON_ERROR)); $origin = WorkflowRun::query()->findOrFail($links[0]->child_workflow_run_id); $retry = WorkflowRun::query()->findOrFail($links[1]->child_workflow_run_id); diff --git a/tests/Feature/V2/V2CompensationScopeTest.php b/tests/Feature/V2/V2CompensationScopeTest.php index bbbdcfd9a..a393a3d31 100644 --- a/tests/Feature/V2/V2CompensationScopeTest.php +++ b/tests/Feature/V2/V2CompensationScopeTest.php @@ -15,12 +15,19 @@ use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Recovery\RecoveryWorkflow; use Workflow\V2\Compensation\Recovery\ScopeRecovery; +use Workflow\V2\Compensation\Recovery\ScopeRetention; use Workflow\V2\Compensation\Security\ScopeAuthorization; +use Workflow\V2\Contracts\WorkflowControlPlane; +use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowHistoryEvent; +use Workflow\V2\Models\WorkflowLink; use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Models\WorkflowTask; use Workflow\V2\StartOptions; -use Workflow\V2\Support\WorkflowReplayer; use Workflow\V2\Support\ConfiguredV2Models; +use Workflow\V2\Support\WorkflowFiberContext; +use Workflow\V2\Support\WorkflowReplayer; +use Workflow\V2\Support\WorkflowRunRetentionCleanup; use Workflow\V2\WorkflowStub; final class V2CompensationScopeTest extends TestCase @@ -108,14 +115,18 @@ public function testFreshRequestsResumeStoppedRecoveryWithoutRebindingOldRequest $stopped = $stop === 'cancel' ? $firstStub->cancel('operator-stop') : $firstStub->terminate('operator-stop'); $this->assertTrue($stopped->accepted()); $firstHistory = WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId']) - ->orderBy('sequence')->get()->toArray(); + ->orderBy('sequence') + ->get() + ->toArray(); $second = app(ScopeRecovery::class)->request($scope, 'recovery-2', 'operator-a', 'cancel'); $this->assertNotSame($first['runId'], $second['runId']); $secondStub = WorkflowStub::loadRun($second['runId']); $stopped = $stop === 'cancel' ? $secondStub->cancel('operator-stop') : $secondStub->terminate('operator-stop'); $this->assertTrue($stopped->accepted()); $secondHistory = WorkflowHistoryEvent::query()->where('workflow_run_id', $second['runId']) - ->orderBy('sequence')->get()->toArray(); + ->orderBy('sequence') + ->get() + ->toArray(); foreach ([['recovery-1', $first], ['recovery-2', $second]] as [$key, $receipt]) { $again = app(ScopeRecovery::class)->request($scope, $key, 'operator-a', 'cancel'); @@ -124,7 +135,10 @@ public function testFreshRequestsResumeStoppedRecoveryWithoutRebindingOldRequest $this->assertSame($stop === 'cancel' ? 'cancelled' : 'terminated', $again['recoveryStatus']); $this->assertSame('submit_fresh_authorized_request', $again['nextAction']); } - $this->assertSame(2, WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count()); + $this->assertSame( + 2, + WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count() + ); Cache::forget('test:scope:unknown-first'); $third = app(ScopeRecovery::class)->request($scope, 'recovery-3', 'operator-a', 'cancel'); $this->assertNotContains($third['runId'], [$first['runId'], $second['runId']]); @@ -135,9 +149,13 @@ public function testFreshRequestsResumeStoppedRecoveryWithoutRebindingOldRequest static fn (string $key): bool => $key === 'second', ))); $this->assertSame($firstHistory, WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId']) - ->orderBy('sequence')->get()->toArray()); + ->orderBy('sequence') + ->get() + ->toArray()); $this->assertSame($secondHistory, WorkflowHistoryEvent::query()->where('workflow_run_id', $second['runId']) - ->orderBy('sequence')->get()->toArray()); + ->orderBy('sequence') + ->get() + ->toArray()); $retained = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); $this->assertSame($first['runId'], $retained['runId']); $this->assertSame($first['commandId'], $retained['commandId']); @@ -166,7 +184,10 @@ public function testRetainedRecoveryRequestCannotChangeItsInitialBudget(): void } catch (\LogicException $error) { $this->assertStringContainsString('profile changed', $error->getMessage()); } - $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count()); + $this->assertSame( + 1, + WorkflowRun::query()->where('workflow_instance_id', $scope->recoveryInstanceId())->count() + ); $this->assertSame($history, WorkflowHistoryEvent::query()->where('workflow_run_id', $first['runId'])->count()); } @@ -175,8 +196,9 @@ public function testRecoveryRefusesCallerTransactionsBeforeHostAuthorization(): [$scope] = $this->completedOrigin(); $calls = 0; $this->app->instance(ScopeAuthorization::class, new class($calls) implements ScopeAuthorization { - public function __construct(private int &$calls) - { + public function __construct( + private int &$calls + ) { } public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string @@ -198,6 +220,43 @@ public function assertAllowed(ScopeIdentity $scope, string $principal, string $o } } + public function testRedriveResponseCannotReturnAnotherCanonicalScopeInTheSameNamespace(): void + { + [$scope] = $this->completedOrigin(); + Cache::put('test:scope:unknown-first', true); + $first = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $this->driveNativeRetries($first['runId']); + $foreign = new ScopeIdentity('shop', 'client-a', 'foreign-response', '1'); + WorkflowStub::make(TestRetainedScopeWorkflow::class, 'foreign-response-origin', 'client-a') + ->start($foreign->toArray(), 'complete'); + $foreignRecovery = app(ScopeRecovery::class)->request($foreign, 'foreign-1', 'operator-a', 'cancel'); + $before = [ + WorkflowRun::query()->count(), + WorkflowCommand::query()->count(), + WorkflowHistoryEvent::query()->count(), + ]; + $runtime = $this->createMock(WorkflowControlPlane::class); + $runtime->expects($this->once()) + ->method('redrive') + ->willReturn([ + 'accepted' => true, + 'workflow_run_id' => $foreignRecovery['runId'], + ]); + $this->app->instance(WorkflowControlPlane::class, $runtime); + try { + app(ScopeRecovery::class)->request($scope, 'recovery-2', 'operator-a', 'cancel'); + $this->fail('The scoped facade accepted a foreign native response.'); + } catch (\LogicException $error) { + $this->assertStringContainsString('exact canonical scope', $error->getMessage()); + } + $this->assertSame($before, [ + WorkflowRun::query()->count(), + WorkflowCommand::query()->count(), + WorkflowHistoryEvent::query()->count(), + ]); + $this->assertSame('requested', (new ScopeRepository())->inspect($scope)['status']); + } + public function testRetryExhaustionStaysUnresolvedAndNativeRedriveReusesVerifiedWork(): void { [$scope] = $this->completedOrigin(); @@ -408,13 +467,19 @@ public function testFailedChildNeedsAnExplicitFreshRequestBeforeParentRecoveryCa $this->assertSame('failed', $childRun->refresh()->status->value); $this->driveNativeRetries($parentRecovery['runId']); $this->assertSame('failed', WorkflowRun::query()->findOrFail($parentRecovery['runId'])->status->value); - $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->count()); + $this->assertSame( + 1, + WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->count() + ); $this->assertNotContains('waiting-parent/first', Cache::get('test:scope:undo-scoped-log')); Cache::forget('test:scope:unknown-first'); $retained = app(ScopeRecovery::class)->request($parent, 'parent-1', 'operator-a', 'cancel'); $this->assertSame($parentRecovery['runId'], $retained['runId']); $this->assertSame('submit_fresh_authorized_request', $retained['nextAction']); - $this->assertSame(1, WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->count()); + $this->assertSame( + 1, + WorkflowRun::query()->where('workflow_instance_id', $child->recoveryInstanceId())->count() + ); app(ScopeRecovery::class)->request($child, 'explicit-child-2', 'operator-a', 'child-recovery'); $this->assertSame('verified', (new ScopeRepository())->inspect($child)['status']); @@ -501,6 +566,115 @@ public function testOrdinaryRunPruningPreservesRetainedScopeRecovery(): void $this->assertSame(['second', 'first'], Cache::get('test:scope:undo-log')); } + public static function retainedRecoveryStates(): array + { + return [['failed'], ['cancelled'], ['terminated'], ['completed']]; + } + + #[DataProvider('retainedRecoveryStates')] + public function testCanonicalRecoveryDetailsRemainPinnedUntilExplicitRelease(string $terminal): void + { + [$scope] = $this->completedOrigin(); + if ($terminal !== 'completed') { + Cache::put('test:scope:unknown-first', true); + } + $first = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $firstStub = WorkflowStub::loadRun($first['runId']); + match ($terminal) { + 'cancelled' => $firstStub->cancel('operator-stop'), + 'terminated' => $firstStub->terminate('operator-stop'), + 'failed' => $this->driveNativeRetries($first['runId']), + default => null, + }; + $this->assertSame($terminal, $firstStub->refresh()->status()); + $this->assertRecoveryPruningHeld($first['runId']); + $again = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $this->assertSame($first['runId'], $again['runId']); + Cache::forget('test:scope:unknown-first'); + $last = $terminal === 'completed' ? $first + : app(ScopeRecovery::class)->request($scope, 'recovery-2', 'operator-a', 'cancel'); + $this->driveNativeRetries($last['runId']); + $this->assertSame('verified', (new ScopeRepository())->inspect($scope)['status']); + foreach (array_unique([$first['runId'], $last['runId']]) as $runId) { + $this->assertRecoveryPruningHeld($runId); + } + $again = app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $this->assertSame($first['runId'], $again['runId']); + app(ScopeRetention::class)->release($scope, 'operator-a'); + $commandsDeleted = 0; + foreach (array_unique([$first['runId'], $last['runId']]) as $runId) { + $this->assertNull(WorkflowRunRetentionCleanup::retentionHoldReason($runId)); + $report = WorkflowRunRetentionCleanup::pruneRun($runId); + $commandsDeleted += $report['commands_deleted']; + $this->assertGreaterThan(0, $report['history_events_deleted']); + $this->assertNotNull(WorkflowRun::query()->findOrFail($runId)->details_pruned_at); + } + $this->assertGreaterThan(0, $commandsDeleted); + } + + public static function invalidReleaseContexts(): array + { + return [['caller-transaction'], ['replay'], ['authorization-transaction']]; + } + + #[DataProvider('invalidReleaseContexts')] + public function testReleaseRequiresUnlockedHostAuthorization(string $context): void + { + [$scope] = $this->completedOrigin(); + app(ScopeRecovery::class)->request($scope, 'recovery-1', 'operator-a', 'cancel'); + $before = (new ScopeRepository())->inspect($scope); + $calls = 0; + $this->app->instance(ScopeAuthorization::class, new class($calls, $context) implements ScopeAuthorization { + public function __construct( + private int &$calls, + private string $context + ) { + } + + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + ++$this->calls; + if ($this->context === 'authorization-transaction') { + ConfiguredV2Models::connection()->beginTransaction(); + } + return 'release-policy-1'; + } + }); + $release = function () use ($scope): void { + try { + app(ScopeRetention::class)->release($scope, 'operator-a'); + $this->fail('Release crossed the unlocked host boundary.'); + } catch (\LogicException $error) { + $this->assertStringContainsString('unlocked host context', $error->getMessage()); + } + }; + $database = ConfiguredV2Models::connection(); + try { + if ($context === 'caller-transaction') { + $database->beginTransaction(); + } + if ($context === 'replay') { + (new \Fiber(static function () use ($release): void { + WorkflowFiberContext::enter(); + try { + $release(); + } finally { + WorkflowFiberContext::leave(); + } + }))->start(); + } else { + $release(); + } + $this->assertSame($context === 'authorization-transaction' ? 1 : 0, $calls); + $this->assertSame($before, (new ScopeRepository())->inspect($scope)); + } finally { + while ($database->transactionLevel() > 0) { + $database->rollBack(); + } + } + $this->assertSame($before, (new ScopeRepository())->inspect($scope)); + } + public function testTamperedRetainedDescriptorIsRefusedBeforeProviderCall(): void { [$scope] = $this->completedOrigin(); @@ -577,7 +751,10 @@ public function testReleaseRefusesUnresolvedWorkThenRetainsAnUnreusableTombstone public function testDefaultEmbeddedNamespaceIsPreservedWithoutInventingAnAlias(): void { - config()->set('workflows.v2.namespaces', [null]); + config([ + 'workflows.v2.namespaces' => null, + 'workflows.v2.namespace' => null, + ]); $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string { @@ -652,6 +829,28 @@ public function testCancellationWhileChildWaitsResolvesItsSagaBeforeEarlierParen ); } + private function assertRecoveryPruningHeld(string $runId): void + { + $snapshot = static fn (): array => [ + WorkflowCommand::query()->orderBy('id')->get()->toArray(), + WorkflowHistoryEvent::query()->orderBy('id')->get()->toArray(), + WorkflowLink::query()->orderBy('id')->get()->toArray(), + WorkflowTask::query()->orderBy('id')->get()->toArray(), + WorkflowRun::query()->findOrFail($runId)->toArray(), + ]; + $before = $snapshot(); + $this->assertNotEmpty($before[0]); + $this->assertNotEmpty($before[1]); + try { + $report = WorkflowRunRetentionCleanup::pruneRun($runId); + $this->fail('Pruned retained recovery receipts: ' . json_encode($report, JSON_THROW_ON_ERROR)); + } catch (\LogicException $error) { + $this->assertStringContainsString('retained_compensation_scope', $error->getMessage()); + } + $this->assertSame($before, $snapshot()); + $this->assertSame('retained_compensation_scope', WorkflowRunRetentionCleanup::retentionHoldReason($runId)); + } + private function driveNativeRetries(string $runId): void { for ($step = 0; $step < 20; $step++) { diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json index 4d5d43c79..914e6050d 100644 --- a/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json +++ b/tests/Fixtures/V2/ReplayRegression/compensation-recovery-version-one-cold-replay.json @@ -24,8 +24,7 @@ "sequence": 1, "event_type": "WorkflowStarted", "payload": { - "workflow_type": "dw.compensation.recovery.v1", - "namespace": "client-a" + "workflow_type": "dw.compensation.recovery.v1" }, "recorded_at": "2026-11-13T00:00:00+00:00" }, @@ -49,8 +48,18 @@ "activity_type": "dw.compensation.prepare-recovery.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\PrepareRecoveryActivity", "activity_execution_id": "corpus-execution-2", - "arguments": "wwHioz3/VYAiNwwCDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAA", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-2", + "idempotency_key": "corpus-execution-2", + "sequence": 2, + "type": "dw.compensation.prepare-recovery.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\PrepareRecoveryActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwCDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAA" + } }, "recorded_at": "2026-11-13T00:00:02+00:00" }, @@ -91,8 +100,18 @@ "activity_type": "dw.compensation.compensate-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", "activity_execution_id": "corpus-execution-3", - "arguments": "wwHioz3/VYAiNwwEDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-3", + "idempotency_key": "corpus-execution-3", + "sequence": 3, + "type": "dw.compensation.compensate-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwEDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmAA==" + } }, "recorded_at": "2026-11-13T00:00:05+00:00" }, @@ -133,8 +152,18 @@ "activity_type": "dw.compensation.compensate-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", "activity_execution_id": "corpus-execution-4", - "arguments": "wwHioz3/VYAiNwwEDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4AA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-4", + "idempotency_key": "corpus-execution-4", + "sequence": 4, + "type": "dw.compensation.compensate-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\CompensateObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwEDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4AA==" + } }, "recorded_at": "2026-11-13T00:00:08+00:00" }, @@ -175,8 +204,18 @@ "activity_type": "dw.compensation.verify-scope.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\VerifyScopeActivity", "activity_execution_id": "corpus-execution-5", - "arguments": "wwHioz3/VYAiNwwCDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAA", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-5", + "idempotency_key": "corpus-execution-5", + "sequence": 5, + "type": "dw.compensation.verify-scope.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\VerifyScopeActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwCDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAA" + } }, "recorded_at": "2026-11-13T00:00:11+00:00" }, diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json index 22c97d69d..8ff10ebec 100644 --- a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-format-refused.json @@ -25,8 +25,7 @@ "sequence": 1, "event_type": "WorkflowStarted", "payload": { - "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", - "namespace": "client-a" + "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow" }, "recorded_at": "2026-11-13T00:00:00+00:00" }, @@ -50,8 +49,18 @@ "activity_type": "dw.compensation.register-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", "activity_execution_id": "corpus-execution-2", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-2", + "idempotency_key": "corpus-execution-2", + "sequence": 2, + "type": "dw.compensation.register-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:02+00:00" }, @@ -92,8 +101,18 @@ "activity_type": "dw.compensation.forward-operation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", "activity_execution_id": "corpus-execution-3", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-3", + "idempotency_key": "corpus-execution-3", + "sequence": 3, + "type": "dw.compensation.forward-operation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:05+00:00" }, @@ -134,8 +153,18 @@ "activity_type": "dw.compensation.register-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", "activity_execution_id": "corpus-execution-4", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-4", + "idempotency_key": "corpus-execution-4", + "sequence": 4, + "type": "dw.compensation.register-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:08+00:00" }, @@ -176,8 +205,18 @@ "activity_type": "dw.compensation.forward-operation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", "activity_execution_id": "corpus-execution-5", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-5", + "idempotency_key": "corpus-execution-5", + "sequence": 5, + "type": "dw.compensation.forward-operation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:11+00:00" }, diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json index 08822fc0d..4e5e68a67 100644 --- a/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-future-marker-refused.json @@ -25,8 +25,7 @@ "sequence": 1, "event_type": "WorkflowStarted", "payload": { - "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", - "namespace": "client-a" + "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow" }, "recorded_at": "2026-11-13T00:00:00+00:00" }, @@ -50,8 +49,18 @@ "activity_type": "dw.compensation.register-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", "activity_execution_id": "corpus-execution-2", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-2", + "idempotency_key": "corpus-execution-2", + "sequence": 2, + "type": "dw.compensation.register-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:02+00:00" }, @@ -92,8 +101,18 @@ "activity_type": "dw.compensation.forward-operation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", "activity_execution_id": "corpus-execution-3", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-3", + "idempotency_key": "corpus-execution-3", + "sequence": 3, + "type": "dw.compensation.forward-operation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:05+00:00" }, @@ -134,8 +153,18 @@ "activity_type": "dw.compensation.register-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", "activity_execution_id": "corpus-execution-4", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-4", + "idempotency_key": "corpus-execution-4", + "sequence": 4, + "type": "dw.compensation.register-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:08+00:00" }, @@ -176,8 +205,18 @@ "activity_type": "dw.compensation.forward-operation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", "activity_execution_id": "corpus-execution-5", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-5", + "idempotency_key": "corpus-execution-5", + "sequence": 5, + "type": "dw.compensation.forward-operation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:11+00:00" }, diff --git a/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json index 0898eeb56..4d4746661 100644 --- a/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json +++ b/tests/Fixtures/V2/ReplayRegression/compensation-scope-version-one-cold-replay.json @@ -25,8 +25,7 @@ "sequence": 1, "event_type": "WorkflowStarted", "payload": { - "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow", - "namespace": "client-a" + "workflow_type": "Tests\\Fixtures\\V2\\TestRetainedScopeWorkflow" }, "recorded_at": "2026-11-13T00:00:00+00:00" }, @@ -50,8 +49,18 @@ "activity_type": "dw.compensation.register-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", "activity_execution_id": "corpus-execution-2", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-2", + "idempotency_key": "corpus-execution-2", + "sequence": 2, + "type": "dw.compensation.register-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgpmaXJzdBZoYW5kbGVyVHlwZQocdGVzdC5vcGVyYXRpb24caGFuZGxlclZlcnNpb24KAjEKaW5wdXQOAghtb2RlChBjb21wbGV0ZQAAChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:02+00:00" }, @@ -92,8 +101,18 @@ "activity_type": "dw.compensation.forward-operation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", "activity_execution_id": "corpus-execution-3", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-3", + "idempotency_key": "corpus-execution-3", + "sequence": 3, + "type": "dw.compensation.forward-operation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAE5OGE3ZTA3YzQzNmFjMmVhZTA3MzhjODdlNjkxYjFiMDljZWExNTkyYTU2Yjg2MDAyZTAwZmVjYjUzNzJhNmI4ChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:05+00:00" }, @@ -134,8 +153,18 @@ "activity_type": "dw.compensation.register-obligation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", "activity_execution_id": "corpus-execution-4", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-4", + "idempotency_key": "corpus-execution-4", + "sequence": 4, + "type": "dw.compensation.register-obligation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\RegisterObligationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAOCg52ZXJzaW9uBAIGa2V5CgxzZWNvbmQWaGFuZGxlclR5cGUKHHRlc3Qub3BlcmF0aW9uHGhhbmRsZXJWZXJzaW9uCgIxCmlucHV0DAAAChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:08+00:00" }, @@ -176,8 +205,18 @@ "activity_type": "dw.compensation.forward-operation.v1", "activity_class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", "activity_execution_id": "corpus-execution-5", - "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==", - "payload_codec": "avro" + "activity": { + "id": "corpus-execution-5", + "idempotency_key": "corpus-execution-5", + "sequence": 5, + "type": "dw.compensation.forward-operation.v1", + "class": "Workflow\\V2\\Compensation\\Activities\\ForwardOperationActivity", + "local_activity": false, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "arguments": "wwHioz3/VYAiNwwGDgoOdmVyc2lvbgQCFmFwcGxpY2F0aW9uCghzaG9wEm5hbWVzcGFjZQoQY2xpZW50LWEGa2V5CipyZXRhaW5lZC1jb3JwdXMtb3JkZXIUZ2VuZXJhdGlvbgoCMQAKgAFiOTY2MGYxZWRmZjlmM2FlMDFkZGM2Mjc0MDQzY2I3YWEzNmZlODc0MjU1ZjUyMzcxYmNjNDU3ZTc5MThhNDFmChRvcGVyYXRvci1hAA==" + } }, "recorded_at": "2026-11-13T00:00:11+00:00" }, diff --git a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php index 980d4350c..e239e27c0 100644 --- a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php +++ b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php @@ -48,6 +48,10 @@ public function testVersionOnePersistedFormatsHaveStableCanonicalIdentities(): v 'reference' => 'booking-1', ]); $this->assertSame('ceb73af611903fb425f8bcdb3b415d52be6f619cc5f1f8f6c3c3996aea4c5a8d', $scope->id()); + $this->assertSame( + 'dw-compensation-ceb73af611903fb425f8bcdb3b415d52be6f619cc5f1f8f6c3c3996aea4c5a8d', + $scope->recoveryInstanceId() + ); $this->assertSame( '43a5906816309c817fafc95d183ade5b5786123115a765b85b13b12440bcfa13', $obligation->fingerprint() diff --git a/tests/Unit/V2/RecoveryRunAdmissionTest.php b/tests/Unit/V2/RecoveryRunAdmissionTest.php index 3f75168e7..693cf8ebc 100644 --- a/tests/Unit/V2/RecoveryRunAdmissionTest.php +++ b/tests/Unit/V2/RecoveryRunAdmissionTest.php @@ -438,7 +438,9 @@ public function testRetainedRecoveryRedriveRefusesAnotherAdmission(string $subst ]; $redrive = app(WorkflowControlPlane::class)->redrive($stub->id(), $first->runId(), $options); self::assertTrue($redrive['accepted']); - $owner = WorkflowCommand::query()->findOrFail($first->commandId())->context['compensation_recovery']['intent']['binding']['original_run_id']; + $owner = WorkflowCommand::query()->findOrFail( + $first->commandId() + )->context['compensation_recovery']['intent']['binding']['original_run_id']; $targetScope = $substitution === 'foreign' ? new ScopeIdentity($scope->application, $scope->namespace, 'different-order', $scope->generation) : $scope; From 89c41414a59d3dca0cb77989ab0e8d1ad275c8fd Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:42:30 +0000 Subject: [PATCH 18/32] Preserve embedded child definition authority across native retries --- src/V2/Support/ChildRunHistory.php | 6 + src/V2/Support/DefaultWorkflowTaskBridge.php | 17 ++ .../V2/V2CompensationScopeConcurrencyTest.php | 48 ++++- ...ld-retry-definition-drift-cold-replay.json | 44 +++++ ...definition-legacy-absence-cold-replay.json | 43 +++++ ...retry-definition-recorded-cold-replay.json | 63 +++++++ .../V2/ChildDefinitionFingerprintTest.php | 166 ++++++++++++++++++ 7 files changed, 380 insertions(+), 7 deletions(-) create mode 100644 tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json create mode 100644 tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json create mode 100644 tests/Fixtures/V2/ReplayRegression/child-retry-definition-recorded-cold-replay.json create mode 100644 tests/Unit/V2/ChildDefinitionFingerprintTest.php diff --git a/src/V2/Support/ChildRunHistory.php b/src/V2/Support/ChildRunHistory.php index 797394a85..eb2b1e65c 100644 --- a/src/V2/Support/ChildRunHistory.php +++ b/src/V2/Support/ChildRunHistory.php @@ -161,9 +161,15 @@ public static function recordRetryStarted( ], $policies, $parallelMetadata), static fn (mixed $value): bool => $value !== null)); } + // A retry remains on its admitted definition. Looking up current code + // here would turn registration drift or legacy absence into new authority. + $fingerprint = WorkflowDefinitionFingerprint::recordedForRun($failedChildRun); WorkflowHistoryEvent::record($retryRun, HistoryEventType::WorkflowStarted, array_merge([ 'workflow_class' => $retryRun->workflow_class, 'workflow_type' => $retryRun->workflow_type, + ...($fingerprint === null ? [] : [ + 'workflow_definition_fingerprint' => $fingerprint, + ]), 'workflow_instance_id' => $retryRun->workflow_instance_id, 'workflow_run_id' => $retryRun->id, 'parent_workflow_instance_id' => $parentRun->workflow_instance_id, diff --git a/src/V2/Support/DefaultWorkflowTaskBridge.php b/src/V2/Support/DefaultWorkflowTaskBridge.php index e59f6d03a..db9964fdc 100644 --- a/src/V2/Support/DefaultWorkflowTaskBridge.php +++ b/src/V2/Support/DefaultWorkflowTaskBridge.php @@ -4703,6 +4703,7 @@ private function applyStartChildWorkflow( WorkflowHistoryEvent::record($childRun, HistoryEventType::WorkflowStarted, [ 'workflow_class' => $workflowType, 'workflow_type' => $workflowType, + ...self::embeddedChildDefinition($workflowType), 'workflow_instance_id' => $childRun->workflow_instance_id, 'workflow_run_id' => $childRun->id, 'parent_workflow_instance_id' => $run->workflow_instance_id, @@ -4738,6 +4739,22 @@ private function applyStartChildWorkflow( return $sequence + 1; } + /** + * @return array{workflow_definition_fingerprint?: string} + */ + private static function embeddedChildDefinition(string $workflowType): array + { + try { + $class = TypeRegistry::resolveWorkflowClass($workflowType, $workflowType); + } catch (LogicException) { + return []; + } + $fingerprint = WorkflowDefinition::fingerprint($class); + return $fingerprint === null ? [] : [ + 'workflow_definition_fingerprint' => $fingerprint, + ]; + } + /** * @param array{type: string, result: string, payload_codec?: string|null} $command */ diff --git a/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php b/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php index cae05c4d4..d2eb94290 100644 --- a/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php +++ b/tests/Feature/V2/V2CompensationScopeConcurrencyTest.php @@ -5,6 +5,7 @@ namespace Tests\Feature\V2; use Closure; +use Illuminate\Database\Events\QueryExecuted; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use RuntimeException; @@ -38,6 +39,9 @@ protected function setUp(): void $this->app->instance(ScopeAuthorization::class, new class() implements ScopeAuthorization { public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string { + if (ConfiguredV2Models::connection()->transactionLevel() !== 0) { + throw new RuntimeException('Host authorization must precede the native request transaction.'); + } if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { throw new RuntimeException('Scope access refused.'); } @@ -52,7 +56,8 @@ public function testConcurrentRequestsStartOneCanonicalRecovery(): void $results = $this->race( static fn (int $index): array => app( ScopeRecovery::class - )->request($scope, 'request-' . $index, 'operator-a', 'cancel') + )->request($scope, 'request-' . $index, 'operator-a', 'cancel'), + holdRequestInsert: true, ); $this->assertSame($results[0]['runId'], $results[1]['runId']); $this->assertSame( @@ -127,7 +132,7 @@ private function openOrigin(): ScopeIdentity return $scope; } - private function race(Closure $operation, bool $expectLock = true): array + private function race(Closure $operation, bool $expectLock = true, bool $holdRequestInsert = false): array { $root = sys_get_temp_dir() . '/dw-conflict-' . bin2hex(random_bytes(6)); mkdir($root); @@ -146,11 +151,33 @@ private function race(Closure $operation, bool $expectLock = true): array DB::statement("SET application_name = 'dw-conflict-probe-{$index}'"); DB::statement("SET lock_timeout = '15s'"); if ($index === 0) { - DB::beginTransaction(); - $result = $operation($index); - touch($root . '/holding'); - self::waitFile($root . '/release'); - DB::commit(); + if ($holdRequestInsert) { + $held = false; + DB::listen(static function (QueryExecuted $query) use ($root, &$held): void { + if ($held || ! str_starts_with(strtolower($query->sql), 'insert') + || ! str_contains($query->sql, 'workflow_compensation_requests')) { + return; + } + if ($query->connection->transactionLevel() === 0) { + throw new RuntimeException( + 'The request barrier requires a native transaction.' + ); + } + $held = true; + touch($root . '/holding'); + self::waitFile($root . '/release'); + }); + $result = $operation($index); + if (! $held) { + throw new RuntimeException('The host request did not reach its native insert.'); + } + } else { + DB::beginTransaction(); + $result = $operation($index); + touch($root . '/holding'); + self::waitFile($root . '/release'); + DB::commit(); + } } else { self::waitFile($root . '/holding'); $result = $operation($index); @@ -228,6 +255,13 @@ private static function waitFile(string $file): void { $deadline = microtime(true) + 10; while (! is_file($file)) { + $failed = dirname($file) . '/0.json'; + if (is_file($failed)) { + $payload = json_decode(file_get_contents($failed), true, flags: JSON_THROW_ON_ERROR); + if (isset($payload['error'])) { + throw new RuntimeException('Conflict probe failed before its barrier: ' . $payload['error']); + } + } if (microtime(true) >= $deadline) { throw new RuntimeException('Probe barrier timed out.'); } diff --git a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json new file mode 100644 index 000000000..b580a769b --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "child-retry-definition-drift-cold-replay", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner", + "query-state-replayer", + "workflow-executor" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "arguments": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "workflow_type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "workflow_instance_id": "child-instance", + "workflow_run_id": "retry-run", + "parent_workflow_instance_id": "parent-instance", + "parent_workflow_run_id": "parent-run", + "parent_sequence": 1, + "workflow_link_id": "retry-link", + "child_call_id": "child-call", + "retry_attempt": 2, + "retry_of_child_workflow_run_id": "failed-child-run", + "workflow_definition_fingerprint": "sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + } + ], + "expected_failure": { + "type": "version_marker_incompatible_range", + "exception": "Workflow\\V2\\Exceptions\\VersionRangeMismatchException" + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json new file mode 100644 index 000000000..a0cd241e5 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "child-retry-definition-legacy-absence-cold-replay", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner", + "query-state-replayer", + "workflow-executor" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "arguments": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "workflow_type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "workflow_instance_id": "child-instance", + "workflow_run_id": "retry-run", + "parent_workflow_instance_id": "parent-instance", + "parent_workflow_run_id": "parent-run", + "parent_sequence": 1, + "workflow_link_id": "retry-link", + "child_call_id": "child-call", + "retry_attempt": 2, + "retry_of_child_workflow_run_id": "failed-child-run" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + } + ], + "expected_failure": { + "type": "version_marker_incompatible_range", + "exception": "Workflow\\V2\\Exceptions\\VersionRangeMismatchException" + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-recorded-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-recorded-cold-replay.json new file mode 100644 index 000000000..5cab79af4 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-recorded-cold-replay.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "child-retry-definition-recorded-cold-replay", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner", + "query-state-replayer", + "workflow-executor" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "arguments": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "workflow_type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", + "workflow_instance_id": "child-instance", + "workflow_run_id": "retry-run", + "parent_workflow_instance_id": "parent-instance", + "parent_workflow_run_id": "parent-run", + "parent_sequence": 1, + "workflow_link_id": "retry-link", + "child_call_id": "child-call", + "retry_attempt": 2, + "retry_of_child_workflow_run_id": "failed-child-run", + "workflow_definition_fingerprint": "sha256:86ebbe8f1dff670d3d4d3c822fed20d605f16acf4ddca2f0e962987fd4869082" + }, + "recorded_at": "2026-11-13T00:00:00+00:00" + }, + { + "sequence": 2, + "event_type": "VersionMarkerRecorded", + "payload": { + "sequence": 1, + "change_id": "step-1", + "version": 2, + "min_supported": 1, + "max_supported": 2 + }, + "recorded_at": "2026-11-13T00:00:01+00:00" + } + ], + "expected": { + "completed": true, + "result": { + "version": 2 + }, + "commands": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/Unit/V2/ChildDefinitionFingerprintTest.php b/tests/Unit/V2/ChildDefinitionFingerprintTest.php new file mode 100644 index 000000000..f474c152a --- /dev/null +++ b/tests/Unit/V2/ChildDefinitionFingerprintTest.php @@ -0,0 +1,166 @@ + 'redis', + 'workflows.v2.namespaces' => ['client-a'], + 'workflows.v2.task_dispatch_mode' => 'queue', + 'workflows.v2.types.workflows' => [ + 'mapped-child' => FingerprintInitialChild::class, + ], + ]); + } + + public static function childTypes(): array + { + return [[FingerprintInitialChild::class, true], ['mapped-child', true], ['portable.unknown-child', false]]; + } + + #[DataProvider('childTypes')] + public function testBridgeOnlyFingerprintsResolvedEmbeddedChildren(string $type, bool $embedded): void + { + $child = $this->admitChild($type); + $recorded = WorkflowDefinitionFingerprint::recordedForRun($child); + $this->assertSame( + $embedded ? WorkflowDefinition::fingerprint(FingerprintInitialChild::class) : null, + $recorded + ); + $resolution = VersionResolver::resolve( + $child, + null, + new VersionCall('child-version', WorkflowStub::DEFAULT_VERSION, 1), + 1 + ); + $this->assertSame($embedded ? 1 : WorkflowStub::DEFAULT_VERSION, $resolution->version); + $this->assertSame($embedded, $resolution->shouldRecordMarker); + } + + public static function sourceDefinitions(): array + { + return [['unchanged'], ['registration-drift'], ['legacy-absence']]; + } + + #[DataProvider('sourceDefinitions')] + public function testRetryKeepsRecordedDefinitionWithoutAcquiringCurrentCode(string $source): void + { + $child = $this->admitChild('mapped-child'); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $child->id) + ->where('event_type', HistoryEventType::WorkflowStarted)->sole(); + $payload = $started->payload; + if ($source === 'legacy-absence') { + unset($payload['workflow_definition_fingerprint']); + $started->forceFill([ + 'payload' => $payload, + ])->save(); + } + if ($source === 'registration-drift') { + config()->set('workflows.v2.types.workflows.mapped-child', FingerprintReplacementChild::class); + $this->assertNotSame( + $payload['workflow_definition_fingerprint'], + WorkflowDefinition::fingerprint(FingerprintReplacementChild::class) + ); + } + $task = WorkflowTask::query()->where('workflow_run_id', $child->id)->sole(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertTrue($bridge->claimStatus($task->id, 'child-worker')['claimed']); + $result = $bridge->complete($task->id, [[ + 'type' => 'fail_workflow', + 'message' => 'retry this child', + ]]); + $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $retry = WorkflowRun::query()->where('workflow_instance_id', $child->workflow_instance_id) + ->where('run_number', 2) + ->sole(); + $this->assertSame($payload, $started->refresh()->payload); + $this->assertSame( + $payload['workflow_definition_fingerprint'] ?? null, + WorkflowDefinitionFingerprint::recordedForRun($retry) + ); + $resolution = VersionResolver::resolve( + $retry, + null, + new VersionCall('child-version', WorkflowStub::DEFAULT_VERSION, 1), + 1 + ); + $this->assertSame($source === 'unchanged' ? 1 : WorkflowStub::DEFAULT_VERSION, $resolution->version); + $this->assertSame($source === 'unchanged', $resolution->shouldRecordMarker); + } + + private function admitChild(string $type): WorkflowRun + { + $parent = app(WorkflowControlPlane::class)->start(FingerprintParent::class, 'fingerprint-parent', [ + 'namespace' => 'client-a', + 'arguments' => Serializer::serializeWithCodec('avro', [$type]), + 'payload_codec' => 'avro', + ]); + $this->assertTrue($parent['started']); + $task = WorkflowTask::query()->where('workflow_run_id', $parent['workflow_run_id'])->sole(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $claimed = $bridge->claimStatus($task->id, 'parent-worker'); + $this->assertTrue($claimed['claimed'], json_encode($claimed, JSON_THROW_ON_ERROR)); + $result = $bridge->complete($task->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => $type, + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'retry_policy' => [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ], + ]]); + $this->assertTrue($result['completed'], json_encode($result, JSON_THROW_ON_ERROR)); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $parent['workflow_run_id'])->sole(); + return WorkflowRun::query()->findOrFail($link->child_workflow_run_id); + } +} + +final class FingerprintParent extends Workflow +{ + public function handle(string $type): mixed + { + return self::child($type); + } +} + +final class FingerprintInitialChild extends Workflow +{ + public function handle(): string + { + return 'initial'; + } +} + +final class FingerprintReplacementChild extends Workflow +{ + public function handle(): string + { + return 'replacement'; + } +} From ed926b5a6b6d4b86c943d4dff8687cb09578f974 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:51:16 +0000 Subject: [PATCH 19/32] Consolidate duplicate child definition refusal corpus evidence --- ...ld-retry-definition-drift-cold-replay.json | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json diff --git a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json deleted file mode 100644 index b580a769b..000000000 --- a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-drift-cold-replay.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "child-retry-definition-drift-cold-replay", - "protocol_version": "1.20", - "bindings": [ - "php" - ], - "consumers": [ - "workflow-fiber-runner", - "query-state-replayer", - "workflow-executor" - ], - "workflow": { - "type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", - "arguments": [], - "payload_codec": "avro" - }, - "history": [ - { - "sequence": 1, - "event_type": "WorkflowStarted", - "payload": { - "workflow_class": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", - "workflow_type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", - "workflow_instance_id": "child-instance", - "workflow_run_id": "retry-run", - "parent_workflow_instance_id": "parent-instance", - "parent_workflow_run_id": "parent-run", - "parent_sequence": 1, - "workflow_link_id": "retry-link", - "child_call_id": "child-call", - "retry_attempt": 2, - "retry_of_child_workflow_run_id": "failed-child-run", - "workflow_definition_fingerprint": "sha256:0000000000000000000000000000000000000000000000000000000000000000" - }, - "recorded_at": "2026-11-13T00:00:00+00:00" - } - ], - "expected_failure": { - "type": "version_marker_incompatible_range", - "exception": "Workflow\\V2\\Exceptions\\VersionRangeMismatchException" - } -} From 7dbb78253f9199c1b67eb3975ee47e83d780b5aa Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:24:04 +0000 Subject: [PATCH 20/32] Preserve historical definition authority in embedded corpus import --- .../V2EmbeddedReplayRegressionCorpusTest.php | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 615b3faaf..1de1f5b0a 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -2558,6 +2558,25 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void continue; } + if ($fixture['id'] === 'child-retry-definition-recorded-cold-replay') { + $run = $this->createRunFromFixture($fixture); + $before = $run->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + $state = (new QueryStateReplayer())->replayState($run->fresh(['historyEvents'])); + $this->assertSame(2, $state->sequence); + $this->assertNull($state->current); + $this->assertSame($before, $run->historyEvents()->orderBy('sequence')->get()->toArray()); + $this->runReadyWorkflowTask($run); + $this->assertSame(RunStatus::Completed, $run->fresh()->status); + $this->assertSame($fixture['expected']['result'], WorkflowStub::loadRun($run->id)->output()); + $this->assertSame(1, $run->historyEvents() + ->where('event_type', HistoryEventType::VersionMarkerRecorded)->count()); + + continue; + } + if ($fixture['id'] === 'child-closure-sibling-history-bound') { $this->assertChildClosureReadsNoSiblingHistory($fixture); @@ -5297,6 +5316,21 @@ private function createRunFromFixture(array $fixture): WorkflowRun foreach ($fixture['history'] as $event) { $eventType = HistoryEventType::from($event['event_type']); if ($eventType === HistoryEventType::WorkflowStarted) { + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::WorkflowStarted)->sole(); + $payload = $started->payload; + // Keep the new run's routing/command IDs, but import definition + // authority exactly. A synthetic current fingerprint must not + // turn legacy absence into a fresh version decision. + foreach (['workflow_definition_fingerprint', 'workflow_definition_fingerprint_source'] as $key) { + unset($payload[$key]); + if (array_key_exists($key, $event['payload'])) { + $payload[$key] = $event['payload'][$key]; + } + } + $started->forceFill([ + 'payload' => $payload, + ])->save(); continue; } From c3b451845ffcb46bb4fa86ed86fc96c19ddc24f2 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:34:45 +0000 Subject: [PATCH 21/32] Exercise native child producers in recorded definition corpus --- .../V2EmbeddedReplayRegressionCorpusTest.php | 41 ++++++++++++++++++ ...definition-legacy-absence-cold-replay.json | 43 ------------------- 2 files changed, 41 insertions(+), 43 deletions(-) delete mode 100644 tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 1de1f5b0a..d02e2bdad 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -103,6 +103,7 @@ use Workflow\V2\Support\ServiceCallScheduler; use Workflow\V2\Support\WorkflowCommandNormalizer; use Workflow\V2\Support\WorkflowDefinition; +use Workflow\V2\Support\WorkflowDefinitionFingerprint; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\Support\WorkflowReplayer; use Workflow\V2\Support\WorkflowRunRetentionCleanup; @@ -2559,6 +2560,7 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void } if ($fixture['id'] === 'child-retry-definition-recorded-cold-replay') { + $this->assertNativeChildProducersRetainDefinition($fixture); $run = $this->createRunFromFixture($fixture); $before = $run->historyEvents() ->orderBy('sequence') @@ -2919,6 +2921,45 @@ private function assertForeignNamespaceTaskIsRefusedByAnotherEmbeddedApp(array $ } } + /** + * @param array $fixture + */ + private function assertNativeChildProducersRetainDefinition(array $fixture): void + { + $parent = WorkflowStub::make(TestGreetingWorkflow::class); + $parent->start('definition-parent'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $parentTask = WorkflowTask::query()->where('workflow_run_id', $parent->runId())->sole(); + $this->assertTrue($bridge->claimStatus($parentTask->id, 'definition-parent-worker')['claimed']); + $admitted = $bridge->complete($parentTask->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => $fixture['workflow']['type'], + 'arguments' => Serializer::serializeWithCodec('avro', $fixture['workflow']['arguments']), + 'payload_codec' => 'avro', + 'retry_policy' => [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ], + ]]); + $this->assertTrue($admitted['completed'], json_encode($admitted, JSON_THROW_ON_ERROR)); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId())->sole(); + $child = WorkflowRun::query()->findOrFail($link->child_workflow_run_id); + $expected = $fixture['history'][0]['payload']['workflow_definition_fingerprint']; + $this->assertSame($expected, WorkflowDefinitionFingerprint::recordedForRun($child)); + + $childTask = WorkflowTask::query()->where('workflow_run_id', $child->id)->sole(); + $this->assertTrue($bridge->claimStatus($childTask->id, 'definition-child-worker')['claimed']); + $failed = $bridge->complete($childTask->id, [[ + 'type' => 'fail_workflow', + 'message' => 'retry the recorded child definition', + ]]); + $this->assertTrue($failed['completed'], json_encode($failed, JSON_THROW_ON_ERROR)); + $retry = WorkflowRun::query()->where('workflow_instance_id', $child->workflow_instance_id) + ->where('run_number', 2) + ->sole(); + $this->assertSame($expected, WorkflowDefinitionFingerprint::recordedForRun($retry)); + } + private function assertHandedOverChildFailureStartsTheRecordedRetry(array $fixture): void { $this->clearWorkflowState(); diff --git a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json deleted file mode 100644 index a0cd241e5..000000000 --- a/tests/Fixtures/V2/ReplayRegression/child-retry-definition-legacy-absence-cold-replay.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "child-retry-definition-legacy-absence-cold-replay", - "protocol_version": "1.20", - "bindings": [ - "php" - ], - "consumers": [ - "workflow-fiber-runner", - "query-state-replayer", - "workflow-executor" - ], - "workflow": { - "type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", - "arguments": [], - "payload_codec": "avro" - }, - "history": [ - { - "sequence": 1, - "event_type": "WorkflowStarted", - "payload": { - "workflow_class": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", - "workflow_type": "Tests\\Fixtures\\V2\\TestVersionMinSupportedWorkflow", - "workflow_instance_id": "child-instance", - "workflow_run_id": "retry-run", - "parent_workflow_instance_id": "parent-instance", - "parent_workflow_run_id": "parent-run", - "parent_sequence": 1, - "workflow_link_id": "retry-link", - "child_call_id": "child-call", - "retry_attempt": 2, - "retry_of_child_workflow_run_id": "failed-child-run" - }, - "recorded_at": "2026-11-13T00:00:00+00:00" - } - ], - "expected_failure": { - "type": "version_marker_incompatible_range", - "exception": "Workflow\\V2\\Exceptions\\VersionRangeMismatchException" - } -} From 4f5c1316913c56e81bd9745d655636c171ae8ba8 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:45:26 +0000 Subject: [PATCH 22/32] Retain embedded legacy definition absence refusal control --- .../V2/V2EmbeddedReplayRegressionCorpusTest.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index d02e2bdad..1d6995969 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -2576,7 +2576,19 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void $this->assertSame(1, $run->historyEvents() ->where('event_type', HistoryEventType::VersionMarkerRecorded)->count()); - continue; + // Legacy absence is a compatibility control, not new regression + // evidence: neither query nor execution may acquire a current + // definition when the historical start recorded none. + $fixture['id'] .= '-legacy-absence-control'; + $fixture['history'] = [$fixture['history'][0]]; + unset( + $fixture['history'][0]['payload']['workflow_definition_fingerprint'], + $fixture['history'][0]['payload']['workflow_definition_fingerprint_source'], + $fixture['expected'], + ); + $fixture['expected_failure'] = [ + 'exception' => VersionRangeMismatchException::class, + ]; } if ($fixture['id'] === 'child-closure-sibling-history-bound') { From 93f3a21d06d3a913407b88014a3975761a76631b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:28:45 +0000 Subject: [PATCH 23/32] Require bound native recovery admission at activity boundaries --- CHANGELOG.md | 4 +- docs/compensation-scopes.md | 5 + .../Compensation/Security/ScopeBoundary.php | 38 +++ .../V2CompensationRecoveryAuthorityTest.php | 244 ++++++++++++++++++ 4 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 tests/Feature/V2/V2CompensationRecoveryAuthorityTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index a1433fe90..54cbc7d61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ - Experimental embedded compensation scopes (#1063): retained versioned obligations, native Saga recovery, sequential child participation and explicit release. Each scope keeps its originating run; continue-as-new, origin redrive - and child retry do not transfer forward ownership. + and child retry do not transfer forward ownership. Recovery activities require + the exact paired native admission receipt and retained scope/original-owner + binding before host authorization and again under native activity locks. - Failed-run redrive accepts a verified linear activity retry chain. A retried completed prefix is reused and only the failed boundary step runs again. diff --git a/docs/compensation-scopes.md b/docs/compensation-scopes.md index a45a5062e..34919bbec 100644 --- a/docs/compensation-scopes.md +++ b/docs/compensation-scopes.md @@ -80,6 +80,11 @@ flowchart TD Y --> Z[Eligible native redrive reuses verified activity prefix] ``` +Recovery activities also require the paired native creation command/history +receipt for their exact run, scope and original owner. A generic start at the +canonical ID cannot borrow a retained request's principal. The boundary checks +fresh receipt data before host authorization and again under the native locks. + `ScopeBoundary` uses the existing native activity status/authority checks while retaining their SQL locks in the ledger transaction. Its lock order is native attempt, execution, run and task, followed by the scope and its journal. It checks diff --git a/src/V2/Compensation/Security/ScopeBoundary.php b/src/V2/Compensation/Security/ScopeBoundary.php index 2ca64a7ea..dedbacf8d 100644 --- a/src/V2/Compensation/Security/ScopeBoundary.php +++ b/src/V2/Compensation/Security/ScopeBoundary.php @@ -11,10 +11,14 @@ use Workflow\V2\Compensation\Definition\ScopeIdentity; use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Persistence\ScopeStorage; +use Workflow\V2\Compensation\Recovery\RecoveryRunAdmission; use Workflow\V2\Compensation\Recovery\RecoveryWorkflow; +use Workflow\V2\Models\WorkflowInstance; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\DefaultActivityTaskBridge; use Workflow\V2\Support\SqliteTransaction; +use Workflow\V2\Support\TypeRegistry; /** * @internal Retains native attempt locks until the ledger transaction commits; owns no new lease. @@ -110,6 +114,40 @@ private function assertScopeOwner( if ((! $origin && ! $recovery) || ($operation === 'forward' && ! $origin)) { throw new NonRetryableException('scope_execution_owner_mismatch'); } + if ($recovery) { + $this->assertRecoveryAdmission($activity, $scope, $snapshot); + } + } + + private function assertRecoveryAdmission(Activity $activity, ScopeIdentity $scope, array $snapshot): void + { + // Read fresh native rows on both sides of host authorization. Inside + // the boundary the native attempt/run locks are already held; avoid + // cached history and adding an instance lock after that lock prefix. + $instance = ConfiguredV2Models::query('instance_model', WorkflowInstance::class) + ->where('namespace', $scope->namespace) + ->find($scope->recoveryInstanceId()); + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->where('namespace', $scope->namespace) + ->find($activity->runId()); + if (! $instance instanceof WorkflowInstance || ! $run instanceof WorkflowRun + || $instance->workflow_class !== RecoveryWorkflow::class + || $run->workflow_class !== RecoveryWorkflow::class + || $instance->workflow_type !== TypeRegistry::for(RecoveryWorkflow::class) + || $run->workflow_type !== $instance->workflow_type) { + throw new NonRetryableException('scope_recovery_admission_unavailable'); + } + try { + $receipt = RecoveryRunAdmission::sourceReceipt($instance, $run); + if ($receipt === null + || ($receipt['intent']['binding']['original_run_id'] ?? null) !== $snapshot['original_run_id'] + || PortableDescriptor::encode($receipt['intent']['binding']['scope'] ?? []) + !== PortableDescriptor::encode($scope->toArray())) { + throw new \LogicException('Recovery activity is not bound to the retained scope and owner.'); + } + } catch (\LogicException|\JsonException) { + throw new NonRetryableException('scope_recovery_admission_unavailable'); + } } private function assertUnlockedHostContext(): void diff --git a/tests/Feature/V2/V2CompensationRecoveryAuthorityTest.php b/tests/Feature/V2/V2CompensationRecoveryAuthorityTest.php new file mode 100644 index 000000000..b12adeb6b --- /dev/null +++ b/tests/Feature/V2/V2CompensationRecoveryAuthorityTest.php @@ -0,0 +1,244 @@ + 'redis', + 'workflows.v2.namespaces' => ['client-a'], + ]); + $this->app->instance(HandlerRegistry::class, new HandlerRegistry([ + 'test.operation' => [ + '1' => new TestScopedOperation(), + ], + ])); + $this->authorize(); + } + + public function testOrdinaryStartWinningAfterTheRetainedRouteCannotPerformRecovery(): void + { + [$scope] = $this->origin(); + $winner = null; + $injected = false; + ConfiguredV2Models::connection()->listen(function (QueryExecuted $query) use ( + $scope, + &$winner, + &$injected + ): void { + if ($injected || ! str_starts_with(strtolower($query->sql), 'update') + || ! str_contains($query->sql, 'workflow_compensation_requests') + || ! str_contains($query->sql, 'recovery_route')) { + return; + } + $injected = true; + $query->connection->afterCommit(function () use ($scope, &$winner): void { + $this->assertSame(0, ConfiguredV2Models::connection()->transactionLevel()); + $winner = WorkflowStub::make(RecoveryWorkflow::class, $scope->recoveryInstanceId(), 'client-a'); + $winner->start($scope->toArray()); + }); + }); + try { + app(ScopeRecovery::class)->request($scope, 'retained-request', 'operator-a', 'cancel'); + $this->fail('An ordinary start was adopted as content-bound recovery.'); + } catch (\LogicException) { + $this->assertTrue($injected); + $this->assertInstanceOf(WorkflowStub::class, $winner); + } + $task = $this->nextActivity($winner->runId(), PrepareRecoveryActivity::class); + $this->assertBoundaryRefuses($scope, $task); + $this->assertSame('requested', app(ScopeRepository::class)->inspect($scope)['status']); + $this->assertNull(Cache::get('test:scope:undo-log')); + } + + public static function invalidReceipts(): iterable + { + foreach ([ + PrepareRecoveryActivity::class, + CompensateObligationActivity::class, + VerifyScopeActivity::class, + ] as $stage) { + foreach (['missing-command', 'missing-history', 'missing-both', 'divergent', 'scope', 'owner'] as $change) { + yield $stage . '/' . $change => [$stage, $change]; + } + } + } + + #[DataProvider('invalidReceipts')] + public function testEveryRecoveryActivityRequiresItsExactAdmission(string $stage, string $change): void + { + [$scope] = $this->origin(); + $result = app(ScopeRecovery::class)->request($scope, 'recovery', 'operator-a', 'cancel'); + $task = $this->nextActivity($result['runId'], $stage); + $this->alterReceipt($result['runId'], $change); + $this->assertBoundaryRefuses($scope, $task); + } + + public function testReceiptIsRecheckedAfterUnlockedHostAuthorization(): void + { + [$scope] = $this->origin(); + $result = app(ScopeRecovery::class)->request($scope, 'recovery', 'operator-a', 'cancel'); + $task = $this->nextActivity($result['runId'], PrepareRecoveryActivity::class); + $changed = false; + $this->authorize(function () use ($result, &$changed): void { + if (! $changed) { + $changed = true; + $this->assertSame(0, ConfiguredV2Models::connection()->transactionLevel()); + $this->alterReceipt($result['runId'], 'owner'); + } + }); + $this->assertBoundaryRefuses($scope, $task); + $this->assertTrue($changed); + } + + private function origin(): array + { + $scope = new ScopeIdentity('shop', 'client-a', 'order-42', '1'); + $stub = WorkflowStub::make(TestRetainedScopeWorkflow::class, 'receipt-origin', 'client-a'); + $stub->start($scope->toArray(), 'complete'); + for ($count = 0; $count < 24 && ! $stub->refresh()->completed(); ++$count) { + $this->execute($this->readyTask($stub->runId())); + } + $this->assertTrue($stub->refresh()->completed()); + return [$scope, $stub]; + } + + private function nextActivity(string $runId, string $class): WorkflowTask + { + for ($count = 0; $count < 24; ++$count) { + $task = $this->readyTask($runId); + $execution = $task->task_type === TaskType::Activity + ? ActivityExecution::query()->find($task->payload['activity_execution_id']) + : null; + if ($execution?->activity_type === TypeRegistry::for($class)) { + return $task; + } + $this->execute($task); + } + $this->fail('The bounded recovery did not reach ' . $class); + } + + private function readyTask(string $runId): WorkflowTask + { + return WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('status', TaskStatus::Ready)->orderBy('created_at')->firstOrFail(); + } + + private function execute(WorkflowTask $task): void + { + $job = $task->task_type === TaskType::Activity + ? new RunActivityTask($task->id) : new RunWorkflowTask($task->id); + $this->app->call([$job, 'handle']); + } + + private function assertBoundaryRefuses(ScopeIdentity $scope, WorkflowTask $task): void + { + $repository = app(ScopeRepository::class); + $before = [$repository->inspect($scope), $repository->journal($scope), Cache::get('test:scope:undo-log')]; + $this->execute($task); + $messages = WorkflowFailure::query()->where('workflow_run_id', $task->workflow_run_id)->pluck('message')->all(); + $this->assertStringContainsString('scope_recovery_admission_unavailable', implode('\n', $messages)); + $this->assertSame( + $before, + [$repository->inspect($scope), $repository->journal($scope), Cache::get('test:scope:undo-log')] + ); + } + + private function alterReceipt(string $runId, string $change): void + { + $command = WorkflowCommand::query()->where('workflow_run_id', $runId) + ->where('command_type', CommandType::Start)->sole(); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::WorkflowStarted)->sole(); + $context = $command->context; + $payload = $started->payload; + $receipt = $context['compensation_recovery']; + if (in_array($change, ['scope', 'owner'], true)) { + if ($change === 'scope') { + $receipt['intent']['binding']['scope']['key'] = 'another-scope'; + } else { + $receipt['intent']['binding']['original_run_id'] = 'another-original-run'; + } + $receipt['intent']['binding_fingerprint'] = hash( + 'sha256', + PortableDescriptor::encode($receipt['intent']['binding']) + ); + $context['compensation_recovery'] = $payload['compensation_recovery'] = $receipt; + } + if (in_array($change, ['missing-command', 'missing-both'], true)) { + unset($context['compensation_recovery']); + } + if (in_array($change, ['missing-history', 'missing-both'], true)) { + unset($payload['compensation_recovery']); + } + if ($change === 'divergent') { + $payload['compensation_recovery']['request_key'] = 'different-request'; + } + $command->forceFill([ + 'context' => $context, + ])->save(); + $started->forceFill([ + 'payload' => $payload, + ])->save(); + } + + private function authorize(?\Closure $afterCheck = null): void + { + $this->app->instance(ScopeAuthorization::class, new class($afterCheck) implements ScopeAuthorization { + public function __construct( + private ?\Closure $afterCheck + ) { + } + + public function assertAllowed(ScopeIdentity $scope, string $principal, string $operation): string + { + if ($scope->namespace !== 'client-a' || $principal !== 'operator-a') { + throw new \LogicException('Scope access refused.'); + } + if ($operation === 'compensate') { + ($this->afterCheck)?->__invoke(); + } + return 'test-policy-1'; + } + }); + } +} From 2e966b8f22d14e507b8bc28eb4377521d6566d6d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:07:31 +0000 Subject: [PATCH 24/32] Keep child closure history locks within the parent run --- CHANGELOG.md | 4 +++ src/V2/Support/ParallelChildGroup.php | 29 ++++++++++++++++--- .../V2/V2CompensationScopeRunBoundaryTest.php | 11 ++++--- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54cbc7d61..45ff7de90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Concurrent service child completions keep parent-history row locks on the + existing per-run index on MySQL and MariaDB. This prevents a parent-history + table scan from deadlocking against an unrelated sibling's completion. + - Experimental embedded compensation scopes (#1063): retained versioned obligations, native Saga recovery, sequential child participation and explicit release. Each scope keeps its originating run; continue-as-new, origin redrive diff --git a/src/V2/Support/ParallelChildGroup.php b/src/V2/Support/ParallelChildGroup.php index ad1a47b93..836364774 100644 --- a/src/V2/Support/ParallelChildGroup.php +++ b/src/V2/Support/ParallelChildGroup.php @@ -4,6 +4,8 @@ namespace Workflow\V2\Support; +use Illuminate\Database\Eloquent\Collection; +use Illuminate\Database\Schema\Blueprint; use Workflow\V2\Enums\ActivityStatus; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; @@ -732,10 +734,10 @@ private static function persistSelectionWinner( $events = $parentRun->historyEvents(); if ($lockHistoryForUpdate) { - $events->lockForUpdate(); + $parentRun->setRelation('historyEvents', self::lockedParentHistory($parentRun)); + } else { + $parentRun->setRelation('historyEvents', $events->get()); } - - $parentRun->setRelation('historyEvents', $events->get()); $groupId = $metadata['parallel_group_id']; $memberBase = self::intValue($metadata['selection_member_base_sequence'] ?? null) ?? ($metadata['parallel_group_base_sequence'] + $metadata['parallel_group_index']); @@ -800,7 +802,7 @@ private static function groupCompletedSuccessfully( if ($lockHistoryForUpdate) { // Observe the resolution event committed by the previous holder of // the parent lock even when this transaction has an older snapshot. - $parentRun->setRelation('historyEvents', $parentRun->historyEvents() ->lockForUpdate() ->get()); + $parentRun->setRelation('historyEvents', self::lockedParentHistory($parentRun)); } $activitiesBySequence = collect(RunActivityView::activitiesForRun($parentRun, decodePayloads: false)) @@ -876,6 +878,25 @@ private static function groupCompletedSuccessfully( return true; } + /** + * @return Collection + */ + private static function lockedParentHistory(WorkflowRun $parentRun): Collection + { + $events = $parentRun->historyEvents(); + $model = $events->getRelated(); + $connection = $model->getConnection(); + if (in_array($connection->getDriverName(), ['mysql', 'mariadb'], true)) { + // A table scan can lock an uncommitted sibling event while that + // sibling waits for our parent-run lock. Use the migration's + // per-run index, including Laravel's configured table/index prefix. + $index = (new Blueprint($connection, $model->getTable())) + ->unique(['workflow_run_id', 'sequence'])->index; + $events->forceIndex($index); + } + return $events->lockForUpdate()->get(); + } + /** * @param array $payload * @return array{ diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 1d04fc009..424db05cc 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -257,11 +257,14 @@ private function drainTasks(?string $excludedRunId = null): void if ($excludedRunId !== null) { $tasks->where('workflow_run_id', '!=', $excludedRunId); } - $task = $tasks->first(); + $task = (clone $tasks)->where(static function ($query): void { + $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u')); + })->first(); if ($task === null) { - return; - } - if ($task->available_at !== null && $task->available_at->isFuture()) { + $task = $tasks->reorder('available_at')->orderBy('created_at')->orderBy('id')->first(); + if ($task === null) { + return; + } Carbon::setTestNow($task->available_at->copy()->addSecond()); } $job = match ($task->task_type) { From 7b05eb83463aeaf950a98bfb358873be456ab00b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:13:25 +0000 Subject: [PATCH 25/32] Apply formatter to parent history lock helper --- src/V2/Support/ParallelChildGroup.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/V2/Support/ParallelChildGroup.php b/src/V2/Support/ParallelChildGroup.php index 836364774..10d77640f 100644 --- a/src/V2/Support/ParallelChildGroup.php +++ b/src/V2/Support/ParallelChildGroup.php @@ -894,7 +894,8 @@ private static function lockedParentHistory(WorkflowRun $parentRun): Collection ->unique(['workflow_run_id', 'sequence'])->index; $events->forceIndex($index); } - return $events->lockForUpdate()->get(); + return $events->lockForUpdate() + ->get(); } /** From 9329222f31cd67ebfa5fe53db1ab4a5844cc65e9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:17:51 +0000 Subject: [PATCH 26/32] Initialize schema grammar before deriving the history index --- src/V2/Support/ParallelChildGroup.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/V2/Support/ParallelChildGroup.php b/src/V2/Support/ParallelChildGroup.php index 10d77640f..ead6f5a56 100644 --- a/src/V2/Support/ParallelChildGroup.php +++ b/src/V2/Support/ParallelChildGroup.php @@ -890,6 +890,7 @@ private static function lockedParentHistory(WorkflowRun $parentRun): Collection // A table scan can lock an uncommitted sibling event while that // sibling waits for our parent-run lock. Use the migration's // per-run index, including Laravel's configured table/index prefix. + $connection->getSchemaBuilder(); $index = (new Blueprint($connection, $model->getTable())) ->unique(['workflow_run_id', 'sequence'])->index; $events->forceIndex($index); From d7ea1b01bd881900719af7988bfe85139d8c37b8 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:35:10 +0000 Subject: [PATCH 27/32] Apply formatter to the scope task drain helper --- tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php index 424db05cc..68f4d8634 100644 --- a/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php +++ b/tests/Feature/V2/V2CompensationScopeRunBoundaryTest.php @@ -258,10 +258,14 @@ private function drainTasks(?string $excludedRunId = null): void $tasks->where('workflow_run_id', '!=', $excludedRunId); } $task = (clone $tasks)->where(static function ($query): void { - $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u')); + $query->whereNull('available_at') + ->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u')); })->first(); if ($task === null) { - $task = $tasks->reorder('available_at')->orderBy('created_at')->orderBy('id')->first(); + $task = $tasks->reorder('available_at') + ->orderBy('created_at') + ->orderBy('id') + ->first(); if ($task === null) { return; } From d4205216fd85a62ca40e7382d1d4d4d00a22d67a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:07:14 +0000 Subject: [PATCH 28/32] Record bounded cold recovery queue diagnostics --- .../CompensationScopeRecoveryProcessTest.php | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/Support/CompensationScopeRecoveryProcessTest.php b/tests/Support/CompensationScopeRecoveryProcessTest.php index eb51d0a86..af0f776b7 100644 --- a/tests/Support/CompensationScopeRecoveryProcessTest.php +++ b/tests/Support/CompensationScopeRecoveryProcessTest.php @@ -4,6 +4,7 @@ namespace Tests\Support; +use Illuminate\Support\Facades\DB; use Orchestra\Testbench\TestCase; use Tests\Fixtures\V2\TestScopedOperation; use Workflow\Providers\WorkflowServiceProvider; @@ -12,6 +13,7 @@ use Workflow\V2\Compensation\Persistence\ScopeRepository; use Workflow\V2\Compensation\Recovery\ScopeRecovery; use Workflow\V2\Compensation\Security\ScopeAuthorization; +use Workflow\V2\Models\WorkflowRun; /** Started by the parent integration test without its container, static state or migration traits. */ final class CompensationScopeRecoveryProcessTest extends TestCase @@ -34,7 +36,9 @@ public function assertAllowed(ScopeIdentity $scope, string $principal, string $o } }); $scope = ScopeIdentity::fromArray($input['scope']); - app(ScopeRecovery::class)->request($scope, 'cold-request', 'operator-a', 'cancel'); + $result = app(ScopeRecovery::class)->request($scope, 'cold-request', 'operator-a', 'cancel'); + $started = hrtime(true); + $progress = [$this->progress($result['runId'], -1, $started)]; for ($step = 0; $step < 20 && app(ScopeRepository::class)->inspect($scope)['status'] !== 'verified'; $step++) { $this->artisan('queue:work', [ '--once' => true, @@ -42,9 +46,10 @@ public function assertAllowed(ScopeIdentity $scope, string $principal, string $o '--sleep' => 0, '--tries' => 1, ])->assertExitCode(0); + $progress[] = $this->progress($result['runId'], $step, $started); } $status = app(ScopeRepository::class)->inspect($scope)['status']; - $this->assertSame('verified', $status); + $this->assertSame('verified', $status, json_encode($progress, JSON_THROW_ON_ERROR)); file_put_contents($input['report'], json_encode([ 'pid' => getmypid(), 'status' => $status, @@ -80,6 +85,48 @@ protected function defineEnvironment($app): void ]); } + private function progress(string $runId, int $step, int $started): array + { + $run = WorkflowRun::query()->findOrFail($runId); + $database = DB::connection(); + return [ + 'step' => $step, + 'elapsed_ms' => (hrtime(true) - $started) / 1_000_000, + 'now' => now() + ->toISOString(), + 'run' => $run->only(['id', 'workflow_instance_id', 'status', 'connection', 'queue']), + 'queue_connection' => config('queue.default'), + 'maintenance' => app() + ->isDownForMaintenance(), + 'worker_output' => $step < 0 ? '' : substr(\Illuminate\Support\Facades\Artisan::output(), 0, 4096), + 'jobs' => $database->table('jobs') + ->orderBy('id') + ->limit(32) + ->get(['id', 'queue', 'attempts', 'reserved_at', 'available_at', 'payload']) + ->map(static function (object $job): array { + $payload = json_decode($job->payload, true, flags: JSON_THROW_ON_ERROR); + return [ + 'id' => $job->id, + 'queue' => $job->queue, + 'attempts' => $job->attempts, + 'reserved_at' => $job->reserved_at, + 'available_at' => $job->available_at, + 'payload_sha256' => hash('sha256', $job->payload), + 'uuid' => $payload['uuid'] ?? null, + 'job' => $payload['displayName'] ?? null, + ]; + })->all(), + 'tasks' => $database->table('workflow_tasks') + ->where('workflow_run_id', $runId) + ->orderBy('id') + ->limit(64) + ->get([ + 'id', 'task_type', 'status', 'connection', 'queue', 'available_at', + 'last_error', 'last_dispatch_error', 'queue_delivery_reference', + ])->all(), + ]; + } + private function input(): array { $path = getenv('DWF_SCOPE_RECOVERY_INPUT'); From 63f964ba3b2b35cff638b4dfe34c0a8fb9e2746f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:16:18 +0000 Subject: [PATCH 29/32] Scope and bound cold recovery progress snapshots --- .../CompensationScopeRecoveryProcessTest.php | 119 +++++++++++++----- 1 file changed, 91 insertions(+), 28 deletions(-) diff --git a/tests/Support/CompensationScopeRecoveryProcessTest.php b/tests/Support/CompensationScopeRecoveryProcessTest.php index af0f776b7..b53eafb7e 100644 --- a/tests/Support/CompensationScopeRecoveryProcessTest.php +++ b/tests/Support/CompensationScopeRecoveryProcessTest.php @@ -49,7 +49,11 @@ public function assertAllowed(ScopeIdentity $scope, string $principal, string $o $progress[] = $this->progress($result['runId'], $step, $started); } $status = app(ScopeRepository::class)->inspect($scope)['status']; - $this->assertSame('verified', $status, json_encode($progress, JSON_THROW_ON_ERROR)); + $this->assertSame( + 'verified', + $status, + json_encode($progress, JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE) + ); file_put_contents($input['report'], json_encode([ 'pid' => getmypid(), 'status' => $status, @@ -87,43 +91,102 @@ protected function defineEnvironment($app): void private function progress(string $runId, int $step, int $started): array { - $run = WorkflowRun::query()->findOrFail($runId); - $database = DB::connection(); - return [ + $snapshot = [ 'step' => $step, 'elapsed_ms' => (hrtime(true) - $started) / 1_000_000, 'now' => now() ->toISOString(), - 'run' => $run->only(['id', 'workflow_instance_id', 'status', 'connection', 'queue']), - 'queue_connection' => config('queue.default'), - 'maintenance' => app() - ->isDownForMaintenance(), - 'worker_output' => $step < 0 ? '' : substr(\Illuminate\Support\Facades\Artisan::output(), 0, 4096), - 'jobs' => $database->table('jobs') - ->orderBy('id') - ->limit(32) - ->get(['id', 'queue', 'attempts', 'reserved_at', 'available_at', 'payload']) - ->map(static function (object $job): array { - $payload = json_decode($job->payload, true, flags: JSON_THROW_ON_ERROR); - return [ - 'id' => $job->id, - 'queue' => $job->queue, - 'attempts' => $job->attempts, - 'reserved_at' => $job->reserved_at, - 'available_at' => $job->available_at, - 'payload_sha256' => hash('sha256', $job->payload), - 'uuid' => $payload['uuid'] ?? null, - 'job' => $payload['displayName'] ?? null, - ]; - })->all(), - 'tasks' => $database->table('workflow_tasks') + ]; + try { + $run = WorkflowRun::query()->findOrFail($runId); + $database = DB::connection(); + $tasks = $database->table('workflow_tasks') ->where('workflow_run_id', $runId) ->orderBy('id') ->limit(64) ->get([ 'id', 'task_type', 'status', 'connection', 'queue', 'available_at', 'last_error', 'last_dispatch_error', 'queue_delivery_reference', - ])->all(), + ]); + $references = []; + foreach ($tasks as $task) { + $reference = $this->deliveryReference($task, $runId, $run->queue); + if ($reference !== null) { + $references[$reference['job_id']] = $reference; + } + } + $snapshot += [ + 'run' => $run->only(['id', 'workflow_instance_id', 'status', 'connection', 'queue']), + 'queue_connection' => config('queue.default'), + 'maintenance' => app() + ->isDownForMaintenance(), + 'worker_output' => $step < 0 ? null : $this->bounded(\Illuminate\Support\Facades\Artisan::output()), + 'queue_jobs_count' => $database->table('jobs') + ->where('queue', $run->queue) + ->count(), + 'jobs' => $database->table('jobs') + ->where('queue', $run->queue) + ->whereIn('id', array_keys($references)) + ->orderBy('id') + ->limit(32) + ->get(['id', 'queue', 'attempts', 'reserved_at', 'available_at', 'payload']) + ->map(static function (object $job) use ($references): array { + $payload = strlen($job->payload) <= 262144 ? json_decode($job->payload, true) : null; + $expected = $references[$job->id]; + return [ + 'id' => $job->id, + 'queue' => $job->queue, + 'attempts' => $job->attempts, + 'reserved_at' => $job->reserved_at, + 'available_at' => $job->available_at, + 'payload_sha256' => hash('sha256', $job->payload), + 'task_id' => $expected['task_id'], + 'expected_uuid' => $expected['payload_uuid'], + 'payload_identity_matches' => is_array($payload) + && ($payload['uuid'] ?? null) === $expected['payload_uuid'], + ]; + })->all(), + 'tasks' => $tasks->map(function (object $task) use ($runId, $run): array { + $value = (array) $task; + foreach (['last_error', 'last_dispatch_error'] as $key) { + $value[$key] = $this->bounded($value[$key]); + } + $value['queue_delivery_reference'] = $this->deliveryReference($task, $runId, $run->queue); + return $value; + })->all(), + ]; + } catch (\Throwable $error) { + $snapshot['diagnostic_error'] = $this->bounded($error->getMessage()); + } + return $snapshot; + } + + private function deliveryReference(object $task, string $runId, ?string $queue): ?array + { + $raw = $task->queue_delivery_reference; + $reference = is_string($raw) && strlen($raw) <= 2048 ? json_decode($raw, true) : null; + if (! is_array($reference) || ($reference['version'] ?? null) !== 1 + || ($reference['task_id'] ?? null) !== $task->id + || ($reference['run_id'] ?? null) !== $runId + || ($reference['queue'] ?? null) !== $queue + || ! is_string($reference['job_id'] ?? null) + || preg_match('/^[1-9]\d{0,19}$/D', $reference['job_id']) !== 1 + || ! is_string($reference['payload_uuid'] ?? null) + || preg_match( + '/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D', + $reference['payload_uuid'] + ) !== 1) { + return null; + } + return array_intersect_key($reference, array_flip(['task_id', 'job_id', 'payload_uuid'])); + } + + private function bounded(?string $value): ?array + { + return $value === null ? null : [ + 'length' => strlen($value), + 'sha256' => hash('sha256', $value), + 'prefix' => substr($value, 0, 256), ]; } From dfe79f76b9b8c1f4b60f03ad92580c20bc821ca1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:33:38 +0000 Subject: [PATCH 30/32] Resolve physical queues and expose bounded native recovery failures --- .../CompensationScopeRecoveryProcessTest.php | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/Support/CompensationScopeRecoveryProcessTest.php b/tests/Support/CompensationScopeRecoveryProcessTest.php index b53eafb7e..09e8022ae 100644 --- a/tests/Support/CompensationScopeRecoveryProcessTest.php +++ b/tests/Support/CompensationScopeRecoveryProcessTest.php @@ -100,6 +100,12 @@ private function progress(string $runId, int $step, int $started): array try { $run = WorkflowRun::query()->findOrFail($runId); $database = DB::connection(); + $delivery = app('queue') + ->connection($run->connection); + if (! $delivery instanceof \Illuminate\Queue\DatabaseQueue) { + throw new \LogicException('The cold recovery diagnostic requires its database queue.'); + } + $physicalQueue = $delivery->getQueue($run->queue); $tasks = $database->table('workflow_tasks') ->where('workflow_run_id', $runId) ->orderBy('id') @@ -110,7 +116,7 @@ private function progress(string $runId, int $step, int $started): array ]); $references = []; foreach ($tasks as $task) { - $reference = $this->deliveryReference($task, $runId, $run->queue); + $reference = $this->deliveryReference($task, $runId, $physicalQueue); if ($reference !== null) { $references[$reference['job_id']] = $reference; } @@ -118,14 +124,15 @@ private function progress(string $runId, int $step, int $started): array $snapshot += [ 'run' => $run->only(['id', 'workflow_instance_id', 'status', 'connection', 'queue']), 'queue_connection' => config('queue.default'), + 'physical_queue' => $physicalQueue, 'maintenance' => app() ->isDownForMaintenance(), 'worker_output' => $step < 0 ? null : $this->bounded(\Illuminate\Support\Facades\Artisan::output()), 'queue_jobs_count' => $database->table('jobs') - ->where('queue', $run->queue) + ->where('queue', $physicalQueue) ->count(), 'jobs' => $database->table('jobs') - ->where('queue', $run->queue) + ->where('queue', $physicalQueue) ->whereIn('id', array_keys($references)) ->orderBy('id') ->limit(32) @@ -146,12 +153,27 @@ private function progress(string $runId, int $step, int $started): array && ($payload['uuid'] ?? null) === $expected['payload_uuid'], ]; })->all(), - 'tasks' => $tasks->map(function (object $task) use ($runId, $run): array { + 'failures' => $database->table('workflow_failures') + ->where('workflow_run_id', $runId) + ->orderBy('id') + ->limit(4) + ->get([ + 'id', 'exception_class', 'failure_category', 'non_retryable', 'message', 'file', 'line', + ])->map(fn (object $failure): array => [ + 'id' => $failure->id, + 'exception_class' => $failure->exception_class, + 'failure_category' => $failure->failure_category, + 'non_retryable' => $failure->non_retryable, + 'message' => $this->bounded($failure->message), + 'file' => $this->bounded($failure->file), + 'line' => $failure->line, + ])->all(), + 'tasks' => $tasks->map(function (object $task) use ($runId, $physicalQueue): array { $value = (array) $task; foreach (['last_error', 'last_dispatch_error'] as $key) { $value[$key] = $this->bounded($value[$key]); } - $value['queue_delivery_reference'] = $this->deliveryReference($task, $runId, $run->queue); + $value['queue_delivery_reference'] = $this->deliveryReference($task, $runId, $physicalQueue); return $value; })->all(), ]; From 230fb46c49f237d57c3e0e1c148d7d8c8b9f6fb7 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:04:28 +0000 Subject: [PATCH 31/32] Run cold recovery workers before pending command destruction --- .../CompensationScopeRecoveryProcessTest.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/Support/CompensationScopeRecoveryProcessTest.php b/tests/Support/CompensationScopeRecoveryProcessTest.php index 09e8022ae..9998b445e 100644 --- a/tests/Support/CompensationScopeRecoveryProcessTest.php +++ b/tests/Support/CompensationScopeRecoveryProcessTest.php @@ -39,21 +39,25 @@ public function assertAllowed(ScopeIdentity $scope, string $principal, string $o $result = app(ScopeRecovery::class)->request($scope, 'cold-request', 'operator-a', 'cancel'); $started = hrtime(true); $progress = [$this->progress($result['runId'], -1, $started)]; - for ($step = 0; $step < 20 && app(ScopeRepository::class)->inspect($scope)['status'] !== 'verified'; $step++) { + for ($step = 0; $step < 20; $step++) { + if (app(ScopeRepository::class)->inspect($scope)['status'] === 'verified' + && WorkflowRun::query()->findOrFail($result['runId'])->status->value === 'completed') { + break; + } + // PHP 8.2/8.3 cannot enter workflow fibers from PendingCommand's destructor. $this->artisan('queue:work', [ '--once' => true, '--queue' => 'scope-recovery-test', '--sleep' => 0, '--tries' => 1, - ])->assertExitCode(0); + ])->assertExitCode(0) + ->run(); $progress[] = $this->progress($result['runId'], $step, $started); } $status = app(ScopeRepository::class)->inspect($scope)['status']; - $this->assertSame( - 'verified', - $status, - json_encode($progress, JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE) - ); + $diagnostic = json_encode($progress, JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE); + $this->assertSame('verified', $status, $diagnostic); + $this->assertSame('completed', WorkflowRun::query()->findOrFail($result['runId'])->status->value, $diagnostic); file_put_contents($input['report'], json_encode([ 'pid' => getmypid(), 'status' => $status, From a600ffe353ae0daec992ee2f6f0a142a154d8f31 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:44:16 +0000 Subject: [PATCH 32/32] test(compensation): cover retained outcome decoding and reuse --- .../V2/Compensation/ScopeDescriptorTest.php | 62 +++++++++++++++++++ .../V2/Compensation/ScopeProgressTest.php | 22 +++++++ 2 files changed, 84 insertions(+) diff --git a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php index e239e27c0..c89d5ba6a 100644 --- a/tests/Unit/V2/Compensation/ScopeDescriptorTest.php +++ b/tests/Unit/V2/Compensation/ScopeDescriptorTest.php @@ -5,6 +5,7 @@ namespace Tests\Unit\V2\Compensation; use InvalidArgumentException; +use JsonException; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use ReflectionClass; @@ -70,6 +71,67 @@ public function testVersionOnePersistedFormatsHaveStableCanonicalIdentities(): v )); } + public function testStoredForwardOutcomesPreserveTheirValueEvidenceAndUnknownState(): void + { + foreach ([ + ForwardResult::observed([ + 'items' => ['second', 'first'], + 'amount' => 1.0, + ], ['provider:proof']), + ForwardResult::unknown('response-lost'), + ] as $result) { + $stored = PortableDescriptor::encode($result->toArray()); + $restored = ForwardResult::fromArray(json_decode($stored, true, flags: JSON_THROW_ON_ERROR)); + $this->assertSame($stored, PortableDescriptor::encode($restored->toArray())); + $this->assertSame($result->observed, $restored->observed); + $this->assertSame($result->evidence, $restored->evidence); + } + } + + public function testStoredResolutionsPreserveAuthoritativeEvidenceAndSettlement(): void + { + foreach ([ + Resolution::verified(['provider:proof'], true), + Resolution::verified(['provider:proof'], false), + Resolution::unknown('provider-unavailable'), + ] as $result) { + $stored = PortableDescriptor::encode($result->toArray()); + $restored = Resolution::fromArray(json_decode($stored, true, flags: JSON_THROW_ON_ERROR)); + $this->assertSame($stored, PortableDescriptor::encode($restored->toArray())); + $this->assertSame($result->verified, $restored->verified); + $this->assertSame($result->forwardSettled, $restored->forwardSettled); + } + } + + public static function claimsWithoutEvidence(): iterable + { + yield 'observed-forward' => ['forward', ForwardResult::observed([], ['proof'])->toArray()]; + yield 'verified-compensation' => ['resolution', Resolution::verified(['proof'], true)->toArray()]; + } + + #[DataProvider('claimsWithoutEvidence')] + public function testStoredSuccessClaimsCannotDiscardTheirEvidence(string $kind, array $payload): void + { + $payload['evidence'] = []; + $this->expectException(InvalidArgumentException::class); + match ($kind) { + 'forward' => ForwardResult::fromArray($payload), + 'resolution' => Resolution::fromArray($payload), + }; + } + + public function testInvalidUtf8CannotBePersistedAsACompensationDescriptor(): void + { + try { + new Obligation('book', 'carrier.cancel', '1', [ + 'reference' => "\xFF", + ]); + $this->fail('The retained descriptor accepted invalid UTF-8.'); + } catch (InvalidArgumentException $error) { + $this->assertInstanceOf(JsonException::class, $error->getPrevious()); + } + } + public static function unsupportedVersions(): iterable { foreach ([null, '1', true, 1.0, 2] as $index => $version) { diff --git a/tests/Unit/V2/Compensation/ScopeProgressTest.php b/tests/Unit/V2/Compensation/ScopeProgressTest.php index 6acf7846e..d747a79f6 100644 --- a/tests/Unit/V2/Compensation/ScopeProgressTest.php +++ b/tests/Unit/V2/Compensation/ScopeProgressTest.php @@ -15,6 +15,28 @@ final class ScopeProgressTest extends TestCase { + public function testObservedForwardIsReturnedWithoutAdmittingAnotherAttempt(): void + { + [$scope, $id] = $this->registered(); + $progress = new ScopeProgress(); + $repository = new ScopeRepository(); + $result = ForwardResult::observed([ + 'reference' => 'booking-1', + ], ['provider:proof']); + $progress->admitForward($scope, $id, 'native-attempt-1'); + $progress->forwardResult($scope, $id, 'native-attempt-1', $result); + $before = $repository->inspect($scope); + $journal = $repository->journal($scope); + + $admission = $progress->admitForward($scope, $id, 'native-attempt-2'); + + $this->assertSame($result->toArray(), $admission['cached']); + $this->assertFalse($admission['uncertain']); + $this->assertSame('native-attempt-1', $admission['row']['forward_attempt_id']); + $this->assertSame($before, $repository->inspect($scope)); + $this->assertSame($journal, $repository->journal($scope)); + } + public function testUnknownForwardIsReconciledOnRetryAndCancellationClosesAdmission(): void { [$scope, $id] = $this->registered();