|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +from collections.abc import AsyncIterator |
| 4 | +from typing import Any |
| 5 | + |
| 6 | +import httpx |
| 7 | +import pytest |
| 8 | + |
| 9 | +from durable_workflow.client import Client |
| 10 | +from durable_workflow.errors import ServerError |
| 11 | +from durable_workflow.metrics import CLIENT_REQUEST_DURATION_SECONDS, CLIENT_REQUESTS, InMemoryMetrics |
| 12 | +from durable_workflow.retry_policy import TransportRetryPolicy |
| 13 | + |
| 14 | +ROUTES = [ |
| 15 | + ("/workflows/wf-{id}/signal/signal-{id}", "/workflows/{workflow_id}/signal/{name}"), |
| 16 | + ("/workflows/wf-{id}/runs/run-{id}/history/export", "/workflows/{workflow_id}/runs/{run_id}/history/export"), |
| 17 | + ("/worker/workflow-tasks/task-{id}/complete", "/worker/workflow-tasks/{task_id}/complete"), |
| 18 | + ("/worker/activity-tasks/task-{id}/heartbeat", "/worker/activity-tasks/{task_id}/heartbeat"), |
| 19 | + ("/worker/query-tasks/task-{id}/status", "/worker/query-tasks/{task_id}/status"), |
| 20 | + ("/worker/update-validation-tasks/task-{id}/approve", "/worker/update-validation-tasks/{task_id}/approve"), |
| 21 | + ("/worker/update-validation-tasks/task-{id}/reject", "/worker/update-validation-tasks/{task_id}/reject"), |
| 22 | + ("/worker/registrations/worker-{id}", "/worker/registrations/{worker_id}"), |
| 23 | + ("/worker/sessions/session-{id}/heartbeat", "/worker/sessions/{session_id}/heartbeat"), |
| 24 | + ("/worker/sessions/session-{id}", "/worker/sessions/{session_id}"), |
| 25 | + ("/activities/activity-{id}", "/activities/{activity_id}"), |
| 26 | + ("/workers/worker-{id}", "/workers/{worker_id}"), |
| 27 | + ("/schedules/schedule-{id}/history", "/schedules/{schedule_id}/history"), |
| 28 | + ("/namespaces/namespace-{id}/external-storage", "/namespaces/{name}/external-storage"), |
| 29 | + ("/task-queues/queue-{id}/build-ids/drain", "/task-queues/{task_queue}/build-ids/drain"), |
| 30 | + ("/workflows/wf-{id}/message-streams/stream-{id}/messages", "/workflows/{workflow_id}/message-streams/{name}/messages"), |
| 31 | + ("/workflows/wf-{id}/runs/run-{id}/streams/stream-{id}/items", "/workflows/{workflow_id}/runs/{run_id}/streams/{name}/items"), |
| 32 | + ("/service-endpoints/endpoint-{id}/services/service-{id}/operations/operation-{id}/execute", |
| 33 | + "/service-endpoints/{endpoint_name}/services/{service_name}/operations/{operation_name}/execute"), |
| 34 | + ("/worker/workflow-tasks/task-{id}/local-activities/attempt-{id}/heartbeat", |
| 35 | + "/worker/workflow-tasks/{task_id}/local-activities/{activity_attempt_id}/heartbeat"), |
| 36 | + ("/unrecognized/private-{id}", "unknown"), |
| 37 | + ("/worker/sessions/session-{id}/unrecognized/private-{id}", "unknown"), |
| 38 | + ("/workflows/wf-{id}/unrecognized/private-{id}", "unknown"), |
| 39 | +] |
| 40 | + |
| 41 | + |
| 42 | +@pytest.fixture |
| 43 | +async def instrumented_client() -> AsyncIterator[tuple[Client, InMemoryMetrics]]: |
| 44 | + metrics = InMemoryMetrics() |
| 45 | + client = Client( |
| 46 | + "https://runtime.example", token="synthetic-credential", metrics=metrics, |
| 47 | + retry_policy=TransportRetryPolicy(max_attempts=1), |
| 48 | + ) |
| 49 | + client._http = httpx.AsyncClient( |
| 50 | + base_url=client.base_url, transport=httpx.MockTransport(lambda _: httpx.Response(200, json={})), |
| 51 | + ) |
| 52 | + client._runtime_external_payload_transport_resolved = True |
| 53 | + async with client: |
| 54 | + yield client, metrics |
| 55 | + |
| 56 | + |
| 57 | +@pytest.mark.parametrize("path,expected", ROUTES) |
| 58 | +async def test_request_emissions_have_bounded_routes( |
| 59 | + path: str, expected: str, instrumented_client: tuple[Client, InMemoryMetrics], |
| 60 | +) -> None: |
| 61 | + client, metrics = instrumented_client |
| 62 | + for identifier in ("synthetic-a", "synthetic-b"): |
| 63 | + actual = path.replace("{id}", identifier) + "?token=synthetic-query-secret&cursor=" + identifier |
| 64 | + await client._request("POST", actual, json={}) |
| 65 | + tags = {"method": "POST", "route": expected, "plane": "control", "status_code": "200", "outcome": "ok"} |
| 66 | + assert metrics.counter_value(CLIENT_REQUESTS, tags) == 2 |
| 67 | + assert len(metrics.observations(CLIENT_REQUEST_DURATION_SECONDS, tags)) == 2 |
| 68 | + assert len(metrics.counters) == len(metrics.histograms) == 1 |
| 69 | + assert "synthetic-" not in repr(metrics.counters) |
| 70 | + |
| 71 | + |
| 72 | +@pytest.mark.parametrize("kind", ["workflow", "activity", "query", "update-validation"]) |
| 73 | +async def test_static_poll_route_is_distinct_from_a_task( |
| 74 | + kind: str, instrumented_client: tuple[Client, InMemoryMetrics], |
| 75 | +) -> None: |
| 76 | + client, metrics = instrumented_client |
| 77 | + await client._request("POST", f"/worker/{kind}-tasks/poll", worker=True, json={}) |
| 78 | + tags = {"method": "POST", "route": f"/worker/{kind}-tasks/poll", "plane": "worker", |
| 79 | + "status_code": "200", "outcome": "ok"} |
| 80 | + assert metrics.counter_value(CLIENT_REQUESTS, tags) == 1 |
| 81 | + |
| 82 | + |
| 83 | +@pytest.mark.parametrize("failure", [False, True], ids=["success", "http-failure"]) |
| 84 | +async def test_public_worker_apis_do_not_emit_task_or_session_ids( |
| 85 | + failure: bool, instrumented_client: tuple[Client, InMemoryMetrics], |
| 86 | +) -> None: |
| 87 | + client, metrics = instrumented_client |
| 88 | + if failure: |
| 89 | + await client._http.aclose() |
| 90 | + client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport( |
| 91 | + lambda _: httpx.Response(503, json={"reason": "unavailable"}), |
| 92 | + )) |
| 93 | + for identifier in ("synthetic-a", "synthetic-b"): |
| 94 | + calls = [ |
| 95 | + client.approve_update_validation_task( |
| 96 | + update_validation_task_id=identifier, lease_owner="synthetic-worker", update_validation_attempt=1, |
| 97 | + ), |
| 98 | + client.reject_update_validation_task( |
| 99 | + update_validation_task_id=identifier, lease_owner="synthetic-worker", update_validation_attempt=1, |
| 100 | + message="synthetic-payload-secret", reason="invalid", |
| 101 | + ), |
| 102 | + client.renew_worker_session("synthetic-worker", identifier, 120), |
| 103 | + client.close_worker_session("synthetic-worker", identifier), |
| 104 | + ] |
| 105 | + for call in calls: |
| 106 | + if failure: |
| 107 | + with pytest.raises(ServerError): |
| 108 | + await call |
| 109 | + else: |
| 110 | + await call |
| 111 | + routes = {dict(tags)["route"] for _, tags in metrics.counters} |
| 112 | + assert routes == { |
| 113 | + "/worker/update-validation-tasks/{task_id}/approve", "/worker/update-validation-tasks/{task_id}/reject", |
| 114 | + "/worker/sessions/{session_id}/heartbeat", "/worker/sessions/{session_id}", |
| 115 | + } |
| 116 | + assert all(count == 2 for count in metrics.counters.values()) |
| 117 | + assert "synthetic-" not in repr(metrics.counters) |
| 118 | + |
| 119 | + |
| 120 | +async def test_transport_failure_uses_the_same_bounded_route( |
| 121 | + instrumented_client: tuple[Client, InMemoryMetrics], |
| 122 | +) -> None: |
| 123 | + client, metrics = instrumented_client |
| 124 | + |
| 125 | + def unavailable(request: httpx.Request) -> Any: |
| 126 | + raise httpx.ConnectError("synthetic-transport-secret", request=request) |
| 127 | + |
| 128 | + await client._http.aclose() |
| 129 | + client._http = httpx.AsyncClient(base_url=client.base_url, transport=httpx.MockTransport(unavailable)) |
| 130 | + for identifier in ("synthetic-a", "synthetic-b"): |
| 131 | + polling = client.renew_worker_session("worker", identifier, 120) |
| 132 | + with pytest.raises(httpx.ConnectError): |
| 133 | + await polling |
| 134 | + assert len(metrics.counters) == 1 |
| 135 | + assert "synthetic-" not in repr(metrics.counters) |
0 commit comments