Skip to content

Commit 52cd64a

Browse files
committed
Document helper-thread activity heartbeats and cancellation
1 parent 5cd660f commit 52cd64a

3 files changed

Lines changed: 235 additions & 0 deletions

File tree

‎docs/reference/activity.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,59 @@ default. They can call `context().heartbeat_sync(details)` and inspect
77
See [activity execution and shutdown](worker.md#activity-execution) for thread
88
capacity, cancellation and process-isolation limits.
99

10+
## Heartbeats during one blocking call
11+
12+
Capture `ctx = activity.context()` inside the activity before starting a helper
13+
thread. A newly created thread does not automatically have the activity's
14+
context. Calling `activity.context()` there can raise "called outside of an
15+
activity execution". Pass the captured context instead. Its
16+
`ctx.heartbeat_sync(details)` schedules the heartbeat on the owning worker loop
17+
with the original task, attempt and worker identity. It can be called from a
18+
helper thread, but must never run on that worker loop. Async activity code uses
19+
`await ctx.heartbeat(details)`.
20+
21+
The complete [threaded-heartbeat example](https://github.com/durable-workflow/sdk-python/blob/main/examples/threaded_heartbeats.py)
22+
provides `call_with_heartbeats(work)`. Copy it into your application and use it
23+
from an ordinary synchronous activity:
24+
25+
```python
26+
from durable_workflow import activity
27+
from my_app.exports import export_service
28+
from my_app.threaded_heartbeats import call_with_heartbeats
29+
30+
31+
@activity.defn(name="wait_for_export")
32+
def wait_for_export(export_id: str) -> dict:
33+
return call_with_heartbeats(
34+
lambda cancelled: export_service.wait(
35+
export_id, timeout=120, cancel_event=cancelled,
36+
),
37+
interval=5,
38+
)
39+
```
40+
41+
`export_service` is your application's client, not a Durable Workflow API. It
42+
must implement the finite timeout and observe `cancel_event` promptly, returning
43+
or raising when it is set. Use the equivalent timeout and cancellation mechanism
44+
from your own library. If it cannot cooperate, give it a finite I/O deadline;
45+
the helper cannot force a blocked thread to exit. A worker process may need
46+
separate isolation for a call that cannot be bounded.
47+
48+
The helper records the original exception, sets the cancellation event, and
49+
stops heartbeating when cancellation, lease-authority loss or a transport error
50+
occurs. After the work unwinds, it joins the helper and re-raises that exception
51+
before a result can be returned. In particular, do not swallow `ActivityCancelled`
52+
or `ActivityLeaseAuthorityLost`, replace them with an application error, or
53+
return a successful result after either. `ActivityCancelled` inherits from
54+
`BaseException`, so `except Exception` alone does not catch it. Ordinary return
55+
and work exceptions also stop and join the helper.
56+
57+
Keep the client and its worker event loop alive until the helper has joined.
58+
An in-flight heartbeat still needs its bounded transport request to finish.
59+
Use finite client timeouts and bounded retry policy as well as a deadline on
60+
the business operation. A helper reports liveness even if that operation hangs,
61+
so its heartbeat is not proof of progress. Choose the interval using the
62+
configured heartbeat timeout and leave room for request, scheduling and retry
63+
delays. Five seconds is an example, not a guaranteed safe cadence.
64+
1065
::: durable_workflow.activity

‎examples/threaded_heartbeats.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
"""Heartbeat a bounded, cooperatively cancellable call from a sync activity."""
2+
from __future__ import annotations
3+
4+
import asyncio
5+
from collections.abc import Callable
6+
from queue import SimpleQueue
7+
from threading import Event, Thread
8+
from typing import TypeVar
9+
10+
from durable_workflow import activity
11+
from durable_workflow.errors import ActivityCancelled
12+
13+
T = TypeVar("T")
14+
15+
16+
def call_with_heartbeats(work: Callable[[Event], T], *, interval: float = 5.0) -> T:
17+
"""Run work(cancelled), which must enforce its own finite deadline.
18+
19+
Work must return or raise promptly when cancelled is set. This helper cannot
20+
interrupt an arbitrary blocking call. Call it from a synchronous activity,
21+
never from the worker event loop.
22+
"""
23+
try:
24+
asyncio.get_running_loop()
25+
except RuntimeError:
26+
pass
27+
else:
28+
raise RuntimeError("call_with_heartbeats requires a synchronous activity thread")
29+
if not 0 < interval < float("inf"):
30+
raise ValueError("interval must be finite and positive")
31+
32+
ctx = activity.context()
33+
finished = Event()
34+
cancelled = Event()
35+
failures: SimpleQueue[BaseException] = SimpleQueue()
36+
37+
def heartbeat() -> None:
38+
while not finished.wait(interval):
39+
try:
40+
ctx.heartbeat_sync({"message": "Waiting for remote operation"})
41+
except (ActivityCancelled, Exception) as error:
42+
# ActivityCancelled inherits BaseException, not Exception.
43+
failures.put(error)
44+
cancelled.set()
45+
return
46+
47+
helper = Thread(target=heartbeat, name="activity-heartbeat")
48+
helper.start()
49+
try:
50+
return work(cancelled)
51+
finally:
52+
finished.set()
53+
# Keep the captured context and client alive until the helper exits.
54+
helper.join()
55+
if not failures.empty():
56+
raise failures.get()
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
"""Exercise the public example with real ActivityContext thread/loop behavior."""
2+
from __future__ import annotations
3+
4+
import asyncio
5+
import importlib.util
6+
import unittest
7+
from pathlib import Path
8+
from threading import Event, Thread
9+
from threading import enumerate as threads
10+
from typing import Any
11+
12+
from durable_workflow import activity
13+
from durable_workflow.activity import ActivityContext, ActivityInfo, _set_context
14+
from durable_workflow.errors import ActivityCancelled, ActivityLeaseAuthorityLost
15+
16+
SPEC = importlib.util.spec_from_file_location(
17+
"threaded_heartbeats_example",
18+
Path(__file__).resolve().parents[1] / "examples" / "threaded_heartbeats.py",
19+
)
20+
assert SPEC is not None and SPEC.loader is not None
21+
EXAMPLE = importlib.util.module_from_spec(SPEC)
22+
SPEC.loader.exec_module(EXAMPLE)
23+
24+
25+
class HeartbeatClient:
26+
def __init__(self, response: Any = None) -> None:
27+
self.response = response if response is not None else {"can_continue": True}
28+
self.calls: list[dict[str, Any]] = []
29+
self.observed = Event()
30+
31+
async def heartbeat_activity_task(self, **request: Any) -> Any:
32+
self.calls.append(request)
33+
self.observed.set()
34+
if isinstance(self.response, Exception):
35+
raise self.response
36+
return self.response
37+
38+
39+
class ThreadedHeartbeatExampleTest(unittest.IsolatedAsyncioTestCase):
40+
async def asyncSetUp(self) -> None:
41+
self.client = HeartbeatClient()
42+
self.ctx = ActivityContext(
43+
info=ActivityInfo(
44+
"task-original", "export", "attempt-original", 2, "reports", "worker-original",
45+
),
46+
client=self.client, # type: ignore[arg-type]
47+
)
48+
_set_context(self.ctx)
49+
50+
async def asyncTearDown(self) -> None:
51+
_set_context(None)
52+
self.assertFalse([thread for thread in threads() if thread.name == "activity-heartbeat"])
53+
54+
async def run_work(self, work: Any) -> Any:
55+
return await asyncio.wait_for(
56+
asyncio.to_thread(EXAMPLE.call_with_heartbeats, work, interval=0.01),
57+
timeout=3,
58+
)
59+
60+
async def test_new_thread_needs_the_captured_context(self) -> None:
61+
errors: list[BaseException] = []
62+
63+
def look_up_context() -> None:
64+
try:
65+
activity.context()
66+
except RuntimeError as error:
67+
errors.append(error)
68+
69+
helper = Thread(target=look_up_context)
70+
helper.start()
71+
helper.join()
72+
self.assertEqual(len(errors), 1)
73+
self.assertIn("outside of an activity", str(errors[0]))
74+
75+
async def test_return_keeps_original_attempt_and_joins(self) -> None:
76+
def work(cancelled: Event) -> dict[str, int]:
77+
self.assertTrue(self.client.observed.wait(2))
78+
self.assertFalse(cancelled.is_set())
79+
return {"count": 7}
80+
81+
self.assertEqual(await self.run_work(work), {"count": 7})
82+
self.assertTrue(self.client.calls)
83+
for request in self.client.calls:
84+
self.assertEqual(request["task_id"], "task-original")
85+
self.assertEqual(request["activity_attempt_id"], "attempt-original")
86+
self.assertEqual(request["lease_owner"], "worker-original")
87+
88+
async def test_work_error_also_joins(self) -> None:
89+
original = ValueError("business call failed")
90+
91+
def work(cancelled: Event) -> None:
92+
self.assertTrue(self.client.observed.wait(2))
93+
raise original
94+
95+
with self.assertRaises(ValueError) as caught:
96+
await self.run_work(work)
97+
self.assertIs(caught.exception, original)
98+
99+
def cancelled_work(self, cancelled: Event) -> str:
100+
self.assertTrue(cancelled.wait(2))
101+
return "must not be returned"
102+
103+
async def test_cancellation_stops_work_and_preserves_type(self) -> None:
104+
self.client.response = {"cancel_requested": True}
105+
with self.assertRaises(ActivityCancelled):
106+
await self.run_work(self.cancelled_work)
107+
108+
async def test_authority_loss_keeps_original_identity_and_reason(self) -> None:
109+
self.client.response = {"can_continue": False, "reason": "activity_attempt_replaced"}
110+
with self.assertRaises(ActivityLeaseAuthorityLost) as caught:
111+
await self.run_work(self.cancelled_work)
112+
self.assertEqual(caught.exception.task_id, "task-original")
113+
self.assertEqual(caught.exception.reason, "activity_attempt_replaced")
114+
115+
async def test_transport_error_is_not_silently_lost(self) -> None:
116+
original = OSError("transport unavailable")
117+
self.client.response = original
118+
with self.assertRaises(OSError) as caught:
119+
await self.run_work(self.cancelled_work)
120+
self.assertIs(caught.exception, original)
121+
122+
async def test_event_loop_call_refuses_before_work(self) -> None:
123+
with self.assertRaisesRegex(RuntimeError, "synchronous activity thread"):
124+
EXAMPLE.call_with_heartbeats(lambda cancelled: self.fail("must not run"))

0 commit comments

Comments
 (0)