diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index 5a93bf7a..a5930b02 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -20,6 +20,9 @@ gh api 'repos/ably/specification/contents/uts/realtime/integration/.md' -- gh api repos/ably/specification/contents/uts/docs/proxy.md --jq '.content' | base64 -d gh api 'repos/ably/specification/contents/uts/rest/integration/proxy/.md' --jq '.content' | base64 -d gh api 'repos/ably/specification/contents/uts/realtime/integration/proxy/.md' --jq '.content' | base64 -d +gh api 'repos/ably/specification/contents/uts/objects/unit/.md' --jq '.content' | base64 -d +gh api 'repos/ably/specification/contents/uts/objects/integration/.md' --jq '.content' | base64 -d +gh api repos/ably/specification/contents/uts/objects/helpers/standard_test_pool.md --jq '.content' | base64 -d ``` The realtime integration tier nests, so list a directory before fetching from it: @@ -39,9 +42,10 @@ carries over as it stands: `channels/channel_publish_test.md` becomes `channels/channel_publish_test.py`. Every directory needs an `__init__.py`, as `test` is a package. -There are two kinds of tier. `rest/unit` and `realtime/unit` serve every request from a -mock and reach no network; `rest/integration` and `realtime/integration` run against the -real Ably sandbox and have no mock at all. See **The integration tier** below. +There are two kinds of tier. `rest/unit`, `realtime/unit` and `objects/unit` reach no +network; `rest/integration`, `realtime/integration` and `objects/integration` run against +the real Ably sandbox and have no mock at all. See **The integration tier** below, and +**The objects tier** for LiveObjects. `test/uts/rest/unit/time_test.py` is the reference example for REST unit, `test/uts/realtime/unit/connection/auto_connect_test.py` for realtime unit, @@ -451,6 +455,103 @@ async def test_rsc15l4_cloudfront_header_fallback(sandbox, proxy_session): assert http_responses(log)[0]['status'] == 403 ``` +## The objects tier + +`uts/objects/unit/.md` becomes `test/uts/objects/unit/_test.py`, the integration +specifications keep their `_test` suffix under `test/uts/objects/integration/`, and +`integration/proxy/objects_faults.md` becomes +`test/uts/objects/integration/proxy/objects_faults_test.py`. Seven unit specifications are +pure and construct internal objects; the other eight drive `channel.object` over the mock +websocket. Follow `internal_live_map_test.py` (pure), `path_object_mutations_test.py` (mock) +and `objects_lifecycle_test.py` (integration). The integration package's own `conftest.py` +provisions a separate app under the realtime tier's fixture name, `realtime_sandbox`, and +all three of its specifications take `use_binary_protocol`. The package is +`ably.pubsub.objects`; the public names are exported from `ably.pubsub.server`. + +**Translate the untyped API through the typed views** (LODR-061, the RTTS partition). The +specifications call every method on one `PathObject` or `Instance` class; ably-python +reaches a type's methods through its view: + +| Pseudocode | ably-python | +|---|---| +| `root.get("score").value()` | `root.get('score').as_live_counter().value()` | +| `root.get("name").value()` | `root.get('name').as_primitive().value()` | +| `pathObject.set("k", v)` | `await path_object.as_live_map().set('k', v)`; `root` is already a `LiveMapPathObject` | +| untyped `value() == null` | `as_primitive().value() is None` **and** `as_live_counter().value() is None` | +| `inst.id()` | `inst.id`, a property, as is `inst.type` | +| `{ depth: n }` | `subscribe(listener, depth=n)`, keyword-only | + +Mutations are `async`; reads, navigation, views and `subscribe` are not. `entries()` yields +tuples. A path's views never raise, and a write through the wrong one raises 92007, or 92005 +for a path that does not resolve; an `Instance`'s views raise 92007. A spelling difference is +not a deviation. Where the partition makes a read unreachable — `value()` on a +`LiveMapInstance` — assert the `type`, `not hasattr(...)` and the 92007, and cite S-5. + +**The pure tier adapts to five shapes**, defined in `deviations.md` and cited by label in the +module docstring: S-1 `apply_operation` returns a boolean, so record updates with +`capture_updates(obj)`; S-2 the sync state machine is a standalone `RealtimeObject()`'s +(`_on_attached(has_objects)`, `_handle_object_sync_messages(msgs, channel_serial)`, +`_sync_state`); S-3 `evaluate(vt, timestamp_ms)`; S-4 the retained create is +`operation.resolved_counter_create` / `resolved_map_create`; S-5 above. Members of a public +class that LODR-061 does not name carry a leading underscore (`channel.object._objects_pool`). + +`test/uts/objects/helpers/standard_test_pool.py` holds what the specifications share: +`setup_synced_channel` (and `_no_ack`), `standard_mock_websocket`, `objects_client`, +`objects_channel_options`, `objects_connected_message`, `objects_attached_message`, the +`build_*` builders, `json_value`, `bytes_value`, `ack_serial`, `remote_serial`, +`below_ack_serial`, `object_message(s)`, `capture_updates`, `build_public_object_message`, +`assert_unchanged_after_quiescence`, `provision_objects_via_rest`, and the wire constants +(`HAS_OBJECTS`, `OBJECT_SUBSCRIBE_FLAG`, `OBJECT_PUBLISH_FLAG`, `LWW`, the actions). +`test/uts/README.md` says what each is. + +### Traps found while deriving and implementing the objects tier + +- **An injected frame is applied on the transport's read task, not by `send_to_client`.** + Read state only after a `poll_until` on its effect. A negative — "did not fire", "the echo + was not applied" — needs a positive control sent *behind* the message under test and + `assert_unchanged_after_quiescence`; an exact count after `poll_until(>= n)` needs + `await settle()` first; and a subscription made straight after a seeding message receives + the seed, so poll for the seed before subscribing. The specifications often skip all three, + and their negatives then pass whatever the SDK does. +- **Process a re-sync ATTACHED before starting what it should hold back.** Send it, poll until + `channel.object._sync_state == ObjectsSyncState.SYNCING`, then start `get()` or the write as + a task; started earlier, it sees SYNCED and resolves at once. +- **A mock ATTACHED grants no modes.** The standard one carries `HAS_OBJECTS` alone, which + empties `channel.modes`, and RTO2 then checks the modes the channel *requested* — so request + them with `objects_channel_options()` or `get()` raises 40024. Grant modes as `flags` bits, + never as the specifications' `modes: [...]`. An injected `flags: 128` at the integration tier + empties them the same way. +- **A write resolves on its ACK.** The standard mock ACKs each OBJECT with + `ack_serial(msgSerial, i)`; under `setup_synced_channel_no_ack`, drive the write as a task. + Those serials land in `appliedOnAckSerials`, so never reuse one as an inbound serial, and + use `remote_serial(n)` for a remote write: a bare `'99'` sorts before `POOL_SERIAL` and is + stale. +- **`json` values are JSON strings on the wire** (OD2g), so compare a captured + `mapSet.value.json` after `json.loads`; `bytes` are base64. Actions are + `ObjectOperationAction`, an `IntEnum`. +- **Numbers decode to `float`.** Read them with `value(float)`; `value(int)` raises + `TypeError`, since `expected` must be exactly one of `str, float, bool, bytes, list, dict`. + Assert booleans with `is True`, since `True == 1`. +- **Creating a `LiveCounter` or `LiveMap` reads `/time`.** `objects_client` answers it; a + client built any other way needs `mock_http=time_mock_http(clock)`. +- **A tombstoned object reads null with no GC at all**, so a GC test asserts that the object + left `_objects_pool`. The GC interval, 300000 ms, is read when the first ATTACHED schedules + the timer on the client's clock: set `channel.object._gc_interval_ms` before `get()`. +- **The public and internal `ObjectMessage` share a name.** Reach the public one as + `publicmessage.ObjectMessage`, from `ably.pubsub.objects`. +- **Check that a test can fail.** Several objects specifications assert something that holds + with the behaviour removed. The derivation caught them by running each module against a + throwaway reference implementation with one fault injected; add the discriminating + assertion under `# UTS SPEC ERROR:` and record the fault. +- **At the sandbox, the echo arrives before the ACK**, so RTO9a3's dedup is only reachable at + the mock tier, and **ACKs are paced at about one per 500 ms per connection**, so each + awaited write after the first can take half a second. +- **Every objects channel gets an OBJECT_SYNC**, an empty one included, and a resumed + ATTACHED on an attached channel restarts the sync without a state change: wait on + `get()`, not on a channel state. A sync cursor can itself contain `:`. +- **uts-proxy matches `action` as a string and names actions only up to AUTH**: OBJECT_SYNC + is `'20'`, OBJECT `'19'`. Its `delay` holds every later frame behind the delayed one. + ## Traps that cost the most time Ordered by how much they cost, not by subject. Every one was hit for real while @@ -896,8 +997,8 @@ the reasoning. The next reader will otherwise reach the same first conclusion. ```bash uv run --frozen --extra crypto --extra dev ruff check ably/ test/ -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/helpers -q -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration test/uts/realtime/integration -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/objects/unit test/uts/helpers test/uts/objects/helpers -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration test/uts/realtime/integration test/uts/objects/integration -q RUN_DEVIATIONS=1 uv run --frozen --extra crypto --extra dev pytest test/uts -q ``` diff --git a/README.md b/README.md index e833df38..3d30e0a7 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,52 @@ async with create_realtime_client(key='your-ably-api-key', client_id='me') as re await channel.publish('test-event', 'hello world') ``` +### LiveObjects + +LiveObjects keeps shared, mutable state on a channel: maps and counters that every client +attached to it reads, updates and subscribes to. The channel needs the object modes, and +`channel.object.get()` attaches it, waits for the objects to sync, and returns the root map. +Values are read through a typed view of their path — `as_live_map()`, `as_live_counter()` or +`as_primitive()` — and reads are synchronous; writes are awaited. + +```python +from ably.pubsub.server import ChannelMode, ChannelOptions, LiveCounter, LiveMap, create_realtime_client + +async with create_realtime_client(key='your-ably-api-key') as realtime_client: + channel = realtime_client.channels.get( + 'my-objects', + ChannelOptions(modes=[ChannelMode.OBJECT_SUBSCRIBE, ChannelMode.OBJECT_PUBLISH]), + ) + + # Attach, wait for the objects to sync, and get the root map + root = await channel.object.get() + + # Create objects by setting them on the root + await root.set('visits', LiveCounter.create(0)) + await root.set('profile', LiveMap.create({'name': 'Alice', 'theme': 'dark'})) + + # Read through a typed view of the path + visits = root.get('visits').as_live_counter() + print(visits.value()) # 0.0 + print(root.at('profile.name').as_primitive().value(str)) # Alice + + # Subscribe to changes at a path + def on_change(event): + print(f'{event.object.path()} changed') + + subscription = root.get('visits').subscribe(on_change) + + # Mutate + await visits.increment(5) + + # Batch several writes into a single message + async with root.get('profile').as_live_map().batch() as profile: + profile.set('name', 'Bob') + profile.remove('theme') + + subscription.unsubscribe() +``` + ## Releases The [CHANGELOG.md](https://github.com/ably/ably-pubsub-python/blob/main/CHANGELOG.md) contains details of the latest releases for this SDK. You can also view all Ably releases on [changelog.ably.com](https://changelog.ably.com). diff --git a/ably/pubsub/objects/__init__.py b/ably/pubsub/objects/__init__.py new file mode 100644 index 00000000..621bd62b --- /dev/null +++ b/ably/pubsub/objects/__init__.py @@ -0,0 +1,7 @@ +"""LiveObjects: shared, conflict-free data structures stored on a realtime channel. + +The public names are re-exported from :mod:`ably.pubsub.server`; the modules here +hold the implementation behind them, and the internal classes the specification +names (``ObjectsPool``, ``InternalLiveMap``, ``InternalLiveCounter`` and the wire +types). +""" diff --git a/ably/pubsub/objects/batch.py b/ably/pubsub/objects/batch.py new file mode 100644 index 00000000..d704c96d --- /dev/null +++ b/ably/pubsub/objects/batch.py @@ -0,0 +1,385 @@ +"""Batched writes, published as one message (RTPO20, RTINS17, RTBC*). + +`batch()` on a typed path or instance returns a `Batch`, an async context manager whose +`__aenter__` checks the write preconditions and resolves the target (RTPO20b-d, +RTINS17b-d) and returns the matching `BatchContext`. Inside the block every context +method is synchronous: reads resolve against the objects as they are held locally, so +they do not see the writes queued before them, and writes only queue. `__aexit__` +publishes everything queued as one message (RTBC16d) when the block exits without +raising, and closes the batch either way (RTPO20g). Any use of a context after its batch +has closed raises `AblyException` 40000 (RTBC16e). + +A write validates its arguments when it is called, so an invalid one raises inside the +block and nothing is published. A `LiveMap` or `LiveCounter` value is validated then too, +but evaluated only when the batch is published, because its object ids need the server +time (RTO16). +""" + +from __future__ import annotations + +from functools import partial +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Generic, TypeVar, overload + +from ably.pubsub.objects.enums import ValueType +from ably.pubsub.objects.instance import ( + Instance, + LiveCounterInstance, + LiveMapInstance, + PrimitiveInstance, + expected_value_type, + primitive_value, + value_type_of, +) +from ably.pubsub.objects.livecounter import counter_inc_message +from ably.pubsub.objects.livemap import map_remove_message, map_set_message, map_set_messages +from ably.pubsub.objects.objectmessage import ObjectMessage +from ably.pubsub.objects.valuetypes import ( + LiveCounter, + LiveMap, + primitive_to_object_data, + validate_amount, + validate_key, + validate_value, +) +from ably.pubsub.util.exceptions import AblyException + +if TYPE_CHECKING: + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + + # RTBC16b: builds the ObjectMessages of one queued write when the batch is flushed + MessageConstructor = Callable[[], Awaitable[list[ObjectMessage]]] + +C = TypeVar('C', bound='BatchContext') +V = TypeVar('V', bound='BatchContext') + + +class Batch(Generic[C]): + """RTBC16: the `async with` block a `batch()` call returns. + + `resolve` returns the value the batch acts on, or None, and is called when the block + is entered; the value must be the live object a `context_type` wraps. `description` + names the target in the error raised when it is not. A `Batch` opens one batch, so it + can be entered once. + """ + + def __init__(self, realtime_object: RealtimeObject, resolve: Callable[[], Any], context_type: type[C], + description: str): + self._realtime_object = realtime_object + self._resolve = resolve + self._context_type = context_type + self._description = description + self._root_context: RootBatchContext | None = None + + async def __aenter__(self) -> C: + """RTPO20b-d, RTINS17b-d: checks the write preconditions, resolves the target and opens + the batch. + + Raises AblyException 92007 if the target is not a live object of the type the batch + was taken on (RTPO20c, RTINS17c), and RuntimeError if this batch was entered before. + """ + if self._root_context is not None: + raise RuntimeError('This batch has already been entered; call batch() again for another') + self._realtime_object._check_write_preconditions() # RTPO20b, RTINS17b + + value = self._resolve() + expected = self._context_type._value_type + actual = value_type_of(value) if value is not None else None + if actual is not expected: + # RTPO20c, RTINS17c: nothing, a primitive or a live object of another type + reason = 'it does not resolve' if actual is None else f'it is of type {actual.value}' + raise AblyException(f'Cannot batch operations on {self._description} as a {expected.value}: ' + f'{reason}', 400, 92007) + + instance = Instance._wrap(self._realtime_object, value) + self._root_context = RootBatchContext(self._realtime_object, instance) # RTPO20d, RTINS17d + return self._root_context.wrap_instance(instance) + + async def __aexit__(self, exc_type, exc, tb) -> None: + """RTPO20f, RTPO20g, RTBC16d: publishes the queued writes unless the block raised, and + closes the batch either way. + + An exception raised in the block propagates, and nothing is published. + """ + if exc_type is not None: + self._root_context.close() # RTPO20g + return + await self._root_context.flush() # RTPO20f, RTINS17f + + +class RootBatchContext: + """RTBC16: the state one batch shares across every context opened from it.""" + + def __init__(self, realtime_object: RealtimeObject, instance: Instance): + self.realtime_object = realtime_object + self.instance = instance + self.wrapped_instances: dict[str, BatchContext] = {} # RTBC16a + self.queued_message_constructors: list[MessageConstructor] = [] # RTBC16b + self.closed = False + + def wrap_instance(self, instance: Instance) -> BatchContext: + """RTBC16c: the context for `instance`, one per object id. + + The context is the subclass matching the `Instance` subclass, so a `LiveMapInstance` + is wrapped in a `LiveMapBatchContext`. + """ + object_id = instance.id + if object_id is not None: + context = self.wrapped_instances.get(object_id) + if context is not None: + return context + + context = _context_type_for(instance)(self, instance) + if object_id is not None: + self.wrapped_instances[object_id] = context + return context + + def queue_message_constructor(self, constructor: MessageConstructor) -> None: + """RTBC16b: queues `constructor`, which builds the ObjectMessages of one write when the + batch is flushed.""" + self.queued_message_constructors.append(constructor) + + async def flush(self) -> None: + """RTBC16d: closes the batch, builds the queued messages in the order they were queued and + publishes them in one message through `RealtimeObject._publish_and_apply`, if there are + any. + + Raises whatever building or publishing the messages raises. Nothing is published if + building any of them fails. + """ + constructors = self.queued_message_constructors + self.close() + if constructors: + # The block may have awaited since its writes checked the write preconditions (RTO26) + self.realtime_object._check_write_preconditions() + object_messages: list[ObjectMessage] = [] + for construct in constructors: + object_messages.extend(await construct()) + if object_messages: + await self.realtime_object._publish_and_apply(object_messages) + + def close(self) -> None: + """RTBC16e: closes the batch, so that every later use of its contexts raises 40000, and + drops the contexts and writes it holds.""" + self.closed = True + self.wrapped_instances = {} + self.queued_message_constructors = [] + + +class BatchContext: + """RTBC1: a synchronous view of an `Instance` inside a batch (RTBC2a). + + As on an `Instance`, the view helpers are checked: asking for a type the context does + not wrap raises `AblyException` 92007 (RTBC1a, RTTS9d). Reads check the access + preconditions (RTO25), and writes the write preconditions (RTO26), before checking + that the batch is still open. + """ + + # The type of live object that a batch opening this subclass of context acts on + _value_type: ValueType | None = None + + def __init__(self, root_context: RootBatchContext, instance: Instance): + self._root_context = root_context # RTBC2b + self._instance = instance # RTBC2a + + @property + def id(self) -> str | None: + """RTBC3: the wrapped object's id, or None for a primitive.""" + self._throw_if_closed() # RTBC3b + return self._instance.id # RTBC3a + + def get(self, key: str) -> BatchContext | None: + """RTBC4: the context for the value at `key` of the wrapped map, or None. + + Raises AblyException 40003 if `key` is not a string. + """ + self._realtime_object._check_access_preconditions() # RTBC4b + self._throw_if_closed() # RTBC4c + instance = self._instance.get(key) # RTBC4d + if instance is None: + return None + return self._root_context.wrap_instance(instance) # RTBC4e + + def compact(self) -> Any: + """RTBC10: `Instance.compact` on the wrapped instance.""" + self._realtime_object._check_access_preconditions() # RTBC10a + self._throw_if_closed() # RTBC10b + return self._instance.compact() # RTBC10c + + def compact_json(self) -> Any: + """RTBC11: `Instance.compact_json` on the wrapped instance.""" + self._realtime_object._check_access_preconditions() # RTBC11a + self._throw_if_closed() # RTBC11b + return self._instance.compact_json() # RTBC11c + + def as_live_map(self) -> LiveMapBatchContext: + """RTBC1a: this context as a map. Raises AblyException 92007 if it wraps something else.""" + self._throw_if_closed() # RTBC16e + return self._view(LiveMapBatchContext, self._instance.as_live_map()) # RTTS9d + + def as_live_counter(self) -> LiveCounterBatchContext: + """RTBC1a: this context as a counter. Raises AblyException 92007 if it wraps something else.""" + self._throw_if_closed() # RTBC16e + return self._view(LiveCounterBatchContext, self._instance.as_live_counter()) # RTTS9d + + def as_primitive(self) -> PrimitiveBatchContext: + """RTBC1a: this context as a primitive. Raises AblyException 92007 if it wraps something else.""" + self._throw_if_closed() # RTBC16e + return self._view(PrimitiveBatchContext, self._instance.as_primitive()) # RTTS9d + + @property + def _realtime_object(self) -> RealtimeObject: + return self._root_context.realtime_object + + def _view(self, context_type: type[V], instance: Instance) -> V: + """This context if it is a `context_type`, else a `context_type` wrapping `instance`.""" + if isinstance(self, context_type): + return self + return context_type(self._root_context, instance) + + def _throw_if_closed(self) -> None: + """RTBC16e: raises AblyException 40000 once the batch has closed.""" + if self._root_context.closed: + raise AblyException('Batch is closed: a batch context can only be used inside its batch block', + 400, 40000) + + def _queue(self, *object_messages: ObjectMessage) -> None: + """Queues a write whose ObjectMessages are already built.""" + async def construct() -> list[ObjectMessage]: + return list(object_messages) + + self._root_context.queue_message_constructor(construct) + + +class LiveMapBatchContext(BatchContext): + """RTBC1a: a batch context wrapping a map. + + A context of this type only ever wraps a map, so its writes never find another type + (RTBC12d, RTBC13d). + """ + + _instance: LiveMapInstance + _value_type = ValueType.LIVE_MAP + + def entries(self) -> list[tuple[str, BatchContext]]: + """RTBC6: `(key, context)` for each entry of the map, as `LiveMapInstance.entries` gives them.""" + self._realtime_object._check_access_preconditions() # RTBC6a + self._throw_if_closed() # RTBC6b + wrap_instance = self._root_context.wrap_instance + return [(key, wrap_instance(instance)) for key, instance in self._instance.entries()] # RTBC6c, RTBC6d + + def keys(self) -> list[str]: + """RTBC7: the keys of the map.""" + self._realtime_object._check_access_preconditions() # RTBC7a + self._throw_if_closed() # RTBC7b + return self._instance.keys() # RTBC7c + + def values(self) -> list[BatchContext]: + """RTBC8: a context for each entry of the map, as `entries` wraps them.""" + return [context for _, context in self.entries()] # RTBC8a-RTBC8c + + def size(self) -> int | None: + """RTBC9: the number of entries in the map. + + This is always an int, as the context always wraps a map; the signature is the one + `size()` has on the path views, where a path can resolve to nothing. + """ + self._realtime_object._check_access_preconditions() # RTBC9a + self._throw_if_closed() # RTBC9b + return self._instance.size() # RTBC9c + + def set(self, key: str, value: LiveMapValue) -> None: + """RTBC12: queues a MAP_SET of `key`, preceded by the creates a `LiveMap` or `LiveCounter` + value evaluates to. + + Raises AblyException 40003 for a key that is not a string and 40013 for a value of + an unsupported type (RTLM20e1), the contents of a `LiveMap` or `LiveCounter` value + included (RTLMV4). Such a value is evaluated when the batch is flushed, as its object + ids need the server time (RTO16); a primitive is encoded now, so later changes to a + dict or list the caller passed do not reach the batch. + """ + realtime_object = self._realtime_object + realtime_object._check_write_preconditions() # RTBC12b + self._throw_if_closed() # RTBC12c + validate_key(key) # RTLM20e1 + object_id = self._instance.id + + if not isinstance(value, (LiveCounter, LiveMap)): + self._queue(map_set_message(object_id, key, primitive_to_object_data(value))) # RTBC12e, RTLM20e1 + return + + validate_value(value) # RTLM20e1 + self._root_context.queue_message_constructor( + partial(map_set_messages, realtime_object, object_id, key, value)) # RTBC12e + + def remove(self, key: str) -> None: + """RTBC13: queues a MAP_REMOVE of `key`. + + Raises AblyException 40003 for a key that is not a string (RTLM21e1). + """ + self._realtime_object._check_write_preconditions() # RTBC13b + self._throw_if_closed() # RTBC13c + validate_key(key) # RTLM21e1 + self._queue(map_remove_message(self._instance.id, key)) # RTBC13e + + +class LiveCounterBatchContext(BatchContext): + """RTBC1a: a batch context wrapping a counter. + + A context of this type only ever wraps a counter, so its writes never find another + type (RTBC14d, RTBC15d). + """ + + _instance: LiveCounterInstance + _value_type = ValueType.LIVE_COUNTER + + def value(self) -> float: + """RTBC5: the counter's value.""" + self._realtime_object._check_access_preconditions() # RTBC5a + self._throw_if_closed() # RTBC5b + return self._instance.value() # RTBC5c + + def increment(self, amount: float = 1) -> None: + """RTBC14: queues a COUNTER_INC of `amount`. + + Raises AblyException 40003 unless `amount` is a finite number (RTLC12e1); a bool is + not a number. + """ + self._realtime_object._check_write_preconditions() # RTBC14b + self._throw_if_closed() # RTBC14c + self._queue(counter_inc_message(self._instance.id, validate_amount(amount))) # RTBC14e, RTLC12e1 + + def decrement(self, amount: float = 1) -> None: + """RTBC15: queues a COUNTER_INC of `-amount`, after the same validation as `increment`.""" + self._realtime_object._check_write_preconditions() # RTBC15b + self._throw_if_closed() # RTBC15c + self.increment(-validate_amount(amount)) # RTBC15e + + +class PrimitiveBatchContext(BatchContext): + """RTBC1a: a batch context wrapping a primitive.""" + + @overload + def value(self) -> Primitive: ... + + @overload + def value(self, expected: type[T]) -> T | None: ... + + def value(self, expected: type | None = None) -> Any: + """RTBC5: the wrapped primitive; with `expected`, only if it is of that type, as for + `PrimitiveInstance.value`.""" + value_type = expected_value_type(expected) + self._realtime_object._check_access_preconditions() # RTBC5a + self._throw_if_closed() # RTBC5b + return primitive_value(self._instance._value, value_type) # RTBC5c + + +def _context_type_for(instance: Instance) -> type[BatchContext]: + """RTBC1a: the `BatchContext` subclass matching the `Instance` subclass of `instance`.""" + if isinstance(instance, LiveMapInstance): + return LiveMapBatchContext + if isinstance(instance, LiveCounterInstance): + return LiveCounterBatchContext + if isinstance(instance, PrimitiveInstance): + return PrimitiveBatchContext + return BatchContext diff --git a/ably/pubsub/objects/defaults.py b/ably/pubsub/objects/defaults.py new file mode 100644 index 00000000..fbfd3c1e --- /dev/null +++ b/ably/pubsub/objects/defaults.py @@ -0,0 +1,10 @@ +"""Default values for the LiveObjects client.""" + +# RTO10a: how often tombstoned objects and map entries are checked for release +GC_INTERVAL_MS = 5 * 60 * 1000 + +# RTO10b3: the grace period used when ConnectionDetails.objectsGCGracePeriod is absent +GC_GRACE_PERIOD_MS = 24 * 60 * 60 * 1000 + +# RTO3b: the object id of the root map, which every ObjectsPool holds +ROOT_OBJECT_ID = 'root' diff --git a/ably/pubsub/objects/enums.py b/ably/pubsub/objects/enums.py new file mode 100644 index 00000000..b42e90ee --- /dev/null +++ b/ably/pubsub/objects/enums.py @@ -0,0 +1,39 @@ +"""The enumerations LiveObjects shares across its modules.""" + +from enum import Enum + + +class ObjectsSyncState(str, Enum): + """RTO17a: the state of synchronising a channel's objects with Ably.""" + + INITIALIZED = 'initialized' # RTO17a1 + SYNCING = 'syncing' # RTO17a2 + SYNCED = 'synced' # RTO17a3 + + +class ObjectsEvent(str, Enum): + """RTO18b: the sync state events `RealtimeObject.on` registers listeners for.""" + + SYNCING = 'syncing' # RTO18b1 + SYNCED = 'synced' # RTO18b2 + + +class ObjectsOperationSource(str, Enum): + """RTO22: where an operation being applied came from.""" + + LOCAL = 'local' # RTO22a: applied on receipt of the ACK for a local publish + CHANNEL = 'channel' # RTO22b: received over the channel + + +class ValueType(Enum): + """RTTS2: the category of value a path or an instance holds.""" + + STRING = 'string' # RTTS2a1 + NUMBER = 'number' # RTTS2a2 + BOOLEAN = 'boolean' # RTTS2a3 + BINARY = 'binary' # RTTS2a4 + JSON_OBJECT = 'json_object' # RTTS2a5 + JSON_ARRAY = 'json_array' # RTTS2a6 + LIVE_MAP = 'live_map' # RTTS2a7 + LIVE_COUNTER = 'live_counter' # RTTS2a8 + UNKNOWN = 'unknown' # RTTS2a9 diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/objects/instance.py new file mode 100644 index 00000000..64c238b6 --- /dev/null +++ b/ably/pubsub/objects/instance.py @@ -0,0 +1,384 @@ +"""Identity-addressed views onto a channel's objects (RTINS*, RTTS7-RTTS10). + +An `Instance` wraps one resolved value, a live object or a primitive, and follows that +object wherever it sits in the graph. Every instance is one of `LiveMapInstance`, +`LiveCounterInstance` or `PrimitiveInstance`, matching the value it wraps. The view +helpers are checked: asking for a type the instance does not wrap raises `AblyException` +92007 (RTTS9d). + +The module also holds what the path and instance views share about resolved values: +their `ValueType`, the type filter `value(expected)` applies, and compaction. +""" + +from __future__ import annotations + +import base64 +import copy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, overload + +from ably.pubsub.objects.enums import ValueType +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.liveobject import LiveObject +from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.util.exceptions import AblyException + +if TYPE_CHECKING: + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext + from ably.pubsub.objects.liveobject import LiveObjectUpdate + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.subscription import Subscription + from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + +# The types `value(expected)` accepts as `expected`, and the category each one selects +_EXPECTED_VALUE_TYPES: tuple[tuple[type, ValueType], ...] = ( + (str, ValueType.STRING), + (float, ValueType.NUMBER), + (bool, ValueType.BOOLEAN), + (bytes, ValueType.BINARY), + (list, ValueType.JSON_ARRAY), + (dict, ValueType.JSON_OBJECT), +) + +_PRIMITIVE_VALUE_TYPES = frozenset(value_type for _, value_type in _EXPECTED_VALUE_TYPES) + + +def value_type_of(value: Any) -> ValueType: + """RTTS2: the category of a resolved value, `UNKNOWN` for one in no known category (RTTS2a9). + + A boolean is never a number. + """ + if isinstance(value, InternalLiveMap): + return ValueType.LIVE_MAP + if isinstance(value, InternalLiveCounter): + return ValueType.LIVE_COUNTER + if isinstance(value, bool): + return ValueType.BOOLEAN + if isinstance(value, (int, float)): + return ValueType.NUMBER + if isinstance(value, str): + return ValueType.STRING + if isinstance(value, (bytes, bytearray)): + return ValueType.BINARY + if isinstance(value, dict): + return ValueType.JSON_OBJECT + if isinstance(value, list): + return ValueType.JSON_ARRAY + return ValueType.UNKNOWN + + +def expected_value_type(expected: type | None) -> ValueType | None: + """The category `value(expected)` filters on, or None for no filter. + + Raises TypeError for an `expected` other than `str`, `float`, `bool`, `bytes`, `list` + or `dict`. + """ + if expected is None: + return None + for python_type, value_type in _EXPECTED_VALUE_TYPES: + if expected is python_type: + return value_type + raise TypeError(f'expected must be one of str, float, bool, bytes, list or dict, not {expected!r}') + + +def primitive_value(value: Any, value_type: ValueType | None = None) -> Any: + """RTPO7d, RTTS6c: `value` if it is a primitive, of category `value_type` when one is given, + else None. + + A number is a float, and a JSON object or array is a copy, so that changing it leaves + the object it was read from unchanged. + """ + if value is None: + return None + actual_type = value_type_of(value) + if actual_type not in _PRIMITIVE_VALUE_TYPES or (value_type is not None and actual_type is not value_type): + return None + if actual_type is ValueType.NUMBER: + return float(value) + if actual_type is ValueType.BINARY: + return bytes(value) + if actual_type in (ValueType.JSON_OBJECT, ValueType.JSON_ARRAY): + return copy.deepcopy(value) + return value + + +def compact_value(value: Any, *, for_json: bool = False) -> Any: + """RTPO13, RTPO14: a plain snapshot of a resolved value. + + A map becomes a dict of its entries that are not tombstoned (RTPO13c1), nested maps + becoming nested dicts (RTPO13c2) and counters their values (RTPO13c3). Maps are visited + depth first in entry order, and a map reached again in the same snapshot, through a + cycle or a second reference, is the dict already built for it (RTPO13c5). With + `for_json` it is `{'objectId': id}` instead (RTPO14b2), and binary is a base64 string + (RTPO14b1), so that the result is JSON-serializable. + """ + if not isinstance(value, InternalLiveMap): + return _compact_leaf(value, for_json) + + result: dict[str, Any] = {} + built: dict[str, dict[str, Any]] = {value.object_id: result} + # A stack in place of recursion, so that the depth of the tree is not limited by the + # interpreter's. Each item is a dict being built and the entries of its map still to add. + stack = [(result, iter(value.entries()))] + while stack: + target, entries = stack[-1] + for key, entry_value in entries: + if not isinstance(entry_value, InternalLiveMap): + target[key] = _compact_leaf(entry_value, for_json) + continue + seen = built.get(entry_value.object_id) + if seen is not None: + target[key] = {'objectId': entry_value.object_id} if for_json else seen + continue + child: dict[str, Any] = {} + built[entry_value.object_id] = child + target[key] = child + stack.append((child, iter(entry_value.entries()))) + break + else: + stack.pop() + return result + + +def _compact_leaf(value: Any, for_json: bool) -> Any: + """RTPO13c3, RTPO13c4, RTPO13d, RTPO13e, RTPO14b1: the snapshot of a value that is not a map.""" + if isinstance(value, InternalLiveCounter): + return value.value() + if for_json and isinstance(value, (bytes, bytearray)): + return base64.b64encode(value).decode('ascii') + if isinstance(value, (dict, list)): + return copy.deepcopy(value) + return value + + +@dataclass +class InstanceSubscriptionEvent: + """RTINS16e: what an `Instance.subscribe` listener receives.""" + + object: Instance # RTINS16e1 + message: ObjectMessage | None = None # RTINS16e2 + + +class Instance: + """RTINS1, RTTS7: a reference to a specific live object or primitive value.""" + + def __init__(self, realtime_object: RealtimeObject, value: LiveObject | Primitive): + self._realtime_object = realtime_object + self._value = value # RTINS2a + + def __repr__(self) -> str: + if isinstance(self._value, LiveObject): + return f'{type(self).__name__}(id={self._value.object_id!r})' + return f'{type(self).__name__}({self._value!r})' + + @staticmethod + def _wrap(realtime_object: RealtimeObject, value: LiveObject | Primitive) -> Instance: + """The `Instance` subclass matching `value`.""" + if isinstance(value, InternalLiveMap): + return LiveMapInstance(realtime_object, value) + if isinstance(value, InternalLiveCounter): + return LiveCounterInstance(realtime_object, value) + return PrimitiveInstance(realtime_object, value) + + @property + def id(self) -> str | None: + """RTINS3: the wrapped object's id, or None for a primitive.""" + if isinstance(self._value, LiveObject): + return self._value.object_id # RTINS3a + return None # RTINS3b + + @property + def type(self) -> ValueType: + """RTTS8a: the type of the wrapped value.""" + return value_type_of(self._value) + + def get(self, key: str) -> Instance | None: + """RTINS5: an instance wrapping the value at `key` of the wrapped map, or None. + + Raises AblyException 40003 if `key` is not a string. + """ + self._realtime_object._check_access_preconditions() # RTINS5b + if not isinstance(key, str): + raise AblyException(f'Map key must be a string, not {type(key).__name__}', 400, 40003) + if not isinstance(self._value, InternalLiveMap): + return None # RTINS5d + value = self._value.get(key) + if value is None: + return None + return Instance._wrap(self._realtime_object, value) # RTINS5c + + def compact(self) -> Any: + """RTINS10: a plain snapshot of the wrapped value.""" + self._realtime_object._check_access_preconditions() # RTINS10a + return compact_value(self._value) # RTINS10b + + def compact_json(self) -> Any: + """RTINS11: `compact`, with binary as base64 and cycles as `{'objectId': ...}`. Never None.""" + self._realtime_object._check_access_preconditions() # RTINS11a + return compact_value(self._value, for_json=True) # RTINS11b + + def as_live_map(self) -> LiveMapInstance: + """RTTS9a: this instance as a map. Raises AblyException 92007 if it wraps something else.""" + if not isinstance(self._value, InternalLiveMap): + raise self._view_mismatch('a map') # RTTS9d + if isinstance(self, LiveMapInstance): + return self + return LiveMapInstance(self._realtime_object, self._value) + + def as_live_counter(self) -> LiveCounterInstance: + """RTTS9b: this instance as a counter. Raises AblyException 92007 if it wraps something else.""" + if not isinstance(self._value, InternalLiveCounter): + raise self._view_mismatch('a counter') # RTTS9d + if isinstance(self, LiveCounterInstance): + return self + return LiveCounterInstance(self._realtime_object, self._value) + + def as_primitive(self) -> PrimitiveInstance: + """RTTS9c: this instance as a primitive. Raises AblyException 92007 if it wraps something else.""" + if isinstance(self._value, LiveObject): + raise self._view_mismatch('a primitive') # RTTS9d + if isinstance(self, PrimitiveInstance): + return self + return PrimitiveInstance(self._realtime_object, self._value) + + def _view_mismatch(self, requested: str) -> AblyException: + return AblyException(f'Cannot view an instance of type {self.type.value} as {requested}', 400, 92007) + + +class LiveMapInstance(Instance): + """RTTS10a: an instance wrapping an `InternalLiveMap`.""" + + _value: InternalLiveMap + + @property + def id(self) -> str: + """RTINS3a: the wrapped map's id.""" + return self._value.object_id + + def batch(self) -> Batch[LiveMapBatchContext]: + """RTINS17: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTINS17b). + """ + # Imported here, as the batch module builds on the views + from ably.pubsub.objects.batch import Batch, LiveMapBatchContext + + return Batch(self._realtime_object, lambda: self._value, LiveMapBatchContext, f'object {self.id!r}') + + def entries(self) -> list[tuple[str, Instance]]: + """RTINS6: `(key, instance)` for each entry of the map. + + An entry referencing an object this client does not hold has no value to wrap, and + is left out. + """ + self._realtime_object._check_access_preconditions() # RTINS6a + # RTINS6b + return [(key, Instance._wrap(self._realtime_object, value)) + for key, value in self._value.entries() if value is not None] + + def keys(self) -> list[str]: + """RTINS7: the keys of the map.""" + self._realtime_object._check_access_preconditions() # RTINS7a + return self._value.keys() # RTINS7b + + def values(self) -> list[Instance]: + """RTINS8: an instance for each entry of the map, as `entries` wraps them.""" + return [instance for _, instance in self.entries()] # RTINS8a, RTINS8b + + def size(self) -> int: + """RTINS9: the number of entries in the map.""" + self._realtime_object._check_access_preconditions() # RTINS9a + return self._value.size() # RTINS9b + + async def set(self, key: str, value: LiveMapValue) -> None: + """RTINS12: sets `key` in the wrapped map.""" + self._realtime_object._check_write_preconditions() # RTINS12b + await self._value.set(key, value) # RTINS12c + + async def remove(self, key: str) -> None: + """RTINS13: removes `key` from the wrapped map.""" + self._realtime_object._check_write_preconditions() # RTINS13b + await self._value.remove(key) # RTINS13c + + def subscribe(self, listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: + """RTINS16: calls `listener` for each update to the wrapped map, wherever it sits (RTINS16g).""" + self._realtime_object._check_access_preconditions() # RTINS16b + return _subscribe(self, listener) + + +class LiveCounterInstance(Instance): + """RTTS10b: an instance wrapping an `InternalLiveCounter`.""" + + _value: InternalLiveCounter + + @property + def id(self) -> str: + """RTINS3a: the wrapped counter's id.""" + return self._value.object_id + + def batch(self) -> Batch[LiveCounterBatchContext]: + """RTINS17: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTINS17b). + """ + # Imported here, as the batch module builds on the views + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext + + return Batch(self._realtime_object, lambda: self._value, LiveCounterBatchContext, f'object {self.id!r}') + + def value(self) -> float: + """RTINS4b: the wrapped counter's value.""" + self._realtime_object._check_access_preconditions() # RTINS4a + return self._value.value() + + async def increment(self, amount: float = 1) -> None: + """RTINS14: increments the wrapped counter by `amount`.""" + self._realtime_object._check_write_preconditions() # RTINS14b + await self._value.increment(amount) # RTINS14c + + async def decrement(self, amount: float = 1) -> None: + """RTINS15: decrements the wrapped counter by `amount`.""" + self._realtime_object._check_write_preconditions() # RTINS15b + await self._value.decrement(amount) # RTINS15c + + def subscribe(self, listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: + """RTINS16: calls `listener` for each update to the wrapped counter, wherever it sits (RTINS16g).""" + self._realtime_object._check_access_preconditions() # RTINS16b + return _subscribe(self, listener) + + +class PrimitiveInstance(Instance): + """RTTS10c, RTTS10d: an instance wrapping a primitive value. It has no `subscribe` (RTTS7b).""" + + @overload + def value(self) -> Primitive: ... + + @overload + def value(self, expected: type[T]) -> T | None: ... + + def value(self, expected: type | None = None) -> Any: + """RTINS4c: the wrapped primitive; with `expected`, only if it is of that type, as for + `PrimitivePathObject.value`.""" + value_type = expected_value_type(expected) + self._realtime_object._check_access_preconditions() # RTINS4a + return primitive_value(self._value, value_type) + + +def _subscribe(instance: LiveMapInstance | LiveCounterInstance, + listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: + """RTINS16d: subscribes `listener` to the updates of the live object `instance` wraps. + + The live object calls its listeners synchronously, logging any that raises, and drops + them once it is tombstoned (RTLO4b4c3c). + """ + realtime_object = instance._realtime_object + + def on_update(update: LiveObjectUpdate) -> None: + object_message = update.object_message + message = None + if object_message is not None and object_message.operation is not None: + message = ObjectMessage._from_internal(object_message, realtime_object._channel_name) # RTINS16e2 + listener(InstanceSubscriptionEvent(instance, message)) # RTINS16e1 + + return instance._value.subscribe(on_update) diff --git a/ably/pubsub/objects/livecounter.py b/ably/pubsub/objects/livecounter.py new file mode 100644 index 00000000..69ce71df --- /dev/null +++ b/ably/pubsub/objects/livecounter.py @@ -0,0 +1,173 @@ +"""The counter CRDT (RTLC*).""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.liveobject import CounterUpdate, LiveCounterUpdate, LiveObject +from ably.pubsub.objects.objectmessage import CounterInc, ObjectMessage, ObjectOperation, ObjectOperationAction +from ably.pubsub.objects.valuetypes import validate_amount +from ably.pubsub.util.clock import Clock +from ably.pubsub.util.exceptions import AblyException + +if TYPE_CHECKING: + from ably.pubsub.objects.objectspool import ObjectsPool + +log = logging.getLogger(__name__) + + +class InternalLiveCounter(LiveObject): + """RTLC1: a counter holding a 64-bit float (RTLC3). + + A new counter has `data` 0 (RTLC4). + """ + + _update_type = LiveCounterUpdate + + def __init__(self, object_id: str, *, pool: ObjectsPool | None = None, clock: Clock | None = None): + super().__init__(object_id, pool=pool, clock=clock) + self.data: float = 0.0 # RTLC3, RTLC4b + + def value(self) -> float: + """RTLC5: the current count.""" + return self.data # RTLC5c + + async def increment(self, amount: float) -> None: + """RTLC12: publishes a COUNTER_INC of `amount` through `RealtimeObject._publish_and_apply`. + + Raises AblyException 40003 if `amount` is not a finite number (RTLC12e1); a bool + is not a number. + """ + await self._publish_counter_inc(validate_amount(amount)) + + async def decrement(self, amount: float) -> None: + """RTLC13: `increment` by `-amount`, after the same validation (RTLC13c).""" + await self._publish_counter_inc(-validate_amount(amount)) # RTLC13b, RTLC13c + + async def _publish_counter_inc(self, number: float) -> None: + realtime_object = self.realtime_object + if realtime_object is None: + raise AblyException('Unable to increment a counter that is not on a channel', 400, 40000) + await realtime_object._publish_and_apply([counter_inc_message(self.object_id, number)]) # RTLC12g + + def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: + """RTLC7: applies `object_message.operation`, returning whether it was applied (RTLC7g). + + The update is emitted through `notify_updated` (RTLC7d1a, RTLC7d5a, RTLC7d4c). + """ + if not self.can_apply_operation(object_message): + # RTLC7b; an operation with invalid serials has already been logged by can_apply_operation + if object_message.serial and object_message.site_code: + log.debug(f'InternalLiveCounter.apply_operation(): skipping an operation whose serial is not ' + f'newer than the one recorded for its site; serial={object_message.serial}, ' + f'site_code={object_message.site_code}, object_id={self.object_id}') + return False + + if source == ObjectsOperationSource.CHANNEL: + # RTLC7c: the serial is recorded whether or not the operation then applies + self.site_timeserials[object_message.site_code] = object_message.serial + + if self.is_tombstone: + return False # RTLC7e + + operation = object_message.operation + if operation.action == ObjectOperationAction.COUNTER_CREATE: + update = self.apply_counter_create(operation, object_message) # RTLC7d1 + elif operation.action == ObjectOperationAction.COUNTER_INC: + update = self.apply_counter_inc(operation.counter_inc, object_message) # RTLC7d5 + elif operation.action == ObjectOperationAction.OBJECT_DELETE: + update = self.tombstone(object_message) # RTLC7d4, RTLO5 + else: + # RTLC7d3 + log.warning(f'InternalLiveCounter.apply_operation(): skipping an object operation message with ' + f'an unsupported action; action={operation.action!r}, object_id={self.object_id}') + return False + + self.notify_updated(update) # RTLC7d1a, RTLC7d5a, RTLC7d4c + return True # RTLC7d1b, RTLC7d5b, RTLC7d4b + + def replace_data(self, object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC6: replaces this counter's data with `object_message.object`, returning the diff.""" + object_state = object_message.object + self.site_timeserials = dict(object_state.site_timeserials) # RTLC6a + + if self.is_tombstone: + return LiveCounterUpdate(noop=True) # RTLC6e, RTLC6e1 + + if object_state.tombstone: + return self.tombstone(object_message) # RTLC6f, RTLC6f2 + + previous_data = self.data # RTLC6g + self.create_operation_is_merged = False # RTLC6b + count = object_state.counter.count if object_state.counter is not None else None + self.data = count if count is not None else 0.0 # RTLC6c + if object_state.create_op is not None: + self.merge_initial_value(object_state.create_op, object_message) # RTLC6d + + update = self.diff(previous_data, self.data) # RTLC6h + update.object_message = object_message + return update + + def apply_counter_create(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC8: applies a COUNTER_CREATE.""" + if self.create_operation_is_merged: + # RTLC8b + log.debug(f'InternalLiveCounter.apply_counter_create(): skipping a COUNTER_CREATE for a counter ' + f'whose create operation is already merged; object_id={self.object_id}') + return LiveCounterUpdate(noop=True) + return self.merge_initial_value(operation, object_message) # RTLC8c, RTLC8e + + def apply_counter_inc(self, counter_inc: CounterInc | None, + object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC9: applies a COUNTER_INC. + + A COUNTER_INC with no `number`, or with no payload at all, is a no-op (RTLC9h). + """ + number = counter_inc.number if counter_inc is not None else None + if number is None: + return LiveCounterUpdate(noop=True) # RTLC9h + + self.data += number # RTLC9f + return LiveCounterUpdate(update=CounterUpdate(amount=number), object_message=object_message) # RTLC9g + + def merge_initial_value(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC16: merges the initial value of a create operation into this counter. + + The `CounterCreate` is `operation.resolved_counter_create`. + """ + counter_create = operation.resolved_counter_create + count = counter_create.count if counter_create is not None else None + self.create_operation_is_merged = True # RTLC16b + if count is None: + return LiveCounterUpdate(noop=True) # RTLC16d + + self.data += count # RTLC16a + return LiveCounterUpdate(update=CounterUpdate(amount=count), object_message=object_message) # RTLC16c + + def clear_data(self) -> None: + """RTO27a1: resets `data` to 0, emitting nothing.""" + self.data = 0.0 # RTLC4b + + @staticmethod + def diff(previous_data: float, new_data: float, *, for_tombstone: bool = False) -> LiveCounterUpdate: + """RTLC14: the update between two counter values. + + A zero difference is a no-op (RTLC14c), unless the diff is computed for a tombstone + (`for_tombstone`, RTLO4e5). + """ + amount = new_data - previous_data # RTLC14b + if amount == 0 and not for_tombstone: + return LiveCounterUpdate(noop=True) # RTLC14c + return LiveCounterUpdate(update=CounterUpdate(amount=amount)) + + +def counter_inc_message(object_id: str, number: float) -> ObjectMessage: + """RTLC12e: the COUNTER_INC of the counter `object_id` by `number`, a validated amount + (`validate_amount`).""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, # RTLC12e2 + object_id=object_id, # RTLC12e3 + counter_inc=CounterInc(number=number), # RTLC12e5 + )) diff --git a/ably/pubsub/objects/livemap.py b/ably/pubsub/objects/livemap.py new file mode 100644 index 00000000..4b8f44f2 --- /dev/null +++ b/ably/pubsub/objects/livemap.py @@ -0,0 +1,451 @@ +"""The last-write-wins map CRDT (RTLM*).""" + +from __future__ import annotations + +import dataclasses +import logging +from typing import TYPE_CHECKING, Any + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.liveobject import MAP_KEY_REMOVED, MAP_KEY_UPDATED, LiveMapUpdate, LiveObject +from ably.pubsub.objects.objectmessage import ( + MapRemove, + MapSet, + ObjectData, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapEntry, + ObjectsMapSemantics, +) +from ably.pubsub.objects.valuetypes import ( + LiveCounter, + LiveMap, + evaluate, + primitive_to_object_data, + validate_key, + validate_value, +) +from ably.pubsub.util.clock import Clock +from ably.pubsub.util.exceptions import AblyException + +if TYPE_CHECKING: + from ably.pubsub.objects.objectspool import ObjectsPool + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.valuetypes import LiveMapValue + +log = logging.getLogger(__name__) + + +class InternalLiveMap(LiveObject): + """RTLM1: a map of `ObjectsMapEntry` values by key, with per-entry last-write-wins (RTLM3). + + A new map has empty `data` and no `clear_timeserial` (RTLM4). A value read from the + map is a primitive, an `InternalLiveMap` or `InternalLiveCounter` looked up in `pool`, + or None. + """ + + _update_type = LiveMapUpdate + + def __init__(self, object_id: str, semantics: ObjectsMapSemantics = ObjectsMapSemantics.LWW, *, + pool: ObjectsPool | None = None, clock: Clock | None = None): + super().__init__(object_id, pool=pool, clock=clock) + self.semantics: ObjectsMapSemantics = semantics # RTLM4b + self.data: dict[str, ObjectsMapEntry] = {} # RTLM3, RTLM4c + self.clear_timeserial: str | None = None # RTLM25, RTLM4d + + def get(self, key: str) -> Any: + """RTLM5: the value at `key`, or None if there is none, it is tombstoned, or this map is.""" + if self.is_tombstone: + return None # RTLM5e + entry = self.data.get(key) + if entry is None: + return None # RTLM5d1 + if self.is_entry_tombstoned(entry): + return None # RTLM5d2h + return self._resolve(entry.data) + + def size(self) -> int: + """RTLM10: the number of entries that are not tombstoned (RTLM14).""" + return sum(1 for entry in self.data.values() if not self.is_entry_tombstoned(entry)) # RTLM10d + + def entries(self) -> list[tuple[str, Any]]: + """RTLM11: `(key, value)` for each entry that is not tombstoned, values as `get` returns them.""" + # RTLM11d1, RTLM11d3; a value that resolves to None is still returned (RTLM11d3a) + return [(key, self._resolve(entry.data)) for key, entry in self.data.items() + if not self.is_entry_tombstoned(entry)] + + def keys(self) -> list[str]: + """RTLM12: the keys `entries` returns.""" + return [key for key, _ in self.entries()] + + def values(self) -> list[Any]: + """RTLM13: the values `entries` returns.""" + return [value for _, value in self.entries()] + + async def set(self, key: str, value: LiveMapValue) -> None: + """RTLM20: publishes a MAP_SET, preceded by the creates a `LiveMap` or `LiveCounter` value + evaluates to, through `RealtimeObject._publish_and_apply`. + + Raises AblyException 40003 for a key that is not a string and 40013 for a value of + an unsupported type (RTLM20e1, RTLMV4b, RTLMV4c), before anything is requested or + published. + """ + validate_key(key) # RTLM20e1 + realtime_object = self._publishing_realtime_object() + object_messages = await map_set_messages(realtime_object, self.object_id, key, value) # RTLM20e + await realtime_object._publish_and_apply(object_messages) # RTLM20h1, RTLM20h2 + + async def remove(self, key: str) -> None: + """RTLM21: publishes a MAP_REMOVE through `RealtimeObject._publish_and_apply`. + + Raises AblyException 40003 for a key that is not a string (RTLM21e1). + """ + validate_key(key) # RTLM21e1 + realtime_object = self._publishing_realtime_object() + await realtime_object._publish_and_apply([map_remove_message(self.object_id, key)]) # RTLM21g + + def _publishing_realtime_object(self) -> RealtimeObject: + realtime_object = self.realtime_object + if realtime_object is None: + raise AblyException('Unable to update a map that is not on a channel', 400, 40000) + return realtime_object + + def is_entry_tombstoned(self, entry: ObjectsMapEntry) -> bool: + """RTLM14: whether `entry` is tombstoned, or references a tombstoned object in `pool`.""" + if entry.tombstone: + return True # RTLM14a + referenced = self._referenced_object(entry.data) + return referenced is not None and referenced.is_tombstone # RTLM14c, RTLM14b + + def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: + """RTLM15: applies `object_message.operation`, returning whether it was applied (RTLM15g). + + The update is emitted through `notify_updated` (RTLM15d1a and its siblings). + """ + if not self.can_apply_operation(object_message): + # RTLM15b; an operation with invalid serials has already been logged by can_apply_operation + if object_message.serial and object_message.site_code: + log.debug(f'InternalLiveMap.apply_operation(): skipping an operation whose serial is not ' + f'newer than the one recorded for its site; serial={object_message.serial}, ' + f'site_code={object_message.site_code}, object_id={self.object_id}') + return False + + if source == ObjectsOperationSource.CHANNEL: + # RTLM15c: the serial is recorded whether or not the operation then applies + self.site_timeserials[object_message.site_code] = object_message.serial + + if self.is_tombstone: + return False # RTLM15e + + operation = object_message.operation + action = operation.action + if action == ObjectOperationAction.MAP_CREATE: + update = self.apply_map_create(operation, object_message) # RTLM15d1 + elif action == ObjectOperationAction.MAP_SET: + map_set = operation.map_set + if map_set is None or map_set.key is None or map_set.value is None: + self._log_missing_payload(action) + return False + update = self.apply_map_set(map_set, object_message.serial, object_message) # RTLM15d6 + elif action == ObjectOperationAction.MAP_REMOVE: + map_remove = operation.map_remove + if map_remove is None or map_remove.key is None: + self._log_missing_payload(action) + return False + update = self.apply_map_remove(map_remove, object_message.serial, object_message.serial_timestamp, + object_message) # RTLM15d7 + elif action == ObjectOperationAction.OBJECT_DELETE: + update = self.tombstone(object_message) # RTLM15d5, RTLO5 + elif action == ObjectOperationAction.MAP_CLEAR: + update = self.apply_map_clear(object_message.serial, object_message) # RTLM15d8 + else: + # RTLM15d4 + log.warning(f'InternalLiveMap.apply_operation(): skipping an object operation message with an ' + f'unsupported action; action={action!r}, object_id={self.object_id}') + return False + + self.notify_updated(update) # RTLM15d1a, RTLM15d6a, RTLM15d7a, RTLM15d5c, RTLM15d8a + return True # RTLM15d1b, RTLM15d6b, RTLM15d7b, RTLM15d5b, RTLM15d8b + + def replace_data(self, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM6: replaces this map's data with `object_message.object`, returning the diff.""" + object_state = object_message.object + self.site_timeserials = dict(object_state.site_timeserials) # RTLM6a + + if self.is_tombstone: + return LiveMapUpdate(noop=True) # RTLM6e, RTLM6e1 + + if object_state.tombstone: + return self.tombstone(object_message) # RTLM6f, RTLM6f2 + + previous_data = self.data # RTLM6g + self.create_operation_is_merged = False # RTLM6b + objects_map = object_state.map + self.clear_timeserial = objects_map.clear_timeserial if objects_map is not None else None # RTLM6i + entries = objects_map.entries if objects_map is not None and objects_map.entries is not None else {} + # RTLM6c. Each entry is copied, so that applying operations to this map never alters the + # ObjectState it was replaced from. + self.data = {} + for key, entry in entries.items(): + entry = dataclasses.replace(entry, tombstoned_at=None) + if entry.tombstone: + entry.tombstoned_at = self._tombstoned_at(entry.serial_timestamp) # RTLM6c1 + self.data[key] = entry + if object_state.create_op is not None: + self.merge_initial_value(object_state.create_op, object_message) # RTLM6d + + update = self.diff(previous_data, self.data) # RTLM6h + update.object_message = object_message + return update + + def apply_map_create(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM16: applies a MAP_CREATE.""" + if self.create_operation_is_merged: + # RTLM16b + log.debug(f'InternalLiveMap.apply_map_create(): skipping a MAP_CREATE for a map whose create ' + f'operation is already merged; object_id={self.object_id}') + return LiveMapUpdate(noop=True) + return self.merge_initial_value(operation, object_message) # RTLM16d, RTLM16f + + def apply_map_set(self, map_set: MapSet, serial: str | None, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM7: applies a MAP_SET for `map_set.key`.""" + key = map_set.key + if self._is_at_or_before_clear(serial): + # RTLM7h + log.debug(f'InternalLiveMap.apply_map_set(): skipping a MAP_SET at or before the map\'s clear ' + f'serial; key={key!r}, serial={serial}, clear_timeserial={self.clear_timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + entry = self.data.get(key) + if entry is not None: + if not self.can_apply_map_operation(entry.timeserial, serial): + # RTLM7a1 + log.debug(f'InternalLiveMap.apply_map_set(): skipping a MAP_SET not newer than the entry; ' + f'key={key!r}, serial={serial}, entry serial={entry.timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + self._release_reference(entry.data, key) # RTLM7a3 + entry.data = map_set.value # RTLM7a2e + entry.timeserial = serial # RTLM7a2b + entry.tombstone = False # RTLM7a2c + entry.tombstoned_at = None # RTLM7a2d + else: + # RTLM7b4, RTLM7b2, RTLM7b3 + self.data[key] = ObjectsMapEntry(data=map_set.value, timeserial=serial, tombstone=False, + tombstoned_at=None) + + value = map_set.value + if value is not None and value.object_id and self.pool is not None: + # RTLM7g + referenced = self.pool.create_zero_value_object_if_not_exists(value.object_id) # RTLM7g1 + if referenced is not None: + referenced.add_parent_reference(self, key) # RTLM7g2 + + return LiveMapUpdate(update={key: MAP_KEY_UPDATED}, object_message=object_message) # RTLM7f + + def apply_map_remove(self, map_remove: MapRemove, serial: str | None, serial_timestamp: int | None, + object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM8: applies a MAP_REMOVE for `map_remove.key`.""" + key = map_remove.key + if self._is_at_or_before_clear(serial): + # RTLM8g + log.debug(f'InternalLiveMap.apply_map_remove(): skipping a MAP_REMOVE at or before the map\'s ' + f'clear serial; key={key!r}, serial={serial}, clear_timeserial={self.clear_timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + entry = self.data.get(key) + if entry is not None: + if not self.can_apply_map_operation(entry.timeserial, serial): + # RTLM8a1 + log.debug(f'InternalLiveMap.apply_map_remove(): skipping a MAP_REMOVE not newer than the ' + f'entry; key={key!r}, serial={serial}, entry serial={entry.timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + self._release_reference(entry.data, key) # RTLM8a3 + entry.data = None # RTLM8a2a + entry.timeserial = serial # RTLM8a2b + entry.tombstone = True # RTLM8a2c + entry.tombstoned_at = self._tombstoned_at(serial_timestamp) # RTLM8a2d + else: + # RTLM8b1, RTLM8b2, RTLM8b3 + self.data[key] = ObjectsMapEntry(data=None, timeserial=serial, tombstone=True, + tombstoned_at=self._tombstoned_at(serial_timestamp)) + + return LiveMapUpdate(update={key: MAP_KEY_REMOVED}, object_message=object_message) # RTLM8e + + def apply_map_clear(self, serial: str | None, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM24: applies a MAP_CLEAR.""" + if not serial: + # A MAP_CLEAR with no serial cannot be ordered against the map's entries. `apply_operation` + # never gets here, as it rejects a message with no serial (RTLO4a3); a direct call can. + log.warning(f'InternalLiveMap.apply_map_clear(): skipping a MAP_CLEAR with no serial; ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + if self.clear_timeserial is not None and self.clear_timeserial > serial: + # RTLM24c + log.debug(f'InternalLiveMap.apply_map_clear(): skipping a MAP_CLEAR older than the map\'s clear ' + f'serial; serial={serial}, clear_timeserial={self.clear_timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + self.clear_timeserial = serial # RTLM24d + removed: dict[str, str] = {} + for key, entry in list(self.data.items()): + # RTLM24e1 + if entry.timeserial is None or serial > entry.timeserial: + self._release_reference(entry.data, key) # RTLM24e1c + del self.data[key] # RTLM24e1a + removed[key] = MAP_KEY_REMOVED # RTLM24e1b + + return LiveMapUpdate(update=removed, object_message=object_message) # RTLM24f + + def merge_initial_value(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM23: merges the initial entries of a create operation into this map. + + The `MapCreate` is `operation.resolved_map_create`. + """ + map_create = operation.resolved_map_create + merged: dict[str, str] = {} + entries = map_create.entries if map_create is not None else {} + for key, entry in entries.items(): + # The serial an entry is applied with is the entry's own, not the message's + if entry.tombstone: + # RTLM23a2 + update = self.apply_map_remove(MapRemove(key=key), entry.timeserial, entry.serial_timestamp, + object_message) + else: + # RTLM23a1 + update = self.apply_map_set(MapSet(key=key, value=entry.data), entry.timeserial, object_message) + if not update.noop: + merged.update(update.update) # RTLM23c + + self.create_operation_is_merged = True # RTLM23b + return LiveMapUpdate(update=merged, object_message=object_message) # RTLM23c + + def gc_tombstoned_entries(self, grace_period_ms: int, now_ms: int) -> None: + """RTLM19: removes tombstoned entries whose `tombstoned_at` is `grace_period_ms` or more + before `now_ms`.""" + for key, entry in list(self.data.items()): + # RTLM19a1 + if (entry.tombstone and entry.tombstoned_at is not None + and now_ms - entry.tombstoned_at >= grace_period_ms): + del self.data[key] + + def clear_data(self) -> None: + """RTO27a1: resets `data` to empty and `clear_timeserial` to None, emitting nothing. + + Each object an entry references stops recording this map as a parent (RTLO4e9). + """ + for key, entry in self.data.items(): + self._release_reference(entry.data, key) + self.data = {} # RTLM4c + self.clear_timeserial = None # RTLM4d + + @staticmethod + def can_apply_map_operation(entry_serial: str | None, operation_serial: str | None) -> bool: + """RTLM9: whether an operation with `operation_serial` may replace an entry with `entry_serial`.""" + if not entry_serial and not operation_serial: + return False # RTLM9b + if not entry_serial: + return True # RTLM9d + if not operation_serial: + return False # RTLM9c + return operation_serial > entry_serial # RTLM9a, RTLM9e + + @staticmethod + def diff(previous_data: dict[str, ObjectsMapEntry], new_data: dict[str, ObjectsMapEntry], *, + for_tombstone: bool = False) -> LiveMapUpdate: + """RTLM22: the update between two versions of a map's data. + + Only entries whose own `tombstone` flag is false are compared. An empty diff is a + no-op (RTLM22c), unless it is computed for a tombstone (`for_tombstone`, RTLO4e5). + """ + previous_live = {key: entry for key, entry in previous_data.items() if not entry.tombstone} + new_live = {key: entry for key, entry in new_data.items() if not entry.tombstone} + update: dict[str, str] = {} + for key in previous_live: + if key not in new_live: + update[key] = MAP_KEY_REMOVED # RTLM22b1 + for key, entry in new_live.items(): + if key not in previous_live or previous_live[key].data != entry.data: + update[key] = MAP_KEY_UPDATED # RTLM22b2, RTLM22b3 + + if not update and not for_tombstone: + return LiveMapUpdate(noop=True) # RTLM22c + return LiveMapUpdate(update=update) + + def _resolve(self, data: ObjectData | None) -> Any: + """RTLM5d2: the value `data` holds: a primitive, the object it references, or None.""" + if data is None: + return None + # RTLM5d2b-RTLM5d2e, and a JSON value (OD2g) + primitive = data.value + if primitive is not None: + return primitive + if data.object_id is not None: + return self._referenced_object(data) # RTLM5d2f1, RTLM5d2f2 + return None # RTLM5d2g + + def _referenced_object(self, data: ObjectData | None) -> LiveObject | None: + """The object in `pool` that `data` references, if there is one.""" + if data is None or data.object_id is None or self.pool is None: + return None + return self.pool.get(data.object_id) + + def _release_reference(self, data: ObjectData | None, key: str) -> None: + """RTLM7a3, RTLM8a3, RTLM24e1c, RTLO4e9: removes this map at `key` from the parent references of + the object `data` references, if it is in `pool`.""" + referenced = self._referenced_object(data) + if referenced is not None: + referenced.remove_parent_reference(self, key) + + def _is_at_or_before_clear(self, serial: str | None) -> bool: + """RTLM7h, RTLM8g: whether an operation with `serial` is no later than the map's last MAP_CLEAR.""" + return self.clear_timeserial is not None and (not serial or self.clear_timeserial >= serial) + + def _log_missing_payload(self, action: ObjectOperationAction) -> None: + log.warning(f'InternalLiveMap.apply_operation(): skipping a {action.name} operation with no key or ' + f'value; object_id={self.object_id}') + + +def map_set_message(object_id: str, key: str, data: ObjectData) -> ObjectMessage: + """RTLM20e: the MAP_SET of `key` in the map `object_id` to `data`.""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_SET, # RTLM20e2 + object_id=object_id, # RTLM20e3 + map_set=MapSet(key=key, value=data), # RTLM20e6, RTLM20e7 + )) + + +def map_remove_message(object_id: str, key: str) -> ObjectMessage: + """RTLM21e: the MAP_REMOVE of `key` from the map `object_id`.""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_REMOVE, # RTLM21e2 + object_id=object_id, # RTLM21e3 + map_remove=MapRemove(key=key), # RTLM21e5 + )) + + +async def map_set_messages(realtime_object: RealtimeObject, object_id: str, key: str, + value: LiveMapValue) -> list[ObjectMessage]: + """RTLM20e: the ObjectMessages setting `key` of the map `object_id` to `value`. + + For a `LiveMap` or `LiveCounter`, they are the creates it evaluates to, with object ids + generated from the server time (RTLM20e7g1), followed by a MAP_SET referencing the last + of them (RTLM20e7g2); for a primitive, just the MAP_SET (RTLM20e7b-RTLM20e7f). + + Raises as `validate_value` does for an invalid value (RTLM20e1), a blueprint before the + server time is fetched. + """ + if not isinstance(value, (LiveCounter, LiveMap)): + return [map_set_message(object_id, key, primitive_to_object_data(value))] + + validate_value(value) + server_time_ms = await realtime_object._get_server_time_ms() + object_messages = evaluate(value, server_time_ms) + data = ObjectData(object_id=object_messages[-1].operation.object_id) + return [*object_messages, map_set_message(object_id, key, data)] diff --git a/ably/pubsub/objects/liveobject.py b/ably/pubsub/objects/liveobject.py new file mode 100644 index 00000000..47bf61b5 --- /dev/null +++ b/ably/pubsub/objects/liveobject.py @@ -0,0 +1,250 @@ +"""The behaviour every live object shares (RTLO*), and the updates objects emit.""" + +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Callable + +from ably.pubsub.objects.defaults import ROOT_OBJECT_ID +from ably.pubsub.objects.objectmessage import ObjectMessage +from ably.pubsub.objects.subscription import Registry, Subscription +from ably.pubsub.util.clock import Clock + +if TYPE_CHECKING: + from ably.pubsub.objects.enums import ObjectsOperationSource + from ably.pubsub.objects.livemap import InternalLiveMap + from ably.pubsub.objects.objectspool import ObjectsPool + from ably.pubsub.objects.realtimeobject import RealtimeObject + +log = logging.getLogger(__name__) + +# RTLM18b: the change recorded against a map key in a LiveMapUpdate +MAP_KEY_UPDATED = 'updated' +MAP_KEY_REMOVED = 'removed' + + +@dataclass +class LiveObjectUpdate: + """RTLO4b4: a change to a live object's data, as the object emits it.""" + + update: Any = None # RTLO4b4a + noop: bool = False # RTLO4b4b + object_message: ObjectMessage | None = None # RTLO4b4d + tombstone: bool = False # RTLO4b4e + + +@dataclass +class CounterUpdate: + """RTLC11b: what a counter update changed.""" + + amount: float # RTLC11b1 + + +@dataclass +class LiveCounterUpdate(LiveObjectUpdate): + """RTLC11: an update emitted by an `InternalLiveCounter`. + + `update` is a `CounterUpdate`, or None on a no-op. + """ + + update: CounterUpdate | None = None + + +@dataclass +class LiveMapUpdate(LiveObjectUpdate): + """RTLM18: an update emitted by an `InternalLiveMap`. + + `update` maps each changed key to `'updated'` or `'removed'` (RTLM18b). + """ + + update: dict[str, str] = field(default_factory=dict) + + +class LiveObject(ABC): + """RTLO1: the state and behaviour `InternalLiveCounter` and `InternalLiveMap` share. + + A live object is constructible on its own, with no channel, so that its CRDT + behaviour can be driven directly. `pool` is the `ObjectsPool` it reads other objects + from (RTLM5d2f, RTLM14c, RTLO4f); an object added to a pool with `pool[object_id] = obj` + is adopted by it. `clock` is what RTLO6b reads the local time from; without one the + object uses its pool's clock, and with neither, the system clock. + """ + + # The update type this object emits, which a no-op it returns is also an instance of + _update_type: type[LiveObjectUpdate] = LiveObjectUpdate + + def __init__(self, object_id: str, *, pool: ObjectsPool | None = None, clock: Clock | None = None): + self.object_id: str = object_id # RTLO3a + self.site_timeserials: dict[str, str] = {} # RTLO3b + self.create_operation_is_merged: bool = False # RTLO3c + self.is_tombstone: bool = False # RTLO3d + self.tombstoned_at: int | None = None # RTLO3e + self.parent_references: dict[str, set[str]] = {} # RTLO3f + self.pool: ObjectsPool | None = pool + self.data: Any = None + self._clock: Clock | None = clock + self._listeners: Registry[Callable[[LiveObjectUpdate], None]] = Registry() + + @property + def clock(self) -> Clock: + """The clock RTLO6b reads the local time from.""" + if self._clock is not None: + return self._clock + if self.pool is not None: + return self.pool.clock + return Clock() + + @property + def realtime_object(self) -> RealtimeObject | None: + """The `RealtimeObject` this object's pool belongs to, if any.""" + return self.pool.realtime_object if self.pool is not None else None + + def subscribe(self, listener: Callable[[LiveObjectUpdate], None]) -> Subscription: + """RTLO4b: registers `listener` for the updates this object emits. + + Listeners are called synchronously from `notify_updated`, in registration order. + """ + return Subscription(self._listeners.register(listener)) + + def notify_updated(self, update: LiveObjectUpdate) -> None: + """RTLO4b4c: emits `update`. + + Every update an operation produces passes through here, a no-op included, and a + no-op stops here (RTLO4b4c1). Otherwise the `subscribe` listeners are called + (RTLO4b4c3a), path subscriptions are dispatched through the `RealtimeObject`, if + there is one (RTLO4b4c3b), and a tombstone update then deregisters the `subscribe` + listeners (RTLO4b4c3c). A listener registered during the dispatch is not called by it, + and one deregistered during it is not called again. + """ + if update.noop: + return + + for registration in self._listeners.snapshot(): + if not registration.active: + continue + try: + registration.item(update) + except Exception: + log.exception(f'LiveObject.notify_updated(): a subscription listener raised; ' + f'object_id={self.object_id}') + + realtime_object = self.realtime_object + if realtime_object is not None: + realtime_object._path_object_subscription_register.dispatch(self, update) + + if update.tombstone: + self._listeners.clear() + + def can_apply_operation(self, object_message: ObjectMessage) -> bool: + """RTLO4a: whether `object_message`'s serial is newer than this object's for its site.""" + serial = object_message.serial + site_code = object_message.site_code + # RTLO4a3 + if not isinstance(serial, str) or not serial or not isinstance(site_code, str) or not site_code: + log.warning(f'LiveObject.can_apply_operation(): object operation message has invalid serial ' + f'values, skipping it; serial={serial!r}, site_code={site_code!r}, ' + f'object_id={self.object_id}') + return False + + site_serial = self.site_timeserials.get(site_code) # RTLO4a4 + if not site_serial: + return True # RTLO4a5 + return serial > site_serial # RTLO4a6 + + @abstractmethod + def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: + """RTLC7, RTLM15: applies `object_message.operation`, returning whether it was applied. + + The resulting update is emitted through `notify_updated`, not returned. + """ + + @abstractmethod + def replace_data(self, object_message: ObjectMessage) -> LiveObjectUpdate: + """RTLC6, RTLM6: replaces this object's data with `object_message.object`. + + The update is returned, not emitted; the caller emits it (RTO5c7). + """ + + def tombstone(self, object_message: ObjectMessage) -> LiveObjectUpdate: + """RTLO4e: tombstones this object, returning the resulting update. + + The update has `tombstone` set and `object_message` populated, and is never a no-op, + except for the root object, which is never tombstoned (RTLO4e10). + """ + if self.object_id == ROOT_OBJECT_ID: + log.warning(f'LiveObject.tombstone(): attempt to tombstone the root object was rejected; ' + f'serial={object_message.serial}, site_code={object_message.site_code}, ' + f'message id={object_message.id}') + return self._update_type(noop=True) + + self.is_tombstone = True # RTLO4e2 + self.tombstoned_at = self._tombstoned_at(object_message.serial_timestamp) # RTLO4e3 + previous_data = self.data + self.clear_data() # RTLO4e9, RTLO4e4 + update = self.diff(previous_data, self.data, for_tombstone=True) # RTLO4e5 + update.tombstone = True # RTLO4e6 + update.object_message = object_message # RTLO4e7 + return update + + @abstractmethod + def clear_data(self) -> None: + """RTO27a1: resets this object's data to that of a new empty object, emitting nothing.""" + + @staticmethod + @abstractmethod + def diff(previous_data: Any, new_data: Any, *, for_tombstone: bool = False) -> LiveObjectUpdate: + """RTLC14, RTLM22: the update between two versions of this type of object's data.""" + + def add_parent_reference(self, parent: InternalLiveMap, key: str) -> None: + """RTLO4g: records that `parent` references this object at `key`.""" + self.parent_references.setdefault(parent.object_id, set()).add(key) + + def remove_parent_reference(self, parent: InternalLiveMap, key: str) -> None: + """RTLO4h: removes the record that `parent` references this object at `key`.""" + keys = self.parent_references.get(parent.object_id) + if keys is None: + return # RTLO4h1 + keys.discard(key) # RTLO4h2 + if not keys: + del self.parent_references[parent.object_id] # RTLO4h3 + + def get_full_paths(self) -> list[list[str]]: + """RTLO4f: every key-path from the root map to this object, each once, in no set order. + + Parents are looked up by object id in this object's pool. + """ + paths: list[list[str]] = [] + seen: set[tuple[str, ...]] = set() + # Each item is an object, the keys from it down to this object, and the ids of the + # objects already on that path, which a simple path does not revisit (RTLO4f2) + stack: list[tuple[LiveObject, list[str], frozenset[str]]] = [(self, [], frozenset())] + while stack: + live_object, path, visited = stack.pop() + if live_object.object_id in visited: + continue + if live_object.object_id == ROOT_OBJECT_ID: + if tuple(path) not in seen: + seen.add(tuple(path)) + paths.append(path) + continue + if self.pool is None: + continue + visited = visited | {live_object.object_id} + for parent_id, keys in live_object.parent_references.items(): + parent = self.pool.get(parent_id) + if parent is None: + continue + for key in keys: + stack.append((parent, [key, *path], visited)) + return paths + + def _tombstoned_at(self, serial_timestamp: int | None) -> int: + """RTLO6: the time something is tombstoned at, given the operation's `serial_timestamp`.""" + if serial_timestamp is not None: + return serial_timestamp # RTLO6a + # RTLO6b, RTLO6b1 + log.debug(f'LiveObject._tombstoned_at(): no serial_timestamp for the operation, using the local ' + f'clock instead; object_id={self.object_id}') + return self.clock.now_ms() diff --git a/ably/pubsub/objects/objectid.py b/ably/pubsub/objects/objectid.py new file mode 100644 index 00000000..8a7a349a --- /dev/null +++ b/ably/pubsub/objects/objectid.py @@ -0,0 +1,31 @@ +"""Object ids generated by the client (RTO14) and parsed by it (RTO6b).""" + +from __future__ import annotations + +import base64 +import hashlib +import os + +# 12 random bytes base64-encode to 16 characters, the RTLCV4d/RTLMV4g minimum +_NONCE_ENTROPY_BYTES = 12 + + +def generate_object_id(object_type: str, initial_value: str, nonce: str, timestamp_ms: int) -> str: + """RTO14: `[type]:[hash]@[timestamp]`, where the hash is the URL-safe, unpadded base64 of + the SHA-256 of `initial_value:nonce` (RTO14b). + + `object_type` is `'map'` or `'counter'` (RTO14a1). + """ + digest = hashlib.sha256(f'{initial_value}:{nonce}'.encode()).digest() # RTO14b1 + hash_part = base64.urlsafe_b64encode(digest).decode('ascii').rstrip('=') # RTO14b2 + return f'{object_type}:{hash_part}@{int(timestamp_ms)}' # RTO14c + + +def generate_nonce() -> str: + """RTLCV4d, RTLMV4g: a random string of at least 16 characters.""" + return base64.urlsafe_b64encode(os.urandom(_NONCE_ENTROPY_BYTES)).decode('ascii') + + +def object_type_of(object_id: str) -> str: + """RTO6b1: the type an object id names, the part before the first `:`.""" + return object_id.split(':', 1)[0] diff --git a/ably/pubsub/objects/objectmessage.py b/ably/pubsub/objects/objectmessage.py new file mode 100644 index 00000000..6dce9407 --- /dev/null +++ b/ably/pubsub/objects/objectmessage.py @@ -0,0 +1,676 @@ +"""The internal object message types and their wire encoding. + +These are the `ObjectMessage` (OM*), `ObjectOperation` (OOP*), `ObjectState` (OST*) +and `ObjectData` (OD*) types of features.md, with the protocol v6 operation payloads +(`MapCreate`, `MapSet`, `MapRemove`, `CounterCreate`, `CounterInc`, `ObjectDelete`, +`MapClear`, `MapCreateWithObjectId`, `CounterCreateWithObjectId`) and the `ObjectsMap`, +`ObjectsMapEntry` and `ObjectsCounter` state types. + +Every type decodes from a wire dictionary with `from_dict(obj, format)` and encodes to +one with `to_dict(format)`, where `format` is `'json'` or `'msgpack'`. The two forms +differ only in how values are carried: + +- `ObjectData.bytes` is a base64 string on the JSON wire and raw binary on msgpack + (OD2d, OD4c2, OD4d2, OD5a1, OD5b2). +- `ObjectData.json` is a JSON-encoded string on both wires (OD2g), and a decoded `dict` + or `list` here. +- Numbers are float64 (OD4c3) and decode to `float`. +- Enums are carried as their integer wire values (OOP2, OMP2). + +Decoded values are what the rest of the package works with: `ObjectData.bytes` holds +`bytes`, `ObjectData.json` holds the parsed value, and wire names are snake_cased. +Fields that are only ever held locally (`ObjectsMapEntry.tombstoned_at`, RTLM3a1, and +the `derived_from` of the `*CreateWithObjectId` payloads, RTLMV4j5 and RTLCV4g5) are +never encoded. + +A ProtocolMessage's `state` is decoded one `ObjectMessage` at a time: a message that fails +to decode is logged and left out, and the others are decoded. +""" + +from __future__ import annotations + +import base64 +import builtins +import json +import logging +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Any, TypeVar + +log = logging.getLogger(__name__) + +WIRE_FORMAT_JSON = 'json' +WIRE_FORMAT_MSGPACK = 'msgpack' + +E = TypeVar('E', bound='_WireEnum') + + +class _WireEnum(IntEnum): + """An enumeration carried as its integer wire value, whose `UNKNOWN` member stands for a wire + value this library does not recognise.""" + + @classmethod + def from_wire(cls: type[E], value: Any) -> E: + """The member for a wire value, or `UNKNOWN` for one that is not recognised (OOP2a, OMP2a).""" + if isinstance(value, bool): + return cls.UNKNOWN + try: + return cls(value) + except ValueError: + return cls.UNKNOWN + + +class ObjectOperationAction(_WireEnum): + """OOP2: the operation an `ObjectOperation` describes, by its wire value. + + `UNKNOWN` stands for a wire value this library does not recognise (OOP2a). Such an + operation decodes without error, is never applied and is never encoded. + """ + + UNKNOWN = -1 + MAP_CREATE = 0 + MAP_SET = 1 + MAP_REMOVE = 2 + COUNTER_CREATE = 3 + COUNTER_INC = 4 + OBJECT_DELETE = 5 + MAP_CLEAR = 6 + + +class ObjectsMapSemantics(_WireEnum): + """OMP2: the conflict-resolution semantics of a map, by its wire value. + + `UNKNOWN` stands for a wire value this library does not recognise (OMP2a). + """ + + UNKNOWN = -1 + LWW = 0 + + +def _check_format(format: str) -> None: + if format not in (WIRE_FORMAT_JSON, WIRE_FORMAT_MSGPACK): + raise ValueError(f"Unknown wire format {format!r}; expected 'json' or 'msgpack'") + + +def _decode_number(value: Any) -> Any: + # OD4c3: numbers are float64 on the wire, so a decoded number is always a float + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return value + + +def _encode_number(value: Any, format: str) -> Any: + if format == WIRE_FORMAT_MSGPACK and isinstance(value, int) and not isinstance(value, bool): + return float(value) + return value + + +def _decode_bytes(value: Any) -> Any: + # OD5a1, OD5b2: raw binary on msgpack, a base64 string on JSON + if isinstance(value, (bytes, bytearray)): + return bytes(value) + if isinstance(value, str): + return base64.b64decode(value) + return value + + +def _encode_bytes(value: Any, format: str) -> Any: + # OD4c2, OD4d2 + if format == WIRE_FORMAT_MSGPACK: + return bytes(value) + return base64.b64encode(value).decode('ascii') + + +def _encode_json(value: Any) -> str: + return json.dumps(value, separators=(',', ':')) + + +@dataclass +class ObjectData: + """OD1: a value held in an object, either a primitive or a reference to another object. + + At most one of the value fields is set (OD2). + """ + + object_id: str | None = None # OD2a + encoding: str | None = None # OD2b + boolean: bool | None = None # OD2c + # Spelled `builtins.bytes`, as the field's own name hides the builtin within the class + bytes: builtins.bytes | None = None # OD2d + number: float | None = None # OD2e + string: str | None = None # OD2f + json: dict | list | None = None # OD2g + + @property + def value(self) -> Any: + """The primitive this data holds, or None when it holds a reference or nothing.""" + for candidate in (self.boolean, self.bytes, self.number, self.string, self.json): + if candidate is not None: + return candidate + return None + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectData: + """Decodes an `ObjectData` per OD5.""" + _check_format(format) + json_value = obj.get('json') + if isinstance(json_value, str): + json_value = json.loads(json_value) + string_value = obj.get('string') + encoding = obj.get('encoding') + # OD5a2, OD5b3: a JSON payload carried in `string` under the legacy `json` encoding + if encoding == 'json' and string_value is not None and json_value is None: + json_value = json.loads(string_value) + string_value = None + encoding = None + return ObjectData( + object_id=obj.get('objectId'), + encoding=encoding, + boolean=obj.get('boolean'), + bytes=_decode_bytes(obj.get('bytes')), + number=_decode_number(obj.get('number')), + string=string_value, + json=json_value, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + """Encodes this `ObjectData` per OD4.""" + _check_format(format) + result: dict = {} + if self.object_id is not None: + result['objectId'] = self.object_id + if self.encoding is not None: + result['encoding'] = self.encoding + if self.boolean is not None: + result['boolean'] = self.boolean + if self.bytes is not None: + result['bytes'] = _encode_bytes(self.bytes, format) + if self.number is not None: + result['number'] = _encode_number(self.number, format) + if self.string is not None: + result['string'] = self.string + if self.json is not None: + result['json'] = _encode_json(self.json) + return result + + +@dataclass +class ObjectsMapEntry: + """OME1: the value at one key of a map. + + `tombstoned_at` (RTLM3a1) is held locally by an `InternalLiveMap` and is never sent + or received. + """ + + tombstone: bool = False # OME2a + timeserial: str | None = None # OME2b + serial_timestamp: int | None = None # OME2d + data: ObjectData | None = None # OME2c + tombstoned_at: int | None = None # RTLM3a1 + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectsMapEntry: + _check_format(format) + data = obj.get('data') + return ObjectsMapEntry( + tombstone=bool(obj.get('tombstone', False)), + timeserial=obj.get('timeserial'), + serial_timestamp=obj.get('serialTimestamp'), + data=ObjectData.from_dict(data, format) if data is not None else None, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.tombstone: + result['tombstone'] = True + if self.timeserial is not None: + result['timeserial'] = self.timeserial + if self.serial_timestamp is not None: + result['serialTimestamp'] = self.serial_timestamp + if self.data is not None: + result['data'] = self.data.to_dict(format) + return result + + +def _decode_entries(entries: dict | None, format: str) -> dict[str, ObjectsMapEntry] | None: + if entries is None: + return None + return {key: ObjectsMapEntry.from_dict(entry or {}, format) for key, entry in entries.items()} + + +def _encode_entries(entries: dict[str, ObjectsMapEntry], format: str) -> dict: + return {key: entry.to_dict(format) for key, entry in entries.items()} + + +@dataclass +class ObjectsMap: + """OMP1: the state of a map object, as an `ObjectState` carries it.""" + + semantics: ObjectsMapSemantics | None = None # OMP3a + entries: dict[str, ObjectsMapEntry] | None = None # OMP3b + clear_timeserial: str | None = None # OMP3c + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectsMap: + _check_format(format) + semantics = obj.get('semantics') + return ObjectsMap( + semantics=ObjectsMapSemantics.from_wire(semantics) if semantics is not None else None, + entries=_decode_entries(obj.get('entries'), format), + clear_timeserial=obj.get('clearTimeserial'), + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.semantics is not None: + result['semantics'] = int(self.semantics) + if self.entries is not None: + result['entries'] = _encode_entries(self.entries, format) + if self.clear_timeserial is not None: + result['clearTimeserial'] = self.clear_timeserial + return result + + +@dataclass +class ObjectsCounter: + """OCN1: the state of a counter object, as an `ObjectState` carries it.""" + + count: float | None = None # OCN2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectsCounter: + _check_format(format) + return ObjectsCounter(count=_decode_number(obj.get('count'))) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.count is None else {'count': _encode_number(self.count, format)} + + +@dataclass +class MapCreate: + """MCR1: the payload of a MAP_CREATE operation.""" + + semantics: ObjectsMapSemantics = ObjectsMapSemantics.LWW # MCR2a + entries: dict[str, ObjectsMapEntry] = field(default_factory=dict) # MCR2b + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapCreate: + _check_format(format) + semantics = obj.get('semantics') + return MapCreate( + semantics=(ObjectsMapSemantics.from_wire(semantics) if semantics is not None + else ObjectsMapSemantics.LWW), + entries=_decode_entries(obj.get('entries'), format) or {}, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {'semantics': int(self.semantics), 'entries': _encode_entries(self.entries, format)} + + +@dataclass +class MapSet: + """MST1: the payload of a MAP_SET operation.""" + + key: str | None = None # MST2a + value: ObjectData | None = None # MST2b + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapSet: + _check_format(format) + value = obj.get('value') + return MapSet(key=obj.get('key'), value=ObjectData.from_dict(value, format) if value is not None else None) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.key is not None: + result['key'] = self.key + if self.value is not None: + result['value'] = self.value.to_dict(format) + return result + + +@dataclass +class MapRemove: + """MRM1: the payload of a MAP_REMOVE operation.""" + + key: str | None = None # MRM2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapRemove: + _check_format(format) + return MapRemove(key=obj.get('key')) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.key is None else {'key': self.key} + + +@dataclass +class CounterCreate: + """CCR1: the payload of a COUNTER_CREATE operation.""" + + count: float | None = None # CCR2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> CounterCreate: + _check_format(format) + return CounterCreate(count=_decode_number(obj.get('count'))) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.count is None else {'count': _encode_number(self.count, format)} + + +@dataclass +class CounterInc: + """CIN1: the payload of a COUNTER_INC operation.""" + + number: float | None = None # CIN2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> CounterInc: + _check_format(format) + return CounterInc(number=_decode_number(obj.get('number'))) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.number is None else {'number': _encode_number(self.number, format)} + + +@dataclass +class ObjectDelete: + """ODE1: the payload of an OBJECT_DELETE operation, which has no attributes (ODE2).""" + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectDelete: + _check_format(format) + return ObjectDelete() + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} + + +@dataclass +class MapClear: + """MCL1: the payload of a MAP_CLEAR operation, which has no attributes (MCL2).""" + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapClear: + _check_format(format) + return MapClear() + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} + + +@dataclass +class MapCreateWithObjectId: + """MCRO1: the payload of a MAP_CREATE operation sent with a client-generated object id. + + `derived_from` is the `MapCreate` the payload was built from, kept for local + application and size calculation (RTLMV4j5) and never sent. + """ + + initial_value: str | None = None # MCRO2a + nonce: str | None = None # MCRO2b + derived_from: MapCreate | None = None # RTLMV4j5 + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapCreateWithObjectId: + _check_format(format) + return MapCreateWithObjectId(initial_value=obj.get('initialValue'), nonce=obj.get('nonce')) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.initial_value is not None: + result['initialValue'] = self.initial_value + if self.nonce is not None: + result['nonce'] = self.nonce + return result + + +@dataclass +class CounterCreateWithObjectId: + """CCRO1: the payload of a COUNTER_CREATE operation sent with a client-generated object id. + + `derived_from` is the `CounterCreate` the payload was built from, kept for local + application and size calculation (RTLCV4g5) and never sent. + """ + + initial_value: str | None = None # CCRO2a + nonce: str | None = None # CCRO2b + derived_from: CounterCreate | None = None # RTLCV4g5 + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> CounterCreateWithObjectId: + _check_format(format) + return CounterCreateWithObjectId(initial_value=obj.get('initialValue'), nonce=obj.get('nonce')) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.initial_value is not None: + result['initialValue'] = self.initial_value + if self.nonce is not None: + result['nonce'] = self.nonce + return result + + +# The operation payloads, by the wire name each is carried under +_PAYLOADS = ( + ('map_create', 'mapCreate', MapCreate), + ('map_set', 'mapSet', MapSet), + ('map_remove', 'mapRemove', MapRemove), + ('counter_create', 'counterCreate', CounterCreate), + ('counter_inc', 'counterInc', CounterInc), + ('object_delete', 'objectDelete', ObjectDelete), + ('map_create_with_object_id', 'mapCreateWithObjectId', MapCreateWithObjectId), + ('counter_create_with_object_id', 'counterCreateWithObjectId', CounterCreateWithObjectId), + ('map_clear', 'mapClear', MapClear), +) + + +@dataclass +class ObjectOperation: + """OOP1: an operation to apply to an object on a channel.""" + + action: ObjectOperationAction # OOP3a + object_id: str # OOP3b + map_create: MapCreate | None = None # OOP3j + map_set: MapSet | None = None # OOP3k + map_remove: MapRemove | None = None # OOP3l + counter_create: CounterCreate | None = None # OOP3m + counter_inc: CounterInc | None = None # OOP3n + object_delete: ObjectDelete | None = None # OOP3o + map_create_with_object_id: MapCreateWithObjectId | None = None # OOP3p + counter_create_with_object_id: CounterCreateWithObjectId | None = None # OOP3q + map_clear: MapClear | None = None # OOP3r + + @property + def resolved_map_create(self) -> MapCreate | None: + """`map_create` if present, else the `MapCreate` `map_create_with_object_id` was derived from. + + This is the `mapCreate` RTLM23 merges and PAOOP3b exposes. + """ + if self.map_create is not None: + return self.map_create + if self.map_create_with_object_id is not None: + return self.map_create_with_object_id.derived_from + return None + + @property + def resolved_counter_create(self) -> CounterCreate | None: + """`counter_create` if present, else the `CounterCreate` `counter_create_with_object_id` was + derived from. + + This is the `counterCreate` RTLC16 merges and PAOOP3c exposes. + """ + if self.counter_create is not None: + return self.counter_create + if self.counter_create_with_object_id is not None: + return self.counter_create_with_object_id.derived_from + return None + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectOperation: + _check_format(format) + operation = ObjectOperation( + action=ObjectOperationAction.from_wire(obj.get('action')), + object_id=obj.get('objectId'), + ) + for attribute, wire_name, payload_type in _PAYLOADS: + payload = obj.get(wire_name) + if payload is not None: + setattr(operation, attribute, payload_type.from_dict(payload, format)) + return operation + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + if self.action is ObjectOperationAction.UNKNOWN: + # OOP2a: an unrecognised action is never sent + raise ValueError('An ObjectOperation with an unrecognised action cannot be encoded') + result: dict = {'action': int(self.action)} + if self.object_id is not None: + result['objectId'] = self.object_id + for attribute, wire_name, _ in _PAYLOADS: + payload = getattr(self, attribute) + if payload is not None: + result[wire_name] = payload.to_dict(format) + return result + + +@dataclass +class ObjectState: + """OST1: the instantaneous state of an object, as an OBJECT_SYNC carries it.""" + + object_id: str # OST2a + site_timeserials: dict[str, str] = field(default_factory=dict) # OST2b + tombstone: bool = False # OST2c + create_op: ObjectOperation | None = None # OST2d + map: ObjectsMap | None = None # OST2e + counter: ObjectsCounter | None = None # OST2f + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectState: + _check_format(format) + create_op = obj.get('createOp') + objects_map = obj.get('map') + counter = obj.get('counter') + return ObjectState( + object_id=obj.get('objectId'), + site_timeserials=dict(obj.get('siteTimeserials') or {}), + tombstone=bool(obj.get('tombstone', False)), + create_op=ObjectOperation.from_dict(create_op, format) if create_op is not None else None, + map=ObjectsMap.from_dict(objects_map, format) if objects_map is not None else None, + counter=ObjectsCounter.from_dict(counter, format) if counter is not None else None, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {'objectId': self.object_id, 'siteTimeserials': dict(self.site_timeserials)} + if self.tombstone: + result['tombstone'] = True + if self.create_op is not None: + result['createOp'] = self.create_op.to_dict(format) + if self.map is not None: + result['map'] = self.map.to_dict(format) + if self.counter is not None: + result['counter'] = self.counter.to_dict(format) + return result + + +@dataclass +class ObjectMessage: + """OM1: an object message sent or received over a channel. + + This is the internal type. The type delivered to listeners is the public + `ably.pubsub.objects.publicmessage.ObjectMessage` (PAOM1). + """ + + id: str | None = None # OM2a + client_id: str | None = None # OM2b + connection_id: str | None = None # OM2c + extras: dict | None = None # OM2d + timestamp: int | None = None # OM2e + operation: ObjectOperation | None = None # OM2f + object: ObjectState | None = None # OM2g + serial: str | None = None # OM2h + serial_timestamp: int | None = None # OM2j + site_code: str | None = None # OM2i + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectMessage: + """Decodes one entry of a ProtocolMessage's `state` array (OM5).""" + _check_format(format) + operation = obj.get('operation') + state = obj.get('object') + return ObjectMessage( + id=obj.get('id'), + client_id=obj.get('clientId'), + connection_id=obj.get('connectionId'), + extras=obj.get('extras'), + timestamp=obj.get('timestamp'), + operation=ObjectOperation.from_dict(operation, format) if operation is not None else None, + object=ObjectState.from_dict(state, format) if state is not None else None, + serial=obj.get('serial'), + serial_timestamp=obj.get('serialTimestamp'), + site_code=obj.get('siteCode'), + ) + + @staticmethod + def from_protocol_message(protocol_message: dict, format: str = WIRE_FORMAT_JSON) -> list[ObjectMessage]: + """Decodes the `state` array of an OBJECT or OBJECT_SYNC ProtocolMessage. + + A message with no `id`, `connectionId` or `timestamp` takes them from the + ProtocolMessage that carried it (OM2a, OM2c, OM2e). A message that fails to decode is + logged and left out, so that it costs neither the other messages nor the handling of + the ProtocolMessage itself, such as the sync cursor of an OBJECT_SYNC. + """ + channel = protocol_message.get('channel') + entries = protocol_message.get('state') or [] + if not isinstance(entries, list): + log.error(f'ObjectMessage.from_protocol_message(): skipping a state that is not an array; ' + f'channel={channel}, state={entries!r}') + return [] + + messages = [] + protocol_id = protocol_message.get('id') + for index, entry in enumerate(entries): + try: + message = ObjectMessage.from_dict(entry, format) + except Exception as e: + log.error(f'ObjectMessage.from_protocol_message(): skipping an object message that failed to ' + f'decode; index={index}, channel={channel}, error={e!r}') + continue + if message.id is None and protocol_id is not None: + message.id = f'{protocol_id}:{index}' + if message.connection_id is None: + message.connection_id = protocol_message.get('connectionId') + if message.timestamp is None: + message.timestamp = protocol_message.get('timestamp') + messages.append(message) + return messages + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + """Encodes this message for the `state` array of a ProtocolMessage (OM4).""" + _check_format(format) + result: dict = {} + for attribute, wire_name in (('id', 'id'), ('client_id', 'clientId'), ('connection_id', 'connectionId'), + ('extras', 'extras'), ('timestamp', 'timestamp'), ('serial', 'serial'), + ('serial_timestamp', 'serialTimestamp'), ('site_code', 'siteCode')): + value = getattr(self, attribute) + if value is not None: + result[wire_name] = value + if self.operation is not None: + result['operation'] = self.operation.to_dict(format) + if self.object is not None: + result['object'] = self.object.to_dict(format) + return result diff --git a/ably/pubsub/objects/objectspool.py b/ably/pubsub/objects/objectspool.py new file mode 100644 index 00000000..41919bc3 --- /dev/null +++ b/ably/pubsub/objects/objectspool.py @@ -0,0 +1,114 @@ +"""The objects held for a channel (RTO3).""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator, MutableMapping +from typing import TYPE_CHECKING + +from ably.pubsub.objects.defaults import ROOT_OBJECT_ID +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.liveobject import LiveObject +from ably.pubsub.objects.objectid import object_type_of +from ably.pubsub.util.clock import Clock + +if TYPE_CHECKING: + from ably.pubsub.objects.realtimeobject import RealtimeObject + +log = logging.getLogger(__name__) + + +class ObjectsPool(MutableMapping): + """RTO3: the live objects on a channel, by object id (RTO3a). + + A pool always holds an `InternalLiveMap` with id `root`, created with the pool + (RTO3b1). It is a mutable mapping, so `pool[object_id]`, `object_id in pool`, + `len(pool)` and `pool.keys()` work as they do on a dict. Setting `pool[object_id] = obj` + adopts `obj`: its `pool` becomes this pool. + + A pool is constructible on its own. `realtime_object` is None for a pool built without + one, in which case objects in it dispatch no path subscriptions and publish nothing. + """ + + def __init__(self, realtime_object: RealtimeObject | None = None, *, clock: Clock | None = None): + self.realtime_object: RealtimeObject | None = realtime_object + self.clock: Clock = clock if clock is not None else Clock() + self._objects: dict[str, LiveObject] = {} + self[ROOT_OBJECT_ID] = InternalLiveMap(ROOT_OBJECT_ID, pool=self) + + @property + def root(self) -> InternalLiveMap: + """The root map (RTO3b).""" + return self._objects[ROOT_OBJECT_ID] + + def __getitem__(self, object_id: str) -> LiveObject: + return self._objects[object_id] + + def __setitem__(self, object_id: str, live_object: LiveObject) -> None: + live_object.pool = self + self._objects[object_id] = live_object + + def __delitem__(self, object_id: str) -> None: + del self._objects[object_id] + + def __iter__(self) -> Iterator[str]: + return iter(self._objects) + + def __len__(self) -> int: + return len(self._objects) + + def __repr__(self) -> str: + return f'ObjectsPool({list(self._objects)!r})' + + def create_zero_value_object_if_not_exists(self, object_id: str) -> LiveObject | None: + """RTO6: the object with `object_id`, created empty from the type its id names if absent. + + Returns None, creating nothing, for an id that names neither a map nor a counter. + """ + existing = self._objects.get(object_id) + if existing is not None: + return existing # RTO6a + + object_type = object_type_of(object_id) if isinstance(object_id, str) else None # RTO6b1 + if object_type == 'map': + live_object: LiveObject = InternalLiveMap(object_id) # RTO6b2 + elif object_type == 'counter': + live_object = InternalLiveCounter(object_id) # RTO6b3 + else: + log.warning(f'ObjectsPool.create_zero_value_object_if_not_exists(): cannot create an object for ' + f'an id of unknown type; object_id={object_id!r}') + return None + + self[object_id] = live_object + return live_object + + def clear_all_data(self) -> None: + """RTO27a1: resets every object's data to empty, keeping the objects, emitting nothing.""" + for live_object in list(self._objects.values()): + live_object.clear_data() + + def rebuild_parent_references(self) -> None: + """RTO5c10: clears every object's `parent_references` and rebuilds them from the map entries.""" + for live_object in self._objects.values(): + live_object.parent_references = {} # RTO5c10a + + # RTO5c10b + for live_object in list(self._objects.values()): + if not isinstance(live_object, InternalLiveMap): + continue + for key, value in live_object.entries(): + if isinstance(value, LiveObject): + value.add_parent_reference(live_object, key) + + def collect_garbage(self, grace_period_ms: int, now_ms: int) -> None: + """RTO10c: releases tombstoned map entries (RTLM19) and objects tombstoned `grace_period_ms` + or more before `now_ms`, never the root (RTO10c1b1).""" + for object_id, live_object in list(self._objects.items()): + # RTO10c1b, RTO10c1b1 + if (object_id != ROOT_OBJECT_ID and live_object.is_tombstone and live_object.tombstoned_at is not None + and now_ms - live_object.tombstoned_at >= grace_period_ms): + del self._objects[object_id] + continue + if isinstance(live_object, InternalLiveMap): + live_object.gc_tombstoned_entries(grace_period_ms, now_ms) # RTO10c1a diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/objects/pathobject.py new file mode 100644 index 00000000..1c8d9928 --- /dev/null +++ b/ably/pubsub/objects/pathobject.py @@ -0,0 +1,319 @@ +"""Path-addressed views onto a channel's objects (RTPO*, RTTS3-RTTS6). + +A `PathObject` holds a path from the root map and resolves it each time a method reads +or writes through it, so it follows whatever object is at that path at the time. The +base class carries what does not depend on the type at the path, navigation included; +`as_live_map()`, `as_live_counter()` and `as_primitive()` return the typed views, without +checking what the path resolves to (RTTS5d). + +Reads never raise for a path that does not resolve or that resolves to another type: +they return None, or an empty list for the collection methods (RTPO3c1, RTTS5d1). Writes +raise `AblyException` 92005 for a path that does not resolve and 92007 for one that +resolves to the wrong type (RTPO3c2, RTTS5d2). Every read and `subscribe` checks the +access preconditions (RTO25), and every write the write preconditions (RTO26), before +resolving the path; navigation and the view helpers check nothing. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, overload + +from ably.pubsub.objects.instance import ( + Instance, + compact_value, + expected_value_type, + primitive_value, + value_type_of, +) +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.util.exceptions import AblyException + +if TYPE_CHECKING: + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext + from ably.pubsub.objects.enums import ValueType + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.subscription import Subscription + from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + + +@dataclass +class PathObjectSubscriptionEvent: + """RTPO19e: what a `PathObject.subscribe` listener receives.""" + + object: PathObject # RTPO19e1: the path where the change occurred + message: ObjectMessage | None = None # RTPO19e2 + + +class PathObject: + """RTPO1, RTTS3: a lazy reference to whatever is at a path from the root map.""" + + def __init__(self, realtime_object: RealtimeObject, root: InternalLiveMap, path: Sequence[str]): + self._realtime_object = realtime_object + self._root = root # RTPO2b + self._path: list[str] = list(path) # RTPO2a + + def __repr__(self) -> str: + return f'{type(self).__name__}({self.path()!r})' + + def path(self) -> str: + """RTPO4: the path as a dotted string, with dots in segments escaped as `\\.`.""" + # RTPO4a, RTPO4b; the empty path is the empty string (RTPO4c) + return '.'.join(segment.replace('.', '\\.') for segment in self._path) + + def type(self) -> ValueType | None: + """RTTS4b: the type of what the path resolves to, or None if it resolves to nothing.""" + self._realtime_object._check_access_preconditions() # RTTS4b1 + resolved = self._resolve() # RTTS4b2 + return value_type_of(resolved) if resolved is not None else None # RTTS4b3 + + def exists(self) -> bool: + """RTTS4a: whether the path resolves to anything.""" + self._realtime_object._check_access_preconditions() # RTTS4a1 + return self._resolve() is not None # RTTS4a2, RTTS4a3 + + def get(self, key: str) -> PathObject: + """RTPO5: the path one key further down. Navigational only (RTPO5d). + + Raises AblyException 40003 if `key` is not a string (RTPO5b). + """ + if not isinstance(key, str): + raise AblyException(f'Path key must be a string, not {type(key).__name__}', 400, 40003) + return PathObject(self._realtime_object, self._root, [*self._path, key]) # RTPO5c + + def at(self, path: str | Sequence[str]) -> PathObject: + """RTPO6: the path further down by a dotted string, `\\.` escaping a literal dot (RTPO6b), + or by a sequence of segments taken as they are. + + Raises AblyException 40003 for anything else. + """ + if isinstance(path, str): + segments = _parse_path(path) # RTPO6b + elif (isinstance(path, Sequence) and not isinstance(path, (bytes, bytearray)) + and all(isinstance(segment, str) for segment in path)): + segments = list(path) + else: + raise AblyException(f'Path must be a string or a sequence of strings, not {type(path).__name__}', + 400, 40003) + return PathObject(self._realtime_object, self._root, [*self._path, *segments]) # RTPO6c + + def instance(self) -> Instance | None: + """RTPO8: an `Instance` wrapping what the path resolves to, or None if it resolves to nothing. + + The instance is the subclass matching the resolved value (`LiveMapInstance`, + `LiveCounterInstance` or `PrimitiveInstance`). + """ + self._realtime_object._check_access_preconditions() # RTPO8a + resolved = self._resolve() # RTPO8b + if resolved is None: + return None # RTPO8e + return Instance._wrap(self._realtime_object, resolved) # RTPO8c, RTPO8f + + def compact(self) -> Any: + """RTPO13: a plain snapshot of what the path resolves to, or None.""" + self._realtime_object._check_access_preconditions() # RTPO13a + resolved = self._resolve() # RTPO13b + if resolved is None: + return None # RTPO13f + return compact_value(resolved) # RTPO13c, RTPO13d, RTPO13e + + def compact_json(self) -> Any: + """RTPO14: `compact`, with binary as base64 and cycles as `{'objectId': ...}`.""" + self._realtime_object._check_access_preconditions() # RTPO14a + resolved = self._resolve() + if resolved is None: + return None + return compact_value(resolved, for_json=True) # RTPO14b + + def subscribe(self, listener: Callable[[PathObjectSubscriptionEvent], None], *, + depth: int | None = None) -> Subscription: + """RTPO19: calls `listener` for changes at this path or, within `depth` levels, below it. + + Raises AblyException 40003 if `depth` is given and is not a positive integer + (RTPO19c1a). Has no effect on the channel (RTPO19g). + """ + self._realtime_object._check_access_preconditions() # RTPO19b + if depth is not None and (isinstance(depth, bool) or not isinstance(depth, int) or depth < 1): + # RTPO19c1a + raise AblyException(f'Subscription depth must be a positive integer, or None for any depth; ' + f'got {depth!r}', 400, 40003) + register = self._realtime_object._path_object_subscription_register + return register.subscribe(self._path, listener, depth) # RTPO19f + + def as_live_map(self) -> LiveMapPathObject: + """RTTS5a: this path, viewed as a map.""" + return LiveMapPathObject(self._realtime_object, self._root, self._path) + + def as_live_counter(self) -> LiveCounterPathObject: + """RTTS5b: this path, viewed as a counter.""" + return LiveCounterPathObject(self._realtime_object, self._root, self._path) + + def as_primitive(self) -> PrimitivePathObject: + """RTTS5c, RTTS6h: this path, viewed as a primitive.""" + return PrimitivePathObject(self._realtime_object, self._root, self._path) + + def _resolve(self) -> Any: + """RTPO3: what the path resolves to, or None if it does not resolve (RTPO3c).""" + current: Any = self._root # RTPO3b + for segment in self._path: # RTPO3a + if not isinstance(current, InternalLiveMap): + return None # RTPO3a1 + current = current.get(segment) # RTPO3a2, RTPO3a3 + if current is None: + return None + return current + + def _resolve_for_write(self, expected_type: type, description: str) -> Any: + """RTPO3c2, RTTS5d2: what the path resolves to, for a write that requires an `expected_type`. + + Raises AblyException 92005 if the path does not resolve and 92007 if it resolves to + something else. + """ + resolved = self._resolve() + if resolved is None: + raise AblyException(f'Could not resolve a value at path {self.path()!r}', 400, 92005) + if not isinstance(resolved, expected_type): + raise AblyException(f'Cannot write to the value at path {self.path()!r} as {description}: it is of ' + f'type {value_type_of(resolved).value}', 400, 92007) + return resolved + + +class LiveMapPathObject(PathObject): + """RTTS6a: a path expected to resolve to a map.""" + + def batch(self) -> Batch[LiveMapBatchContext]: + """RTPO20: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTPO20b) and raises AblyException 92007 if + the path does not resolve to a map (RTPO20c). + """ + # Imported here, as the batch module builds on the views + from ably.pubsub.objects.batch import Batch, LiveMapBatchContext + + return Batch(self._realtime_object, self._resolve, LiveMapBatchContext, f'path {self.path()!r}') + + def entries(self) -> list[tuple[str, PathObject]]: + """RTPO9: `(key, path)` for each key of the map, or `[]`.""" + self._realtime_object._check_access_preconditions() # RTPO9a + resolved = self._resolve() # RTPO9b + if not isinstance(resolved, InternalLiveMap): + return [] # RTPO9d + return [(key, self.get(key)) for key in resolved.keys()] # RTPO9c + + def keys(self) -> list[str]: + """RTPO10: the keys of the map, or `[]`.""" + self._realtime_object._check_access_preconditions() # RTPO10a + resolved = self._resolve() # RTPO10b + if not isinstance(resolved, InternalLiveMap): + return [] # RTPO10d + return resolved.keys() # RTPO10c + + def values(self) -> list[PathObject]: + """RTPO11: a path for each key of the map, or `[]`.""" + return [path_object for _, path_object in self.entries()] # RTPO11a-RTPO11d + + def size(self) -> int | None: + """RTPO12: the number of entries in the map, or None.""" + self._realtime_object._check_access_preconditions() # RTPO12a + resolved = self._resolve() # RTPO12b + if not isinstance(resolved, InternalLiveMap): + return None # RTPO12d + return resolved.size() # RTPO12c + + async def set(self, key: str, value: LiveMapValue) -> None: + """RTPO15: sets `key` in the map at this path.""" + self._realtime_object._check_write_preconditions() # RTPO15b + live_map = self._resolve_for_write(InternalLiveMap, 'a map') # RTPO15c, RTPO15e + await live_map.set(key, value) # RTPO15d + + async def remove(self, key: str) -> None: + """RTPO16: removes `key` from the map at this path.""" + self._realtime_object._check_write_preconditions() # RTPO16b + live_map = self._resolve_for_write(InternalLiveMap, 'a map') # RTPO16c, RTPO16e + await live_map.remove(key) # RTPO16d + + +class LiveCounterPathObject(PathObject): + """RTTS6b: a path expected to resolve to a counter.""" + + def batch(self) -> Batch[LiveCounterBatchContext]: + """RTPO20: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTPO20b) and raises AblyException 92007 if + the path does not resolve to a counter (RTPO20c). + """ + # Imported here, as the batch module builds on the views + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext + + return Batch(self._realtime_object, self._resolve, LiveCounterBatchContext, f'path {self.path()!r}') + + def value(self) -> float | None: + """RTTS6b: the counter's value, or None if the path does not resolve to a counter.""" + self._realtime_object._check_access_preconditions() # RTPO7a + resolved = self._resolve() # RTPO7b + if not isinstance(resolved, InternalLiveCounter): + return None # RTPO7f, RTTS6b + return resolved.value() # RTPO7c + + async def increment(self, amount: float = 1) -> None: + """RTPO17: increments the counter at this path by `amount`.""" + self._realtime_object._check_write_preconditions() # RTPO17b + counter = self._resolve_for_write(InternalLiveCounter, 'a counter') # RTPO17c, RTPO17e + await counter.increment(amount) # RTPO17d + + async def decrement(self, amount: float = 1) -> None: + """RTPO18: decrements the counter at this path by `amount`.""" + self._realtime_object._check_write_preconditions() # RTPO18b + counter = self._resolve_for_write(InternalLiveCounter, 'a counter') # RTPO18c, RTPO18e + await counter.decrement(amount) # RTPO18d + + +class PrimitivePathObject(PathObject): + """RTTS6c, RTTS6h: a path expected to resolve to a primitive.""" + + @overload + def value(self) -> Primitive | None: ... + + @overload + def value(self, expected: type[T]) -> T | None: ... + + def value(self, expected: type | None = None) -> Any: + """RTPO7, RTTS6c: the primitive at this path, or None. + + With `expected` (one of `str`, `float`, `bool`, `bytes`, `list`, `dict`) the value is + returned only if it is of that type, judged by its wire type, so a boolean is never + a number. Any other `expected` raises TypeError. + """ + value_type = expected_value_type(expected) + self._realtime_object._check_access_preconditions() # RTPO7a + return primitive_value(self._resolve(), value_type) # RTPO7b, RTPO7d-RTPO7f, RTTS6c + + +def _parse_path(path: str) -> list[str]: + """RTPO6b: the segments of a dotted path, in which `\\.` is a literal dot. + + A backslash before anything other than a dot is kept as it is. + """ + segments: list[str] = [] + segment: list[str] = [] + escaping = False + for char in path: + if escaping: + if char != '.': + segment.append('\\') + segment.append(char) + escaping = False + elif char == '\\': + escaping = True + elif char == '.': + segments.append(''.join(segment)) + segment = [] + else: + segment.append(char) + if escaping: + segment.append('\\') + segments.append(''.join(segment)) + return segments diff --git a/ably/pubsub/objects/pathobjectsubscriptionregister.py b/ably/pubsub/objects/pathobjectsubscriptionregister.py new file mode 100644 index 00000000..aa2c7ff7 --- /dev/null +++ b/ably/pubsub/objects/pathobjectsubscriptionregister.py @@ -0,0 +1,89 @@ +"""The register of path subscriptions a channel's `RealtimeObject` keeps (RTO24).""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Callable + +from ably.pubsub.objects.liveobject import LiveMapUpdate +from ably.pubsub.objects.pathobject import PathObject, PathObjectSubscriptionEvent +from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.objects.subscription import Registry, Subscription + +if TYPE_CHECKING: + from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate + from ably.pubsub.objects.realtimeobject import RealtimeObject + +log = logging.getLogger(__name__) + + +class _PathSubscription: + """One `PathObject.subscribe` registration.""" + + def __init__(self, path: list[str], listener: Callable[[PathObjectSubscriptionEvent], None], + depth: int | None): + self.path = path + self.listener = listener + self.depth = depth + + +class PathObjectSubscriptionRegister: + """RTO24: every subscription made through `PathObject.subscribe` on one channel (RTO24a).""" + + def __init__(self, realtime_object: RealtimeObject): + self.realtime_object = realtime_object + self._subscriptions: Registry[_PathSubscription] = Registry() + + def subscribe(self, path: list[str], listener: Callable[[PathObjectSubscriptionEvent], None], + depth: int | None = None) -> Subscription: + """RTPO19f: registers `listener` for changes covered by `path` and `depth` (RTO24c1).""" + return Subscription(self._subscriptions.register(_PathSubscription(list(path), listener, depth))) + + def dispatch(self, live_object: LiveObject, update: LiveObjectUpdate) -> None: + """RTO24b: calls each subscription once for each path to `live_object` that it covers. + + A listener that raises is logged and does not stop the others (RTO24b2c). A + subscription made during a dispatch is not called by it, for any path, and one removed + during a dispatch is not called again. + """ + if not self._subscriptions: + return + registrations = self._subscriptions.snapshot() + + realtime_object = self.realtime_object + root = realtime_object._objects_pool.root + object_message = update.object_message + # RTO24b2b2: an update from a sync carries the object's state, not an operation + has_operation = object_message is not None and object_message.operation is not None + updated_keys = list(update.update) if isinstance(update, LiveMapUpdate) else [] + + for path_to_this in live_object.get_full_paths(): # RTO24b1, RTO24b2 + # RTO24b2a1, RTO24b2a2 + candidate_paths = [path_to_this, *([*path_to_this, key] for key in updated_keys)] + for registration in registrations: + if not registration.active: + continue + subscription = registration.item + # RTO24b2b + event_path = next((candidate for candidate in candidate_paths + if self.covers(subscription.path, subscription.depth, candidate)), None) + if event_path is None: + continue + try: + message = None + if has_operation: + message = ObjectMessage._from_internal(object_message, realtime_object._channel_name) + event = PathObjectSubscriptionEvent(PathObject(realtime_object, root, event_path), message) + subscription.listener(event) + except Exception: + # RTO24b2c + log.exception(f'PathObjectSubscriptionRegister.dispatch(): a path subscription listener ' + f'raised; path={event_path}, channel={realtime_object._channel_name}') + + @staticmethod + def covers(subscription_path: list[str], depth: int | None, event_path: list[str]) -> bool: + """RTO24c1: whether a subscription at `subscription_path` with `depth` covers `event_path`.""" + prefix_length = len(subscription_path) + if len(event_path) < prefix_length or list(event_path[:prefix_length]) != list(subscription_path): + return False + return depth is None or len(event_path) - prefix_length + 1 <= depth diff --git a/ably/pubsub/objects/publicmessage.py b/ably/pubsub/objects/publicmessage.py new file mode 100644 index 00000000..8f603c0e --- /dev/null +++ b/ably/pubsub/objects/publicmessage.py @@ -0,0 +1,102 @@ +"""The object message delivered to subscription listeners (PAOM*, PAOOP*). + +The specification calls these `PublicAPI::ObjectMessage` and `PublicAPI::ObjectOperation`. +They are exposed to users as `ObjectMessage` and `ObjectOperation`, sharing their names +with the internal wire types in `ably.pubsub.objects.objectmessage`; code that needs both +imports one of the pairs under a module-qualified name. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from ably.pubsub.objects.objectmessage import ( + CounterCreate, + CounterInc, + MapClear, + MapCreate, + MapRemove, + MapSet, + ObjectDelete, + ObjectOperationAction, +) + +if TYPE_CHECKING: + from ably.pubsub.objects import objectmessage + + +@dataclass +class ObjectOperation: + """PAOOP1: the operation that caused an object change. + + Unlike the wire operation it never carries the `*CreateWithObjectId` payloads: a + create sent with a client-generated object id is shown as the `MapCreate` or + `CounterCreate` it was derived from. + """ + + action: ObjectOperationAction # PAOOP2a + object_id: str # PAOOP2b + map_create: MapCreate | None = None # PAOOP2c + map_set: MapSet | None = None # PAOOP2d + map_remove: MapRemove | None = None # PAOOP2e + counter_create: CounterCreate | None = None # PAOOP2f + counter_inc: CounterInc | None = None # PAOOP2g + object_delete: ObjectDelete | None = None # PAOOP2h + map_clear: MapClear | None = None # PAOOP2i + + @staticmethod + def _from_internal(operation: objectmessage.ObjectOperation) -> ObjectOperation: + """PAOOP3: the public form of an internal `ObjectOperation`. + + The payloads are copies, so a listener that changes them leaves the objects the + operation was applied to unchanged. + """ + return ObjectOperation( + action=operation.action, # PAOOP3a + object_id=operation.object_id, + map_create=copy.deepcopy(operation.resolved_map_create), # PAOOP3b + map_set=copy.deepcopy(operation.map_set), + map_remove=copy.deepcopy(operation.map_remove), + counter_create=copy.deepcopy(operation.resolved_counter_create), # PAOOP3c + counter_inc=copy.deepcopy(operation.counter_inc), + object_delete=copy.deepcopy(operation.object_delete), + map_clear=copy.deepcopy(operation.map_clear), + ) + + +@dataclass +class ObjectMessage: + """PAOM1: the inbound object message that caused an object change.""" + + channel: str # PAOM2e + operation: ObjectOperation # PAOM2f + id: str | None = None # PAOM2a + client_id: str | None = None # PAOM2b + connection_id: str | None = None # PAOM2c + timestamp: int | None = None # PAOM2d + serial: str | None = None # PAOM2g + serial_timestamp: int | None = None # PAOM2h + site_code: str | None = None # PAOM2i + extras: dict | None = None # PAOM2j + + @staticmethod + def _from_internal(message: objectmessage.ObjectMessage, channel_name: str) -> ObjectMessage: + """PAOM3: the public form of an internal `ObjectMessage` received on `channel_name`. + + The caller ensures `message.operation` is populated (PAOM3a1). + """ + return ObjectMessage( + channel=channel_name, # PAOM3b + operation=ObjectOperation._from_internal(message.operation), # PAOM3d + # PAOM3c + id=message.id, + client_id=message.client_id, + connection_id=message.connection_id, + timestamp=message.timestamp, + serial=message.serial, + serial_timestamp=message.serial_timestamp, + site_code=message.site_code, + extras=copy.deepcopy(message.extras), + ) diff --git a/ably/pubsub/objects/realtimeobject.py b/ably/pubsub/objects/realtimeobject.py new file mode 100644 index 00000000..c2276b11 --- /dev/null +++ b/ably/pubsub/objects/realtimeobject.py @@ -0,0 +1,645 @@ +"""The LiveObjects entry point for a channel (RTO*), reached as `channel.object`.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import logging +from typing import TYPE_CHECKING, Any, Callable + +from ably.pubsub.objects.defaults import GC_GRACE_PERIOD_MS, GC_INTERVAL_MS, ROOT_OBJECT_ID +from ably.pubsub.objects.enums import ObjectsEvent, ObjectsOperationSource, ObjectsSyncState +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_JSON, + WIRE_FORMAT_MSGPACK, + ObjectOperationAction, + ObjectsMapSemantics, +) +from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.pathobject import LiveMapPathObject +from ably.pubsub.objects.pathobjectsubscriptionregister import PathObjectSubscriptionRegister +from ably.pubsub.objects.subscription import Registry, StatusSubscription +from ably.pubsub.objects.syncobjectspool import SyncObjectsPool +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.clock import Clock, select_clock +from ably.pubsub.util.exceptions import AblyException + +if TYPE_CHECKING: + from ably.pubsub.objects.objectmessage import ObjectMessage + from ably.pubsub.realtime.channel import RealtimeChannel + from ably.pubsub.types.connectiondetails import ConnectionDetails + from ably.pubsub.types.operations import PublishResult + from ably.pubsub.util.helper import Timer + +log = logging.getLogger(__name__) + +# RTO9a2a: the operation actions `_apply_object_messages` applies +_SUPPORTED_ACTIONS = frozenset(( + ObjectOperationAction.MAP_CREATE, + ObjectOperationAction.MAP_SET, + ObjectOperationAction.MAP_REMOVE, + ObjectOperationAction.COUNTER_CREATE, + ObjectOperationAction.COUNTER_INC, + ObjectOperationAction.OBJECT_DELETE, + ObjectOperationAction.MAP_CLEAR, +)) + +# RTO17b: the event each sync state emits when it is entered +_SYNC_STATE_EVENTS = { + ObjectsSyncState.SYNCING: ObjectsEvent.SYNCING, + ObjectsSyncState.SYNCED: ObjectsEvent.SYNCED, +} + +# RTO23c1, RTO20e1: the channel states that fail a wait for SYNCED +_SYNC_WAIT_FAILURE_STATES = frozenset((ChannelState.DETACHED, ChannelState.SUSPENDED, ChannelState.FAILED)) + +# RTO27a: the channel states after which the objects' data can no longer be known. The channel +# leaves them only by attaching again, so a wait for SYNCED begun in one of them fails at once. +_DATA_CLEARING_STATES = frozenset((ChannelState.DETACHED, ChannelState.FAILED)) + + +class RealtimeObject: + """RTO*: the objects on one channel, and the entry point to them. + + Each `RealtimeChannel` builds one, reachable as `channel.object`. One can also be built + with no channel, so that its sync and apply behaviour can be driven directly; such a + `RealtimeObject` publishes nothing, checks no channel preconditions and schedules no + timers. `pool` adopts an existing `ObjectsPool`; `clock` defaults to the channel's + client's clock. + + Construction is cheap and needs no running event loop: the GC timer (RTO10) is + scheduled on the first ATTACHED, not here. + """ + + def __init__(self, channel: RealtimeChannel | None = None, *, pool: ObjectsPool | None = None, + clock: Clock | None = None): + self._channel = channel + if clock is None: + clock = select_clock(channel.ably.options) if channel is not None else Clock() + self._clock: Clock = clock + if pool is None: + pool = ObjectsPool(self, clock=clock) + else: + pool.realtime_object = self + self._objects_pool: ObjectsPool = pool # RTO3 + self._sync_objects_pool = SyncObjectsPool() # RTO5f + self._sync_state = ObjectsSyncState.INITIALIZED # RTO17a1 + self._buffered_object_operations: list[ObjectMessage] = [] # RTO7a, RTO7a1 + self._applied_on_ack_serials: set[str] = set() # RTO7b, RTO7b1 + self._current_sync_id: str | None = None # RTO5a1 + self._path_object_subscription_register = PathObjectSubscriptionRegister(self) # RTO24a + self._gc_interval_ms: int = GC_INTERVAL_MS # RTO10a, read each time the GC timer is scheduled + self._gc_timer: Timer | None = None + self._gc_grace_period_ms: int = GC_GRACE_PERIOD_MS # RTO10b3, until a CONNECTED gives one + self._server_time_offset_ms: float | None = None # RTO16a + # RTO23c, RTO20e: each pending wait for SYNCED, with the description its 92008 error gives + self._sync_waiters: dict[asyncio.Future[None], str] = {} + # RTO18: the callbacks `on` registered, by event + self._sync_state_listeners: dict[ObjectsEvent, Registry[Callable[[], None]]] = { + event: Registry() for event in ObjectsEvent + } + # Whether the channel has been released, after which it receives nothing more + self._released = False + if channel is not None: + self._on_connected(channel.ably.connection.connection_details) # RTO10b1 + + # Public API + + async def get(self) -> LiveMapPathObject: + """RTO23: the root of the channel's objects, once they have synced. + + Requires the OBJECT_SUBSCRIBE mode (RTO23a, 40024), attaches the channel if it is not + attached (RTO23e, RTL33), and waits for the sync state to reach SYNCED (RTO23c). + Raises AblyException 92008 if the channel is or enters DETACHED or FAILED, or enters + SUSPENDED, while waiting (RTO23c1), 90001 if the channel is FAILED (RTL33c), and the + attach's error if attaching fails (RTO23e). + """ + self._throw_if_missing_channel_mode(ChannelMode.OBJECT_SUBSCRIBE) # RTO23a + if self._channel is not None: + await self._channel._ensure_active() # RTO23e + + if self._sync_state != ObjectsSyncState.SYNCED: + await self._wait_for_synced('The object could not be retrieved') # RTO23c, RTO23c1 + + return LiveMapPathObject(self, self._objects_pool.root, []) # RTO23d + + def on(self, event: ObjectsEvent, callback: Callable[[], None]) -> StatusSubscription: + """RTO18: calls `callback`, with no arguments, whenever the sync state reaches `event`. + + Registering one callback twice calls it twice (RTO18d). Raises AblyException 40003 for + an `event` that is not an `ObjectsEvent`. + """ + return StatusSubscription(self._sync_state_listeners[_objects_event(event)].register(callback)) # RTO18c + + def off(self, event: ObjectsEvent, callback: Callable[[], None]) -> None: + """RTO19: deregisters `callback` from `event`, however many times it was registered. + + Raises AblyException 40003 for an `event` that is not an `ObjectsEvent`. + """ + self._sync_state_listeners[_objects_event(event)].deregister(callback) + + # Internal API: inbound protocol messages and channel state + + @property + def _channel_name(self) -> str | None: + """The channel's name, which PAOM3b puts on every public message.""" + return self._channel.name if self._channel is not None else None + + @property + def _wire_format(self) -> str: + """The wire format the channel's connection decodes with.""" + if self._channel is not None and self._channel.ably.options.use_binary_protocol: + return WIRE_FORMAT_MSGPACK + return WIRE_FORMAT_JSON + + def _on_attached(self, has_objects: bool) -> None: + """RTO4: handles an ATTACHED ProtocolMessage the channel received while ATTACHING or + ATTACHED; the channel ignores one received in any other state. + + `has_objects` is the HAS_OBJECTS flag. Starts the GC timer if it is not running + and there is a channel (RTO10). + """ + log.debug(f'RealtimeObject._on_attached(): channel={self._channel_name}, has_objects={has_objects}') + self._set_sync_state(ObjectsSyncState.SYNCING) # RTO4c + self._buffered_object_operations = [] # RTO4d + # The objects an ATTACHED announces arrive in a sync sequence of their own (RTO4a), so a + # sequence still in flight from before it is abandoned + self._start_sync_sequence(None) + + if not has_objects: + # RTO4b + pool = self._objects_pool + for object_id in list(pool): + if object_id != ROOT_OBJECT_ID: + del pool[object_id] # RTO4b1 + + # RTO4b2: the root is cleared in place, never replaced + root = pool.root + previous_data = root.data + root.clear_data() + # RTO4b2a: the removed keys, with no object message; a no-op if the root was empty + root.notify_updated(InternalLiveMap.diff(previous_data, root.data)) + + self._sync_objects_pool.clear() # RTO4b3 + self._complete_sync() # RTO4b4 + + self._schedule_gc_timer() # RTO10a + + def _on_connected(self, connection_details: ConnectionDetails | None) -> None: + """RTO10b2: takes the GC grace period from the ConnectionDetails of a CONNECTED. + + With no details, as before the first CONNECTED, the grace period is left as it is. + """ + if connection_details is None: + return + grace_period_ms = connection_details.objects_gc_grace_period + # RTO10b1, RTO10b3 + self._gc_grace_period_ms = grace_period_ms if grace_period_ms is not None else GC_GRACE_PERIOD_MS + + def _release(self) -> None: + """Stops this RealtimeObject when its channel is released (RTS4). + + A released channel receives nothing more, so the GC timer stops for good, and every + wait for SYNCED, pending or later, fails with 92008. + """ + self._released = True + self._cancel_gc_timer() + self._fail_sync_waiters('the channel being released', self._channel_error_reason) + + def _handle_object_sync_messages(self, object_messages: list[ObjectMessage], + sync_channel_serial: str | None) -> None: + """RTO5: handles the decoded `state` of an OBJECT_SYNC ProtocolMessage.""" + sync_id, sync_cursor = self._parse_sync_channel_serial(sync_channel_serial) # RTO5a + self._set_sync_state(ObjectsSyncState.SYNCING) # RTO5e + + # RTO5a2: a new sequence id starts a new sequence. A message with no sequence id holds a + # whole sync of its own (RTO5a5), so it too discards anything accumulated before it. + if sync_id is None or sync_id != self._current_sync_id: + self._start_sync_sequence(sync_id) + + self._sync_objects_pool.apply_object_sync_messages(object_messages) # RTO5f + + # RTO5a4, RTO5a5: the sequence is complete once its cursor is empty + if not sync_cursor: + self._complete_sync() + + def _handle_object_messages(self, object_messages: list[ObjectMessage]) -> None: + """RTO8: handles the decoded `state` of an OBJECT ProtocolMessage, buffering while not SYNCED.""" + if self._sync_state != ObjectsSyncState.SYNCED: + self._buffered_object_operations.extend(object_messages) # RTO8a + return + self._apply_object_messages(object_messages, ObjectsOperationSource.CHANNEL) # RTO8b + + def _apply_object_messages(self, object_messages: list[ObjectMessage], + source: ObjectsOperationSource) -> None: + """RTO9: applies operations to the pool, recording LOCAL serials in `_applied_on_ack_serials`. + + An operation that raises as it is applied is logged and skipped, and the rest are + applied. + """ + for object_message in object_messages: + try: + self._apply_object_message(object_message, source) + except Exception: + log.exception(f'RealtimeObject._apply_object_messages(): skipping an object message that ' + f'could not be applied; message id={object_message.id}, ' + f'channel={self._channel_name}') + + def _apply_object_message(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> None: + """RTO9a: applies one operation to the pool.""" + operation = object_message.operation + if operation is None: + # RTO9a1 + log.warning(f'RealtimeObject._apply_object_message(): skipping an object message with no ' + f'operation; message id={object_message.id}, channel={self._channel_name}') + return + + serial = object_message.serial + if serial is not None and serial in self._applied_on_ack_serials: + # RTO9a3 + log.debug(f'RealtimeObject._apply_object_message(): skipping an operation already applied ' + f'on ACK; serial={serial}, channel={self._channel_name}') + self._applied_on_ack_serials.discard(serial) + return + + if operation.action not in _SUPPORTED_ACTIONS: + # RTO9a2b + log.warning(f'RealtimeObject._apply_object_message(): skipping an object message with an ' + f'unsupported action; action={operation.action!r}, ' + f'message id={object_message.id}, channel={self._channel_name}') + return + + # RTO9a2a1, RTO9a2a2 + live_object = self._objects_pool.create_zero_value_object_if_not_exists(operation.object_id) + if live_object is None: + return + applied = live_object.apply_operation(object_message, source) # RTO9a2a3 + if source == ObjectsOperationSource.LOCAL and applied: + self._applied_on_ack_serials.add(serial) # RTO9a2a4 + + def _act_on_channel_state(self, state: ChannelState, reason: AblyException | None = None) -> None: + """RTO27: handles the channel entering `state`, for every state but ATTACHED. + + DETACHED and FAILED clear every object's data (RTO27a); every other state keeps it + (RTO27b). DETACHED, SUSPENDED and FAILED fail a `get` or a `_publish_and_apply` + waiting for SYNCED with 92008 (RTO23c1, RTO20e1), with `reason`, else the channel's + `error_reason`, as the cause. DETACHED and FAILED also stop the GC timer, which the + next ATTACHED starts again. + """ + if state in _SYNC_WAIT_FAILURE_STATES: + cause = reason if reason is not None else self._channel_error_reason + self._fail_sync_waiters(_entering(state), cause) + + if state in _DATA_CLEARING_STATES: + self._objects_pool.clear_all_data() # RTO27a1 + self._sync_objects_pool.clear() # RTO27a2 + self._cancel_gc_timer() + + def _set_sync_state(self, state: ObjectsSyncState) -> None: + """RTO17: moves to `state`, emitting the matching `ObjectsEvent` to `on` listeners (RTO17b).""" + if state == self._sync_state: + return + self._sync_state = state + + if state == ObjectsSyncState.SYNCED: + self._resolve_sync_waiters() # RTO23c, RTO20e + + event = _SYNC_STATE_EVENTS.get(state) + if event is None: + return + for registration in self._sync_state_listeners[event].snapshot(): + if not registration.active: + continue + try: + registration.item() # RTO18e + except Exception: + log.exception(f'RealtimeObject._set_sync_state(): a {event.value} listener raised; ' + f'channel={self._channel_name}') + + # Internal API: the sync sequence + + @staticmethod + def _parse_sync_channel_serial(sync_channel_serial: str | None) -> tuple[str | None, str | None]: + """RTO5a1: the `(sequence id, cursor)` of an OBJECT_SYNC's `channelSerial`. + + Both are None when the serial is absent (RTO5a5) or has no `:` separator (RTO5a6). + """ + if sync_channel_serial is None: + return None, None + sync_id, separator, sync_cursor = sync_channel_serial.partition(':') + if not separator: + log.warning(f'RealtimeObject._parse_sync_channel_serial(): handling an OBJECT_SYNC whose ' + f'channelSerial has no ":" as having none; channelSerial={sync_channel_serial!r}') + return None, None + return sync_id, sync_cursor + + def _start_sync_sequence(self, sync_id: str | None) -> None: + """RTO5a2: discards whatever an earlier sync sequence accumulated and starts `sync_id`.""" + self._sync_objects_pool.clear() # RTO5a2a + self._current_sync_id = sync_id + + def _complete_sync(self) -> None: + """RTO5c: applies the objects the sync sequence delivered, then the operations buffered meanwhile. + + An object whose state raises as it is applied is logged and left as the error left it, + so that the sync still completes. + """ + pool = self._objects_pool + received_object_ids: set[str] = set() + updates: list[tuple[LiveObject, LiveObjectUpdate]] = [] + + # RTO5c1 + for object_id, object_message in self._sync_objects_pool.entries.items(): + received_object_ids.add(object_id) + try: + replaced = self._replace_object_state(object_id, object_message) + except Exception: + log.exception(f'RealtimeObject._complete_sync(): skipping an object state that could not be ' + f'applied; object_id={object_id}, message id={object_message.id}, ' + f'channel={self._channel_name}') + continue + if replaced is not None: + updates.append(replaced) + + # RTO5c2, RTO5c2a + for object_id in list(pool): + if object_id not in received_object_ids and object_id != ROOT_OBJECT_ID: + del pool[object_id] + + pool.rebuild_parent_references() # RTO5c10 + + # RTO5c7: emitted once every object is in place, so that listeners see the synced state + for live_object, update in updates: + live_object.notify_updated(update) + + self._apply_object_messages(self._buffered_object_operations, ObjectsOperationSource.CHANNEL) # RTO5c6 + + self._current_sync_id = None # RTO5c3 + self._sync_objects_pool.clear() # RTO5c4 + self._buffered_object_operations = [] # RTO5c5 + self._applied_on_ack_serials.clear() # RTO5c9 + self._set_sync_state(ObjectsSyncState.SYNCED) # RTO5c8 + + def _replace_object_state(self, object_id: str, + object_message: ObjectMessage) -> tuple[LiveObject, LiveObjectUpdate] | None: + """RTO5c1: applies one synced object state to the pool. + + Returns the object and its update when it replaced an existing object's data, whose + update RTO5c7 emits, and None when it created the object. + """ + existing = self._objects_pool.get(object_id) + if existing is not None: + return existing, existing.replace_data(object_message) # RTO5c1a1, RTO5c1a2 + + # RTO5c1b1 + object_state = object_message.object + if object_state.counter is not None: + live_object: LiveObject = InternalLiveCounter(object_id) # RTO5c1b1a + else: + semantics = object_state.map.semantics + live_object = InternalLiveMap( + object_id, semantics if semantics is not None else ObjectsMapSemantics.LWW) # RTO5c1b1b + self._objects_pool[object_id] = live_object + live_object.replace_data(object_message) + return None + + # Internal API: waiting for SYNCED + + async def _wait_for_synced(self, failure_description: str) -> None: + """RTO23c, RTO20e: waits for the sync state to reach SYNCED. + + Raises AblyException 92008, whose message starts with `failure_description`, if the + channel enters DETACHED, SUSPENDED or FAILED first (RTO23c1, RTO20e1). A channel + already DETACHED or FAILED, or released, can never sync without attaching again, so + the wait then fails at once. + """ + if self._released: + raise _sync_wait_failure(failure_description, 'the channel being released', + self._channel_error_reason) + channel = self._channel + if channel is not None and channel.state in _DATA_CLEARING_STATES: + # The channel left ATTACHED before the wait began, as when an ATTACHED or an ACK and a + # channel ERROR are read together + raise _sync_wait_failure(failure_description, _entering(channel.state), channel.error_reason) + + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._sync_waiters[future] = failure_description + try: + await future + finally: + self._sync_waiters.pop(future, None) + + def _resolve_sync_waiters(self) -> None: + waiters, self._sync_waiters = self._sync_waiters, {} + for future in waiters: + if not future.done(): + future.set_result(None) + + def _fail_sync_waiters(self, reason: str, cause: AblyException | None) -> None: + """RTO23c1, RTO20e1: fails every wait for SYNCED with 92008, `reason` saying why.""" + waiters, self._sync_waiters = self._sync_waiters, {} + for future, failure_description in waiters.items(): + if not future.done(): + future.set_exception(_sync_wait_failure(failure_description, reason, cause)) + + @property + def _channel_error_reason(self) -> AblyException | None: + return self._channel.error_reason if self._channel is not None else None + + # Internal API: publishing + + async def _publish(self, object_messages: list[ObjectMessage]) -> PublishResult: + """RTO15: sends `object_messages` in one OBJECT ProtocolMessage and returns the ACK's result. + + Fails as `RealtimeChannel.publish` does when the connection or channel state does not + allow publishing (RTO15b), and with the NACK's error if the publish is rejected (RTO15g). + """ + channel = self._require_channel() + channel._throw_if_unpublishable_state() # RTO15b + + protocol_message = { + 'action': ProtocolMessageAction.OBJECT, # RTO15e1 + 'channel': channel.name, # RTO15e2 + # RTO15c, RTO15e3 + 'state': [object_message.to_dict(self._wire_format) for object_message in object_messages], + } + + log.debug(f'RealtimeObject._publish(): sending {len(object_messages)} object message(s); ' + f'channel={channel.name}') + + # RTO15f, RTO15g, RTO15h + return await channel.ably.connection.connection_manager.send_protocol_message(protocol_message) + + async def _publish_and_apply(self, object_messages: list[ObjectMessage]) -> None: + """RTO20: publishes `object_messages`, then applies them locally with the serials the ACK + assigned, as LOCAL operations, once synced. + + Raises the publish's error (RTO20b), and AblyException 92008 if the channel is or + enters DETACHED or FAILED, or enters SUSPENDED, while waiting for SYNCED (RTO20e1). An + operation that cannot be applied locally is left for its echo to apply (RTO20c, + RTO20d1), and nothing is applied once the channel is DETACHED or FAILED, as its + objects' data has been cleared (RTO27a). + """ + publish_result = await self._publish(object_messages) # RTO20b + + # RTO20c1 + connection_details = self._channel.ably.connection.connection_details + site_code = connection_details.site_code if connection_details is not None else None + if not site_code: + log.error(f'RealtimeObject._publish_and_apply(): the operations will not be applied locally, ' + f'as the connection details carry no siteCode; channel={self._channel_name}') + return + + # RTO20c2 + serials = publish_result.serials if publish_result is not None else None + if serials is None or len(serials) != len(object_messages): + log.error(f'RealtimeObject._publish_and_apply(): the operations will not be applied locally, as the ' + f'ACK carries {len(serials or [])} serial(s) for {len(object_messages)} object message(s); ' + f'channel={self._channel_name}') + return + + # RTO20d + synthetic_messages = [] + for index, (object_message, serial) in enumerate(zip(object_messages, serials)): + if serial is None: + # RTO20d1 + log.debug(f'RealtimeObject._publish_and_apply(): the operation at index {index} will not be ' + f'applied locally, as the ACK assigned it no serial; channel={self._channel_name}') + continue + # RTO20d2, RTO20d3 + synthetic_messages.append(dataclasses.replace(object_message, serial=serial, site_code=site_code)) + + # RTO20d4 + if not synthetic_messages: + return + + if self._sync_state != ObjectsSyncState.SYNCED: + # RTO20e, RTO20e1 + await self._wait_for_synced('The operation could not be applied locally') + + if self._channel.state in _DATA_CLEARING_STATES: + # The channel left ATTACHED after the ACK, and before this resumed + log.debug(f'RealtimeObject._publish_and_apply(): the operations will not be applied locally, as ' + f'the channel is {self._channel.state.value}; channel={self._channel_name}') + return + + self._apply_object_messages(synthetic_messages, ObjectsOperationSource.LOCAL) # RTO20f + + async def _get_server_time_ms(self) -> int: + """RTO16: the current server time, from a persisted offset where one is known (RTO16a). + + The offset is the one this RealtimeObject persisted, else the one the client's `Auth` + persisted for token requests (RSA10k); with neither, the server is asked for its time. + A `RealtimeObject` with no channel reads its own clock. + """ + now_ms = self._clock.now_ms() + if self._channel is None: + return now_ms + + offset_ms = self._server_time_offset_ms + if offset_ms is None: + offset_ms = self._channel.ably.auth.time_offset + if offset_ms is None: + server_time_ms = await self._channel.ably.time() # RTO16 + self._server_time_offset_ms = server_time_ms - self._clock.now_ms() + return int(server_time_ms) + return int(now_ms + offset_ms) # RTO16a + + def _require_channel(self) -> RealtimeChannel: + if self._channel is None: + raise AblyException('Unable to publish object messages from a RealtimeObject with no channel', + 400, 40000) + return self._channel + + # Internal API: preconditions + + def _throw_if_missing_channel_mode(self, mode: ChannelMode) -> None: + """RTO2: raises AblyException 40024 if `mode` is neither granted nor, before ATTACHED, requested. + + The modes the server granted are checked while the channel is ATTACHED and its ATTACHED + carried mode flags (RTO2a); otherwise, the modes requested in the channel options are + (RTO2b). + """ + if self._channel is None: + return + + modes = self._channel.modes if self._channel.state == ChannelState.ATTACHED else None # RTO2a + if not modes: + modes = (self._channel.options or {}).get('modes') or [] # RTO2b + if mode not in modes: + # RTO2a2, RTO2b2 + raise AblyException(f'"{mode.name}" channel mode must be set for this operation', 400, 40024) + + def _check_access_preconditions(self) -> None: + """RTO25: OBJECT_SUBSCRIBE (40024), and 90001 if the channel is DETACHED or FAILED.""" + self._throw_if_missing_channel_mode(ChannelMode.OBJECT_SUBSCRIBE) # RTO25a + self._throw_if_in_channel_state(ChannelState.DETACHED, ChannelState.FAILED) # RTO25b + + def _check_write_preconditions(self) -> None: + """RTO26: OBJECT_PUBLISH (40024), 90001 if the channel is DETACHED, FAILED or SUSPENDED, + and 40000 if `echo_messages` is disabled.""" + self._throw_if_missing_channel_mode(ChannelMode.OBJECT_PUBLISH) # RTO26a + # RTO26b + self._throw_if_in_channel_state(ChannelState.DETACHED, ChannelState.FAILED, ChannelState.SUSPENDED) + if self._channel is not None and not self._channel.ably.options.echo_messages: + # RTO26c + raise AblyException('"echo_messages" client option must be enabled for this operation', 400, 40000) + + def _throw_if_in_channel_state(self, *states: ChannelState) -> None: + if self._channel is not None and self._channel.state in states: + raise AblyException(f'Channel operation failed as channel state is {self._channel.state.value}', + 400, 90001) + + # Internal API: garbage collection + + def _schedule_gc_timer(self) -> None: + """RTO10a: schedules `_on_gc_interval` on the clock, `_gc_interval_ms` from now, if there is a + channel, it has not been released, and no sweep is scheduled already. + + A `RealtimeObject` with no channel schedules nothing. + """ + if self._channel is None or self._released or self._gc_timer is not None: + return + self._gc_timer = self._clock.timer(self._gc_interval_ms, self._on_gc_interval) + + def _cancel_gc_timer(self) -> None: + if self._gc_timer is not None: + self._gc_timer.cancel() + self._gc_timer = None + + def _on_gc_interval(self) -> None: + """RTO10c: releases what has been tombstoned for the grace period, and reschedules itself. + + A sweep that raises is logged, and the next one is scheduled all the same. + """ + self._gc_timer = None + try: + self._objects_pool.collect_garbage(self._gc_grace_period_ms, self._clock.now_ms()) + except Exception: + log.exception(f'RealtimeObject._on_gc_interval(): the GC sweep raised; channel={self._channel_name}') + self._schedule_gc_timer() + + +def _objects_event(event: Any) -> ObjectsEvent: + """`event` as an `ObjectsEvent`, raising AblyException 40003 if it is not one.""" + try: + return ObjectsEvent(event) + except ValueError: + events = ', '.join(repr(member.value) for member in ObjectsEvent) + raise AblyException(f'Objects event should be one of {events}; got {event!r}', 400, 40003) from None + + +def _entering(state: ChannelState) -> str: + """Why a wait for SYNCED fails, when the channel enters `state`.""" + return f'the channel entering the {state.value} state' + + +def _sync_wait_failure(failure_description: str, reason: str, cause: AblyException | None) -> AblyException: + """RTO23c1, RTO20e1: the 92008 error a wait for SYNCED fails with, `reason` saying why.""" + return AblyException(f'{failure_description} due to {reason} whilst waiting for objects sync to complete', + 400, 92008, cause=cause) diff --git a/ably/pubsub/objects/subscription.py b/ably/pubsub/objects/subscription.py new file mode 100644 index 00000000..63dffb1e --- /dev/null +++ b/ably/pubsub/objects/subscription.py @@ -0,0 +1,86 @@ +"""The handles a subscribe call returns, and the registry of listeners behind them.""" + +from __future__ import annotations + +from typing import Callable, Generic, TypeVar + +T = TypeVar('T') + + +class _Handle: + """Calls `deregister` the first time the handle is released, and never again.""" + + def __init__(self, deregister: Callable[[], None]): + self._deregister: Callable[[], None] | None = deregister + + def _release(self) -> None: + deregister, self._deregister = self._deregister, None + if deregister is not None: + deregister() + + +class Subscription(_Handle): + """SUB1: a registration for receiving events from a `subscribe` call.""" + + def unsubscribe(self) -> None: + """SUB2a: deregisters the listener. Calling it again does nothing (SUB2b).""" + self._release() + + +class StatusSubscription(_Handle): + """RTO18f: the registration `RealtimeObject.on` returns.""" + + def off(self) -> None: + """RTO18f1, RTO18f2: deregisters the listener the corresponding `on` call registered.""" + self._release() + + +class Registration(Generic[T]): + """One registration of `item` in a `Registry`, `active` until it is removed.""" + + def __init__(self, item: T): + self.item = item + self.active = True + + +class Registry(Generic[T]): + """The listeners, or other items, that `subscribe`-style calls register, in registration order. + + Registering one item twice makes two registrations, each removed on its own. A dispatch + iterates one `snapshot()` and skips each registration no longer `active` when it is + reached, so that one made during the dispatch is not called by it, and one removed during + it is not called again. + """ + + def __init__(self) -> None: + self._registrations: list[Registration[T]] = [] + + def __bool__(self) -> bool: + return bool(self._registrations) + + def register(self, item: T) -> Callable[[], None]: + """Registers `item`, returning the function that removes this one registration.""" + registration = Registration(item) + self._registrations.append(registration) + return lambda: self._remove(lambda entry: entry is registration) + + def deregister(self, item: T) -> None: + """Removes every registration of `item`.""" + self._remove(lambda entry: entry.item == item) + + def clear(self) -> None: + """Removes every registration.""" + self._remove(lambda entry: True) + + def snapshot(self) -> list[Registration[T]]: + """The registrations as they stand, for a dispatch to iterate.""" + return list(self._registrations) + + def _remove(self, matches: Callable[[Registration[T]], bool]) -> None: + kept = [] + for registration in self._registrations: + if matches(registration): + registration.active = False + else: + kept.append(registration) + self._registrations = kept diff --git a/ably/pubsub/objects/syncobjectspool.py b/ably/pubsub/objects/syncobjectspool.py new file mode 100644 index 00000000..254ba592 --- /dev/null +++ b/ably/pubsub/objects/syncobjectspool.py @@ -0,0 +1,83 @@ +"""The object states accumulated during an objects sync sequence (RTO5f).""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ably.pubsub.objects.objectmessage import ObjectMessage + +log = logging.getLogger(__name__) + + +class SyncObjectsPool: + """RTO5f: one `ObjectMessage` per object id, received over OBJECT_SYNC and not yet applied. + + A partial map state for an id already held is merged into it (RTO5f2). + """ + + def __init__(self): + self.entries: dict[str, ObjectMessage] = {} + + def apply_object_sync_messages(self, object_messages: list[ObjectMessage]) -> None: + """RTO5f: stores each message with an `object`, merging partial map states (RTO5f2a) and + skipping messages with neither `map` nor `counter` (RTO5f3) and partial counter states + (RTO5f2b).""" + for object_message in object_messages: + object_state = object_message.object + if object_state is None: + # RTO5d: only this message is skipped, so the rest of the sequence still applies + log.warning(f'SyncObjectsPool.apply_object_sync_messages(): skipping an OBJECT_SYNC message ' + f'with no object; message id={object_message.id}') + continue + + if object_state.map is None and object_state.counter is None: + # RTO5f3 + log.warning(f'SyncObjectsPool.apply_object_sync_messages(): skipping an OBJECT_SYNC message ' + f'with an unsupported object type; object_id={object_state.object_id}, ' + f'message id={object_message.id}') + continue + + existing = self.entries.get(object_state.object_id) + if existing is None: + self.entries[object_state.object_id] = object_message # RTO5f1 + continue + + # RTO5f2: a partial state of an object split across several OBJECT_SYNC messages + if object_state.map is not None: + self._merge_partial_map_state(existing, object_message) # RTO5f2a + else: + # RTO5f2b + log.error(f'SyncObjectsPool.apply_object_sync_messages(): skipping an unexpected partial ' + f'state for a counter; object_id={object_state.object_id}, ' + f'message id={object_message.id}') + + def clear(self) -> None: + """RTO5a2a, RTO5c4, RTO27a2: discards every stored message.""" + self.entries.clear() + + def _merge_partial_map_state(self, existing: ObjectMessage, object_message: ObjectMessage) -> None: + """RTO5f2a: merges the partial map state in `object_message` into the stored `existing`.""" + object_state = object_message.object + if object_state.tombstone: + self.entries[object_state.object_id] = object_message # RTO5f2a1 + return + + existing_map = existing.object.map + if existing_map is None: + log.error(f'SyncObjectsPool.apply_object_sync_messages(): skipping a partial map state for an ' + f'object first received as a counter; object_id={object_state.object_id}, ' + f'message id={object_message.id}') + return + + # RTO5f2a2: no two partial states of one map carry the same key + if existing_map.entries is None: + existing_map.entries = {} + existing_map.entries.update(object_state.map.entries or {}) + + def __len__(self) -> int: + return len(self.entries) + + def __contains__(self, object_id: object) -> bool: + return object_id in self.entries diff --git a/ably/pubsub/objects/valuetypes.py b/ably/pubsub/objects/valuetypes.py new file mode 100644 index 00000000..b50281fe --- /dev/null +++ b/ably/pubsub/objects/valuetypes.py @@ -0,0 +1,285 @@ +"""The creation blueprints `LiveCounter` and `LiveMap` (RTLCV*, RTLMV*), and the value unions. + +A blueprint is an inert, immutable description of an object to create. It becomes +ObjectMessages only when a mutation evaluates it (RTLCV4, RTLMV4), so `create` performs +no validation (RTLCV3c, RTLMV3c). + +Evaluation is pure: the caller fetches the server time (RTO16) and passes it in, and +each evaluation generates a fresh nonce, so a blueprint used twice creates two objects. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from typing import Any, TypeVar, Union + +from ably.pubsub.objects.objectid import generate_nonce, generate_object_id +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_JSON, + CounterCreate, + CounterCreateWithObjectId, + MapCreate, + MapCreateWithObjectId, + ObjectData, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapEntry, + ObjectsMapSemantics, +) +from ably.pubsub.util.exceptions import AblyException + +# The primitive values a map entry can hold: String, Number, Boolean, Binary, JsonArray +# and JsonObject (RTLM20a3) +Primitive = Union[str, float, bool, bytes, list, dict] + +T = TypeVar('T', bound=Primitive) + + +@dataclass(frozen=True) +class LiveCounter: + """RTLCV1: the intent to create a counter with an initial count.""" + + _count: Any = 0 # RTLCV2a + + @staticmethod + def create(initial_count: float = 0) -> LiveCounter: + """RTLCV3: a blueprint for a counter starting at `initial_count`.""" + return LiveCounter(initial_count) + + +@dataclass(frozen=True) +class LiveMap: + """RTLMV1: the intent to create a map with initial entries.""" + + _entries: Any = None # RTLMV2a + + @staticmethod + def create(entries: dict[str, LiveMapValue] | None = None) -> LiveMap: + """RTLMV3: a blueprint for a map holding `entries`. + + A dict is copied, along with each JSON object, JSON array and binary value in it, so + the blueprint does not change when the caller's values do (RTLMV3d); anything else is + kept for evaluation to reject (RTLMV3c). + """ + if not isinstance(entries, dict): + return LiveMap(entries) + return LiveMap({key: _snapshot(value) for key, value in entries.items()}) + + +# The values `set` accepts: a primitive, or a blueprint for a new object (RTTS11) +LiveMapValue = Union[Primitive, LiveMap, LiveCounter] + + +def validate_key(key: Any) -> None: + """RTLMV4b: raises AblyException 40003 unless `key` is a string. + + A map's `set` and `remove` validate their key the same way (RTLM20e1, RTLM21e1). + """ + if not isinstance(key, str): + raise AblyException('Map key should be string', 400, 40003) + + +def validate_value(value: Any) -> None: + """RTLM20e1: raises the AblyException a map's `set` raises for `value`, without evaluating it. + + That is 40013 unless `value` is a primitive or a blueprint (RTLMV4c), and for a blueprint, + whatever evaluating it raises (RTLCV4a, RTLMV4a-RTLMV4c), so that a write can reject one + before fetching the server time its object ids are generated from (RTO16). + """ + if isinstance(value, LiveCounter): + _counter_create(value) + elif isinstance(value, LiveMap): + entries, _ = _map_entries_data(value) + for entry in entries.values(): + if isinstance(entry, (LiveCounter, LiveMap)): + validate_value(entry) + else: + primitive_to_object_data(value) + + +def validate_amount(amount: Any) -> float: + """RTLC12e1: `amount` as a float, raising AblyException 40003 unless it is a finite number. + + A bool is not a number, and None is not an omitted amount. A counter's `increment` and + `decrement` validate their amount this way (RTLC12e1, RTLC13c). + """ + number = _finite_number(amount) + if number is None: + raise AblyException('Counter value increment should be a valid number', 400, 40003) + return number + + +def primitive_to_object_data(value: Any) -> ObjectData: + """RTLMV4d3-RTLMV4d7: the `ObjectData` holding the primitive `value`. + + It holds the value as it decodes on receipt: a number as a float (OD4c3), binary as + `bytes`, and a JSON object or array as a copy parsed back from its JSON encoding. + Raises AblyException 40013 for any other value (RTLMV4c, OD4a), which includes a + number that is not finite, a dict or list that is not JSON-encodable, a blueprint, + and a live object or a `PathObject` or `Instance` wrapping one (RTLMV4c1). + """ + if isinstance(value, bool): + return ObjectData(boolean=value) # RTLMV4d6 + if isinstance(value, (int, float)): + number = _finite_number(value) + if number is not None: + return ObjectData(number=number) # RTLMV4d5 + elif isinstance(value, str): + return ObjectData(string=value) # RTLMV4d4 + elif isinstance(value, (bytes, bytearray)): + return ObjectData(bytes=bytes(value)) # RTLMV4d7 + elif isinstance(value, (dict, list)): + try: + encoded = json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + pass + else: + return ObjectData(json=json.loads(encoded)) # RTLMV4d3 + raise AblyException('Map value data type is unsupported', 400, 40013) + + +def evaluate_live_counter(value: LiveCounter, timestamp_ms: int) -> ObjectMessage: + """RTLCV4: the COUNTER_CREATE `ObjectMessage` a `LiveCounter` evaluates to. + + `timestamp_ms` is the server time the object id is generated with (RTLCV4e). The + operation carries `counter_create_with_object_id`, whose `derived_from` is the + `CounterCreate` it was built from (RTLCV4g5). Raises AblyException 40003 for a count + that is not a finite number (RTLCV4a). + """ + counter_create = _counter_create(value) # RTLCV4a, RTLCV4b + initial_value = _initial_value(counter_create) # RTLCV4c + nonce = generate_nonce() # RTLCV4d + object_id = generate_object_id('counter', initial_value, nonce, timestamp_ms) # RTLCV4f + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_CREATE, # RTLCV4g1 + object_id=object_id, # RTLCV4g2 + counter_create_with_object_id=CounterCreateWithObjectId( + initial_value=initial_value, # RTLCV4g4 + nonce=nonce, # RTLCV4g3 + derived_from=counter_create, # RTLCV4g5 + ), + )) + + +def evaluate_live_map(value: LiveMap, timestamp_ms: int) -> list[ObjectMessage]: + """RTLMV4: the ObjectMessages a `LiveMap` evaluates to, nested creates first, depth-first, + and this map's MAP_CREATE last (RTLMV4k). + + The MAP_CREATE carries `map_create_with_object_id`, whose `derived_from` is the + `MapCreate` it was built from (RTLMV4j5). Raises AblyException 40003 for entries that + are not a dict or a key that is not a string (RTLMV4a, RTLMV4b), and 40013 for a + value of an unsupported type (RTLMV4c). + """ + # Every value at this level is validated (RTLMV4c) before any blueprint is evaluated + entries, data = _map_entries_data(value) + + messages: list[ObjectMessage] = [] + for key, entry in entries.items(): + if isinstance(entry, LiveCounter): + counter_create_message = evaluate_live_counter(entry, timestamp_ms) # RTLMV4d1 + messages.append(counter_create_message) + data[key] = ObjectData(object_id=counter_create_message.operation.object_id) + elif isinstance(entry, LiveMap): + nested_messages = evaluate_live_map(entry, timestamp_ms) # RTLMV4d2 + messages.extend(nested_messages) + data[key] = ObjectData(object_id=nested_messages[-1].operation.object_id) + + map_create = MapCreate( # RTLMV4e + semantics=ObjectsMapSemantics.LWW, + entries={key: ObjectsMapEntry(data=entry_data) for key, entry_data in data.items()}, + ) + initial_value = _initial_value(map_create) # RTLMV4f + nonce = generate_nonce() # RTLMV4g + object_id = generate_object_id('map', initial_value, nonce, timestamp_ms) # RTLMV4i + messages.append(ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, # RTLMV4j1 + object_id=object_id, # RTLMV4j2 + map_create_with_object_id=MapCreateWithObjectId( + initial_value=initial_value, # RTLMV4j4 + nonce=nonce, # RTLMV4j3 + derived_from=map_create, # RTLMV4j5 + ), + ))) + return messages # RTLMV4k + + +def evaluate(value: LiveCounter | LiveMap, timestamp_ms: int) -> list[ObjectMessage]: + """The ObjectMessages a blueprint evaluates to, the create for `value` itself last.""" + if isinstance(value, LiveCounter): + return [evaluate_live_counter(value, timestamp_ms)] + if isinstance(value, LiveMap): + return evaluate_live_map(value, timestamp_ms) + raise TypeError(f'Expected a LiveCounter or LiveMap, got {type(value).__name__}') + + +def _counter_create(value: LiveCounter) -> CounterCreate: + """RTLCV4a, RTLCV4b: the `CounterCreate` a `LiveCounter` describes, raising AblyException 40003 + for a count that is not a finite number.""" + count = _finite_number(value._count) + if count is None: + raise AblyException('Counter value should be a valid number', 400, 40003) # RTLCV4a + return CounterCreate(count=count) # RTLCV4b + + +def _map_entries_data(value: LiveMap) -> tuple[dict, dict[str, ObjectData | None]]: + """RTLMV4a-RTLMV4c: a `LiveMap`'s entries, and the `ObjectData` of each, None for a blueprint. + + Raises AblyException 40003 for entries that are not a dict or a key that is not a string, + and 40013 for a value of an unsupported type at this level; a blueprint among the entries + is not looked into. + """ + # `LiveMap.create()` and `LiveMap.create(None)` both leave the entries unset + entries = {} if value._entries is None else value._entries + if not isinstance(entries, dict): + raise AblyException('Map entries should be a dict', 400, 40003) # RTLMV4a + for key in entries: + validate_key(key) # RTLMV4b + # RTLMV4c + data: dict[str, ObjectData | None] = { + key: None if isinstance(entry, (LiveCounter, LiveMap)) else primitive_to_object_data(entry) + for key, entry in entries.items() + } + return entries, data + + +def _snapshot(value: Any) -> Any: + """A copy of a map entry's value that later changes to `value` do not reach (RTLMV3d). + + A JSON object or array is copied through its JSON encoding, which evaluation applies to + it anyway (RTLMV4d3), and binary is copied to `bytes`. A value that does not encode is + kept as it is, as no validation is done at creation (RTLMV3c); evaluation rejects it. + """ + if isinstance(value, bytearray): + return bytes(value) + if isinstance(value, (dict, list)): + try: + return json.loads(json.dumps(value, allow_nan=False)) + except (TypeError, ValueError, RecursionError): + return value + return value + + +def _finite_number(value: Any) -> float | None: + """`value` as a float if it is a finite number, else None. A boolean is not a number.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + number = float(value) + except OverflowError: + return None + return number if math.isfinite(number) else None + + +def _initial_value(create: CounterCreate | MapCreate) -> str: + """RTLCV4c, RTLMV4f: the JSON string of `create` in its JSON-wire encoding (OD4d), whatever + protocol the connection uses. + + The object id is the hash of this exact string (RTO14b1), and it is sent unchanged as + the `initialValue`. Keys keep the order `to_dict` builds them in, with no whitespace + between tokens, non-ASCII characters escaped, and numbers in Python's shortest + round-trip form. + """ + return json.dumps(create.to_dict(WIRE_FORMAT_JSON), separators=(',', ':'), allow_nan=False) diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 0dfd720b..538222da 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -6,6 +6,8 @@ from ably.pubsub.http.channel import Channel from ably.pubsub.http.channel import Channels as HttpChannels +from ably.pubsub.objects.objectmessage import ObjectMessage +from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.realtime.annotations import RealtimeAnnotations from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.realtime.presence import RealtimePresence @@ -92,6 +94,9 @@ def __init__(self, realtime: DefaultPubSubRealtimeClient, name: str, # Initialize realtime annotations for this channel (override REST annotations) self._Channel__annotations = RealtimeAnnotations(self, realtime.connection.connection_manager) + # RTL27: the LiveObjects entry point for this channel + self.__object = RealtimeObject(self) + async def set_options(self, channel_options: ChannelOptions) -> None: """Set channel options""" should_reattach = self.should_reattach_to_set_options(channel_options) @@ -121,12 +126,21 @@ async def attach(self) -> None: AblyException If unable to attach channel """ + await self._attach() + + async def _attach(self) -> ChannelStateChange | None: + """The attach procedure `attach` runs, returning the state change it ended with, or None if + the channel was already ATTACHED. + Raises where `attach` does. An attach that ends in a state other than ATTACHED, + SUSPENDED or FAILED, such as DETACHED when the connection closes first, returns that + state change. + """ log.info(f'RealtimeChannel.attach() called, channel = {self.name}') # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: - return + return None self.__error_reason = None @@ -150,6 +164,7 @@ async def attach(self) -> None: if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): raise state_change.reason + return state_change def _attach_impl(self): log.debug("RealtimeChannel.attach_impl(): sending ATTACH protocol message") @@ -170,6 +185,33 @@ def _attach_impl(self): self._send_message(attach_msg) + # RTL33 + async def _ensure_active(self) -> None: + """The ensure-active-channel procedure: attaches the channel unless it is ATTACHED or SUSPENDED. + + Raises + ------ + AblyException + 90001 if the channel is FAILED (RTL33c), or the error the implicit attach failed + with (RTL33b1): the reason for the state it ended in, else 90001 + """ + # RTL33a + if self.state in (ChannelState.ATTACHED, ChannelState.SUSPENDED): + return + + # RTL33c + if self.state == ChannelState.FAILED: + raise AblyException(f"Channel operation failed as channel state is {self.state.value}", 400, 90001) + + # RTL33b + state_change = await self._attach() + if state_change is not None and state_change.current != ChannelState.ATTACHED: + # RTL33b1, RTL4d: the attach ended without the channel attaching, as when the connection + # closes or `detach` is called first + raise state_change.reason or AblyException( + f'Unable to attach channel; channel state = {state_change.current.value}, ' + f'connection state = {self.__realtime.connection.state.value}', 400, 90001) + # RTL5 async def detach(self) -> None: """Detach from channel @@ -706,6 +748,7 @@ def _on_message(self, proto_msg: dict) -> None: exception = None resumed = False has_presence = False + has_objects = False self.__attach_serial = channel_serial self.__channel_serial = channel_serial @@ -718,6 +761,8 @@ def _on_message(self, proto_msg: dict) -> None: resumed = has_flag(flags, Flag.RESUMED) # RTP1: Check for HAS_PRESENCE flag has_presence = has_flag(flags, Flag.HAS_PRESENCE) + # RTO4: whether an OBJECT_SYNC follows + has_objects = has_flag(flags, Flag.HAS_OBJECTS) # Store channel attach flags self.__modes = decode_channel_mode(flags) @@ -726,8 +771,11 @@ def _on_message(self, proto_msg: dict) -> None: if not resumed: state_change = ChannelStateChange(self.state, ChannelState.ATTACHED, resumed, exception) self._emit("update", state_change) + # RTO4: an ATTACHED restarts the objects sync even while attached, a resumed one included + self.__object._on_attached(has_objects) elif self.state == ChannelState.ATTACHING: self._notify_state(ChannelState.ATTACHED, resumed=resumed, has_presence=has_presence) + self.__object._on_attached(has_objects) # RTO4 else: log.warn("RealtimeChannel._on_message(): ATTACHED received while not attaching") elif action == ProtocolMessageAction.DETACHED: @@ -774,6 +822,18 @@ def _on_message(self, proto_msg: dict) -> None: self.__channel_serial = channel_serial except Exception as e: log.error(f"Annotation processing error {e}. Skip annotations {annotation_data}") + elif action == ProtocolMessageAction.OBJECT: + # RTL15b + if channel_serial: + self.__channel_serial = channel_serial + # An object message that fails to decode is logged and left out, and the rest are handled + object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) + self.__object._handle_object_messages(object_messages) # RTO8 + elif action == ProtocolMessageAction.OBJECT_SYNC: + # RTO5: the channelSerial of an OBJECT_SYNC carries the sync sequence and cursor (RTO5a1), + # which are handled whichever of its object messages fail to decode + object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) + self.__object._handle_object_sync_messages(object_messages, channel_serial) elif action == ProtocolMessageAction.ERROR: error = AblyException.from_dict(proto_msg.get('error')) self._notify_state(ChannelState.FAILED, reason=error) @@ -825,6 +885,10 @@ def _notify_state(self, state: ChannelState, reason: AblyException | None = None # RTP5: Notify presence of channel state change self.__presence.act_on_channel_state(state, has_presence=has_presence, error=reason) + # RTO27: the objects act on every state but ATTACHED, whose ATTACHED message RTO4 handles + if state != ChannelState.ATTACHED: + self.__object._act_on_channel_state(state, reason) + def _send_message(self, msg: dict) -> None: asyncio.create_task(self.__realtime.connection.connection_manager.send_protocol_message(msg)) @@ -925,6 +989,12 @@ def presence(self): def annotations(self) -> RealtimeAnnotations: return self._Channel__annotations + # RTL27 + @property + def object(self) -> RealtimeObject: + """The LiveObjects on this channel""" + return self.__object + @property def modes(self): """Get the list of channel modes""" @@ -1025,7 +1095,9 @@ def release(self, name: str) -> None: """ if name not in self.__all: return - del self.__all[name] + channel = self.__all.pop(name) + # The released channel receives nothing more, so its objects stop their GC timer and their waits + channel.object._release() def _on_channel_message(self, msg: dict) -> None: channel_name = msg.get('channel') @@ -1067,8 +1139,10 @@ def _propagate_connection_interruption(self, state: ConnectionState, reason: Abl channel._notify_state(connection_to_channel_state[state], reason) def _on_connected(self) -> None: + connection_details = self.__ably.connection.connection_details for channel_name in self.__all: channel = self.__all[channel_name] + channel.object._on_connected(connection_details) # RTO10b2 if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: channel._check_pending_state() elif channel.state == ChannelState.SUSPENDED: diff --git a/ably/pubsub/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py index 09ce5eab..407692d2 100644 --- a/ably/pubsub/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -211,6 +211,8 @@ async def __get_transport_params(self) -> dict: # RTN2a: Set format to msgpack if use_binary_protocol is enabled if self.options.use_binary_protocol: params["format"] = "msgpack" + # RTN2h, RTC1a: whether the server echoes this connection's own messages back to it + params["echo"] = "true" if self.options.echo_messages else "false" # Add any custom transport params from options params.update(self.options.transport_params) diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 41efc61e..39826dd9 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -22,6 +22,33 @@ from ably.pubsub.http.channel import Channels as HttpChannels from ably.pubsub.http.http import DefaultPubSubHttpClient as _DefaultPubSubHttpClient from ably.pubsub.http.push import Push +from ably.pubsub.objects.batch import ( + Batch, + BatchContext, + LiveCounterBatchContext, + LiveMapBatchContext, + PrimitiveBatchContext, +) +from ably.pubsub.objects.enums import ObjectsEvent, ValueType +from ably.pubsub.objects.instance import ( + Instance, + InstanceSubscriptionEvent, + LiveCounterInstance, + LiveMapInstance, + PrimitiveInstance, +) +from ably.pubsub.objects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.objects.pathobject import ( + LiveCounterPathObject, + LiveMapPathObject, + PathObject, + PathObjectSubscriptionEvent, + PrimitivePathObject, +) +from ably.pubsub.objects.publicmessage import ObjectMessage, ObjectOperation +from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.objects.subscription import StatusSubscription, Subscription +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, LiveMapValue, Primitive from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection @@ -188,6 +215,9 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: queue_messages: bool Hold messages published while the connection is not yet established and send them once it is, rather than failing them. Defaults to True. + echo_messages: bool + Deliver the messages this client publishes back to it on its own connection. + LiveObjects writes require it. Defaults to True. transport_params: dict Additional query parameters to send when opening the realtime connection. @@ -248,6 +278,8 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'Annotation', 'AnnotationAction', 'Auth', + 'Batch', + 'BatchContext', 'Capability', 'ChannelMode', 'ChannelOptions', @@ -257,21 +289,47 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'HttpChannels', 'HttpPaginatedResponse', 'IncompatibleClientIdException', + 'Instance', + 'InstanceSubscriptionEvent', + 'LiveCounter', + 'LiveCounterBatchContext', + 'LiveCounterInstance', + 'LiveCounterPathObject', + 'LiveMap', + 'LiveMapBatchContext', + 'LiveMapInstance', + 'LiveMapPathObject', + 'LiveMapValue', 'Message', 'MessageAction', 'MessageOperation', 'MessageVersion', + 'ObjectMessage', + 'ObjectOperation', + 'ObjectOperationAction', + 'ObjectsEvent', + 'ObjectsMapSemantics', 'Options', 'PaginatedResult', + 'PathObject', + 'PathObjectSubscriptionEvent', + 'Primitive', + 'PrimitiveBatchContext', + 'PrimitiveInstance', + 'PrimitivePathObject', 'PubSubHttpClient', 'PubSubRealtimeClient', 'PublishResult', 'Push', 'PushChannelSubscription', 'RealtimeChannels', + 'RealtimeObject', + 'StatusSubscription', + 'Subscription', 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', + 'ValueType', 'api_version', 'create_http_client', 'create_realtime_client', diff --git a/ably/pubsub/transport/defaults.py b/ably/pubsub/transport/defaults.py index 40d73e08..ae5310e6 100644 --- a/ably/pubsub/transport/defaults.py +++ b/ably/pubsub/transport/defaults.py @@ -1,5 +1,5 @@ class Defaults: - protocol_version = "5" + protocol_version = "6" connectivity_check_url = "https://internet-up.ably-realtime.com/is-the-internet-up.txt" endpoint = 'main' diff --git a/ably/pubsub/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py index 7177732c..785bae46 100644 --- a/ably/pubsub/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -207,7 +207,9 @@ async def on_protocol_message(self, msg): ProtocolMessageAction.MESSAGE, ProtocolMessageAction.PRESENCE, ProtocolMessageAction.ANNOTATION, - ProtocolMessageAction.SYNC + ProtocolMessageAction.SYNC, + ProtocolMessageAction.OBJECT, + ProtocolMessageAction.OBJECT_SYNC, ): self.connection_manager.on_channel_message(msg) diff --git a/ably/pubsub/types/channelmode.py b/ably/pubsub/types/channelmode.py index b16b355d..47a5f9c5 100644 --- a/ably/pubsub/types/channelmode.py +++ b/ably/pubsub/types/channelmode.py @@ -12,6 +12,8 @@ class ChannelMode(int, Enum): PRESENCE_SUBSCRIBE = Flag.PRESENCE_SUBSCRIBE ANNOTATION_PUBLISH = Flag.ANNOTATION_PUBLISH ANNOTATION_SUBSCRIBE = Flag.ANNOTATION_SUBSCRIBE + OBJECT_SUBSCRIBE = Flag.OBJECT_SUBSCRIBE + OBJECT_PUBLISH = Flag.OBJECT_PUBLISH def encode_channel_mode(modes: list[ChannelMode]) -> int: diff --git a/ably/pubsub/types/connectiondetails.py b/ably/pubsub/types/connectiondetails.py index a281daed..d1b461f7 100644 --- a/ably/pubsub/types/connectiondetails.py +++ b/ably/pubsub/types/connectiondetails.py @@ -8,13 +8,19 @@ class ConnectionDetails: connection_key: str def __init__(self, connection_state_ttl: int, max_idle_interval: int, - connection_key: str, client_id: str): + connection_key: str, client_id: str, site_code: str = None, + objects_gc_grace_period: int = None): self.connection_state_ttl = connection_state_ttl self.max_idle_interval = max_idle_interval self.connection_key = connection_key self.client_id = client_id + # CD2j: the site the connection reached, under which LiveObjects applies its own operations + self.site_code = site_code + # CD2i: how long tombstoned objects and map entries are kept before release, in milliseconds + self.objects_gc_grace_period = objects_gc_grace_period @staticmethod def from_dict(json_dict: dict): return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval'), - json_dict.get('connectionKey'), json_dict.get('clientId')) + json_dict.get('connectionKey'), json_dict.get('clientId'), + json_dict.get('siteCode'), json_dict.get('objectsGCGracePeriod')) diff --git a/ably/pubsub/types/flags.py b/ably/pubsub/types/flags.py index 86666019..ff8fe3a0 100644 --- a/ably/pubsub/types/flags.py +++ b/ably/pubsub/types/flags.py @@ -8,6 +8,7 @@ class Flag(int, Enum): RESUMED = 1 << 2 TRANSIENT = 1 << 4 ATTACH_RESUME = 1 << 5 + HAS_OBJECTS = 1 << 7 # Channel mode flags PRESENCE = 1 << 16 PUBLISH = 1 << 17 @@ -15,6 +16,8 @@ class Flag(int, Enum): PRESENCE_SUBSCRIBE = 1 << 19 ANNOTATION_PUBLISH = 1 << 21 ANNOTATION_SUBSCRIBE = 1 << 22 + OBJECT_SUBSCRIBE = 1 << 24 + OBJECT_PUBLISH = 1 << 25 def has_flag(message_flags: int, flag: Flag): diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 51f6016e..0a4fc932 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -34,8 +34,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, connectivity_check_url=None, channel_retry_timeout=Defaults.channel_retry_timeout, add_request_ids=False, - vcdiff_decoder: VCDiffDecoder = None, transport_params=None, _test_options=None, - **kwargs): + vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True, + _test_options=None, **kwargs): super().__init__(**kwargs) @@ -108,6 +108,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__tls_port = tls_port self.__use_binary_protocol = use_binary_protocol self.__queue_messages = queue_messages + self.__echo_messages = echo_messages self.__recover = recover self.__endpoint = endpoint self.__http_open_timeout = http_open_timeout @@ -189,6 +190,11 @@ def queue_messages(self): def queue_messages(self, value): self.__queue_messages = value + # TO3h + @property + def echo_messages(self): + return self.__echo_messages + @property def recover(self): return self.__recover diff --git a/ably/pubsub/version.py b/ably/pubsub/version.py index 8389a068..59f747d4 100644 --- a/ably/pubsub/version.py +++ b/ably/pubsub/version.py @@ -5,5 +5,5 @@ ``ably-pubsub-*`` distributions, and so cannot carry one. """ -api_version = '5' +api_version = '6' lib_version = '3.1.4' diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index e89fb1c8..7f0bfa04 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -190,7 +190,7 @@ async def test_request_headers(self): # API assert 'X-Ably-Version' in r.request.headers - assert r.request.headers['X-Ably-Version'] == '5' + assert r.request.headers['X-Ably-Version'] == '6' # Agent assert 'Ably-Agent' in r.request.headers diff --git a/test/unit/objects/__init__.py b/test/unit/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/objects/batch_test.py b/test/unit/objects/batch_test.py new file mode 100644 index 00000000..5d9f703a --- /dev/null +++ b/test/unit/objects/batch_test.py @@ -0,0 +1,700 @@ +"""Batched writes: `batch()` on the typed path and instance views, and the batch contexts. + +Spec points: RTPO20, RTINS17 and RTBC1-RTBC16 (ably/specification#471), with RTO26 as the +precondition a batch checks. No UTS specification covers batch, so these tests are +written against the features specification and LODR-061. + +Each test drives the standard synced pool over the mock websocket, which ACKs every OBJECT +message, so a batch's writes have been applied locally (RTO20) by the time its block exits. +""" + +import json +from typing import NamedTuple + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.server import ( + LiveCounter, + LiveCounterBatchContext, + LiveMap, + LiveMapBatchContext, + PrimitiveBatchContext, +) +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.helpers.clock import FakeClock, settle +from test.uts.objects.helpers.standard_test_pool import ( + HAS_OBJECTS, + OBJECT, + OBJECT_SUBSCRIBE_FLAG, + build_map_set, + build_object_message, + remote_serial, + setup_synced_channel, + standard_mock_websocket, + time_mock_http, +) + +MAP_CREATE = ObjectOperationAction.MAP_CREATE +MAP_SET = ObjectOperationAction.MAP_SET +MAP_REMOVE = ObjectOperationAction.MAP_REMOVE +COUNTER_CREATE = ObjectOperationAction.COUNTER_CREATE +COUNTER_INC = ObjectOperationAction.COUNTER_INC + +# The server time a test with a fake clock creates objects at +SERVER_TIME_MS = 1_700_000_123_000 + + +async def _synced(**client_options): + """The standard synced channel, and a list of each OBJECT ProtocolMessage the client sends on it.""" + published = [] + mock_ws = standard_mock_websocket(on_object=published.append) + synced = await setup_synced_channel('test', mock_ws=mock_ws, **client_options) + return synced, published + + +def _operations(protocol_message): + """The wire operations of an OBJECT ProtocolMessage the client sent, in order.""" + return [object_message['operation'] for object_message in protocol_message['state']] + + +def _map_set(object_id, key, value): + return {'action': MAP_SET, 'objectId': object_id, 'mapSet': {'key': key, 'value': value}} + + +def _counter_inc(object_id, number): + return {'action': COUNTER_INC, 'objectId': object_id, 'counterInc': {'number': number}} + + +def _assert_error(excinfo, code): + assert excinfo.value.code == code + assert excinfo.value.status_code == 400 + + +# --- Publishing --------------------------------------------------------------------------- + +async def test_rtpo20f_batch_publishes_one_object_message(): + """RTPO20f, RTBC16d: the writes queued in a batch are published in order as one OBJECT + ProtocolMessage, then applied locally.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.remove('age') + ctx.get('score').as_live_counter().increment(5) + ctx.get('profile').as_live_map().set('email', 'bob@example.com') + + assert len(published) == 1 + assert published[0]['action'] == OBJECT + assert published[0]['channel'] == 'test' + assert _operations(published[0]) == [ + _map_set('root', 'name', {'string': 'Bob'}), + {'action': MAP_REMOVE, 'objectId': 'root', 'mapRemove': {'key': 'age'}}, + _counter_inc('counter:score@1000', 5), + _map_set('map:profile@1000', 'email', {'string': 'bob@example.com'}), + ] + + assert root.get('name').as_primitive().value() == 'Bob' + assert root.get('age').exists() is False + assert root.get('score').as_live_counter().value() == 105 + assert root.at('profile.email').as_primitive().value() == 'bob@example.com' + + +async def test_rtbc12e_nested_value_types_published_in_the_batch(): + """RTBC12e, RTLM20e7g: a `LiveMap` value is published as the creates it evaluates to, nested + ones first, ahead of the MAP_SET that references it, all in the batch's one message. Their + object ids carry the server time (RTO16), which the mock answers from the client's clock.""" + (client, channel, root, mock_ws), published = await _synced(clock=FakeClock(epoch_ms=SERVER_TIME_MS)) + + async with root.batch() as ctx: + ctx.set('team', LiveMap.create({'lead': 'Carol', 'points': LiveCounter.create(10)})) + ctx.get('score').as_live_counter().increment(1) + + assert len(published) == 1 + counter_create, map_create, map_set, counter_inc = _operations(published[0]) + + assert counter_create['action'] == COUNTER_CREATE + assert counter_create['objectId'].startswith('counter:') + assert counter_create['objectId'].endswith(f'@{SERVER_TIME_MS}') + assert json.loads(counter_create['counterCreateWithObjectId']['initialValue']) == {'count': 10} + + assert map_create['action'] == MAP_CREATE + assert map_create['objectId'].startswith('map:') + assert map_create['objectId'].endswith(f'@{SERVER_TIME_MS}') + assert json.loads(map_create['mapCreateWithObjectId']['initialValue'])['entries'] == { + 'lead': {'data': {'string': 'Carol'}}, + 'points': {'data': {'objectId': counter_create['objectId']}}, + } + + assert map_set == _map_set('root', 'team', {'objectId': map_create['objectId']}) + assert counter_inc == _counter_inc('counter:score@1000', 1) + + assert root.get('team').instance().id == map_create['objectId'] + assert root.at('team.lead').as_primitive().value() == 'Carol' + assert root.at('team.points').as_live_counter().value() == 10 + + +async def test_rtbc12e_value_type_set_twice_creates_two_objects(): + """RTBC12e, RTLMV4d1: each write of a `LiveCounter` value is evaluated on its own, so one + value set at two keys creates two counters.""" + (client, channel, root, mock_ws), published = await _synced() + counter = LiveCounter.create(3) + + async with root.batch() as ctx: + ctx.set('first', counter) + ctx.set('second', counter) + + first_create, first_set, second_create, second_set = _operations(published[0]) + assert first_create['objectId'] != second_create['objectId'] + assert first_set == _map_set('root', 'first', {'objectId': first_create['objectId']}) + assert second_set == _map_set('root', 'second', {'objectId': second_create['objectId']}) + assert root.get('first').as_live_counter().value() == 3 + assert root.get('second').as_live_counter().value() == 3 + + +async def test_rtbc16d_empty_batch_publishes_nothing(): + """RTBC16d: a batch that queues nothing publishes nothing.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.get('name').as_primitive().value() == 'Alice' + + assert published == [] + + +async def test_rtbc16b_reads_inside_the_block_see_the_state_before_the_batch(): + """RTBC4-RTBC9, RTBC16b: the writes in a block only queue, so reads inside it resolve against + the objects as they were, and nothing is sent until the block exits.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + score = ctx.get('score').as_live_counter() + ctx.set('name', 'Bob') + ctx.set('city', 'Paris') + ctx.remove('age') + score.increment(5) + + assert ctx.get('name').as_primitive().value() == 'Alice' + assert ctx.get('city') is None + assert ctx.get('age').as_primitive().value() == 30 + assert ctx.keys() == ['name', 'age', 'active', 'score', 'profile', 'data', 'avatar'] + assert ctx.size() == 7 + assert score.value() == 100 + assert root.get('name').as_primitive().value() == 'Alice' + + # Nothing is sent while the block runs, however long it yields to the event loop + await settle() + assert published == [] + + assert root.get('name').as_primitive().value() == 'Bob' + assert root.get('city').as_primitive().value() == 'Paris' + assert root.get('age').exists() is False + assert root.get('score').as_live_counter().value() == 105 + + +async def test_rtpo20g_exception_in_the_block_publishes_nothing(): + """RTPO20g: an exception raised in the block propagates, nothing queued is published, and the + batch is closed.""" + (client, channel, root, mock_ws), published = await _synced() + + with pytest.raises(ValueError, match='abandoned'): + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.get('score').as_live_counter().increment(5) + raise ValueError('abandoned') + + assert published == [] + assert root.get('name').as_primitive().value() == 'Alice' + assert root.get('score').as_live_counter().value() == 100 + + with pytest.raises(AblyException) as excinfo: + ctx.keys() + _assert_error(excinfo, 40000) + + +async def test_rtbc12_invalid_value_inside_a_value_type_raises_at_the_call(): + """RTBC12, RTLMV4c: a `LiveMap` value is evaluated when the batch is flushed, but its + contents are validated when it is set, so an unsupported value inside one raises from the + `set` call, before the server time is fetched, and queues nothing; the batch's other writes + are published.""" + mock_http = time_mock_http() + (client, channel, root, mock_ws), published = await _synced(mock_http=mock_http) + + async with root.batch() as ctx: + with pytest.raises(AblyException) as excinfo: + ctx.set('team', LiveMap.create({'lead': object()})) + ctx.set('name', 'Bob') + + _assert_error(excinfo, 40013) + assert len(published) == 1 + assert _operations(published[0]) == [_map_set('root', 'name', {'string': 'Bob'})] + assert [request for request in mock_http.captured_requests if request.path == '/time'] == [] + + +async def test_rtpo20f_batch_checks_preconditions_again_before_publishing(): + """RTPO20f, RTO26b: a block can await after its writes, so the batch checks the write + preconditions again before publishing; a channel detached in the meantime makes the end of + the block raise 90001, with nothing published.""" + (client, channel, root, mock_ws), published = await _synced() + + with pytest.raises(AblyException) as excinfo: + async with root.batch() as ctx: + ctx.set('name', 'Bob') + await channel.detach() + + _assert_error(excinfo, 90001) + assert published == [] + + +async def test_rtpo20f_rejected_publish_raises_from_the_end_of_the_block(): + """RTPO20f, RTPO20g, RTO20b: when the batch's publish is rejected, the end of the block raises + the rejection's error, nothing is applied locally, and the batch is closed.""" + published = [] + + def reject(message): + published.append(message) + mock_ws.send_to_client({ + 'action': int(ProtocolMessageAction.NACK), + 'msgSerial': message['msgSerial'], + 'count': 1, + 'error': {'code': 40160, 'statusCode': 401, 'message': 'Operation not permitted'}, + }) + + mock_ws = standard_mock_websocket(auto_ack=False, on_object=reject) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + with pytest.raises(AblyException) as excinfo: + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.get('score').as_live_counter().increment(5) + + assert excinfo.value.code == 40160 + assert len(published) == 1 + assert len(_operations(published[0])) == 2 + assert root.get('name').as_primitive().value() == 'Alice' + assert root.get('score').as_live_counter().value() == 100 + + with pytest.raises(AblyException) as closed: + ctx.keys() + _assert_error(closed, 40000) + + +# --- Entering ----------------------------------------------------------------------------- + +@pytest.mark.parametrize('open_batch', [ + lambda root: root.get('name').as_live_map().batch(), + lambda root: root.get('missing').as_live_map().batch(), + lambda root: root.get('name').get('first').as_live_counter().batch(), + lambda root: root.get('score').as_live_map().batch(), + lambda root: root.as_live_counter().batch(), +], ids=['primitive', 'missing-key', 'below-primitive', 'counter-as-map', 'map-as-counter']) +async def test_rtpo20c_batch_on_a_path_without_the_live_object_raises(open_batch): + """RTPO20c: entering a batch on a path that does not resolve to a live object of the view's + type raises 92007, and publishes nothing.""" + (client, channel, root, mock_ws), published = await _synced() + batch = open_batch(root) + + with pytest.raises(AblyException) as excinfo: + async with batch: + pytest.fail('the batch was entered') + + _assert_error(excinfo, 92007) + assert published == [] + + +async def test_rtpo20c_path_is_resolved_when_the_block_is_entered(): + """RTPO20c: a path batch resolves its path when the block is entered, not when `batch()` is + called.""" + (client, channel, root, mock_ws), published = await _synced() + batch = root.get('visits').as_live_counter().batch() + + await root.set('visits', LiveCounter.create(1)) + async with batch as visits: + assert isinstance(visits, LiveCounterBatchContext) + visits.increment(2) + + assert root.get('visits').as_live_counter().value() == 3 + + +async def test_rtpo20b_batch_requires_publish_mode(): + """RTPO20b, RTO26a: entering a batch on a channel without the OBJECT_PUBLISH mode raises 40024.""" + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws, + modes=(ChannelMode.OBJECT_SUBSCRIBE,)) + + with pytest.raises(AblyException) as excinfo: + async with root.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 40024) + + +async def test_rtpo20b_batch_requires_echo_messages(): + """RTPO20b, RTO26c: entering a batch on a client with `echo_messages` disabled raises 40000.""" + (client, channel, root, mock_ws), published = await _synced(echo_messages=False) + + with pytest.raises(AblyException) as excinfo: + async with root.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 40000) + + +async def test_rtpo20b_batch_checks_preconditions_before_resolving(): + """RTPO20b, RTO26b: entering a batch on a detached channel raises 90001, which is checked + before the path is resolved.""" + (client, channel, root, mock_ws), published = await _synced() + score = root.get('score').as_live_counter() + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + # The detach cleared the objects (RTO27a), so `score` no longer resolves: a 92007 here + # would mean the path was resolved first + with pytest.raises(AblyException) as excinfo: + async with score.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 90001) + + +async def test_rtins17b_instance_batch_checks_preconditions(): + """RTINS17b, RTO26b: entering a batch on an instance of a detached channel raises 90001.""" + (client, channel, root, mock_ws), published = await _synced() + score = root.get('score').instance().as_live_counter() + + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + async with score.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 90001) + + +async def test_rtpo20d_batch_can_be_entered_once(): + """RTPO20d: each `batch()` call opens one batch, so its block can only be entered once.""" + (client, channel, root, mock_ws), published = await _synced() + batch = root.batch() + + async with batch as ctx: + ctx.set('name', 'Bob') + + with pytest.raises(RuntimeError): + async with batch: + pytest.fail('the batch was entered again') + + assert len(published) == 1 + + +# --- Instances ---------------------------------------------------------------------------- + +async def test_rtins17_batch_on_a_map_instance(): + """RTINS17d-RTINS17f: a batch on a map instance acts on the map it wraps, and is published as + one message.""" + (client, channel, root, mock_ws), published = await _synced() + profile = root.get('profile').instance().as_live_map() + + async with profile.batch() as ctx: + assert isinstance(ctx, LiveMapBatchContext) + assert ctx.id == 'map:profile@1000' + ctx.set('email', 'carol@example.com') + ctx.remove('prefs') + ctx.get('nested_counter').as_live_counter().increment(3) + + assert len(published) == 1 + assert _operations(published[0]) == [ + _map_set('map:profile@1000', 'email', {'string': 'carol@example.com'}), + {'action': MAP_REMOVE, 'objectId': 'map:profile@1000', 'mapRemove': {'key': 'prefs'}}, + _counter_inc('counter:nested@1000', 3), + ] + assert root.at('profile.email').as_primitive().value() == 'carol@example.com' + assert root.at('profile.prefs').exists() is False + assert root.at('profile.nested_counter').as_live_counter().value() == 8 + + +async def test_rtins17_batch_on_a_counter_instance(): + """RTINS17d-RTINS17f: a batch on a counter instance increments the counter it wraps.""" + (client, channel, root, mock_ws), published = await _synced() + score = root.get('score').instance().as_live_counter() + + async with score.batch() as ctx: + assert isinstance(ctx, LiveCounterBatchContext) + ctx.increment(10) + ctx.decrement(4) + + assert _operations(published[0]) == [ + _counter_inc('counter:score@1000', 10), + _counter_inc('counter:score@1000', -4), + ] + assert score.value() == 106 + + +async def test_rtins17d_instance_batch_follows_the_object_not_the_path(): + """RTINS17d, RTPO20c: an instance batch acts on the object the instance wraps wherever it + sits, while a path batch acts on what the path resolves to when it is entered.""" + (client, channel, root, mock_ws), published = await _synced() + profile = root.get('profile').instance().as_live_map() + + await root.remove('profile') + + with pytest.raises(AblyException) as excinfo: + async with root.get('profile').as_live_map().batch(): + pytest.fail('the path batch was entered') + _assert_error(excinfo, 92007) + + async with profile.batch() as ctx: + ctx.set('email', 'dana@example.com') + + assert _operations(published[-1]) == [_map_set('map:profile@1000', 'email', {'string': 'dana@example.com'})] + assert profile.get('email').as_primitive().value() == 'dana@example.com' + + +# --- Counter contexts --------------------------------------------------------------------- + +async def test_rtbc14_rtbc15_counter_batch_increments_and_decrements(): + """RTBC14, RTBC15: a counter context queues a COUNTER_INC for each increment, and one of the + negated amount for each decrement, the amount defaulting to 1.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.get('score').as_live_counter().batch() as score: + assert isinstance(score, LiveCounterBatchContext) + score.increment(5) + score.decrement(2) + score.increment() + score.decrement() + score.increment(0.5) + assert score.value() == 100 + + assert len(published) == 1 + assert _operations(published[0]) == [ + _counter_inc('counter:score@1000', number) for number in (5, -2, 1, -1, 0.5) + ] + assert root.get('score').as_live_counter().value() == 103.5 + + +@pytest.mark.parametrize('write, code', [ + (lambda ctx, score: ctx.set(1, 'value'), 40003), + (lambda ctx, score: ctx.set('key', object()), 40013), + (lambda ctx, score: ctx.set('key', None), 40013), + (lambda ctx, score: ctx.set('key', float('nan')), 40013), + (lambda ctx, score: ctx.remove(None), 40003), + (lambda ctx, score: score.increment('5'), 40003), + (lambda ctx, score: score.increment(None), 40003), + (lambda ctx, score: score.increment(float('inf')), 40003), + (lambda ctx, score: score.decrement(True), 40003), +], ids=['key-not-str', 'value-object', 'value-none', 'value-nan', 'remove-key-none', 'amount-str', + 'amount-none', 'amount-inf', 'amount-bool']) +async def test_rtbc12_rtbc15_invalid_arguments_raise_at_the_call(write, code): + """RTBC12-RTBC15, RTLM20e1, RTLM21e1, RTLC12e1: a write with an invalid key, value or amount + raises where it is called, and queues nothing.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + with pytest.raises(AblyException) as excinfo: + write(ctx, ctx.get('score').as_live_counter()) + _assert_error(excinfo, code) + ctx.set('name', 'Bob') + + assert _operations(published[0]) == [_map_set('root', 'name', {'string': 'Bob'})] + + +# --- Reads and views ---------------------------------------------------------------------- + +async def test_rtbc3_rtbc4_id_and_get(): + """RTBC3, RTBC4: `id` is the wrapped object's id, None for a primitive; `get` wraps the value + at a key in the context of its type, and is None for a key with no value or on anything but + a map.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.id == 'root' + profile, score, name = ctx.get('profile'), ctx.get('score'), ctx.get('name') + assert isinstance(profile, LiveMapBatchContext) + assert isinstance(score, LiveCounterBatchContext) + assert isinstance(name, PrimitiveBatchContext) + assert profile.id == 'map:profile@1000' + assert score.id == 'counter:score@1000' + assert name.id is None + + assert ctx.get('missing') is None + assert name.get('first') is None + assert score.get('first') is None + + with pytest.raises(AblyException) as excinfo: + ctx.get(1) + _assert_error(excinfo, 40003) + + +async def test_rtbc5_value(): + """RTBC5: a counter context's value is the count; a primitive context's is the primitive, + filtered by `expected` as `PrimitiveInstance.value` filters it.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.get('score').as_live_counter().value() == 100 + assert ctx.get('name').as_primitive().value() == 'Alice' + assert ctx.get('name').as_primitive().value(str) == 'Alice' + assert ctx.get('name').as_primitive().value(float) is None + assert ctx.get('active').as_primitive().value(bool) is True + assert ctx.get('active').as_primitive().value(float) is None + assert ctx.get('avatar').as_primitive().value(bytes) == b'\x01\x02\x03' + assert ctx.get('data').as_primitive().value(dict) == {'tags': ['a', 'b']} + + age = ctx.get('age').as_primitive().value(float) + assert age == 30 + assert type(age) is float + + with pytest.raises(TypeError): + ctx.get('age').as_primitive().value(int) + + +async def test_rtbc6_rtbc9_map_enumeration(): + """RTBC6-RTBC9: a map context's entries and values are contexts of each entry's type, its keys + and size those of the map.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.get('profile').as_live_map().batch() as profile: + entries = profile.entries() + assert [key for key, _ in entries] == ['email', 'nested_counter', 'prefs'] + assert [type(context) for _, context in entries] == [ + PrimitiveBatchContext, LiveCounterBatchContext, LiveMapBatchContext] + values = profile.values() + assert [type(context) for context in values] == [type(context) for _, context in entries] + # A live object has one context per batch (RTBC16c); a primitive gets a new one each time + assert values[1:] == [context for _, context in entries[1:]] + assert values[0].value() == 'alice@example.com' + assert profile.keys() == ['email', 'nested_counter', 'prefs'] + assert profile.size() == 3 + assert entries[1][1].value() == 5 + assert entries[2][1].get('theme').as_primitive().value() == 'dark' + + +async def test_rtbc10_rtbc11_compact(): + """RTBC10, RTBC11: `compact` and `compact_json` snapshot the wrapped value as `Instance` does.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.get('profile').compact() == { + 'email': 'alice@example.com', 'nested_counter': 5, 'prefs': {'theme': 'dark'}} + assert ctx.compact()['avatar'] == b'\x01\x02\x03' + assert ctx.compact_json()['avatar'] == 'AQID' + assert ctx.get('score').compact_json() == 100 + + +async def test_rtbc1a_view_helpers_are_checked(): + """RTBC1a, RTTS9d: a context's view helpers return the context itself for the type it wraps, + and raise 92007 for any other.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + score, name = ctx.get('score'), ctx.get('name') + assert ctx.as_live_map() is ctx + assert score.as_live_counter() is score + assert name.as_primitive() is name + + for view_helper in (ctx.as_live_counter, ctx.as_primitive, score.as_live_map, score.as_primitive, + name.as_live_map, name.as_live_counter): + with pytest.raises(AblyException) as excinfo: + view_helper() + _assert_error(excinfo, 92007) + + assert published == [] + + +async def test_rtbc16c_one_context_per_object(): + """RTBC16c: a batch wraps each object in one context however it is reached, the object the + batch was opened on included.""" + (client, channel, root, mock_ws), published = await _synced() + # A reference from `profile.prefs` back to the root, so that the root is reachable from inside + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'home', {'objectId': 'root'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: 'home' in root.at('profile.prefs').as_live_map().keys(), + description='the back reference to be applied') + + async with root.batch() as ctx: + profile = ctx.get('profile') + assert ctx.get('profile') is profile + assert dict(ctx.entries())['profile'] is profile + assert ctx.get('score') is ctx.get('score') + assert profile.get('prefs').get('home') is ctx + + +class _Contexts(NamedTuple): + root: LiveMapBatchContext + profile: LiveMapBatchContext + score: LiveCounterBatchContext + name: PrimitiveBatchContext + + +# A call of each context method that reads the objects, and of each that writes +_READS = { + 'get': lambda c: c.root.get('name'), + 'compact': lambda c: c.root.compact(), + 'compact_json': lambda c: c.root.compact_json(), + 'entries': lambda c: c.root.entries(), + 'keys': lambda c: c.root.keys(), + 'values': lambda c: c.root.values(), + 'size': lambda c: c.root.size(), + 'child-get': lambda c: c.profile.get('email'), + 'counter-value': lambda c: c.score.value(), + 'primitive-value': lambda c: c.name.value(), +} +_WRITES = { + 'set': lambda c: c.root.set('name', 'Bob'), + 'set-value-type': lambda c: c.root.set('team', LiveMap.create()), + 'remove': lambda c: c.root.remove('name'), + 'child-set': lambda c: c.profile.set('email', 'bob@example.com'), + 'increment': lambda c: c.score.increment(1), + 'decrement': lambda c: c.score.decrement(1), +} +# The calls that check whether the batch is open, but no channel preconditions +_VIEWS = { + 'id': lambda c: c.root.id, + 'primitive-id': lambda c: c.name.id, + 'as_live_map': lambda c: c.root.as_live_map(), + 'as_live_counter': lambda c: c.score.as_live_counter(), + 'as_primitive': lambda c: c.name.as_primitive(), +} + + +def _contexts(ctx): + return _Contexts(ctx, ctx.get('profile'), ctx.get('score'), ctx.get('name')) + + +@pytest.mark.parametrize('call', [*_READS.values(), *_WRITES.values()], ids=[*_READS, *_WRITES]) +async def test_rtbc4b_rtbc15b_contexts_check_preconditions_on_each_call(call): + """RTBC4b-RTBC15b: the contexts check the channel preconditions on every read (RTO25) and + write (RTO26), so once the channel detaches inside the block each of them raises 90001, and + nothing is published.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + contexts = _contexts(ctx) + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + call(contexts) + _assert_error(excinfo, 90001) + + assert published == [] + + +@pytest.mark.parametrize('call', [*_VIEWS.values(), *_READS.values(), *_WRITES.values()], + ids=[*_VIEWS, *_READS, *_WRITES]) +async def test_rtbc16e_contexts_raise_once_the_batch_has_closed(call): + """RTBC16e, RTBC3b-RTBC15c: once its block has exited, every method of a batch's contexts, + those opened inside the block included, raises 40000.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + contexts = _contexts(ctx) + + with pytest.raises(AblyException) as excinfo: + call(contexts) + _assert_error(excinfo, 40000) diff --git a/test/unit/objects/conftest.py b/test/unit/objects/conftest.py new file mode 100644 index 00000000..6b7cf0d3 --- /dev/null +++ b/test/unit/objects/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from test.uts.helpers.client import close_open_clients + + +@pytest.fixture(autouse=True) +async def close_clients(): + """Closes the clients a test built through the UTS helpers, whether or not its assertions held.""" + yield + await close_open_clients() diff --git a/test/unit/objects/realtime_object_test.py b/test/unit/objects/realtime_object_test.py new file mode 100644 index 00000000..bdc2184a --- /dev/null +++ b/test/unit/objects/realtime_object_test.py @@ -0,0 +1,342 @@ +"""A channel's `RealtimeObject`: waits for SYNCED the channel can no longer satisfy, object +messages that fail to decode or to apply, and the lifetime of the GC timer. + +Spec points: RTL33b1, RTO5, RTO5c, RTO8, RTO9, RTO10, RTO10c, RTO18, RTO19, RTO20e1, RTO23c1, +RTO27a. No UTS specification covers these cases, so they are written against the features +specification. + +Several cases race a channel state change against a coroutine resuming. The mock websocket +answers one client message with two frames, which the transport handles one after the other +before the coroutine awaiting the first of them resumes, as when the server's frames arrive in +one read. +""" + +import asyncio +import logging + +import pytest + +from ably.pubsub.objects.defaults import GC_INTERVAL_MS +from ably.pubsub.objects.enums import ObjectsSyncState +from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_connection_state, poll_until +from test.uts.helpers.clock import FakeClock, settle +from test.uts.helpers.mock_websocket import MockWebSocket, channel_error_message +from test.uts.objects.helpers.standard_test_pool import ( + ATTACH, + HAS_OBJECTS, + LWW, + STANDARD_POOL_OBJECTS, + build_ack_message, + build_counter_inc, + build_map_set, + build_object_message, + build_object_state, + objects_attached_message, + objects_channel_options, + objects_client, + objects_connected_message, + remote_serial, + setup_synced_channel, + standard_mock_websocket, +) + +# How long a test waits on an operation it expects to complete or fail, so that one which +# never settles fails with a timeout rather than running into the suite's own +OPERATION_TIMEOUT = 5 + +SCORE = 'counter:score@1000' + + +def _score(channel): + return channel.object._objects_pool[SCORE].data + + +def _assert_sync_wait_failure(excinfo, state_text, cause_code=None): + """Asserts an RTO23c1/RTO20e1 failure: 92008, status 400, naming why, with the channel's error.""" + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + assert state_text in excinfo.value.message + if cause_code is not None: + assert excinfo.value.cause.code == cause_code + + +async def _restart_sync(channel, mock_ws): + """Sends an ATTACHED while attached, which restarts the sync (RTO4) with no OBJECT_SYNC to follow.""" + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor', HAS_OBJECTS)) + await poll_until(lambda: channel.object._sync_state == ObjectsSyncState.SYNCING, + description='the sync to restart') + + +def _ack_then_channel_error(mock_ws_ref): + """An `on_object` handler answering each OBJECT with its ACK and, in the same read, a channel ERROR.""" + def on_object(message): + mock_ws_ref[0].send_to_client(build_ack_message(message['msgSerial'], ['t:9:0'])) + mock_ws_ref[0].send_to_client(channel_error_message('test', 90000, 'Channel error', 400)) + return on_object + + +# --- B1: a wait for SYNCED on a channel that has already left ATTACHED ---------------------- + +async def test_rto23c1_get_fails_when_the_channel_fails_with_its_attached(): + """RTO23c1: an ATTACHED and a channel ERROR read together leave the channel FAILED before + `get()` resumes from its attach, so `get()` begins its wait on a channel that will never + sync. It fails at once with 92008, the channel's error as the cause, rather than waiting + for a transition that has already happened.""" + mock_ws = None + + def on_message_from_client(message): + if message.get('action') == ATTACH: + channel_name = message.get('channel') + mock_ws.send_to_client(objects_attached_message(channel_name, 'sync1:cursor', HAS_OBJECTS)) + mock_ws.send_to_client(channel_error_message(channel_name, 90000, 'Channel error', 400)) + + mock_ws = MockWebSocket( + on_connection_attempt=lambda conn: conn.respond_with_success(objects_connected_message()), + on_message_from_client=on_message_from_client, + ) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + _assert_sync_wait_failure(excinfo, 'failed', cause_code=90000) + assert channel.state == ChannelState.FAILED + assert channel.object._sync_waiters == {} + + +async def test_rto20e1_publish_and_apply_fails_when_the_channel_fails_with_the_ack(): + """RTO20e1: an ACK and a channel ERROR read together leave the channel FAILED before the + write resumes, so its wait for SYNCED begins on a channel that will never sync. It fails at + once with 92008, the channel's error as the cause.""" + mock_ws_ref = [] + mock_ws = standard_mock_websocket(auto_ack=False, on_object=_ack_then_channel_error(mock_ws_ref)) + mock_ws_ref.append(mock_ws) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + await _restart_sync(channel, mock_ws) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(root.set('name', 'Bob'), OPERATION_TIMEOUT) + + _assert_sync_wait_failure(excinfo, 'failed', cause_code=90000) + assert channel.state == ChannelState.FAILED + assert channel.object._sync_waiters == {} + + +async def test_rto20e1_publish_and_apply_fails_when_the_channel_detached_before_the_ack(): + """RTO20e1: a write whose channel detaches while its publish awaits the ACK begins its wait + for SYNCED on a DETACHED channel, and fails at once with 92008.""" + published = [] + mock_ws = standard_mock_websocket(auto_ack=False, on_object=published.append) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + await poll_until(lambda: len(published) == 1, description='the OBJECT to be published') + await channel.detach() + mock_ws.send_to_client(build_ack_message(published[0]['msgSerial'], ['t:9:0'])) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + _assert_sync_wait_failure(excinfo, 'detached') + assert channel.object._sync_waiters == {} + + +async def test_rto27a_publish_and_apply_does_not_apply_once_the_channel_has_failed(): + """RTO27a, RTO20f: a write on a SYNCED channel whose ACK is read together with a channel + ERROR completes, as it was published, but is not applied locally: the channel's FAILED + has cleared the objects' data, and applying it would rebuild that data and notify + subscribers on a failed channel.""" + mock_ws_ref = [] + mock_ws = standard_mock_websocket(auto_ack=False, on_object=_ack_then_channel_error(mock_ws_ref)) + mock_ws_ref.append(mock_ws) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + calls = [] + root.get('score').subscribe(calls.append) + + await asyncio.wait_for(root.get('score').as_live_counter().increment(10), OPERATION_TIMEOUT) + await settle() + + assert channel.state == ChannelState.FAILED + assert _score(channel) == 0 + assert calls == [] + + +async def test_rtl33b1_get_fails_when_the_client_closes_during_the_attach(): + """RTL33b1, RTO23e: closing the client while `get()` attaches moves the ATTACHING channel to + DETACHED, which ends the attach without attaching. `get()` fails with the attach's error, + rather than waiting for a sync on a channel that has detached.""" + mock_ws = MockWebSocket( + on_connection_attempt=lambda conn: conn.respond_with_success(objects_connected_message()), + on_message_from_client=lambda message: None, # never answers the ATTACH + ) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + await await_connection_state(client, ConnectionState.CONNECTED) + + task = asyncio.ensure_future(channel.object.get()) + await poll_until(lambda: channel.state == ChannelState.ATTACHING, description='the channel to attach') + await client.close() + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + assert channel.state == ChannelState.DETACHED + assert channel.object._sync_waiters == {} + + +# --- R1: object messages that fail to decode or to apply ------------------------------------ + +async def test_rto5_an_object_sync_message_that_fails_to_decode_is_skipped(caplog): + """RTO5, RTO5a4: one undecodable value in the last OBJECT_SYNC of a sequence costs only the + ObjectMessage carrying it. The rest of the ProtocolMessage, its cursor included, is + handled, so the sync completes and `get()` returns.""" + bad = build_object_state('map:bad@1000', {'aaa': 't:0'}, map={ + 'semantics': LWW, 'entries': {'k': {'data': {'json': '{not json'}, 'timeserial': 't:0'}}}) + mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + client, channel, root, mock_ws = await asyncio.wait_for( + setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) + + assert channel.object._sync_state == ObjectsSyncState.SYNCED + assert root.get('score').as_live_counter().value() == 100 + assert 'map:bad@1000' not in channel.object._objects_pool + assert any('failed to decode' in record.getMessage() for record in caplog.records) + + +async def test_rto8_an_object_message_that_fails_to_decode_costs_only_itself(caplog): + """RTO8: an OBJECT carrying an undecodable value and a valid operation applies the valid + one.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'data', {'json': '{not json'}, remote_serial(0), 'remote'), + build_counter_inc(SCORE, 1, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: _score(channel) == 101, description='the valid increment to apply') + + assert root.get('data').as_primitive().value() == {'tags': ['a', 'b']} + assert any('failed to decode' in record.getMessage() for record in caplog.records) + + +async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(caplog): + """RTO9: an operation that decodes but raises as it is applied, here an increment by a + string, is logged and skipped, and the operations after it in the OBJECT still apply.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 'one', remote_serial(0), 'remote'), + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: root.get('name').as_primitive().value() == 'Bob', + description='the MAP_SET after the failing increment to apply') + + assert _score(channel) == 100 + assert any('could not be applied' in record.getMessage() for record in caplog.records) + + +async def test_rto5c_an_object_state_that_fails_to_apply_does_not_stall_the_sync(caplog): + """RTO5c: an object state that decodes but raises as it is applied, here a counter whose + count is a string, is logged and skipped, and the sync still completes.""" + bad = build_object_state('counter:bad@1000', {'aaa': 't:0'}, counter={'count': 'many'}) + mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + client, channel, root, mock_ws = await asyncio.wait_for( + setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) + + assert channel.object._sync_state == ObjectsSyncState.SYNCED + assert root.get('score').as_live_counter().value() == 100 + assert any('could not be applied' in record.getMessage() for record in caplog.records) + + +# --- R2: the GC timer and waits of a released channel --------------------------------------- + +async def test_rto10_a_released_channel_stops_its_gc_timer(): + """RTO10a: the GC timer every ATTACHED schedules stops when its channel is released. A + released channel is out of the client's collection, so closing the client never detaches + it; the release itself stops the timer, for good.""" + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + timer = channel.object._gc_timer + assert timer is not None + + client.channels.release('test') + await client.close() + await clock.advance(GC_INTERVAL_MS) + + assert timer.cancelled + assert channel.object._gc_timer is None + + +async def test_rto23c1_a_wait_for_synced_fails_when_its_channel_is_released(): + """RTO23c1: a released channel receives nothing more, so it can never sync. A `get()` + waiting for SYNCED when the channel is released fails with 92008, and so does a later one, + at once.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + task = asyncio.ensure_future(channel.object.get()) + await settle() + assert not task.done() + + client.channels.release('test') + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + _assert_sync_wait_failure(excinfo, 'released') + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + _assert_sync_wait_failure(excinfo, 'released') + + +# --- R4: a GC sweep that raises ------------------------------------------------------------- + +async def test_rto10c_a_gc_sweep_that_raises_is_logged_and_the_next_still_runs(caplog, monkeypatch): + """RTO10c: a sweep that raises is logged, and the next is scheduled all the same; the error + does not escape the timer's callback.""" + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + sweeps = [] + + def collect_garbage(grace_period_ms, now_ms): + sweeps.append(now_ms) + if len(sweeps) == 1: + raise RuntimeError('the sweep failed') + + monkeypatch.setattr(channel.object._objects_pool, 'collect_garbage', collect_garbage) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + await clock.advance(GC_INTERVAL_MS) + assert len(sweeps) == 1 + assert any('GC sweep raised' in record.getMessage() for record in caplog.records) + assert channel.object._gc_timer is not None + + await clock.advance(GC_INTERVAL_MS) + assert len(sweeps) == 2 + + +# --- on and off -------------------------------------------------------------------------- + +@pytest.mark.parametrize('register', [ + lambda realtime_object: realtime_object.on('synched', lambda: None), + lambda realtime_object: realtime_object.off('synched', lambda: None), +]) +def test_rto18_rto19_an_unknown_event_raises_40003(register): + """RTO18, RTO19: `on` and `off` take an `ObjectsEvent`; any other event is an invalid + argument, raised as AblyException 40003 like the other argument errors.""" + with pytest.raises(AblyException) as excinfo: + register(RealtimeObject()) + + assert excinfo.value.code == 40003 + assert excinfo.value.status_code == 400 diff --git a/test/unit/objects/subscription_test.py b/test/unit/objects/subscription_test.py new file mode 100644 index 00000000..17fd56ab --- /dev/null +++ b/test/unit/objects/subscription_test.py @@ -0,0 +1,69 @@ +"""Which listeners a dispatch calls when listeners are added or removed during it. + +Spec points: RTLO4b4c3a (`subscribe` listeners on a live object) and RTO24b (path +subscriptions). The specification does not say how a dispatch treats registrations changed +by its own listeners; the library takes one snapshot per dispatch, so that a listener added +during it is not called by it, and skips a listener removed during it, for both kinds. +""" + +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_inc, + build_map_set, + build_object_message, + remote_serial, + setup_synced_channel, +) + +SCORE = 'counter:score@1000' + + +async def test_rtlo4b4c3a_a_listener_unsubscribed_during_a_dispatch_is_not_called(): + """RTLO4b4c3a: a listener that another listener unsubscribes earlier in the same dispatch + is not called for that update.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + counter = root.get('score').instance().as_live_counter() + calls = [] + second = None + + def first(event): + calls.append('first') + second.unsubscribe() + + counter.subscribe(first) + second = counter.subscribe(lambda event: calls.append('second')) + await counter.increment(1) + await settle() + + assert calls == ['first'] + + +async def test_rto24b_a_subscription_made_during_a_dispatch_is_not_called_by_it(): + """RTO24b: a path subscription made by a listener is not called by the dispatch that made + it, even for a later path to the same object, and is called by the next update.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + # root.alias references the same counter as root.score, so the counter has two paths + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'alias', {'objectId': SCORE}, remote_serial(0), 'remote')])) + await settle() + assert root.get('alias').as_live_counter().value() == 100 + + late_calls = [] + added = [] + + def first(event): + if not added: + added.append(root.subscribe(lambda late_event: late_calls.append(late_event.object.path()))) + + root.get('score').subscribe(first) + root.get('alias').subscribe(first) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 1, remote_serial(1), 'remote')])) + await settle() + assert added + assert late_calls == [] + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 1, remote_serial(2), 'remote')])) + await settle() + assert sorted(late_calls) == ['alias', 'score'] diff --git a/test/unit/objects/typed_views_test.py b/test/unit/objects/typed_views_test.py new file mode 100644 index 00000000..6121dc0f --- /dev/null +++ b/test/unit/objects/typed_views_test.py @@ -0,0 +1,386 @@ +"""The typed views: `type()` and `exists()`, the unchecked path views and the checked instance views. + +Spec points: RTTS2-RTTS10 (ably/specification#491), as LODR-061 binds them: one +`PrimitivePathObject` and `PrimitiveInstance` whose `value(expected)` stands for the six +per-primitive classes (RTTS6h), and `ValueType.UNKNOWN` for a value in none of the known +categories. No UTS specification covers the typed views, so these tests are written against +the features specification and LODR-061. + +Each test drives the standard synced pool over the mock websocket. +""" + +import typing + +import pytest + +from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate +from ably.pubsub.server import ( + Instance, + LiveCounterInstance, + LiveCounterPathObject, + LiveMapInstance, + LiveMapPathObject, + PathObject, + PrimitiveInstance, + PrimitivePathObject, + ValueType, +) +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.objects.helpers.standard_test_pool import ( + build_map_set, + build_object_message, + remote_serial, + setup_synced_channel, +) + +# The type of each value in the standard pool, and of the JSON array each test adds to it +POOL_VALUE_TYPES = { + '': ValueType.LIVE_MAP, + 'name': ValueType.STRING, + 'age': ValueType.NUMBER, + 'active': ValueType.BOOLEAN, + 'avatar': ValueType.BINARY, + 'data': ValueType.JSON_OBJECT, + 'tags': ValueType.JSON_ARRAY, + 'score': ValueType.LIVE_COUNTER, + 'profile': ValueType.LIVE_MAP, + 'profile.prefs.theme': ValueType.STRING, +} + + +async def _synced_root(): + """The standard synced root, with a JSON array added at `tags`.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + await root.set('tags', ['a', 'b']) + return root + + +def _at(root, path): + """The path object at the dotted `path` from `root`, `root` itself for the empty path.""" + return root.at(path) if path else root + + +def _assert_error(excinfo, code): + assert excinfo.value.code == code + assert excinfo.value.status_code == 400 + + +class _FutureLiveObject(LiveObject): + """A live object of a type this SDK does not know, standing in for one a later protocol adds.""" + + def apply_operation(self, object_message, source): + return False + + def replace_data(self, object_message): + return LiveObjectUpdate(noop=True) + + def clear_data(self): + pass + + @staticmethod + def diff(previous_data, new_data, *, for_tombstone=False): + return LiveObjectUpdate(noop=True) + + +# --- type() and exists() ------------------------------------------------------------------ + +@pytest.mark.parametrize('path, value_type', POOL_VALUE_TYPES.items(), ids=[p or 'root' for p in POOL_VALUE_TYPES]) +async def test_rtts4b_type_is_the_type_of_the_resolved_value(path, value_type): + """RTTS4a, RTTS4b, RTTS2a1-RTTS2a8: `type()` is the `ValueType` of the value the path resolves to, + and `exists()` is True for it.""" + root = await _synced_root() + + assert _at(root, path).type() is value_type + assert _at(root, path).exists() is True + + +@pytest.mark.parametrize('path', ['missing', 'name.first', 'score.value', 'data.tags', 'profile.prefs.missing']) +async def test_rtts4b_type_is_none_where_nothing_resolves(path): + """RTTS4a3, RTTS4b3: `type()` is None and `exists()` False for a path that does not resolve, + including one that continues below a primitive or a counter.""" + root = await _synced_root() + + assert root.at(path).type() is None + assert root.at(path).exists() is False + + +async def test_rtts4b_type_follows_the_value_at_the_path(): + """RTTS4b, RTTS8a: a path's type is read when it is asked for, so it follows a change to the + value at the path; an instance's type is fixed by the value it wraps.""" + root = await _synced_root() + score = root.get('score') + instance = score.instance() + + await root.set('score', 'retired') + assert score.type() is ValueType.STRING + assert instance.type is ValueType.LIVE_COUNTER + + await root.remove('score') + assert score.type() is None + assert score.exists() is False + + +async def test_rtts2a9_type_is_unknown_for_a_value_in_no_known_category(): + """RTTS2a9: a value that resolves but falls into none of the known categories is + `ValueType.UNKNOWN`, which is distinct from None for no value, and the typed views read + None from it.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + channel.object._objects_pool['future:thing@1000'] = _FutureLiveObject('future:thing@1000') + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'thing', {'objectId': 'future:thing@1000'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: 'thing' in root.keys(), description='the reference to be applied') + + thing = root.get('thing') + assert thing.type() is ValueType.UNKNOWN + assert thing.exists() is True + assert thing.as_primitive().value() is None + assert thing.as_live_counter().value() is None + assert thing.as_live_map().size() is None + assert thing.as_live_map().keys() == [] + assert thing.instance().type is ValueType.UNKNOWN + + +async def test_rtts4a1_rtts4b1_type_and_exists_check_access_preconditions(): + """RTTS4a1, RTTS4b1, RTO25b: `type()` and `exists()` raise 90001 on a detached channel.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + for read in (root.get('name').type, root.get('name').exists): + with pytest.raises(AblyException) as excinfo: + read() + _assert_error(excinfo, 90001) + + +# --- The unchecked path views ------------------------------------------------------------- + +async def test_rtts5_path_views_rewrap_the_same_path(): + """RTTS5, RTTS6d: `channel.object.get()` is a `LiveMapPathObject`; navigation gives base + `PathObject`s, and each view helper gives the typed view of the same path, without reading + anything.""" + root = await _synced_root() + assert isinstance(root, LiveMapPathObject) + + path = root.get('profile').get('prefs') + assert type(path) is PathObject + for view, view_type in ((path.as_live_map(), LiveMapPathObject), + (path.as_live_counter(), LiveCounterPathObject), + (path.as_primitive(), PrimitivePathObject)): + assert type(view) is view_type + assert view.path() == 'profile.prefs' + + assert path.as_live_map().get('theme').as_primitive().value() == 'dark' + # A view of a path that does not resolve is still a view + assert root.get('missing').as_live_counter().path() == 'missing' + + +async def test_rtts5d1_mismatched_path_views_read_none(): + """RTTS5d, RTTS5d1: a path view never raises for the type at its path; a read through a view + that does not match it gives None, or [] for the collections.""" + root = await _synced_root() + + name_as_map = root.get('name').as_live_map() + assert name_as_map.keys() == [] + assert name_as_map.entries() == [] + assert name_as_map.values() == [] + assert name_as_map.size() is None + + assert root.get('name').as_live_counter().value() is None + assert root.get('score').as_primitive().value() is None + assert root.get('score').as_live_map().size() is None + assert root.as_primitive().value() is None + assert root.as_live_counter().value() is None + assert root.get('missing').as_primitive().value() is None + + +async def test_rtts5d2_writes_through_mismatched_path_views_raise(): + """RTTS5d2: a write through a path view raises 92007 when the path resolves to another type, + and 92005 when it does not resolve.""" + root = await _synced_root() + + for write, code in ((lambda: root.get('name').as_live_map().set('k', 'v'), 92007), + (lambda: root.get('profile').as_live_counter().increment(), 92007), + (lambda: root.get('score').as_live_map().remove('k'), 92007), + (lambda: root.get('missing').as_live_counter().decrement(), 92005), + (lambda: root.at('name.first').as_live_map().set('k', 'v'), 92005)): + with pytest.raises(AblyException) as excinfo: + await write() + _assert_error(excinfo, code) + + assert root.get('name').as_primitive().value() == 'Alice' + + +async def test_rtts6b_counter_view_value_is_only_a_counters(): + """RTTS6b: `LiveCounterPathObject.value()` is the counter's value as a float, and None for + anything else, a number included.""" + root = await _synced_root() + + score = root.get('score').as_live_counter().value() + assert score == 100 + assert type(score) is float + assert root.get('age').as_live_counter().value() is None + assert root.as_live_counter().value() is None + + +@pytest.mark.parametrize('path, expected, value', [ + ('name', str, 'Alice'), + ('name', float, None), + ('age', float, 30.0), + ('age', str, None), + ('age', bool, None), + ('active', bool, True), + ('active', float, None), + ('avatar', bytes, b'\x01\x02\x03'), + ('avatar', str, None), + ('data', dict, {'tags': ['a', 'b']}), + ('data', list, None), + ('tags', list, ['a', 'b']), + ('tags', dict, None), + ('score', float, None), + ('profile', dict, None), + ('missing', str, None), +], ids=lambda param: param.__name__ if isinstance(param, type) else None) +async def test_rtts6c_value_expected_filters_by_type(path, expected, value): + """RTTS6c, RTTS6h: `value(expected)` is the primitive at the path only if it is of the expected + type, judged by its wire type, so a boolean is never a number and a counter or map is never + a primitive.""" + root = await _synced_root() + + result = root.get(path).as_primitive().value(expected) + assert result == value + assert type(result) is type(value) + + +async def test_rtts6c_value_without_expected_is_any_primitive(): + """RTTS6c, RTTS6h: `value()` with no `expected` is the primitive at the path whatever its type, + and None for a live object.""" + root = await _synced_root() + + assert {path: _at(root, path).as_primitive().value() for path in POOL_VALUE_TYPES} == { + '': None, + 'name': 'Alice', + 'age': 30.0, + 'active': True, + 'avatar': b'\x01\x02\x03', + 'data': {'tags': ['a', 'b']}, + 'tags': ['a', 'b'], + 'score': None, + 'profile': None, + 'profile.prefs.theme': 'dark', + } + + +async def test_rtts6c_value_rejects_an_unsupported_expected_type(): + """RTTS6c: `expected` is one of str, float, bool, bytes, list and dict; anything else, a + subclass of one of them included, is a TypeError, raised before the preconditions are + checked.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + age = root.get('age').as_primitive() + name = root.get('name').as_primitive() + + for expected in (int, object, tuple, 'str', typing.List, type('Text', (str,), {})): + with pytest.raises(TypeError): + name.value(expected) + + await channel.detach() + with pytest.raises(TypeError): + age.value(int) + with pytest.raises(AblyException) as excinfo: + age.value(float) + _assert_error(excinfo, 90001) + + +async def test_rtts6c_value_is_a_copy_of_a_json_value(): + """RTTS6c: a JSON value read through a view is the reader's own, so changing it leaves the + object it was read from unchanged.""" + root = await _synced_root() + + data = root.get('data').as_primitive().value(dict) + data['tags'].append('c') + + assert root.get('data').as_primitive().value(dict) == {'tags': ['a', 'b']} + + +# --- The checked instance views ----------------------------------------------------------- + +@pytest.mark.parametrize('path, value_type', POOL_VALUE_TYPES.items(), ids=[p or 'root' for p in POOL_VALUE_TYPES]) +async def test_rtts8a_instance_type_is_the_type_of_the_wrapped_value(path, value_type): + """RTTS7e, RTTS8a: every instance is the typed subclass matching the value it wraps, and its + `type` property is that value's `ValueType`.""" + root = await _synced_root() + + instance = _at(root, path).instance() + + assert instance.type is value_type + expected_class = {ValueType.LIVE_MAP: LiveMapInstance, ValueType.LIVE_COUNTER: LiveCounterInstance}.get( + value_type, PrimitiveInstance) + assert type(instance) is expected_class + + +async def test_rtts9d_instance_views_are_checked(): + """RTTS9, RTTS9d: an instance's view helper returns the instance for the type it wraps and + raises 92007 for any other, where the same helper on a path never raises.""" + root = await _synced_root() + instances = { + 'profile': (root.get('profile').instance(), 'as_live_map'), + 'score': (root.get('score').instance(), 'as_live_counter'), + 'name': (root.get('name').instance(), 'as_primitive'), + } + + for path, (instance, matching_helper) in instances.items(): + assert getattr(instance, matching_helper)() is instance + for helper in {'as_live_map', 'as_live_counter', 'as_primitive'} - {matching_helper}: + with pytest.raises(AblyException) as excinfo: + getattr(instance, helper)() + _assert_error(excinfo, 92007) + # The same request on the path is an unchecked expectation + assert getattr(root.get(path), helper)().path() == path + + +async def test_rtts10_instance_id_is_a_property(): + """RTTS10a, RTTS10b, RTINS3: `id` is a property: the object id of a map or counter instance, + and None for a primitive one.""" + root = await _synced_root() + + assert root.instance().id == 'root' + assert root.get('profile').instance().id == 'map:profile@1000' + assert root.get('score').instance().id == 'counter:score@1000' + assert root.get('name').instance().id is None + + +@pytest.mark.parametrize('typed_class, present, absent', [ + (PathObject, + {'path', 'type', 'exists', 'get', 'at', 'instance', 'compact', 'compact_json', 'subscribe', + 'as_live_map', 'as_live_counter', 'as_primitive'}, + {'value', 'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'batch'}), + (LiveMapPathObject, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'batch'}, + {'value', 'increment', 'decrement'}), + (LiveCounterPathObject, + {'value', 'increment', 'decrement', 'batch'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove'}), + (PrimitivePathObject, + {'value'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'batch'}), + (Instance, + {'id', 'type', 'get', 'compact', 'compact_json', 'as_live_map', 'as_live_counter', 'as_primitive'}, + {'value', 'entries', 'size', 'set', 'remove', 'increment', 'decrement', 'subscribe', 'batch'}), + (LiveMapInstance, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'subscribe', 'batch'}, + {'value', 'increment', 'decrement'}), + (LiveCounterInstance, + {'value', 'increment', 'decrement', 'subscribe', 'batch'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove'}), + (PrimitiveInstance, + {'value'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'subscribe', 'batch'}), +], ids=lambda param: param.__name__ if isinstance(param, type) else '') +def test_rtts3_rtts10_methods_are_partitioned_by_type(typed_class, present, absent): + """RTTS3, RTTS6, RTTS7, RTTS10: each typed view carries the methods of its own type only, so + `subscribe` is on map and counter instances alone and only live objects can be batched.""" + assert {name for name in present if not hasattr(typed_class, name)} == set() + assert {name for name in absent if hasattr(typed_class, name)} == set() diff --git a/test/unit/objects/value_types_test.py b/test/unit/objects/value_types_test.py new file mode 100644 index 00000000..e4a0e368 --- /dev/null +++ b/test/unit/objects/value_types_test.py @@ -0,0 +1,43 @@ +"""The `LiveMap` and `LiveCounter` blueprints a write evaluates: what `create` captures, and when +their contents are validated. + +Spec points: RTLMV3d (a blueprint is immutable), RTLM20e1 and RTLMV4c (value validation), RTO16 +(the server time an object id is generated from). +""" + +import pytest + +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate +from ably.pubsub.util.exceptions import AblyException +from test.uts.objects.helpers.standard_test_pool import setup_synced_channel, time_mock_http + +SERVER_TIME_MS = 1_700_000_123_000 + + +def test_rtlmv3d_a_blueprint_does_not_change_with_the_values_it_was_created_from(): + """RTLMV3d: changing a JSON value or a `bytearray` after passing it to `LiveMap.create` does + not change the blueprint, at any depth.""" + meta = {'tags': ['a']} + raw = bytearray(b'\x01') + blueprint = LiveMap.create({'meta': meta, 'raw': raw}) + + meta['tags'].append('b') + meta['owner'] = 'Bob' + raw.append(2) + + map_create = evaluate(blueprint, SERVER_TIME_MS)[-1].operation.map_create_with_object_id.derived_from + assert map_create.entries['meta'].data.json == {'tags': ['a']} + assert map_create.entries['raw'].data.bytes == b'\x01' + + +async def test_rtlm20e1_an_invalid_blueprint_is_rejected_before_the_server_time_is_fetched(): + """RTLM20e1, RTLMV4c, RTO16: `set` validates a blueprint's contents before it asks the server + for the time its object ids are generated from, so an invalid one costs no request.""" + mock_http = time_mock_http() + client, channel, root, mock_ws = await setup_synced_channel('test', mock_http=mock_http) + + with pytest.raises(AblyException) as excinfo: + await root.set('team', LiveMap.create({'lead': 'Carol', 'points': LiveCounter.create('ten')})) + + assert excinfo.value.code == 40003 + assert [request for request in mock_http.captured_requests if request.path == '/time'] == [] diff --git a/test/uts/README.md b/test/uts/README.md index 8690d290..2a7e5976 100644 --- a/test/uts/README.md +++ b/test/uts/README.md @@ -20,12 +20,13 @@ helpers/ shared infrastructure the specifications assume, and its own tests assets/ fixtures the specifications name, vendored from elsewhere rest/ specifications under uts/rest realtime/ specifications under uts/realtime +objects/ specifications under uts/objects (LiveObjects), with helpers/ of its own ``` Every directory holding tests needs an `__init__.py`, because `test` is a package. Unit tests serve every request from a mock and reach no network — neither the REST -suite nor the realtime one. The seams are installed per client, so a test holds to +suite, the realtime one nor the objects one. The seams are installed per client, so a test holds to that by installing them; one that omits a seam, or that lets the host fallback loop run, reaches the real internet. Integration tests run against a sandbox app. @@ -86,10 +87,11 @@ that closes its own leaves nothing to clean up if it fails first. | [helpers/presence.py](helpers/presence.py) | the presence-map stubs and wire-message builders the presence specifications share | | [helpers/sandbox.py](helpers/sandbox.py) | the sandbox app the integration tier provisions, the presence-fixture cipher, `random_id()` and the JWT signing the auth specification asks a library for | | [helpers/deviations.py](helpers/deviations.py) | the `@deviation` and `@spec_error` gates | +| [objects/helpers/standard_test_pool.py](objects/helpers/standard_test_pool.py) | the LiveObjects fixtures, from `uts/objects/helpers/standard_test_pool.md`; see the objects tier below | `SKILL.md` lists every name in each. The helpers have their own tests -(`helpers/*_test.py`), which are not derived from a specification and are not counted -in the derived-test totals. +(`helpers/*_test.py`, `objects/helpers/standard_test_pool_test.py`), which are not derived +from a specification and are not counted in the derived-test totals. ## The integration tier @@ -253,6 +255,55 @@ the session; the modules here sign an Ably JWT locally instead, through a file-l nothing to the log a test is counting. Tests in this package are given 300 seconds each, as in the REST proxy package. +## The objects tier + +`objects/` holds the LiveObjects specifications, laid out as the other two: +`objects/unit/`, fifteen specifications that reach no network; `objects/integration/`, three +against the sandbox; and `objects/integration/proxy/`, one through `uts-proxy`. Seven of the +unit specifications are pure — they build `InternalLiveCounter`, `InternalLiveMap`, +`ObjectsPool` or a channel-less `RealtimeObject()` and connect nothing — and the other eight +drive `channel.object` over the mock websocket. The specifications are written against an +untyped `PathObject` and `Instance`; the tests reach a type's methods through LODR-061's +views, `root.get('score').as_live_counter().value()`, which [deviations.md](deviations.md) +explains along with the shapes the pure tier adapts to (S-1 to S-5). + +Everything the objects specifications share is in +[objects/helpers/standard_test_pool.py](objects/helpers/standard_test_pool.py), named as +`standard_test_pool.md` names it: + +| Helper | Is | +|---|---| +| `setup_synced_channel(channel_name='test', mock_ws=None, clock=None, modes=OBJECTS_MODES, **client_options)` | the specifications' synced channel: a client on `standard_mock_websocket()`, the channel with both object modes, and `await channel.object.get()`. Unpacks as `client, channel, root, mock_ws`. `setup_synced_channel_no_ack` records OBJECT messages without ACKing them | +| `standard_mock_websocket(auto_ack=True, on_object=None, connected=None, ...)` | answers an ATTACH with an ATTACHED and an OBJECT_SYNC of `STANDARD_POOL_OBJECTS`, a DETACH with a DETACHED, and each OBJECT, after `on_object`, with an ACK | +| `objects_client(mock_ws, clock=None, mock_http=None, **kwargs)` | the client underneath: it connects on its own, speaks JSON, and has `GET /time` answered by `time_mock_http(clock)`, since creating an object reads the server time | +| `objects_channel_options(*modes)`, `objects_connected_message(...)`, `objects_attached_message(channel, channel_serial, flags)` | the channel options and harness messages. Granted modes are `flags` bits, `HAS_OBJECTS \| OBJECT_SUBSCRIBE_FLAG`, not a `modes` list | +| `build_counter_inc`, `build_map_set`, `build_map_remove`, `build_map_clear`, `build_object_delete`, `build_counter_create`, `build_map_create`, `build_object_state`, `build_object_message`, `build_object_sync_message`, `build_ack_message`, `json_value`, `bytes_value` | the builders, each returning the JSON-wire dictionary `send_to_client` takes: camelCase keys, numeric actions, `json` values as JSON strings, `bytes` as base64 | +| `ack_serial(msg_serial, index)`, `remote_serial(index)`, `below_ack_serial(index)`, `POOL_SERIAL` | serials that sort where the specifications need them. A bare `'99'` sorts before `POOL_SERIAL` and is rejected as stale | +| `object_message(wire)`, `object_messages(protocol_message)`, `capture_updates(obj)` | the pure tier's: a builder's output decoded to the internal type, and the updates an object emits, since `apply_operation` returns a boolean | +| `build_public_object_message(message, channel_name)` | the public `ObjectMessage` a subscription event should carry, built independently of the library | +| `assert_unchanged_after_quiescence(count_under_test, control_delivered)` | the specifications' negative-assertion pattern | +| `provision_objects_via_rest(api_key, channel_name, operations)` | REST provisioning for the integration tier, over `X-Ably-Version: 6` | + +The client reads frames on a task of its own, so a test reads state after a `poll_until` on +the frame's effect, never straight after `send_to_client`, and asserts a negative or an exact +count only once a positive control behind it has arrived and the loop has settled. + +```python +async def test_rtpo17_increment_delegates_to_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(25) + + assert root.get('score').as_live_counter().value() == 125 +``` + +`objects/integration/` provisions its own sandbox app from its own `conftest.py`, separate +from the realtime tier's but under the same fixture name, `realtime_sandbox`. All three of +its specifications carry Protocol Variants, so every test takes `use_binary_protocol`, and +each is given 120 seconds, as in the other integration tiers. Its `proxy/` package repeats +the realtime proxy package's `proxy_control` and `proxy_session` fixtures and 300-second +timeout, and its clients sign an Ably JWT locally, as the realtime proxy modules do. + ## Running ``` @@ -262,15 +313,16 @@ uv run --frozen --extra crypto --extra dev pytest test/uts -q The offline tiers alone, which need no network: ``` -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/helpers -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/objects/unit test/uts/helpers test/uts/objects/helpers -q ``` -Either integration tier alone, each of which provisions a sandbox app and needs network +Any integration tier alone, each of which provisions a sandbox app and needs network access: ``` uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration -q uv run --frozen --extra crypto --extra dev pytest test/uts/realtime/integration -q +uv run --frozen --extra crypto --extra dev pytest test/uts/objects/integration -q ``` `--frozen` is required: without it dependency resolution reaches past the diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 498f0ad3..939b4a0b 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -24,35 +24,43 @@ One Test ID can become more than one derived test: five Test IDs in `rest/unit` `error_types_test.py`, `fallback_test.py`, `rest_client_test.py` (two) and `paginated_result_test.py` — assert several independent things under a single id, and the derivation writes a function for each rather than one function with an unrelated -second half. That turns 1132 Test IDs into 1141 derived tests. Going the other way, one +second half. That turns 1471 Test IDs into 1480 derived tests. Going the other way, one derived test can become more than one case: five of the twelve `rest/integration` -specifications and five of the twenty `realtime/integration` ones carry a `## Protocol -Variants` section and run every one of their tests twice, once per protocol, and nine -`rest/unit` tests are parametrized over a table of fixtures the specification gives -inline. That turns 1141 derived tests into 1234 pytest cases. - -Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 904 Test IDs -(913 tests, 1001 cases) pass, 213 (213 tests, 218 cases) are gated behind +specifications, five of the twenty `realtime/integration` ones and the three +`objects/integration` ones outside `proxy` carry a `## Protocol Variants` section and run +every one of their tests twice, once per protocol; nine `rest/unit` tests are +parametrized over a table of fixtures the specification gives inline; and five +`objects/unit` tests are parametrized, three over a table and two over the scenarios the +specification loops through. That turns 1480 derived tests into 1607 pytest cases. + +Of **1471 Test IDs, derived as 1480 tests and run as 1607 pytest cases**: 1243 Test IDs +(1252 tests, 1374 cases) pass, 213 (213 tests, 218 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are -disjoint: two Test IDs, and one parametrized test, have a gated part and a passing part, -and are counted with the gated. Every gated test has been confirmed to fail when -enabled, so none of them passes under both behaviours. 494 of the Test IDs come from -`uts/rest/unit` (503 tests, 536 cases), 481 from `uts/realtime/unit` (481, 481), 84 -from `uts/rest/integration` (84, 122) and 73 from `uts/realtime/integration` (73, 95); 8 -of the REST integration ids (8, 8) and 30 of the realtime ones (30, 30) come from the -`proxy` package within each. Of the gated Test IDs 122 are REST and 91 realtime, which is -126 REST cases and 92 realtime. -A further 122 pytest cases under `helpers/` cover the mock infrastructure itself and are -not derived from a specification. - -The 203 gated Test IDs that record SDK non-compliance — 203 tests, 208 cases — reduce to +disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and +`TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, +while their passing parts count among the passing tests and cases. Every gated test has +been confirmed to fail when enabled, so none of them passes under both behaviours. 494 of +the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 481 from +`uts/realtime/unit` (481, 481), 322 from `uts/objects/unit` (322, 344), 84 from +`uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from +`uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the +realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package +within each. Of the gated Test IDs 123 are REST and 90 realtime, which is 127 REST cases +and 91 realtime. None of the 339 objects Test IDs is gated or unrunnable. +A further 148 pytest cases cover the harness itself and are not derived from a +specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for +the objects builders and the synced-channel mock. + +The 202 gated Test IDs that record SDK non-compliance — 202 tests, 207 cases — reduce to **71 distinct root causes**, 27 on the REST side and 44 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **74 SDK root causes** in all. The remaining 10 gated Test IDs are -specification faults, and reduce to 7. +carries **74 SDK root causes** in all. The remaining 11 gated Test IDs are +specification faults, and reduce to 8. The objects tier records no SDK non-compliance; +its specification faults are all corrected or worked around in tests that pass, and +reduce to the seven candidate issues O.1–O.7, none of them filed. Entries closed by a fix are removed rather than kept as history; `git log` holds that. @@ -75,7 +83,7 @@ from the specification text, so that correcting the specification is all it take it pass, and it is marked `@spec_error` — a skip gated on `RUN_DEVIATIONS`, the same gate `@deviation` uses, with a reason naming the specification rather than the SDK. The suite stays green, a real regression still shows, and the failure is one environment variable -away. Each is filed upstream, in the issues named below. Eleven tests are gated +away. Each but RSC7d's is filed upstream, in the issues named below. Eleven tests are gated this way: | Test | Spec error | @@ -99,7 +107,8 @@ site. The entries below cover both kinds and say which applies. Almost every rea fault is of the second kind, which is why only one realtime test is gated as a spec error while fourteen `realtime/unit` entries appear below. The eight `realtime/integration` faults are of that kind without exception, so none of them is -gated either. +gated either, and so is every `uts/objects` fault: the objects entries close this +section, and not one objects test is gated. The three sections that follow this one record SDK behaviour rather than specification faults. @@ -143,8 +152,11 @@ step or a header label — so the derived test keeps the corrected fixture and p none of them is among the eleven gated above. Not every entry has an issue of its own: the URL-safe base64 alphabet is recorded below -and not filed, because ably-python's own encoding settles the tests either way. Line -references in these entries are against `ably/specification@d9a04ca`. +and not filed, because ably-python's own encoding settles the tests either way. None of +the `uts/objects` faults is filed yet; *Candidate issues* classifies them as O.1–O.7 for a +maintainer to file. Line references in these entries are against +`ably/specification@d9a04ca`, and in the `uts/objects` entries against +`ably/specification@747796f`. ### `/time` is stubbed as an object rather than an array @@ -781,10 +793,213 @@ Filed as [#554](https://github.com/ably/specification/issues/554). | `batch_presence.md` | The restricted-key setup's comment reads "only has access to \"batch-allowed\" channel" while the setup fixes `allowed_channel = "channel6"`; `batch-allowed` appears nowhere in the file. Filed with [#548](https://github.com/ably/specification/issues/548), whose fix replaces the same lines | | `batch_presence.md` | BGR2 says a channel with no members "returns a success result with an empty `presence` array", and the unit tier's mocks all send `'presence': []`. The server sends no `presence` key at all, so an implementation has to default the field for the assertion to hold. The derived test asserts the specification's `length == 0`, with the wire shape in a comment. Filed as [#550](https://github.com/ably/specification/issues/550) | +### The objects unit specifications model three internal interfaces against `objects-features.md` + +**Spec points:** RTLC7, RTLM15, RTO3–RTO9, RTO17, RTLCV4g5, RTLMV4j5, PAOOP3. Not yet +filed; candidate O.1. + +A conforming SDK passes these only by adapting, so each is also a shape adaptation under +*Adapted Tests*, which defines the labels S-1, S-2 and S-4. + +- `internal_live_counter.md` and `internal_live_map.md` read + `update = obj.applyOperation(...)` and, of the same call elsewhere in the same files, + assert `result == true` or `false`. RTLC7g and RTLM15g make the return a boolean and + have the update *emitted* (RTLC7d1a, RTLM15d1a and their siblings), so the files + contradict themselves and `objects-features.md` settles it. (S-1) +- `objects_pool.md`, and the three RTO5c10 cases of `parent_references.md`, put the sync + and apply state machine — `processAttached`, `processObjectSync`, + `processObjectMessage`, `applyObjectMessages`, `syncState` — on `ObjectsPool`, and build + `RealtimeObject(pool: pool)`. RTO3a makes the pool a `Dict`, and + RTO7a, RTO7b and RTO17 give the sync state, the buffer and `appliedOnAckSerials` to + `RealtimeObject`. `uts/objects/PLAN.md` repeats the pool model. (S-2) +- `value_types.md` reads the create an evaluated value type retains as + `operation.counterCreate` / `operation.mapCreate`, and `public_object_message.md` reads + it as `derivedFrom` on the `*CreateWithObjectId` payload. RTLCV4g5 and RTLMV4j5 say only + that it is retained alongside, locally, and is not sent — which the first reading would + put on the wire. The same file's PAOOP3b2 and PAOOP3c2 sources give + `mapCreateWithObjectId` the fields `objectId`, `semantics` and `entries`, and + `counterCreateWithObjectId` `objectId` and `count`, where MCRO2 and CCRO2 define only + `initialValue` and `nonce`. (S-4) + +**Tests affected:** the 42 tests in `internal_live_counter_test.py` (13) and +`internal_live_map_test.py` (29) that read an update or a `false`; 30 in +`objects_pool_test.py` and 3 in `parent_references_test.py`; 8 in `value_types_test.py`; +and `test_paoop3_map_create_from_with_object_id` and +`test_paoop3_counter_create_from_with_object_id`, which build the payload MCRO2 and CCRO2 +describe, with `initialValue` the JSON of the derived payload and the derived payload in +`derived_from`. All pass. + +### Objects fixtures that put a value where the protocol does not + +**Spec points:** RTO2, RTO25a, RTO26a, RTLMV4d, RTLMV4f1, and `standard_test_pool.md`. Not yet +filed; candidate O.2. + +| Spec | Fixture | Why it cannot hold | Derived as | +|---|---|---|---| +| `realtime_object.md` RTO2 `mode-enforcement-0`, RTO25a `access-requires-subscribe-mode-0`, RTO26a `write-requires-publish-mode-0` | An ATTACHED carrying `modes: ["OBJECT_SUBSCRIBE"]` | The modes a server grants travel as bits of the ATTACHED's `flags` (RTL4m, TR3), which is where a client decodes them; a `modes` field is read by nothing | `flags: HAS_OBJECTS \| OBJECT_SUBSCRIBE_FLAG`, through `objects_attached_message(..., flags=...)` | +| `standard_test_pool.md`'s CONNECTED, and the hand-written ones in `internal_live_counter_api.md` and `internal_live_map_api.md` | `connectionId` inside `connectionDetails` | `connectionId` is a ProtocolMessage field, which is where a client reads it | `objects_connected_message()` puts it on the message | +| `value_types.md` `RTLMV4d/map-set-all-types-table-0` | Each row compares the in-memory MapCreate's entry with a decoded value — the json rows expect `[1, "a", null]` and `{"k": "v"}` — except the bytes row, whose `"AQID"` is the JSON wire's base64 | In memory the entry holds bytes; base64 is what `initialValue` carries (RTLMV4f1). ably-js's derived test reads `initialValue` for that reason | `bytes([1, 2, 3])` on the retained MapCreate and `"AQID"` in the decoded `initialValue`, so both readings are pinned. The row should read `bytes([1, 2, 3])` | +| `standard_test_pool.md`'s `provision_objects_via_rest` | "a JSON array of result objects (one per batch entry)" | Measured: one result object for the whole batch, whose `objectIds` lists every object created or updated. `X-Ably-Version` 5 and 6 both accept the v6 operation format | `provision_objects_via_rest` flattens either shape | + +### Objects mock-tier steps read state as though an injected frame were applied at once + +**Spec points:** RTO5c9, RTO10c1b1, RTO17, RTO18d, RTO19, RTO20, RTO20d4, RTO20e, RTO20e1, +RTO23c1, RTO24b1, RTO24c1, RTINS16, RTINS16h, RTLO4b, RTPO19, and at the integration tier +RTO5a2, RTO7, RTO8 and RTO17. Not yet filed; candidate O.3. + +`path_object.md`'s own compact tests say that an SDK may apply an inbound OBJECT message +asynchronously, and wait for it; `standard_test_pool.md` defines a quiescence pattern for +negative assertions. The steps below use neither, and are written as though +`send_to_client` returned with the frame applied. A client that reads frames on a task of +its own, as ably-python does, meets four shapes: + +| Shape | Where | What it costs | +|---|---|---| +| A read or a negative straight after `send_to_client` | `RTO20/echo-dedup-0`, `RTO5c9-RTO20/ack-serials-cleared-on-resync-0`, `RTO19/off-deregisters-0`, `RTO10c1b1/gc-root-never-removed-0`; and `RTINS16h/subscribe-no-side-effects-0`, straight after `subscribe` | Each reads the state from before the frame, so the negatives — no double apply, no callback, the root not removed, no side effect — pass whether or not the SDK is right | +| An operation started before the ATTACHED that should hold it back has been processed | the three `RTO23c1` tests, the two `RTO20e1` tests, `RTO20e/waits-for-synced-0`, `RTO20d4/empty-synthetic-list-skips-sync-wait-0` | `get()` or the write still sees SYNCED and resolves at once, so a **conforming** SDK fails `IS NOT complete`. ably-js's derived tests insert `flushAsync()` at that point | +| A seed sent on the line before the subscription it must not reach | `RTO24c1/depth-filtering-formula-0`, `RTO24b1/multi-path-dispatch-0` | A seed applied after the subscription is delivered to it, and `events.length == 2` reads 3, `events_alias.length == 1` reads 2 | +| An exact count straight after `poll_until(count >= n)` | throughout `live_object_subscribe.md` and `path_object_subscribe.md`, `RTINS16/subscribe-receives-events-0`, and `RTO18d`, `RTO24c1` and `RTO17-RTO18` in `realtime_object.md` | The "exactly" half is a negative, read before a second, wrong event could arrive | + +`objects_faults.md` has the integration tier's version. `RTO5a2-RTO17/sync-interrupted-reconnect-0` +and `RTO7-RTO8/mutations-buffered-during-resync-0` `AWAIT_STATE` DISCONNECTED, which RTN15a +leaves at once — `writing-test-specs.md`'s *Verifying Transient States* says not to — and then +CONNECTED, which the client may still hold from before the drop. `RTO17/server-detach-resync-0` +awaits ATTACHED on a channel that has not yet processed the DETACHED injected onto it, so the +wait returns at once and says nothing about the re-attach. + +The derived tests keep every assertion and add the barrier: a `poll_until` on the frame's +effect before a read; for a negative, a positive control delivered behind the message under +test and `assert_unchanged_after_quiescence` (echo-dedup's control is an increment of the +nested counter, RTO10c1b1's an OBJECT_DELETE of the score counter, RTO19's a second SYNCED +listener); `_restart_sync`, which polls until `_sync_state` is SYNCING before the operation +starts; a poll on the seeded key before subscribing; and `await settle()` between a positive +poll and an exact count. The integration tests wait on a state recorder registered before the +fault, as the realtime tier does. The specifications want `process_pending_events()` after +each injected frame they then read from, and their own quiescence pattern before each negative. + +### Objects tests that pass without the behaviour they are named for + +Not yet filed; candidate O.4, the same class as +[#543](https://github.com/ably/specification/issues/543). + +Where the fix is an assertion the specification omits, the derived test carries every one of +the specification's assertions and adds it, with a `# UTS SPEC ERROR:` comment or a comment +citing the spec point at the site — the RSL2b3 precedent above. "A mutant passed" means +measured: a throwaway reference implementation with that single fault passed every assertion +the specification makes. + +| Test ID | Why it cannot fail | What the derived test adds | +|---|---|---| +| `RTLM8g/map-remove-clear-timeserial-floor-0` | The entry's own timeserial, `"04"`, is later than the MAP_REMOVE's `"03"`, so RTLM8a1 and RTLM9e reject it whether or not the RTLM8g floor exists. A mutant ignoring the floor passed | A MAP_REMOVE of a key with no entry at `"04"`: at or below the floor and newer than the site's serial, so only RTLM8g can reject it | +| `RTLM9b/both-empty-reject-0` | Its own note: RTLO4a3 rejects the empty `ObjectMessage.serial` first, so RTLM9b is never reached | `InternalLiveMap.can_apply_map_operation('', '')` and `(None, None)` are False, checked directly | +| `RTO10/gc-tombstoned-objects-0`, `RTO10c1b1/gc-root-never-removed-0` | `score` reads null from the moment the OBJECT_DELETE applies (RTLM14c, RTLM5d2h), so both pass with no GC sweep at all | That the counter has left `_objects_pool`; for RTO10c1b1, a control OBJECT_DELETE whose removal shows a sweep ran | +| `RTO10b1/gc-grace-period-source-0` | As RTO10, and it advances 6000 ms, less than one GC interval — RTO10a's example is five minutes — so no sweep runs. Its stated claim, that it fails if `objectsGCGracePeriod` is ignored, cannot hold | `_gc_interval_ms = 1000` before `get()`, `_gc_grace_period_ms == 5000`, and the counter still in the pool at 4000 ms and gone at 6000 ms | +| `RTPO6/at-escaped-dots-0` | `at("a\\.b.c").path() == "a\\.b.c"` holds for an `at` that splits on every dot too, since `path()` renders `['a\\', 'b', 'c']` back the same way | `_path == ['a.b', 'c']` | +| `RTPO19/map-clear-triggers-child-events-0` | The MAP_CLEAR's `"99"` sorts before the pool's `"t:0"`, so RTLM24e1 removes nothing; and nothing subscribes at a child path, which is what the title is about | The clear at `remote_serial(0)`, and a subscription at `name` that must receive an event for `name` | +| `RTO4b/attached-no-objects-synced-0` | Lists RTO4b2's "must not create a new root"; a mutant replacing the root passed | `pool['root'] is root` | +| `RTO14/objectid-format-counter-0` | Lists RTO14b1's SHA-256 of `[initialValue]:[nonce]` and asserts only the alphabet; a mutant hashing the wrong input passed | The digest, computed in the test, and that the hash decodes | +| `RTINS16g/subscription-follows-identity-0` | Nothing checks that the MAP_SET repointing `score` applied, so a rejected repoint passes | That `score` resolves to the new counter, that the event is the increment of 10, and that the counter reads 110 | +| `RTO20e/waits-for-synced-0` | `inc_future IS NOT complete` is asserted before the publish could have been sent | Pending only once the OBJECT has left the client and its ACK has been processed | +| `RTO23c/get-waits-for-synced-0` | Asserts only the eventual result, which a `get()` that does not wait also produces | `get()` still pending after the ATTACHED has been processed | +| `RTO20c/missing-site-code-0`, `RTO20d1/null-serial-skipped-0` | "The value stays 100" also holds if nothing was published | Exactly one OBJECT was published | +| `RTO20e1/fails-on-channel-detached-0`, `RTO20e1/fails-on-channel-failed-0` | RTO20e1's text gives the status code, and for FAILED the cause, and the assertions read neither | `status_code == 400`, and `cause.code == 90000` for FAILED | + +Six more are derived as written, there being no assertion to add that is not the +specification's to choose: + +- `objects_pool.md`'s `RTO4-RTO5/attached-during-syncing-resets-0` gives the second sync a new + sequence id, so RTO5a2 discards the old objects whether or not the ATTACHED reset anything; + only removing RTO5a2a's clear made it fail. `objects-features.md` does not say that an + ATTACHED with HAS_OBJECTS resets a sync in flight either; see the features entry below. +- `objects_faults.md`'s `RTO7-RTO8/mutations-buffered-during-resync-0` and + `RTO5-RTO7/publish-during-sync-echo-after-0` are titled for the buffering of RTO7 and RTO8 + and never exercise it. Measured: in the first, client B resumes about 60 ms after the + proxy's disconnect and the server delivers A's write inside the resync data, with no OBJECT + after the resume; in the second, uts-proxy's `delay` holds every later frame behind the + delayed OBJECT_SYNC, so A's OBJECT reaches B once B is SYNCED and is applied directly. + Neither can fail on a missing RTO8a buffer. +- `internal_live_counter_api.md`'s `RTLC11/counter-update-on-inc-0` cites RTLC11b1, the + update's `amount`, and asserts the inbound operation's `counterInc.number`, which is what + `Instance#subscribe`'s event carries; RTLC11b1 is covered at the pure tier only. +- `live_object_subscribe.md`'s `RTLO4b/subscribe-map-update-0` states that the update names + the key and asserts only `updates.length == 1`. +- `objects_pool.md`'s `RTO5c10/empty-sync-parent-refs-0` asserts + `pool["root"].parentReferences == {}`, which holds in every state the test can build; the + rest of the test has substance. +- `objects_lifecycle_test.md`'s `RTO23-RTPO15/set-primitive-propagates-0` subscribes + `events_b` and never reads it. + +### Boundaries no objects test reaches + +**Spec points:** RTLM7h, RTLM8g, RTLM24c, RTLM19a1, RTLC7b, RTLC7c, RTLC7e, RTLM15b, RTLM15c, +RTLM15e, RTLC6a, RTLC6e, RTLM6a, RTLM6e, RTLM7a1, RTLM7a3, RTLM7g, RTLM8a3, RTLM23c, RTLO6b, +RTINS3b. Not yet filed; candidate O.5. + +Missing tests rather than faulty ones. Each but the last was found as a single-point mutant +of a reference implementation that every test in `internal_live_counter.md` and +`internal_live_map.md` let through; the last is a point a specification's table lists and no +assertion reads. The implementation was checked against all of them outside the suite. The +derived tests do not add them, since a test with no Test ID cannot be traced back. + +| Spec point | Not covered | +|---|---| +| RTLM7h, RTLM8g | A clear timeserial **equal** to the operation's serial rejects the operation | +| RTLM24c | An equal clear serial is **not** stale; only a strictly greater one is | +| RTLM19a1 | An entry exactly `gracePeriod` old is released; the fixture uses `gracePeriod + 1` | +| RTLC7c before RTLC7e, RTLM15c before RTLM15e | A tombstoned object still records the CHANNEL serial before it rejects the operation | +| RTLC6a before RTLC6e, RTLM6a before RTLM6e | `replaceData` on a tombstoned object still replaces `siteTimeserials` | +| RTLM7a1 and RTLM7h before RTLM7a3 and RTLM7g, and RTLM8a3 | A MAP_SET or MAP_REMOVE rejected by LWW or by the clear floor keeps the old child's parent reference and creates no zero-value object | +| RTLC7b, RTLM15b | A rejected operation leaves `siteTimeserials` untouched | +| RTLM23c | A no-op entry inside a MAP_CREATE is left out of the merged update | +| RTLO6b | A MAP_REMOVE or `replaceData` tombstoning a map entry with no `serialTimestamp` reads the local clock | +| RTINS3b | `instance.md`'s `RTINS3/id-returns-objectid-0` lists it — `id` is null for a primitive — and asserts nothing about it | + +### Objects labels, commentary and harmless fixture slips + +Not yet filed; candidate O.6, the housekeeping class of +[#532](https://github.com/ably/specification/issues/532). No assertion depends on any of them. + +| Spec | Fault | +|---|---| +| `internal_live_map.md` `RTLM24/map-clear-preserves-newer-0` | The "Spec requirement" line removes entries with a timeserial `<=` the clear's serial; RTLM24e1 removes only strictly older ones, and the same file's `map-clear-basic-0` keeps the equal entry. The fixture has no equal entry | +| `internal_live_map.md` `RTLM14c/tombstoned-ref-yields-null-0` | Titled "MAP_SET referencing tombstoned objectId"; the body seeds `map.data` directly and performs no MAP_SET | +| `realtime_object.md` `RTO20d4/mixed-null-serials-applies-non-null-0` | The commentary has `child` dangle and read undefined per RTLM5d2f1. Applying the MAP_SET creates the zero-value counter (RTLM7g1) and references it (RTLM7g2), so `child` resolves to a counter reading 0. `LiveCounter.create(5)` in the fixture would let the test show the create was skipped | +| `realtime_object.md` `RTO19/off-deregisters-0` | Calls `sub.off()` on what `on()` returned, which is RTO18f2. `RealtimeObject#off` itself, RTO19, is exercised by no test in the file | +| `realtime_object.md` `RTO20/ack-after-echo-no-double-apply-0` | The requirement line credits RTO9a3 with the single apply; with the echo first it is the RTLO4a newness check on the LOCAL apply that rejects the second. The assertion is the same either way | +| `path_object.md` `RTPO6b/at-non-string-throws-0` | Asserts 40003 for a path that is not a string, where `objects-features.md`'s RTPO6 defines no error; RTPO5b gives one for `get` only. Derived as written, with a note at the site, 40003 being what RTPO5b gives the same mistake | +| `internal_live_map.md` `RTLM15e/tombstoned-reject-ops-0` | Sets `isTombstone` without `tombstonedAt`, breaking RTLO3e's invariant that one is set exactly when the other is. Kept as written; the counter twin sets both | +| `internal_live_map.md`, the pool-backed tests (RTLM7g, RTLM14, RTLM14c, every `parentReferences` test) | Build `InternalLiveMap(objectId: "root", pool: pool)` beside the root `ObjectsPool()` already holds (RTO3b1), so the map under test is a second `root`, and the parent references it writes resolve to the pool's. The module docstring says so | +| `objects_pool.md`, `parent_references.md` | Every ATTACHED carries a `channelSerial`, and `RTO5-RTO7/new-sync-keeps-buffer-0` calls `seq2:` new relative to it. RTO4 reads only the HAS_OBJECTS flag; the sync cursor is the OBJECT_SYNC's (RTO5a1) | +| `path_object_subscribe.md` | Mixes `remote_serial(n)` with bare `"98"`, `"99"`, `"100"`, `"50"` and `"serial-1"` from one site. Each applies only because no test sends two of them to one object: `"100"` sorts before `"99"` | +| `value_types.md` `RTLMV4a/evaluate-validates-entries-0` | Carries no language note for `LiveMap.create(null)`, where RTLC12e1's table and RTLMV4b both carry one; see *Adapted Tests* | +| `internal_live_map_api.md` `RTLM20/set-invalid-values-table-0` | Its invalid values — a function, `undefined`, a symbol — are JavaScript's; see *Adapted Tests* | +| `uts/docs/integration-testing.md` | Names `objects_lifecycle_test.md` and `objects_sync_test.md` as the objects specifications with Protocol Variants; `objects_gc_test.md` carries the section too, and runs both | + +### `objects-features.md` gaps the derivation and implementation met + +Not faults in a UTS specification, and not yet filed; candidate O.7. The right-hand column is +what ably-python does, and names ably-js where it agrees. + +| Spec point | Gap | ably-python | +|---|---|---| +| RTLM5d2 | Lists boolean, bytes, number, string and objectId, so an `ObjectData.json` entry falls to RTLM5d2g and reads null, though OD2g defines `json` and `path_object.md`'s compact test expects the decoded value | The decoded JSON, as ably-js | +| OD4c5, OD4d5 | Still put a JSON payload in `string` with `encoding: "json"`, where OD2g and ably-js use the `json` field, a JSON string on both wires | Sends `json`, decodes both | +| RTO4 | With HAS_OBJECTS set, does RTO4c and RTO4d only; whether a sync in flight, its SyncObjectsPool and its sequence id are abandoned is unsaid | Abandons them, as ably-js | +| RTO5a5, RTO5a6 | Whether an OBJECT_SYNC with no parseable `channelSerial` discards a sequence already accumulating | Discards it | +| RTO5d | "Skip processing that `ProtocolMessage`" when an ObjectMessage has no `object`. Skipping the ProtocolMessage loses its cursor and the sync never completes; the UTS and ably-js skip the ObjectMessage | Skips the ObjectMessage | +| RTO4b2 | Resets the root's data to RTLM4c's zero value, and is silent on `clearTimeserial` | Resets it with the data | +| RTO14b2 | Cites RFC 4648 §5, which allows padding; the UTS and ably-js require none | No padding | +| RTO2a | Checks the granted modes when they are "populated"; an ATTACHED with no mode bits decodes to none at all | No granted modes counts as unpopulated, so the requested ones are checked (RTO2b), as ably-js. A real ATTACHED always carries the bits; the standard mock's does not, nor does `objects_faults.md`'s injected `flags: 128` | +| RTPO6 | No error for a path that is not a string | 40003, as RTPO5b gives `get` | +| RTLMV4c1 | A live object, PathObject or Instance as a value is 40013 only in the unmerged [#491](https://github.com/ably/specification/pull/491) | 40013 | +| RTPO20c ([#471](https://github.com/ably/specification/pull/471)) | 92007 when the path "does not resolve to a `LiveObject`"; ably-js resolves through `_resolvePath` and gives **92005** for a path that does not resolve | 92007, as written | +| RTBC16d ([#471](https://github.com/ably/specification/pull/471)) | Publishes a batch through RTO15, so its writes would apply only on their echo | Through RTO20, applying on ACK like every other write, as ably-js | + ## Failing Tests The specification's assertion is preserved and gated behind `@deviation`. Removing -the mark is the only change needed once the SDK behaviour lands. +the mark is the only change needed once the SDK behaviour lands. Nothing derived from +`uts/objects` is gated, so no LiveObjects entry appears in this section. ### Unimplemented features @@ -823,7 +1038,7 @@ on the `recover` parameter the connection never sends. | TP3a, TP3d, TP3g | Presence attributes defaulted from the encapsulating ProtocolMessage. There is no ProtocolMessage type; `ably/realtime/channel.py:751-761` passes the presence array through without context. Matters for synthesized-leave detection and `memberKey` | 3 | | TB4, RTL7h, RTP6e | `attachOnSubscribe`. `ChannelOptions.__init__` (`channeloptions.py:22-26`) takes only `cipher`, `params` and `modes`, and `subscribe()` on the channel, on presence and on annotations all end unconditionally with `await attach()`. This absence also forces the largest single adaptation in the suite, below | 3 | | RSL7 | `RestChannel#setOptions`. The realtime channel implements it; the REST `options` setter expects the kwargs dict `Channels.get` collected, so a `ChannelOptions` raises `TypeError` | 2 | -| RTC1a (TO3h), RTL7f | `echoMessages`, in both the forms RTL7f allows. There is no `echo_messages` client option — passing one raises `TypeError` — and no `echo` connect parameter, so every message the server sends is delivered whatever its `connectionId` | 2 | +| RTL7f | `echoMessages` in its client-filter form. The `echo_messages` option sends `echo=false` (RTC1a), which the server honours, but the client does not itself drop a message carrying its own `connectionId`, and that is the only form a mock-backed test can observe | 1 | | RTP12, RTP12a, RTP12c | `RealtimePresence#history`. The realtime *channel* does delegate `history` to the REST implementation; only the presence object is missing it | 2 | | RTN23c1, RTN23c2 | PING/PONG. `ProtocolMessageAction` stops at `ANNOTATION` (21), so PING (22) and PONG (23) are not modelled and action 22 matches no branch of `on_protocol_message` (`websockettransport.py:37-59`, `:143-199`). The message is counted as activity and discarded | 2 | | TI4, TI1/TI5 | `href` anywhere in the SDK, and `cause` when deserialising. `AblyException.from_dict` and `raise_for_response` read only `message`, `statusCode` and `code`, so both fields are dropped from server errors | 2 | @@ -2183,10 +2398,56 @@ pseudocode is not mistaken for non-compliance. | RTC17 (RSA7b1) | `client.clientId == client.auth.clientId` | `AblyRealtime.client_id` reads `options.client_id` and returns the configured value, while `Auth.__init__` sets `self.__client_id = None` whenever `ably._is_realtime` (`rest/auth.py:34-41`), deferring it to whatever a CONNECTED confirms. The two disagree on a client that has not connected | Open bug. RSA12b only allows the realtime clientId to be unknown while it has not been *configured* | | RTC1f | a `transportParams` boolean appears as `"true"` / `"false"` | `True` / `False`, because `WebSocketTransport.connect` builds the query string with `urllib.parse.urlencode`, which renders each value through `str()` (`websockettransport.py:89`). Integers are unaffected | Open bug. A caller can pass the strings directly, but a bool is what the spec's Stringifiable type admits | +### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-5 + +The objects unit specifications drive internal classes, and ably-python's differ in shape from +the ones they are written against — three of them because the specification contradicts +`objects-features.md` (the first UTS Spec Error entry on the objects tier), two because of how +ably-python is built. Each shape has a label, cited in the module docstring and at the site, so +that one entry here covers every test it affects. The behaviour each test asserts is the +specification's; only the reach into the object differs. + +| Label | The specification | ably-python | Tests | +|---|---|---|---| +| S-1 | `update = obj.applyOperation(...)`, or `result == false` | `apply_operation` returns whether it applied (RTLC7g, RTLM15g) and emits the update through `notify_updated`, no-ops included. `capture_updates(obj)` records them: a specification's `update` is the one update an applied operation emitted, and its `false` is a False return with nothing emitted | the 42 in `internal_live_counter_test.py` and `internal_live_map_test.py` that read an update or a `false` | +| S-2 | The sync and apply state machine on `ObjectsPool`: `processAttached`, `processObjectSync`, `processObjectMessage`, `applyObjectMessages`, `syncState` | On `RealtimeObject`, as in `objects-features.md`: `_on_attached(has_objects)`, `_handle_object_sync_messages(messages, channel_serial)`, `_handle_object_messages`, `_apply_object_messages`, `_sync_state`. A bare `ObjectsPool()` that is then driven is the pool a standalone `RealtimeObject()` holds; `RealtimeObject(pool: pool)` is `RealtimeObject(pool=pool)` | 30 of the 31 in `objects_pool_test.py`, every one but RTO3's; the three RTO5c10 cases in `parent_references_test.py` | +| S-3 | `evaluate(vt)` | `evaluate(vt, timestamp_ms)`: evaluation needs the server time (RTLCV4e, RTLMV4h), which is an async read (RTO16), so the caller fetches it and passes it in. The tests pass a fixed one | the 13 in `value_types_test.py` that evaluate | +| S-4 | The create an evaluated value type retains, as `operation.counterCreate` / `operation.mapCreate` | `operation.resolved_counter_create` / `resolved_map_create`, the `derived_from` of the `*CreateWithObjectId` payload. The two retention tests also assert that `counterCreate` / `mapCreate` is absent from `operation.to_dict()`, which is what "not sent" means and what keeps this reading honest | `test_rtlcv4g5_retains_local_counter_create`, `test_rtlcv4_evaluate_zero_count`, `test_rtlmv4j5_retains_local_map_create`, `test_rtlmv4d_entry_value_types`, `test_rtlmv4d1_nested_value_types`, `test_rtlmv4a_evaluate_validates_entries`, `test_rtlmv4e2_empty_entries`, `test_rtlmv4d_map_set_all_types_table` | +| S-5 | Three reads on the untyped `Instance` that answer null or fail: `value()` on a map (RTINS4d), `size()` on a counter (RTINS9c), and `subscribe` on a primitive failing 92007 (RTINS16c) | The typed instances have no such method (RTTS10a–c; `PrimitiveInstance` has no `subscribe` at all, RTTS7b), and the checked views that do have it refuse the wrapped type with 92007 (RTTS9d). The tests assert the wrapped `type`, that the method is absent, and the 92007 from each view in its place | `test_rtins4_value_counter` and `test_rtins9_size` (their second halves), `test_rtins16c_subscribe_primitive_throws` | + +**Status:** shape adaptations, not defects. S-1, S-2 and S-4 close when the specifications are +corrected (candidate O.1); S-3 and S-5 are how ably-python is built. + +### LiveObjects: inputs that do not exist in Python + +| Test | The specification | The test | Status | +|---|---|---|---| +| `test_rtlmv4a_evaluate_validates_entries` | `LiveMap.create(null)` fails 40003 | `LiveMap.create(entries=None)` is LODR-061's default, so `None` is the omitted argument and evaluates to an empty map, which the test pins. The reachable half of RTLMV4a, a non-dict, is asserted as 40003 | Flips if `create` takes a sentinel default instead; the specification carries no language note here (see O.6) | +| `test_rtlm20_set_invalid_values_table` | A function, `undefined` and a symbol are 40013 | `lambda: None`, `None` and `object()`, each 40013; the parametrize ids keep the specification's labels | JavaScript values given their nearest Python counterparts | +| `test_rtlc12e1_increment_invalid_amounts_table` | The null row applies "where null is distinguishable" | It applies: Python tells `increment(None)` from `increment()`, whose default is 1. `True` is in the table and fails 40003 though `bool` is an `int`; NaN and the infinities are `float('nan')`, `float('inf')` and `float('-inf')` | The specification's own reading, for Python | + +### LiveObjects translation notes + +Not deviations, recorded so the difference from the pseudocode is not mistaken for one. The +largest, the typed views, has its own section at the end of this file. + +| Subject | Note | +|---|---| +| Granted modes | The modes a server grants are bits of the ATTACHED `flags`, so the specifications' `modes: [...]` is `flags=HAS_OBJECTS \| OBJECT_SUBSCRIBE_FLAG`. The standard mock's ATTACHED carries HAS_OBJECTS alone, which leaves `channel.modes` empty, and RTO2 then checks the modes the channel *requested* — which is why every objects channel asks for both through `objects_channel_options()`, and `get()` raises 40024 on one that asked for neither. See the RTO2a row of the features entry above | +| Wire values | The mock tier speaks JSON. A `json` value is a JSON-encoded **string** on the v6 wire (OD2g), so a captured `mapSet.value.json` is compared after `json.loads`; a `bytes` value is base64; an action the specification writes by name compares equal to its `ObjectOperationAction`, an `IntEnum` | +| Numbers | Every number decodes to a `float`, and a locally created value holds one too, so that a value reads the same before and after its echo; a number is read with `value(float)`. A boolean is asserted with `is True` / `is False`, since `True == 1` | +| The public message | `PublicObjectMessage.fromObjectMessage(source, channel)` is `publicmessage.ObjectMessage._from_internal(source, channel_name)`. The public and internal `ObjectMessage` share a name, so the tests reach the public one through the `publicmessage` module | +| Channel state | `channel.object.processChannelState(S)` is `channel.object._act_on_channel_state(ChannelState.S)`, which acts on its argument: RTO23c1's SUSPENDED case and RTO27 call it while the channel is still ATTACHED | +| Loops | RTO17-RTO18's scenario loop and RTO27's DETACHED, FAILED and SUSPENDED blocks are one parametrized test each, keeping one Test ID | +| Listeners | `subscribe` on a path or an instance takes any callable, a bound built-in such as `events.append` included — unlike the channel and connection `EventEmitter` | +| ACK serials | `ack-after-echo-no-double-apply` ACKs `msgSerial` 0, relying, as the specification does, on the first OBJECT of a fresh connection taking it (RTN7b) | +| Integration | An un-awaited `channel.attach()` or write is a task, awaited later inside `pytest.raises` where the specification expects it to fail. `poll_until_success` treats only an `AblyException` as "not yet", so that any other error fails at once. `channel.object.get()`, which the lifecycle and sync specifications leave unbounded, is bounded at the fifteen seconds `objects_gc_test.md` gives it | + ## Mock Infrastructure Limitations Tests that cannot be implemented as written, kept as skipped stubs carrying their Test -IDs so the specification's coverage is still accounted for. Fifteen in total. Two of the +IDs so the specification's coverage is still accounted for. Fifteen in total, none of +them from `uts/objects`. Two of the entries are caused by the SDK rather than by the mock, but they land here because the effect is the same: no test can observe the behaviour. @@ -2394,6 +2655,81 @@ is not shared between them. duplicates, on both protocols. The test asserts the full set rather than only the specification's `>= 5`, since the fixture is fixed. +### The sandbox delivers the echo before the ACK, so RTO9a3 is reached at the mock tier only + +Against the sandbox the echo of a client's own write arrived before its ACK in every trace, +the first write included, where both frames came in one read with the OBJECT first. The echo +is therefore applied as a CHANNEL operation (RTO8b), the ACK's LOCAL apply is rejected by the +RTLO4a site-serial check, and `_applied_on_ack_serials` (RTO9a3) is never written on that path. +Both orders are correct per the specification, and nothing is applied twice — a probe writing +every value type in every protocol combination found the writer's state unchanged by its +echoes. The ACK-first order, where RTO9a3 does the work, is covered at the mock tier, whose +standard mock ACKs before any echo is injected (`test_rto20_echo_dedup`, and RTO9a3 in +`objects_pool_test.py`). + +The same traces show the sandbox pacing ACKs at about one per 500 ms per connection: the first +ACK arrives with the echo, and later ones on a half-second tick. An awaited write after the +first can take that long without anything being wrong, which is why the GC integration tests +take over a second each. + +### A Python-formatted `initialValue` is accepted, because the server hashes the string it receives + +An object id is a hash over `[initialValue]:[nonce]` (RTO14b1), and ably-python formats +`initialValue` as Python's `json.dumps` does — `{"count":42.0}` where ably-js writes +`{"count":42}`, and non-ASCII as `\uXXXX` escapes where ably-js writes raw UTF-8 — so it looked +as though the server, or another SDK, would compute a different id. Measured against the +sandbox's REST objects endpoint with client-generated ids: Python's formatting, ably-js's +formatting and raw UTF-8 are each accepted (201) when the id is hashed over the string sent, +while an id hashed over `{"count":7.0}` and sent with `{"count":7}` is rejected 400/92000, +"object id does not match expected value". The server hashes the exact string it receives. The +evaluation computes the id from the one string it then sends, so the formatting cannot diverge, +and the integration tier confirms it over realtime in both protocols. + +### A sync cursor can itself contain `:`, and splitting on the first one is right + +A sync the sandbox split into twenty OBJECT_SYNC messages carried channelSerials of the shape +`4d58ad59:map:0TeL…@1791551583365` and ended on `4d58ad59:`, so the cursor contains `:` too. +RTO5a1 splits on the first `:`, which yields the right sequence id; late joiners on both +protocols matched the writer exactly. + +### `RTO5a2-RTO17/sync-interrupted-reconnect-0` does interrupt a sync + +Its two siblings in `objects_faults.md` never reach what they are named for (see the objects +entry under UTS Spec Errors), so this one was measured too, and it does: the proxy drops the +first OBJECT_SYNC and closes the socket, the client resumes on the same connectionId, receives +an ATTACHED carrying RESUMED and HAS_OBJECTS and a fresh OBJECT_SYNC, and `get()` resolves from +that. The ATTACHED arrives on a channel already ATTACHED, which emits no state change, so the +re-sync depends on the channel handing every ATTACHED it accepts to the objects — which it does. + +### Where ably-python departs from ably-js, it follows `objects-features.md` + +Recorded because anyone comparing the two implementations will meet these first. + +| Spec point | ably-js | ably-python | +|---|---|---| +| RTO5c2, RTO5c10 | `_applySync` returns early when the SyncObjectsPool is empty, skipping both the removal of absent objects and the parent-reference rebuild | No early return; both run, as written | +| RTO5e | Moves to SYNCING only on a new sequence, so an OBJECT_SYNC with no sequence id while SYNCED emits nothing | Applies RTO5e to every OBJECT_SYNC, emitting SYNCING then SYNCED | +| RTO5a6 | `^([\w-]+):(.*)$`, so `:x` reads as having no channelSerial | Malformed means "does not contain the `:` separator", as written | +| RTO2 | Requires a granted mode to be among the requested ones as well | RTO2b applies only "otherwise", when no modes were granted | +| RTLC9 | A COUNTER_INC with no `counterInc` is not applied | The RTLC9h no-op, applied, reading RTLC7d5's "passing in `operation.counterInc`" as written | +| RTLMV4c | A non-finite number in a map's entries is sent as `null` | 40013: it has no JSON representation (OD4d3), and `initialValue` is always JSON | + +### More `Task was destroyed` warnings, and a GC timer on every attached channel, are not faults + +A plain run of `test/uts` prints `Task was destroyed but it is pending!` 174 times where the +baseline printed 80. Every one is a `ConnectionManager.connect_base()` task, the teardown +warning the baseline prints already, and the rise comes from the objects cases that connect a +mock-backed client; no new kind of warning appears. + +The GC timer is scheduled on the first ATTACHED of every channel, whether or not it uses +objects (RTO10a; ably-js does the same whenever the plugin is loaded), and cancelled on +DETACHED and FAILED, so a client's `close()` stops it. A non-objects test advancing a +`FakeClock` by five minutes therefore runs one empty sweep. `Channels.release()` stops it +too, although release does not detach — the RTS4a defect under *Failing Tests*. And a burst +of creates on a channel that has +not yet read the server time each request `/time`, until the first answer persists the offset +(RTO16a); wasteful, and not wrong. + ## Candidate issues `writing-derived-tests.md` asks for the deviations above to be classified into distinct @@ -2725,7 +3061,7 @@ Each row is one feature and one issue. None is a bug in existing code. | Retry backoff, jitter and `retryIn` on both state-change types | RTB1, RTB1a, RTB1b | 4 | `connection/backoff_jitter_test.py` | | `RealtimeChannel#whenState` (the connection has a private equivalent) | RTL25, RTL25a, RTL25b | 4 | `channels/channel_when_state_test.py` | | `attachOnSubscribe` on `ChannelOptions`. Also forces the suite's largest adaptation — 21 tests attach explicitly to work around it | TB4, RTL7h, RTP6e | 3 | `channels/channel_subscribe_test.py -k rtl7h`, `channels/channel_options_test.py -k tb4`, `presence/realtime_presence_subscribe_test.py -k rtp6e` | -| `echoMessages`, in both the client-filter and `echo`-parameter forms | RTC1a, RTL7f | 2 | `client/realtime_client_test.py -k rtc1a`, `channels/channel_subscribe_test.py -k rtl7f` | +| `echoMessages` in its client-filter form; the `echo` connect parameter is sent | RTL7f | 1 | `channels/channel_subscribe_test.py -k rtl7f` | | `RealtimePresence#history` (the realtime *channel* does delegate `history`) | RTP12, RTP12a, RTP12c | 2 | `presence/realtime_presence_history_test.py` | | PING/PONG handling — actions 22 and 23 are not modelled | RTN23c1, RTN23c2 | 2 | `connection/heartbeat_test.py -k rtn23c1` | | The `heartbeats` connect parameter. Binding on ably-python, which cannot observe ping frames | RTN23a | 1 | `connection/heartbeat_test.py -k rtn23a_heartbeats_true` | @@ -2829,6 +3165,87 @@ the unit tier adapts to that; RTN14b does not, which is what makes this a defect than a choice. `test/uts/realtime/integration/proxy/connection_open_failures_test.py -k rtn14b` +### From the objects derivation + +These are against `ably/specification`, not ably-python: the objects tier found no SDK +non-compliance, and each item is a fault in `uts/objects` or, for O.7, a gap in +`objects-features.md`. **None is filed.** Each matches an entry under *UTS Spec Errors*, which +carries the detail, and the command after it runs the derived tests that correct or work +around it — they pass, and are the reference for what the specification should say; +`RUN_DEVIATIONS` makes no difference to them. They are ranked by what an SDK deriving from the +specification as written would suffer: first where a conforming SDK fails or has to adapt +(O.1–O.3), then where a non-conforming one passes (O.4, O.5), then housekeeping (O.6), with the +features-spec gaps last, being questions for a different document. + +**O.1 The unit specifications model three internal interfaces against `objects-features.md`.** +RTLC7g, RTLM15g, RTO3a, RTO7, RTO17, RTLCV4g5, RTLMV4j5, MCRO2, CCRO2. +`internal_live_counter.md` and `internal_live_map.md` take an update from `applyOperation` +while asserting, of the same call, the boolean RTLC7g and RTLM15g say it returns; +`objects_pool.md` puts the sync state machine on `ObjectsPool`, and `PLAN.md` repeats it; +`value_types.md` and `public_object_message.md` read the retained create in two different +places, one of which would put it on the wire, and the latter gives `*CreateWithObjectId` fields +MCRO2 and CCRO2 do not define. Every SDK has to adapt all of it — S-1, S-2 and S-4 here, and +ably-js's derived suite likewise. One issue: write the files against the features spec's shapes. +`test/uts/objects/unit/internal_live_map_test.py test/uts/objects/unit/objects_pool_test.py test/uts/objects/unit/value_types_test.py test/uts/objects/unit/public_object_message_test.py` + +**O.2 Fixtures put values where the protocol does not.** RTO2, RTO25a, RTO26a, RTLMV4d, +RTLMV4f1. Granted modes written as an ATTACHED `modes: [...]` field rather than as `flags` bits; +`connectionId` inside `connectionDetails` in the helper specification and two hand-written +mocks; RTLMV4d's bytes row comparing an in-memory entry with its base64 wire form; and the +provisioning comment's one result per batch entry, where the server answers one result for +the batch. A conforming SDK fails the first three as written. +`test/uts/objects/unit/realtime_object_test.py test/uts/objects/unit/value_types_test.py -k "rto2_mode or rto25a or rto26a or rtlmv4d_map_set_all_types_table"` + +**O.3 Steps read state as though an injected frame were applied at once.** RTO5c9, RTO10c1b1, +RTO17, RTO18d, RTO19, RTO20, RTO20d4, RTO20e, RTO20e1, RTO23c1, RTO24b1, RTO24c1, RTINS16, +RTINS16h, RTLO4b, RTPO19, and at the integration tier RTO5a2, RTO7, RTO8, RTO17. An SDK that +processes frames on a task of its own **fails** the three RTO23c1 tests, the two RTO20e1 tests, +RTO20e and RTO20d4's empty-list test as written — the operation starts before the re-sync +ATTACHED has been processed, sees SYNCED and resolves — while the negatives of RTO20's +echo-dedup, RTO19, RTO10c1b1 and RTINS16h pass whatever the SDK does, and every exact count +after a `poll_until(>= n)` is half a negative. `path_object.md` already waits for an inbound +frame in its compact tests and `standard_test_pool.md` already defines the quiescence pattern; +the fix is to use them throughout, with `process_pending_events()` after each frame that is +read from, and, in `objects_faults.md`, a recorded state sequence in place of `AWAIT_STATE` for +DISCONNECTED and for a re-attach. +`test/uts/objects/unit/realtime_object_test.py test/uts/objects/unit/instance_test.py -k "rto23c1 or rto20e or rto20d4_empty or rto20_echo or rto19 or rto10c1b1 or rto5c9_rto20 or rtins16h"` +`test/uts/objects/integration/proxy/objects_faults_test.py` + +**O.4 Tests that pass without the behaviour they are named for.** RTLM8g, RTLM9b, RTO10, +RTO10b1, RTO10c1b1, RTPO6b, RTPO19 with RTO24b2a2, RTO4b2, RTO14b1, RTINS16g, RTO20e, RTO23c, +RTO20c, RTO20d1, RTO20e1; and, derived as written, RTO4 with RTO5a2, RTO7 and RTO8 through the +proxy, RTLC11b1, RTLO4b, RTO5c10 and the lifecycle test's unread `events_b`. Sixteen derived +tests carry the assertion or the direct check their specification omits. The sharpest are the GC +tests, which read a value that is null with no GC at all — RTO10b1 also advances less than one +GC interval — and the two `objects_faults.md` tests titled for RTO8a's buffer, which +measurement shows never buffer. Same class as +[#543](https://github.com/ably/specification/issues/543). +`test/uts/objects/unit -k "rtlm8g or rtlm9b or rto10 or rtpo6_at_escaped or rtpo19_map_clear or rto4b_attached or rto14_objectid_format_counter or rtins16g or rto20e or rto23c_get_waits or rto20c_missing or rto20d1"` + +**O.5 Boundaries no test reaches.** RTLM7h, RTLM8g, RTLM24c, RTLM19a1, RTLC7c, RTLM15c, +RTLC6a, RTLM6a, RTLM7a3, RTLC7b, RTLM15b, RTLM23c, RTLO6b, RTINS3b. Equal serials against the +clear floor, an entry exactly one grace period old, the order of serial bookkeeping against the +tombstone checks, a rejected MAP_SET's parent references, the serials a rejected operation +leaves, MAP_CREATE no-ops, the clock a map-entry tombstone reads, and a primitive instance's +`id`. Each but the last was found as a mutant every existing test lets through, so each new +test can be written against that mutant. No reproduction: there is no test to run. + +**O.6 Labels, commentary and harmless fixture slips.** The thirteen rows of the objects +housekeeping entry: RTLM24's `<=`, RTLM14c's title, RTO20d4's commentary, an RTO19 test that +exercises RTO18f2, RTO20's credit to RTO9a3, RTPO6b's undefined 40003, RTLM15e's half-set +tombstone, the second `root` beside the pool's, the ATTACHED `channelSerial`s, fragile bare +serials, two missing language notes, and `integration-testing.md`'s Protocol Variants list. +None changes an outcome. Same class as +[#532](https://github.com/ably/specification/issues/532). + +**O.7 `objects-features.md` gaps.** RTLM5d2, OD4c5, OD4d5, RTO4, RTO5a5, RTO5a6, RTO5d, RTO4b2, +RTO14b2, RTO2a, RTPO6; and, on the unmerged pull requests, RTLMV4c1 +([#491](https://github.com/ably/specification/pull/491)) and RTPO20c and RTBC16d +([#471](https://github.com/ably/specification/pull/471)). The widest reaching are RTLM5d2, +under which every JSON map entry reads null, and RTO5d, under which one ObjectMessage with no +`object` stalls the whole sync. The two #471 points want raising on that pull request before it +merges, since ably-js already disagrees with its text on the first. + ## How the specifications are adopted here Choices about the approach, as against the behaviour recorded above. @@ -3182,6 +3599,102 @@ and each message still gets a distinct id, so only the setup latency differs. Ea then polls history until the expected count is visible before paginating, which is the specifications' own `poll_until`, because history is not immediately consistent. +### The objects tier mirrors the other two, with a pure tier inside its unit tier + +`uts/objects/unit/.md` becomes `test/uts/objects/unit/_test.py`, +`uts/objects/integration/_test.md` becomes `test/uts/objects/integration/_test.py`, +and `integration/proxy/objects_faults.md` becomes `integration/proxy/objects_faults_test.py`, +whose Test IDs read `objects/proxy/...`. `uts/objects/helpers/standard_test_pool.md` becomes +`test/uts/objects/helpers/standard_test_pool.py`, with tests of its own beside it, which are +the 18 harness cases the header counts separately. + +Seven of the fifteen unit specifications — `internal_live_counter`, `internal_live_map`, +`objects_pool`, `parent_references`, `object_id`, `value_types` and `public_object_message`, +161 Test IDs — construct internal objects and connect nothing. The other eight, 161 Test IDs +more, drive `channel.object` over the mock websocket. The specifications draw no such line, +but it decides which shapes apply: S-1 to S-4 are pure-tier adaptations, and the mock tier +reaches the library only through its public surface and a handful of underscore readers. + +### The specifications' untyped `PathObject` and `Instance` become LODR-061's typed views + +The specifications are written against the merged features spec's `PathObject` and +`Instance`, each one class carrying every method. ably-python follows LODR-061 and the RTTS +partition of [#491](https://github.com/ably/specification/pull/491): navigation, `instance`, +`compact`, `compact_json` and `subscribe` are on the base `PathObject`, and a type-specific +method is reached through the view for that type. So `root.get("score").value()` is +`root.get('score').as_live_counter().value()`, `root.get("name").value()` is +`root.get('name').as_primitive().value()`, and `pathObject.set(k, v)` is +`await path_object.as_live_map().set(k, v)`; `root` is already a `LiveMapPathObject` and +needs no view. A path's views are unchecked — a view of the wrong type reads `None`, or `[]` +for a collection (RTTS5d1), and a write through it raises 92007 or 92005 (RTTS5d2) — so +where a specification expects the untyped `value()` to be null, the test asserts it through +both value views. An `Instance`'s views are checked and raise 92007 (RTTS9d). + +That is a difference of spelling, which `writing-derived-tests.md` says is not a deviation, +and nothing above records it test by test. Where the partition makes a read unreachable +rather than differently spelled, the test is an S-5 adaptation. Mutations are `async` and +awaited; reads, navigation, the views and `subscribe` are synchronous; `entries()` yields +`(key, value)` tuples; `Instance.id` and `Instance.type` are properties. + +### A standalone `RealtimeObject` drives the pure tier + +`objects-features.md` puts the sync state machine on `RealtimeObject` (S-2), so the pure +tier needs one without a channel. `RealtimeObject()` built with none, or +`RealtimeObject(pool=pool)` around a pool a test filled, is a working state machine: `get()` +waits for SYNCED and returns the root, `_publish` raises 40000, the server time is the local +clock, the channel-mode and channel-state preconditions check nothing, and a path +subscription's events carry no channel name. None of that is reachable through the public +API; it exists so that a pure test exercises the code the mock tier runs. Members of a +public class that LODR-061 does not name carry a leading underscore — `_objects_pool`, +`_sync_state`, `_on_attached`, `_gc_interval_ms` — and the tests reach them by those names. + +### The objects mock tier speaks JSON, and answers `/time` + +`setup_synced_channel` connects through `objects_client`, which sets +`use_binary_protocol=False`, so the builders' dictionaries go over the mock exactly as the +specification describes the JSON wire: camelCase keys, numeric actions and semantics, a `json` +value as a JSON-encoded string and `bytes` as base64. The msgpack wire is covered by the +integration tier, which runs its three specifications under both protocols, and by +`standard_test_pool_test.py`, which round-trips every wire type through both encodings. + +Creating a `LiveCounter` or `LiveMap` reads the server time (RTO16), which is a REST request, +so `objects_client` also installs a `MockHttpClient` answering `GET /time` from the test's +clock and 404 to anything else. Without it a mock-backed client would reach the network for +the time. The harness CONNECTED carries a `siteCode`, an `objectsGCGracePeriod` and a +`maxIdleInterval` of 0, so that a `FakeClock` test — the GC tests, chiefly — has no idle timer +to fire. + +### The objects integration tier provisions its own sandbox app, and its proxy clients sign a JWT + +`test/uts/objects/integration/conftest.py` provisions a sandbox app for the tier, separate +from the realtime tier's, whose fixtures are not visible from `test/uts/objects`; it is +called `realtime_sandbox` all the same, so the tests read as the realtime tier's do. All three +specifications outside `proxy` carry Protocol Variants, so every test there takes the +`use_binary_protocol` fixture. Objects are provisioned over REST with `X-Ably-Version: 6`, in +JSON whichever protocol the realtime client speaks. The proxy package repeats the realtime +tier's `proxy_control` and `proxy_session` fixtures and its 300-second marker, prepended so +that it wins over the parent's 120. A client under test there authenticates with an +`auth_callback` returning a locally signed Ably JWT, which costs no round trip and so puts +nothing in the event log beside the frames a test counts; a second client, where a test has +one, goes straight to the sandbox with the key, as the specification builds it. + +### Batch and the typed views are tested outside `test/uts`, for want of a specification + +No UTS specification covers batched writes (RTPO20, RTINS17, RTBC1–RTBC16, in the unmerged +[#471](https://github.com/ably/specification/pull/471)) or the typed views (RTTS2–RTTS10, in +#491). Both are implemented, so both are tested, in `test/unit/objects/`: `batch_test.py`, 29 +functions run as 76 cases, and `typed_views_test.py`, 17 functions run as 61. They are written +against `objects-features.md`, the two pull requests and LODR-061, carry no `# UTS:` comment, +and are not counted in the header. Beside them, `realtime_object_test.py`, +`subscription_test.py` and `value_types_test.py` (17 functions, 18 cases) hold regression +tests for behaviour no specification test reaches: a sync wait on a channel that is already +detached, failed or released, an object message that fails to decode, a GC sweep that raises, +and the validation of a creation value before the server time is read. They drive the standard synced pool over the mock +websocket with this tier's helpers, and their own `conftest.py` closes clients as +`test/uts/conftest.py` does. Single-point mutants of the implementation checked that they +bite: they kill 15 of 16 in the batch code — the survivor removes a check another layer +repeats — and all 7 in the view code. + ### Deviation records are consolidated, not accumulated Each round of derivation runs a specification area per agent, and each writes its own @@ -3197,7 +3710,11 @@ new ones: `Auth#revokeTokens`, `Rest#batchPresence`, the `PushChannel` surface a same root cause as RTN15h1's, and sits in that entry. The realtime integration round did the same for five more: the RTN15h3 stall, the connection-level ERROR bypass, the unused `connectionStateTtl`, connection recovery, and the missing channel-level handling for a -decode error other than 40018. +decode error other than 40018. The objects round found its specification faults several +times over in the same way: the internal shapes the unit specifications assume were +reported by three agents, the missing barriers after an injected frame by four, and the +RTO8a buffer that `objects_faults.md` never exercises by the agent that wrote the contract, +the one that derived the tier and the one that measured it. A verdict can change the same way, when a second specification reaches a behaviour the first was content with. `errorReason` surviving a successful reconnect is permitted by @@ -3217,16 +3734,21 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 217 failures and 15 skips with the variable set, and -232 skips and 1124 passes without it, the 1124 being 1002 derived cases and 122 -`helpers/` ones. +it. As of this writing that is 218 failures, 1522 passes and 15 skips with the variable +set, and 233 skips and 1522 passes without it, the 1522 being 1374 derived cases and 148 +harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. The tiers are told +apart by the junit report's `classname`, `test.uts..`, with `proxy` the +package within an integration tier. The other two counts are measured from the source rather than from a run. The number of -**derived tests** is the number of `# UTS:` comments, 1141. The number of **Test IDs** is -the number of *distinct* ids in them, 1132 — not the same figure, because five ids in +**derived tests** is the number of `# UTS:` comments, 1480. The number of **Test IDs** is +the number of *distinct* ids in them, 1471 — not the same figure, because five ids in `rest/unit` are carried by more than one test function. Counting the comments and calling the result Test IDs is the easy mistake here, and it overstates the specification coverage -by nine. +by nine. The other is to gate or un-gate a test, record it in the body, and leave the +header alone. A test moved from passing to `@spec_error` moves the passing, gated, +per-side and specification-fault figures together, and this file's were a case short for a +while after `test_rsc7d_ably_agent_header_format` was gated that way. Anyone changing the suite should re-run both and update the header, rather than copying the previous numbers forward. Keep the three units apart while doing it: one Test ID is diff --git a/test/uts/objects/__init__.py b/test/uts/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/helpers/__init__.py b/test/uts/objects/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/helpers/standard_test_pool.py b/test/uts/objects/helpers/standard_test_pool.py new file mode 100644 index 00000000..1507b1ea --- /dev/null +++ b/test/uts/objects/helpers/standard_test_pool.py @@ -0,0 +1,514 @@ +"""The fixtures every LiveObjects specification shares. + +Derived from uts/objects/helpers/standard_test_pool.md in ably/specification: the +standard pool of objects, the canonical serials, the ObjectMessage and ProtocolMessage +builders, the synced-channel setup the mock-backed specifications open with, the +negative-assertion quiescence pattern, and REST provisioning for the integration tier. + +Every builder returns the JSON-wire dictionary the specification describes, ready for +`mock_ws.send_to_client(...)`: camelCase keys, numeric operation actions and map +semantics, `json` values as JSON-encoded strings and `bytes` values as base64 strings. +The specifications write actions and semantics by name; here they are the numbers the +wire carries, so pass `LWW` where a specification writes `semantics: "LWW"`. Templates +shared at module level (`STANDARD_POOL_OBJECTS`) must not be mutated; build a variant +instead. A test that drives the msgpack wire converts a dictionary first with +`ObjectMessage.from_dict(d).to_dict('msgpack')`. + +The pure unit specifications construct internal objects rather than wire messages; +`object_message()` and `object_messages()` decode a builder's output into the internal +`ObjectMessage` they take, and `capture_updates()` records the updates an object emits, +since `apply_operation` returns whether it applied rather than the update itself. +""" + +import base64 +import copy +import json +from typing import NamedTuple +from urllib.parse import quote + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_JSON, + ObjectMessage, + ObjectOperationAction, + ObjectsMapSemantics, +) +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.flags import Flag +from ably.pubsub.util.clock import Clock +from test.uts.helpers.client import poll_until, realtime_client, sandbox_rest_client +from test.uts.helpers.clock import settle +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import MockWebSocket + +# --- Canonical constants --------------------------------------------------------------- + +# The siteCode of the harness connection, under which the client applies its own +# operations on ACK +SITE_CODE = 'test-site' + +# The timeserial every standard-pool object and entry is seeded with. Serials compare +# lexicographically (RTLM9e), so every synthetic serial below is defined relative to it. +POOL_SERIAL = 't:0' + +CONNECTION_ID = 'conn-1' +CONNECTION_KEY = 'conn-key-1' + +# RTO10b3's default, which the harness CONNECTED message states explicitly +GC_GRACE_PERIOD = 86400000 + +# The wire values the specifications write by name +HAS_OBJECTS = int(Flag.HAS_OBJECTS) +OBJECT_SUBSCRIBE_FLAG = int(Flag.OBJECT_SUBSCRIBE) +OBJECT_PUBLISH_FLAG = int(Flag.OBJECT_PUBLISH) +LWW = int(ObjectsMapSemantics.LWW) + +ACK = int(ProtocolMessageAction.ACK) +ATTACH = int(ProtocolMessageAction.ATTACH) +ATTACHED = int(ProtocolMessageAction.ATTACHED) +CONNECTED = int(ProtocolMessageAction.CONNECTED) +DETACH = int(ProtocolMessageAction.DETACH) +DETACHED = int(ProtocolMessageAction.DETACHED) +ERROR = int(ProtocolMessageAction.ERROR) +OBJECT = int(ProtocolMessageAction.OBJECT) +OBJECT_SYNC = int(ProtocolMessageAction.OBJECT_SYNC) + +# The modes every standard channel requests +OBJECTS_MODES = (ChannelMode.OBJECT_SUBSCRIBE, ChannelMode.OBJECT_PUBLISH) + +# The X-Ably-Version the objects REST endpoint is called with: protocol v6, whose +# operation payloads (`mapSet`, `counterInc` and the rest) are what the specification's +# provisioning operations are written in +OBJECTS_REST_API_VERSION = '6' + + +# --- Serials ---------------------------------------------------------------------------- + +def ack_serial(msg_serial, index): + """The serial the harness ACKs operation `index` of the publish `msg_serial` with. + + The first publish's first operation is `ack_serial(0, 0) == 't:1:0'`, which sorts after + `POOL_SERIAL`. These serials are recorded in appliedOnAckSerials (RTO9a2a4) and an echo + carrying one is discarded (RTO9a3), so never reuse one as an inbound serial meant to apply. + """ + return f't:{msg_serial + 1}:{index}' + + +def remote_serial(index): + """A serial for a remote MAP_SET or MAP_REMOVE that wins LWW against a pool entry. + + 0-based: `remote_serial(0) == 't:1'`. A bare number such as `'99'` sorts before `'t:0'` + and would be rejected as stale. + """ + return f't:{index + 1}' + + +def below_ack_serial(index): + """A serial that is not an ACK serial, sorting after `POOL_SERIAL` but before `ack_serial(0, 0)`. + + 0-based: `below_ack_serial(9) == 't:0:9'`. + """ + return f't:0:{index}' + + +# --- Values ----------------------------------------------------------------------------- + +def json_value(value): + """An `ObjectData` wire dictionary carrying `value` as JSON (OD2g).""" + return {'json': json.dumps(value)} + + +def bytes_value(value): + """An `ObjectData` wire dictionary carrying `value` as binary (OD2d).""" + return {'bytes': base64.b64encode(value).decode('ascii')} + + +# --- ProtocolMessage builders ----------------------------------------------------------- + +def build_object_sync_message(channel, channel_serial, object_messages): + """An OBJECT_SYNC ProtocolMessage. A `channel_serial` of None leaves it out (RTO5a5).""" + message = {'action': OBJECT_SYNC, 'channel': channel, 'state': list(object_messages)} + if channel_serial is not None: + message['channelSerial'] = channel_serial + return message + + +def build_object_message(channel, object_messages): + """An OBJECT ProtocolMessage.""" + return {'action': OBJECT, 'channel': channel, 'state': list(object_messages)} + + +def build_ack_message(msg_serial, serials): + """An ACK for one ProtocolMessage, carrying a `PublishResult` with `serials` (TR4s).""" + return {'action': ACK, 'msgSerial': msg_serial, 'count': 1, 'res': [{'serials': list(serials)}]} + + +# --- ObjectMessage builders: operations ------------------------------------------------- + +def _operation_message(serial, site_code, operation, serial_timestamp=None): + message = {'serial': serial, 'siteCode': site_code, 'operation': operation} + if serial_timestamp is not None: + message['serialTimestamp'] = serial_timestamp + return message + + +def build_counter_inc(object_id, number, serial, site_code): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': object_id, + 'counterInc': {'number': number}, + }) + + +def build_map_set(object_id, key, value, serial, site_code): + """A MAP_SET; `value` is an `ObjectData` wire dictionary, such as `{'string': 'Bob'}`.""" + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_SET), + 'objectId': object_id, + 'mapSet': {'key': key, 'value': value}, + }) + + +def build_map_remove(object_id, key, serial, site_code, serial_timestamp=None): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_REMOVE), + 'objectId': object_id, + 'mapRemove': {'key': key}, + }, serial_timestamp) + + +def build_map_clear(object_id, serial, site_code): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_CLEAR), + 'objectId': object_id, + }) + + +def build_object_delete(object_id, serial, site_code, serial_timestamp=None): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.OBJECT_DELETE), + 'objectId': object_id, + }, serial_timestamp) + + +def build_counter_create(object_id, counter_create, serial, site_code): + """A COUNTER_CREATE; `counter_create` is the wire payload, such as `{'count': 42}`.""" + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.COUNTER_CREATE), + 'objectId': object_id, + 'counterCreate': counter_create, + }) + + +def build_map_create(object_id, map_create, serial, site_code): + """A MAP_CREATE; `map_create` is the wire payload, `{'semantics': LWW, 'entries': {...}}`.""" + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_CREATE), + 'objectId': object_id, + 'mapCreate': map_create, + }) + + +# --- ObjectMessage builders: state ------------------------------------------------------ + +def build_object_state(object_id, site_timeserials, map=None, counter=None, tombstone=None, create_op=None): + """An OBJECT_SYNC ObjectMessage wrapping an `ObjectState`. + + `map`, `counter`, `tombstone` and `create_op` are the specification's `opts`, as wire + dictionaries. A terse `create_op` such as `{'counterCreate': {...}}` has its mandatory + `objectId` and `action` filled in (OOP2), as the specification's builder does. + """ + state = {'objectId': object_id, 'siteTimeserials': dict(site_timeserials)} + if map is not None: + state['map'] = map + if counter is not None: + state['counter'] = counter + if tombstone is not None: + state['tombstone'] = tombstone + if create_op is not None: + create_op = dict(create_op) + create_op.setdefault('objectId', object_id) + if 'action' not in create_op: + create_op['action'] = int(ObjectOperationAction.COUNTER_CREATE if 'counterCreate' in create_op + else ObjectOperationAction.MAP_CREATE) + state['createOp'] = create_op + return {'object': state} + + +def build_object_message_with_state(object_state): + """An ObjectMessage wrapping an existing `ObjectState` wire dictionary.""" + return {'object': object_state} + + +# The standard tree. Counters carry their value in the create operation, with +# `counter.count` 0, so that applying the state yields 100 and 5 rather than double that. +STANDARD_POOL_OBJECTS = ( + build_object_state('root', {'aaa': POOL_SERIAL}, map={ + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'Alice'}, 'timeserial': POOL_SERIAL}, + 'age': {'data': {'number': 30}, 'timeserial': POOL_SERIAL}, + 'active': {'data': {'boolean': True}, 'timeserial': POOL_SERIAL}, + 'score': {'data': {'objectId': 'counter:score@1000'}, 'timeserial': POOL_SERIAL}, + 'profile': {'data': {'objectId': 'map:profile@1000'}, 'timeserial': POOL_SERIAL}, + 'data': {'data': json_value({'tags': ['a', 'b']}), 'timeserial': POOL_SERIAL}, + 'avatar': {'data': {'bytes': 'AQID'}, 'timeserial': POOL_SERIAL}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:score@1000', {'aaa': POOL_SERIAL}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + build_object_state('map:profile@1000', {'aaa': POOL_SERIAL}, map={ + 'semantics': LWW, + 'entries': { + 'email': {'data': {'string': 'alice@example.com'}, 'timeserial': POOL_SERIAL}, + 'nested_counter': {'data': {'objectId': 'counter:nested@1000'}, 'timeserial': POOL_SERIAL}, + 'prefs': {'data': {'objectId': 'map:prefs@1000'}, 'timeserial': POOL_SERIAL}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:nested@1000', {'aaa': POOL_SERIAL}, counter={'count': 0}, + create_op={'counterCreate': {'count': 5}}), + build_object_state('map:prefs@1000', {'aaa': POOL_SERIAL}, map={ + 'semantics': LWW, + 'entries': { + 'theme': {'data': {'string': 'dark'}, 'timeserial': POOL_SERIAL}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), +) + + +# --- Internal objects for the pure unit specifications ---------------------------------- + +def object_message(wire, format=WIRE_FORMAT_JSON): + """The internal `ObjectMessage` a builder's wire dictionary decodes to.""" + return ObjectMessage.from_dict(wire, format) + + +def object_messages(protocol_message, format=WIRE_FORMAT_JSON): + """The internal ObjectMessages an OBJECT or OBJECT_SYNC ProtocolMessage's `state` decodes to.""" + return ObjectMessage.from_protocol_message(protocol_message, format) + + +def capture_updates(live_object): + """Records every update `live_object` emits, the no-ops included, and returns the list. + + `apply_operation` returns whether the operation applied (RTLC7g, RTLM15g) and emits the + resulting update through `notify_updated`, so a specification's + `update = obj.applyOperation(...)` is `updates[-1]` after the call. The object's own + `notify_updated` still runs. + """ + updates = [] + original = live_object.notify_updated + + def notify_updated(update): + updates.append(update) + original(update) + + live_object.notify_updated = notify_updated + return updates + + +def build_public_object_message(message, channel_name): + """The public `ObjectMessage` PAOM3 derives from `message` received on `channel_name`. + + `message` is an internal `ObjectMessage` or a builder's wire dictionary. This is the + expected value a subscription test compares against, built here independently of the + library's own derivation. + """ + if isinstance(message, dict): + message = object_message(message) + operation = message.operation + public_operation = publicmessage.ObjectOperation( + action=operation.action, + object_id=operation.object_id, + map_create=operation.resolved_map_create, + map_set=operation.map_set, + map_remove=operation.map_remove, + counter_create=operation.resolved_counter_create, + counter_inc=operation.counter_inc, + object_delete=operation.object_delete, + map_clear=operation.map_clear, + ) + return publicmessage.ObjectMessage( + channel=channel_name, + operation=public_operation, + id=message.id, + client_id=message.client_id, + connection_id=message.connection_id, + timestamp=message.timestamp, + serial=message.serial, + serial_timestamp=message.serial_timestamp, + site_code=message.site_code, + extras=message.extras, + ) + + +# --- The mock-backed client ------------------------------------------------------------- + +def objects_connected_message(site_code=SITE_CODE, objects_gc_grace_period=GC_GRACE_PERIOD, + connection_id=CONNECTION_ID, connection_key=CONNECTION_KEY): + """The harness CONNECTED message, carrying `siteCode` and `objectsGCGracePeriod`. + + Passing None for either leaves it out. `connectionId` travels on the ProtocolMessage + itself, which is where the library reads it, though the specification writes it among + the `connectionDetails`. `maxIdleInterval` is 0 so that the transport schedules no + idle timer, which a `FakeClock` test would otherwise fire. + """ + details = {'connectionKey': connection_key, 'connectionStateTtl': 120000, 'maxIdleInterval': 0} + if site_code is not None: + details['siteCode'] = site_code + if objects_gc_grace_period is not None: + details['objectsGCGracePeriod'] = objects_gc_grace_period + return {'action': CONNECTED, 'connectionId': connection_id, 'connectionDetails': details} + + +def objects_attached_message(channel, channel_serial='sync1:', flags=HAS_OBJECTS): + """An ATTACHED for `channel`. + + The specifications write the modes a server grants as `modes: [...]`; on the wire they + are bits of `flags`, so pass `HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG` for + `flags: HAS_OBJECTS, modes: ["OBJECT_SUBSCRIBE"]`. + """ + return {'action': ATTACHED, 'channel': channel, 'channelSerial': channel_serial, 'flags': flags} + + +def standard_mock_websocket(auto_ack=True, on_object=None, connected=None, attached_channel_serial='sync1:', + attached_flags=HAS_OBJECTS, sync_objects=STANDARD_POOL_OBJECTS, + sync_channel_serial='sync1:'): + """The specification's synced-channel `MockWebSocket`. + + It accepts every connection with `connected` (by default `objects_connected_message()`); + answers an ATTACH with an ATTACHED and then an OBJECT_SYNC of `sync_objects`, or no + OBJECT_SYNC if `sync_objects` is None; answers a DETACH with a DETACHED; and passes each + OBJECT the client sends to `on_object`, then, with `auto_ack`, ACKs it with + `ack_serial(msgSerial, i)` for each of its operations. + """ + connected_message = connected if connected is not None else objects_connected_message() + mock_ws = MockWebSocket(on_connection_attempt=lambda conn: conn.respond_with_success(connected_message)) + + def on_message_from_client(message): + action = message.get('action') + if action == ATTACH: + channel = message.get('channel') + mock_ws.send_to_client(objects_attached_message(channel, attached_channel_serial, attached_flags)) + if sync_objects is not None: + mock_ws.send_to_client(build_object_sync_message(channel, sync_channel_serial, sync_objects)) + elif action == OBJECT: + if on_object is not None: + on_object(message) + if auto_ack: + msg_serial = message['msgSerial'] + serials = [ack_serial(msg_serial, i) for i in range(len(message.get('state') or []))] + mock_ws.send_to_client(build_ack_message(msg_serial, serials)) + elif action == DETACH: + mock_ws.send_to_client({'action': DETACHED, 'channel': message.get('channel')}) + + mock_ws.on_message_from_client = on_message_from_client + return mock_ws + + +def time_mock_http(clock=None): + """A `MockHttpClient` answering `GET /time` with the time on `clock`. + + Creating an object reads the server time (RTLCV4e, RTLMV4h, RTO16), and a mock-backed + client must not reach the network for it. Any other request is answered 404. + """ + time_source = clock if clock is not None else Clock() + + def on_request(request): + if request.method == 'GET' and request.path == '/time': + request.respond_with(200, [time_source.now_ms()]) + else: + request.respond_with(404, {'error': {'code': 40400, 'statusCode': 404, 'message': 'Not found'}}) + + return MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_success(), on_request=on_request) + + +def objects_client(mock_ws, clock=None, mock_http=None, **kwargs): + """A realtime client on `mock_ws`, as the specifications' `Realtime(options: {key: ...})`. + + It connects on its own, speaks JSON so that the builders' dictionaries go over the mock + as they are, and has `GET /time` answered by `time_mock_http(clock)` unless `mock_http` + is given. `clock` is a `FakeClock` for a specification's `enable_fake_timers()`. + """ + kwargs.setdefault('auto_connect', True) + kwargs.setdefault('use_binary_protocol', False) + if mock_http is None: + mock_http = time_mock_http(clock) + return realtime_client(mock_ws, mock_http=mock_http, clock=clock, **kwargs) + + +def objects_channel_options(*modes): + """`ChannelOptions` requesting `modes`, by default OBJECT_SUBSCRIBE and OBJECT_PUBLISH.""" + return ChannelOptions(modes=list(modes or OBJECTS_MODES)) + + +class SyncedChannel(NamedTuple): + """What `setup_synced_channel` returns. It unpacks as `client, channel, root, mock_ws`.""" + + client: object + channel: object + root: object + mock_ws: MockWebSocket + + +async def setup_synced_channel(channel_name='test', mock_ws=None, clock=None, modes=OBJECTS_MODES, + **client_options): + """The specification's `setup_synced_channel(channel_name)`. + + Connects a client to `mock_ws` (by default `standard_mock_websocket()`), gets + `channel_name` with `modes`, and awaits `channel.object.get()`, which attaches the + channel and waits for the standard pool to sync. `clock` and `client_options` go to + `objects_client`. + """ + if mock_ws is None: + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws, clock=clock, **client_options) + channel = client.channels.get(channel_name, objects_channel_options(*modes)) + root = await channel.object.get() + return SyncedChannel(client, channel, root, mock_ws) + + +async def setup_synced_channel_no_ack(channel_name='test', **kwargs): + """The specification's `setup_synced_channel_no_ack`: OBJECT messages are recorded but not ACKed.""" + return await setup_synced_channel(channel_name, mock_ws=standard_mock_websocket(auto_ack=False), **kwargs) + + +# --- Negative-assertion quiescence ------------------------------------------------------ + +async def assert_unchanged_after_quiescence(count_under_test, control_delivered, timeout=5.0, + description='the control to be delivered'): + """The specification's `assert_unchanged_after_quiescence(count_under_test, control)`. + + Reads `count_under_test()`, waits until `control_delivered()` holds, which a control + listener (or a follow-up message) on the same dispatch makes true, settles, and asserts + the count has not moved. Send the message under test before calling this, and the + control's stimulus after it. + """ + before = count_under_test() + await poll_until(control_delivered, timeout=timeout, description=description) + await settle() + after = count_under_test() + assert after == before, f'expected the count to stay at {before} once {description}, but it was {after}' + + +# --- REST provisioning for the integration tier ---------------------------------------- + +async def provision_objects_via_rest(api_key, channel_name, operations, client=None): + """POSTs `operations` to the channel's objects REST endpoint, before any realtime client connects. + + `operations` is one operation dictionary or a list of them, in the REST API's format: + `{'mapSet': {'key': ..., 'value': {...}}, 'objectId': 'root'}` and so on. Returns the + `objectIds` of every result, flattened in request order. The REST client goes straight + to the sandbox and is closed when the test ends; `client` replaces it. + """ + if client is None: + client = sandbox_rest_client(api_key) + path = f"/channels/{quote(channel_name, safe='')}/object" + response = await client.request('POST', path, OBJECTS_REST_API_VERSION, body=copy.deepcopy(operations)) + if not response.success: + raise AssertionError( + f'Provisioning objects on {channel_name!r} failed: {response.status_code} ' + f'{response.error_code} {response.error_message}') + return [object_id for result in response.items for object_id in (result.get('objectIds') or [])] diff --git a/test/uts/objects/helpers/standard_test_pool_test.py b/test/uts/objects/helpers/standard_test_pool_test.py new file mode 100644 index 00000000..0f07d103 --- /dev/null +++ b/test/uts/objects/helpers/standard_test_pool_test.py @@ -0,0 +1,292 @@ +"""Tests for the `standard_test_pool` helper and the object wire types it builds on. + +These are not derived from a specification. They pin the builders to the wire shapes +`standard_test_pool.md` describes, the canonical serials to the ordering the +specifications rely on, the wire types to a lossless round trip on both protocols, and +the mock to the synced-channel conversation, which it can hold before any LiveObjects +behaviour exists. +""" + +import json + +import msgpack + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_MSGPACK, + CounterCreate, + MapCreate, + MapCreateWithObjectId, + ObjectData, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapSemantics, +) +from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelmode import ChannelMode, decode_channel_mode +from ably.pubsub.types.channelstate import ChannelState +from test.uts.helpers.client import await_channel_state, await_connection_state, poll_until, rest_client +from test.uts.helpers.clock import settle +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import MockEventType +from test.uts.objects.helpers.standard_test_pool import ( + GC_GRACE_PERIOD, + HAS_OBJECTS, + LWW, + OBJECT_SUBSCRIBE_FLAG, + OBJECT_SYNC, + POOL_SERIAL, + SITE_CODE, + STANDARD_POOL_OBJECTS, + ack_serial, + assert_unchanged_after_quiescence, + below_ack_serial, + build_ack_message, + build_counter_inc, + build_map_remove, + build_map_set, + build_object_delete, + build_object_message, + build_object_state, + build_object_sync_message, + build_public_object_message, + bytes_value, + capture_updates, + json_value, + object_message, + object_messages, + objects_channel_options, + objects_client, + provision_objects_via_rest, + remote_serial, + standard_mock_websocket, +) + + +def msgpack_round_trip(message): + """`message` encoded for the msgpack wire, sent through msgpack and decoded again.""" + raw = msgpack.packb(message.to_dict(WIRE_FORMAT_MSGPACK), use_bin_type=True) + return ObjectMessage.from_dict(msgpack.unpackb(raw, raw=False), WIRE_FORMAT_MSGPACK) + + +def test_serials_sort_as_the_specification_requires(): + assert ack_serial(0, 0) == 't:1:0' + assert remote_serial(0) == 't:1' + assert below_ack_serial(9) == 't:0:9' + # A remote write and the first ACK both beat the pool, and the probe sits between them + assert POOL_SERIAL < below_ack_serial(9) < ack_serial(0, 0) + assert remote_serial(0) > POOL_SERIAL + + +def test_operation_builders_produce_the_json_wire_shape(): + assert build_counter_inc('counter:a@1', 5, '01', 'site1') == { + 'serial': '01', 'siteCode': 'site1', + 'operation': {'action': 4, 'objectId': 'counter:a@1', 'counterInc': {'number': 5}}, + } + assert build_map_set('root', 'k', {'string': 'v'}, '02', 'site1')['operation'] == { + 'action': 1, 'objectId': 'root', 'mapSet': {'key': 'k', 'value': {'string': 'v'}}, + } + assert 'serialTimestamp' not in build_map_remove('root', 'k', '03', 'site1') + assert build_object_delete('counter:a@1', '04', 'site1', 1700000000000)['serialTimestamp'] == 1700000000000 + + +def test_protocol_message_builders(): + sync = build_object_sync_message('test', 'sync1:', STANDARD_POOL_OBJECTS) + assert sync['action'] == OBJECT_SYNC + assert sync['channelSerial'] == 'sync1:' + assert len(sync['state']) == 5 + # RTO5a5: a sync with no channelSerial carries no key at all + assert 'channelSerial' not in build_object_sync_message('test', None, []) + assert build_object_message('test', [])['action'] == 19 + assert build_ack_message(3, ['a', None]) == {'action': 1, 'msgSerial': 3, 'count': 1, + 'res': [{'serials': ['a', None]}]} + + +def test_object_state_builder_fills_in_the_create_op(): + counter = build_object_state('counter:a@1', {'s': '1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 7}}) + assert counter['object']['createOp'] == {'counterCreate': {'count': 7}, 'objectId': 'counter:a@1', + 'action': int(ObjectOperationAction.COUNTER_CREATE)} + a_map = build_object_state('map:a@1', {}, map={'semantics': LWW, 'entries': {}}, + create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}, tombstone=True) + assert a_map['object']['createOp']['action'] == int(ObjectOperationAction.MAP_CREATE) + assert a_map['object']['tombstone'] is True + assert 'tombstone' not in counter['object'] + + +def test_standard_pool_decodes_to_the_specified_tree(): + root, score, profile, nested, prefs = (object_message(message).object for message in STANDARD_POOL_OBJECTS) + + assert root.object_id == 'root' + assert root.site_timeserials == {'aaa': POOL_SERIAL} + assert root.map.semantics is ObjectsMapSemantics.LWW + entries = root.map.entries + assert entries['name'].data == ObjectData(string='Alice') + assert entries['age'].data == ObjectData(number=30.0) + assert entries['active'].data == ObjectData(boolean=True) + assert entries['score'].data == ObjectData(object_id='counter:score@1000') + assert entries['data'].data.json == {'tags': ['a', 'b']} + assert entries['avatar'].data.bytes == bytes([1, 2, 3]) + assert all(entry.timeserial == POOL_SERIAL and not entry.tombstone for entry in entries.values()) + assert root.create_op.action is ObjectOperationAction.MAP_CREATE + assert root.create_op.object_id == 'root' + + assert score.counter.count == 0 + assert score.create_op.counter_create == CounterCreate(count=100.0) + assert nested.create_op.counter_create.count == 5 + assert set(profile.map.entries) == {'email', 'nested_counter', 'prefs'} + assert prefs.map.entries['theme'].data.string == 'dark' + + +def test_values_carry_json_and_binary_as_the_json_wire_does(): + assert json_value({'a': [1]}) == {'json': '{"a": [1]}'} + assert bytes_value(bytes([1, 2, 3])) == {'bytes': 'AQID'} + data = ObjectData.from_dict({'json': '{"a": [1]}'}) + assert data.json == {'a': [1]} + assert data.to_dict() == {'json': '{"a":[1]}'} + + +def test_wire_types_round_trip_through_json(): + for wire in [*STANDARD_POOL_OBJECTS, build_counter_inc('counter:a@1', 5, '01', 'site1'), + build_map_set('root', 'k', bytes_value(b'\x00\xff'), '02', 'site1'), + build_map_remove('root', 'k', '03', 'site1', 1700000000000)]: + decoded = object_message(wire) + assert object_message(json.loads(json.dumps(decoded.to_dict()))) == decoded + + +def test_wire_types_round_trip_through_msgpack(): + for wire in [*STANDARD_POOL_OBJECTS, build_map_set('root', 'k', bytes_value(b'\x00\xff'), '02', 'site1')]: + decoded = object_message(wire) + assert msgpack_round_trip(decoded) == decoded + # Binary is raw on the msgpack wire and base64 on the JSON one (OD2d) + avatar = object_message(STANDARD_POOL_OBJECTS[0]).object.map.entries['avatar'].data + assert avatar.to_dict(WIRE_FORMAT_MSGPACK) == {'bytes': bytes([1, 2, 3])} + assert avatar.to_dict() == {'bytes': 'AQID'} + + +def test_local_only_fields_are_never_encoded(): + derived = MapCreate(semantics=ObjectsMapSemantics.LWW, entries={}) + operation = ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, object_id='map:a@1', + map_create_with_object_id=MapCreateWithObjectId(initial_value='{}', nonce='n' * 16, derived_from=derived)) + assert operation.to_dict() == { + 'action': 0, 'objectId': 'map:a@1', 'mapCreateWithObjectId': {'initialValue': '{}', 'nonce': 'n' * 16}, + } + assert operation.resolved_map_create is derived + entry = object_message(STANDARD_POOL_OBJECTS[0]).object.map.entries['name'] + entry.tombstoned_at = 1700000000000 + assert 'tombstonedAt' not in json.dumps(entry.to_dict()) + + +def test_unknown_action_decodes_without_failing(): + message = object_message({'serial': '01', 'siteCode': 's', 'operation': {'action': 99, 'objectId': 'x'}}) + assert message.operation.action is ObjectOperationAction.UNKNOWN + + +def test_object_messages_take_identity_from_the_protocol_message(): + protocol_message = build_object_message('test', [build_counter_inc('counter:a@1', 1, '01', 's'), + build_counter_inc('counter:a@1', 2, '02', 's')]) + protocol_message.update(id='pm-id', connectionId='conn-x', timestamp=1234) + first, second = object_messages(protocol_message) + assert (first.id, second.id) == ('pm-id:0', 'pm-id:1') + assert first.connection_id == 'conn-x' + assert second.timestamp == 1234 + + +def test_public_object_message_follows_paom3(): + wire = build_counter_inc('counter:score@1000', 42, 'serial-1', 'site-a') + wire['clientId'] = 'client-1' + public = build_public_object_message(wire, 'test') + assert isinstance(public, publicmessage.ObjectMessage) + assert public.channel == 'test' + assert public.serial == 'serial-1' + assert public.site_code == 'site-a' + assert public.client_id == 'client-1' + assert public.operation.action is ObjectOperationAction.COUNTER_INC + assert public.operation.counter_inc.number == 42 + assert public.operation.map_create is None + + +def test_capture_updates_records_and_delegates(): + delivered = [] + + class Recorder: + def notify_updated(self, update): + delivered.append(update) + + recorder = Recorder() + updates = capture_updates(recorder) + recorder.notify_updated('first') + recorder.notify_updated('second') + assert updates == ['first', 'second'] + assert delivered == ['first', 'second'] + + +async def test_quiescence_passes_when_only_the_control_moves(): + under_test = [] + control = ['delivered'] + await assert_unchanged_after_quiescence(lambda: len(under_test), lambda: len(control) >= 1) + + +async def test_standard_mock_holds_the_synced_channel_conversation(): + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + await await_connection_state(client, ConnectionState.CONNECTED) + + details = client.connection.connection_details + assert details.site_code == SITE_CODE + assert details.objects_gc_grace_period == GC_GRACE_PERIOD + + channel = client.channels.get('test', objects_channel_options()) + await channel.attach() + await poll_until(lambda: any(event.data.get('action') == OBJECT_SYNC + for event in mock_ws.events_of_type(MockEventType.MESSAGE_TO_CLIENT)), + description='the OBJECT_SYNC to follow the ATTACHED') + attach = next(m for m in mock_ws.messages_from_client if m.get('channel') == 'test') + assert decode_channel_mode(attach['flags']) == [ChannelMode.OBJECT_SUBSCRIBE, ChannelMode.OBJECT_PUBLISH] + + await channel.detach() + await await_channel_state(channel, ChannelState.DETACHED) + + +async def test_granted_modes_decode_from_the_attached_flags(): + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG, sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + await channel.attach() + assert channel.modes == [ChannelMode.OBJECT_SUBSCRIBE] + await settle() + assert not any(event.data.get('action') == OBJECT_SYNC + for event in mock_ws.events_of_type(MockEventType.MESSAGE_TO_CLIENT)) + + +async def test_each_channel_has_one_realtime_object(): + client = objects_client(standard_mock_websocket(), auto_connect=False) + channel = client.channels.get('test') + assert isinstance(channel.object, RealtimeObject) + assert channel.object is channel.object + assert channel.object._channel is channel + assert 'root' in channel.object._objects_pool + + +async def test_provisioning_posts_operations_and_flattens_object_ids(): + captured = [] + + def on_request(request): + captured.append(request) + request.respond_with(201, [{'objectIds': ['root', 'counter:a@1']}, {'objectIds': ['map:b@2']}]) + + mock_http = MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_success(), + on_request=on_request) + client = rest_client(mock_http, use_binary_protocol=False) + operations = [{'mapSet': {'key': 'k', 'value': {'string': 'v'}}, 'objectId': 'root'}, + {'counterCreate': {'count': 1}}] + + object_ids = await provision_objects_via_rest(None, 'a channel:1', operations, client=client) + + assert object_ids == ['root', 'counter:a@1', 'map:b@2'] + assert captured[0].method == 'POST' + assert captured[0].url.raw_path == '/channels/a%20channel%3A1/object' + assert json.loads(captured[0].body) == operations diff --git a/test/uts/objects/integration/__init__.py b/test/uts/objects/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/integration/conftest.py b/test/uts/objects/integration/conftest.py new file mode 100644 index 00000000..62ee721c --- /dev/null +++ b/test/uts/objects/integration/conftest.py @@ -0,0 +1,61 @@ +"""Fixtures the LiveObjects integration specifications share. + +The sandbox app is provisioned once and read by every test that asks for it, as +it is for the realtime integration tier: each specification's `BEFORE ALL TESTS` +provisions an app, and provisioning one per file would make the tier slower and +invite the sandbox's rate limiting. The fixtures in +`test/uts/realtime/integration/conftest.py` are not visible from this package, +so the same shape is defined here; the app is a separate one from the realtime +tier's, so that a channel one tier writes objects to is never one the other +reads. +""" + +import os + +import pytest +import pytest_asyncio + +from test.uts.helpers.sandbox import delete_app, provision_app + +# `integration-testing.md` puts a suite of this size at 120 seconds. A LiveObjects +# test opens one or two connections, attaches, waits for the objects sync and then +# waits again for a mutation to reach a second client, each bounded at 10 to 30 +# seconds. The repository default in `pyproject.toml` is 30 seconds, which suits a +# test served from a mock; the marker applies to this package alone. +SUITE_TIMEOUT = 120 + +__package_dir = os.path.dirname(os.path.abspath(__file__)) + + +def pytest_collection_modifyitems(items): + for item in items: + if os.path.abspath(str(item.fspath)).startswith(__package_dir + os.sep): + item.add_marker(pytest.mark.timeout(SUITE_TIMEOUT)) + + +@pytest_asyncio.fixture(scope='session') +async def realtime_sandbox(): + """The provisioned sandbox app, as a specification's `app_config`. + + This is the specifications' `BEFORE ALL TESTS` / `AFTER ALL TESTS` pair. + `realtime_sandbox.key_str` is the full-access key they call `api_key`. + """ + app = await provision_app() + yield app + await delete_app(app) + + +@pytest.fixture(params=[False, True], ids=['json', 'msgpack']) +def use_binary_protocol(request): + """Runs the test once per protocol, which is the specifications' `PROTOCOL`. + + The three specifications directly in this package carry a + `## Protocol Variants` section, so their tests run against both json and + msgpack and pass this straight to every client they build: + + client = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + + The proxy specification has no such section and runs json only, which the + proxy requires in any case. + """ + return request.param diff --git a/test/uts/objects/integration/objects_gc_test.py b/test/uts/objects/integration/objects_gc_test.py new file mode 100644 index 00000000..416cb31c --- /dev/null +++ b/test/uts/objects/integration/objects_gc_test.py @@ -0,0 +1,109 @@ +"""Derived from uts/objects/integration/objects_gc_test.md in ably/specification. + +Spec points: RTO10, RTLM19, RTLM5d2h, RTLM7 + +Tombstone semantics end to end against the sandbox: removing a map entry tombstones it +(RTLM7), a tombstoned entry reads back as absent (RTLM5d2h), and the key can be set again, +with a fresh object id where the new value is an object (RTO10, RTLM19). Only the public +API is used. + +The GC sweep itself is not exercised here, as the specification's Scope section says: its +five-minute cadence and the server's grace period are not observable within an +integration test's budget, so the sweep is covered at the unit tier against a `FakeClock`. +Every wait here is on real time. + +The specification carries a `## Protocol Variants` section, so every test runs once per +protocol and passes `use_binary_protocol` to the client it builds. + +The specification's `value() == null` is `None` in Python. A counter's `value()` is reached +through `as_live_counter()` and a primitive's through `as_primitive()`, since ably-python's +`PathObject` is partitioned by type (RTTS); either view answers `None` once the path no +longer resolves (RTTS5d1), and each test has already seen a value at that path, so the +`None` is the removal rather than a value that never arrived. Clients are closed by the +suite's fixture. +""" + +import asyncio + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import LiveCounter +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options + +# The specification's `AWAIT_STATE ... WITH timeout` and `AWAIT channel.object.get() WITH +# timeout`, in seconds. +CONNECT_TIMEOUT = 15.0 +SYNC_TIMEOUT = 15.0 + + +async def synced_root(api_key, channel_name, use_binary_protocol): + """The setup both tests share: a CONNECTED client and the synced root of `channel_name`.""" + client = sandbox_realtime_client(api_key, auto_connect=False, use_binary_protocol=use_binary_protocol) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + channel = client.channels.get(channel_name, objects_channel_options()) + try: + return await asyncio.wait_for(channel.object.get(), SYNC_TIMEOUT) + except asyncio.TimeoutError: + raise AssertionError( + f'Timed out after {SYNC_TIMEOUT}s waiting for channel.object.get() on {channel_name!r}') from None + + +# UTS: objects/integration/RTO10/tombstoned-object-gc-recreate-0 +async def test_rto10_tombstoned_object_gc_recreate(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-gc-object-' + random_id() + root = await synced_root(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Create a counter + await root.set('counter', LiveCounter.create(42)) + await wall_clock_poll_until( + lambda: root.get('counter').as_live_counter().value() == 42, + description='counter to read 42') + + counter_id = root.get('counter').instance().id + + # Remove it (tombstones the entry and the object, RTLM7) + await root.remove('counter') + + # RTLM5d2h: tombstoned entries read back as None + await wall_clock_poll_until( + lambda: root.get('counter').as_live_counter().value() is None, + description='the removed counter to read None') + + # Create a new counter at the same key + await root.set('counter', LiveCounter.create(99)) + await wall_clock_poll_until( + lambda: root.get('counter').as_live_counter().value() == 99, + description='the new counter to read 99') + + assert root.get('counter').as_live_counter().value() == 99 + assert root.get('counter').instance().id != counter_id + + +# UTS: objects/integration/RTLM19/tombstoned-entry-gc-reset-0 +async def test_rtlm19_tombstoned_entry_gc_reset(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-gc-entry-' + random_id() + root = await synced_root(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Set then remove a key + await root.set('ephemeral', 'temporary') + await wall_clock_poll_until( + lambda: root.get('ephemeral').as_primitive().value() == 'temporary', + description="ephemeral to read 'temporary'") + + await root.remove('ephemeral') + + # RTLM5d2h: tombstoned entries read back as None + await wall_clock_poll_until( + lambda: root.get('ephemeral').as_primitive().value() is None, + description='the removed entry to read None') + + # Set the same key again + await root.set('ephemeral', 'revived') + await wall_clock_poll_until( + lambda: root.get('ephemeral').as_primitive().value() == 'revived', + description="ephemeral to read 'revived'") + + assert root.get('ephemeral').as_primitive().value() == 'revived' diff --git a/test/uts/objects/integration/objects_lifecycle_test.py b/test/uts/objects/integration/objects_lifecycle_test.py new file mode 100644 index 00000000..e43edf2a --- /dev/null +++ b/test/uts/objects/integration/objects_lifecycle_test.py @@ -0,0 +1,167 @@ +"""Derived from uts/objects/integration/objects_lifecycle_test.md in ably/specification. + +Spec points: RTO23, RTPO15, RTPO17 + +End to end against the sandbox: connect, sync, create and mutate objects through the +root `PathObject`, and read the result from a second client. Only the public API is used. + +The specification carries a `## Protocol Variants` section, so every test runs once per +protocol and passes `use_binary_protocol` to each client it builds. Provisioning over REST +is a JSON request whichever protocol the realtime client speaks. + +The specification writes its reads and writes against an untyped `PathObject`. +ably-python's `PathObject` is partitioned by type (RTTS), so a counter's `value()` and +`increment()` are reached through `as_live_counter()` and a primitive's `value()` through +`as_primitive()`; `root` is already a `LiveMapPathObject`, so `set` and `size` are called on +it directly. That is a difference of spelling, not of behaviour. + +`AWAIT_STATE` for CONNECTED waits ten seconds, the figure the realtime integration tier +uses for a connect over the network. The specification leaves `channel.object.get()` +unbounded; it is bounded here at the fifteen seconds `objects_gc_test.md` gives the same +call, so that a sync that never completes fails with a message naming it rather than at the +package timeout. Clients are closed by the suite's fixture. +""" + +import asyncio + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import LiveCounter, LiveMap, PathObject +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options, provision_objects_via_rest + +CONNECT_TIMEOUT = 10.0 +SYNC_TIMEOUT = 15.0 + + +async def connected_client(api_key, use_binary_protocol): + """The specification's `Realtime(options: {...})`, connected and CONNECTED.""" + client = sandbox_realtime_client(api_key, auto_connect=False, use_binary_protocol=use_binary_protocol) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + return client + + +async def synced_root(channel): + """The specification's `AWAIT channel.object.get()`, bounded at `SYNC_TIMEOUT`.""" + try: + return await asyncio.wait_for(channel.object.get(), SYNC_TIMEOUT) + except asyncio.TimeoutError: + raise AssertionError( + f'Timed out after {SYNC_TIMEOUT}s waiting for channel.object.get() on {channel.name!r}') from None + + +async def synced_pair(api_key, channel_name, use_binary_protocol): + """The two clients, channels and roots every propagation test opens with.""" + client_a = await connected_client(api_key, use_binary_protocol) + client_b = await connected_client(api_key, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + + root_a = await synced_root(channel_a) + root_b = await synced_root(channel_b) + return root_a, root_b + + +# UTS: objects/integration/RTO23-RTPO15/set-primitive-propagates-0 +async def test_rto23_rtpo15_set_primitive_propagates(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-lifecycle-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Client A sets a value + await root_a.set('greeting', 'hello') + + # Client B subscribes and waits for the update + events_b = [] + root_b.subscribe(events_b.append) + await wall_clock_poll_until( + lambda: root_b.get('greeting').as_primitive().value() == 'hello', + description="client B to read greeting == 'hello'") + + assert root_b.get('greeting').as_primitive().value() == 'hello' + + +# UTS: objects/integration/RTPO15/set-counter-value-type-0 +async def test_rtpo15_set_counter_value_type(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-counter-create-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + await root_a.set('my_counter', LiveCounter.create(42)) + await wall_clock_poll_until( + lambda: root_b.get('my_counter').as_live_counter().value() == 42, + description='client B to read my_counter == 42') + + assert root_b.get('my_counter').as_live_counter().value() == 42 + assert root_b.get('my_counter').instance() is not None + + +# UTS: objects/integration/RTPO17/increment-propagates-0 +async def test_rtpo17_increment_propagates(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-increment-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Create a counter first + await root_a.set('hits', LiveCounter.create(0)) + await wall_clock_poll_until( + lambda: root_b.get('hits').as_live_counter().value() == 0, + description='client B to read hits == 0') + + # Increment it + await root_a.get('hits').as_live_counter().increment(10) + await wall_clock_poll_until( + lambda: root_b.get('hits').as_live_counter().value() == 10, + description='client B to read hits == 10') + + assert root_a.get('hits').as_live_counter().value() == 10 + assert root_b.get('hits').as_live_counter().value() == 10 + + +# UTS: objects/integration/RTPO15/set-map-value-type-0 +async def test_rtpo15_set_map_value_type(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-map-create-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + await root_a.set('settings', LiveMap.create({ + 'theme': 'dark', + 'fontSize': 14, + })) + await wall_clock_poll_until( + lambda: root_b.get('settings').get('theme').as_primitive().value() == 'dark', + description="client B to read settings.theme == 'dark'") + + assert root_b.get('settings').get('theme').as_primitive().value() == 'dark' + assert root_b.get('settings').get('fontSize').as_primitive().value() == 14 + + +# UTS: objects/integration/RTO23/get-returns-path-object-0 +async def test_rto23_get_returns_path_object(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-get-root-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + + root = await synced_root(channel) + + assert isinstance(root, PathObject) + assert root.path() == '' + assert root.size() == 0 + + +# UTS: objects/integration/RTPO15/rest-provisioned-data-sync-0 +async def test_rtpo15_rest_provisioned_data_sync(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-rest-provision-' + random_id() + + # Provision data via REST before any realtime client connects + await provision_objects_via_rest(realtime_sandbox.key_str, channel_name, [ + { + 'mapSet': {'key': 'provisioned', 'value': {'string': 'from_rest'}}, + 'objectId': 'root', + }, + ]) + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + root = await synced_root(channel) + + assert root.get('provisioned').as_primitive().value() == 'from_rest' diff --git a/test/uts/objects/integration/objects_sync_test.py b/test/uts/objects/integration/objects_sync_test.py new file mode 100644 index 00000000..338c395a --- /dev/null +++ b/test/uts/objects/integration/objects_sync_test.py @@ -0,0 +1,123 @@ +"""Derived from uts/objects/integration/objects_sync_test.md in ably/specification. + +Spec points: RTO4, RTO5, RTO17 + +The objects sync sequence against the sandbox: attaching with object modes gets an +ATTACHED carrying HAS_OBJECTS and an OBJECT_SYNC, after which `channel.object.get()` +resolves; a second client syncs what a first one wrote; and a detach and re-attach syncs +the pool afresh. Only the public API is used. + +The specification carries a `## Protocol Variants` section, so every test runs once per +protocol and passes `use_binary_protocol` to each client it builds. + +A primitive's `value()` is reached through `as_primitive()`, since ably-python's +`PathObject` is partitioned by type (RTTS); `root` is a `LiveMapPathObject`, so `set` and +`size` are called on it directly. + +`AWAIT_STATE` for CONNECTED waits ten seconds, the figure the realtime integration tier +uses for a connect over the network, and `channel.object.get()` is bounded at the fifteen +seconds `objects_gc_test.md` gives the same call. Clients are closed by the suite's fixture. +""" + +import asyncio + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import ChannelMode, PathObject +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options + +CONNECT_TIMEOUT = 10.0 +SYNC_TIMEOUT = 15.0 + + +async def connected_client(api_key, use_binary_protocol): + """The specification's `Realtime(options: {...})`, connected and CONNECTED.""" + client = sandbox_realtime_client(api_key, auto_connect=False, use_binary_protocol=use_binary_protocol) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + return client + + +async def synced_root(channel): + """The specification's `AWAIT channel.object.get()`, bounded at `SYNC_TIMEOUT`.""" + try: + return await asyncio.wait_for(channel.object.get(), SYNC_TIMEOUT) + except asyncio.TimeoutError: + raise AssertionError( + f'Timed out after {SYNC_TIMEOUT}s waiting for channel.object.get() on {channel.name!r}') from None + + +# UTS: objects/integration/RTO4-RTO5/attach-sync-get-0 +async def test_rto4_rto5_attach_sync_get(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-sync-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + + root = await synced_root(channel) + + assert isinstance(root, PathObject) + assert root.path() == '' + + +# UTS: objects/integration/RTO5-RTO17/two-clients-sync-0 +async def test_rto5_rto17_two_clients_sync(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-two-sync-' + random_id() + + client_a = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + client_b = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + + # Client A creates data + root_a = await synced_root(channel_a) + await root_a.set('key1', 'value1') + + # Client B attaches and syncs, and should see the data + root_b = await synced_root(channel_b) + await wall_clock_poll_until( + lambda: root_b.get('key1').as_primitive().value() == 'value1', + description="client B to read key1 == 'value1'") + + assert root_b.get('key1').as_primitive().value() == 'value1' + + +# UTS: objects/integration/RTO17/reattach-resyncs-0 +async def test_rto17_reattach_resyncs(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-reattach-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + root = await synced_root(channel) + + # Set some data. The write is applied locally once the server ACKs it (RTO20), so + # it reads back as soon as `set` returns. + await root.set('before_detach', 'hello') + assert root.get('before_detach').as_primitive().value() == 'hello' + + # Detach and re-attach + await channel.detach() + await channel.attach() + + # Re-sync should restore data + root = await synced_root(channel) + await wall_clock_poll_until( + lambda: root.get('before_detach').as_primitive().value() == 'hello', + description="before_detach to read 'hello' after the re-sync") + + assert root.get('before_detach').as_primitive().value() == 'hello' + + +# UTS: objects/integration/RTO4/attach-subscribe-only-0 +async def test_rto4_attach_subscribe_only(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-subscribe-only-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options(ChannelMode.OBJECT_SUBSCRIBE)) + + root = await synced_root(channel) + + assert isinstance(root, PathObject) + assert root.size() == 0 diff --git a/test/uts/objects/integration/proxy/__init__.py b/test/uts/objects/integration/proxy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/integration/proxy/conftest.py b/test/uts/objects/integration/proxy/conftest.py new file mode 100644 index 00000000..9491c7ce --- /dev/null +++ b/test/uts/objects/integration/proxy/conftest.py @@ -0,0 +1,78 @@ +"""Fixtures the LiveObjects proxy integration specification shares. + +The specification in this package runs its traffic through `uts-proxy`, so each +test opens a proxy session and closes it again afterwards. The `realtime_sandbox` +app the clients connect to comes from the parent package, which provisions it +once for the whole LiveObjects integration tier. The fixtures in +`test/uts/realtime/integration/proxy/conftest.py` are not visible from here, so +the same two are defined again; `ensure_proxy()` hands every caller the one +control process, so a run that collects both packages still starts only one. + +`test/uts/helpers/proxy.py` describes the proxy itself and the two environment +variables that change how it is obtained and started. +""" + +import os + +import pytest +import pytest_asyncio + +from test.uts.helpers.proxy import create_proxy_session, ensure_proxy, stop_proxy + +# What a test in this package gets, in seconds, in place of the 120 the parent +# package gives the rest of the tier. The first proxy test to run waits for the +# binary to be downloaded on a cold cache and for the control process to come up, +# and a fault test sits through a disconnect, a reconnection and a second objects +# sync before it can assert anything. +SUITE_TIMEOUT = 300 + +__package_dir = os.path.dirname(os.path.abspath(__file__)) + + +def pytest_collection_modifyitems(items): + # The parent package marks everything beneath it, this package included, with + # its own shorter timeout, and pytest-timeout reads the first of the item's own + # markers. Putting this one at the front is what makes it the one read, + # whichever order the two hooks happen to run in. + for item in items: + if os.path.abspath(str(item.fspath)).startswith(__package_dir + os.sep): + item.add_marker(pytest.mark.timeout(SUITE_TIMEOUT), append=False) + + +@pytest_asyncio.fixture(scope='session') +async def proxy_control(): + """The running `uts-proxy` control API, shared by every test in the package. + + One control process serves any number of sessions, each on a port of its own, + so it is started once and reaped when the run ends. `proxy_session` asks for + this fixture, so a test that opens sessions does not have to. + """ + await ensure_proxy() + yield + stop_proxy() + + +@pytest_asyncio.fixture +async def proxy_session(proxy_control): + """Opens proxy sessions, and closes every one of them when the test ends. + + This is the specification's `create_proxy_session(...)` together with its + `AFTER EACH TEST: IF session IS NOT null: session.close()`. A test calls it as + it would the function: + + session = await proxy_session(rules=[...]) + + and leaves the closing alone, so a test that fails part way through still + gives its sessions back. + """ + sessions = [] + + async def open_session(**options): + session = await create_proxy_session(**options) + sessions.append(session) + return session + + yield open_session + + for session in reversed(sessions): + await session.close() diff --git a/test/uts/objects/integration/proxy/objects_faults_test.py b/test/uts/objects/integration/proxy/objects_faults_test.py new file mode 100644 index 00000000..75afbda9 --- /dev/null +++ b/test/uts/objects/integration/proxy/objects_faults_test.py @@ -0,0 +1,450 @@ +"""Derived from uts/objects/integration/proxy/objects_faults.md in ably/specification. + +Spec points: RTO5a2, RTO7, RTO8, RTO17, RTO20e, RTO20e1 + +Each test routes a client through a `uts-proxy` session and faults the objects sync +against the real sandbox: the connection dropped on the first OBJECT_SYNC, a disconnect +while another client writes, a DETACHED injected onto an attached channel, an ATTACHED +injected to restart the sync followed by a channel ERROR while a write waits for it, and +the first OBJECT_SYNC delayed while another client writes. Only the public API is used; +the proxy's event log is the second witness where the specification reads it. + +**Clients.** A client under test points at the session (`endpoint='localhost'`, +`port=session.proxy_port`, `tls=False`, `use_binary_protocol=False`) and authenticates with +an `auth_callback` returning a locally signed Ably JWT: the session speaks plain WebSocket, +over which basic auth is refused, and a JWT signed here puts nothing in the event log. The +specification's `key: api_key` on those clients is that callback. Client A, where a test +has one, goes straight to the sandbox with the key, as the specification builds it. There +is no `## Protocol Variants` section, so every client speaks JSON, which the proxy requires +in any case. + +**Rules and the log.** A frame rule matches `action` as a string, and the proxy resolves +names only up to AUTH, so OBJECT_SYNC is matched as `'20'`. The log's frames carry +`direction` and a decoded `message` whose `action` is an integer. + +**Transient and already-held states.** The specification's `AWAIT_STATE ... DISCONNECTED` +waits for a state the client leaves at once: it reconnects immediately (RTN15a), so the +state can be gone before a wait that starts after the fault is registered. A recorder is +registered on the connection before it connects and the waits read the recorded list, +which also tells the reconnection's CONNECTED from the first one. Likewise RTO17's +`AWAIT_STATE channel.state == attached` follows a DETACHED injected onto an attached +channel, so the wait is for ATTACHING followed by ATTACHED in the recorded sequence. + +**Un-awaited calls.** `channel.attach()` and RTO20e's `pending = root.set(...)` are written +without `AWAIT`; they run as tasks. `poll_until_success` is the specification's: a read that +raises `AblyException` (the channel transiently not attached, RTO25b) means "not yet", and +the most recent such error is raised if the wait runs out. Any other error fails at once, +since it means the read is wrong rather than early. + +**Cleanup.** The specification's `AFTER EACH TEST` is the fixtures': the suite closes every +client a test built and `proxy_session` closes every session it opened. + +A primitive's `value()` is reached through `as_primitive()`, since ably-python's +`PathObject` is partitioned by type (RTTS); `root` is a `LiveMapPathObject`, so `set` is +called on it directly. +""" + +import asyncio + +import pytest + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import AblyException, PathObject +from ably.pubsub.types.channelstate import ChannelState +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options + +# The specification's `WITH timeout` values, as wall-clock seconds. +CONNECT_TIMEOUT = 15.0 +DISCONNECT_TIMEOUT = 15.0 +RECONNECT_TIMEOUT = 30.0 +SYNC_TIMEOUT = 15.0 +RESYNC_TIMEOUT = 30.0 +REATTACH_TIMEOUT = 30.0 +FAILED_TIMEOUT = 15.0 +PENDING_TIMEOUT = 15.0 +ACK_TIMEOUT = 10.0 +VALUE_TIMEOUT = 15.0 + +# `poll_until_success`'s defaults, as `uts/README.md` gives them. +POLL_SUCCESS_TIMEOUT = 10.0 +POLL_SUCCESS_INTERVAL = 0.5 + +# How often a recorded state list is re-read. The list is append-only, so nothing is +# missed at any interval; a short one keeps a wait from outlasting the reconnection. +STATE_POLL_INTERVAL = 0.05 + +# RTO20e's `WAIT 500ms`, for the ACKed operation to reach the wait for SYNCED. +PARK_DELAY = 0.5 + +# The actions `uts/docs/proxy.md` tabulates, as a logged frame's `message.action`. +ACK_ACTION = 1 +OBJECT_ACTION = 19 + + +def jwt_auth_callback(api_key): + """The credentials of a proxied client: an Ably JWT signed locally for `api_key`.""" + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + + async def auth_callback(params): + return generate_jwt(key_name, key_secret) + + return auth_callback + + +def proxied_client(session, api_key): + """The specification's `Realtime(options: ClientOptions(...))` pointed at `session`.""" + return sandbox_realtime_client( + auth_callback=jwt_auth_callback(api_key), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + + +def direct_client(api_key): + """The specification's client A, which goes straight to the sandbox with the key.""" + return sandbox_realtime_client(api_key, auto_connect=False) + + +def record_states(emitter): + """Records every state `emitter` enters from now on, in order.""" + states = [] + + def on_state_change(change): + states.append(change.current) + + emitter.on(on_state_change) + return states + + +async def await_recorded_state(states, state, count=1, timeout=CONNECT_TIMEOUT): + """This file's `AWAIT_STATE` for a connection: waits until `states` holds `count` entries of `state`.""" + try: + await wall_clock_poll_until( + lambda: states.count(state) >= count, + timeout=timeout, + description=f'{count} state change(s) to {state.value}', + interval=STATE_POLL_INTERVAL, + ) + except AssertionError as error: + recorded = [recorded_state.value for recorded_state in states] + raise AssertionError(f'{error}; the states recorded were {recorded}') from None + + +def contains_in_order(recorded, expected): + """The specifications' `CONTAINS_IN_ORDER`: `expected` as a subsequence of `recorded`.""" + remaining = list(expected) + for item in recorded: + if remaining and item == remaining[0]: + remaining.pop(0) + return not remaining + + +async def bounded(awaitable, timeout, description): + """An `AWAIT ... WITH timeout`, failing with `description` when the time runs out.""" + try: + return await asyncio.wait_for(awaitable, timeout) + except asyncio.TimeoutError: + raise AssertionError(f'Timed out after {timeout}s waiting for {description}') from None + + +def in_background(coroutine): + """Starts `coroutine` without awaiting it, as an un-awaited call in the specification. + + What it ends with is retrieved when it ends, so an outcome the specification does not + assert on is not reported again as an exception nobody retrieved. + """ + task = asyncio.ensure_future(coroutine) + task.add_done_callback(lambda done: done.cancelled() or done.exception()) + return task + + +async def poll_until_success(condition, timeout=POLL_SUCCESS_TIMEOUT, description='condition'): + """The specification's `poll_until_success`, for a read across a fault and its recovery.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + last_error = None + while True: + try: + result = condition() + if result: + return result + except AblyException as error: + last_error = error + if loop.time() >= deadline: + if last_error is not None: + raise last_error + raise AssertionError(f'Timed out after {timeout}s waiting for {description}') + await asyncio.sleep(POLL_SUCCESS_INTERVAL) + + +def object_publish_acked(log): + """Whether the log holds the server's ACK for an OBJECT frame the client sent. + + The specification polls for any server-to-client ACK; the client sends nothing else + that is ACKed before the `set`, so requiring the OBJECT frame first names the same + frame and cannot be satisfied by anything else. + """ + published = False + for event in log: + if event['type'] != 'ws_frame': + continue + action = (event.get('message') or {}).get('action') + if event.get('direction') == 'client_to_server' and action == OBJECT_ACTION: + published = True + elif published and event.get('direction') == 'server_to_client' and action == ACK_ACTION: + return True + return False + + +# UTS: objects/proxy/RTO5a2-RTO17/sync-interrupted-reconnect-0 +async def test_rto5a2_rto17_sync_interrupted_reconnect(realtime_sandbox, proxy_session): + channel_name = 'objects-sync-interrupt-' + random_id() + + # Disconnect after first OBJECT_SYNC frame + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': '20'}, + 'action': {'type': 'disconnect'}, + 'times': 1, + 'comment': 'RTO5a2: Disconnect after first OBJECT_SYNC to interrupt sync', + }]) + + client = proxied_client(session, realtime_sandbox.key_str) + channel = client.channels.get(channel_name, objects_channel_options()) + states = record_states(client.connection) + + client.connect() + await await_recorded_state(states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # First attach triggers sync; proxy disconnects mid-sync + in_background(channel.attach()) + await await_recorded_state(states, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # Client auto-reconnects; re-attach triggers fresh sync + await await_recorded_state(states, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + # get() waits for SYNCED, and will only resolve if the re-sync completes + root = await bounded(channel.object.get(), RESYNC_TIMEOUT, 'channel.object.get() after the re-sync') + + assert isinstance(root, PathObject) + assert root.path() == '' + + +# UTS: objects/proxy/RTO7-RTO8/mutations-buffered-during-resync-0 +async def test_rto7_rto8_mutations_buffered_during_resync(realtime_sandbox, proxy_session): + channel_name = 'objects-buffer-resync-' + random_id() + + # Client A: direct connection (no proxy), publishes mutations + client_a = direct_client(realtime_sandbox.key_str) + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + root_a = await bounded(channel_a.object.get(), SYNC_TIMEOUT, "client A's channel.object.get()") + + # Set initial data + await root_a.set('key1', 'initial') + + # Client B: through proxy, will be disconnected + session = await proxy_session(rules=[]) + + client_b = proxied_client(session, realtime_sandbox.key_str) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + states_b = record_states(client_b.connection) + + # Client B connects and syncs + client_b.connect() + await await_recorded_state(states_b, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + root_b = await bounded(channel_b.object.get(), SYNC_TIMEOUT, "client B's channel.object.get()") + await poll_until_success( + lambda: root_b.get('key1').as_primitive().value() == 'initial', + description="client B to read key1 == 'initial'") + + # Disconnect client B + await session.trigger_action({'type': 'disconnect'}) + await await_recorded_state(states_b, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # While B is disconnected, A publishes a mutation + await root_a.set('key1', 'updated_during_disconnect') + + # Client B reconnects and re-syncs; the mutation should be visible + await await_recorded_state(states_b, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + root_b = await bounded( + channel_b.object.get(), SYNC_TIMEOUT, "client B's channel.object.get() after reconnecting") + await poll_until_success( + lambda: root_b.get('key1').as_primitive().value() == 'updated_during_disconnect', + timeout=VALUE_TIMEOUT, + description="client B to read key1 == 'updated_during_disconnect'") + + assert root_b.get('key1').as_primitive().value() == 'updated_during_disconnect' + + +# UTS: objects/proxy/RTO17/server-detach-resync-0 +async def test_rto17_server_detach_resync(realtime_sandbox, proxy_session): + channel_name = 'objects-detach-resync-' + random_id() + + session = await proxy_session(rules=[]) + + client = proxied_client(session, realtime_sandbox.key_str) + channel = client.channels.get(channel_name, objects_channel_options()) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + root = await bounded(channel.object.get(), SYNC_TIMEOUT, 'channel.object.get()') + + # Set some data. The write is applied locally once the server ACKs it (RTO20), so it + # reads back as soon as `set` returns. + await root.set('before_detach', 'hello') + assert root.get('before_detach').as_primitive().value() == 'hello' + + channel_states = record_states(channel) + + # Inject server-initiated DETACHED + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 13, + 'channel': channel_name, + }, + }) + + # Client should auto-re-attach (RTL13a). The channel is ATTACHED when the wait begins, + # so it is the recorded re-attach that is waited for. + try: + await wall_clock_poll_until( + lambda: contains_in_order(channel_states, [ChannelState.ATTACHING, ChannelState.ATTACHED]), + timeout=REATTACH_TIMEOUT, + description='the channel to re-attach after the injected DETACHED', + interval=STATE_POLL_INTERVAL) + except AssertionError as error: + recorded = [recorded_state.value for recorded_state in channel_states] + raise AssertionError(f'{error}; the channel states recorded were {recorded}') from None + assert channel.state == ChannelState.ATTACHED + + # Re-sync should restore data + root = await bounded(channel.object.get(), SYNC_TIMEOUT, 'channel.object.get() after the re-attach') + await poll_until_success( + lambda: root.get('before_detach').as_primitive().value() == 'hello', + timeout=VALUE_TIMEOUT, + description="before_detach to read 'hello' after the re-sync") + + assert root.get('before_detach').as_primitive().value() == 'hello' + + +# UTS: objects/proxy/RTO20e/publish-fails-on-channel-failed-0 +async def test_rto20e_publish_fails_on_channel_failed(realtime_sandbox, proxy_session): + channel_name = 'objects-publish-failed-' + random_id() + + session = await proxy_session(rules=[]) + + client = proxied_client(session, realtime_sandbox.key_str) + channel = client.channels.get(channel_name, objects_channel_options()) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + root = await bounded(channel.object.get(), SYNC_TIMEOUT, 'channel.object.get()') + + # Force the objects back into SYNCING: inject an ATTACHED (action 11) carrying the + # HAS_OBJECTS flag (bit 7, i.e. flags: 128). RTO4c starts a new sync sequence on every + # ATTACHED protocol message; the server never sent this ATTACHED, so no OBJECT_SYNC + # follows and the objects remain SYNCING. The channel itself stays ATTACHED. + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': {'action': 11, 'channel': channel_name, 'flags': 128}, + }) + + # Mutate WHILE SYNCING: the channel is ATTACHED so the write preconditions (RTO26) + # pass and the publish + ACK complete against the real server; publishAndApply then + # waits for a SYNCED that will never arrive (RTO20e). Do not await yet. + pending = asyncio.ensure_future(root.set('key', 'value')) + + # Ensure the operation is in the RTO20e sync-wait, not still publishing: wait until + # the proxy log shows the server's ACK for the OBJECT publish, then allow a brief + # real-time yield for the client to move the ACKed operation into the wait. + async def acked(): + return object_publish_acked(await session.get_log()) + + await wall_clock_poll_until(acked, timeout=ACK_TIMEOUT, description='the server to ACK the OBJECT publish') + await asyncio.sleep(PARK_DELAY) + + # The channel enters FAILED whilst the operation waits for SYNCED (RTO20e1) + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 9, + 'channel': channel_name, + 'error': {'statusCode': 400, 'code': 90000, 'message': 'injected error'}, + }, + }) + + await await_channel_state(channel, ChannelState.FAILED, timeout=FAILED_TIMEOUT) + + with pytest.raises(AblyException) as excinfo: + await bounded(pending, PENDING_TIMEOUT, 'the pending set() to fail') + + error = excinfo.value + assert error.code == 92008 + assert error.status_code == 400 + # RTO20e1: cause is set to RealtimeChannel.errorReason, the injected channel ERROR + assert error.cause is not None + assert error.cause.code == 90000 + + +# UTS: objects/proxy/RTO5-RTO7/publish-during-sync-echo-after-0 +async def test_rto5_rto7_publish_during_sync_echo_after(realtime_sandbox, proxy_session): + channel_name = 'objects-publish-during-sync-' + random_id() + + # Client A: direct, no proxy + client_a = direct_client(realtime_sandbox.key_str) + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + root_a = await bounded(channel_a.object.get(), SYNC_TIMEOUT, "client A's channel.object.get()") + + # Set up initial data + await root_a.set('existing', 'before') + + # Client B: through proxy with delayed OBJECT_SYNC + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': '20'}, + 'action': {'type': 'delay', 'delayMs': 3000}, + 'times': 1, + 'comment': 'Delay first OBJECT_SYNC to keep B in SYNCING state', + }]) + + client_b = proxied_client(session, realtime_sandbox.key_str) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + + # Start client B, which stays SYNCING while its first OBJECT_SYNC is delayed + client_b.connect() + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + in_background(channel_b.attach()) + + # While B is syncing, A publishes a mutation + await root_a.set('existing', 'after') + + # B's get() will resolve once the delayed sync completes + root_b = await bounded(channel_b.object.get(), RESYNC_TIMEOUT, "client B's channel.object.get()") + + # The mutation from A should be visible (either in sync data or buffered OBJECT) + await poll_until_success( + lambda: root_b.get('existing').as_primitive().value() == 'after', + timeout=VALUE_TIMEOUT, + description="client B to read existing == 'after'") + + assert root_b.get('existing').as_primitive().value() == 'after' diff --git a/test/uts/objects/unit/__init__.py b/test/uts/objects/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/unit/instance_test.py b/test/uts/objects/unit/instance_test.py new file mode 100644 index 00000000..53eb9a48 --- /dev/null +++ b/test/uts/objects/unit/instance_test.py @@ -0,0 +1,353 @@ +"""Derived from uts/objects/unit/instance.md in ably/specification. + +Spec points: RTINS3, RTINS4, RTINS5, RTINS6, RTINS9, RTINS10, RTINS12, RTINS13, RTINS14, +RTINS15, RTINS16, RTTS8a, RTTS9d, RTTS10 + +The specification is written against the untyped `Instance`, one class carrying every +method. ably-python partitions it per RTTS7-RTTS10: `id` and `type` are properties, `get`, +`compact` and the three view helpers are on the base `Instance`, and a type-specific method +is reached through the checked view helper for the wrapped type, so `inst.value()` on a +counter is `inst.as_live_counter().value()` and `inst.set(k, v)` on a map is +`await inst.as_live_map().set(k, v)`. A view helper asked for a type the instance does not +wrap raises AblyException 92007 (RTTS9d). + +That makes three of the specification's reads unreachable: `value()` on a map instance +(RTINS4d), `size()` on a counter instance (RTINS9c) and `subscribe` on a primitive instance +(RTINS16c), since the typed instance for those types has no such method and the views +that do have it refuse the wrapped type. Those tests assert the wrapped `type`, the +absence of the method, and the 92007 the view helpers raise in its place (S-5). + +`setup_synced_channel` uses the standard mock, which ACKs every OBJECT the client sends, +so each write is awaited to completion and is applied locally by the time it returns +(RTO20). +""" + +import pytest + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.enums import ValueType +from ably.pubsub.objects.instance import Instance, InstanceSubscriptionEvent, PrimitiveInstance +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + assert_unchanged_after_quiescence, + build_counter_inc, + build_map_set, + build_object_message, + build_public_object_message, + remote_serial, + setup_synced_channel, +) + + +def assert_view_refused(view_helper): + """Asserts that calling an `Instance` view helper fails fast with 92007 (RTTS9d).""" + with pytest.raises(AblyException) as excinfo: + view_helper() + assert excinfo.value.code == 92007 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTINS3/id-returns-objectid-0 +async def test_rtins3_id_returns_objectid(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + counter_inst = root.get('score').instance() + assert counter_inst.id == 'counter:score@1000' + + map_inst = root.get('profile').instance() + assert map_inst.id == 'map:profile@1000' + + +# UTS: objects/unit/RTINS4/value-counter-0 +async def test_rtins4_value_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + counter_inst = root.get('score').instance() + assert counter_inst.as_live_counter().value() == 100 + + map_inst = root.instance() + # S-5: the specification asserts `map_inst.value() == null` (RTINS4d). A map instance has + # no `value()` (RTTS10a), and the two views that do have one refuse a map (RTTS9d) + assert map_inst.type is ValueType.LIVE_MAP + assert not hasattr(map_inst, 'value') + assert_view_refused(map_inst.as_primitive) + assert_view_refused(map_inst.as_live_counter) + + +# UTS: objects/unit/RTINS5/get-wraps-entry-0 +async def test_rtins5_get_wraps_entry(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + name_inst = root_inst.get('name') + assert isinstance(name_inst, Instance) + assert name_inst.as_primitive().value() == 'Alice' + + score_inst = root_inst.get('score') + assert score_inst.id == 'counter:score@1000' + + null_inst = root_inst.get('nonexistent') + assert null_inst is None + + +# UTS: objects/unit/RTINS6/entries-yields-instances-0 +async def test_rtins6_entries_yields_instances(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + entries = {} + for key, inst in root_inst.as_live_map().entries(): + entries[key] = inst + + assert len(entries) == 7 + assert isinstance(entries['name'], Instance) + assert entries['name'].as_primitive().value() == 'Alice' + + +# UTS: objects/unit/RTINS9/size-0 +async def test_rtins9_size(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + root_inst = root.instance() + assert root_inst.as_live_map().size() == 7 + + counter_inst = root.get('score').instance() + # S-5: the specification asserts `counter_inst.size() == null` (RTINS9c). A counter + # instance has no `size()` (RTTS10b), and the map view that has one refuses a counter + # (RTTS9d) + assert counter_inst.type is ValueType.LIVE_COUNTER + assert not hasattr(counter_inst, 'size') + assert_view_refused(counter_inst.as_live_map) + + +# UTS: objects/unit/RTINS10/compact-0 +async def test_rtins10_compact(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + result = root_inst.compact() + + assert result['name'] == 'Alice' + assert result['score'] == 100 + assert result['profile']['email'] == 'alice@example.com' + + +# UTS: objects/unit/RTINS12/set-delegates-0 +async def test_rtins12_set_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + await root_inst.as_live_map().set('name', 'Bob') + + assert root.get('name').as_primitive().value() == 'Bob' + + +# UTS: objects/unit/RTINS12d/set-non-map-throws-0 +async def test_rtins12d_set_non_map_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + # `set` is reached through the map view, which refuses a counter with the 92007 and + # status 400 that RTINS12d requires of `set` (RTTS9d) + with pytest.raises(AblyException) as excinfo: + await counter_inst.as_live_map().set('key', 'value') + + assert excinfo.value.code == 92007 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTINS13/remove-delegates-0 +async def test_rtins13_remove_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + assert root.get('name').as_primitive().value() == 'Alice' + + await root_inst.as_live_map().remove('name') + + assert root.get('name').as_primitive().value() is None + + +# UTS: objects/unit/RTINS14/increment-delegates-0 +async def test_rtins14_increment_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().increment(25) + + assert root.get('score').as_live_counter().value() == 125 + + +# UTS: objects/unit/RTINS14d/increment-non-counter-throws-0 +async def test_rtins14d_increment_non_counter_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + map_inst = root.instance() + + # `increment` is reached through the counter view, which refuses a map with the 92007 + # and status 400 that RTINS14d requires of `increment` (RTTS9d) + with pytest.raises(AblyException) as excinfo: + await map_inst.as_live_counter().increment(5) + + assert excinfo.value.code == 92007 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTINS15/decrement-delegates-0 +async def test_rtins15_decrement_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().decrement(10) + + assert root.get('score').as_live_counter().value() == 90 + + +# UTS: objects/unit/RTINS14a/increment-default-0 +async def test_rtins14a_increment_default(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().increment() + + assert root.get('score').as_live_counter().value() == 101 + + +# UTS: objects/unit/RTINS15a/decrement-default-0 +async def test_rtins15a_decrement_default(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().decrement() + + assert root.get('score').as_live_counter().value() == 99 + + +# UTS: objects/unit/RTINS16/subscribe-receives-events-0 +async def test_rtins16_subscribe_receives_events(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + events = [] + sub = counter_inst.as_live_counter().subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, timeout=5, description='the listener to receive the COUNTER_INC') + await settle() + + assert isinstance(sub, Subscription) + assert len(events) == 1 + assert isinstance(events[0], InstanceSubscriptionEvent) + assert isinstance(events[0].object, Instance) + assert events[0].object.id == 'counter:score@1000' + + +# UTS: objects/unit/RTINS16c/subscribe-primitive-throws-0 +async def test_rtins16c_subscribe_primitive_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + name_inst = root.instance().get('name') + + # S-5: the specification calls `name_inst.subscribe(...)` and expects 92007 (RTINS16c). + # A primitive instance has no `subscribe` (RTTS7b, RTTS10c), and the two views that do + # have one refuse a primitive with 92007 (RTTS9d) + assert isinstance(name_inst, PrimitiveInstance) + assert name_inst.type is ValueType.STRING + assert not hasattr(name_inst, 'subscribe') + assert_view_refused(name_inst.as_live_map) + assert_view_refused(name_inst.as_live_counter) + + +# UTS: objects/unit/RTINS16e2/subscription-event-message-0 +async def test_rtins16e2_subscription_event_message(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + events = [] + root_inst.as_live_map().subscribe(events.append) + + map_set = build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote') + mock_ws.send_to_client(build_object_message('test', [map_set])) + await poll_until(lambda: len(events) >= 1, timeout=5, description='the listener to receive the MAP_SET') + + assert isinstance(events[0].object, Instance) + assert events[0].object.id == 'root' + assert events[0].message is not None + assert isinstance(events[0].message, publicmessage.ObjectMessage) + assert events[0].message.channel == 'test' + assert events[0].message.operation.action == ObjectOperationAction.MAP_SET + assert events[0].message.operation.object_id == 'root' + assert events[0].message.operation.map_set.key == 'name' + # The whole message is the one PAOM3 derives from the MAP_SET as it arrived + assert events[0].message == build_public_object_message(map_set, 'test') + + +# UTS: objects/unit/RTINS16f/subscribe-returns-subscription-0 +async def test_rtins16f_subscribe_returns_subscription(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + events = [] + sub = counter_inst.as_live_counter().subscribe(events.append) + sub.unsubscribe() + + # Quiescence control: a listener still subscribed to the same counter, which the + # same dispatch reaches + control_events = [] + counter_inst.as_live_counter().subscribe(control_events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + + await assert_unchanged_after_quiescence( + lambda: len(events), lambda: len(control_events) >= 1, + description='the control listener receives the COUNTER_INC') + assert len(events) == 0 + + +# UTS: objects/unit/RTINS16g/subscription-follows-identity-0 +async def test_rtins16g_subscription_follows_identity(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + events = [] + counter_inst.as_live_counter().subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, timeout=5, description='the listener to receive the COUNTER_INC') + + assert len(events) >= 1 + # RTINS16e1: the delivered event carries the Instance wrapping the object that fired, + # so the assertion reads the event rather than the `counter_inst` handle + assert isinstance(events[0].object, Instance) + assert events[0].object.id == 'counter:score@1000' + # The frames are processed in order, so `score` already pointed at the new counter + # when the original one was incremented, and the event is that increment + assert root.get('score').instance().id == 'counter:new@2000' + assert events[0].message.operation.counter_inc.number == 10 + assert events[0].object.as_live_counter().value() == 110 + + +# UTS: objects/unit/RTINS16h/subscribe-no-side-effects-0 +async def test_rtins16h_subscribe_no_side_effects(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + channel_state_before = channel.state + sent_before = len(mock_ws.messages_from_client) + assert channel_state_before == ChannelState.ATTACHED + + sub = counter_inst.as_live_counter().subscribe(lambda event: None) + await settle() + + assert isinstance(sub, Subscription) + assert channel.state == channel_state_before + # Nothing was sent to the server either, such as an attach or a detach + assert len(mock_ws.messages_from_client) == sent_before diff --git a/test/uts/objects/unit/internal_live_counter_api_test.py b/test/uts/objects/unit/internal_live_counter_api_test.py new file mode 100644 index 00000000..00ebb54a --- /dev/null +++ b/test/uts/objects/unit/internal_live_counter_api_test.py @@ -0,0 +1,142 @@ +"""Derived from uts/objects/unit/internal_live_counter_api.md in ably/specification. + +Spec points: RTLC5, RTLC5c, RTLC11, RTLC11b1, RTLC12, RTLC12e1, RTLC12e2, RTLC12e3, RTLC12e5, +RTLC12g, RTLC13, RTLC13b + +The specification reaches `InternalLiveCounter` through the untyped `PathObject` and +`Instance`; ably-python partitions those by type (RTTS), so a counter's `value`, +`increment`, `decrement` and `subscribe` are read through `as_live_counter()`. + +The tests that capture what the client sends use the standard synced-channel mock, which +is the specification's hand-written one: it records each OBJECT message and then ACKs it +with `ack_serial(msgSerial, i)`, so every awaited write resolves (RTO20). Captured messages +are the decoded JSON wire, and each captured operation is also compared with the whole +protocol v6 operation, so a legacy (v5) payload field fails the test. +""" + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_inc, + build_object_message, + objects_connected_message, + setup_synced_channel, + standard_mock_websocket, +) + + +def _capturing_mock_websocket(captured): + """The specification's capturing mock: the standard synced-channel conversation, with each + OBJECT message the client sends appended to `captured` before it is ACKed.""" + return standard_mock_websocket( + on_object=captured.append, + connected=objects_connected_message(connection_key='key-1'), + ) + + +# UTS: objects/unit/RTLC5/value-returns-data-0 +async def test_rtlc5_value_returns_data(): + ctx = await setup_synced_channel('test') + + counter = ctx.root.get('score') + assert counter.as_live_counter().value() == 100 + + +# UTS: objects/unit/RTLC12/increment-sends-counter-inc-0 +async def test_rtlc12_increment_sends_counter_inc(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.get('score').as_live_counter().increment(25) + + assert len(captured) == 1 + obj_msg = captured[0]['state'][0] + assert obj_msg['operation']['action'] == ObjectOperationAction.COUNTER_INC + assert obj_msg['operation']['objectId'] == 'counter:score@1000' + assert obj_msg['operation']['counterInc']['number'] == 25 + # The whole operation is the v6 wire shape: a numeric action and a `counterInc` payload + assert obj_msg['operation'] == { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:score@1000', + 'counterInc': {'number': 25}, + } + + +# UTS: objects/unit/RTLC12/increment-applies-locally-0 +async def test_rtlc12_increment_applies_locally(): + ctx = await setup_synced_channel('test') + + await ctx.root.get('score').as_live_counter().increment(50) + + assert ctx.root.get('score').as_live_counter().value() == 150 + + +# UTS: objects/unit/RTLC12e1/increment-non-number-0 +async def test_rtlc12e1_increment_non_number(): + ctx = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await ctx.root.get('score').as_live_counter().increment('not_a_number') + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTLC13/decrement-negates-0 +async def test_rtlc13_decrement_negates(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.get('score').as_live_counter().decrement(15) + + assert captured[0]['state'][0]['operation']['counterInc']['number'] == -15 + assert ctx.root.get('score').as_live_counter().value() == 85 + assert captured[0]['state'][0]['operation'] == { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:score@1000', + 'counterInc': {'number': -15}, + } + + +# UTS: objects/unit/RTLC11/counter-update-on-inc-0 +async def test_rtlc11_counter_update_on_inc(): + ctx = await setup_synced_channel('test') + + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote-site'), + ])) + + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + + assert updates[0].message.operation.counter_inc.number == 7 + + +# The `null` row applies: in Python `None` is distinguishable from an omitted amount, which +# defaults to 1. `True` is an `int` in Python and must still be rejected as a non-number. +INVALID_AMOUNTS = [ + pytest.param(None, id='null'), + pytest.param(float('nan'), id='NaN'), + pytest.param(float('inf'), id='Infinity'), + pytest.param(float('-inf'), id='-Infinity'), + pytest.param('10', id='string'), + pytest.param(True, id='boolean'), + pytest.param([1, 2], id='array'), + pytest.param({'n': 1}, id='object'), +] + + +# UTS: objects/unit/RTLC12e1/increment-invalid-amounts-table-0 +@pytest.mark.parametrize('amount', INVALID_AMOUNTS) +async def test_rtlc12e1_increment_invalid_amounts_table(amount): + ctx = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await ctx.root.get('score').as_live_counter().increment(amount) + + assert excinfo.value.code == 40003 diff --git a/test/uts/objects/unit/internal_live_counter_test.py b/test/uts/objects/unit/internal_live_counter_test.py new file mode 100644 index 00000000..2376671e --- /dev/null +++ b/test/uts/objects/unit/internal_live_counter_test.py @@ -0,0 +1,471 @@ +"""Derived from uts/objects/unit/internal_live_counter.md in ably/specification. + +Spec points: RTLC1, RTLC3, RTLC4, RTLC6, RTLC7, RTLC8, RTLC9, RTLC14, RTLC16, RTLO3, RTLO4a, +RTLO4b4d, RTLO4b4e, RTLO4e, RTLO5, RTLO6 + +The specification drives an `InternalLiveCounter` directly and reads the update from +`update = counter.applyOperation(...)`, or `false` when the operation is rejected. +`apply_operation` returns whether the operation applied (RTLC7g) and emits the update +through `notify_updated` (RTLC7d1a, RTLC7d5a, RTLC7d4c), so each test records the +emitted updates with `capture_updates` (shape deviation S-1): where the specification +reads `update`, the operation applied and emitted exactly one update, a no-op included; +where it reads `result == false`, the operation did not apply and emitted nothing. +`replace_data` returns its update (RTLC6h), as the specification has it. +""" + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from test.uts.helpers.clock import FakeClock +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_create, + build_counter_inc, + build_map_set, + build_object_delete, + build_object_state, + capture_updates, + object_message, +) + +COUNTER_ID = 'counter:abc@1000' + +CHANNEL = ObjectsOperationSource.CHANNEL +LOCAL = ObjectsOperationSource.LOCAL + + +def _emitted_update(applied, updates): + """S-1: the specification's `update = counter.applyOperation(...)`. + + An operation that applies returns True and emits exactly one update, a no-op included. + """ + assert applied is True, 'expected the operation to apply' + assert len(updates) == 1, f'expected the operation to emit exactly one update, got {updates!r}' + return updates[0] + + +def _assert_not_applied(applied, updates): + """S-1: the specification's `result == false`: the operation did not apply and emitted nothing.""" + assert applied is False + assert updates == [] + + +# UTS: objects/unit/RTLC4/zero-value-0 +def test_rtlc4_zero_value(): + counter = InternalLiveCounter(COUNTER_ID) + + assert counter.data == 0 + assert counter.object_id == COUNTER_ID + assert counter.is_tombstone is False + assert counter.tombstoned_at is None + assert counter.create_operation_is_merged is False + assert counter.site_timeserials == {} + + +# UTS: objects/unit/RTLC9/counter-inc-basic-0 +def test_rtlc9_counter_inc_basic(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 5 + update = _emitted_update(applied, updates) + assert update.noop is False + assert update.update.amount == 5 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC9/counter-inc-negative-0 +def test_rtlc9_counter_inc_negative(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 10 + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, -3, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 7 + update = _emitted_update(applied, updates) + assert update.update.amount == -3 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC9/counter-inc-missing-number-0 +def test_rtlc9_counter_inc_missing_number(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 10 + updates = capture_updates(counter) + + msg = object_message({ + 'serial': '01', + 'siteCode': 'site1', + 'operation': { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': COUNTER_ID, + 'counterInc': {}, + }, + }) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 10 + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLC9/counter-inc-accumulate-0 +def test_rtlc9_counter_inc_accumulate(): + counter = InternalLiveCounter(COUNTER_ID) + + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, 10, '01', 'site1')), CHANNEL) + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, 20, '02', 'site1')), CHANNEL) + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, -5, '01', 'site2')), CHANNEL) + + assert counter.data == 25 + + +# UTS: objects/unit/RTLC8/counter-create-merge-0 +def test_rtlc8_counter_create_merge(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_counter_create(COUNTER_ID, {'count': 42}, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 42 + assert counter.create_operation_is_merged is True + update = _emitted_update(applied, updates) + assert update.update.amount == 42 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC8/counter-create-already-merged-0 +def test_rtlc8_counter_create_already_merged(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 42 + counter.create_operation_is_merged = True + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_counter_create(COUNTER_ID, {'count': 99}, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 42 + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLC16/counter-create-no-count-0 +def test_rtlc16_counter_create_no_count(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_counter_create(COUNTER_ID, {}, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 0 + assert counter.create_operation_is_merged is True + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLO4a/apply-empty-site-serial-0 +def test_rtlo4a_apply_empty_site_serial(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + # S-1: the specification's `result IS NOT false` is a True return + assert result is True + assert counter.data == 5 + + +# UTS: objects/unit/RTLO4a/reject-stale-serial-0 +def test_rtlo4a_reject_stale_serial(): + counter = InternalLiveCounter(COUNTER_ID) + counter.site_timeserials = {'site1': '05'} + counter.data = 10 + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 99, '03', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 10 + + +# UTS: objects/unit/RTLO4a/reject-equal-serial-0 +def test_rtlo4a_reject_equal_serial(): + counter = InternalLiveCounter(COUNTER_ID) + counter.site_timeserials = {'site1': '05'} + counter.data = 10 + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 99, '05', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 10 + + +# UTS: objects/unit/RTLO4a/warn-invalid-serial-0 +def test_rtlo4a_warn_invalid_serial(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg_no_serial = object_message(build_counter_inc(COUNTER_ID, 5, '', 'site1')) + result1 = counter.apply_operation(msg_no_serial, CHANNEL) + + msg_no_site = object_message(build_counter_inc(COUNTER_ID, 5, '01', '')) + result2 = counter.apply_operation(msg_no_site, CHANNEL) + + assert counter.data == 0 + assert result1 is False + assert result2 is False + # S-1: neither rejected operation emitted an update + assert updates == [] + + +# UTS: objects/unit/RTLC7c/channel-source-updates-serials-0 +def test_rtlc7c_channel_source_updates_serials(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + counter.apply_operation(msg, CHANNEL) + + assert counter.site_timeserials['site1'] == '01' + + +# UTS: objects/unit/RTLC7c/local-source-no-serial-update-0 +def test_rtlc7c_local_source_no_serial_update(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + counter.apply_operation(msg, LOCAL) + + assert counter.site_timeserials == {} + assert counter.data == 5 + + +# UTS: objects/unit/RTLC7g/apply-returns-true-0 +def test_rtlc7g_apply_returns_true(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + assert result is True + + +# UTS: objects/unit/RTLO5/object-delete-tombstones-0 +def test_rtlo5_object_delete_tombstones(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 42 + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1', 1700000000000)) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.is_tombstone is True + assert counter.data == 0 + assert counter.tombstoned_at == 1700000000000 + update = _emitted_update(applied, updates) + assert update.update.amount == -42 + assert update.tombstone is True + assert update.object_message is msg + + +# UTS: objects/unit/RTLO5/tombstone-zero-value-counter-emits-update-0 +def test_rtlo5_tombstone_zero_value_counter_emits_update(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 0 + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1', 1700000000000)) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.is_tombstone is True + assert counter.data == 0 + update = _emitted_update(applied, updates) + # The RTLC14c zero-delta exception does not apply to a tombstone diff (RTLO4e5) + assert update.noop is False + assert update.tombstone is True + assert update.update.amount == 0 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC7e/tombstoned-reject-ops-0 +def test_rtlc7e_tombstoned_reject_ops(): + counter = InternalLiveCounter(COUNTER_ID) + counter.is_tombstone = True + counter.tombstoned_at = 1700000000000 + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 0 + + +# UTS: objects/unit/RTLO6/tombstoned-at-from-serial-timestamp-0 +def test_rtlo6_tombstoned_at_from_serial_timestamp(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1', 1700000050000)) + counter.apply_operation(msg, CHANNEL) + + assert counter.tombstoned_at == 1700000050000 + + +# UTS: objects/unit/RTLO6/tombstoned-at-local-clock-0 +def test_rtlo6_tombstoned_at_local_clock(): + # The local clock is the counter's own. A fake one, at an epoch no fixture uses, holds + # still between the two readings, so the bounds pin the value RTLO6b must take. + clock = FakeClock(epoch_ms=1_700_000_123_456) + counter = InternalLiveCounter(COUNTER_ID, clock=clock) + before_time = clock.now_ms() + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1')) + counter.apply_operation(msg, CHANNEL) + + after_time = clock.now_ms() + assert counter.tombstoned_at >= before_time + assert counter.tombstoned_at <= after_time + + +# UTS: objects/unit/RTLC7d3/unsupported-action-0 +def test_rtlc7d3_unsupported_action(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_map_set(COUNTER_ID, 'x', {'string': 'y'}, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 0 + + +# UTS: objects/unit/RTLC6/replace-data-basic-0 +def test_rtlc6_replace_data_basic(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 10 + counter.create_operation_is_merged = True + counter.site_timeserials = {'site1': '00'} + + state_msg = object_message(build_object_state(COUNTER_ID, {'site2': '05'}, counter={'count': 50})) + update = counter.replace_data(state_msg) + + assert counter.data == 50 + assert counter.site_timeserials == {'site2': '05'} + assert counter.create_operation_is_merged is False + assert update.update.amount == 40 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC6/replace-data-with-create-op-0 +def test_rtlc6_replace_data_with_create_op(): + counter = InternalLiveCounter(COUNTER_ID) + + state_msg = object_message(build_object_state( + COUNTER_ID, {'site1': '01'}, counter={'count': 100}, create_op={'counterCreate': {'count': 50}})) + update = counter.replace_data(state_msg) + + assert counter.data == 150 + assert counter.create_operation_is_merged is True + assert update.update.amount == 150 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC6e/replace-data-tombstoned-noop-0 +def test_rtlc6e_replace_data_tombstoned_noop(): + counter = InternalLiveCounter(COUNTER_ID) + counter.is_tombstone = True + counter.tombstoned_at = 1700000000000 + counter.data = 0 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={'count': 999})) + update = counter.replace_data(state_msg) + + assert counter.data == 0 + assert update.noop is True + + +# UTS: objects/unit/RTLC6f/replace-data-tombstone-flag-0 +def test_rtlc6f_replace_data_tombstone_flag(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 30 + + state_msg = object_message(build_object_state( + COUNTER_ID, {'site1': '01'}, counter={'count': 0}, tombstone=True)) + update = counter.replace_data(state_msg) + + assert counter.is_tombstone is True + assert counter.data == 0 + assert update.update.amount == -30 + assert update.tombstone is True + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC6/replace-data-missing-count-0 +def test_rtlc6_replace_data_missing_count(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 42 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={})) + update = counter.replace_data(state_msg) + + assert counter.data == 0 + assert update.update.amount == -42 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC14/diff-calculation-0 +def test_rtlc14_diff_calculation(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 20 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={'count': 75})) + update = counter.replace_data(state_msg) + + assert update.update.amount == 55 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC14c/zero-delta-diff-is-noop-0 +def test_rtlc14c_zero_delta_diff_is_noop(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 100 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={'count': 100})) + update = counter.replace_data(state_msg) + + assert update.noop is True + assert counter.data == 100 + + +# UTS: objects/unit/RTLC8/create-then-inc-0 +def test_rtlc8_create_then_inc(): + counter = InternalLiveCounter(COUNTER_ID) + + counter.apply_operation( + object_message(build_counter_create(COUNTER_ID, {'count': 100}, '01', 'site1')), CHANNEL) + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, 25, '02', 'site1')), CHANNEL) + + assert counter.data == 125 + assert counter.create_operation_is_merged is True + + +# UTS: objects/unit/RTLO3/live-object-init-properties-0 +def test_rtlo3_live_object_init_properties(): + counter = InternalLiveCounter('counter:test@2000') + + assert counter.object_id == 'counter:test@2000' + assert counter.site_timeserials == {} + assert counter.create_operation_is_merged is False + assert counter.is_tombstone is False + assert counter.tombstoned_at is None diff --git a/test/uts/objects/unit/internal_live_map_api_test.py b/test/uts/objects/unit/internal_live_map_api_test.py new file mode 100644 index 00000000..37c23d42 --- /dev/null +++ b/test/uts/objects/unit/internal_live_map_api_test.py @@ -0,0 +1,300 @@ +"""Derived from uts/objects/unit/internal_live_map_api.md in ably/specification. + +Spec points: RTLM5, RTLM5d2, RTLM10, RTLM10d, RTLM11, RTLM11d, RTLM12, RTLM20, RTLM20a3, +RTLM20e1, RTLM20e2, RTLM20e3, RTLM20e6, RTLM20e7b, RTLM20e7c, RTLM20e7d, RTLM20e7e, +RTLM20e7f, RTLM20e7g, RTLM20e7g1, RTLM20e7g2, RTLM20h1, RTLM20h2, RTLM21, RTLM21e1, +RTLM21e2, RTLM21e5, RTLMV4c, RTLMV4d1, RTLMV4d2, RTLCV4 + +The specification reaches `InternalLiveMap` through the untyped `PathObject`; ably-python +partitions it by type (RTTS). `root` is already a `LiveMapPathObject`, so `size`, +`entries`, `keys`, `set` and `remove` are called on it directly, and a primitive or counter +below it is read through `as_primitive()` or `as_live_counter()`. + +The tests that capture what the client sends use the standard synced-channel mock, which +is the specification's hand-written one: it records each OBJECT message and then ACKs it +with `ack_serial(msgSerial, i)`, so every awaited write resolves (RTO20). Captured messages +are the decoded JSON wire, where a `json` value is a JSON-encoded string (OD2g), a `bytes` +value is base64, and an action is its number. Each captured operation is also checked +against the whole protocol v6 shape: a primitive MAP_SET or a MAP_REMOVE is compared +exactly, and a create carries only its `*CreateWithObjectId` payload, the `CounterCreate` +or `MapCreate` it was derived from staying local (RTLCV4g5, RTLMV4j5). +""" + +import json + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap +from ably.pubsub.util.exceptions import AblyException +from test.uts.objects.helpers.standard_test_pool import ( + LWW, + objects_connected_message, + setup_synced_channel, + standard_mock_websocket, +) + + +def _capturing_mock_websocket(captured): + """The specification's capturing mock: the standard synced-channel conversation, with each + OBJECT message the client sends appended to `captured` before it is ACKed.""" + return standard_mock_websocket( + on_object=captured.append, + connected=objects_connected_message(connection_key='key-1'), + ) + + +def _map_set_operation(object_id, key, value): + """The v6 wire form of a MAP_SET operation; `value` is the `ObjectData` wire dictionary.""" + return { + 'action': int(ObjectOperationAction.MAP_SET), + 'objectId': object_id, + 'mapSet': {'key': key, 'value': value}, + } + + +def _assert_create_wire_shape(operation, payload_name): + """Asserts that a captured create carries only its v6 `*CreateWithObjectId` payload, with + an `initialValue` string and a nonce, and returns the decoded initial value.""" + assert set(operation) == {'action', 'objectId', payload_name} + payload = operation[payload_name] + assert set(payload) == {'initialValue', 'nonce'} + assert isinstance(payload['nonce'], str) + assert isinstance(payload['initialValue'], str) + return json.loads(payload['initialValue']) + + +# UTS: objects/unit/RTLM5/get-string-value-0 +async def test_rtlm5_get_string_value(): + ctx = await setup_synced_channel('test') + + assert ctx.root.get('name').as_primitive().value() == 'Alice' + assert ctx.root.get('age').as_primitive().value() == 30 + assert ctx.root.get('active').as_primitive().value() is True + + +# UTS: objects/unit/RTLM5/get-nonexistent-key-0 +async def test_rtlm5_get_nonexistent_key(): + ctx = await setup_synced_channel('test') + + # The specification's untyped `value()`: a path that resolves to nothing answers None + # through every typed view (RTTS5d1) + assert ctx.root.get('nonexistent').as_primitive().value() is None + assert ctx.root.get('nonexistent').as_live_counter().value() is None + + +# UTS: objects/unit/RTLM5/get-objectid-reference-0 +async def test_rtlm5_get_objectid_reference(): + ctx = await setup_synced_channel('test') + + assert ctx.root.get('score').as_live_counter().value() == 100 + assert ctx.root.get('profile').get('email').as_primitive().value() == 'alice@example.com' + + +# UTS: objects/unit/RTLM10/size-non-tombstoned-0 +async def test_rtlm10_size_non_tombstoned(): + ctx = await setup_synced_channel('test') + + assert ctx.root.size() == 7 + + +# UTS: objects/unit/RTLM11/entries-yields-pairs-0 +async def test_rtlm11_entries_yields_pairs(): + ctx = await setup_synced_channel('test') + + entries = [] + for key, _path_object in ctx.root.entries(): + entries.append(key) + + assert 'name' in entries + assert 'age' in entries + assert 'active' in entries + assert 'score' in entries + assert 'profile' in entries + assert 'data' in entries + assert 'avatar' in entries + assert len(entries) == 7 + + +# UTS: objects/unit/RTLM12/keys-0 +async def test_rtlm12_keys(): + ctx = await setup_synced_channel('test') + + keys = list(ctx.root.keys()) + + assert len(keys) == 7 + assert 'name' in keys + + +# UTS: objects/unit/RTLM20/set-sends-map-set-0 +async def test_rtlm20_set_sends_map_set(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('name', 'Bob') + + assert len(captured) == 1 + obj_msg = captured[0]['state'][0] + assert obj_msg['operation']['action'] == ObjectOperationAction.MAP_SET + assert obj_msg['operation']['objectId'] == 'root' + assert obj_msg['operation']['mapSet']['key'] == 'name' + assert obj_msg['operation']['mapSet']['value']['string'] == 'Bob' + assert obj_msg['operation'] == _map_set_operation('root', 'name', {'string': 'Bob'}) + + +# UTS: objects/unit/RTLM20/set-value-types-0 +async def test_rtlm20_set_value_types(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('num_key', 42) + await ctx.root.set('bool_key', False) + await ctx.root.set('json_key', {'nested': True}) + + assert captured[0]['state'][0]['operation']['mapSet']['value']['number'] == 42 + assert captured[1]['state'][0]['operation']['mapSet']['value']['boolean'] is False + # A `json` value travels as a JSON-encoded string (OD2g); the specification compares the + # decoded value + json_value = captured[2]['state'][0]['operation']['mapSet']['value'] + assert json.loads(json_value['json']) == {'nested': True} + + assert captured[0]['state'][0]['operation'] == _map_set_operation('root', 'num_key', {'number': 42}) + assert captured[1]['state'][0]['operation'] == _map_set_operation('root', 'bool_key', {'boolean': False}) + assert set(json_value) == {'json'} + assert isinstance(json_value['json'], str) + + +# UTS: objects/unit/RTLM20e7g/set-counter-value-type-0 +async def test_rtlm20e7g_set_counter_value_type(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('new_counter', LiveCounter.create(50)) + + assert len(captured) == 1 + state = captured[0]['state'] + assert len(state) == 2 + assert state[0]['operation']['action'] == ObjectOperationAction.COUNTER_CREATE + assert state[0]['operation']['objectId'].startswith('counter:') + assert state[1]['operation']['action'] == ObjectOperationAction.MAP_SET + assert state[1]['operation']['mapSet']['value']['objectId'] == state[0]['operation']['objectId'] + + initial_value = _assert_create_wire_shape(state[0]['operation'], 'counterCreateWithObjectId') + assert initial_value == {'count': 50} + assert state[1]['operation'] == _map_set_operation( + 'root', 'new_counter', {'objectId': state[0]['operation']['objectId']}) + + +# UTS: objects/unit/RTLM20e7g/set-map-value-type-0 +async def test_rtlm20e7g_set_map_value_type(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('nested_map', LiveMap.create({'key1': 'value1'})) + + assert len(captured) == 1 + state = captured[0]['state'] + assert len(state) == 2 + assert state[0]['operation']['action'] == ObjectOperationAction.MAP_CREATE + assert state[0]['operation']['objectId'].startswith('map:') + assert state[1]['operation']['action'] == ObjectOperationAction.MAP_SET + assert state[1]['operation']['mapSet']['key'] == 'nested_map' + assert state[1]['operation']['mapSet']['value']['objectId'] == state[0]['operation']['objectId'] + + initial_value = _assert_create_wire_shape(state[0]['operation'], 'mapCreateWithObjectId') + assert initial_value['semantics'] == LWW + assert set(initial_value['entries']) == {'key1'} + assert initial_value['entries']['key1']['data'] == {'string': 'value1'} + assert state[1]['operation'] == _map_set_operation( + 'root', 'nested_map', {'objectId': state[0]['operation']['objectId']}) + + +# UTS: objects/unit/RTLM20h1/set-nested-value-types-0 +async def test_rtlm20h1_set_nested_value_types(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('stats', LiveMap.create({ + 'count': LiveCounter.create(0), + 'label': 'test', + })) + + assert len(captured) == 1 + state = captured[0]['state'] + # COUNTER_CREATE, MAP_CREATE, MAP_SET: depth-first, then the MAP_SET at root + assert len(state) == 3 + assert state[0]['operation']['action'] == ObjectOperationAction.COUNTER_CREATE + assert state[0]['operation']['objectId'].startswith('counter:') + assert state[1]['operation']['action'] == ObjectOperationAction.MAP_CREATE + assert state[1]['operation']['objectId'].startswith('map:') + assert state[2]['operation']['action'] == ObjectOperationAction.MAP_SET + assert state[2]['operation']['mapSet']['key'] == 'stats' + assert state[2]['operation']['mapSet']['value']['objectId'] == state[1]['operation']['objectId'] + + counter_initial_value = _assert_create_wire_shape(state[0]['operation'], 'counterCreateWithObjectId') + assert counter_initial_value == {'count': 0} + # RTLMV4d1: the nested counter's entry references the counter the first message creates + map_initial_value = _assert_create_wire_shape(state[1]['operation'], 'mapCreateWithObjectId') + assert map_initial_value['entries']['count']['data'] == {'objectId': state[0]['operation']['objectId']} + assert map_initial_value['entries']['label']['data'] == {'string': 'test'} + assert state[2]['operation'] == _map_set_operation( + 'root', 'stats', {'objectId': state[1]['operation']['objectId']}) + + +# UTS: objects/unit/RTLM21/remove-sends-map-remove-0 +async def test_rtlm21_remove_sends_map_remove(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.remove('name') + + obj_msg = captured[0]['state'][0] + assert obj_msg['operation']['action'] == ObjectOperationAction.MAP_REMOVE + assert obj_msg['operation']['objectId'] == 'root' + assert obj_msg['operation']['mapRemove']['key'] == 'name' + assert obj_msg['operation'] == { + 'action': int(ObjectOperationAction.MAP_REMOVE), + 'objectId': 'root', + 'mapRemove': {'key': 'name'}, + } + + +# UTS: objects/unit/RTLM20/set-applies-locally-0 +async def test_rtlm20_set_applies_locally(): + ctx = await setup_synced_channel('test') + + await ctx.root.set('name', 'Bob') + + assert ctx.root.get('name').as_primitive().value() == 'Bob' + + +# The specification's rows are JavaScript values. Their Python counterparts: a function is a +# lambda, `undefined` is `None`, and a symbol is an arbitrary `object()`. None of them is a +# value a map entry can hold (RTLM20a3). +INVALID_VALUES = [ + pytest.param(lambda: None, id='function'), + pytest.param(None, id='undefined'), + pytest.param(object(), id='symbol'), +] + + +# UTS: objects/unit/RTLM20/set-invalid-values-table-0 +@pytest.mark.parametrize('value', INVALID_VALUES) +async def test_rtlm20_set_invalid_values_table(value): + ctx = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await ctx.root.set('key', value) + + assert excinfo.value.code == 40013 + + +# UTS: objects/unit/RTLM20/set-bytes-value-0 +async def test_rtlm20_set_bytes_value(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('binary_data', bytes([1, 2, 3])) + + assert captured[0]['state'][0]['operation']['mapSet']['value']['bytes'] == 'AQID' + assert captured[0]['state'][0]['operation'] == _map_set_operation('root', 'binary_data', {'bytes': 'AQID'}) diff --git a/test/uts/objects/unit/internal_live_map_test.py b/test/uts/objects/unit/internal_live_map_test.py new file mode 100644 index 00000000..4bb1e873 --- /dev/null +++ b/test/uts/objects/unit/internal_live_map_test.py @@ -0,0 +1,928 @@ +"""Derived from uts/objects/unit/internal_live_map.md in ably/specification. + +Spec points: RTLM1-RTLM9, RTLM14-RTLM16, RTLM18, RTLM19, RTLM22-RTLM25, RTLO3, RTLO4a, RTLO4e, +RTLO4g, RTLO4h, RTLO5, RTLO6 + +The specification drives an `InternalLiveMap` directly and reads the update from +`update = map.applyOperation(...)`, or `false` when the operation is rejected. +`apply_operation` returns whether the operation applied (RTLM15g) and emits the update +through `notify_updated` (RTLM15d1a and its siblings), so each test records the emitted +updates with `capture_updates` (shape deviation S-1): where the specification reads +`update`, the operation applied and emitted exactly one update, a no-op included; where +it reads `result == false` or `update == false`, the operation did not apply and emitted +nothing. `replace_data` returns its update (RTLM6h), and `diff` is the static +`InternalLiveMap.diff`, as the specification has them. + +Where the specification passes a pool, the map under test reads it but is not the pool's +own root: `ObjectsPool()` creates that (RTO3b1), and the specification builds a second map +with the id `root` alongside it. +""" + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.objectmessage import ObjectData, ObjectsMapEntry, ObjectsMapSemantics +from ably.pubsub.objects.objectspool import ObjectsPool +from test.uts.objects.helpers.standard_test_pool import ( + LWW, + build_counter_inc, + build_map_clear, + build_map_create, + build_map_remove, + build_map_set, + build_object_delete, + build_object_state, + capture_updates, + object_message, +) + +CHANNEL = ObjectsOperationSource.CHANNEL + + +def _emitted_update(applied, updates): + """S-1: the specification's `update = map.applyOperation(...)`. + + An operation that applies returns True and emits exactly one update, a no-op included. + """ + assert applied is True, 'expected the operation to apply' + assert len(updates) == 1, f'expected the operation to emit exactly one update, got {updates!r}' + return updates[0] + + +def _assert_not_applied(applied, updates): + """S-1: the specification's `result == false`: the operation did not apply and emitted nothing.""" + assert applied is False + assert updates == [] + + +def _references(child, parent_id, key): + """Whether `child.parent_references` records the map `parent_id` referencing it at `key`. + + `not _references(...)` is the specification's + `parent_id NOT IN child.parentReferences OR key NOT IN child.parentReferences[parent_id]`. + """ + return key in child.parent_references.get(parent_id, set()) + + +# UTS: objects/unit/RTLM4/zero-value-0 +def test_rtlm4_zero_value(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + + assert live_map.data == {} + assert live_map.clear_timeserial is None + assert live_map.is_tombstone is False + assert live_map.create_operation_is_merged is False + assert live_map.site_timeserials == {} + + +# UTS: objects/unit/RTLM7/map-set-new-entry-0 +def test_rtlm7_map_set_new_entry(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Alice'}, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['name'].timeserial == '01' + assert live_map.data['name'].tombstone is False + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7/map-set-update-entry-0 +def test_rtlm7_map_set_update_entry(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Bob') + assert live_map.data['name'].timeserial == '02' + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM9/lww-reject-stale-0 +def test_rtlm9_lww_reject_stale(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='05', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '03', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM9/lww-reject-equal-0 +def test_rtlm9_lww_reject_equal(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='05', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM9b/both-empty-reject-0 +def test_rtlm9b_both_empty_reject(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + # The empty ObjectMessage.serial fails the object-level gate (RTLO4a3) before the + # entry-level RTLM9b comparison is reached, so apply_operation returns False (RTLM15b). + _assert_not_applied(applied, updates) + # RTLM9b itself, which apply_operation cannot reach: an empty or missing entry serial + # and operation serial are equal, so the operation is not applied + assert InternalLiveMap.can_apply_map_operation('', '') is False + assert InternalLiveMap.can_apply_map_operation(None, None) is False + + +# UTS: objects/unit/RTLM9d/missing-entry-serial-allows-0 +def test_rtlm9d_missing_entry_serial_allows(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial=None, tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Bob') + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7h/map-set-clear-timeserial-floor-0 +def test_rtlm7h_map_set_clear_timeserial_floor(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '05' + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Alice'}, '03', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert 'name' not in live_map.data + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM7g/map-set-objectid-creates-zero-value-0 +def test_rtlm7g_map_set_objectid_creates_zero_value(): + pool = ObjectsPool() + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + + msg = object_message(build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, '01', 'site1')) + live_map.apply_operation(msg, CHANNEL) + + assert 'counter:new@2000' in pool + assert isinstance(pool['counter:new@2000'], InternalLiveCounter) + assert pool['counter:new@2000'].data == 0 + + +# UTS: objects/unit/RTLM8/map-remove-existing-0 +def test_rtlm8_map_remove_existing(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'name', '02', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data is None + assert live_map.data['name'].tombstone is True + assert live_map.data['name'].timeserial == '02' + assert live_map.data['name'].tombstoned_at == 1700000000000 + update = _emitted_update(applied, updates) + assert update.update == {'name': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8/map-remove-nonexistent-0 +def test_rtlm8_map_remove_nonexistent(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'ghost', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['ghost'].tombstone is True + assert live_map.data['ghost'].tombstoned_at == 1700000000000 + update = _emitted_update(applied, updates) + assert update.update == {'ghost': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8g/map-remove-clear-timeserial-floor-0 +def test_rtlm8g_map_remove_clear_timeserial_floor(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '05' + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='04', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'name', '03', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['name'].tombstone is False + update = _emitted_update(applied, updates) + assert update.noop is True + + # UTS SPEC ERROR: the entry's own timeserial ('04') is later than the operation's ('03'), + # so LWW (RTLM8a1, RTLM9e) rejects the remove with or without the clear floor, and the + # steps above pass either way. A remove for a key with no entry, still at or below the + # floor, reaches RTLM8g alone: without it the key would gain a tombstoned entry (RTLM8b). + ghost_updates = capture_updates(live_map) + ghost_msg = object_message(build_map_remove('root', 'ghost', '04', 'site1', 1700000000000)) + ghost_applied = live_map.apply_operation(ghost_msg, CHANNEL) + + assert 'ghost' not in live_map.data + ghost_update = _emitted_update(ghost_applied, ghost_updates) + assert ghost_update.noop is True + + +# UTS: objects/unit/RTLM24/map-clear-basic-0 +def test_rtlm24_map_clear_basic(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'old': ObjectsMapEntry(data=ObjectData(string='old'), timeserial='02', tombstone=False), + 'new': ObjectsMapEntry(data=ObjectData(string='new'), timeserial='06', tombstone=False), + 'same': ObjectsMapEntry(data=ObjectData(string='same'), timeserial='04', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '04', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.clear_timeserial == '04' + assert 'old' not in live_map.data + # RTLM24e1: an entry is removed only if the clear serial is lexicographically greater + # than the entry's timeserial; 'same' has the clear serial itself, so it is kept + assert 'same' in live_map.data + assert 'new' in live_map.data + update = _emitted_update(applied, updates) + assert update.update == {'old': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM24c/map-clear-stale-0 +def test_rtlm24c_map_clear_stale(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '10' + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.clear_timeserial == '10' + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM16/map-create-merge-0 +def test_rtlm16_map_create_merge(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_map_create('map:test@1000', { + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'Alice'}, 'timeserial': '01'}, + 'removed_key': {'tombstone': True, 'timeserial': '01', 'serialTimestamp': 1700000000000}, + }, + }, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['removed_key'].tombstone is True + assert live_map.create_operation_is_merged is True + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated', 'removed_key': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM16b/map-create-already-merged-0 +def test_rtlm16b_map_create_already_merged(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.create_operation_is_merged = True + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_map_create('map:test@1000', { + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'Bob'}, 'timeserial': '01'}}, + }, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert 'name' not in live_map.data + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM15c/channel-source-updates-serials-0 +def test_rtlm15c_channel_source_updates_serials(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + + msg = object_message(build_map_set('root', 'x', {'number': 1}, '01', 'site1')) + live_map.apply_operation(msg, CHANNEL) + + assert live_map.site_timeserials['site1'] == '01' + + +# UTS: objects/unit/RTLM15e/tombstoned-reject-ops-0 +def test_rtlm15e_tombstoned_reject_ops(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.is_tombstone = True + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'x', {'number': 1}, '01', 'site1')) + result = live_map.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert live_map.data == {} + + +# UTS: objects/unit/RTLO5/object-delete-tombstones-map-0 +def test_rtlo5_object_delete_tombstones_map(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + 'age': ObjectsMapEntry(data=ObjectData(number=30), timeserial='01', tombstone=False), + } + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('map:test@1000', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is True + assert live_map.data == {} + update = _emitted_update(applied, updates) + assert update.update == {'name': 'removed', 'age': 'removed'} + assert update.tombstone is True + assert update.object_message is msg + + +# UTS: objects/unit/RTLO5/tombstone-empty-map-emits-update-0 +def test_rtlo5_tombstone_empty_map_emits_update(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=True, + tombstoned_at=1600000000000), + 'age': ObjectsMapEntry(data=ObjectData(number=30), timeserial='01', tombstone=True, + tombstoned_at=1600000000000), + } + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('map:test@1000', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is True + assert live_map.data == {} + update = _emitted_update(applied, updates) + # The RTLM22c empty-diff exception does not apply to a tombstone diff (RTLO4e5) + assert update.noop is False + assert update.tombstone is True + assert update.update == {} + assert update.object_message is msg + + +# UTS: objects/unit/RTLO4e10/object-delete-root-noop-0 +def test_rtlo4e10_object_delete_root_noop(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('root', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is False + # The data is untouched + assert live_map.data['name'].data.string == 'Alice' + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM14/tombstone-check-objectid-ref-0 +def test_rtlm14_tombstone_check_objectid_ref(): + pool = ObjectsPool() + tombstoned_counter = InternalLiveCounter('counter:dead@1000') + tombstoned_counter.is_tombstone = True + pool['counter:dead@1000'] = tombstoned_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'alive': ObjectsMapEntry(data=ObjectData(string='ok'), timeserial='01', tombstone=False), + 'dead_entry': ObjectsMapEntry(data=None, timeserial='01', tombstone=True), + 'dead_ref': ObjectsMapEntry(data=ObjectData(object_id='counter:dead@1000'), timeserial='01', + tombstone=False), + } + + assert live_map.is_entry_tombstoned(live_map.data['alive']) is False + assert live_map.is_entry_tombstoned(live_map.data['dead_entry']) is True + assert live_map.is_entry_tombstoned(live_map.data['dead_ref']) is True + + +# UTS: objects/unit/RTLM6/replace-data-basic-0 +def test_rtlm6_replace_data_basic(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'old': ObjectsMapEntry(data=ObjectData(string='old'), timeserial='01', tombstone=False), + } + live_map.create_operation_is_merged = True + + state_msg = object_message(build_object_state('root', {'site2': '05'}, map={ + 'semantics': LWW, + 'clearTimeserial': '03', + 'entries': { + 'new': {'data': {'string': 'new'}, 'timeserial': '04', 'tombstone': False}, + }, + })) + update = live_map.replace_data(state_msg) + + assert live_map.site_timeserials == {'site2': '05'} + assert live_map.create_operation_is_merged is False + assert live_map.clear_timeserial == '03' + assert 'old' not in live_map.data + assert live_map.data['new'].data == ObjectData(string='new') + assert update.update == {'old': 'removed', 'new': 'updated'} + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLM6c1/replace-data-tombstoned-entries-0 +def test_rtlm6c1_replace_data_tombstoned_entries(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + + state_msg = object_message(build_object_state('root', {'site1': '01'}, map={ + 'semantics': LWW, + 'entries': { + 'dead': {'tombstone': True, 'timeserial': '01', 'serialTimestamp': 1700000050000}, + }, + })) + live_map.replace_data(state_msg) + + assert live_map.data['dead'].tombstoned_at == 1700000050000 + + +# UTS: objects/unit/RTLM6d/replace-data-with-create-op-0 +def test_rtlm6d_replace_data_with_create_op(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + + state_msg = object_message(build_object_state('map:test@1000', {'site1': '01'}, map={ + 'semantics': LWW, + 'entries': { + 'from_sync': {'data': {'string': 'synced'}, 'timeserial': '01'}, + }, + }, create_op={ + 'mapCreate': { + 'semantics': LWW, + 'entries': { + 'from_create': {'data': {'string': 'created'}, 'timeserial': '00'}, + }, + }, + })) + live_map.replace_data(state_msg) + + assert live_map.data['from_sync'].data == ObjectData(string='synced') + assert live_map.data['from_create'].data == ObjectData(string='created') + assert live_map.create_operation_is_merged is True + + +# UTS: objects/unit/RTLM6f/replace-data-tombstone-flag-0 +def test_rtlm6f_replace_data_tombstone_flag(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + + state_msg = object_message(build_object_state( + 'map:test@1000', {'site1': '01'}, map={'semantics': LWW, 'entries': {}}, tombstone=True)) + update = live_map.replace_data(state_msg) + + assert live_map.is_tombstone is True + assert live_map.data == {} + assert update.update == {'name': 'removed'} + assert update.tombstone is True + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLO4e10/replace-data-tombstone-root-noop-0 +def test_rtlo4e10_replace_data_tombstone_root_noop(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + + state_msg = object_message(build_object_state( + 'root', {'site1': '01'}, map={'semantics': LWW, 'entries': {}}, tombstone=True)) + update = live_map.replace_data(state_msg) + + assert live_map.is_tombstone is False + # The data is untouched + assert live_map.data['name'].data.string == 'Alice' + assert update.noop is True + + +# UTS: objects/unit/RTLM19/gc-tombstoned-entries-0 +def test_rtlm19_gc_tombstoned_entries(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + grace_period = 86400000 + now = 1700100000000 + + live_map.data = { + 'recent_dead': ObjectsMapEntry(data=None, timeserial='01', tombstone=True, tombstoned_at=now - 1000), + 'old_dead': ObjectsMapEntry(data=None, timeserial='01', tombstone=True, + tombstoned_at=now - grace_period - 1), + 'alive': ObjectsMapEntry(data=ObjectData(string='ok'), timeserial='01', tombstone=False), + } + + live_map.gc_tombstoned_entries(grace_period, now) + + assert 'recent_dead' in live_map.data + assert 'old_dead' not in live_map.data + assert 'alive' in live_map.data + + +# UTS: objects/unit/RTLM22/diff-calculation-0 +def test_rtlm22_diff_calculation(): + previous_data = { + 'removed': ObjectsMapEntry(data=ObjectData(string='gone'), timeserial='01', tombstone=False), + 'changed': ObjectsMapEntry(data=ObjectData(string='old'), timeserial='01', tombstone=False), + 'unchanged': ObjectsMapEntry(data=ObjectData(string='same'), timeserial='01', tombstone=False), + 'was_dead': ObjectsMapEntry(data=None, timeserial='01', tombstone=True), + } + new_data = { + 'added': ObjectsMapEntry(data=ObjectData(string='new'), timeserial='02', tombstone=False), + 'changed': ObjectsMapEntry(data=ObjectData(string='new_val'), timeserial='02', tombstone=False), + 'unchanged': ObjectsMapEntry(data=ObjectData(string='same'), timeserial='01', tombstone=False), + 'now_dead': ObjectsMapEntry(data=None, timeserial='02', tombstone=True), + } + + update = InternalLiveMap.diff(previous_data, new_data) + + assert update.update['removed'] == 'removed' + assert update.update['added'] == 'updated' + assert update.update['changed'] == 'updated' + assert 'unchanged' not in update.update + assert 'was_dead' not in update.update + assert 'now_dead' not in update.update + + +# UTS: objects/unit/RTLM22c/empty-diff-is-noop-0 +def test_rtlm22c_empty_diff_is_noop(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='alice'), timeserial='01', tombstone=False), + } + + state_msg = object_message(build_object_state('root', {'site1': '02'}, map={ + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'alice'}, 'timeserial': '02', 'tombstone': False}, + }, + })) + update = live_map.replace_data(state_msg) + + assert update.noop is True + assert live_map.data['name'].data == ObjectData(string='alice') + + +# UTS: objects/unit/RTLM15d4/unsupported-action-0 +def test_rtlm15d4_unsupported_action(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_counter_inc('root', 5, '01', 'site1')) + result = live_map.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + + +# UTS: objects/unit/RTLM6i/replace-data-resets-clear-timeserial-0 +def test_rtlm6i_replace_data_resets_clear_timeserial(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '05' + live_map.data = { + 'x': ObjectsMapEntry(data=ObjectData(number=1), timeserial='03', tombstone=False), + } + + state_msg = object_message(build_object_state('root', {'site1': '01'}, map={ + 'semantics': LWW, + 'entries': { + 'y': {'data': {'number': 2}, 'timeserial': '01'}, + }, + })) + live_map.replace_data(state_msg) + + assert live_map.clear_timeserial is None + assert 'y' in live_map.data + + +# UTS: objects/unit/RTLM14c/tombstoned-ref-yields-null-0 +def test_rtlm14c_tombstoned_ref_yields_null(): + pool = ObjectsPool() + tombstoned_counter = InternalLiveCounter('counter:dead@1000') + tombstoned_counter.is_tombstone = True + pool['counter:dead@1000'] = tombstoned_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref': ObjectsMapEntry(data=ObjectData(object_id='counter:dead@1000'), timeserial='01', tombstone=False), + } + + # The entry itself is not tombstoned, but the object it references is + assert live_map.data['ref'].tombstone is False + # RTLM14c makes the entry tombstoned, so size() does not count it + assert live_map.size() == 0 + assert live_map.get('ref') is None + + +# UTS: objects/unit/RTLM7/map-set-revives-tombstoned-0 +def test_rtlm7_map_set_revives_tombstoned(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=None, timeserial='01', tombstone=True, tombstoned_at=1700000000000), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Alice'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['name'].tombstone is False + assert live_map.data['name'].tombstoned_at is None + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM24/map-clear-preserves-newer-0 +def test_rtlm24_map_clear_preserves_newer(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'before': ObjectsMapEntry(data=ObjectData(string='a'), timeserial='03', tombstone=False), + 'after': ObjectsMapEntry(data=ObjectData(string='b'), timeserial='07', tombstone=False), + 'no_ts': ObjectsMapEntry(data=ObjectData(string='c'), timeserial=None, tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert 'before' not in live_map.data + assert 'no_ts' not in live_map.data + assert live_map.data['after'].data == ObjectData(string='b') + update = _emitted_update(applied, updates) + assert 'before' in update.update + assert 'no_ts' in update.update + assert 'after' not in update.update + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7a3/map-set-overwrite-objectid-parent-refs-0 +def test_rtlm7a3_map_set_overwrite_objectid_parent_refs(): + pool = ObjectsPool() + old_counter = InternalLiveCounter('counter:old@1000') + new_counter = InternalLiveCounter('counter:new@2000') + pool['counter:old@1000'] = old_counter + pool['counter:new@2000'] = new_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref': ObjectsMapEntry(data=ObjectData(object_id='counter:old@1000'), timeserial='01', tombstone=False), + } + # The existing reference, as the map would have recorded it + old_counter.parent_references = {'root': {'ref'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'ref', {'objectId': 'counter:new@2000'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['ref'].data == ObjectData(object_id='counter:new@2000') + # removeParentReference was called on the old child + assert not _references(old_counter, 'root', 'ref') + # addParentReference was called on the new child + assert 'root' in new_counter.parent_references + assert 'ref' in new_counter.parent_references['root'] + update = _emitted_update(applied, updates) + assert update.update == {'ref': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7g2/map-set-new-entry-add-parent-ref-0 +def test_rtlm7g2_map_set_new_entry_add_parent_ref(): + pool = ObjectsPool() + child_counter = InternalLiveCounter('counter:child@1000') + pool['counter:child@1000'] = child_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'score', {'objectId': 'counter:child@1000'}, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['score'].data == ObjectData(object_id='counter:child@1000') + assert 'root' in child_counter.parent_references + assert 'score' in child_counter.parent_references['root'] + update = _emitted_update(applied, updates) + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7/map-set-primitive-no-parent-refs-0 +def test_rtlm7_map_set_primitive_no_parent_refs(): + pool = ObjectsPool() + old_counter = InternalLiveCounter('counter:old@1000') + pool['counter:old@1000'] = old_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref': ObjectsMapEntry(data=ObjectData(object_id='counter:old@1000'), timeserial='01', tombstone=False), + } + old_counter.parent_references = {'root': {'ref'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'ref', {'string': 'plain_value'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['ref'].data == ObjectData(string='plain_value') + # removeParentReference was called on the old child, as the entry referenced it; the new + # value is a primitive, so there is no child to call addParentReference on + assert not _references(old_counter, 'root', 'ref') + update = _emitted_update(applied, updates) + assert update.update == {'ref': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8a3/map-remove-objectid-parent-refs-0 +def test_rtlm8a3_map_remove_objectid_parent_refs(): + pool = ObjectsPool() + child_counter = InternalLiveCounter('counter:child@1000') + pool['counter:child@1000'] = child_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'score': ObjectsMapEntry(data=ObjectData(object_id='counter:child@1000'), timeserial='01', + tombstone=False), + } + child_counter.parent_references = {'root': {'score'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'score', '02', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['score'].tombstone is True + # removeParentReference was called on the child + assert not _references(child_counter, 'root', 'score') + update = _emitted_update(applied, updates) + assert update.update == {'score': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8/map-remove-primitive-no-parent-refs-0 +def test_rtlm8_map_remove_primitive_no_parent_refs(): + pool = ObjectsPool() + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'name', '02', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].tombstone is True + update = _emitted_update(applied, updates) + assert update.update == {'name': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM24e1c/map-clear-parent-refs-0 +def test_rtlm24e1c_map_clear_parent_refs(): + pool = ObjectsPool() + counter_a = InternalLiveCounter('counter:a@1000') + counter_b = InternalLiveCounter('counter:b@1000') + pool['counter:a@1000'] = counter_a + pool['counter:b@1000'] = counter_b + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref_a': ObjectsMapEntry(data=ObjectData(object_id='counter:a@1000'), timeserial='02', tombstone=False), + 'ref_b': ObjectsMapEntry(data=ObjectData(object_id='counter:b@1000'), timeserial='02', tombstone=False), + 'primitive': ObjectsMapEntry(data=ObjectData(string='hello'), timeserial='02', tombstone=False), + 'newer': ObjectsMapEntry(data=ObjectData(string='kept'), timeserial='09', tombstone=False), + } + counter_a.parent_references = {'root': {'ref_a'}} + counter_b.parent_references = {'root': {'ref_b'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + # ref_a, ref_b and primitive are removed (timeserial '02' < '05'); newer is kept ('09' > '05') + assert 'ref_a' not in live_map.data + assert 'ref_b' not in live_map.data + assert 'primitive' not in live_map.data + assert 'newer' in live_map.data + # removeParentReference was called on both child counters + assert not _references(counter_a, 'root', 'ref_a') + assert not _references(counter_b, 'root', 'ref_b') + update = _emitted_update(applied, updates) + assert update.update == {'ref_a': 'removed', 'ref_b': 'removed', 'primitive': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLO4e9/tombstone-map-parent-refs-0 +def test_rtlo4e9_tombstone_map_parent_refs(): + pool = ObjectsPool() + child_counter = InternalLiveCounter('counter:child@1000') + child_map = InternalLiveMap('map:child@1000', ObjectsMapSemantics.LWW) + pool['counter:child@1000'] = child_counter + pool['map:child@1000'] = child_map + + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'counter_ref': ObjectsMapEntry(data=ObjectData(object_id='counter:child@1000'), timeserial='01', + tombstone=False), + 'map_ref': ObjectsMapEntry(data=ObjectData(object_id='map:child@1000'), timeserial='01', tombstone=False), + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + live_map.site_timeserials = {'site1': '00'} + child_counter.parent_references = {'map:test@1000': {'counter_ref'}} + child_map.parent_references = {'map:test@1000': {'map_ref'}} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('map:test@1000', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is True + assert live_map.data == {} + # removeParentReference was called on both children + assert not _references(child_counter, 'map:test@1000', 'counter_ref') + assert not _references(child_map, 'map:test@1000', 'map_ref') + update = _emitted_update(applied, updates) + assert update.update == {'counter_ref': 'removed', 'map_ref': 'removed', 'name': 'removed'} + assert update.tombstone is True + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7a3/map-set-replace-objectid-both-refs-0 +def test_rtlm7a3_map_set_replace_objectid_both_refs(): + pool = ObjectsPool() + old_map = InternalLiveMap('map:old@1000', ObjectsMapSemantics.LWW) + new_map = InternalLiveMap('map:new@2000', ObjectsMapSemantics.LWW) + pool['map:old@1000'] = old_map + pool['map:new@2000'] = new_map + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'child': ObjectsMapEntry(data=ObjectData(object_id='map:old@1000'), timeserial='01', tombstone=False), + } + old_map.parent_references = {'root': {'child'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'child', {'objectId': 'map:new@2000'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['child'].data == ObjectData(object_id='map:new@2000') + # The old child no longer records the reference from root + assert not _references(old_map, 'root', 'child') + # The new child records it + assert 'root' in new_map.parent_references + assert 'child' in new_map.parent_references['root'] + update = _emitted_update(applied, updates) + assert update.update == {'child': 'updated'} + assert update.object_message is msg diff --git a/test/uts/objects/unit/live_object_subscribe_test.py b/test/uts/objects/unit/live_object_subscribe_test.py new file mode 100644 index 00000000..d240e8c8 --- /dev/null +++ b/test/uts/objects/unit/live_object_subscribe_test.py @@ -0,0 +1,321 @@ +"""Derived from uts/objects/unit/live_object_subscribe.md in ably/specification. + +Spec points: RTLO4b, RTLO4b3, RTLO4b4c1, RTLO4b4c3a, RTLO4b4c3c, RTLO4b4d, RTLO4b4e, +RTLO4b6, RTLO4b7, RTINS16e, RTLC14c + +`LiveObject#subscribe` is exercised through `Instance#subscribe` (RTINS16), as the +specification does. ably-python partitions `Instance` by type (RTTS) and only the live +object views have `subscribe`, so a counter's instance is subscribed through +`as_live_counter()` and the root map's through `as_live_map()`. Each listener receives an +`InstanceSubscriptionEvent`, whose `message` is the public `ObjectMessage`. + +Inbound frames are processed on the transport's read task, so a positive is awaited with +`poll_until`, and an exact count or a "listener did not fire" is asserted only once a +control listener on a later dispatch has fired and the loop has settled (the +specification's negative-assertion quiescence pattern). +""" + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.subscription import Subscription +from test.uts.helpers.client import poll_until +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + assert_unchanged_after_quiescence, + build_counter_inc, + build_map_set, + build_object_delete, + build_object_message, + build_public_object_message, + remote_serial, + setup_synced_channel, +) + + +# UTS: objects/unit/RTLO4b/subscribe-receives-updates-0 +async def test_rtlo4b_subscribe_receives_updates(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + sub = instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + await settle() + + assert isinstance(sub, Subscription) + assert len(updates) == 1 + + +# UTS: objects/unit/RTLO4b7/subscribe-returns-subscription-0 +async def test_rtlo4b7_subscribe_returns_subscription(): + ctx = await setup_synced_channel('test') + instance = ctx.root.get('score').instance() + + sub = instance.as_live_counter().subscribe(lambda event: None) + + assert isinstance(sub, Subscription) + assert callable(sub.unsubscribe) + + +# UTS: objects/unit/RTLO4b7/subscription-unsubscribe-stops-delivery-0 +async def test_rtlo4b7_subscription_unsubscribe_stops_delivery(): + ctx = await setup_synced_channel('test') + updates = [] + control = [] + instance = ctx.root.get('score').instance() + sub = instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, '01', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the first update to be delivered') + + sub.unsubscribe() + + # A control listener fires on the same dispatch as the message under test, so once it + # has fired the unsubscribed listener would also have run had it still been registered + instance.as_live_counter().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, '02', 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: len(updates), lambda: len(control) >= 1, + description='the control listener received the second update') + + assert len(updates) == 1 + + +# UTS: objects/unit/RTLO4b7/subscription-unsubscribe-idempotent-0 +async def test_rtlo4b7_subscription_unsubscribe_idempotent(): + ctx = await setup_synced_channel('test') + instance = ctx.root.get('score').instance() + sub = instance.as_live_counter().subscribe(lambda event: None) + + # Neither call raises + sub.unsubscribe() + sub.unsubscribe() + + +# UTS: objects/unit/RTLO4b4c1/noop-no-trigger-0 +async def test_rtlo4b4c1_noop_no_trigger(): + ctx = await setup_synced_channel('test') + updates = [] + control = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, '01', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the first update to be delivered') + + # Serial "02" passes the newness check (RTLO4a6), and an increment with no `number` is + # the no-op (RTLC9h). `number: 0` would be present per RTLC9g and give a real update of 0. + ctx.mock_ws.send_to_client(build_object_message('test', [{ + 'serial': '02', + 'siteCode': 'remote', + 'operation': { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:score@1000', + 'counterInc': {}, + }, + }])) + # The follow-up "03" is dispatched after the no-op, so once its separate control + # listener fires the no-op has certainly been processed + control_sub = instance.as_live_counter().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 3, '03', 'remote'), + ])) + await poll_until(lambda: len(control) >= 1, timeout=5, description='the control listener to receive "03"') + await settle() + control_sub.unsubscribe() + + # The original listener fired for "01" and "03" only; had the no-op fired it would be 3 + assert len(updates) == 2 + + +# UTS: objects/unit/RTLO4b6/subscribe-no-side-effects-0 +async def test_rtlo4b6_subscribe_no_side_effects(): + ctx = await setup_synced_channel('test') + state_before = ctx.channel.state + sync_state_before = ctx.channel.object._sync_state + sent_before = len(ctx.mock_ws.messages_from_client) + instance = ctx.root.get('score').instance() + + instance.as_live_counter().subscribe(lambda event: None) + await settle() + + assert ctx.channel.state == state_before + # Nor does it change the objects' sync state or send anything on the channel + assert ctx.channel.object._sync_state == sync_state_before + assert len(ctx.mock_ws.messages_from_client) == sent_before + + +# UTS: objects/unit/RTLO4b/subscribe-map-update-0 +async def test_rtlo4b_subscribe_map_update(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.instance() + instance.as_live_map().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the map update to be delivered') + await settle() + + assert len(updates) == 1 + + +# UTS: objects/unit/RTLO4b4c3c/tombstone-deregisters-listeners-0 +async def test_rtlo4b4c3c_tombstone_deregisters_listeners(): + ctx = await setup_synced_channel('test') + updates_a = [] + updates_b = [] + control = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates_a.append(event)) + instance.as_live_counter().subscribe(lambda event: updates_b.append(event)) + + # An OBJECT_DELETE tombstones the counter + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '50', 'remote'), + ])) + # Both listeners are awaited on this dispatch before either count is asserted + await poll_until(lambda: len(updates_a) >= 1, timeout=5, description='listener A to receive the tombstone') + await poll_until(lambda: len(updates_b) >= 1, timeout=5, description='listener B to receive the tombstone') + await settle() + + # Both listeners received the tombstone update + assert len(updates_a) == 1 + assert updates_a[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + assert len(updates_b) == 1 + assert updates_b[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + + # A tombstoned object ignores further operations (RTLC7e), so no listener on the counter + # can serve as the control. A listener on another live object can: the "52" MAP_SET on + # the profile map is processed after the "51" increment, so once it fires "51" has been too. + control_inst = ctx.root.get('profile').instance() + control_inst.as_live_map().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 3, '51', 'remote'), + ])) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'quiescence_probe', {'string': 'x'}, '52', 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: (len(updates_a), len(updates_b)), lambda: len(control) >= 1, + description='the control listener on the profile map fired') + + # The tombstone deregistered both listeners (RTLO4b4c3c) + assert len(updates_a) == 1 + assert len(updates_b) == 1 + + +# UTS: objects/unit/RTLO4b4c3c/tombstone-zero-value-counter-tears-down-0 +async def test_rtlo4b4c3c_tombstone_zero_value_counter_tears_down(): + ctx = await setup_synced_channel('test') + updates_a = [] + updates_b = [] + control = [] + instance = ctx.root.get('score').instance() + + # The counter (100 in the standard pool) is driven to 0 before the listeners under test + # are registered, so they observe only the tombstone + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', -100, '40', 'remote'), + ])) + await poll_until(lambda: ctx.root.get('score').as_live_counter().value() == 0, timeout=5, + description='the counter to reach 0') + + instance.as_live_counter().subscribe(lambda event: updates_a.append(event)) + instance.as_live_counter().subscribe(lambda event: updates_b.append(event)) + + # The OBJECT_DELETE tombstones an already-zero counter: a zero-delta diff, which for a + # tombstone is not a no-op (RTLC14c) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '50', 'remote'), + ])) + await poll_until(lambda: len(updates_a) >= 1, timeout=5, description='listener A to receive the tombstone') + await poll_until(lambda: len(updates_b) >= 1, timeout=5, description='listener B to receive the tombstone') + await settle() + + # Both listeners received the tombstone update although the counter's data did not change + assert len(updates_a) == 1 + assert updates_a[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + assert len(updates_b) == 1 + assert updates_b[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + + # A separate live object is the quiescence barrier, as in the populated case + control_inst = ctx.root.get('profile').instance() + control_inst.as_live_map().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 3, '51', 'remote'), + ])) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'quiescence_probe', {'string': 'x'}, '52', 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: (len(updates_a), len(updates_b)), lambda: len(control) >= 1, + description='the control listener on the profile map fired') + + # The tombstone deregistered both listeners (RTLO4b4c3c) + assert len(updates_a) == 1 + assert len(updates_b) == 1 + + +# UTS: objects/unit/RTLO4b4d/update-has-object-message-0 +async def test_rtlo4b4d_update_has_object_message(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + counter_inc = build_counter_inc('counter:score@1000', 7, '99', 'remote') + ctx.mock_ws.send_to_client(build_object_message('test', [counter_inc])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + await settle() + + assert len(updates) == 1 + assert updates[0].message is not None + assert updates[0].message.serial == '99' + assert updates[0].message.site_code == 'remote' + assert updates[0].message.operation.action == ObjectOperationAction.COUNTER_INC + assert updates[0].message.operation.object_id == 'counter:score@1000' + # RTINS16e2: the message is the public ObjectMessage derived from the source per PAOM3 + assert isinstance(updates[0].message, publicmessage.ObjectMessage) + assert updates[0].message == build_public_object_message(counter_inc, 'test') + + +# UTS: objects/unit/RTLO4b4e/tombstone-flag-true-0 +async def test_rtlo4b4e_tombstone_flag_true(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '50', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the tombstone update to be delivered') + await settle() + + assert len(updates) == 1 + assert updates[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + + +# UTS: objects/unit/RTLO4b4e/tombstone-flag-false-0 +async def test_rtlo4b4e_tombstone_flag_false(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + await settle() + + assert len(updates) == 1 + assert updates[0].message.operation.action == ObjectOperationAction.COUNTER_INC diff --git a/test/uts/objects/unit/object_id_test.py b/test/uts/objects/unit/object_id_test.py new file mode 100644 index 00000000..d6f43184 --- /dev/null +++ b/test/uts/objects/unit/object_id_test.py @@ -0,0 +1,99 @@ +"""Derived from uts/objects/unit/object_id.md in ably/specification. + +Spec points: RTO14, RTO14a1, RTO14b, RTO14b1, RTO14b2, RTO14c + +A pure function: `generate_object_id(object_type, initial_value, nonce, timestamp_ms)` is the +specification's `generateObjectId(type:, initialValue:, nonce:, timestamp:)`, producing +`[type]:[hash]@[timestamp]`. +""" + +import base64 +import hashlib +import re + +from ably.pubsub.objects.objectid import generate_object_id + +# RFC 4648 s.5's URL-safe alphabet, with no padding +BASE64URL_UNPADDED = re.compile(r'[A-Za-z0-9_-]+') + + +def _is_base64url(value): + """Whether `value` is unpadded base64url that decodes: the alphabet, and a length no + encoding produces (one more than a multiple of four) excluded.""" + if not BASE64URL_UNPADDED.fullmatch(value) or len(value) % 4 == 1: + return False + base64.urlsafe_b64decode(value + '=' * (-len(value) % 4)) + return True + + +def _expected_hash(initial_value, nonce): + """RTO14b, computed here independently: the unpadded base64url SHA-256 digest of the + UTF-8 string `initial_value:nonce`.""" + digest = hashlib.sha256(f'{initial_value}:{nonce}'.encode()).digest() + return base64.urlsafe_b64encode(digest).decode('ascii').rstrip('=') + + +# UTS: objects/unit/RTO14/objectid-format-counter-0 +def test_rto14_objectid_format_counter(): + object_id = generate_object_id('counter', '{"counter":{"count":42}}', 'test-nonce-12345678', 1700000000000) + + assert object_id.startswith('counter:') + assert '@1700000000000' in object_id + parts = object_id.split(':') + # The shape is asserted before indexing, so a malformed id fails on an assertion + assert len(parts) == 2 + type_part = parts[0] + rest = parts[1] + hash_and_ts = rest.split('@') + assert len(hash_and_ts) == 2 + hash_part = hash_and_ts[0] + ts_part = hash_and_ts[1] + assert type_part == 'counter' + assert ts_part == '1700000000000' + assert _is_base64url(hash_part) + assert '+' not in hash_part + assert '/' not in hash_part + assert '=' not in hash_part + # The hash itself, which RTO14b1 and RTO14b2 determine and the specification's + # assertions leave open: the digest of the initial value and the nonce, joined by ':' + assert hash_part == _expected_hash('{"counter":{"count":42}}', 'test-nonce-12345678') + + +# UTS: objects/unit/RTO14/objectid-format-map-0 +def test_rto14_objectid_format_map(): + object_id = generate_object_id( + 'map', '{"map":{"semantics":"LWW","entries":{}}}', 'test-nonce-12345678', 1700000000000) + + assert object_id.startswith('map:') + assert '@1700000000000' in object_id + + +# UTS: objects/unit/RTO14/deterministic-0 +def test_rto14_deterministic(): + id1 = generate_object_id('counter', '{"counter":{"count":0}}', 'same-nonce-1234567', 1700000000000) + id2 = generate_object_id('counter', '{"counter":{"count":0}}', 'same-nonce-1234567', 1700000000000) + + assert id1 == id2 + + +# UTS: objects/unit/RTO14/different-nonce-0 +def test_rto14_different_nonce(): + id1 = generate_object_id('counter', '{"counter":{"count":0}}', 'nonce-aaaaaaaaaaaaa', 1700000000000) + id2 = generate_object_id('counter', '{"counter":{"count":0}}', 'nonce-bbbbbbbbbbbbb', 1700000000000) + + assert id1 != id2 + + +# UTS: objects/unit/RTO14b/base64url-encoding-0 +def test_rto14b_base64url_encoding(): + object_id = generate_object_id('counter', '{"counter":{"count":0}}', 'test-nonce-12345678', 1700000000000) + parts = object_id.split(':') + # The shape is asserted before indexing, so a malformed id fails on an assertion + assert len(parts) == 2 + hash_and_ts = parts[1].split('@') + assert len(hash_and_ts) == 2 + hash_part = hash_and_ts[0] + + assert '+' not in hash_part + assert '/' not in hash_part + assert not hash_part.endswith('=') diff --git a/test/uts/objects/unit/objects_pool_test.py b/test/uts/objects/unit/objects_pool_test.py new file mode 100644 index 00000000..04843e5e --- /dev/null +++ b/test/uts/objects/unit/objects_pool_test.py @@ -0,0 +1,687 @@ +"""Derived from uts/objects/unit/objects_pool.md in ably/specification. + +Spec points: RTO3, RTO3a, RTO3b, RTO3b1, RTO4, RTO4a, RTO4b, RTO4b1, RTO4b2, RTO4b2a, RTO4b4, +RTO4c, RTO4d, RTO5, RTO5a1, RTO5a2, RTO5a2a, RTO5a4, RTO5a5, RTO5a6, RTO5c, RTO5c2, RTO5c2a, +RTO5c6, RTO5c7, RTO5c8, RTO5c9, RTO5c10, RTO5c10a, RTO5c10b, RTO5d, RTO5e, RTO5f1, RTO5f2, +RTO5f2a2, RTO5f2b, RTO5f3, RTO6, RTO6a, RTO6b1, RTO6b2, RTO6b3, RTO7, RTO7a, RTO8a, RTO8b, +RTO9a1, RTO9a2a4, RTO9a2b, RTO9a3, RTLM23 + +A pure unit specification: nothing is mocked and nothing is connected. The specification +puts the sync and apply state machine on `ObjectsPool` (`processAttached`, +`processObjectSync`, `processObjectMessage`, `applyObjectMessages`, `syncState`). Here, as +in features.md (RTO4-RTO9, RTO17), it is `RealtimeObject`'s, and the pool is a mapping of +object ids to objects (RTO3a); this is shape deviation S-2. A test that builds only +`pool = ObjectsPool()` and then drives the state machine works on the pool a standalone +`RealtimeObject()` holds (`_driven_pool()` below); one that builds +`RealtimeObject(pool: pool)` does the same with `RealtimeObject(pool=pool)`. The +`_process_*` helpers below are the specification's three `process*` calls. + +The channel hands `RealtimeObject` an ATTACHED's HAS_OBJECTS flag and nothing else (RTO4), +so the `channelSerial` the specification gives its ATTACHED messages goes no further than +the message dictionary. An OBJECT_SYNC's `channelSerial` is the sync cursor (RTO5a1) and is +passed on. + +Updates are emitted synchronously (`LiveObject.notify_updated`), so a listener registered +with `subscribe` has seen every update by the time the call that caused it returns. The +tests that drive a `RealtimeObject` are coroutines only so that an implementation is free to +touch the event loop. +""" + +from ably.pubsub.objects.enums import ObjectsOperationSource, ObjectsSyncState +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.objectmessage import ObjectData, ObjectOperationAction, ObjectsMapEntry +from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.realtimeobject import RealtimeObject +from test.uts.objects.helpers.standard_test_pool import ( + ATTACHED, + HAS_OBJECTS, + LWW, + build_counter_inc, + build_map_set, + build_object_message, + build_object_state, + build_object_sync_message, + object_message, + object_messages, + objects_attached_message, +) + + +def _driven_pool(): + """The specification's `pool = ObjectsPool()`, in a test that then drives the sync state machine. + + S-2: the state machine is `RealtimeObject`'s, so the pool is the one a standalone + `RealtimeObject` holds. Returns `(realtime_object, pool)`. + """ + realtime_object = RealtimeObject() + return realtime_object, realtime_object._objects_pool + + +def _process_attached(realtime_object, attached): + """The specification's `pool.processAttached(attached)` (S-2). + + `RealtimeObject` is given the ATTACHED's HAS_OBJECTS flag; a message with no `flags` + has the flag unset (RTO4b). + """ + realtime_object._on_attached(has_objects=bool(attached.get('flags', 0) & HAS_OBJECTS)) + + +def _process_object_sync(realtime_object, protocol_message): + """The specification's `pool.processObjectSync(protocol_message)` (S-2).""" + realtime_object._handle_object_sync_messages( + object_messages(protocol_message), protocol_message.get('channelSerial')) + + +def _process_object_message(realtime_object, protocol_message): + """The specification's `pool.processObjectMessage(protocol_message)` (S-2).""" + realtime_object._handle_object_messages(object_messages(protocol_message)) + + +def _empty_root_state(): + """The root state most syncs carry: an empty LWW map whose create operation adds nothing.""" + return build_object_state('root', {'aaa': 't:0'}, map={'semantics': LWW, 'entries': {}}, + create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}) + + +# UTS: objects/unit/RTO3/pool-init-root-0 +def test_rto3_pool_init_root(): + pool = ObjectsPool() + + assert 'root' in pool + assert isinstance(pool['root'], InternalLiveMap) + assert pool['root'].data == {} + assert pool['root'].object_id == 'root' + + +# UTS: objects/unit/RTO4/attached-has-objects-syncing-0 +async def test_rto4_attached_has_objects_syncing(): + realtime_object, pool = _driven_pool() + + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + + +# UTS: objects/unit/RTO4b/attached-no-objects-synced-0 +async def test_rto4b_attached_no_objects_synced(): + realtime_object, pool = _driven_pool() + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + pool['root'].data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + root = pool['root'] + + updates = [] + pool['root'].subscribe(updates.append) + + _process_attached(realtime_object, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:abc@1000' not in pool + assert 'root' in pool + # RTO4b2: the root map's data is cleared in place; the root is never replaced + assert pool['root'] is root + assert pool['root'].data == {} + assert len(updates) >= 1 + assert updates[0].update == {'name': 'removed'} + assert updates[0].object_message is None + + +# UTS: objects/unit/RTO4b2a/reset-of-empty-root-emits-no-update-0 +async def test_rto4b2a_reset_of_empty_root_emits_no_update(): + realtime_object, pool = _driven_pool() + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + # The root is already empty, the zero value of RTLM4c + pool['root'].data = {} + + updates = [] + pool['root'].subscribe(updates.append) + + _process_attached(realtime_object, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + # RTO4b1: the objects other than the root are still removed + assert 'counter:abc@1000' not in pool + assert 'root' in pool + assert pool['root'].data == {} + # RTO4b2a: no key was removed, so the update has no changed keys, is a no-op and is + # not delivered + assert len(updates) == 0 + + # Liveness control: a reset that does remove a key is delivered, so the count of zero + # above is the empty root's no-op and not a dead subscription. Emission is synchronous + # at this tier, so there is nothing to wait for. + realtime_object2, pool2 = _driven_pool() + pool2['root'].data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + control = [] + pool2['root'].subscribe(control.append) + _process_attached(realtime_object2, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + assert len(control) >= 1 + assert control[0].update == {'name': 'removed'} + + +# UTS: objects/unit/RTO5/sync-complete-sequence-0 +async def test_rto5_sync_complete_sequence(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}}, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 42}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'root' in pool + assert 'counter:abc@1000' in pool + assert pool['root'].data['name'].data == ObjectData(string='Alice') + assert pool['counter:abc@1000'].data == 42 + + +# UTS: objects/unit/RTO5a2/new-sequence-discards-old-0 +async def test_rto5a2_new_sequence_discards_old(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'seq1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'seq1:more', [ + build_object_state('counter:old@1000', {'aaa': 't:0'}, counter={'count': 10}), + ])) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'seq2:', [ + _empty_root_state(), + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:old@1000' not in pool + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5a5/absent-channel-serial-0 +async def test_rto5a5_absent_channel_serial(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # No channelSerial: the whole sync is contained in this one message (RTO5a5) + _process_object_sync(realtime_object, build_object_sync_message('test', None, [ + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5a6/malformed-channel-serial-treated-as-absent-0 +async def test_rto5a6_malformed_channel_serial_treated_as_absent(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # 'malformedserialnocolon' has no ':' separator, so it cannot be parsed per RTO5a1; + # RTO5a6 handles it as if the channelSerial were absent (RTO5a5) + _process_object_sync(realtime_object, build_object_sync_message('test', 'malformedserialnocolon', [ + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + # Treated as absent (RTO5a5): the message was applied and the sync ended + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5f2a/partial-map-merge-0 +async def test_rto5f2a_partial_map_merge(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}}, + }), + ])) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'age': {'data': {'number': 30}, 'timeserial': 't:0'}}, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + ])) + + assert pool['root'].data['name'].data == ObjectData(string='Alice') + assert pool['root'].data['age'].data == ObjectData(number=30) + + +# UTS: objects/unit/RTO5c2/remove-absent-objects-0 +async def test_rto5c2_remove_absent_objects(): + realtime_object, pool = _driven_pool() + pool['counter:old@1000'] = InternalLiveCounter('counter:old@1000') + pool['counter:old@1000'].data = 99 + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [_empty_root_state()])) + + assert 'counter:old@1000' not in pool + assert 'root' in pool + + +# UTS: objects/unit/RTO5c9/clear-applied-on-ack-serials-0 +async def test_rto5c9_clear_applied_on_ack_serials(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + realtime_object._applied_on_ack_serials = {'serial-1', 'serial-2'} + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [_empty_root_state()])) + + # The specification's `{}` is the empty set + assert realtime_object._applied_on_ack_serials == set() + + +# UTS: objects/unit/RTO8a/buffer-during-syncing-0 +async def test_rto8a_buffer_during_syncing(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + assert len(realtime_object._buffered_object_operations) == 1 + assert 'counter:abc@1000' not in pool + + +# UTS: objects/unit/RTO5c6/apply-buffered-on-sync-0 +async def test_rto5c6_apply_buffered_on_sync(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 10, '02', 'site1'), + ])) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + _empty_root_state(), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + ])) + + assert pool['counter:abc@1000'].data == 110 + assert len(realtime_object._buffered_object_operations) == 0 + + +# UTS: objects/unit/RTO9a1/null-operation-warning-0 +async def test_rto9a1_null_operation_warning(): + realtime_object, pool = _driven_pool() + realtime_object._sync_state = ObjectsSyncState.SYNCED + + # The specification's `ObjectMessage(serial: "01", siteCode: "site1", operation: null)` + _process_object_message(realtime_object, build_object_message('test', [ + {'serial': '01', 'siteCode': 'site1'}, + ])) + + assert len(pool) == 1 + + +# UTS: objects/unit/RTO9a3/dedup-applied-on-ack-0 +async def test_rto9a3_dedup_applied_on_ack(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + realtime_object._sync_state = ObjectsSyncState.SYNCED + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + pool['counter:abc@1000'].data = 10 + realtime_object._applied_on_ack_serials = {'echo-serial-1'} + + _process_object_message(realtime_object, build_object_message('test', [ + { + 'serial': 'echo-serial-1', + 'siteCode': 'site1', + 'operation': { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:abc@1000', + 'counterInc': {'number': 5}, + }, + }, + ])) + + assert pool['counter:abc@1000'].data == 10 + assert 'echo-serial-1' not in realtime_object._applied_on_ack_serials + + +# UTS: objects/unit/RTO9a2a4/local-source-adds-serial-0 +async def test_rto9a2a4_local_source_adds_serial(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + realtime_object._sync_state = ObjectsSyncState.SYNCED + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + + realtime_object._apply_object_messages([ + object_message(build_counter_inc('counter:abc@1000', 5, 'local-serial-1', 'test-site')), + ], ObjectsOperationSource.LOCAL) + + assert 'local-serial-1' in realtime_object._applied_on_ack_serials + assert pool['counter:abc@1000'].data == 5 + + +# UTS: objects/unit/RTO9a2b/unsupported-action-warning-0 +async def test_rto9a2b_unsupported_action_warning(): + realtime_object, pool = _driven_pool() + realtime_object._sync_state = ObjectsSyncState.SYNCED + + # The specification's `action: "UNKNOWN_ACTION"`: 99 is no ObjectOperationAction, so it + # decodes to UNKNOWN (OOP2a) + _process_object_message(realtime_object, build_object_message('test', [ + {'serial': '01', 'siteCode': 'site1', 'operation': {'action': 99, 'objectId': 'counter:abc@1000'}}, + ])) + + assert len(pool) == 1 + + +# UTS: objects/unit/RTO6/zero-value-from-prefix-0 +async def test_rto6_zero_value_from_prefix(): + realtime_object, pool = _driven_pool() + realtime_object._sync_state = ObjectsSyncState.SYNCED + + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:new@2000', 5, '01', 'site1'), + ])) + _process_object_message(realtime_object, build_object_message('test', [ + build_map_set('map:new@2000', 'key', {'string': 'val'}, '01', 'site1'), + ])) + + assert 'counter:new@2000' in pool + assert isinstance(pool['counter:new@2000'], InternalLiveCounter) + assert pool['counter:new@2000'].data == 5 + + assert 'map:new@2000' in pool + assert isinstance(pool['map:new@2000'], InternalLiveMap) + assert pool['map:new@2000'].data['key'].data == ObjectData(string='val') + + +# UTS: objects/unit/RTO5d/null-object-skipped-0 +async def test_rto5d_null_object_skipped(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # The specification's `ObjectMessage(object: null)` is the empty message `{}` + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + {}, + _empty_root_state(), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + + +# UTS: objects/unit/RTO5f3/unsupported-type-skipped-0 +async def test_rto5f3_unsupported_type_skipped(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + _empty_root_state(), + {'object': {'objectId': 'unknown:xyz@1000', 'siteTimeserials': {}}}, + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'unknown:xyz@1000' not in pool + + +# UTS: objects/unit/RTO5e/object-sync-transitions-syncing-0 +async def test_rto5e_object_sync_transitions_syncing(): + realtime_object, pool = _driven_pool() + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('root', {'aaa': 't:0'}, map={'semantics': LWW, 'entries': {}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + + +# UTS: objects/unit/RTO5c7/sync-emits-updates-0 +async def test_rto5c7_sync_emits_updates(): + realtime_object, pool = _driven_pool() + pool['root'].data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Old'), timeserial='01', tombstone=False), + } + + updates = [] + pool['root'].subscribe(updates.append) + + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'New'}, 'timeserial': 't:0'}}, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + ])) + + assert len(updates) >= 1 + assert 'name' in updates[0].update + assert updates[0].update['name'] == 'updated' + + +# UTS: objects/unit/RTO5f2b/partial-counter-error-0 +async def test_rto5f2b_partial_counter_error(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 10}), + ])) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + _empty_root_state(), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 5}), + ])) + + assert pool['counter:abc@1000'].data == 10 + + +# UTS: objects/unit/RTO4d/attached-clears-buffer-0 +async def test_rto4d_attached_clears_buffer(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + assert len(realtime_object._buffered_object_operations) == 1 + + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + + assert len(realtime_object._buffered_object_operations) == 0 + + +# UTS: objects/unit/RTO4-RTO5/attached-during-syncing-resets-0 +async def test_rto4_rto5_attached_during_syncing_resets(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('counter:old@1000', {'aaa': 't:0'}, counter={'count': 10}), + ])) + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync2:', [ + _empty_root_state(), + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:old@1000' not in pool + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5-RTO7/new-sync-keeps-buffer-0 +async def test_rto5_rto7_new_sync_keeps_buffer(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + assert len(realtime_object._buffered_object_operations) == 1 + + _process_object_sync(realtime_object, build_object_sync_message('test', 'seq2:', [ + _empty_root_state(), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert pool['counter:abc@1000'].data == 105 + + +# UTS: objects/unit/RTO7-RTO8/buffer-without-attached-0 +async def test_rto7_rto8_buffer_without_attached(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + assert realtime_object._sync_state == ObjectsSyncState.INITIALIZED + + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + + assert len(realtime_object._buffered_object_operations) == 1 + + +# UTS: objects/unit/RTO5c-RTLM23/sync-clear-timeserial-hides-create-entries-0 +async def test_rto5c_rtlm23_sync_clear_timeserial_hides_create_entries(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {}, + 'clearTimeserial': '05', + }, create_op={'mapCreate': { + 'semantics': LWW, + 'entries': { + 'old_key': {'data': {'string': 'old'}, 'timeserial': '03'}, + 'new_key': {'data': {'string': 'new'}, 'timeserial': '07'}, + }, + }}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'old_key' not in pool['root'].data + assert pool['root'].data['new_key'].data == ObjectData(string='new') + + +# UTS: objects/unit/RTO5c10/sync-rebuilds-parent-refs-0 +async def test_rto5c10_sync_rebuilds_parent_refs(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'score': {'data': {'objectId': 'counter:score@1000'}, 'timeserial': 't:0'}, + 'profile': {'data': {'objectId': 'map:profile@1000'}, 'timeserial': 't:0'}, + 'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:score@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + build_object_state('map:profile@1000', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'nested_counter': {'data': {'objectId': 'counter:nested@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:nested@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 5}}), + ])) + + # root is not referenced by any parent + assert pool['root'].parent_references == {} + # counter:score@1000 is referenced by root at key 'score' + assert pool['counter:score@1000'].parent_references == {'root': {'score'}} + # map:profile@1000 is referenced by root at key 'profile' + assert pool['map:profile@1000'].parent_references == {'root': {'profile'}} + # counter:nested@1000 is referenced by map:profile@1000 at key 'nested_counter' + assert pool['counter:nested@1000'].parent_references == {'map:profile@1000': {'nested_counter'}} + # The primitive-valued entry 'name' appears in no parent_references, which the + # equalities above already establish + + +# UTS: objects/unit/RTO5c10/resync-rebuilds-parent-refs-0 +async def test_rto5c10_resync_rebuilds_parent_refs(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # First sync: counter:abc@1000 is a child of root + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'counter_key': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 10}}), + ])) + assert pool['counter:abc@1000'].parent_references == {'root': {'counter_key'}} + + # Second sync: counter:abc@1000 is a child of map:wrapper@1000, not of root + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync2:', [ + build_object_state('root', {'aaa': 't:1'}, map={ + 'semantics': LWW, + 'entries': { + 'wrapper': {'data': {'objectId': 'map:wrapper@1000'}, 'timeserial': 't:1'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('map:wrapper@1000', {'aaa': 't:1'}, map={ + 'semantics': LWW, + 'entries': { + 'moved_counter': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:1'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 20}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + # root is not referenced by any parent + assert pool['root'].parent_references == {} + # map:wrapper@1000 is a child of root at key 'wrapper' + assert pool['map:wrapper@1000'].parent_references == {'root': {'wrapper'}} + # counter:abc@1000 is a child of map:wrapper@1000, and no longer of root + assert pool['counter:abc@1000'].parent_references == {'map:wrapper@1000': {'moved_counter'}} + + +# UTS: objects/unit/RTO5c10/empty-sync-parent-refs-0 +async def test_rto5c10_empty_sync_parent_refs(): + realtime_object, pool = _driven_pool() + + # A normal sync first, to populate parent_references + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'child': {'data': {'objectId': 'counter:child@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:child@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 1}}), + ])) + assert pool['counter:child@1000'].parent_references == {'root': {'child'}} + + # An empty sync: ATTACHED without HAS_OBJECTS + _process_attached(realtime_object, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + # counter:child@1000 was removed from the pool (RTO4b1) + assert 'counter:child@1000' not in pool + # root remains, with empty data and empty parent_references + assert 'root' in pool + assert pool['root'].data == {} + assert pool['root'].parent_references == {} diff --git a/test/uts/objects/unit/parent_references_test.py b/test/uts/objects/unit/parent_references_test.py new file mode 100644 index 00000000..20697c74 --- /dev/null +++ b/test/uts/objects/unit/parent_references_test.py @@ -0,0 +1,433 @@ +"""Derived from uts/objects/unit/parent_references.md in ably/specification. + +Spec points: RTLO3f, RTLO3f2, RTLO4g, RTLO4g1, RTLO4g2, RTLO4h, RTLO4h1, RTLO4h2, RTLO4h3, +RTLO4f, RTLO4f1, RTLO4f2, RTLO4f3, RTLO4f4, RTO5c10, RTO5c10a, RTO5c10b + +A pure unit specification: nothing is mocked and nothing is connected. Objects are built on +their own and then put in a pool with `pool[object_id] = obj`, which adopts them, so +`get_full_paths()` resolves their parents' ids through that pool. A path is a list of keys, +so the specification's `paths CONTAINS ["a", "x"]` is `['a', 'x'] in paths`. + +The three RTO5c10 cases drive a sync, which the specification does through +`ObjectsPool.processAttached` and `processObjectSync`. The sync state machine is +`RealtimeObject`'s (shape deviation S-2), so those cases work on the pool a standalone +`RealtimeObject()` holds and drive the `RealtimeObject`, through the helpers below; they are +coroutines only so that an implementation is free to touch the event loop. +""" + +from ably.pubsub.objects.enums import ObjectsSyncState +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.objectmessage import ObjectsMapSemantics +from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.realtimeobject import RealtimeObject +from test.uts.objects.helpers.standard_test_pool import ( + HAS_OBJECTS, + LWW, + build_object_state, + build_object_sync_message, + object_messages, + objects_attached_message, +) + + +def _driven_pool(): + """The specification's `pool = ObjectsPool()`, in a test that then drives the sync state machine. + + S-2: the state machine is `RealtimeObject`'s, so the pool is the one a standalone + `RealtimeObject` holds. Returns `(realtime_object, pool)`. + """ + realtime_object = RealtimeObject() + return realtime_object, realtime_object._objects_pool + + +def _process_attached(realtime_object, attached): + """The specification's `pool.processAttached(attached)` (S-2). + + `RealtimeObject` is given the ATTACHED's HAS_OBJECTS flag and nothing else (RTO4). + """ + realtime_object._on_attached(has_objects=bool(attached.get('flags', 0) & HAS_OBJECTS)) + + +def _process_object_sync(realtime_object, protocol_message): + """The specification's `pool.processObjectSync(protocol_message)` (S-2).""" + realtime_object._handle_object_sync_messages( + object_messages(protocol_message), protocol_message.get('channelSerial')) + + +# UTS: objects/unit/RTLO3f2/init-empty-counter-0 +def test_rtlo3f2_init_empty_counter(): + counter = InternalLiveCounter('counter:abc@1000') + + assert counter.parent_references == {} + + +# UTS: objects/unit/RTLO3f2/init-empty-map-0 +def test_rtlo3f2_init_empty_map(): + live_map = InternalLiveMap('map:abc@1000', ObjectsMapSemantics.LWW) + + assert live_map.parent_references == {} + + +# UTS: objects/unit/RTLO4g2/first-reference-new-entry-0 +def test_rtlo4g2_first_reference_new_entry(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + + child.add_parent_reference(parent, 'score') + + assert 'map:parent@1000' in child.parent_references + assert child.parent_references['map:parent@1000'] == {'score'} + + +# UTS: objects/unit/RTLO4g1/second-key-same-parent-0 +def test_rtlo4g1_second_key_same_parent(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score'}} + + child.add_parent_reference(parent, 'points') + + assert child.parent_references['map:parent@1000'] == {'score', 'points'} + + +# UTS: objects/unit/RTLO4g/different-parent-separate-entry-0 +def test_rtlo4g_different_parent_separate_entry(): + child = InternalLiveCounter('counter:child@1000') + parent_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + parent_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + + child.add_parent_reference(parent_a, 'x') + child.add_parent_reference(parent_b, 'y') + + assert child.parent_references['map:a@1000'] == {'x'} + assert child.parent_references['map:b@1000'] == {'y'} + + +# UTS: objects/unit/RTLO4g/multiple-parents-multiple-keys-0 +def test_rtlo4g_multiple_parents_multiple_keys(): + child = InternalLiveCounter('counter:child@1000') + parent_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + parent_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + + child.add_parent_reference(parent_a, 'x') + child.add_parent_reference(parent_a, 'y') + child.add_parent_reference(parent_b, 'p') + child.add_parent_reference(parent_b, 'q') + + assert child.parent_references['map:a@1000'] == {'x', 'y'} + assert child.parent_references['map:b@1000'] == {'p', 'q'} + + +# UTS: objects/unit/RTLO4h1/nonexistent-parent-noop-0 +def test_rtlo4h1_nonexistent_parent_noop(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + + child.remove_parent_reference(parent, 'score') + + assert child.parent_references == {} + + +# UTS: objects/unit/RTLO4h2/remove-key-leaves-others-0 +def test_rtlo4h2_remove_key_leaves_others(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score', 'points'}} + + child.remove_parent_reference(parent, 'score') + + assert child.parent_references['map:parent@1000'] == {'points'} + + +# UTS: objects/unit/RTLO4h3/remove-last-key-removes-entry-0 +def test_rtlo4h3_remove_last_key_removes_entry(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score'}} + + child.remove_parent_reference(parent, 'score') + + assert 'map:parent@1000' not in child.parent_references + assert child.parent_references == {} + + +# UTS: objects/unit/RTLO4h/remove-nonexistent-key-0 +def test_rtlo4h_remove_nonexistent_key(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score'}} + + child.remove_parent_reference(parent, 'nonexistent') + + assert child.parent_references['map:parent@1000'] == {'score'} + + +# UTS: objects/unit/RTLO4f2/root-returns-empty-path-0 +def test_rtlo4f2_root_returns_empty_path(): + pool = ObjectsPool() + root = pool['root'] + + paths = root.get_full_paths() + assert len(paths) == 1 + assert [] in paths + + +# UTS: objects/unit/RTLO4f/direct-child-single-path-0 +def test_rtlo4f_direct_child_single_path(): + pool = ObjectsPool() + counter = InternalLiveCounter('counter:score@1000') + pool['counter:score@1000'] = counter + + root = pool['root'] + counter.add_parent_reference(root, 'score') + + paths = counter.get_full_paths() + assert len(paths) == 1 + assert ['score'] in paths + + +# UTS: objects/unit/RTLO4f/deep-nesting-0 +def test_rtlo4f_deep_nesting(): + # root --'profile'--> map:profile --'prefs'--> map:prefs --'theme_counter'--> counter:theme + pool = ObjectsPool() + root = pool['root'] + + profile = InternalLiveMap('map:profile@1000', ObjectsMapSemantics.LWW) + pool['map:profile@1000'] = profile + profile.add_parent_reference(root, 'profile') + + prefs = InternalLiveMap('map:prefs@1000', ObjectsMapSemantics.LWW) + pool['map:prefs@1000'] = prefs + prefs.add_parent_reference(profile, 'prefs') + + theme_counter = InternalLiveCounter('counter:theme@1000') + pool['counter:theme@1000'] = theme_counter + theme_counter.add_parent_reference(prefs, 'theme_counter') + + paths = theme_counter.get_full_paths() + assert len(paths) == 1 + assert ['profile', 'prefs', 'theme_counter'] in paths + + +# UTS: objects/unit/RTLO4f/diamond-graph-0 +def test_rtlo4f_diamond_graph(): + # root --'a'--> map:A --'x'--> counter:leaf, and root --'b'--> map:B --'y'--> counter:leaf + pool = ObjectsPool() + root = pool['root'] + + map_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + pool['map:a@1000'] = map_a + map_a.add_parent_reference(root, 'a') + + map_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + pool['map:b@1000'] = map_b + map_b.add_parent_reference(root, 'b') + + leaf = InternalLiveCounter('counter:leaf@1000') + pool['counter:leaf@1000'] = leaf + leaf.add_parent_reference(map_a, 'x') + leaf.add_parent_reference(map_b, 'y') + + paths = leaf.get_full_paths() + assert len(paths) == 2 + assert ['a', 'x'] in paths + assert ['b', 'y'] in paths + + +# UTS: objects/unit/RTLO4f/single-parent-multiple-keys-0 +def test_rtlo4f_single_parent_multiple_keys(): + pool = ObjectsPool() + root = pool['root'] + + child = InternalLiveCounter('counter:child@1000') + pool['counter:child@1000'] = child + child.add_parent_reference(root, 'primary') + child.add_parent_reference(root, 'alias') + + paths = child.get_full_paths() + assert len(paths) == 2 + assert ['primary'] in paths + assert ['alias'] in paths + + +# UTS: objects/unit/RTLO4f/orphan-returns-empty-0 +def test_rtlo4f_orphan_returns_empty(): + pool = ObjectsPool() + + orphan = InternalLiveCounter('counter:orphan@1000') + pool['counter:orphan@1000'] = orphan + + paths = orphan.get_full_paths() + assert len(paths) == 0 + + +# UTS: objects/unit/RTLO4f/cycle-suppression-0 +def test_rtlo4f_cycle_suppression(): + # root --'a'--> map:A --'b'--> map:B --'a'--> map:A, a cycle. The only simple path to + # map:B is ['a', 'b']. + pool = ObjectsPool() + root = pool['root'] + + map_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + pool['map:a@1000'] = map_a + map_a.add_parent_reference(root, 'a') + + map_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + pool['map:b@1000'] = map_b + map_b.add_parent_reference(map_a, 'b') + + # The cycle: map:A also has map:B as a parent + map_a.add_parent_reference(map_b, 'a') + + paths_b = map_b.get_full_paths() + assert len(paths_b) == 1 + assert ['a', 'b'] in paths_b + + paths_a = map_a.get_full_paths() + assert len(paths_a) == 1 + assert ['a'] in paths_a + + +# UTS: objects/unit/RTLO4f/complex-diamond-deep-0 +def test_rtlo4f_complex_diamond_deep(): + # root --'left'--> map:L --'mid'--> map:M --'target'--> counter:T, and + # root --'right'--> map:R --'target'--> counter:T + pool = ObjectsPool() + root = pool['root'] + + map_l = InternalLiveMap('map:l@1000', ObjectsMapSemantics.LWW) + pool['map:l@1000'] = map_l + map_l.add_parent_reference(root, 'left') + + map_r = InternalLiveMap('map:r@1000', ObjectsMapSemantics.LWW) + pool['map:r@1000'] = map_r + map_r.add_parent_reference(root, 'right') + + map_m = InternalLiveMap('map:m@1000', ObjectsMapSemantics.LWW) + pool['map:m@1000'] = map_m + map_m.add_parent_reference(map_l, 'mid') + + target = InternalLiveCounter('counter:t@1000') + pool['counter:t@1000'] = target + target.add_parent_reference(map_m, 'target') + target.add_parent_reference(map_r, 'target') + + paths = target.get_full_paths() + assert len(paths) == 2 + assert ['left', 'mid', 'target'] in paths + assert ['right', 'target'] in paths + + +# UTS: objects/unit/RTO5c10/rebuild-from-sync-0 +async def test_rto5c10_rebuild_from_sync(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'score': {'data': {'objectId': 'counter:score@1000'}, 'timeserial': 't:0'}, + 'profile': {'data': {'objectId': 'map:profile@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:score@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + build_object_state('map:profile@1000', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'nested': {'data': {'objectId': 'counter:nested@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:nested@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 5}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + + # counter:score@1000 is referenced by root at key 'score' + score = pool['counter:score@1000'] + assert score.parent_references['root'] == {'score'} + + # map:profile@1000 is referenced by root at key 'profile' + profile = pool['map:profile@1000'] + assert profile.parent_references['root'] == {'profile'} + + # counter:nested@1000 is referenced by map:profile@1000 at key 'nested' + nested = pool['counter:nested@1000'] + assert nested.parent_references['map:profile@1000'] == {'nested'} + + # root has no parent references + assert pool['root'].parent_references == {} + + # get_full_paths resolves through the rebuilt references + assert ['score'] in score.get_full_paths() + assert ['profile', 'nested'] in nested.get_full_paths() + + +# UTS: objects/unit/RTO5c10a/rebuild-clears-stale-refs-0 +async def test_rto5c10a_rebuild_clears_stale_refs(): + realtime_object, pool = _driven_pool() + + # First sync: root --'score'--> counter:abc@1000 + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'score': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 10}}), + ])) + assert pool['counter:abc@1000'].parent_references['root'] == {'score'} + + # Second sync: root --'points'--> counter:abc@1000, the key changed from 'score' to 'points' + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync2:', [ + build_object_state('root', {'aaa': 't:1'}, map={ + 'semantics': LWW, + 'entries': { + 'points': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:1'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 20}}), + ])) + + counter = pool['counter:abc@1000'] + + # The 'score' reference is gone, replaced by 'points' + assert counter.parent_references['root'] == {'points'} + assert ['points'] in counter.get_full_paths() + + paths = counter.get_full_paths() + assert len(paths) == 1 + + +# UTS: objects/unit/RTO5c10/unreferenced-empty-refs-0 +async def test_rto5c10_unreferenced_empty_refs(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:orphan@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 42}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + + # The counter is in the pool, but no InternalLiveMap entry refers to it + orphan = pool['counter:orphan@1000'] + assert orphan.parent_references == {} + + # get_full_paths returns no key-path for an unreferenced object + assert len(orphan.get_full_paths()) == 0 diff --git a/test/uts/objects/unit/path_object_mutations_test.py b/test/uts/objects/unit/path_object_mutations_test.py new file mode 100644 index 00000000..0f62f201 --- /dev/null +++ b/test/uts/objects/unit/path_object_mutations_test.py @@ -0,0 +1,152 @@ +"""Derived from uts/objects/unit/path_object_mutations.md in ably/specification. + +Spec points: RTPO3c2, RTPO15, RTPO15a2, RTPO15b, RTPO15c, RTPO15d, RTPO15e, RTPO16, RTPO16b, RTPO16c, +RTPO16d, RTPO16e, RTPO17, RTPO17a1, RTPO17b, RTPO17c, RTPO17d, RTPO17e, RTPO18, RTPO18a1, RTPO18b, +RTPO18c, RTPO18d, RTPO18e, RTTS5d2 + +The specification writes through the untyped `PathObject`. ably-python partitions it (LODR-061, +RTTS3-RTTS6): `set` and `remove` are on `LiveMapPathObject` and `increment` and `decrement` on +`LiveCounterPathObject`, reached through the unchecked view helpers. `root` is already a +`LiveMapPathObject`, so `root.set(...)` needs no view; `root.get('score').increment(25)` is +`root.get('score').as_live_counter().increment(25)`. A view never raises for the type at the path; +the write through it raises 92007 for a path resolving to the wrong type and 92005 for one that does +not resolve (RTTS5d2), which is what the specification's failure cases assert. + +Every write is awaited against the standard mock, which ACKs each OBJECT message, so the operation +has been applied locally (RTO20) by the time the write returns and the read that follows needs no +wait. Reads go through the view matching the type written: `as_primitive()` for map entries, +`as_live_counter()` for the counter. +""" + +import pytest + +from ably.pubsub.util.exceptions import AblyException +from test.uts.objects.helpers.standard_test_pool import setup_synced_channel + + +# UTS: objects/unit/RTPO15/set-delegates-to-map-0 +async def test_rtpo15_set_delegates_to_map(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.set('name', 'Bob') + + assert root.get('name').as_primitive().value() == 'Bob' + + +# UTS: objects/unit/RTPO15/set-nested-path-0 +async def test_rtpo15_set_nested_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('profile').as_live_map().set('email', 'bob@example.com') + + assert root.get('profile').get('email').as_primitive().value() == 'bob@example.com' + + +# UTS: objects/unit/RTPO15d/set-non-map-throws-0 +async def test_rtpo15d_set_non_map_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # The map view is unchecked (RTTS5d); `set` through it finds a counter at the path + with pytest.raises(AblyException) as excinfo: + await root.get('score').as_live_map().set('key', 'value') + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO16/remove-delegates-to-map-0 +async def test_rtpo16_remove_delegates_to_map(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.remove('name') + + assert root.get('name').as_primitive().value() is None + + +# UTS: objects/unit/RTPO16d/remove-non-map-throws-0 +async def test_rtpo16d_remove_non_map_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.get('score').as_live_map().remove('key') + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO17/increment-delegates-to-counter-0 +async def test_rtpo17_increment_delegates_to_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(25) + + assert root.get('score').as_live_counter().value() == 125 + + +# UTS: objects/unit/RTPO17/increment-default-amount-0 +async def test_rtpo17_increment_default_amount(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment() + + assert root.get('score').as_live_counter().value() == 101 + + +# UTS: objects/unit/RTPO17d/increment-non-counter-throws-0 +async def test_rtpo17d_increment_non_counter_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # `root` is a `LiveMapPathObject` and has no `increment`; the counter view over the root + # path is the translation, and the root map is the wrong type for it + with pytest.raises(AblyException) as excinfo: + await root.as_live_counter().increment(5) + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO18/decrement-delegates-to-counter-0 +async def test_rtpo18_decrement_delegates_to_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().decrement(10) + + assert root.get('score').as_live_counter().value() == 90 + + +# UTS: objects/unit/RTPO18/decrement-default-amount-0 +async def test_rtpo18_decrement_default_amount(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().decrement() + + assert root.get('score').as_live_counter().value() == 99 + + +# UTS: objects/unit/RTPO18d/decrement-non-counter-throws-0 +async def test_rtpo18d_decrement_non_counter_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.as_live_counter().decrement(5) + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO3c2/set-unresolvable-throws-0 +async def test_rtpo3c2_set_unresolvable_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.get('nonexistent').get('deep').as_live_map().set('key', 'value') + + assert excinfo.value.code == 92005 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTPO3c2/increment-unresolvable-throws-0 +async def test_rtpo3c2_increment_unresolvable_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.get('nonexistent').as_live_counter().increment(5) + + assert excinfo.value.code == 92005 + assert excinfo.value.status_code == 400 diff --git a/test/uts/objects/unit/path_object_subscribe_test.py b/test/uts/objects/unit/path_object_subscribe_test.py new file mode 100644 index 00000000..06e3a62c --- /dev/null +++ b/test/uts/objects/unit/path_object_subscribe_test.py @@ -0,0 +1,553 @@ +"""Derived from uts/objects/unit/path_object_subscribe.md in ably/specification. + +Spec points: RTPO19, RTPO19b, RTPO19c1, RTPO19c1a, RTPO19d, RTPO19e1, RTPO19e2, RTPO19f, RTPO19g, +RTO24b1, RTO24b2, RTO24b2a, RTO24b2a1, RTO24b2a2, RTO24b2b, RTO24b2b2, RTO24b2c, RTO24c1, RTO24c2a, +RTO24c2b, RTO24c2c, RTO24c2d, RTO25b, RTLM24 + +`subscribe` is on the base `PathObject` (LODR-061, RTTS3d) and is synchronous; the specification's +`{ depth: n }` option is the keyword argument `depth=n`. The listener receives a +`PathObjectSubscriptionEvent` whose `object` is a base `PathObject`, so the counter value RTPO19e1 +reads is `event.object.as_live_counter().value()`, and whose `message` is the public `ObjectMessage` +with snake_case fields (`site_code`, `operation.object_id`, `operation.counter_inc`) and the action as +an `ObjectOperationAction`. + +Each stimulus is an inbound OBJECT or OBJECT_SYNC message, which the client applies on the +transport's read task, so every positive expectation is reached with `poll_until`. Where the +specification then asserts an exact count, or that a listener stayed silent, the test first awaits +the delivery of a control (a further event, or an unlimited-depth listener on the same dispatch) and +settles, so that a late or spurious callback would already have run. Where the specification +subscribes straight after sending a seeding message, the test waits for the seed to be applied +first, since a seed applied after the subscription would itself be delivered to it. +""" + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.pathobject import PathObject +from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_channel_state, poll_until +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_inc, + build_map_clear, + build_map_set, + build_object_message, + build_object_state, + build_object_sync_message, + objects_channel_options, + objects_client, + remote_serial, + setup_synced_channel, + standard_mock_websocket, +) + + +# UTS: objects/unit/RTPO19/subscribe-receives-events-0 +async def test_rtpo19_subscribe_receives_events(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + sub = root.get('score').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the COUNTER_INC event') + await settle() + + assert isinstance(sub, Subscription) + assert len(events) == 1 + assert isinstance(events[0].object, PathObject) + assert events[0].object.path() == 'score' + assert events[0].message is not None + assert events[0].message.serial == '99' + assert events[0].message.site_code == 'remote' + assert events[0].message.operation is not None + assert events[0].message.operation.action == ObjectOperationAction.COUNTER_INC + assert events[0].message.channel == 'test' + + +# UTS: objects/unit/RTPO19b/subscribe-precondition-detached-0 +async def test_rtpo19b_subscribe_precondition_detached(): + # The specification's hand-written mock is the standard one: CONNECTED with siteCode and + # objectsGCGracePeriod, ATTACH answered with ATTACHED and the standard pool's OBJECT_SYNC, + # DETACH answered with DETACHED + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + root = await channel.object.get() + + await channel.detach() + await await_channel_state(channel, ChannelState.DETACHED) + + with pytest.raises(AblyException) as excinfo: + root.subscribe(lambda event: None) + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTPO19c1a/subscribe-non-positive-depth-throws-0 +async def test_rtpo19c1a_subscribe_non_positive_depth_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + root.subscribe(lambda event: None, depth=0) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO19c1a/subscribe-negative-depth-throws-0 +async def test_rtpo19c1a_subscribe_negative_depth_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + root.subscribe(lambda event: None, depth=-1) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO19c1/subscribe-depth-1-self-only-0 +async def test_rtpo19c1_subscribe_depth_1_self_only(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append, depth=1) + # Quiescence control: an unlimited-depth root listener covers the out-of-scope child path + control = [] + root.subscribe(control.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the self event') + await settle() + + control_before = len(control) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '100', 'remote'), + ])) + await poll_until(lambda: len(control) > control_before, description='the control to see the child event') + await settle() + + assert len(events) == 1 + + +# UTS: objects/unit/RTPO19c1/subscribe-depth-2-children-0 +async def test_rtpo19c1_subscribe_depth_2_children(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append, depth=2) + # Quiescence control: an unlimited-depth root listener covers the out-of-scope grandchild path + control = [] + root.subscribe(control.append) + + # Self event (root map update): the candidate [] is covered at depth 2 + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the self event') + + # Child event (the counter at ["score"]): relative depth 1 - 0 + 1 = 2 <= 2, covered + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the child event') + await settle() + + # Grandchild event (the counter at ["profile", "nested_counter"]): relative depth + # 2 - 0 + 1 = 3 > 2, not covered. A COUNTER_INC yields this single candidate only. + control_before = len(control) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 1, '101', 'remote'), + ])) + await poll_until(lambda: len(control) > control_before, description='the control to see the grandchild event') + await settle() + + assert len(events) == 2 + + +# UTS: objects/unit/RTPO19c1/subscribe-unlimited-depth-0 +async def test_rtpo19c1_subscribe_unlimited_depth(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the root event') + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the child event') + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'theme', {'string': 'light'}, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: len(events) >= 3, description='the descendant event') + + assert len(events) >= 3 + + +# UTS: objects/unit/RTPO19d/subscribe-returns-subscription-0 +async def test_rtpo19d_subscribe_returns_subscription(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + sub = root.get('score').subscribe(events.append) + # Quiescence control: a separate listener at the same path, which stays subscribed + control = [] + root.get('score').subscribe(control.append) + + assert isinstance(sub, Subscription) + sub.unsubscribe() + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(control) >= 1, description='the control listener to fire') + await settle() + + assert len(events) == 0 + + +# UTS: objects/unit/RTPO19e1/event-path-object-correct-0 +async def test_rtpo19e1_event_path_object_correct(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the COUNTER_INC event') + + assert isinstance(events[0].object, PathObject) + assert events[0].object.path() == 'score' + assert events[0].object.as_live_counter().value() == 107 + + +# UTS: objects/unit/RTPO19e2/event-message-delivery-0 +async def test_rtpo19e2_event_message_delivery(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('score').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 42, 'serial-1', 'site-a'), + ])) + await poll_until(lambda: len(events) >= 1, description='the COUNTER_INC event') + + assert events[0].message is not None + assert events[0].message.channel == 'test' + assert events[0].message.serial == 'serial-1' + assert events[0].message.site_code == 'site-a' + assert events[0].message.operation is not None + assert events[0].message.operation.action == ObjectOperationAction.COUNTER_INC + assert events[0].message.operation.object_id == 'counter:score@1000' + assert events[0].message.operation.counter_inc.number == 42 + + +# UTS: objects/unit/RTPO19e2/event-message-omitted-no-operation-0 +async def test_rtpo19e2_event_message_omitted_no_operation(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + + # An OBJECT_SYNC that moves counter:score@1000 from 100 to 200 through replaceData (RTLC6), so + # the update's ObjectMessage has no operation. It omits root, which is never removed from the + # pool (RTO5c2a) and still references "score", so the counter stays reachable from the root + # subscription. + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', [ + build_object_state('counter:score@1000', {'aaa': 't:1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 200}}), + ])) + await poll_until(lambda: len(events) >= 1, description='the sync-triggered event') + await settle() + + # Events from sync-triggered updates carry no message + for event in events: + assert event.message is None + + +# UTS: objects/unit/RTPO19f/subscribe-follows-path-0 +async def test_rtpo19f_subscribe_follows_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('score').subscribe(events.append) + + # Replace the counter at "score" with a new counter + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + + # Increment the new counter at "score" + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:new@2000', 10, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the MAP_SET and COUNTER_INC events') + await settle() + + # The subscription follows the path, so it sees the new counter + found_new = False + for event in events: + if event.object.path() == 'score': + found_new = True + assert found_new is True + # The second dispatch is the increment on the new counter, which is what shows the + # subscription followed the path to the replacement rather than staying bound to the old + # object; the MAP_SET dispatch alone would satisfy found_new + assert len(events) == 2 + assert events[1].object.path() == 'score' + assert events[1].message.operation.action == ObjectOperationAction.COUNTER_INC + assert events[1].message.operation.object_id == 'counter:new@2000' + + +# UTS: objects/unit/RTPO19g/subscribe-no-side-effects-0 +async def test_rtpo19g_subscribe_no_side_effects(): + client, channel, root, mock_ws = await setup_synced_channel('test') + state_before = channel.state + # RTPO19g also rules out side effects on the RealtimeObject and on the channel itself, which + # the channel state alone does not show while it is attached + sync_state_before = channel.object._sync_state + sent_before = len(mock_ws.messages_from_client) + + root.get('score').subscribe(lambda event: None) + await settle() + + assert channel.state == state_before + assert channel.object._sync_state == sync_state_before + assert len(mock_ws.messages_from_client) == sent_before + + +# UTS: objects/unit/RTPO19/subscribe-primitive-path-0 +async def test_rtpo19_subscribe_primitive_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('name').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the MAP_SET event at "name"') + await settle() + + assert len(events) == 1 + assert events[0].object.path() == 'name' + + +# UTS: objects/unit/RTPO19/map-clear-triggers-child-events-0 +async def test_rtpo19_map_clear_triggers_child_events(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + # UTS SPEC ERROR: the specification clears with serial "99", which sorts before the pool's + # entry serial "t:0", so RTLM24e1 removes no entry and the update carries no key; and it + # subscribes at the root only, which fires for any root update. Neither reaches the child + # paths the test is named for. The clear uses a serial after the pool's, and a subscription + # at the child path "name" is asserted as well. + child_events = [] + root.get('name').subscribe(child_events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_clear('root', remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the MAP_CLEAR event at the root') + + assert len(events) >= 1 + await poll_until(lambda: len(child_events) >= 1, description='the MAP_CLEAR event at "name"') + assert child_events[0].object.path() == 'name' + + +# UTS: objects/unit/RTPO19/child-events-bubble-0 +async def test_rtpo19_child_events_bubble(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('profile').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'email', {'string': 'bob@example.com'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the profile event') + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 3, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the nested counter event') + + assert len(events) >= 2 + + +# UTS: objects/unit/RTO24c1/depth-filtering-formula-0 +async def test_rto24c1_depth_filtering_formula(): + client, channel, root, mock_ws = await setup_synced_channel('test') + # Seed a grandchild object under profile.prefs (path ["profile", "prefs", "deep"]), so the + # grandchild stimulus below can be a COUNTER_INC yielding only that depth-3 candidate (RTO6 + # creates counter:deep@3000 at its zero value) + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'deep', {'objectId': 'counter:deep@3000'}, '50', 'remote'), + ])) + # The specification subscribes straight after the send, relying on the seed not reaching + # the listener under test; it is applied on the read task, and its ["profile", "prefs"] + # candidate is covered at depth 2, so it must land before the subscription + await poll_until(lambda: 'deep' in root.get('profile').get('prefs').as_live_map().keys(), + description='the seeding MAP_SET to be applied') + events = [] + # Subscribed at "profile" with depth 2: + # self (profile) -> ["profile"], 1 - 1 + 1 = 1 <= 2, covered + # child (profile.nested) -> ["profile", "nested_counter"], 2 - 1 + 1 = 2 <= 2, covered + # grandchild (prefs.deep) -> ["profile", "prefs", "deep"], 3 - 1 + 1 = 3 > 2, not covered + root.get('profile').subscribe(events.append, depth=2) + # Quiescence control: an unlimited-depth root listener covers the out-of-scope grandchild path + control = [] + root.subscribe(control.append) + + # Self event (profile map update): the first covered candidate is ["profile"] + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'email', {'string': 'bob@example.com'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the self event') + + # Child event (the nested counter, relative depth 2): covered + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 3, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the child event') + await settle() + + # Grandchild event (counter:deep, relative depth 3): not covered. A COUNTER_INC yields only + # this candidate, where a MAP_SET on map:prefs would also yield the covered ["profile", "prefs"]. + control_before = len(control) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:deep@3000', 1, '101', 'remote'), + ])) + await poll_until(lambda: len(control) > control_before, description='the control to see the grandchild event') + await settle() + + assert len(events) == 2 + + +# UTS: objects/unit/RTO24c1/prefix-mismatch-0 +async def test_rto24c1_prefix_mismatch(): + client, channel, root, mock_ws = await setup_synced_channel('test') + profile_events = [] + root.get('profile').subscribe(profile_events.append) + # Control listener at the root, which fires on both out-of-scope sends below + control_events = [] + root.subscribe(control_events.append) + + # Change at "score": "profile" is not a prefix of "score" + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + + # Change at "name": "profile" is not a prefix of "name" + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(control_events) >= 2, description='the control to see both events') + await settle() + + assert len(profile_events) == 0 + + +# UTS: objects/unit/RTO24b2a/candidate-paths-map-keys-0 +async def test_rto24b2a_candidate_paths_map_keys(): + client, channel, root, mock_ws = await setup_synced_channel('test') + score_events = [] + root_events = [] + # Subscribed at the child path ["score"] + root.get('score').subscribe(score_events.append) + # Subscribed at the root path [] + root.subscribe(root_events.append) + + # A MAP_SET on root with key "score" yields the candidates [] (root itself) and ["score"] + # (from the map update key), so both subscriptions fire + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(score_events) >= 1, description='the event at "score"') + await poll_until(lambda: len(root_events) >= 1, description='the event at the root') + await settle() + + assert len(score_events) == 1 + assert score_events[0].object.path() == 'score' + assert len(root_events) == 1 + + +# UTS: objects/unit/RTO24b2c/listener-exception-caught-0 +async def test_rto24b2c_listener_exception_caught(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + + def boom(event): + raise Exception('boom') + + root.subscribe(boom) + root.subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the second listener to fire') + await settle() + + assert len(events) == 1 + + +# UTS: objects/unit/RTO24b1/multi-path-dispatch-0 +async def test_rto24b1_multi_path_dispatch(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events_score = [] + events_alias = [] + + # "score" already points to counter:score@1000; a second reference "alias" gives it two paths + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'alias', {'objectId': 'counter:score@1000'}, '98', 'remote'), + ])) + # The specification subscribes straight after the send; the MAP_SET is applied on the read + # task, and its ["alias"] candidate would reach the "alias" subscription if it landed after it + await poll_until(lambda: 'alias' in root.keys(), description='the alias MAP_SET to be applied') + + root.get('score').subscribe(events_score.append) + root.get('alias').subscribe(events_alias.append) + + # Increment counter:score@1000, whose full paths are ["score"] and ["alias"] + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, '99', 'remote'), + ])) + await poll_until(lambda: len(events_score) >= 1, description='the event at "score"') + await poll_until(lambda: len(events_alias) >= 1, description='the event at "alias"') + await settle() + + assert len(events_score) == 1 + assert events_score[0].object.path() == 'score' + assert len(events_alias) == 1 + assert events_alias[0].object.path() == 'alias' + + +# UTS: objects/unit/RTO24b2b/fires-once-per-dispatch-0 +async def test_rto24b2b_fires_once_per_dispatch(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + # Subscribed at the root with unlimited depth, covering both [] and ["score"] + root.subscribe(events.append) + + # A MAP_SET on root with key "score" yields the candidates [] and ["score"]; the root + # subscription covers both and fires once, with the first + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the MAP_SET event') + + # Quiescence: a second, single-candidate dispatch is the control delivery, so a spurious + # second callback from the first dispatch would already have run + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:new@2000', 1, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the control COUNTER_INC event') + await settle() + + # One event per dispatch, though the first had two covered candidates + assert len(events) == 2 diff --git a/test/uts/objects/unit/path_object_test.py b/test/uts/objects/unit/path_object_test.py new file mode 100644 index 00000000..2172f068 --- /dev/null +++ b/test/uts/objects/unit/path_object_test.py @@ -0,0 +1,363 @@ +"""Derived from uts/objects/unit/path_object.md in ably/specification. + +Spec points: RTPO3, RTPO3a1, RTPO3b, RTPO3c1, RTPO4, RTPO4a, RTPO4b, RTPO4c, RTPO5, RTPO5b, RTPO5c, +RTPO5d, RTPO6, RTPO6b, RTPO6d, RTPO7, RTPO7a, RTPO7c, RTPO7d, RTPO7e, RTPO7f, RTPO8, RTPO8a, RTPO8c, +RTPO8f, RTPO9, RTPO9a, RTPO9c, RTPO9d, RTPO10, RTPO10a, RTPO10c, RTPO10d, RTPO11, RTPO11a, RTPO11c, +RTPO11d, RTPO12, RTPO12a, RTPO12c, RTPO12d, RTPO13, RTPO13a, RTPO13c1, RTPO13c2, RTPO13c3, RTPO13c4, +RTPO13c5, RTPO13d, RTPO14, RTPO14a, RTPO14b1, RTPO14b2, RTINS3b, RTINS4c, RTTS5d1 + +The specification reads through the untyped `PathObject`, which carries every method. ably-python +partitions it (LODR-061, RTTS3-RTTS6): navigation, `instance`, `compact` and `compact_json` are on the +base `PathObject`, and the type-specific reads are reached through the unchecked view helpers, so +`po.value()` on a counter is `po.as_live_counter().value()`, on a primitive `po.as_primitive().value()`, +and `po.keys()` is `po.as_live_map().keys()`. A view whose type does not match what the path resolves +to answers None, or `[]` for the collection reads (RTTS5d1), which is what the specification's +null and empty-array expectations become. Where the specification expects the untyped `value()` to +be null, neither value view may answer, so both are asserted. + +Every test opens with the standard synced channel; the module issues no writes, so no ACK is involved. +""" + +import pytest + +from ably.pubsub.objects.instance import Instance +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.objects.helpers.standard_test_pool import ( + build_map_set, + build_object_message, + setup_synced_channel, +) + + +async def _apply_back_ref(root, mock_ws): + """Sends the specification's `prefs.back_ref -> profile` MAP_SET, closing a cycle, and waits for it. + + The MAP_SET is an inbound OBJECT message, applied on the transport's read task, so the + specification's quiescence barrier is a poll until the new key is visible. + """ + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'back_ref', {'objectId': 'map:profile@1000'}, '99', 'remote'), + ])) + await poll_until(lambda: 'back_ref' in root.get('profile').get('prefs').as_live_map().keys(), + description='the back_ref MAP_SET to be applied') + + +# UTS: objects/unit/RTPO4/path-string-representation-0 +async def test_rtpo4_path_string_representation(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.path() == '' + assert root.get('profile').path() == 'profile' + assert root.get('profile').get('email').path() == 'profile.email' + + +# UTS: objects/unit/RTPO4b/path-escapes-dots-0 +async def test_rtpo4b_path_escapes_dots(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + po = root.get('a.b').get('c') + + # The specification's "a\\.b.c" is the string a\.b.c: one backslash before the dot in the segment + assert po.path() == 'a\\.b.c' + + +# UTS: objects/unit/RTPO5/get-appends-key-0 +async def test_rtpo5_get_appends_key(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + child = root.get('profile') + grandchild = child.get('email') + + assert child.path() == 'profile' + assert grandchild.path() == 'profile.email' + assert child is not root + + +# UTS: objects/unit/RTPO5b/get-non-string-throws-0 +async def test_rtpo5b_get_non_string_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + root.get(123) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO6/at-parses-path-0 +async def test_rtpo6_at_parses_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + po = root.at('profile.email') + + assert po.path() == 'profile.email' + assert po.as_primitive().value() == 'alice@example.com' + + +# UTS: objects/unit/RTPO6/at-escaped-dots-0 +async def test_rtpo6_at_escaped_dots(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + po = root.at('a\\.b.c') + + assert po.path() == 'a\\.b.c' + # UTS SPEC ERROR: the round trip through path() alone does not show that `\.` stayed inside + # its segment: an `at` that split on every dot gives the segments ['a\\', 'b', 'c'], which + # path() renders as the same string. RTPO6b makes `\.` a literal dot within a segment, so + # the parsed segments are asserted as well. + assert po._path == ['a.b', 'c'] + + +# UTS: objects/unit/RTPO6b/at-non-string-throws-0 +async def test_rtpo6b_at_non_string_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # NOTE: the features spec's RTPO6 defines no error for a path that is not a string (RTPO5b + # covers `get` only); 40003 is asserted as the specification has it, consistently with RTPO5b. + # ably-python's `at` also takes a sequence of segments (LODR-061), and an int is neither. + with pytest.raises(AblyException) as excinfo: + root.at(123) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO7/value-counter-0 +async def test_rtpo7_value_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('score').as_live_counter().value() == 100 + + +# UTS: objects/unit/RTPO7/value-primitive-0 +async def test_rtpo7_value_primitive(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('name').as_primitive().value() == 'Alice' + assert root.get('age').as_primitive().value() == 30 + assert root.get('active').as_primitive().value() is True + + +# UTS: objects/unit/RTPO7d/value-livemap-null-0 +async def test_rtpo7d_value_livemap_null(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + profile = root.get('profile') + + # value() on a map is null (RTPO7e); neither value view answers for a map (RTTS6b, RTTS6c) + assert profile.as_primitive().value() is None + assert profile.as_live_counter().value() is None + + +# UTS: objects/unit/RTPO7e/value-unresolvable-null-0 +async def test_rtpo7e_value_unresolvable_null(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + deep = root.get('nonexistent').get('deep') + + assert deep.as_primitive().value() is None + assert deep.as_live_counter().value() is None + + +# UTS: objects/unit/RTPO8/instance-live-object-0 +async def test_rtpo8_instance_live_object(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + counter_inst = root.get('score').instance() + assert isinstance(counter_inst, Instance) + assert counter_inst.id == 'counter:score@1000' + + map_inst = root.get('profile').instance() + assert isinstance(map_inst, Instance) + assert map_inst.id == 'map:profile@1000' + + +# UTS: objects/unit/RTPO8f/instance-primitive-wrapped-0 +async def test_rtpo8f_instance_primitive_wrapped(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + name_inst = root.get('name').instance() + + assert isinstance(name_inst, Instance) + assert name_inst.id is None + assert name_inst.as_primitive().value() == 'Alice' + + +# UTS: objects/unit/RTPO9/entries-yields-pairs-0 +async def test_rtpo9_entries_yields_pairs(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + entries = {} + for key, path_obj in root.entries(): + entries[key] = path_obj.path() + + assert entries['name'] == 'name' + assert entries['profile'] == 'profile' + assert len(entries) == 7 + + +# UTS: objects/unit/RTPO9d/entries-non-map-empty-0 +async def test_rtpo9d_entries_non_map_empty(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + entries = root.get('score').as_live_map().entries() + + assert len(entries) == 0 + + +# UTS: objects/unit/RTPO10/keys-returns-array-0 +async def test_rtpo10_keys_returns_array(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + keys = root.keys() + + assert isinstance(keys, list) + assert len(keys) == 7 + assert 'name' in keys + assert 'profile' in keys + assert 'score' in keys + + +# UTS: objects/unit/RTPO10d/keys-non-map-empty-0 +async def test_rtpo10d_keys_non_map_empty(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + keys = root.get('score').as_live_map().keys() + + assert isinstance(keys, list) + assert len(keys) == 0 + + +# UTS: objects/unit/RTPO11/values-returns-array-0 +async def test_rtpo11_values_returns_array(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + vals = root.values() + + assert isinstance(vals, list) + assert len(vals) == 7 + # Each element is a PathObject whose path is the key + paths = {} + for v in vals: + paths[v.path()] = True + assert paths['name'] is True + assert paths['profile'] is True + assert paths['score'] is True + + +# UTS: objects/unit/RTPO11d/values-non-map-empty-0 +async def test_rtpo11d_values_non_map_empty(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + vals = root.get('score').as_live_map().values() + + assert isinstance(vals, list) + assert len(vals) == 0 + + +# UTS: objects/unit/RTPO12/size-count-0 +async def test_rtpo12_size_count(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.size() == 7 + assert root.get('profile').as_live_map().size() == 3 + + +# UTS: objects/unit/RTPO12c/size-non-map-null-0 +async def test_rtpo12c_size_non_map_null(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('score').as_live_map().size() is None + assert root.get('name').as_live_map().size() is None + + +# UTS: objects/unit/RTPO13/compact-recursive-0 +async def test_rtpo13_compact_recursive(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + result = root.compact() + + assert result['name'] == 'Alice' + assert result['age'] == 30 + assert result['active'] is True + assert result['score'] == 100 + assert result['data'] == {'tags': ['a', 'b']} + assert result['avatar'] == bytes([1, 2, 3]) + assert result['profile']['email'] == 'alice@example.com' + assert result['profile']['nested_counter'] == 5 + assert result['profile']['prefs']['theme'] == 'dark' + + +# UTS: objects/unit/RTPO13c5/compact-cycle-detection-0 +async def test_rtpo13c5_compact_cycle_detection(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _apply_back_ref(root, mock_ws) + + result = root.get('profile').compact() + + assert result['prefs']['back_ref'] is result + + +# UTS: objects/unit/RTPO13c/compact-counter-0 +async def test_rtpo13c_compact_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('score').compact() == 100 + + +# UTS: objects/unit/RTPO14/compact-json-0 +async def test_rtpo14_compact_json(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _apply_back_ref(root, mock_ws) + + result = root.get('profile').compact_json() + + assert result['prefs']['back_ref'] == {'objectId': 'map:profile@1000'} + + +# UTS: objects/unit/RTPO3/path-resolution-walk-0 +async def test_rtpo3_path_resolution_walk(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # The empty path resolves to the root map (RTPO3b), for which value() is null (RTPO7e) + assert root.as_primitive().value() is None + assert root.as_live_counter().value() is None + assert root.get('profile').get('prefs').get('theme').as_primitive().value() == 'dark' + + +# UTS: objects/unit/RTPO3a1/intermediate-not-map-0 +async def test_rtpo3a1_intermediate_not_map(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + something = root.get('score').get('something') + + assert something.as_primitive().value() is None + assert something.as_live_counter().value() is None + + +# UTS: objects/unit/RTPO3c1/read-null-on-failure-0 +async def test_rtpo3c1_read_null_on_failure(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + nonexistent = root.get('nonexistent') + + assert nonexistent.as_primitive().value() is None + assert nonexistent.as_live_counter().value() is None + assert nonexistent.instance() is None + assert nonexistent.as_live_map().size() is None + assert nonexistent.compact() is None + + +# UTS: objects/unit/RTPO7/value-bytes-0 +async def test_rtpo7_value_bytes(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('avatar').as_primitive().value() == bytes([1, 2, 3]) + + +# UTS: objects/unit/RTPO14/compact-json-bytes-0 +async def test_rtpo14_compact_json_bytes(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + result = root.compact_json() + + assert result['avatar'] == 'AQID' diff --git a/test/uts/objects/unit/public_object_message_test.py b/test/uts/objects/unit/public_object_message_test.py new file mode 100644 index 00000000..1e412c77 --- /dev/null +++ b/test/uts/objects/unit/public_object_message_test.py @@ -0,0 +1,345 @@ +"""Derived from uts/objects/unit/public_object_message.md in ably/specification. + +Spec points: PAOM1, PAOM2, PAOM3, PAOOP1, PAOOP2, PAOOP3 + +The specification's `PublicObjectMessage.fromObjectMessage(source, channel)` is +`publicmessage.ObjectMessage._from_internal(source, channel_name)`, which takes the +channel's name rather than the channel (PAOM3b reads only `channel.name`), and +`PublicObjectOperation.fromObjectOperation(op)` is +`publicmessage.ObjectOperation._from_internal(op)`. The public types share their names with +the internal wire types, so the public pair is reached through the `publicmessage` module. + +Each source is built as the internal dataclasses directly. An attribute the specification +writes as `null` or omitted is None, operation actions are `ObjectOperationAction` members +and map semantics `ObjectsMapSemantics` members. + +The specification gives `mapCreateWithObjectId` and `counterCreateWithObjectId` the fields +of the payload they were derived from (`objectId`, `semantics`, `entries`, `count`), which +those payloads do not carry: MCRO2 and CCRO2 define only `initialValue` and `nonce`. Here +they carry those two, with `initialValue` the JSON encoding of the derived payload as +RTLMV4f and RTLCV4c produce it, and the derived payload in `derived_from` (RTLMV4j5, +RTLCV4g5). +""" + +import json + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ( + CounterCreate, + CounterCreateWithObjectId, + CounterInc, + MapClear, + MapCreate, + MapCreateWithObjectId, + MapRemove, + MapSet, + ObjectData, + ObjectDelete, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapEntry, + ObjectsMapSemantics, +) + +NONCE = 'test-nonce-1234567890' + + +# UTS: objects/unit/PAOM3/construction-all-fields-0 +def test_paom3_construction_all_fields(): + source = ObjectMessage( + id='msg-id-1', + client_id='client-1', + connection_id='conn-1', + timestamp=1700000000000, + serial='01', + serial_timestamp=1700000001000, + site_code='site1', + extras={'key': 'value'}, + operation=ObjectOperation( + action=ObjectOperationAction.MAP_SET, + object_id='map:abc@1000', + map_set=MapSet(key='name', value=ObjectData(string='Alice')), + ), + ) + + public_msg = publicmessage.ObjectMessage._from_internal(source, 'test-channel') + + assert isinstance(public_msg, publicmessage.ObjectMessage) + assert public_msg.id == 'msg-id-1' + assert public_msg.client_id == 'client-1' + assert public_msg.connection_id == 'conn-1' + assert public_msg.timestamp == 1700000000000 + assert public_msg.channel == 'test-channel' + assert public_msg.serial == '01' + assert public_msg.serial_timestamp == 1700000001000 + assert public_msg.site_code == 'site1' + assert public_msg.extras == {'key': 'value'} + assert public_msg.operation is not None + assert isinstance(public_msg.operation, publicmessage.ObjectOperation) + assert public_msg.operation.action == ObjectOperationAction.MAP_SET + assert public_msg.operation.object_id == 'map:abc@1000' + assert public_msg.operation.map_set.key == 'name' + assert public_msg.operation.map_set.value.string == 'Alice' + + +# UTS: objects/unit/PAOM3/construction-optional-fields-missing-0 +def test_paom3_construction_optional_fields_missing(): + source = ObjectMessage( + operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, + object_id='counter:abc@1000', + counter_inc=CounterInc(number=5.0), + ), + ) + + public_msg = publicmessage.ObjectMessage._from_internal(source, 'my-channel') + + assert public_msg.id is None + assert public_msg.client_id is None + assert public_msg.connection_id is None + assert public_msg.timestamp is None + assert public_msg.channel == 'my-channel' + assert public_msg.serial is None + assert public_msg.serial_timestamp is None + assert public_msg.site_code is None + assert public_msg.extras is None + assert public_msg.operation is not None + assert public_msg.operation.action == ObjectOperationAction.COUNTER_INC + + +# UTS: objects/unit/PAOM3/channel-from-channel-name-0 +def test_paom3_channel_from_channel_name(): + source = ObjectMessage( + operation=ObjectOperation( + action=ObjectOperationAction.OBJECT_DELETE, + object_id='counter:abc@1000', + ), + ) + + public_msg = publicmessage.ObjectMessage._from_internal(source, 'different-channel-name') + + assert public_msg.channel == 'different-channel-name' + + +# UTS: objects/unit/PAOOP3/map-set-copies-fields-0 +def test_paoop3_map_set_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_SET, + object_id='map:abc@1000', + map_set=MapSet(key='color', value=ObjectData(string='blue')), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_SET + assert public_op.object_id == 'map:abc@1000' + assert public_op.map_set.key == 'color' + assert public_op.map_set.value.string == 'blue' + assert public_op.map_create is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/map-remove-copies-fields-0 +def test_paoop3_map_remove_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_REMOVE, + object_id='map:abc@1000', + map_remove=MapRemove(key='old-key'), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_REMOVE + assert public_op.object_id == 'map:abc@1000' + assert public_op.map_remove.key == 'old-key' + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/counter-inc-copies-fields-0 +def test_paoop3_counter_inc_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, + object_id='counter:abc@1000', + counter_inc=CounterInc(number=42.0), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.COUNTER_INC + assert public_op.object_id == 'counter:abc@1000' + assert public_op.counter_inc.number == 42 + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.object_delete is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/object-delete-copies-fields-0 +def test_paoop3_object_delete_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.OBJECT_DELETE, + object_id='counter:abc@1000', + object_delete=ObjectDelete(), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.OBJECT_DELETE + assert public_op.object_id == 'counter:abc@1000' + assert public_op.object_delete is not None + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/map-clear-copies-fields-0 +def test_paoop3_map_clear_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_CLEAR, + object_id='map:abc@1000', + map_clear=MapClear(), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_CLEAR + assert public_op.object_id == 'map:abc@1000' + assert public_op.map_clear is not None + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + + +# UTS: objects/unit/PAOOP3/map-create-direct-0 +def test_paoop3_map_create_direct(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, + object_id='map:new@2000', + map_create=MapCreate( + semantics=ObjectsMapSemantics.LWW, + entries={'key1': ObjectsMapEntry(data=ObjectData(string='val1'))}, + ), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_CREATE + assert public_op.object_id == 'map:new@2000' + assert public_op.map_create is not None + assert public_op.map_create.semantics == ObjectsMapSemantics.LWW + assert public_op.map_create.entries['key1'].data.string == 'val1' + assert public_op.counter_create is None + + +# UTS: objects/unit/PAOOP3/map-create-from-with-object-id-0 +def test_paoop3_map_create_from_with_object_id(): + derived_map_create = MapCreate( + semantics=ObjectsMapSemantics.LWW, + entries={'x': ObjectsMapEntry(data=ObjectData(number=10.0))}, + ) + + # The specification also lists `objectId`, `semantics` and `entries` inside + # `mapCreateWithObjectId`, which MCRO2 does not define; see the module docstring + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, + object_id='map:derived@3000', + map_create_with_object_id=MapCreateWithObjectId( + initial_value=json.dumps(derived_map_create.to_dict()), + nonce=NONCE, + derived_from=derived_map_create, + ), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_CREATE + assert public_op.object_id == 'map:derived@3000' + assert public_op.map_create is not None + assert public_op.map_create.semantics == ObjectsMapSemantics.LWW + assert public_op.map_create.entries['x'].data.number == 10 + assert public_op.counter_create is None + # PAOOP1: the public operation carries the derived payload, never the WithObjectId one + assert public_op.map_create == derived_map_create + assert not hasattr(public_op, 'map_create_with_object_id') + + +# UTS: objects/unit/PAOOP3/counter-create-from-with-object-id-0 +def test_paoop3_counter_create_from_with_object_id(): + derived_counter_create = CounterCreate(count=100.0) + + # The specification also lists `objectId` and `count` inside + # `counterCreateWithObjectId`, which CCRO2 does not define; see the module docstring + source_operation = ObjectOperation( + action=ObjectOperationAction.COUNTER_CREATE, + object_id='counter:derived@3000', + counter_create_with_object_id=CounterCreateWithObjectId( + initial_value=json.dumps(derived_counter_create.to_dict()), + nonce=NONCE, + derived_from=derived_counter_create, + ), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.COUNTER_CREATE + assert public_op.object_id == 'counter:derived@3000' + assert public_op.counter_create is not None + assert public_op.counter_create.count == 100 + assert public_op.map_create is None + # PAOOP1: the public operation carries the derived payload, never the WithObjectId one + assert public_op.counter_create == derived_counter_create + assert not hasattr(public_op, 'counter_create_with_object_id') + + +# UTS: objects/unit/PAOOP3/create-payloads-omitted-0 +def test_paoop3_create_payloads_omitted(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_SET, + object_id='map:abc@1000', + map_set=MapSet(key='k', value=ObjectData(string='v')), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.map_create is None + assert public_op.counter_create is None + + +# UTS: objects/unit/PAOOP3/only-relevant-field-per-action-0 +def test_paoop3_only_relevant_field_per_action(): + source_operation = ObjectOperation( + action=ObjectOperationAction.COUNTER_CREATE, + object_id='counter:new@2000', + counter_create=CounterCreate(count=50.0), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.COUNTER_CREATE + assert public_op.object_id == 'counter:new@2000' + assert public_op.counter_create is not None + assert public_op.counter_create.count == 50 + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + assert public_op.map_clear is None diff --git a/test/uts/objects/unit/realtime_object_test.py b/test/uts/objects/unit/realtime_object_test.py new file mode 100644 index 00000000..33252655 --- /dev/null +++ b/test/uts/objects/unit/realtime_object_test.py @@ -0,0 +1,951 @@ +"""Derived from uts/objects/unit/realtime_object.md in ably/specification. + +Spec points: RTO2, RTO2a, RTO2b, RTO5c9, RTO10, RTO10a, RTO10b1, RTO10c1b, RTO10c1b1, RTO15, +RTO15e1, RTO15e2, RTO15e3, RTO17, RTO17b, RTO18, RTO18b, RTO18d, RTO18e, RTO19, RTO20, RTO20b, +RTO20c1, RTO20d1, RTO20d2, RTO20d4, RTO20e, RTO20e1, RTO20f, RTO23, RTO23a, RTO23c, RTO23c1, +RTO23d, RTO23e, RTO24a, RTO24c1, RTO25a, RTO25b, RTO26a, RTO26b, RTO26c, RTO27a, RTO27b, RTL33b, +RTL33c, RTLO4e10 + +The specification drives `channel.object` against the mock websocket, and reaches into the +RealtimeObject where nothing public observes what it asserts: the channel-state handler +(`_act_on_channel_state`), the pool (`_objects_pool`), the sync state (`_sync_state`) and the +GC interval (`_gc_interval_ms`). + +Several translations recur through the file. + +Every OBJECT the client publishes is ACKed, by the standard mock or by a test's own handler, +because a publish resolves only on its ACK (RTO15g). The one test whose specification sends +the ACK by hand, `ack-after-echo-no-double-apply`, drives the write as a task and ACKs it once +the OBJECT has left the client. + +The specifications' untyped `PathObject` calls go through the typed views: +`root.get("score").value()` is `root.get('score').as_live_counter().value()`, and +`root.get("name").value()` is `root.get('name').as_primitive().value()`. + +The modes a server grants travel as bits of the ATTACHED `flags`, so the specification's +`modes: ["OBJECT_SUBSCRIBE"]` is `flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG`. + +An inbound frame is processed on the transport's read task, so state is read after a +`poll_until` or a `settle()`, and a negative assertion ("the listener did not fire", "the echo +did not apply") follows a positive control delivered behind the message under test. + +`enable_fake_timers()` is a `FakeClock` given to the client, which is where the GC timer +(RTO10a) is scheduled and where `now_ms()` is read for a sweep (RTO10c1b). +""" + +import asyncio + +import pytest + +from ably.pubsub.objects.enums import ObjectsEvent, ObjectsSyncState +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.pathobject import PathObject +from ably.pubsub.objects.valuetypes import LiveCounter +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_channel_state, poll_until +from test.uts.helpers.clock import FakeClock, settle +from test.uts.helpers.mock_websocket import MockWebSocket, channel_error_message, contains_in_order +from test.uts.objects.helpers.standard_test_pool import ( + ATTACH, + HAS_OBJECTS, + OBJECT, + OBJECT_PUBLISH_FLAG, + OBJECT_SUBSCRIBE_FLAG, + SITE_CODE, + STANDARD_POOL_OBJECTS, + ack_serial, + assert_unchanged_after_quiescence, + below_ack_serial, + build_ack_message, + build_counter_inc, + build_map_set, + build_object_delete, + build_object_message, + build_object_sync_message, + objects_attached_message, + objects_channel_options, + objects_client, + objects_connected_message, + remote_serial, + setup_synced_channel, + setup_synced_channel_no_ack, + standard_mock_websocket, +) + +# How long a test waits on an operation it expects to complete or fail, so that one which +# never settles fails with a timeout rather than running into the suite's own +OPERATION_TIMEOUT = 5 + +# RTO10b3's default grace period and RTO10a's example interval, which the GC tests advance past +DEFAULT_GC_GRACE_PERIOD_MS = 86_400_000 +DEFAULT_GC_INTERVAL_MS = 300_000 + + +def _score(root): + """The value of the standard pool's `score` counter, read through the counter view.""" + return root.get('score').as_live_counter().value() + + +def _nested_counter(root): + """The value of the standard pool's `profile.nested_counter` counter.""" + return root.get('profile').get('nested_counter').as_live_counter().value() + + +def _name(root): + """The standard pool's `name` primitive, read through the primitive view.""" + return root.get('name').as_primitive().value() + + +def _sent(mock_ws, action): + """How many protocol messages carrying `action` the client has sent.""" + return sum(1 for message in mock_ws.messages_from_client if message.get('action') == int(action)) + + +def _raise_if_failed(task): + """Re-raises the exception `task` finished with, so that an early failure reports itself.""" + if task.done() and not task.cancelled() and task.exception() is not None: + raise task.exception() + + +def _assert_pending(task, description): + """Asserts `task` is still waiting; a task that already failed re-raises its own error.""" + _raise_if_failed(task) + assert not task.done(), f'{description} completed while it should still be waiting' + + +async def _await_sent(mock_ws, action, task): + """Waits for the client to send a message carrying `action` on behalf of `task`.""" + await poll_until(lambda: task.done() or _sent(mock_ws, action) >= 1, + description=f'the client to send a message with action {int(action)}') + _raise_if_failed(task) + + +async def _restart_sync(channel, mock_ws, channel_serial='sync2:cursor'): + """Sends an ATTACHED carrying a sync cursor and waits for the sync state to leave SYNCED. + + A new ATTACHED moves the sync state to SYNCING (RTO4c), and the cursor keeps it there + until an OBJECT_SYNC completes the sequence. + """ + mock_ws.send_to_client(objects_attached_message(channel.name, channel_serial)) + await poll_until(lambda: channel.object._sync_state == ObjectsSyncState.SYNCING, + description='the ATTACHED to restart the objects sync') + + +def _acking_mock_websocket(serials, captured=None, **kwargs): + """The standard mock, ACKing each OBJECT with the serials `serials(message)` gives. + + Each OBJECT is recorded in `captured` first, if given. `kwargs` go to + `standard_mock_websocket`. + """ + mock_ws = None + + def on_object(message): + if captured is not None: + captured.append(message) + mock_ws.send_to_client(build_ack_message(message['msgSerial'], serials(message))) + + mock_ws = standard_mock_websocket(auto_ack=False, on_object=on_object, **kwargs) + return mock_ws + + +# UTS: objects/unit/RTO23/get-returns-path-object-0 +async def test_rto23_get_returns_path_object(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert isinstance(root, PathObject) + assert root._path == [] + # RTO23d: the root is the pool's InternalLiveMap with id `root` + assert root._root is channel.object._objects_pool['root'] + + +# UTS: objects/unit/RTO23a/get-requires-subscribe-mode-0 +async def test_rto23a_get_requires_subscribe_mode(): + mock_ws = standard_mock_websocket(sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options(ChannelMode.OBJECT_PUBLISH)) + + with pytest.raises(AblyException) as excinfo: + await channel.object.get() + + assert excinfo.value.code == 40024 + + +# UTS: objects/unit/RTO23e/get-reattaches-detached-0 +async def test_rto23e_get_reattaches_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test', modes=(ChannelMode.OBJECT_SUBSCRIBE,)) + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + root = await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + assert isinstance(root, PathObject) + assert root._path == [] + assert channel.state == ChannelState.ATTACHED + + +# UTS: objects/unit/RTO23c/get-waits-for-synced-0 +async def test_rto23c_get_waits_for_synced(): + mock_ws = standard_mock_websocket(attached_channel_serial='sync1:cursor', sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + task = asyncio.ensure_future(channel.object.get()) + await _await_sent(mock_ws, ProtocolMessageAction.ATTACH, task) + + # The ATTACHED carries a sync cursor and no OBJECT_SYNC has arrived, so get() must still + # be waiting for SYNCED + await settle() + _assert_pending(task, 'get()') + + mock_ws.send_to_client(build_object_sync_message('test', 'sync1:', STANDARD_POOL_OBJECTS)) + root = await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert isinstance(root, PathObject) + assert root._path == [] + + +# UTS: objects/unit/RTO23c1/fails-on-channel-detached-0 +async def test_rto23c1_fails_on_channel_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(channel.object.get()) + await settle() + _assert_pending(task, 'get()') + + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO23c1/fails-on-channel-suspended-0 +async def test_rto23c1_fails_on_channel_suspended(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(channel.object.get()) + await settle() + _assert_pending(task, 'get()') + + # The mock cannot drive the channel to SUSPENDED, so the RealtimeObject's channel-state + # handler is called directly, as the RTO27 test does + channel.object._act_on_channel_state(ChannelState.SUSPENDED) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO23c1/fails-on-channel-failed-0 +async def test_rto23c1_fails_on_channel_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(channel.object.get()) + await settle() + _assert_pending(task, 'get()') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel failed', 400)) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + # The cause is the channel's errorReason, the injected channel error + assert excinfo.value.cause.code == 90000 + + +# UTS: objects/unit/RTO15/publish-sends-object-pm-0 +async def test_rto15_publish_sends_object_pm(): + captured = [] + mock_ws = _acking_mock_websocket(lambda message: ['serial-0'], captured) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.get('score').as_live_counter().increment(5) + + assert len(captured) == 1 + assert captured[0]['action'] == OBJECT + assert captured[0]['channel'] == 'test' + assert len(captured[0]['state']) == 1 + # RTO15e3: the state entry is the encoded ObjectMessage for the increment + operation = captured[0]['state'][0]['operation'] + assert operation['action'] == ObjectOperationAction.COUNTER_INC + assert operation['objectId'] == 'counter:score@1000' + assert operation['counterInc']['number'] == 5 + + +# UTS: objects/unit/RTO20/publish-and-apply-local-0 +async def test_rto20_publish_and_apply_local(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO20c/missing-site-code-0 +async def test_rto20c_missing_site_code(): + captured = [] + mock_ws = _acking_mock_websocket(lambda message: ['serial-0'], captured, + connected=objects_connected_message(site_code=None)) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.get('score').as_live_counter().increment(10) + + # The increment was published and ACKed, so its not being applied is the siteCode's doing + assert len(captured) == 1 + assert _score(root) == 100 + + +# UTS: objects/unit/RTO20d1/null-serial-skipped-0 +async def test_rto20d1_null_serial_skipped(): + captured = [] + mock_ws = _acking_mock_websocket(lambda message: [None], captured) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.get('score').as_live_counter().increment(10) + + # The increment was published and ACKed, so its not being applied is the null serial's doing + assert len(captured) == 1 + assert _score(root) == 100 + + +# UTS: objects/unit/RTO20d4/empty-synthetic-list-skips-sync-wait-0 +async def test_rto20d4_empty_synthetic_list_skips_sync_wait(): + mock_ws = _acking_mock_websocket(lambda message: [None]) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + # Back to SYNCING, where a write with something to apply would wait for SYNCED (RTO20e). + # No OBJECT_SYNC is ever sent, so the increment completing at all shows the wait was skipped. + await _restart_sync(channel, mock_ws) + + await asyncio.wait_for(root.get('score').as_live_counter().increment(10), OPERATION_TIMEOUT) + + assert channel.object._sync_state == ObjectsSyncState.SYNCING + assert _score(root) == 100 + + +# UTS: objects/unit/RTO20d4/mixed-null-serials-applies-non-null-0 +async def test_rto20d4_mixed_null_serials_applies_non_null(): + # The set() publishes [COUNTER_CREATE, MAP_SET] (RTLM20h1); the CREATE is ACKed with a + # null serial and skipped (RTO20d1), and the MAP_SET with a real one and applied + mock_ws = _acking_mock_websocket(lambda message: [None, ack_serial(message['msgSerial'], 1)]) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.set('child', LiveCounter.create(0)) + + assert 'child' in root.keys() + # The seven standard-pool entries and `child` + assert root.size() == 8 + + +# UTS: objects/unit/RTO20e/waits-for-synced-0 +async def test_rto20e_waits_for_synced(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The OBJECT is published and ACKed; while still SYNCING the increment must wait, unapplied + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + await settle() + _assert_pending(task, 'increment()') + assert _score(root) == 100 + + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO20e1/fails-on-channel-detached-0 +async def test_rto20e1_fails_on_channel_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The publish and its ACK complete against the mock, and the increment waits for SYNCED + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + await settle() + _assert_pending(task, 'increment()') + + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + # RTO20e1 gives the status code as well + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO20e1/fails-on-channel-failed-0 +async def test_rto20e1_fails_on_channel_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The publish and its ACK complete against the mock, and the increment waits for SYNCED + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + await settle() + _assert_pending(task, 'increment()') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel failed', 400)) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + # RTO20e1 gives the status code, and the channel's errorReason as the cause + assert excinfo.value.status_code == 400 + assert excinfo.value.cause.code == 90000 + + +# UTS: objects/unit/RTO17/sync-state-events-0 +async def test_rto17_sync_state_events(): + mock_ws = standard_mock_websocket(attached_channel_serial='sync1:cursor', sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + events = [] + channel.object.on(ObjectsEvent.SYNCING, lambda: events.append('SYNCING')) + channel.object.on(ObjectsEvent.SYNCED, lambda: events.append('SYNCED')) + + task = asyncio.ensure_future(channel.object.get()) + await poll_until(lambda: task.done() or len(events) >= 1, description='the SYNCING event') + _raise_if_failed(task) + + mock_ws.send_to_client(build_object_sync_message('test', 'sync1:', STANDARD_POOL_OBJECTS)) + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert contains_in_order(events, ['SYNCING', 'SYNCED']), events + + +# UTS: objects/unit/RTO18d/duplicate-listener-0 +async def test_rto18d_duplicate_listener(): + client, channel, root, mock_ws = await setup_synced_channel('test') + calls = [] + + def listener(): + calls.append('SYNCED') + + channel.object.on(ObjectsEvent.SYNCED, listener) + channel.object.on(ObjectsEvent.SYNCED, listener) + + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await poll_until(lambda: len(calls) >= 2, description='the listener to be called twice') + await settle() + + assert len(calls) == 2 + + +# UTS: objects/unit/RTO19/off-deregisters-0 +async def test_rto19_off_deregisters(): + client, channel, root, mock_ws = await setup_synced_channel('test') + calls = [] + + def listener(): + calls.append('SYNCED') + + subscription = channel.object.on(ObjectsEvent.SYNCED, listener) + subscription.off() + + # A second listener on the same event is the control which shows the SYNCED was emitted + control = [] + channel.object.on(ObjectsEvent.SYNCED, lambda: control.append('SYNCED')) + + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await assert_unchanged_after_quiescence(lambda: len(calls), lambda: len(control) >= 1, + description='the control listener receives SYNCED') + + assert len(calls) == 0 + + +# UTS: objects/unit/RTO2/mode-enforcement-0 +async def test_rto2_mode_enforcement(): + # The server grants OBJECT_SUBSCRIBE only, where the channel requested both modes + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 40024 + + +# UTS: objects/unit/RTO23e/get-rejects-failed-0 +async def test_rto23e_get_rejects_failed(): + mock_ws = None + + def on_message_from_client(message): + if message.get('action') == ATTACH: + mock_ws.send_to_client(channel_error_message(message.get('channel'), 90000, 'Channel error', 400)) + + mock_ws = MockWebSocket( + on_connection_attempt=lambda conn: conn.respond_with_success(objects_connected_message()), + on_message_from_client=on_message_from_client, + ) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options(ChannelMode.OBJECT_SUBSCRIBE)) + + # The attach fails, which leaves the channel FAILED + with pytest.raises(AblyException): + await channel.attach() + assert channel.state == ChannelState.FAILED + + with pytest.raises(AblyException) as excinfo: + await channel.object.get() + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO25a/access-requires-subscribe-mode-0 +async def test_rto25a_access_requires_subscribe_mode(): + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_PUBLISH_FLAG) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options(ChannelMode.OBJECT_PUBLISH)) + + with pytest.raises(AblyException) as excinfo: + await channel.object.get() + + assert excinfo.value.code == 40024 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO25b/access-throws-detached-0 +async def test_rto25b_access_throws_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + with pytest.raises(AblyException) as excinfo: + root.keys() + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO25b/access-throws-failed-0 +async def test_rto25b_access_throws_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel error', 400)) + await await_channel_state(channel, ChannelState.FAILED) + + with pytest.raises(AblyException) as excinfo: + root.keys() + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26a/write-requires-publish-mode-0 +async def test_rto26a_write_requires_publish_mode(): + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws, + modes=(ChannelMode.OBJECT_SUBSCRIBE,)) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 40024 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26b/write-throws-detached-0 +async def test_rto26b_write_throws_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26b/write-throws-failed-0 +async def test_rto26b_write_throws_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel error', 400)) + await await_channel_state(channel, ChannelState.FAILED) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26c/write-throws-echo-disabled-0 +async def test_rto26c_write_throws_echo_disabled(): + client, channel, root, mock_ws = await setup_synced_channel('test', echo_messages=False) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 40000 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO24a/single-register-instance-0 +async def test_rto24a_single_register_instance(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events_root = [] + events_score = [] + + root.subscribe(events_root.append) + score_path = root.get('score') + score_path.subscribe(events_score.append) + + # `remote` has no entry in the counter's siteTimeserials, so the increment is new (RTLO4a) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, 't:1', 'remote'), + ])) + await poll_until(lambda: len(events_score) >= 1, description='the score subscription to fire') + + # Both subscriptions are held by the one register, so both fire + assert len(events_root) >= 1 + assert len(events_score) >= 1 + + +# UTS: objects/unit/RTO24c1/coverage-prefix-depth-0 +async def test_rto24c1_coverage_prefix_depth(): + client, channel, root, mock_ws = await setup_synced_channel('test') + shallow_events = [] + deep_events = [] + + # With depth 1 a subscription at the root covers the root's own path only; a child such + # as ['score'] is at relative depth 2 (RTO24c2b) + root.subscribe(shallow_events.append, depth=1) + root.subscribe(deep_events.append) + + # A MAP_SET on the root itself, at path [] + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(deep_events) >= 1, description='the deep subscription to see the root update') + + # An increment on the root's child, at path ['score'] + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, 't:2', 'remote'), + ])) + await poll_until(lambda: len(deep_events) >= 2, description='the deep subscription to see the child update') + await poll_until(lambda: len(shallow_events) >= 1, + description='the shallow subscription to see the root update') + await settle() + + # The shallow subscription sees the root update only; the deep one both + assert len(shallow_events) == 1 + assert len(deep_events) >= 2 + + +# UTS: objects/unit/RTO10/gc-tombstoned-objects-0 +async def test_rto10_gc_tombstoned_objects(): + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + pool = channel.object._objects_pool + + # A tombstone stamped now, which only the advance below makes eligible for collection + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '99', 'site1', clock.now_ms()), + ])) + await poll_until(lambda: pool['counter:score@1000'].is_tombstone, + description='the OBJECT_DELETE to tombstone the score counter') + + await clock.advance(DEFAULT_GC_GRACE_PERIOD_MS + DEFAULT_GC_INTERVAL_MS) + + assert _score(root) is None + # UTS SPEC ERROR: the specification asserts only that `score` reads null, which holds + # from the moment the counter is tombstoned (RTLM14c, RTLM5d2h), with or without a GC + # sweep. The sweep itself is observed as the counter leaving the pool (RTO10c1b). + assert 'counter:score@1000' not in pool + + +# UTS: objects/unit/RTO10c1b1/gc-root-never-removed-0 +async def test_rto10c1b1_gc_root_never_removed(): + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + pool = channel.object._objects_pool + root_map = pool['root'] + + # A rogue OBJECT_DELETE targeting the root, which RTLO4e10 rejects + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('root', remote_serial(0), 'remote', clock.now_ms()), + ])) + # The score counter is tombstoned behind it: the control which shows the rogue delete has + # been processed, and, once collected, that a GC sweep has run past the grace period + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '99', 'site1', clock.now_ms()), + ])) + await poll_until(lambda: pool['counter:score@1000'].is_tombstone, + description='the control OBJECT_DELETE to tombstone the score counter') + + # The root is not tombstoned and its data is untouched + assert _name(root) == 'Alice' + assert root_map.is_tombstone is False + + await clock.advance(DEFAULT_GC_GRACE_PERIOD_MS + DEFAULT_GC_INTERVAL_MS) + + assert 'counter:score@1000' not in pool + assert pool['root'] is root_map + + # The root is still live: an operation still applies to the root object the client holds + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: _name(root) == 'Bob', description='the MAP_SET to apply to the root') + + assert _name(root) == 'Bob' + + +# UTS: objects/unit/RTO20/echo-dedup-0 +async def test_rto20_echo_dedup(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + score_after_apply = _score(root) + + # The echo of the increment, carrying the serial it was ACKed and applied with + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, ack_serial(0, 0), SITE_CODE), + ])) + # A control behind the echo on the same connection, which shows the echo has been processed + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 1, remote_serial(0), 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: _score(root), lambda: _nested_counter(root) == 6, + description='the control increment has applied') + score_after_echo = _score(root) + + assert score_after_apply == 110 + assert score_after_echo == 110 + + +# UTS: objects/unit/RTO20f/ack-no-site-timeserials-update-0 +async def test_rto20f_ack_no_site_timeserials_update(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + assert _score(root) == 110 + + # `below_ack_serial(9)` is not the ACK serial, so the echo dedup (RTO9a3) leaves it alone, + # and it sorts below `ack_serial(0, 0)`, so it is rejected as stale (RTLO4a) if and only if + # the LOCAL apply recorded the ACK serial in siteTimeserials + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, below_ack_serial(9), SITE_CODE), + ])) + await poll_until(lambda: _score(root) == 120, description='the inbound increment to apply') + + assert _score(root) == 120 + + +# UTS: objects/unit/RTO20/ack-after-echo-no-double-apply-0 +async def test_rto20_ack_after_echo_no_double_apply(): + client, channel, root, mock_ws = await setup_synced_channel_no_ack('test') + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The OBJECT must be pending on the connection before its ACK can be matched to it + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + + # The echo arrives before the ACK + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, ack_serial(0, 0), SITE_CODE), + ])) + mock_ws.send_to_client(build_ack_message(0, [ack_serial(0, 0)])) + + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO5c9-RTO20/ack-serials-cleared-on-resync-0 +async def test_rto5c9_rto20_ack_serials_cleared_on_resync(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + assert _score(root) == 110 + + # A re-sync replaces the data and clears appliedOnAckSerials (RTO5c9) + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await poll_until(lambda: _score(root) == 100, description='the re-sync to replace the score') + assert _score(root) == 100 + + # The serial the increment was applied with on ACK applies normally once the set is cleared; + # were it still recorded, the dedup (RTO9a3) would discard it and the score would stay 100 + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, ack_serial(0, 0), SITE_CODE), + ])) + await poll_until(lambda: _score(root) == 110, description='the replayed increment to apply') + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO20/subscription-fires-on-ack-apply-0 +async def test_rto20_subscription_fires_on_ack_apply(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('score').subscribe(events.append) + + await root.get('score').as_live_counter().increment(10) + + assert len(events) >= 1 + assert _score(root) == 110 + + +# UTS: objects/unit/RTO23/get-implicit-attach-0 +async def test_rto23_get_implicit_attach(): + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + assert channel.state == ChannelState.INITIALIZED + root = await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + assert isinstance(root, PathObject) + assert root._path == [] + assert channel.state == ChannelState.ATTACHED + + +# UTS: objects/unit/RTO23d/get-resolves-immediately-synced-0 +async def test_rto23d_get_resolves_immediately_synced(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + root2 = await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + assert isinstance(root2, PathObject) + assert root2._path == [] + + +# UTS: objects/unit/RTO10b1/gc-grace-period-source-0 +async def test_rto10b1_gc_grace_period_source(): + clock = FakeClock() + mock_ws = standard_mock_websocket(connected=objects_connected_message(objects_gc_grace_period=5000)) + client = objects_client(mock_ws, clock=clock) + channel = client.channels.get('test', objects_channel_options()) + # UTS SPEC ERROR: the specification advances 6000 ms, which RTO10a's example interval of + # five minutes never sweeps within, so as written no GC check runs at all. The interval is + # shortened before the first ATTACHED schedules the GC timer, so that sweeps fall inside it. + channel.object._gc_interval_ms = 1000 + root = await channel.object.get() + pool = channel.object._objects_pool + + assert channel.object._gc_grace_period_ms == 5000 + + # A tombstone stamped now: eligible after 6000 ms under the 5000 ms grace period from + # ConnectionDetails, but not under the 24 hour default + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '99', 'site1', clock.now_ms()), + ])) + await poll_until(lambda: pool['counter:score@1000'].is_tombstone, + description='the OBJECT_DELETE to tombstone the score counter') + + # The advance is taken in two steps so that the object is seen to survive the sweeps that + # fall inside the grace period (RTO10c1b) + await clock.advance(4000) + assert 'counter:score@1000' in pool + + await clock.advance(2000) + + assert _score(root) is None + # UTS SPEC ERROR: as in RTO10, `score` reads null from the moment the counter is + # tombstoned; the collection is observed as the counter leaving the pool + assert 'counter:score@1000' not in pool + + +# UTS: objects/unit/RTO17-RTO18/sync-event-sequences-0 +@pytest.mark.parametrize('scenario', [ + 'initial attach', + 're-sync on new ATTACHED', + 'ATTACHED without HAS_OBJECTS', +]) +async def test_rto17_rto18_sync_event_sequences(scenario): + if scenario == 'initial attach': + # A first attach is only observable on a channel that has not attached, with the + # listeners registered before attach() is called + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + else: + client, channel, root, mock_ws = await setup_synced_channel('test') + + events = [] + channel.object.on(ObjectsEvent.SYNCING, lambda: events.append('SYNCING')) + channel.object.on(ObjectsEvent.SYNCED, lambda: events.append('SYNCED')) + + if scenario == 'initial attach': + await channel.attach() + elif scenario == 're-sync on new ATTACHED': + mock_ws.send_to_client(objects_attached_message('test', 'sync3:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync3:', STANDARD_POOL_OBJECTS)) + else: + # RTO4c moves the sync state to SYNCING on any ATTACHED, and with no HAS_OBJECTS the + # sync completes at once (RTO4b4), moving it to SYNCED + mock_ws.send_to_client(objects_attached_message('test', 'sync4:', flags=0)) + + expected_events = ['SYNCING', 'SYNCED'] + await poll_until(lambda: len(events) >= len(expected_events), description=f'the {scenario} events') + await settle() + + assert events == expected_events + + +# UTS: objects/unit/RTO27/channel-state-data-lifecycle-0 +@pytest.mark.parametrize('state', [ChannelState.DETACHED, ChannelState.FAILED, ChannelState.SUSPENDED], + ids=['RTO27a-detached', 'RTO27a-failed', 'RTO27b-suspended']) +async def test_rto27_channel_state_data_lifecycle(state): + client, channel, root, mock_ws = await setup_synced_channel('test') + pool = channel.object._objects_pool + + # The standard pool has been materialised: the root map, a counter and a nested map + assert 'name' in pool['root'].data + assert pool['counter:score@1000'].value() == 100 + assert 'email' in pool['map:profile@1000'].data + + updates = [] + pool['root'].subscribe(updates.append) + pool['counter:score@1000'].subscribe(updates.append) + pool['map:profile@1000'].subscribe(updates.append) + + # The mock cannot drive every state, so the channel-state handler is called directly + channel.object._act_on_channel_state(state) + + if state == ChannelState.SUSPENDED: + # RTO27b: the data is retained unchanged + assert 'name' in pool['root'].data + assert pool['counter:score@1000'].value() == 100 + assert 'email' in pool['map:profile@1000'].data + else: + # RTO27a1: every object's data is cleared, the nested map's checked through the pool + # rather than through the root, while the objects themselves stay in the pool + assert pool['root'].data == {} + assert 'counter:score@1000' in pool + assert pool['counter:score@1000'].value() == 0 + assert 'map:profile@1000' in pool + assert pool['map:profile@1000'].data == {} + + # RTO27a1: and no update is emitted for the clear + await settle() + assert updates == [] diff --git a/test/uts/objects/unit/value_types_test.py b/test/uts/objects/unit/value_types_test.py new file mode 100644 index 00000000..a424b108 --- /dev/null +++ b/test/uts/objects/unit/value_types_test.py @@ -0,0 +1,291 @@ +"""Derived from uts/objects/unit/value_types.md in ably/specification. + +Spec points: RTLCV1, RTLCV2, RTLCV3, RTLCV3a1, RTLCV3b, RTLCV3c, RTLCV3d, RTLCV4, RTLCV4a, +RTLCV4b1, RTLCV4c, RTLCV4d, RTLCV4f, RTLCV4g1, RTLCV4g2, RTLCV4g3, RTLCV4g4, RTLCV4g5, +RTLMV1, RTLMV2, RTLMV3, RTLMV3a1, RTLMV3b, RTLMV3d, RTLMV4, RTLMV4a, RTLMV4b, RTLMV4c, +RTLMV4d, RTLMV4d1, RTLMV4d2, RTLMV4d3, RTLMV4d4, RTLMV4d5, RTLMV4d6, RTLMV4d7, RTLMV4e1, +RTLMV4e2, RTLMV4f, RTLMV4g, RTLMV4i, RTLMV4j1, RTLMV4j3, RTLMV4j4, RTLMV4j5, RTLMV4k + +Pure tests: no client and no mock. The blueprints' internal `count` and `entries` are +`_count` and `_entries`. The specification's `evaluate(vt)` is `evaluate(vt, timestamp_ms)`: +evaluation reads the server time (RTLCV4e, RTLMV4h), which the caller fetches and passes +in (S-3), so these tests pass a fixed one. + +Where the specification reads `operation.counterCreate` or `operation.mapCreate` of an +evaluated message, it means the create retained alongside the `*CreateWithObjectId` +payload (RTLCV4g5, RTLMV4j5). The evaluated operation carries only the +`*CreateWithObjectId` payload, and the retained create is read through +`resolved_counter_create` / `resolved_map_create`, its `derived_from` (S-4). Its entries +hold decoded values, so a `bytes` entry holds bytes; the base64 form is in the +`initialValue` JSON, which is encoded for the JSON wire (RTLMV4f1). +""" + +import json + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate +from ably.pubsub.util.exceptions import AblyException + +# The server time evaluation generates object ids with (S-3) +SERVER_TIME_MS = 1_700_000_000_000 + + +# UTS: objects/unit/RTLCV3/create-with-count-0 +def test_rtlcv3_create_with_count(): + vt = LiveCounter.create(42) + + assert isinstance(vt, LiveCounter) + assert vt._count == 42 + + +# UTS: objects/unit/RTLCV3/create-default-zero-0 +def test_rtlcv3_create_default_zero(): + vt = LiveCounter.create() + + assert vt._count == 0 + + +# UTS: objects/unit/RTLCV3c/no-validation-at-create-0 +def test_rtlcv3c_no_validation_at_create(): + # Does not raise + vt = LiveCounter.create('not_a_number') + + assert isinstance(vt, LiveCounter) + + +# UTS: objects/unit/RTLCV4/evaluate-generates-message-0 +def test_rtlcv4_evaluate_generates_message(): + vt = LiveCounter.create(42) + messages = evaluate(vt, SERVER_TIME_MS) + + assert len(messages) == 1 + msg = messages[0] + assert msg.operation.action == ObjectOperationAction.COUNTER_CREATE + assert msg.operation.object_id.startswith('counter:') + assert '@' in msg.operation.object_id + assert msg.operation.counter_create_with_object_id is not None + assert msg.operation.counter_create_with_object_id.nonce is not None + assert len(msg.operation.counter_create_with_object_id.nonce) >= 16 + assert msg.operation.counter_create_with_object_id.initial_value is not None + # RTLCV4c: the initial value is the JSON of the CounterCreate + assert json.loads(msg.operation.counter_create_with_object_id.initial_value) == {'count': 42} + + +# UTS: objects/unit/RTLCV4g5/retains-local-counter-create-0 +def test_rtlcv4g5_retains_local_counter_create(): + vt = LiveCounter.create(42) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the specification's `operation.counterCreate` is the retained CounterCreate + assert msg.operation.resolved_counter_create is not None + assert msg.operation.resolved_counter_create.count == 42 + # The retained CounterCreate is local only and never sent (RTLCV4g5) + assert 'counterCreate' not in msg.operation.to_dict() + + +# UTS: objects/unit/RTLCV4a/evaluate-validates-count-0 +def test_rtlcv4a_evaluate_validates_count(): + vt = LiveCounter.create('not_a_number') + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTLCV4/evaluate-zero-count-0 +def test_rtlcv4_evaluate_zero_count(): + vt = LiveCounter.create(0) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the retained CounterCreate + assert msg.operation.resolved_counter_create.count == 0 + + +# UTS: objects/unit/RTLMV3/create-with-entries-0 +def test_rtlmv3_create_with_entries(): + vt = LiveMap.create({ + 'name': 'Alice', + 'age': 30, + }) + + assert isinstance(vt, LiveMap) + assert vt._entries['name'] == 'Alice' + assert vt._entries['age'] == 30 + + +# UTS: objects/unit/RTLMV3/create-no-entries-0 +def test_rtlmv3_create_no_entries(): + vt = LiveMap.create() + + assert isinstance(vt, LiveMap) + + +# UTS: objects/unit/RTLMV4/evaluate-generates-message-0 +def test_rtlmv4_evaluate_generates_message(): + vt = LiveMap.create({'name': 'Alice'}) + messages = evaluate(vt, SERVER_TIME_MS) + + assert len(messages) == 1 + msg = messages[0] + assert msg.operation.action == ObjectOperationAction.MAP_CREATE + assert msg.operation.object_id.startswith('map:') + assert msg.operation.map_create_with_object_id is not None + assert len(msg.operation.map_create_with_object_id.nonce) >= 16 + assert msg.operation.map_create_with_object_id.initial_value is not None + # RTLMV4f: the initial value is the JSON of the encoded MapCreate + initial_value = json.loads(msg.operation.map_create_with_object_id.initial_value) + assert initial_value['semantics'] == ObjectsMapSemantics.LWW + assert initial_value['entries']['name']['data'] == {'string': 'Alice'} + + +# UTS: objects/unit/RTLMV4j5/retains-local-map-create-0 +def test_rtlmv4j5_retains_local_map_create(): + vt = LiveMap.create({'name': 'Alice'}) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the specification's `operation.mapCreate` is the retained MapCreate + assert msg.operation.resolved_map_create is not None + assert msg.operation.resolved_map_create.semantics == ObjectsMapSemantics.LWW + assert msg.operation.resolved_map_create.entries['name'].data.string == 'Alice' + # The retained MapCreate is local only and never sent (RTLMV4j5) + assert 'mapCreate' not in msg.operation.to_dict() + + +# UTS: objects/unit/RTLMV4d/entry-value-types-0 +def test_rtlmv4d_entry_value_types(): + vt = LiveMap.create({ + 'str': 'hello', + 'num': 42, + 'bool': True, + 'json_arr': [1, 2, 3], + 'json_obj': {'key': 'value'}, + }) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the retained MapCreate + entries = msg.operation.resolved_map_create.entries + assert entries['str'].data.string == 'hello' + assert entries['num'].data.number == 42 + # `bool` is an `int` in Python, so a boolean is told apart from a number by identity + assert entries['bool'].data.boolean is True + assert entries['json_arr'].data.json == [1, 2, 3] + assert entries['json_obj'].data.json == {'key': 'value'} + + +# UTS: objects/unit/RTLMV4d1/nested-value-types-0 +def test_rtlmv4d1_nested_value_types(): + inner_counter = LiveCounter.create(10) + inner_map = LiveMap.create({ + 'nested_count': inner_counter, + }) + outer = LiveMap.create({ + 'child': inner_map, + }) + messages = evaluate(outer, SERVER_TIME_MS) + + assert len(messages) == 3 + assert messages[0].operation.action == ObjectOperationAction.COUNTER_CREATE + assert messages[0].operation.object_id.startswith('counter:') + assert messages[1].operation.action == ObjectOperationAction.MAP_CREATE + assert messages[1].operation.object_id.startswith('map:') + assert messages[2].operation.action == ObjectOperationAction.MAP_CREATE + assert messages[2].operation.object_id.startswith('map:') + + inner_counter_id = messages[0].operation.object_id + inner_map_id = messages[1].operation.object_id + + # S-4: the retained MapCreates + assert messages[1].operation.resolved_map_create.entries['nested_count'].data.object_id == inner_counter_id + assert messages[2].operation.resolved_map_create.entries['child'].data.object_id == inner_map_id + + +# UTS: objects/unit/RTLMV4a/evaluate-validates-entries-0 +def test_rtlmv4a_evaluate_validates_entries(): + # The specification's input is `LiveMap.create(null)`. `LiveMap.create` takes None as its + # default, so `None` cannot be told apart from an omitted argument and the null input is + # not applicable. The other half of RTLMV4a, entries that are not a dict, is the + # reachable failure. + vt = LiveMap.create(['not', 'a', 'dict']) + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40003 + + # `None` is the omitted argument, and evaluates as an empty map (RTLMV4e2) + messages = evaluate(LiveMap.create(None), SERVER_TIME_MS) + assert messages[-1].operation.resolved_map_create.entries == {} + + +# UTS: objects/unit/RTLMV4b/evaluate-validates-keys-0 +def test_rtlmv4b_evaluate_validates_keys(): + vt = LiveMap.create({123: 'value'}) + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTLMV4c/evaluate-validates-values-0 +def test_rtlmv4c_evaluate_validates_values(): + vt = LiveMap.create({'fn': lambda: None}) + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40013 + + +# UTS: objects/unit/RTLMV4e2/empty-entries-0 +def test_rtlmv4e2_empty_entries(): + vt = LiveMap.create() + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the retained MapCreate + assert msg.operation.resolved_map_create.entries == {} + + +# The specification's `expected_value` for the bytes row, "AQID", is the JSON-wire form of +# bytes([1, 2, 3]). The retained MapCreate holds the decoded bytes, and the base64 is what the +# `initialValue` JSON carries, so the test asserts both. +MAP_SET_TYPE_SCENARIOS = [ + pytest.param('hello', 'string', 'hello', id='string'), + pytest.param(42, 'number', 42, id='number-42'), + pytest.param(3.14, 'number', 3.14, id='number-3.14'), + pytest.param(0, 'number', 0, id='number-0'), + pytest.param(-1, 'number', -1, id='number-minus-1'), + pytest.param(True, 'boolean', True, id='boolean-true'), + pytest.param(False, 'boolean', False, id='boolean-false'), + pytest.param([1, 'a', None], 'json', [1, 'a', None], id='json-array'), + pytest.param({'k': 'v'}, 'json', {'k': 'v'}, id='json-object'), + pytest.param(bytes([1, 2, 3]), 'bytes', bytes([1, 2, 3]), id='bytes'), +] + + +# UTS: objects/unit/RTLMV4d/map-set-all-types-table-0 +@pytest.mark.parametrize('value,expected_field,expected_value', MAP_SET_TYPE_SCENARIOS) +def test_rtlmv4d_map_set_all_types_table(value, expected_field, expected_value): + vt = LiveMap.create({'test_key': value}) + messages = evaluate(vt, SERVER_TIME_MS) + + # S-4: the retained MapCreate + entry = messages[0].operation.resolved_map_create.entries['test_key'] + actual = getattr(entry.data, expected_field) + if isinstance(expected_value, bool): + # `bool` is an `int` in Python, so a boolean is told apart from a number by identity + assert actual is expected_value + else: + assert actual == expected_value + + if expected_field == 'bytes': + initial_value = json.loads(messages[0].operation.map_create_with_object_id.initial_value) + assert initial_value['entries']['test_key']['data']['bytes'] == 'AQID' diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index 3c548191..b680edd5 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -151,9 +151,6 @@ async def test_rtc17_client_id_attribute(): # UTS: realtime/unit/RTC1a/echo-messages-option-0 -# DEVIATION: ably-python has no `echo_messages` option and sends no `echo` query -# parameter. See deviations.md. -@deviation async def test_rtc1a_echo_messages_option(): # RTC1a_1: echoMessages defaults to true mock_ws = MockWebSocket()