diff --git a/docs/last_known_state.md b/docs/last_known_state.md new file mode 100644 index 0000000..4405574 --- /dev/null +++ b/docs/last_known_state.md @@ -0,0 +1,29 @@ +# Last-known robot state + +A robot that loses Wi-Fi, gets stuck or is switched off stops answering, and from then on nothing says where +it was or what it last reported. The server keeps that for every robot it manages, so a client can show the last +known picture instead of an empty screen. + +## What is kept + +Per robot, in `last_known/` under the data directory (it survives a restart): + +- When the robot last spoke, and when its connection last closed. +- The last status it sent: state, battery and error code. +- Its last position on the map and the map it was drawn on. +- Its room table (which map segment is which room), with the room names from the home data. + +It is fed from the MQTT proxy: status pushes (`dps` 120, 121 and 122), the answers to `get_status` and +`get_room_mapping`, and the `get_map_v1` reply, which is decrypted with the nonce of the request that asked for +it. On startup the store is seeded from the existing MQTT log, so a freshly updated server does not begin empty. + +## Reading it + +```text +GET /user/devices/{duid}/last_known +GET /user/devices/{duid}/last_known?map=1 +``` + +The answer is the usual `{"code": 200, "data": {...}}` envelope. `data.known` is `false` for a robot the server has +never heard from, which is how a client tells "never paired" from "paired and currently offline". With `?map=1` the +last map is included as base64 of the same bytes `get_map_v1` returns once decrypted and decompressed. diff --git a/mkdocs.yml b/mkdocs.yml index 7dd2153..be7c8d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ nav: - Camera (Live Video): camera.md - Reference: - Log Retention: logging.md + - Last-Known State: last_known_state.md - Known Limitations: known_limitations.md - Scene Validity Reports: scene_validity.md - Tested Vacuums: tested_vacuums.md diff --git a/src/roborock_local_server/bundled_backend/https_server/endpoint_rules.py b/src/roborock_local_server/bundled_backend/https_server/endpoint_rules.py index f2f2dcf..e5ad7dc 100644 --- a/src/roborock_local_server/bundled_backend/https_server/endpoint_rules.py +++ b/src/roborock_local_server/bundled_backend/https_server/endpoint_rules.py @@ -121,6 +121,8 @@ from .routes.user.deviceshare import match_rooms as _match_get_shared_device_rooms from .routes.user.devices.jobs import build as _build_get_schedules from .routes.user.devices.jobs import match as _match_get_schedules +from .routes.user.devices.last_known import build as _build_last_known +from .routes.user.devices.last_known import match as _match_last_known from .routes.user.devices.newadd import build as _build_add_device from .routes.user.devices.newadd import match as _match_add_device from .routes.user.homes.item import build as _build_get_home_data @@ -609,6 +611,7 @@ def default_endpoint_rules() -> Sequence[EndpointRule]: EndpointRule("get_device", _match_get_device, _build_get_device), EndpointRule("get_device_extra", _match_get_device_extra, _build_get_device_extra), EndpointRule("get_schedules", _match_get_schedules, _build_get_schedules), + EndpointRule("get_device_last_known", _match_last_known, _build_last_known), EndpointRule("post_app_info", _match_post_app_info, _build_post_app_info), EndpointRule("get_inbox_latest", _match_get_inbox_latest, _build_get_inbox_latest), EndpointRule("get_products_v4", _match_get_products_v4, _build_get_products_v4), diff --git a/src/roborock_local_server/bundled_backend/https_server/routes/user/devices/last_known.py b/src/roborock_local_server/bundled_backend/https_server/routes/user/devices/last_known.py new file mode 100644 index 0000000..c4fe2f4 --- /dev/null +++ b/src/roborock_local_server/bundled_backend/https_server/routes/user/devices/last_known.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import re +from typing import Any + +from shared.context import ServerContext +from shared.http_helpers import wrap_response + +from ..homes.service import home_rooms_payload + + +def match(path: str) -> bool: + clean = path.rstrip("/") + return bool(re.fullmatch(r"/user/devices/[^/]+/last_known", clean)) + + +def build( + ctx: ServerContext, + query_params: dict[str, list[str]], + _body_params: dict[str, list[str]], + clean_path: str, +) -> dict[str, Any]: + """The robot's last-known state: status, position, and when it last spoke or went offline. + + ``?map=1`` adds the last map the robot sent (base64 of the same bytes ``get_map_v1`` returns + once decrypted and decompressed), so a client can draw it while the robot cannot be asked. + """ + parts = [part for part in clean_path.rstrip("/").split("/") if part] + duid = parts[-2] if len(parts) >= 2 else "" + store = ctx.snapshot_store + snapshot = None + if store is not None: + include_map = (query_params.get("map") or [""])[0] in ("1", "true") + snapshot = store.get(duid, include_map=include_map) + if snapshot is None: + return wrap_response({"duid": duid, "known": False}) + # The robot's room table says which segment is which room; the home data knows the names. + names = {str(room.get("id")): room.get("name") for room in home_rooms_payload(ctx) if isinstance(room, dict)} + for room in snapshot.get("rooms") or []: + room["name"] = names.get(str(room.get("iot_id"))) + return wrap_response({**snapshot, "known": True}) diff --git a/src/roborock_local_server/bundled_backend/mqtt_tls_proxy_server/server.py b/src/roborock_local_server/bundled_backend/mqtt_tls_proxy_server/server.py index e497909..02003fb 100644 --- a/src/roborock_local_server/bundled_backend/mqtt_tls_proxy_server/server.py +++ b/src/roborock_local_server/bundled_backend/mqtt_tls_proxy_server/server.py @@ -19,6 +19,7 @@ from shared.protocol_auth import ProtocolAuthStore from shared.runtime_credentials import RuntimeCredentialsStore, parse_mqtt_connect_packet from shared.runtime_state import RuntimeState +from shared.device_snapshot_store import DeviceSnapshotStore, is_status_answer from shared.zone_ranges_store import ZoneRangesStore from .command_handlers import RpcCommandRegistry @@ -46,6 +47,7 @@ def __init__( runtime_state: RuntimeState | None = None, runtime_credentials: RuntimeCredentialsStore | None = None, zone_ranges_store: ZoneRangesStore | None = None, + snapshot_store: DeviceSnapshotStore | None = None, tls_enabled: bool = True, on_onboarding_credentials_learned: Callable[[], None] | None = None, ) -> None: @@ -65,6 +67,8 @@ def __init__( self.runtime_state = runtime_state self.runtime_credentials = runtime_credentials self.zone_ranges_store = zone_ranges_store + self.snapshot_store = snapshot_store + self._conn_duids: dict[str, set[str]] = {} self._on_onboarding_credentials_learned = on_onboarding_credentials_learned self._server_socket: socket.socket | None = None self._running = False @@ -521,6 +525,56 @@ def _parse_v1_rpc_payload(payload_utf8: str | None, protocol_value: int) -> dict parsed["error"] = rpc_obj.get("error") return parsed + def _duid_for_topic(self, topic: str) -> str: + parts = topic.split("/") + if topic.startswith("rr/m/") and len(parts) >= 6: + return parts[5] + if topic.startswith("rr/d/") and len(parts) >= 5 and self.runtime_credentials is not None: + device = self.runtime_credentials.resolve_device(did=parts[3]) + return str(device.get("duid") or "") if device else "" + return "" + + def _observe_snapshot( + self, + conn_id: str, + direction: str, + topic: str, + proto_value: int, + decoded_entry: dict[str, Any], + payload_bytes: bytes, + ) -> None: + """Feed what the robot says into the last-known-state store.""" + if self.snapshot_store is None: + return + try: + duid = self._duid_for_topic(topic) + if not duid: + return + payload_utf8 = decoded_entry.get("payload_utf8") + payload_utf8 = payload_utf8 if isinstance(payload_utf8, str) else None + if proto_value == 101: + # Any client's request: the nonce in it is what decrypts the map reply it gets. + self.snapshot_store.note_request(duid, payload_utf8) + return + if not (topic.startswith("rr/d/i/") and direction == "c2b"): + return + with self._lock: + self._conn_duids.setdefault(conn_id, set()).add(duid) + self.snapshot_store.mark_seen(duid) + if proto_value == 301: + self.snapshot_store.record_map(duid, payload_bytes) + elif proto_value == 102: + response_to = decoded_entry.get("response_to") + if isinstance(response_to, dict): + if is_status_answer(response_to) and response_to.get("error") is None: + self.snapshot_store.record_status_result(duid, response_to.get("result")) + elif response_to.get("request_method") == "get_room_mapping" and response_to.get("error") is None: + self.snapshot_store.record_room_mapping(duid, response_to.get("result")) + else: + self.snapshot_store.record_dps(duid, payload_utf8) + except Exception: + self.logger.exception("[conn %s] snapshot update failed", conn_id) + def _trace_packet(self, conn_id: str, direction: str, packet: bytes) -> None: packet_type = packet[0] >> 4 if packet_type in (12, 13): # PINGREQ, PINGRESP @@ -661,6 +715,7 @@ def _trace_packet(self, conn_id: str, direction: str, packet: bytes) -> None: response_to.get("request_method"), response_to.get("error"), ) + self._observe_snapshot(conn_id, direction, topic, proto_value, decoded_entry, payload_bytes) decoded_messages.append(decoded_entry) self.logger.info( "[conn %s %s] PUBLISH topic=%s proto=%s(%d) ver=%s seq=%s key_source=%s payload=%s", @@ -815,6 +870,10 @@ def _handle_client(self, tls_conn: socket.socket | ssl.SSLSocket, addr: tuple[st self.runtime_state.record_mqtt_disconnect(conn_id=conn_id) with self._lock: self._conn_protocol_levels.pop(conn_id, None) + gone_duids = self._conn_duids.pop(conn_id, set()) + if self.snapshot_store is not None: + for duid in gone_duids: + self.snapshot_store.mark_offline(duid) self.logger.info("[conn %s] closed", conn_id) def start(self) -> threading.Thread: diff --git a/src/roborock_local_server/bundled_backend/shared/context.py b/src/roborock_local_server/bundled_backend/shared/context.py index 69dc72c..e91cf4e 100644 --- a/src/roborock_local_server/bundled_backend/shared/context.py +++ b/src/roborock_local_server/bundled_backend/shared/context.py @@ -15,6 +15,7 @@ from .http_helpers import pick_first from .runtime_credentials import RuntimeCredentialsStore from .runtime_state import RuntimeState +from .device_snapshot_store import DeviceSnapshotStore from .zone_ranges_store import ZoneRangesStore _DEVICE_ID_KEYS = ("did", "d", "duid", "device_id", "deviceId", "device_did", "deviceDid") @@ -70,6 +71,7 @@ class ServerContext: runtime_state: RuntimeState | None = None runtime_credentials: RuntimeCredentialsStore | None = None zone_ranges_store: ZoneRangesStore | None = None + snapshot_store: DeviceSnapshotStore | None = None timezone: str | None = None # TURN/STUN advertised to the robot for camera sessions (see routes/fw/createca.py). turn_enabled: bool = False diff --git a/src/roborock_local_server/bundled_backend/shared/device_snapshot_store.py b/src/roborock_local_server/bundled_backend/shared/device_snapshot_store.py new file mode 100644 index 0000000..b7d55cb --- /dev/null +++ b/src/roborock_local_server/bundled_backend/shared/device_snapshot_store.py @@ -0,0 +1,451 @@ +"""Persistent last-known state of each robot the server manages. + +A robot that dies, loses Wi-Fi or gets stuck stops answering, and from then on nothing +can say where it was or what it last reported. This store keeps, per device, the last +status, battery and error the robot sent, its last position on the map and the map it +was drawn on, and when it last spoke or went offline. It survives a server restart. + +It is fed from the MQTT proxy: status pushes (``dps`` 120/121/122), ``get_status`` +answers, and the ``get_map_v1`` reply, which is decrypted with the nonce of the request +that asked for it. Position needs that map; nothing else in the protocol carries it. +""" + +from __future__ import annotations + +import base64 +import json +import logging +import re +import struct +import threading +import time +from pathlib import Path +from collections.abc import Callable +from typing import Any + +from roborock_local_server.log_retention import iter_log_lines + +from .data_helpers import utcnow_iso + +logger = logging.getLogger(__name__) + +_DIR_NAME = "device_snapshots" +_MAX_NONCES_PER_DEVICE = 16 +_SAFE_DUID = re.compile(r"^[A-Za-z0-9_-]+$") + +# dps keys the robot pushes on its own. +_DPS_ERROR = "120" +_DPS_STATE = "121" +_DPS_BATTERY = "122" + +_SECURITY_RE = re.compile(r'"endpoint\\?":\s*\\?"(\w+)\\?",\s*\\?"nonce\\?":\s*\\?"([0-9a-fA-F]+)') + + +def _bytes_from_hex(value: Any) -> bytes | None: + if not isinstance(value, str): + return None + try: + return bytes.fromhex(value) + except ValueError: + return None + + +def _int_or_none(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + return None + + +def _point(point: Any) -> dict[str, int] | None: + if point is None: + return None + try: + return {"x": int(point.x), "y": int(point.y), "angle": int(getattr(point, "a", 0) or 0)} + except (AttributeError, TypeError, ValueError): + return None + + +def _parse_map(map_bytes: bytes) -> tuple[dict[str, int] | None, dict[str, int] | None]: + """(robot position, charger position) in robot millimetres, or Nones if it will not parse.""" + from roborock.map.map_parser import MapParser, MapParserConfig + + parsed = MapParser(MapParserConfig()).parse(map_bytes) + map_data = parsed.map_data if parsed is not None else None + if map_data is None: + return None, None + return _point(map_data.vacuum_position), _point(map_data.charger) + + +def _dps_update(payload_utf8: str | None) -> dict[str, int]: + """The status a push carries: ``{"dps": {"122": 5}}`` -> ``{"battery": 5}``.""" + if not payload_utf8: + return {} + try: + payload = json.loads(payload_utf8) + except (TypeError, json.JSONDecodeError): + return {} + dps = payload.get("dps") if isinstance(payload, dict) else None + if not isinstance(dps, dict): + return {} + update: dict[str, int] = {} + for key, name in ((_DPS_STATE, "state"), (_DPS_BATTERY, "battery"), (_DPS_ERROR, "error_code")): + value = _int_or_none(dps.get(key)) + if value is not None: + update[name] = value + return update + + +def is_status_answer(response_to: dict[str, Any]) -> bool: + """True for the answer to ``get_status`` or ``get_prop ["get_status"]`` (both carry the same status).""" + method = response_to.get("request_method") + if method == "get_status": + return True + return method == "get_prop" and response_to.get("request_params") == ["get_status"] + + +def _status_update(result: Any) -> dict[str, int]: + """The status a ``get_status`` answer carries: a one-item list holding the status object.""" + status = result[0] if isinstance(result, list) and result else result + if not isinstance(status, dict): + return {} + update: dict[str, int] = {} + for key in ("state", "battery", "error_code", "clean_percent"): + value = _int_or_none(status.get(key)) + if value is not None: + update[key] = value + return update + + +def _room_table(result: Any) -> list[dict[str, Any]]: + """The rows of a ``get_room_mapping`` answer, ``[segment, iot room id, type tag]``, as dicts.""" + rows: list[dict[str, Any]] = [] + for row in result if isinstance(result, list) else []: + if not isinstance(row, (list, tuple)) or len(row) < 2: + continue + segment = _int_or_none(row[0]) + if segment is None or row[1] in (None, ""): + continue + rows.append({"segment": segment, "iot_id": str(row[1]), "tag": _int_or_none(row[2]) if len(row) > 2 else None}) + return rows + + +def _security_of(payload_utf8: str | None) -> tuple[str, str] | None: + """(endpoint, nonce) from a request that asked for a map, or None.""" + if not payload_utf8 or "security" not in payload_utf8: + return None + found = _SECURITY_RE.search(payload_utf8) + return (found.group(1), found.group(2)) if found else None + + +def _decode_map(payload: bytes, nonce: str) -> tuple[bytes, dict[str, int] | None, dict[str, int] | None]: + """Decrypt a map reply: (the map bytes, robot position, charger position).""" + from roborock.protocol import Utils + + # 24-byte header (endpoint, request id), then the gzipped map encrypted with the nonce. + map_bytes = Utils.decompress(Utils.decrypt_cbc(payload[24:], bytes.fromhex(nonce))) + position, charger = _parse_map(map_bytes) + return map_bytes, position, charger + + +class DeviceSnapshotStore: + """Thread-safe, file-backed last-known state per device, keyed by duid.""" + + def __init__(self, directory: Path, *, flush_interval_s: float = 30.0, map_interval_s: float = 60.0) -> None: + self._dir = directory / _DIR_NAME + self._flush_interval_s = flush_interval_s + self._map_interval_s = map_interval_s + self._lock = threading.RLock() + self._data: dict[str, dict[str, Any]] = {} + self._maps: dict[str, bytes] = {} + self._nonces: dict[str, dict[str, str]] = {} + self._dirty: set[str] = set() + self._dirty_maps: set[str] = set() + self._last_flush = 0.0 + self._last_map_at: dict[str, float] = {} + self._load() + + # ---- persistence ------------------------------------------------------------- + + def _load(self) -> None: + try: + files = sorted(self._dir.glob("*.json")) + except OSError: + return + for path in files: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + duid = str(raw.get("duid") or path.stem) if isinstance(raw, dict) else "" + if not isinstance(raw, dict) or not duid or not _SAFE_DUID.match(duid): + continue + self._data[duid] = raw + map_path = self._dir / f"{duid}.map" + if map_path.exists(): + try: + self._maps[duid] = map_path.read_bytes() + except OSError: + pass + + def flush(self) -> None: + """Write everything changed since the last flush.""" + with self._lock: + dirty, dirty_maps = set(self._dirty), set(self._dirty_maps) + if not dirty and not dirty_maps: + return + try: + self._dir.mkdir(parents=True, exist_ok=True) + except OSError: + return + for duid in dirty: + self._write(self._dir / f"{duid}.json", json.dumps(self._data[duid], separators=(",", ":")).encode()) + for duid in dirty_maps: + if duid in self._maps: + self._write(self._dir / f"{duid}.map", self._maps[duid]) + self._dirty.clear() + self._dirty_maps.clear() + self._last_flush = time.monotonic() + + @staticmethod + def _write(path: Path, content: bytes) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + try: + tmp.write_bytes(content) + tmp.replace(path) + except OSError: + logger.warning("Could not write %s", path) + + def _touch(self, duid: str, *, force_flush: bool = False) -> None: + self._dirty.add(duid) + if force_flush or time.monotonic() - self._last_flush >= self._flush_interval_s: + self.flush() + + # ---- recording --------------------------------------------------------------- + + def _entry(self, duid: str) -> dict[str, Any]: + entry = self._data.get(duid) + if entry is None: + entry = {"duid": duid, "last_seen_at": None, "offline_at": None, "status": {}, "position": None} + self._data[duid] = entry + return entry + + def mark_seen(self, duid: str) -> None: + """The robot spoke: it is online, so any earlier offline mark is cleared.""" + if not _SAFE_DUID.match(duid or ""): + return + with self._lock: + entry = self._entry(duid) + entry["last_seen_at"] = utcnow_iso() + if entry.get("offline_at"): + entry["offline_at"] = None + self._touch(duid, force_flush=True) + else: + self._touch(duid) + + def mark_offline(self, duid: str) -> None: + """The robot's connection closed. Written at once, since the next restart may lose the rest.""" + if not _SAFE_DUID.match(duid or ""): + return + with self._lock: + entry = self._entry(duid) + if not entry.get("offline_at"): + entry["offline_at"] = utcnow_iso() + self._touch(duid, force_flush=True) + + def record_dps(self, duid: str, payload_utf8: str | None) -> None: + """A status push, ``{"dps": {"122": 5}}``.""" + update = _dps_update(payload_utf8) + if update and _SAFE_DUID.match(duid or ""): + self._update_status(duid, update) + + def record_status_result(self, duid: str, result: Any) -> None: + """The answer to ``get_status``.""" + update = _status_update(result) + if update and _SAFE_DUID.match(duid or ""): + self._update_status(duid, update) + + def record_room_mapping(self, duid: str, result: Any) -> None: + """The answer to ``get_room_mapping``: which room each map segment is, and its type.""" + rooms = _room_table(result) + if not rooms or not _SAFE_DUID.match(duid or ""): + return + with self._lock: + entry = self._entry(duid) + if entry.get("rooms") != rooms: + entry["rooms"] = rooms + self._touch(duid) + + def _update_status(self, duid: str, update: dict[str, int]) -> None: + with self._lock: + entry = self._entry(duid) + status = entry.setdefault("status", {}) + status.update(update) + status["updated_at"] = utcnow_iso() + self._touch(duid) + + def note_request(self, duid: str, payload_utf8: str | None) -> None: + """Remember the nonce a client sent with a request, so its map reply can be decrypted.""" + security = _security_of(payload_utf8) + if security is None or not _SAFE_DUID.match(duid or ""): + return + endpoint, nonce = security + with self._lock: + nonces = self._nonces.setdefault(duid, {}) + nonces.pop(endpoint, None) + nonces[endpoint] = nonce + while len(nonces) > _MAX_NONCES_PER_DEVICE: + nonces.pop(next(iter(nonces))) + + def record_map(self, duid: str, payload: bytes) -> None: + """A ``get_map_v1`` reply (protocol 301): decrypt, keep the map, and read the robot's position.""" + if not payload or len(payload) < 24 or not _SAFE_DUID.match(duid or ""): + return + now = time.monotonic() + with self._lock: + if now - self._last_map_at.get(duid, 0.0) < self._map_interval_s: + return + endpoint = struct.unpack("<8s", payload[:8])[0].decode("ascii", "replace") + nonce = self._nonces.get(duid, {}).get(endpoint) + if nonce is None: + return + try: + map_bytes, position, charger = _decode_map(payload, nonce) + except Exception: + logger.debug("Could not decode a map reply for %s", duid, exc_info=True) + return + with self._lock: + self._last_map_at[duid] = now + entry = self._entry(duid) + self._maps[duid] = map_bytes + self._dirty_maps.add(duid) + entry["position"] = { + "robot": position, + "charger": charger, + "map_updated_at": utcnow_iso(), + } + self._touch(duid) + + # ---- backfill ---------------------------------------------------------------- + + @staticmethod + def _duid_in_topic(topic: str, resolve_duid: Callable[[str], str]) -> str: + parts = topic.split("/") + if topic.startswith("rr/m/") and len(parts) >= 6: + return parts[5] + if topic.startswith("rr/d/") and len(parts) >= 5: + return resolve_duid(parts[3]) + return "" + + def seed_from_mqtt_jsonl(self, mqtt_jsonl: Path, resolve_duid: Callable[[str], str]) -> int: + """Fill in devices the store knows nothing about from the decoded MQTT log. + + The log holds what each robot last said, so a robot that stopped answering before this + store existed still gets its last status and map. A device that already has a snapshot is + left alone. ``resolve_duid`` turns the short id in a robot's topic into its duid. Returns + how many devices were added; it can take a while on a large log, so run it off the main + thread. + """ + with self._lock: + known = set(self._data) + nonces: dict[str, dict[str, str]] = {} + seen: dict[str, dict[str, Any]] = {} + for line in iter_log_lines(mqtt_jsonl): + # Most lines are neither a robot's own frame nor a map request; skip them unparsed. + if "rr/d/i/" not in line and "security" not in line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + topic = str(entry.get("topic") or "") + at = str(entry.get("time") or "") + duid = self._duid_in_topic(topic, resolve_duid) + if not duid or duid in known or not _SAFE_DUID.match(duid): + continue + from_robot = topic.startswith("rr/d/i/") and entry.get("direction") == "c2b" + for message in entry.get("decoded_messages") or []: + if not isinstance(message, dict): + continue + payload_utf8 = message.get("payload_utf8") + payload_utf8 = payload_utf8 if isinstance(payload_utf8, str) else None + proto = message.get("protocol_value") + if proto == 101: + security = _security_of(payload_utf8) + if security is not None: + nonces.setdefault(duid, {})[security[0]] = security[1] + continue + if not from_robot: + continue + acc = seen.setdefault(duid, {"status": {}, "status_at": None, "map": None, "rooms": []}) + acc["last_seen_at"] = at + if proto == 301: + raw = _bytes_from_hex(message.get("payload_hex")) + if raw is not None and len(raw) >= 24: + nonce = nonces.get(duid, {}).get(raw[:8].decode("ascii", "replace")) + if nonce: + acc["map"] = (at, raw, nonce) + elif proto == 102: + response_to = message.get("response_to") + if isinstance(response_to, dict) and response_to.get("request_method") == "get_room_mapping": + rooms = _room_table(response_to.get("result")) if response_to.get("error") is None else [] + if rooms: + acc["rooms"] = rooms + continue + if isinstance(response_to, dict): + update = ( + _status_update(response_to.get("result")) + if is_status_answer(response_to) and response_to.get("error") is None + else {} + ) + else: + update = _dps_update(payload_utf8) + if update: + acc["status"].update(update) + acc["status_at"] = at + + added = 0 + for duid, acc in seen.items(): + map_bytes = position = charger = None + if acc["map"] is not None: + try: + map_bytes, position, charger = _decode_map(acc["map"][1], acc["map"][2]) + except Exception: + logger.debug("Could not decode the last logged map for %s", duid, exc_info=True) + with self._lock: + if duid in self._data: + continue + entry = self._entry(duid) + entry["last_seen_at"] = acc.get("last_seen_at") + if acc["status"]: + entry["status"] = {**acc["status"], "updated_at": acc["status_at"]} + if acc["rooms"]: + entry["rooms"] = acc["rooms"] + if map_bytes is not None: + self._maps[duid] = map_bytes + self._dirty_maps.add(duid) + entry["position"] = {"robot": position, "charger": charger, "map_updated_at": acc["map"][0]} + self._dirty.add(duid) + added += 1 + if added: + self.flush() + return added + + # ---- reading ----------------------------------------------------------------- + + def get(self, duid: str, *, include_map: bool = False) -> dict[str, Any] | None: + with self._lock: + entry = self._data.get(duid) + if entry is None: + return None + result = json.loads(json.dumps(entry)) + map_bytes = self._maps.get(duid) + result["has_map"] = map_bytes is not None + if include_map and map_bytes is not None: + result["map"] = base64.b64encode(map_bytes).decode("ascii") + return result diff --git a/src/roborock_local_server/server.py b/src/roborock_local_server/server.py index c0412ef..6e4322c 100644 --- a/src/roborock_local_server/server.py +++ b/src/roborock_local_server/server.py @@ -61,6 +61,7 @@ load_cloud_full_snapshot, with_current_server_urls, ) +from .bundled_backend.shared.device_snapshot_store import DeviceSnapshotStore from .bundled_backend.shared.zone_ranges_store import ZoneRangesStore from .security import AdminSessionManager, verify_password @@ -454,6 +455,7 @@ def __init__( backup_count=self.config.logging.backup_count, ) + snapshot_store = DeviceSnapshotStore(self.paths.http_jsonl_path.parent) self.context = ServerContext( api_host=self.config.network.stack_fqdn, mqtt_host=self.config.network.stack_fqdn, @@ -475,6 +477,7 @@ def __init__( runtime_state=self.runtime_state, runtime_credentials=self.runtime_credentials, zone_ranges_store=zone_ranges_store, + snapshot_store=snapshot_store, timezone=self.config.network.timezone or None, turn_enabled=self.config.turn.mode != "disabled", turn_host=self.config.turn.host or self.config.network.stack_fqdn, @@ -484,9 +487,26 @@ def __init__( turn_realm=self.config.turn.realm or self.config.turn.host or self.config.network.stack_fqdn, turn_ttl=self.config.turn.ttl, ) + # A robot that went quiet before this store existed still has its last words in the MQTT log. + threading.Thread( + target=self._seed_snapshots, args=(snapshot_store,), daemon=True, name="snapshot-seed", + ).start() self.endpoint_rules = default_endpoint_rules() self.app = self._create_app() + def _seed_snapshots(self, store: DeviceSnapshotStore) -> None: + def resolve_duid(did: str) -> str: + device = self.runtime_credentials.resolve_device(did=did) + return str(device.get("duid") or "") if device else "" + + try: + added = store.seed_from_mqtt_jsonl(self.paths.mqtt_jsonl_path, resolve_duid) + except Exception: + self.root_logger.exception("Could not seed last-known device state from the MQTT log") + return + if added: + self.root_logger.info("Seeded last-known state for %d device(s) from the MQTT log", added) + def _init_zone_ranges_store(self) -> ZoneRangesStore: store = ZoneRangesStore(self.paths.http_jsonl_path.parent) if not store._data: @@ -1904,6 +1924,7 @@ def _start_mqtt_proxy(self) -> None: runtime_state=self.runtime_state, runtime_credentials=self.runtime_credentials, zone_ranges_store=self.context.zone_ranges_store, + snapshot_store=self.context.snapshot_store, tls_enabled=self._uses_local_tls(), on_onboarding_credentials_learned=self.persist_active_onboarding_device, ) diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index 13341a3..3a4a55b 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -1602,3 +1602,56 @@ def test_create_room_returns_bare_id_in_result_and_object_in_data(tmp_path: Path assert body["data"]["id"] == body["result"] assert body["data"]["name"] == "Study" assert body["data"]["created"] is True + + +def test_last_known_route_returns_the_stored_snapshot(tmp_path: Path) -> None: + config_file = write_release_config(tmp_path) + config = load_config(config_file) + paths = resolve_paths(config_file, config) + _seed_protocol_snapshot(paths.cloud_snapshot_path) + supervisor = ReleaseSupervisor(config=config, paths=paths) + client = TestClient(supervisor.app) + device_id = "6HL2zfniaoYYV01CkVuhkO" + path = f"/user/devices/{device_id}/last_known" + + from https_server.routes.user.devices.last_known import build as build_last_known + + assert build_last_known(supervisor.context, {}, {}, path)["data"] == {"duid": device_id, "known": False} + + store = supervisor.context.snapshot_store + store.record_dps(device_id, json.dumps({"dps": {"122": 5, "121": 12}})) + store.mark_offline(device_id) + + known = client.get(path, headers=_hawk_headers(paths.cloud_snapshot_path, path)) + assert known.status_code == 200 + data = known.json()["data"] + assert data["known"] is True + assert data["status"]["battery"] == 5 + assert data["offline_at"] + assert data["has_map"] is False + assert "map" not in data + + +def test_last_known_route_names_each_room_of_the_robots_room_table(tmp_path: Path, monkeypatch) -> None: + config_file = write_release_config(tmp_path) + config = load_config(config_file) + paths = resolve_paths(config_file, config) + _seed_protocol_snapshot(paths.cloud_snapshot_path) + supervisor = ReleaseSupervisor(config=config, paths=paths) + device_id = "6HL2zfniaoYYV01CkVuhkO" + path = f"/user/devices/{device_id}/last_known" + + from https_server.routes.user.devices import last_known as route + + monkeypatch.setattr(route, "home_rooms_payload", lambda ctx: [{"id": 1, "name": "Living Room"}, {"id": 4, "name": "Kitchen"}]) + store = supervisor.context.snapshot_store + store.record_dps(device_id, json.dumps({"dps": {"122": 5}})) + store.record_room_mapping(device_id, [[21, "1", 6], [19, "4", 14], [18, "999", 1]]) + + rooms = route.build(supervisor.context, {}, {}, path)["data"]["rooms"] + + assert rooms == [ + {"segment": 21, "iot_id": "1", "tag": 6, "name": "Living Room"}, + {"segment": 19, "iot_id": "4", "tag": 14, "name": "Kitchen"}, + {"segment": 18, "iot_id": "999", "tag": 1, "name": None}, + ] diff --git a/tests/test_device_snapshot_store.py b/tests/test_device_snapshot_store.py new file mode 100644 index 0000000..613e40c --- /dev/null +++ b/tests/test_device_snapshot_store.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import base64 +import gzip +import json +from pathlib import Path + +import pytest + +from roborock_local_server.bundled_backend.shared.device_snapshot_store import DeviceSnapshotStore, is_status_answer + +DUID = "2e2txkqKqxVifcIdyOT1lY" + + +def _store(tmp_path: Path) -> DeviceSnapshotStore: + return DeviceSnapshotStore(tmp_path, flush_interval_s=0.0, map_interval_s=0.0) + + +def test_unknown_device_has_no_snapshot(tmp_path: Path) -> None: + assert _store(tmp_path).get(DUID) is None + + +def test_status_push_records_battery_state_and_error(tmp_path: Path) -> None: + store = _store(tmp_path) + store.record_dps(DUID, json.dumps({"t": 1, "dps": {"122": 5}})) + store.record_dps(DUID, json.dumps({"t": 2, "dps": {"121": 12, "120": 4}})) + + status = store.get(DUID)["status"] + + assert (status["battery"], status["state"], status["error_code"]) == (5, 12, 4) + assert status["updated_at"] + + +def test_get_status_answer_is_read_from_the_one_item_list(tmp_path: Path) -> None: + store = _store(tmp_path) + store.record_status_result(DUID, [{"state": 8, "battery": 100, "error_code": 0, "ignored": "x"}]) + + status = store.get(DUID)["status"] + + assert (status["state"], status["battery"], status["error_code"]) == (8, 100, 0) + assert "ignored" not in status + + +def test_dps_ignores_unrelated_or_malformed_payloads(tmp_path: Path) -> None: + store = _store(tmp_path) + store.record_dps(DUID, "not json") + store.record_dps(DUID, json.dumps({"dps": {"101": "{}"}})) + store.record_dps(DUID, json.dumps({"dps": {"122": True}})) + + assert store.get(DUID) is None + + +def test_offline_is_marked_and_cleared_when_the_robot_speaks_again(tmp_path: Path) -> None: + store = _store(tmp_path) + store.mark_seen(DUID) + store.mark_offline(DUID) + assert store.get(DUID)["offline_at"] + + store.mark_seen(DUID) + assert store.get(DUID)["offline_at"] is None + + +def test_state_survives_a_restart(tmp_path: Path) -> None: + store = _store(tmp_path) + store.record_dps(DUID, json.dumps({"dps": {"122": 5}})) + store.mark_offline(DUID) + store.flush() + + reopened = _store(tmp_path).get(DUID) + + assert reopened["status"]["battery"] == 5 + assert reopened["offline_at"] + + +def test_unsafe_device_ids_are_refused(tmp_path: Path) -> None: + store = _store(tmp_path) + for bad in ("", "../x", "a/b", "a b"): + store.mark_seen(bad) + store.mark_offline(bad) + store.record_dps(bad, json.dumps({"dps": {"122": 5}})) + + assert not (tmp_path / "device_snapshots").exists() + + +def test_request_nonce_is_found_in_the_escaped_rpc_payload(tmp_path: Path) -> None: + store = _store(tmp_path) + payload = ( + '{"dps":{"101":"{\\"id\\":29755,\\"method\\":\\"get_map_v1\\",\\"params\\":[],' + '\\"security\\":{\\"endpoint\\":\\"w5Mpv215\\",\\"nonce\\":\\"0fd7440a89d9e25af17aa00493c29025\\"}}"},"t":1}' + ) + + store.note_request(DUID, payload) + + assert store._nonces[DUID] == {"w5Mpv215": "0fd7440a89d9e25af17aa00493c29025"} + + +def test_map_reply_without_a_known_nonce_is_ignored(tmp_path: Path) -> None: + store = _store(tmp_path) + + store.record_map(DUID, b"w5Mpv215" + b"\x00" * 64) + + assert store.get(DUID) is None + + +def test_map_reply_is_decrypted_and_its_position_stored(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from roborock.protocol import Utils + + from roborock_local_server.bundled_backend.shared import device_snapshot_store as module + + class Point: + def __init__(self, x: int, y: int, a: int) -> None: + self.x, self.y, self.a = x, y, a + + map_bytes = b"fake map bytes" + nonce = bytes(range(16)) + header = b"w5Mpv215" + b"\x00" * 8 + b"\x01\x00" + b"\x00" * 6 + body = Utils.encrypt_cbc(gzip.compress(map_bytes), nonce) + monkeypatch.setattr(module, "_parse_map", lambda data: (module._point(Point(24343, 38108, 170)), module._point(Point(26306, 25598, -177))) if data == map_bytes else (None, None)) + + store = _store(tmp_path) + store.note_request( + DUID, + '{"dps":{"101":"{\\"security\\":{\\"endpoint\\":\\"w5Mpv215\\",\\"nonce\\":\\"%s\\"}}"}}' % nonce.hex(), + ) + store.record_map(DUID, header + body) + + snapshot = store.get(DUID, include_map=True) + + assert snapshot["position"]["robot"] == {"x": 24343, "y": 38108, "angle": 170} + assert snapshot["position"]["charger"] == {"x": 26306, "y": 25598, "angle": -177} + assert base64.b64decode(snapshot["map"]) == map_bytes + assert snapshot["has_map"] is True + assert "map" not in store.get(DUID) + + +def _log_line(time: str, direction: str, topic: str, *messages: dict) -> str: + return json.dumps({"time": time, "direction": direction, "topic": topic, "decoded_messages": list(messages)}) + "\n" + + +def _seed_log(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, map_bytes: bytes = b"last map bytes") -> Path: + from roborock.protocol import Utils + + from roborock_local_server.bundled_backend.shared import device_snapshot_store as module + + class Point: + def __init__(self, x: int, y: int, a: int) -> None: + self.x, self.y, self.a = x, y, a + + monkeypatch.setattr( + module, + "_parse_map", + lambda data: (module._point(Point(24343, 38108, 170)), module._point(Point(26306, 25598, -177))) + if data == map_bytes + else (None, None), + ) + nonce = bytes(range(16)) + header = b"w5Mpv215" + b"\x00" * 8 + b"\x01\x00" + b"\x00" * 6 + reply = header + Utils.encrypt_cbc(gzip.compress(map_bytes), nonce) + request = '{"dps":{"101":"{\\"security\\":{\\"endpoint\\":\\"w5Mpv215\\",\\"nonce\\":\\"%s\\"}}"}}' % nonce.hex() + robot = "rr/d/i/1103829226788/abc" + path = tmp_path / "decoded.jsonl" + path.write_text( + _log_line("2026-10-22T00:20:00+00:00", "c2b", f"rr/m/i/user/abcd1234/{DUID}", {"protocol_value": 101, "payload_utf8": request}) + + _log_line("2026-10-22T00:24:17+00:00", "c2b", robot, {"protocol_value": 102, "payload_utf8": '{"dps":{"122":7}}'}) + + _log_line("2026-10-22T00:25:00+00:00", "c2b", robot, {"protocol_value": 301, "payload_hex": reply.hex()}) + + _log_line( + "2026-10-22T00:30:00+00:00", "c2b", robot, + {"protocol_value": 102, "response_to": {"request_method": "get_status", "error": None, "result": [{"state": 2, "battery": 6}]}}, + ) + + _log_line("2026-10-22T00:33:57+00:00", "c2b", robot, {"protocol_value": 102, "payload_utf8": '{"dps":{"122":5}}'}) + # An app's own traffic and unrelated lines must not count as the robot speaking. + + _log_line("2026-10-22T03:00:00+00:00", "b2c", "rr/d/o/1103829226788/abc", {"protocol_value": 102, "payload_utf8": '{"dps":{"122":99}}'}) + + "not json at all\n", + encoding="utf-8", + ) + return path + + +def test_seed_recovers_the_last_status_map_and_time_from_the_log(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + log = _seed_log(tmp_path, monkeypatch) + store = _store(tmp_path / "state") + + added = store.seed_from_mqtt_jsonl(log, lambda did: DUID if did == "1103829226788" else "") + + snapshot = store.get(DUID, include_map=True) + assert added == 1 + assert snapshot["last_seen_at"] == "2026-10-22T00:33:57+00:00" + assert snapshot["status"]["battery"] == 5 + assert snapshot["status"]["state"] == 2 + assert snapshot["position"]["robot"] == {"x": 24343, "y": 38108, "angle": 170} + assert snapshot["position"]["map_updated_at"] == "2026-10-22T00:25:00+00:00" + assert base64.b64decode(snapshot["map"]) == b"last map bytes" + + +def test_seed_persists_so_a_restart_keeps_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + log = _seed_log(tmp_path, monkeypatch) + _store(tmp_path / "state").seed_from_mqtt_jsonl(log, lambda did: DUID) + + reopened = _store(tmp_path / "state").get(DUID) + + assert reopened["status"]["battery"] == 5 + assert reopened["position"]["robot"]["x"] == 24343 + + +def test_seed_leaves_a_device_that_already_has_a_snapshot_alone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + log = _seed_log(tmp_path, monkeypatch) + store = _store(tmp_path / "state") + store.record_dps(DUID, json.dumps({"dps": {"122": 80}})) + + added = store.seed_from_mqtt_jsonl(log, lambda did: DUID) + + assert added == 0 + assert store.get(DUID)["status"]["battery"] == 80 + + +def test_seed_skips_robots_it_cannot_name_and_missing_logs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + log = _seed_log(tmp_path, monkeypatch) + store = _store(tmp_path / "state") + + assert store.seed_from_mqtt_jsonl(log, lambda did: "") == 0 + assert store.seed_from_mqtt_jsonl(tmp_path / "missing.jsonl", lambda did: DUID) == 0 + + +def test_room_mapping_answer_is_kept_and_survives_a_restart(tmp_path: Path) -> None: + store = _store(tmp_path) + store.record_room_mapping(DUID, [[21, "1001", 6], [16, "1002", 8], [18, "1003"]]) + store.flush() + + rooms = _store(tmp_path).get(DUID)["rooms"] + + assert rooms == [ + {"segment": 21, "iot_id": "1001", "tag": 6}, + {"segment": 16, "iot_id": "1002", "tag": 8}, + {"segment": 18, "iot_id": "1003", "tag": None}, + ] + + +def test_room_mapping_ignores_empty_and_malformed_answers(tmp_path: Path) -> None: + store = _store(tmp_path) + store.record_room_mapping(DUID, []) + store.record_room_mapping(DUID, "nope") + store.record_room_mapping(DUID, [["x", "1"], [3], [4, None]]) + + assert store.get(DUID) is None + + +def test_get_prop_get_status_counts_as_a_status_answer() -> None: + assert is_status_answer({"request_method": "get_status"}) + assert is_status_answer({"request_method": "get_prop", "request_params": ["get_status"]}) + assert not is_status_answer({"request_method": "get_prop", "request_params": ["get_consumable"]}) + assert not is_status_answer({"request_method": "get_prop"}) + assert not is_status_answer({"request_method": "get_room_mapping"}) diff --git a/tests/test_mqtt_tls_proxy.py b/tests/test_mqtt_tls_proxy.py index 61d7e33..07866c5 100644 --- a/tests/test_mqtt_tls_proxy.py +++ b/tests/test_mqtt_tls_proxy.py @@ -1195,3 +1195,78 @@ def test_stalled_tls_handshake_is_closed_after_timeout(start_tls_proxy) -> None: proxy = start_tls_proxy(handshake_timeout_seconds=0.5) with socket.create_connection(("127.0.0.1", proxy.listen_port), timeout=5) as stalled: assert stalled.recv(1) == b"" + + +class _FakeCredentials: + def resolve_device(self, *, did: str = "", duid: str = "", model: str = "") -> dict[str, str] | None: + return {"duid": "DUID1"} if did == "1103829226788" else None + + +def _snapshot_proxy(tmp_path): + from roborock_local_server.bundled_backend.shared.device_snapshot_store import DeviceSnapshotStore + + store = DeviceSnapshotStore(tmp_path, flush_interval_s=0.0, map_interval_s=0.0) + proxy = MqttTlsProxy( + cert_file=tmp_path / "fullchain.pem", + key_file=tmp_path / "privkey.pem", + listen_host="127.0.0.1", + listen_port=8883, + backend_host="127.0.0.1", + backend_port=1883, + localkey="test-local-key", + logger=logging.getLogger("test.mqtt_tls_proxy"), + decoded_jsonl=tmp_path / "decoded.jsonl", + runtime_credentials=_FakeCredentials(), + snapshot_store=store, + ) + return proxy, store + + +def test_robot_status_push_updates_the_snapshot_and_remembers_the_connection(tmp_path) -> None: + proxy, store = _snapshot_proxy(tmp_path) + + proxy._observe_snapshot( + "7", "c2b", "rr/d/i/1103829226788/abc", 102, {"payload_utf8": '{"t":1,"dps":{"122":5}}'}, b"", + ) + + assert store.get("DUID1")["status"]["battery"] == 5 + assert proxy._conn_duids["7"] == {"DUID1"} + + +def test_get_status_answer_updates_the_snapshot(tmp_path) -> None: + proxy, store = _snapshot_proxy(tmp_path) + + proxy._observe_snapshot( + "7", + "c2b", + "rr/d/i/1103829226788/abc", + 102, + {"response_to": {"request_method": "get_status", "error": None, "result": [{"state": 8, "battery": 100}]}}, + b"", + ) + + assert store.get("DUID1")["status"]["state"] == 8 + + +def test_frames_that_are_not_from_the_robot_do_not_mark_it_seen(tmp_path) -> None: + proxy, store = _snapshot_proxy(tmp_path) + + # An app's request travels on rr/m/i; the broker's copy toward the robot is rr/d/o. + proxy._observe_snapshot( + "9", "c2b", "rr/m/i/user/abcd1234/DUID1", 102, {"payload_utf8": '{"dps":{"122":5}}'}, b"", + ) + proxy._observe_snapshot( + "9", "b2c", "rr/d/o/1103829226788/abc", 102, {"payload_utf8": '{"dps":{"122":5}}'}, b"", + ) + + assert store.get("DUID1") is None + assert "9" not in proxy._conn_duids + + +def test_a_request_with_a_nonce_is_remembered_for_its_map_reply(tmp_path) -> None: + proxy, store = _snapshot_proxy(tmp_path) + payload = '{"dps":{"101":"{\\"security\\":{\\"endpoint\\":\\"w5Mpv215\\",\\"nonce\\":\\"00ff\\"}}"}}' + + proxy._observe_snapshot("9", "c2b", "rr/m/i/user/abcd1234/DUID1", 101, {"payload_utf8": payload}, b"") + + assert store._nonces["DUID1"] == {"w5Mpv215": "00ff"}