From 2084031e8a14f14339e10c5bb45968865b113126 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Thu, 15 Oct 2026 16:09:49 -0700 Subject: [PATCH 01/12] feat(api_v2): add client file selection Signed-off-by: Set Paing --- apps/action2v/tests/test_application.py | 4 + apps/t2v/t2v/testing.py | 4 + apps/t2v/tests/test_cli.py | 4 + flashdreams/flashdreams/api_v2/README.md | 9 +- .../flashdreams/api_v2/client_window.py | 248 +++++++- flashdreams/flashdreams/api_v2/loop.py | 155 ++++- flashdreams/flashdreams/runtime_v2/README.md | 17 +- .../runtime_v2/mp4_client_window.py | 13 +- .../runtime_v2/native_window_client_window.py | 162 +++++- .../runtime_v2/null_client_window.py | 15 +- .../flashdreams/runtime_v2/selected_file.py | 183 ++++++ .../flashdreams/runtime_v2/serving/web/app.js | 323 ++++++++++- .../runtime_v2/serving/web/index.html | 109 ++++ .../runtime_v2/serving/webrtc_server.py | 372 +++++++++++- .../flashdreams/runtime_v2/session_runner.py | 6 + .../runtime_v2/user_input_event.py | 35 ++ .../runtime_v2/webrtc_client_window.py | 24 + .../test_v2/test_application_runner.py | 4 + flashdreams/test_v2/test_cli.py | 4 + flashdreams/test_v2/test_client_window.py | 4 + .../test_native_window_client_window.py | 235 ++++++++ flashdreams/test_v2/test_selected_files.py | 407 +++++++++++++ flashdreams/test_v2/test_session_runner.py | 98 +++- .../test_v2/test_webrtc_client_window.py | 541 +++++++++++++++++- flashdreams/tests/test_profiler.py | 4 + integrations_v2/imgui_ui_demo/README.md | 20 + .../imgui_ui_demo/imgui_ui_demo/__init__.py | 3 + .../imgui_ui_demo/file_picker_app.py | 248 ++++++++ .../imgui_ui_demo/tests/test_imgui_ui_demo.py | 200 +++++++ integrations_v2/imgui_ui_demo/pyproject.toml | 1 + .../red_screen/tests/test_red_screen.py | 4 + tests/test_step_agreement.py | 3 + 32 files changed, 3434 insertions(+), 25 deletions(-) create mode 100644 flashdreams/flashdreams/runtime_v2/selected_file.py create mode 100644 flashdreams/test_v2/test_selected_files.py create mode 100644 integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py diff --git a/apps/action2v/tests/test_application.py b/apps/action2v/tests/test_application.py index c767d5950..e1dbf23dc 100644 --- a/apps/action2v/tests/test_application.py +++ b/apps/action2v/tests/test_application.py @@ -22,6 +22,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.session_runner import run_session from flashdreams.runtime_v2.step_result import StepResult @@ -138,6 +139,9 @@ def __init__(self) -> None: self.opened = False self.cursor_requests: list[tuple[str, bool]] = [] + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc self.opened = True diff --git a/apps/t2v/t2v/testing.py b/apps/t2v/t2v/testing.py index 648639a68..a20337d9c 100644 --- a/apps/t2v/t2v/testing.py +++ b/apps/t2v/t2v/testing.py @@ -21,6 +21,7 @@ import torch from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.blit_model_output_to_screen_loop import ( BlitModelOutputToScreenLoop, ) @@ -410,6 +411,9 @@ class _DiscardingClientWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc diff --git a/apps/t2v/tests/test_cli.py b/apps/t2v/tests/test_cli.py index 8521ddeea..0cadccec3 100644 --- a/apps/t2v/tests/test_cli.py +++ b/apps/t2v/tests/test_cli.py @@ -27,6 +27,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2 import cli from flashdreams.runtime_v2.application_registry import ( APPLICATION_ENTRY_POINT_GROUP, @@ -160,6 +161,9 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc diff --git a/flashdreams/flashdreams/api_v2/README.md b/flashdreams/flashdreams/api_v2/README.md index 4bdad8a4b..cb46848bb 100644 --- a/flashdreams/flashdreams/api_v2/README.md +++ b/flashdreams/flashdreams/api_v2/README.md @@ -245,7 +245,14 @@ UI control that needs a fresh application session calls `request_new_session(session_desc)` with a fully resolved replacement description; the runtime cleans the current session and passes that description to `ApplicationRunner` unchanged. Remaining timeout and remaining model steps -still apply to the replacement. +still apply to the replacement. To let the user pick a client file, call +`request_file_selector(request_id, ...)` when they ask (for ImGui, when +`imgui.button(...)` returns `True`), with one stable `request_id` per control. +The result arrives on a later tick as one `SelectedFilesUserInputEvent` +carrying that `request_id`; read its `status` before its `files`. One picker +is open at a time, and a call made while one is open is answered `BUSY`. The +`request_file_selector` docstring is the full contract: timing, every status, +size and type limits, and what WebRTC shows. For SlangPy's smaller retained widget API, subclass `SlangPyUILoop` from `flashdreams.runtime_v2.slangpy_ui_loop`. The diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 25b55131c..a483363a4 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -3,13 +3,33 @@ """Client window abstract interface.""" +import logging import math -from abc import ABC -from dataclasses import dataclass +import threading +import time +from abc import ABC, abstractmethod +from collections.abc import Callable +from dataclasses import dataclass, replace +from typing import final from urllib.parse import urlsplit +from numpy import uint64 + +from flashdreams.runtime_v2.selected_file import ( + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS, + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + clamp_selected_file_max_bytes, + selected_files_policy_status, +) +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent + from .input_source import InputSource from .output_sink import OutputSink +from .user_input_event import UserInputEvent + +_LOGGER = logging.getLogger(__name__) @dataclass(frozen=True, slots=True) @@ -86,8 +106,43 @@ class IClientWindow(InputSource, OutputSink, ABC): A window does not describe the output shape. The session does, and the window is given that description in :meth:`OutputSink.open`. - One thread makes every call on a window, so an implementation needs no - locking except when its backend delivers input from another thread. + The runtime makes every call on a window from one thread. A window still + needs locking where its backend or file picker delivers input from another + thread, such as :meth:`_put_user_input_event`. + + File selection lifecycle. This class holds the rules; a window only shows + a picker and reports what it returned. + + - **Request.** The runtime calls :meth:`request_file_selector` on the UI + thread, once for each call an app made, except in a tick that also + replaces the session (that session gets no results). The first request + opens a picker + with a new internal ``selector_id``. A request made while a picker is + open is answered at once as ``BUSY``. + - **Showing.** :meth:`_request_file_selector` runs on the UI thread and + must not block. The window calls :meth:`_file_selector_opened` from any + thread once the picker is on screen, which ends its deadline. + - **Deadline.** A picker not shown within + :data:`FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS` completes as + ``UNAVAILABLE``. :meth:`_expire_file_selectors` checks it; a window with a + picker calls it each time its input is read, or that answer waits for the + next request. + - **Result.** The window calls :meth:`_complete_file_selector` from any + thread. This class checks the result against the request (non-``OK`` + results carry no files; ``OK`` needs files that pass ``accept``, the + clamped size limits, and ``multiple``) and queues exactly one event + through :meth:`_put_user_input_event`. A result for a ``selector_id`` + that is no longer active is dropped. + - **Withdrawal.** :meth:`_cancel_active_file_selectors` runs on the UI + thread after a timeout or a flush, so the window can close what it shows. + - **Session end.** When a session ends, for replacement or shutdown, the + runtime calls :meth:`flush_file_selectors`: the open picker is withdrawn + and its result is dropped. :meth:`OutputSink.open` drops events still + queued for the previous session. An in-session reset + (``ResetUserInputEvent``) does not flush; the picker stays open and its + result arrives after the reset. + - **Wrappers.** A window that wraps another forwards + :meth:`request_file_selector` and :meth:`flush_file_selectors` to it. Created by the runtime, never by an application. """ @@ -117,3 +172,188 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: new_window_size: Requested ``(width, height)`` in pixels. """ pass + + # Plain class defaults so subclasses need no constructor call; assigning + # one on an instance shadows it for that window only. + _active_file_selector: tuple[int, FileSelectionRequest] | None = None + _file_selector_deadline: float | None = None + _last_file_selector_id = 0 + _file_selector_clock: Callable[[], float] = staticmethod(time.monotonic) + + def request_file_selector(self, request: FileSelectionRequest) -> None: + """Show a file selector; its result arrives later as an input event. + + The result is one ``SelectedFilesUserInputEvent`` from + :meth:`InputSource.get_user_input_events` carrying + ``request.request_id``. A request made while a picker is open is + answered as ``BUSY``. See the class docstring for the lifecycle. + + Args: + request: Selector request queued by the UI loop. + """ + self._expire_file_selectors() + with _FILE_SELECTOR_LOCK: + if self._active_file_selector is not None: + self._put_user_input_event( + _selected_files_event(request, SelectedFilesStatus.BUSY) + ) + return + selector_id = self._last_file_selector_id + 1 + self._last_file_selector_id = selector_id + self._active_file_selector = (selector_id, request) + self._file_selector_deadline = ( + self._file_selector_clock() + + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS + ) + try: + self._request_file_selector(selector_id, request) + except Exception: + _LOGGER.exception("File selector %r failed to start.", request.request_id) + self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + + def flush_file_selectors(self) -> None: + """Forget the open picker without a result, for a session that ended. + + The window withdraws a picker still on screen where it can; whatever + that picker returns is dropped. + """ + with _FILE_SELECTOR_LOCK: + active = self._active_file_selector + self._active_file_selector = None + self._file_selector_deadline = None + if active is not None: + self._cancel_active_file_selectors() + + # Optional to implement + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Show a picker for ``request`` without blocking. + + Call :meth:`_file_selector_opened` once it is shown, then + :meth:`_complete_file_selector` with ``selector_id`` when the user + finishes, from any thread. The default has no picker and completes as + ``UNAVAILABLE``. + """ + del request + self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + + # Optional to implement + def _cancel_active_file_selectors(self) -> None: + """Withdraw the picker that was flushed or that timed out.""" + pass + + @abstractmethod + def _put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event`` for :meth:`InputSource.get_user_input_events`. + + Required even for a window without a picker, which answers every + request through it. Only this class calls it. It may run on any thread + and runs under the file-selector lock, so it must not call back into + file selection. + """ + + @final + def _file_selector_opened(self, selector_id: int) -> None: + """Stop the deadline because the picker for ``selector_id`` is shown.""" + with _FILE_SELECTOR_LOCK: + active = self._active_file_selector + if active is not None and active[0] == selector_id: + self._file_selector_deadline = None + + @final + def _complete_file_selector( + self, + selector_id: int, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), + ) -> None: + """Report the result for ``selector_id``; a stale id is dropped. + + A result that breaks the request's rules is reported by the status it + earns instead; see the class docstring. + """ + with _FILE_SELECTOR_LOCK: + active = self._active_file_selector + if active is None or active[0] != selector_id: + return + self._active_file_selector = None + self._file_selector_deadline = None + request = active[1] + status, files = _checked_result(request, status, files) + # Queued under the lock so flush_file_selectors, which runs before + # the next session's open(), cannot slip in before the put. + self._put_user_input_event(_selected_files_event(request, status, files)) + + @final + def _expire_file_selectors(self) -> None: + """Complete a picker not shown by its deadline as ``UNAVAILABLE``. + + Windows with a picker call this each time input is read. + """ + with _FILE_SELECTOR_LOCK: + active = self._active_file_selector + deadline = self._file_selector_deadline + if ( + active is None + or deadline is None + or self._file_selector_clock() < deadline + ): + return + _LOGGER.warning( + "File selector %r was not shown within %g seconds.", + active[1].request_id, + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS, + ) + self._complete_file_selector(active[0], SelectedFilesStatus.UNAVAILABLE) + self._cancel_active_file_selectors() + + +def _selected_files_event( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), +) -> SelectedFilesUserInputEvent: + """Return the event answering ``request``.""" + return SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=request.request_id, + status=status, + files=files, + ) + + +def _checked_result( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], +) -> tuple[SelectedFilesStatus, tuple[SelectedFile, ...]]: + """Return the status and files a window's result earns under ``request``.""" + if status is SelectedFilesStatus.BUSY: + status = SelectedFilesStatus.UNAVAILABLE + if status is not SelectedFilesStatus.OK: + return status, () + if not files or (len(files) > 1 and not request.multiple): + _LOGGER.warning("File selector %r returned no valid set.", request.request_id) + return SelectedFilesStatus.UNAVAILABLE, () + files = tuple(replace(chosen, name=_base_name(chosen.name)) for chosen in files) + rejected = selected_files_policy_status( + [(chosen.name, len(chosen.data)) for chosen in files], + accept=request.accept, + max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + ) + if rejected is not None: + return rejected, () + return status, files + + +def _base_name(name: str) -> str: + """Return ``name`` without any POSIX or Windows directory part.""" + base = name.replace("\\", "/").rsplit("/", 1)[-1] + return "upload" if base in ("", ".", "..") else base + + +# ponytail: one lock for every window keeps subclasses constructor-free; it only +# guards a few assignments and one event put. Upgrade path: a per-window lock +# set in ``__init__``. +_FILE_SELECTOR_LOCK = threading.Lock() diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 11078776b..c1545015e 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -9,9 +9,10 @@ import threading import time from abc import ABC, abstractmethod -from collections.abc import Callable -from dataclasses import dataclass +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field from enum import Enum +from pathlib import Path from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar, final import torch @@ -25,9 +26,14 @@ bind_inference_profiler, unbind_inference_profiler, ) +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + normalize_selected_file_accept, +) from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( CloseUserInputEvent, + SelectedFilesUserInputEvent, UserInputEvent, ) from flashdreams.runtime_v2.user_input_events import UserInputEvents @@ -101,6 +107,9 @@ class UILoopRequests: links: tuple[ClientWindowLink, ...] | None = None """Replacement clickable regions, or ``None`` to leave them unchanged.""" + file_selections: list[FileSelectionRequest] = field(default_factory=list) + """File-selector requests in call order. The window answers each one.""" + class ILoop(ABC, Generic[StateT]): """Shared state, messaging, and lifecycle for a session loop. @@ -256,8 +265,16 @@ def _finish_run( """ if step_completed: self.latest_result = result + self._pending_user_events.clear() + else: + # Input that reaches a run with no step is dropped, except a file + # result: it answers a request and arrives once. + self._pending_user_events = [ + event + for event in self._pending_user_events + if isinstance(event, SelectedFilesUserInputEvent) + ] self._step_index += 1 - self._pending_user_events.clear() @final def _shutdown(self) -> None: @@ -604,6 +621,137 @@ def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: raise ValueError("At most sixteen client-window links are supported.") self.get_or_create_ui_loop_requests().links = links + @final + def request_file_selector( + self, + request_id: str, + initial_path: str | None = None, + *, + accept: Sequence[str] = (), + max_file_bytes: int | None = None, + multiple: bool = False, + ) -> None: + """Ask the client to show a file selector. + + The call only queues the request and returns nothing. The window sees + it after this step, so the result is never in :attr:`user_events` + during the step that asked. It arrives on a later tick as one + :class:`~flashdreams.runtime_v2.user_input_event.SelectedFilesUserInputEvent` + carrying ``request_id``. The model loop's ``step`` receives the same + event. + + Every call gets exactly one event, unless the session ends first: a + session replaced or shut down before the answer gets none. One + selector is open at a time, so a call made while one is open, including + a second call in the same tick, is answered ``BUSY``. Use one stable + ``request_id`` per control and reuse it. A control that is already + waiting for its own result can get a ``BUSY`` for a repeat click + before that result. + + For the full lifecycle, including session reset versus replacement, + see :class:`~flashdreams.api_v2.client_window.IClientWindow`. + + Read ``status`` first. ``files`` is empty unless it is ``OK``, and one + rejected file fails the whole set: + + - ``OK``: ``files`` holds each chosen file as its base ``name`` and its + ``data`` bytes. There is no host path. Empty files are allowed. + - ``CANCELLED``: the user dismissed the selector. + - ``BUSY``: another selector was open, so this one was not shown. + - ``DISALLOWED_TYPE``: a file did not match ``accept``. Suffixes + compare case-insensitively, and this wins over ``TOO_LARGE``. + - ``TOO_LARGE``: a file exceeded ``max_file_bytes``, or the set + exceeded + :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_BATCH_BYTES` + or :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_COUNT`. + - ``UNAVAILABLE``: the window has no picker (MP4, null), the picker + was not shown within + :data:`~flashdreams.runtime_v2.selected_file.FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS`, + the viewer left, the chooser failed, or a file could not be read. + + ``accept`` checks filename suffixes only, never contents, so validate + and decode ``data`` as untrusted input. + + Once the picker is shown there is no deadline. On WebRTC, shown means + the browser's "The app requested a file" prompt; its Select… button + opens the browser's file chooser. The native window uses a Tk dialog, + which needs a Python with Tk and a display; without them the result is + ``UNAVAILABLE``. Tk cannot close its dialog from another thread, so one + left open when its session ends stays on screen, and whatever it + returns is dropped. + + Example: + + .. code-block:: python + + from flashdreams.runtime_v2.selected_file import SelectedFilesStatus + from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, + ) + + if imgui.button("Open image"): + self.request_file_selector("open-image", accept=(".png",)) + for event in self.user_events.get_events(): + if not isinstance(event, SelectedFilesUserInputEvent): + continue + if event.request_id != "open-image": + continue + if event.status is SelectedFilesStatus.OK: + image_bytes = event.files[0].data + else: + self.state.message = f"No image: {event.status.value}" + + Args: + request_id: Names the control that asked, so its result can be + told apart from another control's. Echoed on the result. + initial_path: Directory hint for the native selector; ``None`` + uses the host user's home directory. The browser ignores it. + accept: Filename suffixes such as ``.png``. Empty allows any type. + max_file_bytes: Maximum size of each file in bytes, or ``None`` for + the ceiling. Larger values are clamped to + :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_BYTES` + without notice. + multiple: Whether the selector may return more than one file. The + whole set is also capped by + :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_BATCH_BYTES`. + + Raises: + TypeError: ``request_id`` or ``initial_path`` is not a string, or + ``accept`` / ``max_file_bytes`` / ``multiple`` has the wrong + type. + ValueError: ``request_id`` or ``initial_path`` is empty, or + ``accept`` / ``max_file_bytes`` is invalid. + """ + if not isinstance(request_id, str): + raise TypeError("request_id must be a string.") + if not request_id: + raise ValueError("request_id must be a non-empty string.") + if initial_path is None: + resolved_initial_path = str(Path.home()) + else: + if not isinstance(initial_path, str): + raise TypeError("initial_path must be a string.") + if not initial_path: + raise ValueError("initial_path must be a non-empty string.") + resolved_initial_path = initial_path + suffixes = normalize_selected_file_accept(accept) + if max_file_bytes is not None: + if isinstance(max_file_bytes, bool) or not isinstance(max_file_bytes, int): + raise TypeError("max_file_bytes must be an integer.") + if max_file_bytes <= 0: + raise ValueError("max_file_bytes must be > 0.") + if not isinstance(multiple, bool): + raise TypeError("multiple must be a bool.") + self.get_or_create_ui_loop_requests().file_selections.append( + FileSelectionRequest( + request_id=request_id, + initial_path=resolved_initial_path, + accept=suffixes, + max_bytes=max_file_bytes, + multiple=multiple, + ) + ) + def get_or_create_ui_loop_requests(self) -> UILoopRequests: if self._ui_loop_requests is None: self._ui_loop_requests = UILoopRequests() @@ -699,6 +847,7 @@ def invoke_async(loop: ILoop[StateT], operation: Callable[[StateT], None]) -> No __all__ = [ + "FileSelectionRequest", "ILoop", "IModelLoop", "IUILoop", diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 6f2cd1398..3fcd0554a 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -370,16 +370,25 @@ delegates over the thing that does the work — `Mp4ClientWindow` over the output shape, because the session already did. They differ in what they can do rather than in how they are driven. A session -chooses its UI independently of the client window. Browser keyboard, mouse and -focus events arrive on the server's own thread, so it queues them and hands them -over in batches when the session asks. +chooses its UI independently of the client window. Browser keyboard, mouse, +focus, and file-selector events arrive on the server's own thread, so it queues +them and hands them over in batches when the session asks. + +File selection rules live in `IClientWindow`, and its class docstring is the +full lifecycle: requests, the 30 second deadline, result checks, withdrawal, +session end versus in-session reset, threads, and wrappers. A window only shows +a picker and reports what it returned. Every window implements the abstract +`_put_user_input_event`; a window with a picker also implements +`_request_file_selector` and calls `_expire_file_selectors()` each time its +input is read. A UI loop may call `request_new_session(session_desc)` during its step with a fully resolved replacement description. `run_session` stops and cleans the current session, leaves the interactive window open, and returns that description unchanged. `ApplicationRunner` creates the replacement from it. A WebRTC browser disconnect releases only its peer connection, so refreshing the -page does not stop the session, server, or application. +page does not stop the session, server, or application. Leftover and in-flight +file picks complete as `unavailable`; a replacement viewer does not inherit them. The UI thread owns WebRTC cadence. Each `write` synchronously materializes one owned video frame and admits it to a two-frame FIFO of unsent frames. The WebRTC diff --git a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py index 3e2c72ac6..40439ffda 100644 --- a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py @@ -6,6 +6,7 @@ from pathlib import Path from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.mp4_output_sink import Mp4OutputSink from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -13,7 +14,7 @@ class Mp4ClientWindow(IClientWindow): - """Write UI frames to an MP4 file and report no input. + """Write UI frames to an MP4 file and report no input except file results. The session must finish on its own because this window never sends a close event. Use ``BackpressureMode.BLOCK`` with @@ -27,6 +28,7 @@ def __init__(self, path: str | Path) -> None: """ self._path = Path(path) self._video_sink = Mp4OutputSink(path) + self._input_events: list[UserInputEvent] = [] @property def path(self) -> Path: @@ -34,8 +36,12 @@ def path(self) -> Path: return self._path def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return the file-selection results queued since the last call.""" + events, self._input_events = self._input_events, [] + return UserInputEvents(events) + + def _put_user_input_event(self, event: UserInputEvent) -> None: + self._input_events.append(event) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the MP4 presentation dimensions without replacing the session. @@ -52,6 +58,7 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: def open(self, session_desc: SessionDesc) -> None: """Prepare to write a session's output.""" + self._input_events = [] self._video_sink.open(session_desc) def write(self, result: StepResult) -> None: diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index e2a8c2af1..77a42ff8f 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -7,12 +7,14 @@ import logging import queue +import stat import threading import time import webbrowser from collections import deque -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import replace +from pathlib import Path from typing import TYPE_CHECKING, Any, cast import numpy as np @@ -22,6 +24,14 @@ from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow from flashdreams.api_v2.user_input_event import UserInputEvent +from flashdreams.runtime_v2.selected_file import ( + MAX_SELECTED_FILE_BYTES, + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + clamp_selected_file_max_bytes, + selected_files_policy_status, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -40,6 +50,107 @@ _LOGGER = logging.getLogger(__name__) + +def _ask_open_filename( + *, + initial_dir: str, + accept: tuple[str, ...] = (), + multiple: bool = False, + on_presented: Callable[[], None] | None = None, +) -> tuple[str, ...]: + """Return paths from the Tk file dialog, or an empty tuple when cancelled. + + Tk shows Explorer on Windows, the macOS panel, and its own dialog on Linux. + ``on_presented`` runs just before the dialog opens. It does not mean a + remote host made that dialog visible. + + Raises: + ImportError: This Python was built without Tk. + tkinter.TclError: Tk cannot open a dialog, such as without a display. + """ + # ponytail: this runs on a worker thread, which macOS may abort for Tk. + # Upgrade path: launch ``osascript`` "choose file" as a separate process. + import tkinter as tk + from tkinter import filedialog + + root = tk.Tk() + root.withdraw() + try: + root.attributes("-topmost", True) + except tk.TclError: + pass + # ponytail: calling the Tk dialog is the presentation signal. Tk does not + # report when the dialog is actually visible. Upgrade path: + # IFileDialogEvents on Windows or NSOpenPanel visibility on macOS. + if on_presented is not None: + on_presented() + try: + kwargs: dict[str, Any] = { + "initialdir": initial_dir, + "title": "Select a file", + } + if accept: + kwargs["filetypes"] = [ + ("Accepted", " ".join(f"*{suffix}" for suffix in accept)) + ] + if multiple: + selected = filedialog.askopenfilenames(**kwargs) + return tuple(str(path) for path in selected if path) + selected = filedialog.askopenfilename(**kwargs) + finally: + root.destroy() + return (str(selected),) if selected else () + + +def _selected_files_from_paths( + paths: Sequence[str], + *, + accept: tuple[str, ...] = (), + max_bytes: int | None = None, +) -> tuple[tuple[SelectedFile, ...], SelectedFilesStatus]: + """Load chosen files, or empty files and a failure status.""" + budget = MAX_SELECTED_FILE_BYTES if max_bytes is None else max_bytes + rejected = selected_files_policy_status( + [(Path(path).name, None) for path in paths], + accept=accept, + max_bytes=budget, + ) + if rejected is not None: + return (), rejected + sized: list[tuple[str, int]] = [] + for path in paths: + try: + info = Path(path).stat() + except OSError: + return (), SelectedFilesStatus.UNAVAILABLE + if not stat.S_ISREG(info.st_mode): + return (), SelectedFilesStatus.UNAVAILABLE + sized.append((path, info.st_size)) + rejected = selected_files_policy_status( + [(Path(path).name, size) for path, size in sized], + accept=accept, + max_bytes=budget, + ) + if rejected is not None: + return (), rejected + files: list[SelectedFile] = [] + for path, _size in sized: + try: + with open(path, "rb") as stream: + data = stream.read(budget + 1) + except OSError: + return (), SelectedFilesStatus.UNAVAILABLE + files.append(SelectedFile(name=Path(path).name, data=data)) + rejected = selected_files_policy_status( + [(chosen.name, len(chosen.data)) for chosen in files], + accept=accept, + max_bytes=budget, + ) + if rejected is not None: + return (), rejected + return tuple(files), SelectedFilesStatus.OK + + _PRINTABLE_KEY_NAMES = { "space": " ", "apostrophe": "'", @@ -187,6 +298,48 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: ) self._window_size = presenter.resize(*new_window_size) + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Open the Tk file dialog on a worker thread so the window keeps presenting.""" + # ponytail: Tk cannot close this dialog from another thread, so one + # left open after its session ended can sit beside a newer one; its + # result is dropped. Upgrade path: run the dialog in a child process. + threading.Thread( + target=self._complete_file_selection, + args=(selector_id, request), + name="flashdreams-file-picker", + daemon=True, + ).start() + + def _complete_file_selection( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Run the file dialog, then report it opened and its result.""" + files: tuple[SelectedFile, ...] = () + status = SelectedFilesStatus.UNAVAILABLE + try: + selected = _ask_open_filename( + initial_dir=request.initial_path or str(Path.home()), + accept=request.accept, + multiple=request.multiple, + on_presented=lambda: self._file_selector_opened(selector_id), + ) + except Exception: + _LOGGER.exception( + "Native file picker failed for request id %r.", request.request_id + ) + else: + if selected: + files, status = _selected_files_from_paths( + selected if request.multiple else selected[:1], + accept=request.accept, + max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + ) + else: + status = SelectedFilesStatus.CANCELLED + self._complete_file_selector(selector_id, status, files) + def open(self, session_desc: SessionDesc) -> None: """Open a session in the GLFW window on the runtime's UI thread. @@ -251,6 +404,7 @@ def open(self, session_desc: SessionDesc) -> None: def get_user_input_events(self) -> UserInputEvents: """Pump GLFW and return native input events not yet read.""" + self._expire_file_selectors() presenter = self._presenter if presenter is not None: pending_before_poll = tuple(self._pending_printable_keys) @@ -453,6 +607,12 @@ def _on_window_closed(self) -> None: self._close_event_enqueued = True self._put_input(CloseUserInputEvent(timestamp=uint64(0))) + def _put_user_input_event(self, event: UserInputEvent) -> None: + """Queue one event from a picker thread, bypassing the poll-time buffer.""" + started_ns = self._session_started_ns + elapsed_ns = 0 if started_ns is None else max(0, self._clock_ns() - started_ns) + self._input_events.put(replace(event, timestamp=uint64(elapsed_ns // 1_000))) + class _SlangPyNativeWindowPresenter: """Own the SlangPy window and forward frames to its presentation context.""" diff --git a/flashdreams/flashdreams/runtime_v2/null_client_window.py b/flashdreams/flashdreams/runtime_v2/null_client_window.py index 76ff107fa..377889abe 100644 --- a/flashdreams/flashdreams/runtime_v2/null_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/null_client_window.py @@ -4,21 +4,30 @@ """Client window that discards output and reports no input.""" from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_events import UserInputEvents class NullClientWindow(IClientWindow): - """Discard output and report no input.""" + """Discard output and report no input except file-selection results.""" + + def __init__(self) -> None: + self._input_events: list[UserInputEvent] = [] def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return the file-selection results queued since the last call.""" + events, self._input_events = self._input_events, [] + return UserInputEvents(events) + + def _put_user_input_event(self, event: UserInputEvent) -> None: + self._input_events.append(event) def open(self, session_desc: SessionDesc) -> None: """Accept a session without opening an output.""" del session_desc + self._input_events = [] def write(self, result: StepResult) -> None: """Discard one result.""" diff --git a/flashdreams/flashdreams/runtime_v2/selected_file.py b/flashdreams/flashdreams/runtime_v2/selected_file.py new file mode 100644 index 000000000..ce100e183 --- /dev/null +++ b/flashdreams/flashdreams/runtime_v2/selected_file.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Selected-file types, request policy, and size/type checks.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum + +MAX_SELECTED_FILE_BYTES = 32 * 1024 * 1024 +"""Hard ceiling for one selected file. Windows clamp requested budgets to this.""" + +# ponytail: 8× the per-file ceiling is the batch cap; raise it or +# make it per-request if an app needs a larger set in one pick. +MAX_SELECTED_FILE_BATCH_BYTES = 8 * MAX_SELECTED_FILE_BYTES +"""Hard ceiling for the total bytes of one selected-files request.""" + +MAX_SELECTED_FILE_COUNT = 256 +"""Hard ceiling for the number of files in one selected-files request.""" + +FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS = 30.0 +"""Seconds to wait for a client window to show a picker before ``UNAVAILABLE``. + +The deadline starts when a request is handed to the window. Once the window +reports the picker is shown, the user may take as long as they want. +""" + + +def clamp_selected_file_max_bytes(max_bytes: int | None) -> int: + """Return a file-size budget that cannot exceed ``MAX_SELECTED_FILE_BYTES``. + + Args: + max_bytes: Requested budget in bytes, or ``None`` for the ceiling. + + Returns: + The requested budget when it is below the ceiling, otherwise the ceiling. + + Raises: + TypeError: ``max_bytes`` is not ``None`` or an integer. + ValueError: ``max_bytes`` is not positive. + """ + if max_bytes is None: + return MAX_SELECTED_FILE_BYTES + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int): + raise TypeError("max_bytes must be an integer.") + if max_bytes <= 0: + raise ValueError("max_bytes must be > 0.") + return min(max_bytes, MAX_SELECTED_FILE_BYTES) + + +def normalize_selected_file_accept(accept: Sequence[str] = ()) -> tuple[str, ...]: + """Return accepted filename suffixes such as ``.png``. + + Args: + accept: Suffixes the client may choose. Empty means any type. + + Returns: + The suffixes as a tuple, unchanged except for the sequence type. + + Raises: + TypeError: ``accept`` is not a sequence of strings. + ValueError: A suffix is missing the leading ``.`` or contains a path. + """ + if isinstance(accept, str) or not isinstance(accept, Sequence): + raise TypeError("accept must be a sequence of suffixes.") + suffixes: list[str] = [] + for suffix in accept: + if not isinstance(suffix, str): + raise TypeError("accept suffixes must be strings.") + if ( + not suffix.startswith(".") + or suffix == "." + or "/" in suffix + or "\\" in suffix + ): + raise ValueError( + "accept suffixes must look like ``.png`` and cannot contain a path." + ) + suffixes.append(suffix) + return tuple(suffixes) + + +def selected_file_suffix_allowed(name: str, accept: tuple[str, ...]) -> bool: + """Return whether ``name`` matches ``accept``. + + Empty ``accept`` allows any name. Comparison is case-insensitive. + """ + if not accept: + return True + lower = name.lower() + return any(lower.endswith(suffix.lower()) for suffix in accept) + + +class SelectedFilesStatus(Enum): + """Outcome of one client file-selector request.""" + + OK = "ok" + """The client chose files that passed type and size checks.""" + + CANCELLED = "cancelled" + """The user dismissed the selector.""" + + TOO_LARGE = "too_large" + """A file exceeded its size budget, or the set exceeded the batch size or + file-count ceiling.""" + + DISALLOWED_TYPE = "disallowed_type" + """A chosen file did not match ``accept``. Checked before size.""" + + BUSY = "busy" + """Another selector was already open, so this request was not shown. The + open selector keeps going and answers its own request.""" + + UNAVAILABLE = "unavailable" + """No pick was made: the client has no picker or left, the picker was not + shown in time, or a chosen file was unreadable.""" + + +def selected_files_policy_status( + files: Sequence[tuple[str, int | None]], + *, + accept: tuple[str, ...] = (), + max_bytes: int, +) -> SelectedFilesStatus | None: + """Return why a chosen set is rejected, or ``None`` if every file is allowed. + + Type is checked before size and count, across the whole set. + + Args: + files: ``(name, size)`` pairs. ``size`` of ``None`` skips that file's + size check and omits it from the batch total. + accept: Filename suffixes such as ``.png``. Empty allows any type. + max_bytes: Maximum allowed size of each file in bytes. + + Returns: + ``DISALLOWED_TYPE``, ``TOO_LARGE``, or ``None`` when the set passes. + """ + for name, _size in files: + if not selected_file_suffix_allowed(name, accept): + return SelectedFilesStatus.DISALLOWED_TYPE + if len(files) > MAX_SELECTED_FILE_COUNT: + return SelectedFilesStatus.TOO_LARGE + total = 0 + for _name, size in files: + if size is None: + continue + if size > max_bytes: + return SelectedFilesStatus.TOO_LARGE + total += size + if total > MAX_SELECTED_FILE_BATCH_BYTES: + return SelectedFilesStatus.TOO_LARGE + return None + + +@dataclass(frozen=True, slots=True) +class SelectedFile: + """One file chosen by a client file selector.""" + + name: str + """Base name of the chosen file, without any host path.""" + + data: bytes + """File contents.""" + + +@dataclass(frozen=True, slots=True) +class FileSelectionRequest: + """One client file-selector request queued by a UI loop.""" + + request_id: str + """Stable id for which button this request is for. Echoed on the matching event.""" + + initial_path: str + """Directory hint for the selector.""" + + accept: tuple[str, ...] = () + """Filename suffixes the client may choose, empty for any type.""" + + max_bytes: int | None = None + """Requested size budget per file in bytes, or ``None`` for the ceiling.""" + + multiple: bool = False + """Whether the selector may return more than one file.""" diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 57a659621..4f6740425 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -7,6 +7,12 @@ const pointerControls = peer.createDataChannel("pointer-controls"); peer.addTransceiver("video", {direction: "recvonly"}); const video = document.getElementById("video"); const status = document.getElementById("status"); +const promptDialog = document.getElementById("prompt-dialog"); +const promptBackdrop = document.getElementById("prompt-backdrop"); +const promptDialogTitle = document.getElementById("prompt-dialog-title"); +const promptDialogDetails = document.getElementById("prompt-dialog-details"); +const promptDialogConfirm = document.getElementById("prompt-dialog-confirm"); +const promptDialogCancel = document.getElementById("prompt-dialog-cancel"); const pressedKeys = new Map(); const pressedButtons = new Set(); const gamepadSnapshots = new Map(); @@ -74,6 +80,313 @@ const applyCursorOptions = options => { } }; +let promptSettled = false; +let promptOnConfirm = null; +let promptOnCancel = null; + +const setPromptBackdropVisible = visible => { + if (promptBackdrop !== null) { + promptBackdrop.hidden = !visible; + } +}; + +const closePromptDialog = () => { + setPromptBackdropVisible(false); + if (promptDialog !== null && promptDialog.open) { + promptDialog.close(); + } +}; + +const fillPromptDetails = details => { + if (promptDialogDetails === null) { + return; + } + promptDialogDetails.replaceChildren(); + if (!Array.isArray(details) || details.length === 0) { + promptDialogDetails.hidden = true; + return; + } + for (const section of details) { + const row = document.createElement("div"); + row.className = "prompt-dialog-row"; + const label = document.createElement("div"); + label.className = "prompt-dialog-detail-label"; + label.textContent = section.label; + const value = document.createElement("div"); + value.className = "prompt-dialog-row-value"; + const values = Array.isArray(section.items) ? section.items : []; + for (const item of values) { + const chip = document.createElement("span"); + chip.className = "prompt-dialog-chip"; + chip.textContent = item; + value.append(chip); + } + row.append(label, value); + promptDialogDetails.append(row); + } + promptDialogDetails.hidden = false; +}; + +const showPromptDialog = ({ + title, + details = [], + confirmLabel = "OK", + cancelLabel = "Cancel", + onConfirm = null, + onCancel = null, +} = {}) => { + if ( + promptDialog === null + || promptDialogTitle === null + || promptDialogConfirm === null + || promptDialogCancel === null + ) { + return; + } + if (promptDialog.open) { + promptSettled = true; + promptOnConfirm = null; + promptOnCancel = null; + promptDialog.close(); + } + promptDialogTitle.textContent = title; + fillPromptDetails(details); + promptDialogConfirm.textContent = confirmLabel; + promptDialogCancel.textContent = cancelLabel; + promptOnConfirm = onConfirm; + promptOnCancel = onCancel; + promptSettled = false; + setPromptBackdropVisible(true); + promptDialog.show(); + promptDialogConfirm.focus(); +}; + +if (promptDialog !== null) { + promptDialogConfirm?.addEventListener("click", () => { + if (promptSettled) { + return; + } + promptSettled = true; + const onConfirm = promptOnConfirm; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + onConfirm?.(); + }); + promptDialogCancel?.addEventListener("click", () => { + if (promptSettled) { + return; + } + promptSettled = true; + const onCancel = promptOnCancel; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + onCancel?.(); + }); + promptDialog.addEventListener("close", () => { + setPromptBackdropVisible(false); + if (promptSettled) { + return; + } + promptSettled = true; + const onCancel = promptOnCancel; + promptOnConfirm = null; + promptOnCancel = null; + onCancel?.(); + }); +} + +window.addEventListener("keydown", event => { + if (event.key !== "Escape" || promptDialog === null || !promptDialog.open) { + return; + } + event.preventDefault(); + event.stopImmediatePropagation(); + promptDialogCancel?.click(); +}); + +if (promptBackdrop !== null) { + promptBackdrop.addEventListener("click", () => { + promptDialogCancel?.click(); + }); +} + +let pendingFileRequest = null; + +const hideFilePicker = () => { + pendingFileRequest = null; + if (promptDialog !== null && promptDialog.open) { + promptSettled = true; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + } +}; + +const completeFileSelection = (requestId, status) => { + send({type: "file_selector_result", id: requestId, status}); +}; + +const formatByteBudget = bytes => { + if (!Number.isFinite(bytes) || bytes <= 0) { + return "the allowed size"; + } + const mib = bytes / (1024 * 1024); + if (mib >= 1) { + return Number.isInteger(mib) ? `${mib} MiB` : `${mib.toFixed(1)} MiB`; + } + const kib = bytes / 1024; + if (kib >= 1) { + return Number.isInteger(kib) ? `${kib} KiB` : `${kib.toFixed(1)} KiB`; + } + return `${bytes} bytes`; +}; + +const fileNameMatchesAccept = (name, accept) => { + if (!Array.isArray(accept) || accept.length === 0) { + return true; + } + const lower = name.toLowerCase(); + return accept.some( + suffix => typeof suffix === "string" && lower.endsWith(suffix.toLowerCase()), + ); +}; + +const startFileInput = pending => { + const requestId = pending.id; + const input = document.createElement("input"); + input.type = "file"; + if (pending.accept.length > 0) { + input.accept = pending.accept.join(","); + } + input.multiple = pending.multiple === true; + let settled = false; + const finishFileSelection = status => { + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + completeFileSelection(requestId, status); + hideFilePicker(); + }; + input.addEventListener("change", () => { + if (pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + const files = pending.multiple + ? Array.from(input.files || []) + : Array.from(input.files || []).slice(0, 1); + if (files.length === 0) { + finishFileSelection("cancelled"); + return; + } + let rejected = null; + for (const file of files) { + if (!fileNameMatchesAccept(file.name || "", pending.accept)) { + rejected = "disallowed_type"; + break; + } + } + if (rejected === null) { + for (const file of files) { + if (file.size > pending.maxBytes) { + rejected = "too_large"; + break; + } + } + } + if (rejected !== null) { + finishFileSelection(rejected); + return; + } + const body = new FormData(); + for (const file of files) { + body.append("file", file, file.name); + } + fetch(`/api/files?id=${encodeURIComponent(requestId)}`, { + method: "POST", + body, + }) + .then(response => { + if (!response.ok) { + // A proxy may refuse the upload before the server can answer it. + console.debug("File upload was rejected.", response.status); + finishFileSelection("unavailable"); + return; + } + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + hideFilePicker(); + }) + .catch(error => { + console.debug("Unable to upload the selected file.", error); + finishFileSelection("unavailable"); + }); + }); + input.addEventListener("cancel", () => finishFileSelection("cancelled")); + input.click(); +}; + +const openFileSelector = options => { + const requestId = options?.id; + if (typeof requestId !== "string" || !requestId) { + return; + } + if (pendingFileRequest !== null) { + if (pendingFileRequest.id !== requestId) { + completeFileSelection(requestId, "unavailable"); + } + return; + } + const accept = Array.isArray(options.accept) ? options.accept : []; + const maxBytes = Number(options.max_bytes); + pendingFileRequest = { + id: requestId, + accept, + maxBytes: Number.isFinite(maxBytes) ? maxBytes : 0, + multiple: options.multiple === true, + }; + showPromptDialog({ + title: pendingFileRequest.multiple + ? "The app requested files" + : "The app requested a file", + details: [ + { + label: "Accepted types", + items: accept.length > 0 ? accept : ["Any"], + }, + { + label: "Max size per file", + items: [formatByteBudget(pendingFileRequest.maxBytes)], + }, + ], + confirmLabel: "Select…", + onConfirm: () => { + const pending = pendingFileRequest; + if (pending === null || pending.id !== requestId) { + return; + } + startFileInput(pending); + }, + onCancel: () => { + if (pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + completeFileSelection(requestId, "cancelled"); + hideFilePicker(); + }, + }); + if (promptDialog === null || !promptDialog.open) { + completeFileSelection(requestId, "unavailable"); + hideFilePicker(); + return; + } + send({type: "file_selector_ack", id: requestId}); +}; + window.addEventListener("focus", updateCursorVisibility); window.addEventListener("blur", updateCursorVisibility); @@ -91,6 +404,14 @@ controls.addEventListener("message", event => { openUrl(payload.url); return; } + if (payload?.type === "file_selector") { + openFileSelector(payload); + return; + } + if (payload?.type === "file_selector_cancel") { + hideFilePicker(); + return; + } if (payload?.type === "error") { console.warn(`WebRTC server: ${payload.message ?? "unknown error"}`); } @@ -124,7 +445,7 @@ peer.addEventListener("connectionstatechange", () => { window.addEventListener("keydown", event => { const keyId = event.code || event.key; - if (pressedKeys.has(keyId)) { + if (pressedKeys.has(keyId) || (promptDialog !== null && promptDialog.open)) { return; } const wasSent = sendInput({ diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/index.html b/flashdreams/flashdreams/runtime_v2/serving/web/index.html index 1b73e1dc8..eb0d4ad2c 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/index.html +++ b/flashdreams/flashdreams/runtime_v2/serving/web/index.html @@ -48,11 +48,120 @@ color: #ff8a8a; } + #prompt-backdrop { + position: fixed; + inset: 0; + z-index: 2; + background: rgb(0 0 0 / 28%); + } + + #prompt-backdrop[hidden] { + display: none; + } + + #prompt-dialog { + position: fixed; + inset: 0; + z-index: 3; + margin: auto; + height: fit-content; + color-scheme: light dark; + max-width: 26rem; + width: min(26rem, calc(100% - 2rem)); + padding: 1.25rem 1.35rem 1rem; + border: 1px solid color-mix(in srgb, CanvasText 18%, Canvas); + border-radius: 12px; + background: Canvas; + color: CanvasText; + font: inherit; + box-shadow: 0 18px 40px rgb(0 0 0 / 45%); + } + + #prompt-dialog-title { + margin: 0 0 0.9rem; + font-size: 1.1rem; + font-weight: 650; + } + + #prompt-dialog-details { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + align-items: baseline; + column-gap: 0.85rem; + row-gap: 0.4rem; + margin: 0 0 1.15rem; + } + + #prompt-dialog-details[hidden] { + display: none; + } + + .prompt-dialog-row { + display: contents; + } + + .prompt-dialog-detail-label { + margin: 0; + font-size: 0.85rem; + color: color-mix(in srgb, CanvasText 62%, Canvas); + text-align: end; + white-space: nowrap; + } + + .prompt-dialog-detail-label::after { + content: ":"; + } + + .prompt-dialog-row-value { + display: flex; + flex-wrap: wrap; + column-gap: 0.55rem; + row-gap: 0.15rem; + } + + .prompt-dialog-chip { + font-size: 0.95rem; + } + + .prompt-dialog-actions { + display: flex; + justify-content: flex-end; + gap: 0.55rem; + } + + .prompt-dialog-actions button { + font: inherit; + padding: 0.45rem 0.95rem; + border-radius: 0.45rem; + cursor: pointer; + } + + #prompt-dialog-cancel { + border: 1px solid color-mix(in srgb, CanvasText 28%, Canvas); + background: transparent; + color: CanvasText; + } + + #prompt-dialog-confirm { + border: 0; + background: CanvasText; + color: Canvas; + } +
Waiting for the server…
+ + +

+ +
+ + +
+
diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 8550aed5d..4a6dea337 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -8,6 +8,7 @@ import asyncio import json import math +import secrets import socket import threading import time @@ -21,6 +22,7 @@ import numpy as np import torch from aiohttp import web +from aiohttp.multipart import BodyPartReader, MultipartReader from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription from aiortc.mediastreams import MediaStreamError from av import VideoFrame @@ -29,6 +31,17 @@ from flashdreams.api_v2.client_window import ClientWindowLink from flashdreams.runtime_v2.cuda_utils import resolve_cuda_device +from flashdreams.runtime_v2.selected_file import ( + MAX_SELECTED_FILE_BATCH_BYTES, + MAX_SELECTED_FILE_COUNT, + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + clamp_selected_file_max_bytes, + normalize_selected_file_accept, + selected_file_suffix_allowed, + selected_files_policy_status, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -295,6 +308,23 @@ async def _next_queued_frame(self) -> _QueuedRGBFrame: await self._frame_available.wait() +@dataclass(frozen=True, slots=True) +class _PendingFileSelection: + """Accept suffixes and size budget for one armed file-selector request.""" + + selector_id: str + """Window selector this upload answers. Never sent to the browser.""" + + accept: tuple[str, ...] + """Filename suffixes the client may upload, empty for any type.""" + + max_bytes: int + """Maximum file bytes, already clamped to the window ceiling.""" + + multiple: bool + """Whether this request may include more than one file.""" + + class WebRTCServer: """Own the HTTP, signaling, input buffering, and media transport.""" @@ -350,6 +380,16 @@ def __init__( self._client_connected = False self._hide_cursor = False self._lock_cursor_to_window = False + self._pending_file_requests: dict[str, _PendingFileSelection] = {} + self._uploading_file_requests: dict[str, _PendingFileSelection] = {} + self._file_selector_callbacks: ( + tuple[ + Callable[[str], None], + Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], + ] + | None + ) = None + self._file_selector_lock = threading.Lock() self._thread = threading.Thread( target=self._run_server, name="flashdreams-webrtc", @@ -495,6 +535,96 @@ def _send_cursor_options(self) -> None: self._sent_cursor_options = cursor_options + def request_file_selector( + self, selector_id: str, request: FileSelectionRequest + ) -> None: + """Ask the connected browser to open a file selector. + + The browser cannot use a host path, so ``initial_path`` is ignored. + Without a connected viewer the request completes as ``UNAVAILABLE``. + + Args: + selector_id: Window selector reported back with this result. + request: File selector to show. + """ + policy = _PendingFileSelection( + selector_id=selector_id, + accept=normalize_selected_file_accept(request.accept), + max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + multiple=request.multiple, + ) + loop = self._loop + if loop is None: + raise RuntimeError("WebRTC server is not running.") + loop.call_soon_threadsafe(self._arm_file_selector, policy) + + def cancel_file_selectors(self) -> None: + """Forget every armed selector and close the viewer's prompt. + + A later upload for a forgotten selector is refused. + """ + loop = self._loop + if loop is None: + with self._file_selector_lock: + self._pending_file_requests.clear() + return + # On the event loop, after any arm still queued for a forgotten selector. + loop.call_soon_threadsafe(self._withdraw_file_selectors) + + def _withdraw_file_selectors(self) -> None: + """Forget armed selectors and tell the viewer to close its prompt.""" + with self._file_selector_lock: + self._pending_file_requests.clear() + channel = self._control_channel + if channel is not None and channel.readyState == "open": + channel.send(json.dumps({"type": "file_selector_cancel"})) + + def _arm_file_selector(self, policy: _PendingFileSelection) -> None: + """Send the selector to the browser and remember its upload policy. + + The browser gets an unguessable token instead of the selector id, so a + page that cannot read the data channel cannot answer the selector. + """ + channel = self._control_channel + if channel is None or channel.readyState != "open": + self._emit_selected_files( + policy.selector_id, status=SelectedFilesStatus.UNAVAILABLE + ) + return + token = secrets.token_urlsafe(16) + with self._file_selector_lock: + self._pending_file_requests[token] = policy + channel.send( + json.dumps( + { + "type": "file_selector", + "id": token, + "accept": list(policy.accept), + "max_bytes": policy.max_bytes, + "multiple": policy.multiple, + } + ) + ) + + def _take_pending_file_request(self, token: str) -> _PendingFileSelection | None: + """Return and consume the policy for ``token``, if it was pending.""" + with self._file_selector_lock: + return self._pending_file_requests.pop(token, None) + + def _complete_abandoned_file_selectors(self) -> None: + """Answer every armed pick as ``UNAVAILABLE`` because the viewer left.""" + with self._file_selector_lock: + abandoned = ( + *self._pending_file_requests.values(), + *self._uploading_file_requests.values(), + ) + self._pending_file_requests.clear() + self._uploading_file_requests.clear() + for policy in abandoned: + self._emit_selected_files( + policy.selector_id, status=SelectedFilesStatus.UNAVAILABLE + ) + def event_timestamp_us(self) -> np.uint64: """Return the current timestamp relative to this server's first session. @@ -558,6 +688,8 @@ def open(self, session_desc: SessionDesc) -> None: # Keep timestamps comparable across sessions so events buffered # during a handoff retain their real order. self._event_origin_ns = time.monotonic_ns() + with self._file_selector_lock: + self._pending_file_requests.clear() def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the dimensions used to present subsequent UI frames. @@ -614,6 +746,22 @@ def register_input_callback( raise RuntimeError("An input callback is already registered.") self._input_callback = callback + def register_file_selector_callbacks( + self, + *, + opened: Callable[[str], None], + completed: Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], + ) -> None: + """Register how armed file selectors report back, by selector id. + + Both run on the server thread. + + Args: + opened: Called when the browser shows the selector. + completed: Called once with the selector's status and files. + """ + self._file_selector_callbacks = (opened, completed) + def write(self, result: StepResult) -> None: """Materialize and admit one generated result to the sender mailbox. @@ -781,11 +929,14 @@ def _run_server(self) -> None: async def _start_server(self) -> None: """Create and bind the standalone aiohttp application.""" self._offer_lock = asyncio.Lock() - app = web.Application() + app = web.Application( + client_max_size=MAX_SELECTED_FILE_BATCH_BYTES + 1024 * 1024 + ) app.router.add_get("/", self._serve_browser) app.router.add_get("/app.js", self._serve_browser_script) app.router.add_get("/healthz", self._health) app.router.add_post("/api/webrtc/offer", self._offer) + app.router.add_post("/api/files", self._upload_selected_file) runner = web.AppRunner(app) await runner.setup() address_family = socket.AF_INET6 if ":" in self._host else socket.AF_INET @@ -823,6 +974,120 @@ async def _health(self, _: web.Request) -> web.Response: } ) + async def _upload_selected_file(self, request: web.Request) -> web.Response: + """Turn a browser file upload into a selected-files input event.""" + if self._closed: + raise web.HTTPServiceUnavailable(reason="WebRTC server is closed.") + if self._timestamp_us() is None: + raise web.HTTPConflict(reason="WebRTC server is not open.") + token = request.query.get("id") + if not isinstance(token, str) or not token: + raise web.HTTPBadRequest(reason="File upload requires an id.") + policy = self._take_pending_file_request(token) + if policy is None: + raise web.HTTPConflict(reason="No pending file selection for this id.") + # Tracked until it ends so a viewer that leaves mid-upload is answered. + with self._file_selector_lock: + self._uploading_file_requests[token] = policy + + emitted = False + + def emit( + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), + ) -> None: + nonlocal emitted + self._emit_selected_files(policy.selector_id, status=status, files=files) + emitted = True + + try: + if not request.content_type.startswith("multipart/"): + emit(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File upload requires a file.") + try: + reader = await request.multipart() + except web.HTTPRequestEntityTooLarge: + emit(SelectedFilesStatus.TOO_LARGE) + raise + chosen: list[tuple[str, bytes]] = [] + file_count = 0 + total_bytes = 0 + too_large = False + while True: + part = await reader.next() + if part is None: + break + if not isinstance(part, BodyPartReader): + continue + if part.name != "file": + await part.release() + continue + if not policy.multiple and (chosen or too_large): + await part.release() + continue + if not _identity_encoded(part): + emit(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File parts must not be encoded.") + file_count += 1 + if file_count > MAX_SELECTED_FILE_COUNT: + too_large = True + chosen.clear() + name = part.filename or "upload" + if not selected_file_suffix_allowed(name, policy.accept): + emit(SelectedFilesStatus.DISALLOWED_TYPE) + await _drain_multipart_reader(reader, part) + return web.Response(status=204) + if too_large: + # Keep reading names so a disallowed type still wins. + await part.release() + continue + data = await _read_upload_part_bounded(part, max_bytes=policy.max_bytes) + if data is not None: + total_bytes += len(data) + if data is None or total_bytes > MAX_SELECTED_FILE_BATCH_BYTES: + too_large = True + chosen.clear() + continue + chosen.append((name, data)) + if too_large: + emit(SelectedFilesStatus.TOO_LARGE) + raise web.HTTPRequestEntityTooLarge( + max_size=policy.max_bytes, actual_size=policy.max_bytes + 1 + ) + if not chosen: + emit(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File upload requires a file.") + rejected = selected_files_policy_status( + [(name, len(data)) for name, data in chosen], + accept=policy.accept, + max_bytes=policy.max_bytes, + ) + if rejected is SelectedFilesStatus.TOO_LARGE: + emit(rejected) + raise web.HTTPRequestEntityTooLarge( + max_size=policy.max_bytes, + actual_size=max(len(data) for _name, data in chosen), + ) + if rejected is not None: + emit(rejected) + return web.Response(status=204) + emit( + SelectedFilesStatus.OK, + files=tuple( + SelectedFile(name=name, data=data) for name, data in chosen + ), + ) + return web.Response(status=204) + except web.HTTPRequestEntityTooLarge: + if not emitted: + emit(SelectedFilesStatus.TOO_LARGE) + raise + finally: + with self._file_selector_lock: + self._uploading_file_requests.pop(token, None) + if not emitted: + emit(SelectedFilesStatus.UNAVAILABLE) + async def _offer(self, request: web.Request) -> web.Response: """Negotiate one browser peer connection.""" if self._closed: @@ -1172,6 +1437,36 @@ def _buffer_browser_message( elif event_type == "close": self._cancel_link_press() event = CloseUserInputEvent(timestamp=timestamp_us) + elif event_type == "file_selector_ack": + token = payload.get("id") + if not isinstance(token, str) or not token: + raise ValueError( + "File selector acknowledgement requires a non-empty id." + ) + with self._file_selector_lock: + policy = self._pending_file_requests.get(token) + if policy is None: + return + self._file_selector_callbacks_or_raise()[0](policy.selector_id) + return + elif event_type == "file_selector_result": + token = payload.get("id") + if not isinstance(token, str) or not token: + raise ValueError("File selector result requires a non-empty id.") + try: + status = SelectedFilesStatus(payload.get("status")) + except ValueError as error: + raise ValueError("File selector result status is invalid.") from error + if status is SelectedFilesStatus.OK: + raise ValueError( + "File selector result must not set status ok; send files " + "with POST /api/files." + ) + policy = self._take_pending_file_request(token) + if policy is None: + return + self._emit_selected_files(policy.selector_id, status=status) + return else: raise ValueError("Unsupported browser event type.") self._append_event(event) @@ -1183,6 +1478,27 @@ def _append_event(self, event: UserInputEvent) -> None: raise RuntimeError("WebRTC input callback is not registered.") callback(event) + def _emit_selected_files( + self, + selector_id: str, + *, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), + ) -> None: + """Report the result of one armed file selector to the window.""" + self._file_selector_callbacks_or_raise()[1](selector_id, status, files) + + def _file_selector_callbacks_or_raise( + self, + ) -> tuple[ + Callable[[str], None], + Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], + ]: + callbacks = self._file_selector_callbacks + if callbacks is None: + raise RuntimeError("WebRTC file selector callbacks are not registered.") + return callbacks + async def _release_peer_connection( self, peer_connection: RTCPeerConnection, @@ -1202,6 +1518,7 @@ async def _release_peer_connection( self._video_track = None self._sent_cursor_options = None self._cancel_link_press() + self._complete_abandoned_file_selectors() else: track = detached_video_track failures: list[BaseException] = [] @@ -1278,6 +1595,59 @@ async def _shutdown(self) -> None: raise primary +_UPLOAD_READ_CHUNK_BYTES = 64 * 1024 +"""Bytes read per multipart chunk while enforcing ``max_bytes``.""" + + +def _identity_encoded(part: BodyPartReader) -> bool: + """Return whether ``part`` carries its bytes as-is, as browsers send them.""" + transfer = part.headers.get("Content-Transfer-Encoding", "").lower() + content = part.headers.get("Content-Encoding", "").lower() + return transfer in ("", "binary", "7bit", "8bit") and content in ("", "identity") + + +async def _read_upload_part_bounded( + part: BodyPartReader, *, max_bytes: int +) -> bytes | None: + """Return the decoded part body, or ``None`` when it exceeds ``max_bytes``. + + Args: + part: Multipart file part whose headers have already been read. + max_bytes: Maximum allowed size of the decoded file in bytes. + + Returns: + The decoded bytes, or ``None`` when they exceed ``max_bytes``. + """ + chunks: list[bytes] = [] + total = 0 + while True: + chunk = await part.read_chunk(_UPLOAD_READ_CHUNK_BYTES) + if not chunk: + break + async for decoded in part.decode_iter(chunk): + piece = bytes(decoded) + total += len(piece) + if total > max_bytes: + await part.release() + return None + chunks.append(piece) + return b"".join(chunks) + + +async def _drain_multipart_reader( + reader: MultipartReader, part: BodyPartReader | None = None +) -> None: + """Consume leftover multipart parts so the request can finish.""" + if part is not None: + await part.release() + while True: + leftover = await reader.next() + if leftover is None: + break + if isinstance(leftover, BodyPartReader): + await leftover.release() + + def _same_stream_format(first: SessionDesc, second: SessionDesc) -> bool: """Return whether two sessions can share one negotiated video track.""" return ( diff --git a/flashdreams/flashdreams/runtime_v2/session_runner.py b/flashdreams/flashdreams/runtime_v2/session_runner.py index b306ac05e..b6a051646 100644 --- a/flashdreams/flashdreams/runtime_v2/session_runner.py +++ b/flashdreams/flashdreams/runtime_v2/session_runner.py @@ -399,6 +399,8 @@ def should_end_ui() -> bool: cleanup(sink.close) # Both loops have stopped, so nothing will close an input wait still open. cleanup(profiler.reset_counts) + if window is not None: + cleanup(window.flush_file_selectors) if next_session_desc is None and window is not None: cleanup(window.close) cleanup(session.close) @@ -455,6 +457,10 @@ def _apply_window_requests(window: IClientWindow, request: UILoopRequests) -> No window.request_new_window_size(request.new_window_size) if request.links is not None: window.request_links(request.links) + # A replaced session gets no results, so its requests are not shown. + if request.new_session is None: + for file_selection in request.file_selections: + window.request_file_selector(file_selection) def _open_chunk_trace(path_value: object) -> _ChunkTraceLog: diff --git a/flashdreams/flashdreams/runtime_v2/user_input_event.py b/flashdreams/flashdreams/runtime_v2/user_input_event.py index 6f9c4ca50..7c3e2bacf 100644 --- a/flashdreams/flashdreams/runtime_v2/user_input_event.py +++ b/flashdreams/flashdreams/runtime_v2/user_input_event.py @@ -8,6 +8,7 @@ from typing import Literal from flashdreams.api_v2.user_input_event import UserInputEvent +from flashdreams.runtime_v2.selected_file import SelectedFile, SelectedFilesStatus class KeyboardInputState(Enum): @@ -245,6 +246,40 @@ def get_type_name(cls) -> str: """Optional controller quaternion in client XR space.""" +@dataclass(frozen=True, slots=True, eq=False) +class SelectedFilesUserInputEvent(UserInputEvent): + """Result of one file-selector request. + + The answer to :meth:`IUILoop.request_file_selector`, read from + ``user_events`` like any other input. + ``OK`` is the only status with files; every other status has empty ``files``. + """ + + @classmethod + def get_type_name(cls) -> str: + """Return the event type name.""" + return "selected_files" + + request_id: str + """``request_id`` of the control that asked for this result.""" + + status: SelectedFilesStatus + """Why this request completed. ``OK`` is the only value with files.""" + + files: tuple[SelectedFile, ...] = () + """Chosen files. Empty unless ``status`` is ``OK``.""" + + def __post_init__(self) -> None: + """Reject an ``ok`` event without files, or files on a failed pick.""" + has_files = bool(self.files) + if self.status is SelectedFilesStatus.OK: + if not has_files: + raise ValueError("ok selected-files events must include files.") + return + if has_files: + raise ValueError("failed selected-files events must not include files.") + + @dataclass(frozen=True, slots=True, eq=False) class UnknownUserInputEvent(UserInputEvent): """User input event for an unknown input modality.""" diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 9eb4ba640..5331b3372 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -10,6 +10,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow +from flashdreams.runtime_v2.selected_file import FileSelectionRequest from flashdreams.runtime_v2.serving.webrtc_server import WebRTCServer from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -71,6 +72,12 @@ def handle_input(event: UserInputEvent) -> None: self._input_events.append(event) self.server.register_input_callback(handle_input) + self.server.register_file_selector_callbacks( + opened=lambda selector_id: self._file_selector_opened(int(selector_id)), + completed=lambda selector_id, status, files: self._complete_file_selector( + int(selector_id), status, files + ), + ) def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace links opened locally by the browser viewing this window.""" @@ -104,6 +111,22 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """ self.server.request_new_window_size(new_window_size) + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Ask the connected browser to open a file selector.""" + self.server.request_file_selector(str(selector_id), request) + + def _cancel_active_file_selectors(self) -> None: + """Refuse a later upload for the selector that was forgotten.""" + self.server.cancel_file_selectors() + + def _put_user_input_event(self, event: UserInputEvent) -> None: + """Queue a file-selection event on the server's clock, like browser input.""" + event = replace(event, timestamp=self.server.event_timestamp_us()) + with self._input_lock: + self._input_events.append(event) + def open(self, session_desc: SessionDesc) -> None: """Implement ``OutputSink.open`` by configuring WebRTC output. @@ -122,6 +145,7 @@ def get_user_input_events(self) -> UserInputEvents: Returns: Buffered browser events in timestamp order, each returned once. """ + self._expire_file_selectors() with self._input_lock: events = list(self._input_events) self._input_events.clear() diff --git a/flashdreams/test_v2/test_application_runner.py b/flashdreams/test_v2/test_application_runner.py index 606c3899d..89329e07a 100644 --- a/flashdreams/test_v2/test_application_runner.py +++ b/flashdreams/test_v2/test_application_runner.py @@ -16,6 +16,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop, IUILoop from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.core.distributed.parallel import ParallelContext from flashdreams.runtime_v2.application_runner import ApplicationRunner from flashdreams.runtime_v2.metrics_output_sink import MetricsOutputSink @@ -158,6 +159,9 @@ def get_user_input_events(self) -> UserInputEvents: ) return UserInputEvents([]) + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc self._calls.append("window.open") diff --git a/flashdreams/test_v2/test_cli.py b/flashdreams/test_v2/test_cli.py index a9ecd7d97..1baf6864c 100644 --- a/flashdreams/test_v2/test_cli.py +++ b/flashdreams/test_v2/test_cli.py @@ -18,6 +18,7 @@ from flashdreams.api_v2.application import IApplication from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2 import cli, client_window_factory from flashdreams.runtime_v2.application_runner import ApplicationRunner, Unbound from flashdreams.runtime_v2.client_window_factory import ClientWindowMode @@ -42,6 +43,9 @@ class _StubWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc diff --git a/flashdreams/test_v2/test_client_window.py b/flashdreams/test_v2/test_client_window.py index 720ab4acc..7e66a592f 100644 --- a/flashdreams/test_v2/test_client_window.py +++ b/flashdreams/test_v2/test_client_window.py @@ -11,6 +11,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.input_source import InputSource from flashdreams.api_v2.output_sink import OutputSink +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -36,6 +37,9 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return self._input + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc self._is_open = True diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index a36f00c23..ed6e376e5 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -8,8 +8,10 @@ import queue import sys import threading +import time import webbrowser from collections.abc import Callable, Sequence +from pathlib import Path from types import SimpleNamespace from typing import TYPE_CHECKING, Any, cast from unittest.mock import Mock, call @@ -22,6 +24,10 @@ from flashdreams.runtime_v2.native_window_client_window import ( NativeWindowClientWindow, ) +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFilesStatus, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -31,6 +37,7 @@ KeyboardInputState, KeyboardUserInputEvent, MouseUserInputEvent, + SelectedFilesUserInputEvent, ) from flashdreams.runtime_v2.video_encoder import result_to_rgb24_tensor from flashdreams.runtime_v2.video_tensor import VideoTensorLayout @@ -41,6 +48,32 @@ pytestmark = pytest.mark.ci_cpu +def _file_request( + request_id: str, + initial_path: str, + *, + accept: tuple[str, ...] = (), + max_bytes: int | None = None, + multiple: bool = False, +) -> FileSelectionRequest: + return FileSelectionRequest( + request_id=request_id, + initial_path=initial_path, + accept=accept, + max_bytes=max_bytes, + multiple=multiple, + ) + + +def _wait_for_input_events(window: NativeWindowClientWindow) -> list[object]: + for _ in range(100): + events = window.get_user_input_events().get_events() + if events: + return list(events) + time.sleep(0.01) + return [] + + @pytest.mark.parametrize("fullscreen", [False, True]) def test_fullscreen_creation_reports_size_on_first_poll(fullscreen: bool) -> None: presenter = _Presenter() @@ -966,6 +999,208 @@ def test_native_link_warns_when_browser_is_unavailable( window.close() +def test_native_window_reports_a_selected_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + chosen = tmp_path / "seed.png" + chosen.write_bytes(b"png-bytes") + monkeypatch.setattr( + native_window_module, + "_ask_open_filename", + lambda *, initial_dir, accept=(), multiple=False, on_presented=None: ( + str(chosen), + ), + ) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.OK + assert event.files[0].name == "seed.png" + assert event.files[0].data == b"png-bytes" + + +def test_native_window_reports_cancelled_file_selection( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr( + native_window_module, + "_ask_open_filename", + lambda *, initial_dir, accept=(), multiple=False, on_presented=None: (), + ) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.CANCELLED + assert event.files == () + + +def test_native_window_reports_unavailable_when_picker_raises( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + def fail_picker( + *, + initial_dir: str, + accept: tuple[str, ...] = (), + multiple: bool = False, + on_presented: object = None, + ) -> tuple[str, ...]: + del initial_dir, accept, multiple, on_presented + raise RuntimeError("Native file picker needs a desktop chooser.") + + monkeypatch.setattr(native_window_module, "_ask_open_filename", fail_picker) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert event.files == () + + +def test_selected_files_from_paths_skips_unreadable_and_oversize( + tmp_path: Path, +) -> None: + missing = tmp_path / "gone.png" + files, status = native_window_module._selected_files_from_paths((str(missing),)) + assert files == () + assert status is SelectedFilesStatus.UNAVAILABLE + + too_big = tmp_path / "big.bin" + too_big.write_bytes(b"abcd") + files, status = native_window_module._selected_files_from_paths( + (str(too_big),), max_bytes=3 + ) + assert files == () + assert status is SelectedFilesStatus.TOO_LARGE + + wrong_type = tmp_path / "notes.txt" + wrong_type.write_bytes(b"hi") + files, status = native_window_module._selected_files_from_paths( + (str(wrong_type),), accept=(".png",) + ) + assert files == () + assert status is SelectedFilesStatus.DISALLOWED_TYPE + + too_big_wrong_type = tmp_path / "huge.txt" + too_big_wrong_type.write_bytes(b"abcd") + files, status = native_window_module._selected_files_from_paths( + (str(too_big_wrong_type),), accept=(".png",), max_bytes=1 + ) + assert files == () + assert status is SelectedFilesStatus.DISALLOWED_TYPE + + +@pytest.mark.skipif(not Path("/dev/zero").exists(), reason="needs /dev/zero") +def test_selected_files_from_paths_rejects_a_non_regular_file(tmp_path: Path) -> None: + endless = tmp_path / "endless.bin" + endless.symlink_to("/dev/zero") + files, status = native_window_module._selected_files_from_paths( + (str(endless),), accept=(".bin",) + ) + assert files == () + assert status is SelectedFilesStatus.UNAVAILABLE + + +def test_native_window_shown_picker_has_no_deadline( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + shown = threading.Event() + release = threading.Event() + + def picker( + *, + initial_dir: str, + accept: tuple[str, ...] = (), + multiple: bool = False, + on_presented: Callable[[], None] | None = None, + ) -> tuple[str, ...]: + del initial_dir, accept, multiple + assert on_presented is not None + on_presented() + shown.set() + release.wait(5) + return () + + monkeypatch.setattr(native_window_module, "_ask_open_filename", picker) + now = [0.0] + presenter = _Presenter() + window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) + window._file_selector_clock = lambda: now[0] + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + assert shown.wait(5) + now[0] = 1_000.0 + assert window.get_user_input_events().get_events() == [] + release.set() + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.status is SelectedFilesStatus.CANCELLED + + +def test_native_window_reports_multiple_selected_files( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + first = tmp_path / "a.bin" + second = tmp_path / "b.bin" + first.write_bytes(b"one") + second.write_bytes(b"two") + monkeypatch.setattr( + native_window_module, + "_ask_open_filename", + lambda *, initial_dir, accept=(), multiple=False, on_presented=None: ( + str(first), + str(second), + ), + ) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path), multiple=True)) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.status is SelectedFilesStatus.OK + assert [chosen.name for chosen in event.files] == ["a.bin", "b.bin"] + assert [chosen.data for chosen in event.files] == [b"one", b"two"] + + def test_native_window_reports_standard_gamepad_events() -> None: presenter = _Presenter() clock_values = iter((1_000_000, 1_001_000, 1_002_000, 1_003_000)) diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py new file mode 100644 index 000000000..46c24186d --- /dev/null +++ b/flashdreams/test_v2/test_selected_files.py @@ -0,0 +1,407 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for client file selection through the v2 window contract.""" + +import queue +import threading +from pathlib import Path + +import pytest +from numpy import uint64 + +from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.loop import IUILoop +from flashdreams.runtime_v2.mp4_client_window import Mp4ClientWindow +from flashdreams.runtime_v2.null_client_window import NullClientWindow +from flashdreams.runtime_v2.presentation_manager import PresentationManager +from flashdreams.runtime_v2.selected_file import ( + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS, + MAX_SELECTED_FILE_BATCH_BYTES, + MAX_SELECTED_FILE_BYTES, + MAX_SELECTED_FILE_COUNT, + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + clamp_selected_file_max_bytes, + normalize_selected_file_accept, + selected_file_suffix_allowed, + selected_files_policy_status, +) +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents + +pytestmark = pytest.mark.ci_cpu + + +class _UILoop(IUILoop[None]): + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + del step_index, events + return [] + + def reset(self) -> None: + return + + +def _ui_loop() -> _UILoop: + loop = _UILoop() + loop.register_session_loop_objects( + state=None, + frequency=0, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(), + presentation_manager=PresentationManager(), + ) + return loop + + +def test_ui_loop_queues_file_selection_policy_without_clamping() -> None: + loop = _ui_loop() + loop.request_file_selector( + "open-1", + "/tmp", + accept=(".png", ".JPG"), + max_file_bytes=MAX_SELECTED_FILE_BYTES * 2, + multiple=True, + ) + + requests = loop.flush_ui_loop_requests() + assert requests is not None + [selection] = requests.file_selections + assert selection.request_id == "open-1" + assert selection.initial_path == "/tmp" + assert selection.accept == (".png", ".JPG") + assert selection.max_bytes == MAX_SELECTED_FILE_BYTES * 2 + assert selection.multiple is True + + +def test_ui_loop_uses_home_when_initial_path_is_omitted() -> None: + loop = _ui_loop() + loop.request_file_selector("open-1") + + requests = loop.flush_ui_loop_requests() + assert requests is not None + assert requests.file_selections[0].initial_path == str(Path.home()) + + +def test_ui_loop_forwards_every_file_selection_of_a_tick() -> None: + loop = _ui_loop() + loop.request_file_selector("open-1", "/tmp") + loop.request_file_selector("open-2", "/var") + + requests = loop.flush_ui_loop_requests() + assert requests is not None + assert [item.request_id for item in requests.file_selections] == [ + "open-1", + "open-2", + ] + + +def test_ui_loop_keeps_file_results_until_a_step_runs() -> None: + loop = _ui_loop() + picked = SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id="open-1", + status=SelectedFilesStatus.CANCELLED, + ) + + loop._begin_run(UserInputEvents([picked]), generation=0) + loop._finish_run(None, step_completed=False) + loop._begin_run(UserInputEvents([]), generation=0) + assert loop.user_events.get_events() == [picked] + + loop._finish_run([], step_completed=True) + loop._begin_run(UserInputEvents([]), generation=0) + assert loop.user_events.get_events() == [] + + +class _PickerWindow(NullClientWindow): + def __init__(self) -> None: + super().__init__() + self.now = 0.0 + self._file_selector_clock = lambda: self.now + self.started: list[tuple[int, str]] = [] + self.cancelled = 0 + + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + self.started.append((selector_id, request.request_id)) + + def _cancel_active_file_selectors(self) -> None: + self.cancelled += 1 + + +def _request(request_id: str) -> FileSelectionRequest: + return FileSelectionRequest(request_id=request_id, initial_path="/tmp") + + +def _statuses(window: IClientWindow) -> list[tuple[str, SelectedFilesStatus]]: + return [ + (event.request_id, event.status) + for event in window.get_user_input_events().get_events() + if isinstance(event, SelectedFilesUserInputEvent) + ] + + +def test_window_shows_one_picker_at_a_time() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window.request_file_selector(_request("open-1")) + window.request_file_selector(_request("open-2")) + assert [request_id for _, request_id in window.started] == ["open-1"] + assert _statuses(window) == [ + ("open-1", SelectedFilesStatus.BUSY), + ("open-2", SelectedFilesStatus.BUSY), + ] + + selector_id = window.started[0][0] + window._complete_file_selector(selector_id, SelectedFilesStatus.CANCELLED) + window._complete_file_selector(selector_id, SelectedFilesStatus.CANCELLED) + assert _statuses(window) == [("open-1", SelectedFilesStatus.CANCELLED)] + window.request_file_selector(_request("open-2")) + assert [request_id for _, request_id in window.started] == ["open-1", "open-2"] + + +def test_window_checks_a_result_against_its_request() -> None: + def complete( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], + ) -> list[SelectedFilesUserInputEvent]: + window = _PickerWindow() + window.request_file_selector(request) + window._complete_file_selector(window.started[0][0], status, files) + events = window.get_user_input_events().get_events() + selected = [e for e in events if isinstance(e, SelectedFilesUserInputEvent)] + assert selected == events + return selected + + png = FileSelectionRequest( + request_id="open-1", initial_path="/tmp", accept=(".png",), max_bytes=4 + ) + a = SelectedFile(name="a.png", data=b"1") + b = SelectedFile(name="b.png", data=b"2") + cases = [ + (png, SelectedFilesStatus.OK, (), SelectedFilesStatus.UNAVAILABLE), + (png, SelectedFilesStatus.OK, (a, b), SelectedFilesStatus.UNAVAILABLE), + ( + png, + SelectedFilesStatus.OK, + (SelectedFile(name="a.txt", data=b"1"),), + SelectedFilesStatus.DISALLOWED_TYPE, + ), + ( + png, + SelectedFilesStatus.OK, + (SelectedFile(name="a.png", data=b"12345"),), + SelectedFilesStatus.TOO_LARGE, + ), + (png, SelectedFilesStatus.CANCELLED, (a,), SelectedFilesStatus.CANCELLED), + (png, SelectedFilesStatus.BUSY, (), SelectedFilesStatus.UNAVAILABLE), + ] + for request, status, files, expected in cases: + [event] = complete(request, status, files) + assert (event.status, event.files) == (expected, ()) + + many = FileSelectionRequest(request_id="open-1", initial_path="/tmp", multiple=True) + [event] = complete(many, SelectedFilesStatus.OK, (a, b)) + assert (event.status, event.files) == (SelectedFilesStatus.OK, (a, b)) + + pathy = ( + SelectedFile(name="../outside.png", data=b"1"), + SelectedFile(name="C:\\dir\\in.png", data=b"2"), + SelectedFile(name="..", data=b"3"), + ) + [event] = complete(many, SelectedFilesStatus.OK, pathy) + assert [chosen.name for chosen in event.files] == [ + "outside.png", + "in.png", + "upload", + ] + + +def test_window_without_put_user_input_event_cannot_be_created() -> None: + class _Incomplete(IClientWindow): + def get_user_input_events(self) -> UserInputEvents: + return UserInputEvents([]) + + def open(self, session_desc: SessionDesc) -> None: + del session_desc + + def write(self, result: StepResult) -> None: + del result + + def close(self) -> None: + return + + with pytest.raises(TypeError, match="_put_user_input_event"): + _Incomplete() # type: ignore[abstract] + + +def test_window_times_out_unshown_picker_and_drops_its_late_result() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window.now = FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS + window._expire_file_selectors() + assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] + assert window.cancelled == 1 + + window._complete_file_selector(window.started[0][0], SelectedFilesStatus.CANCELLED) + assert _statuses(window) == [] + + +def test_null_window_open_drops_a_result_left_by_the_previous_session() -> None: + window = NullClientWindow() + window.request_file_selector(_request("open-1")) + window.open(SessionDesc()) + assert _statuses(window) == [] + + +def test_window_expires_a_stuck_picker_when_the_next_request_arrives() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window.now = FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS + window.request_file_selector(_request("open-2")) + assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] + assert [request_id for _, request_id in window.started] == ["open-1", "open-2"] + + +def test_window_shown_picker_waits_for_the_user_without_deadline() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window._file_selector_opened(window.started[0][0]) + window.now = 1_000.0 + window._expire_file_selectors() + assert _statuses(window) == [] + + +def test_flush_forgets_the_open_picker_without_a_result() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window.flush_file_selectors() + window._complete_file_selector(window.started[0][0], SelectedFilesStatus.CANCELLED) + assert _statuses(window) == [] + assert window.cancelled == 1 + + window.flush_file_selectors() + assert window.cancelled == 1 + window.request_file_selector(_request("open-1")) + assert len(window.started) == 2 + + +def test_window_answers_unavailable_when_its_picker_fails_to_start() -> None: + class _BrokenWindow(NullClientWindow): + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + raise RuntimeError("no picker") + + window = _BrokenWindow() + window.request_file_selector(_request("open-1")) + assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] + + +def test_selected_file_policy_helpers() -> None: + assert clamp_selected_file_max_bytes(None) == MAX_SELECTED_FILE_BYTES + assert clamp_selected_file_max_bytes(16) == 16 + assert clamp_selected_file_max_bytes(MAX_SELECTED_FILE_BYTES * 2) == ( + MAX_SELECTED_FILE_BYTES + ) + assert normalize_selected_file_accept([".png", ".jpg"]) == (".png", ".jpg") + assert selected_file_suffix_allowed("seed.PNG", (".png",)) + assert not selected_file_suffix_allowed("seed.txt", (".png",)) + assert selected_file_suffix_allowed("seed.txt", ()) + assert ( + selected_files_policy_status( + [("a.bin", 0)] * (MAX_SELECTED_FILE_COUNT + 1), max_bytes=3 + ) + is SelectedFilesStatus.TOO_LARGE + ) + assert ( + selected_files_policy_status( + [("a.bin", 2), ("b.txt", 2)], + accept=(".bin",), + max_bytes=3, + ) + is SelectedFilesStatus.DISALLOWED_TYPE + ) + assert ( + selected_files_policy_status( + [("a.bin", 2), ("b.bin", 99)], + accept=(".bin",), + max_bytes=3, + ) + is SelectedFilesStatus.TOO_LARGE + ) + assert ( + selected_files_policy_status( + [("a.bin", MAX_SELECTED_FILE_BATCH_BYTES), ("b.bin", 1)], + accept=(".bin",), + max_bytes=MAX_SELECTED_FILE_BYTES, + ) + is SelectedFilesStatus.TOO_LARGE + ) + assert ( + selected_files_policy_status( + [("a.bin", 2), ("b.bin", 2)], + accept=(".bin",), + max_bytes=3, + ) + is None + ) + + +def test_ui_loop_rejects_empty_request_id() -> None: + loop = _ui_loop() + with pytest.raises(ValueError, match="non-empty string"): + loop.request_file_selector("") + + +def test_ui_loop_rejects_invalid_accept() -> None: + loop = _ui_loop() + with pytest.raises(ValueError, match="accept suffixes"): + loop.request_file_selector("open-1", accept=("png",)) + + +def test_ui_loop_rejects_invalid_multiple() -> None: + loop = _ui_loop() + with pytest.raises(TypeError, match="multiple must be a bool"): + loop.request_file_selector("open-1", multiple=1) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + + +def test_selected_files_event_rejects_inconsistent_status() -> None: + with pytest.raises(ValueError, match="must include files"): + SelectedFilesUserInputEvent( + timestamp=uint64(1), + request_id="open-1", + status=SelectedFilesStatus.OK, + ) + with pytest.raises(ValueError, match="must not include files"): + SelectedFilesUserInputEvent( + timestamp=uint64(1), + request_id="open-1", + status=SelectedFilesStatus.CANCELLED, + files=(SelectedFile(name="seed.png", data=b"png"),), + ) + + +@pytest.mark.parametrize("kind", ["mp4", "null"]) +def test_window_without_a_selector_answers_unavailable( + kind: str, tmp_path: Path +) -> None: + window = ( + Mp4ClientWindow(tmp_path / "clip.mp4") if kind == "mp4" else NullClientWindow() + ) + window.request_file_selector( + FileSelectionRequest(request_id="open-1", initial_path="/tmp") + ) + + assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] diff --git a/flashdreams/test_v2/test_session_runner.py b/flashdreams/test_v2/test_session_runner.py index 359ef819b..990ca76b6 100644 --- a/flashdreams/test_v2/test_session_runner.py +++ b/flashdreams/test_v2/test_session_runner.py @@ -21,6 +21,7 @@ IModelLoop, IUILoop, ModelInferenceState, + UILoopRequests, invoke_async, ) from flashdreams.api_v2.session import ISession @@ -35,18 +36,23 @@ PresentationManager, _PresentationClock, ) +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFilesStatus, +) from flashdreams.runtime_v2.session_desc import ( BackpressureMode, PresentationMode, SessionDesc, ) -from flashdreams.runtime_v2.session_runner import run_session +from flashdreams.runtime_v2.session_runner import _apply_window_requests, run_session from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( CloseUserInputEvent, KeyboardInputState, KeyboardUserInputEvent, ResetUserInputEvent, + SelectedFilesUserInputEvent, ) from flashdreams.runtime_v2.user_input_events import UserInputEvents from flashdreams.runtime_v2.video_tensor import VideoTensorLayout @@ -663,6 +669,8 @@ def __init__( self.session_desc: SessionDesc | None = None self.results: list[StepResult] = [] self.cursor_requests: list[tuple[str, bool]] = [] + self.file_selection_requests: list[FileSelectionRequest] = [] + self._file_events: list[UserInputEvent] = [] def request_hide_cursor(self, hide_cursor: bool) -> None: """Record one cursor visibility request.""" @@ -674,12 +682,25 @@ def request_lock_cursor_to_window(self, lock_cursor_to_window: bool) -> None: self._log.record("window.request_lock_cursor_to_window") self.cursor_requests.append(("lock", lock_cursor_to_window)) + def request_file_selector(self, request: FileSelectionRequest) -> None: + """Record one file-selector request and answer with the default.""" + self._log.record("window.request_file_selector") + self.file_selection_requests.append(request) + super().request_file_selector(request) + + def _put_user_input_event(self, event: UserInputEvent) -> None: + with self._lock: + self._file_events.append(event) + def get_user_input_events(self) -> UserInputEvents: self._log.record("window.get_user_input_events") with self._lock: + file_events, self._file_events = self._file_events, [] if self._scripted: - return self._scripted.pop(0) - return UserInputEvents([]) + return UserInputEvents( + [*file_events, *self._scripted.pop(0).get_events()] + ) + return UserInputEvents(file_events) def open(self, session_desc: SessionDesc) -> None: self._log.record("window.open") @@ -1370,6 +1391,77 @@ def init(self) -> None: assert "window.close" not in log.calls +def test_run_session_forwards_file_selection_requests_to_the_window() -> None: + log = CallLog() + + class RequestingSession(FakeSession): + def init(self) -> None: + super().init() + + def open_selector(_: object) -> None: + self.ui_loop.request_file_selector( + "open-1", + "/tmp", + accept=(".png",), + max_file_bytes=1024, + ) + + invoke_async(self.ui_loop, open_selector) + + window = RecordingClientWindow(log) + run_session(RequestingSession(_session_desc(), log), window, steps=1) + + assert window.file_selection_requests == [ + FileSelectionRequest( + request_id="open-1", + initial_path="/tmp", + accept=(".png",), + max_bytes=1024, + ) + ] + assert "window.request_file_selector" in log.calls + + +def test_window_requests_skip_a_file_selector_for_a_replaced_session() -> None: + window = RecordingClientWindow(CallLog()) + _apply_window_requests( + window, + UILoopRequests( + new_session=_session_desc(), + file_selections=[ + FileSelectionRequest(request_id="open-1", initial_path="/tmp") + ], + ), + ) + assert window.file_selection_requests == [] + + +def test_run_session_delivers_an_immediate_file_selection_result() -> None: + log = CallLog() + picks: list[SelectedFilesUserInputEvent] = [] + + class PickingSession(FakeSession): + def run_ui(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + if self.ui_steps == 0: + self.ui_loop.request_file_selector("open-1") + self.ui_steps += 1 + picks.extend( + event + for event in self.ui_loop.user_events.get_events() + if isinstance(event, SelectedFilesUserInputEvent) + ) + return super().run_ui(step_index, events) + + session_desc = replace( + _session_desc(), presentation_mode=PresentationMode.CONTINUOUS + ) + run_session(PickingSession(session_desc, log), RecordingClientWindow(log), steps=4) + + assert [(pick.request_id, pick.status) for pick in picks] == [ + ("open-1", SelectedFilesStatus.UNAVAILABLE) + ] + + def test_timeout_wins_over_a_ui_requested_replacement() -> None: log = CallLog() resolved = _session_desc() diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 8300f4f97..0a04d8eab 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -21,7 +21,7 @@ pytest.importorskip("aiohttp") pytest.importorskip("aiortc") -from aiohttp import ClientSession +from aiohttp import ClientSession, FormData, MultipartWriter, web from aiortc import ( MediaStreamTrack, RTCDataChannel, @@ -33,6 +33,12 @@ from yarl import URL from flashdreams.api_v2.client_window import ClientWindowLink +from flashdreams.runtime_v2.selected_file import ( + MAX_SELECTED_FILE_BYTES, + FileSelectionRequest, + SelectedFilesStatus, + clamp_selected_file_max_bytes, +) from flashdreams.runtime_v2.serving import webrtc_server from flashdreams.runtime_v2.serving.webrtc_server import _VideoTrack from flashdreams.runtime_v2.session_desc import PresentationMode, SessionDesc @@ -45,6 +51,7 @@ KeyboardUserInputEvent, MouseUserInputEvent, QueryStringUserInputEvent, + SelectedFilesUserInputEvent, TouchUserInputEvent, XRControllerUserInputEvent, ) @@ -52,6 +59,102 @@ from flashdreams.runtime_v2.webrtc_client_window import WebRTCClientWindow +def _file_request( + request_id: str, + initial_path: str = "/tmp", + *, + accept: tuple[str, ...] = (), + max_bytes: int | None = None, + multiple: bool = False, +) -> FileSelectionRequest: + return FileSelectionRequest( + request_id=request_id, + initial_path=initial_path, + accept=accept, + max_bytes=max_bytes, + multiple=multiple, + ) + + +def _files_url(window: WebRTCClientWindow, token: str) -> str: + return f"{window.server.url}api/files?id={token}" + + +async def _wait_file_armed(window: WebRTCClientWindow) -> str: + """Return the token the browser was sent once a selector is armed.""" + for _ in range(100): + if window.server._pending_file_requests: + return next(iter(window.server._pending_file_requests)) + await asyncio.sleep(0.01) + raise AssertionError("file selection was not armed") + + +def _arm(window: WebRTCClientWindow, request: FileSelectionRequest) -> str: + """Arm a selector as if the browser had been sent ``request``; return its token.""" + selector_id = window._last_file_selector_id + 1 + window._last_file_selector_id = selector_id + window._active_file_selector = (selector_id, request) + wire_id = f"token-{selector_id}" + window.server._pending_file_requests[wire_id] = webrtc_server._PendingFileSelection( + selector_id=str(selector_id), + accept=request.accept, + max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + multiple=request.multiple, + ) + return wire_id + + +async def _post_selected_file( + window: WebRTCClientWindow, + request_id: str, + data: bytes, + filename: str, +) -> int: + form = FormData() + form.add_field( + "file", + data, + filename=filename, + content_type="application/octet-stream", + ) + async with ClientSession() as client: + async with client.post(_files_url(window, request_id), data=form) as response: + return response.status + + +@pytest.mark.asyncio +async def test_file_upload_accepts_multiple_files() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer, _, _ = await _connect_browser(window) + try: + window.request_file_selector(_file_request("open-1", multiple=True)) + wire_id = await _wait_file_armed(window) + form = FormData() + form.add_field( + "file", + b"one", + filename="a.bin", + content_type="application/octet-stream", + ) + form.add_field( + "file", + b"two", + filename="b.bin", + content_type="application/octet-stream", + ) + async with ClientSession() as client: + async with client.post(_files_url(window, wire_id), data=form) as response: + assert response.status == 204 + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.OK + assert [chosen.name for chosen in event.files] == ["a.bin", "b.bin"] + assert [chosen.data for chosen in event.files] == [b"one", b"two"] + finally: + await peer.close() + window.close() + + def _session_desc( presentation_mode: PresentationMode = PresentationMode.ON_DEMAND, ) -> SessionDesc: @@ -143,6 +246,439 @@ async def test_browser_query_string_reaches_the_input_stream() -> None: window.close() +@pytest.mark.asyncio +async def test_file_selector_without_a_viewer_is_unavailable() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + window.request_file_selector(_file_request("open-1")) + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert window.server._pending_file_requests == {} + finally: + window.close() + + +@pytest.mark.asyncio +async def test_peer_loss_completes_armed_file_selections_as_unavailable() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, _, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + await _wait_file_armed(window) + await peer.close() + peer = None + + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert window.server._pending_file_requests == {} + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_in_flight_upload_is_unavailable_after_peer_loss( + monkeypatch: pytest.MonkeyPatch, +) -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, _, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + original_multipart = web.Request.multipart + disconnected = False + + async def multipart_after_disconnect(request: web.Request) -> Any: + nonlocal disconnected + if not disconnected: + disconnected = True + peer_connection = window.server._peer_connection + if peer_connection is not None: + await window.server._release_peer_connection(peer_connection) + return await original_multipart(request) + + monkeypatch.setattr(web.Request, "multipart", multipart_after_disconnect) + assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 204 + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + await asyncio.sleep(0.05) + assert window.get_user_input_events().get_events() == [] + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_encoded_file_part_is_unavailable() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1")) + with MultipartWriter("form-data") as body: + part = body.append(b"abc=\n") + part.set_content_disposition("form-data", name="file", filename="a.bin") + part.headers["Content-Transfer-Encoding"] = "quoted-printable" + async with ClientSession() as client: + async with client.post(_files_url(window, wire_id), data=body) as response: + assert response.status == 400 + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.UNAVAILABLE + finally: + window.close() + + +@pytest.mark.asyncio +async def test_upload_without_the_armed_token_is_rejected() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + assert await _post_selected_file(window, "1", b"hello", "hello.txt") == 409 + _arm(window, _file_request("open-1")) + assert await _post_selected_file(window, "1", b"hello", "hello.txt") == 409 + assert window.get_user_input_events().get_events() == [] + assert len(window.server._pending_file_requests) == 1 + finally: + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_sends_clamped_policy_without_host_path() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + messages: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + messages.append(json.loads(message)) + + window.request_file_selector( + _file_request( + "open-1", + "/home/secret", + accept=(".png", ".jpg"), + max_bytes=MAX_SELECTED_FILE_BYTES * 2, + ) + ) + for _ in range(100): + if any(item.get("type") == "file_selector" for item in messages): + break + await asyncio.sleep(0.01) + payload = next(item for item in messages if item.get("type") == "file_selector") + token = payload.pop("id") + assert isinstance(token, str) and len(token) >= 16 + assert token in window.server._pending_file_requests + assert payload == { + "type": "file_selector", + "accept": [".png", ".jpg"], + "max_bytes": MAX_SELECTED_FILE_BYTES, + "multiple": False, + } + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_flushed_file_selector_closes_the_browser_prompt() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + messages: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + messages.append(json.loads(message)) + + window.request_file_selector(_file_request("open-1")) + await _wait_file_armed(window) + window.flush_file_selectors() + for _ in range(100): + if any(item.get("type") == "file_selector_cancel" for item in messages): + break + await asyncio.sleep(0.01) + assert {"type": "file_selector_cancel"} in messages + assert window.server._pending_file_requests == {} + finally: + if peer is not None: + await peer.close() + window.close() + + +async def _wait_selected_files_event( + window: WebRTCClientWindow, +) -> SelectedFilesUserInputEvent: + events: list[object] = [] + for _ in range(100): + events.extend(window.get_user_input_events().get_events()) + if events: + break + await asyncio.sleep(0.01) + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + return event + + +@pytest.mark.asyncio +async def test_file_upload_without_file_is_unavailable() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1")) + form = FormData() + form.add_field("not_file", "hello") + async with ClientSession() as client: + async with client.post(_files_url(window, wire_id), data=form) as response: + assert response.status == 400 + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_rejects_oversize() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", max_bytes=1)) + status = await _post_selected_file(window, wire_id, b"x" * 8192, "big.bin") + assert status == 413 + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.TOO_LARGE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_stops_once_the_batch_exceeds_its_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(webrtc_server, "MAX_SELECTED_FILE_BATCH_BYTES", 5) + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", multiple=True)) + form = FormData() + for name in ("a.bin", "b.bin", "c.bin"): + form.add_field( + "file", b"xyz", filename=name, content_type="application/octet-stream" + ) + async with ClientSession() as client: + async with client.post(_files_url(window, wire_id), data=form) as response: + assert response.status == 413 + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.TOO_LARGE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_over_the_batch_cap_still_reports_a_disallowed_type( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(webrtc_server, "MAX_SELECTED_FILE_BATCH_BYTES", 5) + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", accept=(".bin",), multiple=True)) + form = FormData() + for name in ("a.bin", "b.bin", "c.exe"): + form.add_field( + "file", b"xyz", filename=name, content_type="application/octet-stream" + ) + async with ClientSession() as client: + async with client.post(_files_url(window, wire_id), data=form) as response: + assert response.status == 204 + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.DISALLOWED_TYPE + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_prefers_disallowed_suffix_over_body_limit() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", accept=(".png",), max_bytes=1)) + assert ( + await _post_selected_file( + window, wire_id, b"x" * (2 * 1024 * 1024), "hello.txt" + ) + == 204 + ) + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.DISALLOWED_TYPE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +async def test_browser_ack_reports_the_file_selector_opened() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + channel.send(json.dumps({"type": "file_selector_ack", "id": "other"})) + channel.send(json.dumps({"type": "file_selector_ack", "id": wire_id})) + for _ in range(100): + if window._file_selector_deadline is None: + break + await asyncio.sleep(0.01) + assert window._file_selector_deadline is None + assert window.get_user_input_events().get_events() == [] + channel.send( + json.dumps( + { + "type": "file_selector_result", + "id": wire_id, + "status": "cancelled", + } + ) + ) + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.CANCELLED + assert event.files == () + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_result_reports_status() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + channel.send( + json.dumps( + { + "type": "file_selector_result", + "id": wire_id, + "status": "cancelled", + } + ) + ) + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.CANCELLED + assert event.files == () + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_result_cannot_claim_ok() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + errors: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + payload = json.loads(message) + if payload.get("type") == "error": + errors.append(payload) + + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + channel.send( + json.dumps( + { + "type": "file_selector_result", + "id": wire_id, + "status": "ok", + } + ) + ) + for _ in range(100): + if errors: + break + await asyncio.sleep(0.01) + assert errors + assert "POST /api/files" in str(errors[0].get("message")) + assert window.get_user_input_events().get_events() == [] + assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 204 + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.OK + assert event.files[0].data == b"hello" + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_browser_script_checks_type_and_size_before_upload() -> None: + window = WebRTCClientWindow() + try: + async with ClientSession() as client: + async with client.get(f"{window.server.url}app.js") as response: + script = await response.text() + assert "fileNameMatchesAccept" in script + assert "file.size" in script + assert '"too_large"' in script + assert '"disallowed_type"' in script + assert "fetch(`/api/files?id=${encodeURIComponent(requestId)}`" in script + assert "input.multiple" in script + assert "file_selector_ack" in script + assert "event.stopImmediatePropagation();" in script + assert "(promptDialog !== null && promptDialog.open)" in script + finally: + window.close() + + +@pytest.mark.asyncio +async def test_session_handoff_clears_pending_file_selection() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1")) + window.open(_session_desc()) + assert window.server._pending_file_requests == {} + assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 409 + finally: + window.close() + + @pytest.mark.asyncio async def test_window_buffers_browser_events_until_drained() -> None: window = WebRTCClientWindow() @@ -422,6 +958,9 @@ def _link_server() -> tuple[webrtc_server.WebRTCServer, Mock, Mock]: server._link_lock = threading.Lock() server._links = () server._pressed_link = None + server._file_selector_lock = threading.Lock() + server._pending_file_requests = {} + server._uploading_file_requests = {} channel = Mock(readyState="open") server._control_channel = channel server._loop = Mock(spec=asyncio.AbstractEventLoop) diff --git a/flashdreams/tests/test_profiler.py b/flashdreams/tests/test_profiler.py index ffad1ff1e..935494a13 100644 --- a/flashdreams/tests/test_profiler.py +++ b/flashdreams/tests/test_profiler.py @@ -35,6 +35,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.profiler import IProfiler +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.infra import profiler as event_timer_module from flashdreams.runtime_v2 import profiler as profiler_module from flashdreams.runtime_v2.event_buffer import EventBuffer @@ -350,6 +351,9 @@ def get_user_input_events(self) -> UserInputEvents: self.batches += 1 return UserInputEvents([_key_press()]) + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: pass diff --git a/integrations_v2/imgui_ui_demo/README.md b/integrations_v2/imgui_ui_demo/README.md index dcca7516a..3ca20bdb9 100644 --- a/integrations_v2/imgui_ui_demo/README.md +++ b/integrations_v2/imgui_ui_demo/README.md @@ -8,6 +8,13 @@ SPDX-License-Identifier: Apache-2.0 Small ImGui applications for the FlashDreams v2 loop runtime: - `imgui-ui-text-input` renders an editable text field over model output. +- `imgui-ui-file-picker` opens the client file selector and shows the chosen + name and size on one status line per button. Each Open button calls + `request_file_selector` with its own `request_id`, and the result event + carrying that id updates the button's line. One picker is open at a time; + a click on any button while it is open shows `BUSY` on that button's line. Dropping the WebRTC tab completes in-flight picks as unavailable. A picker + the host never shows within 30 seconds also completes as unavailable; a + picker that is shown stays open as long as the user wants. - `imgui-ui-query-string` sets background from a `?(r,g,b)` browser URL. - `imgui-ui-window-size` resizes native window or UI render target without resetting session. @@ -24,6 +31,19 @@ Run the text-input application: uv run --no-sync flashdreams-run-v2 imgui-ui-text-input --mode native-window ``` +Run the file-picker application. The native window uses the Tk file dialog +(on Linux, install Python's Tk package, such as `python3-tk`). WebRTC shows a +confirmation dialog in the viewer (the picker must open from a browser click). +The demo asks for `.bin` / `.raw` at the per-file ceiling +(`MAX_SELECTED_FILE_BYTES`). Controls are **Open file (A)**, **Open file (B)**, and **Open files** +(more than one file). Failed picks show in red with the +`SelectedFilesStatus` name and why. +```bash +uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode native-window +uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode webrtc \ + --host 127.0.0.1 --port 8080 +``` + Run the query-string application: ```bash uv run --no-sync flashdreams-run-v2 imgui-ui-query-string \ diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py index b3fcbf18f..fa3acf920 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py @@ -3,10 +3,13 @@ """Interactive ImGui UI applications for the v2 loop runtime.""" +from .file_picker_app import FilePickerApplication, FilePickerSession from .text_input_app import TextInputApplication, TextInputSession, create_app from .window_size_app import WindowSizeApplication, WindowSizeSession __all__ = [ + "FilePickerApplication", + "FilePickerSession", "TextInputApplication", "TextInputSession", "WindowSizeApplication", diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py new file mode 100644 index 000000000..c39767795 --- /dev/null +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ImGui file-picker application for the v2 loop runtime.""" + +import math +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +import torch +from torch import Tensor + +from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop +from flashdreams.runtime_v2.selected_file import ( + MAX_SELECTED_FILE_BYTES, + SelectedFilesStatus, +) +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout + +_BREATHE_PERIOD_S = 3.0 +"""Seconds for one black → gray → black cycle in the demo background.""" + +_BREATHE_BLACK = -1.0 +"""Model-frame value for black in ``[-1, 1]``.""" + +_BREATHE_GRAY = -0.25 +"""Brightest gray in the cycle. Stops short of white.""" + +_FILE_SELECTION_STATUS_TEXT = { + SelectedFilesStatus.CANCELLED: ( + "SelectedFilesStatus.CANCELLED: user dismissed the selector." + ), + SelectedFilesStatus.BUSY: ( + "SelectedFilesStatus.BUSY: another picker was already open." + ), + SelectedFilesStatus.TOO_LARGE: ( + "SelectedFilesStatus.TOO_LARGE: a file exceeded this request's " + f"max_file_bytes={MAX_SELECTED_FILE_BYTES}, or the set exceeded the " + "batch limits." + ), + SelectedFilesStatus.DISALLOWED_TYPE: ( + "SelectedFilesStatus.DISALLOWED_TYPE: suffix not in accept=('.bin', '.raw')." + ), + SelectedFilesStatus.UNAVAILABLE: ( + "SelectedFilesStatus.UNAVAILABLE: picker could not complete " + "(no or gone client, picker not shown in time, failed chooser, or " + "unreadable file)." + ), +} +_FILE_SELECTION_ERROR_COLOR = (1.0, 0.25, 0.25, 1.0) +"""RGBA used for cancelled and other failed selector results.""" + +_OPEN_FILE_A_REQUEST_ID = "open-file-a" +"""Selector slot for Open file (A).""" + +_OPEN_FILE_B_REQUEST_ID = "open-file-b" +"""Selector slot for Open file (B).""" + +_OPEN_FILE_A_LABEL = "Open file (A)" +"""ImGui label for selector slot A.""" + +_OPEN_FILE_B_LABEL = "Open file (B)" +"""ImGui label for selector slot B.""" + +_OPEN_FILES_REQUEST_ID = "open-files" +"""Selector slot for Open files. Asks for more than one file.""" + +_OPEN_FILES_LABEL = "Open files" +"""ImGui label for a multi-file selector slot.""" + +_FILE_SELECTOR_SLOTS = ( + (_OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID, False), + (_OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID, False), + (_OPEN_FILES_LABEL, _OPEN_FILES_REQUEST_ID, True), +) +"""``(label, request_id, multiple)`` for each Open button, in display order.""" + +_NO_FILE_SELECTED = "No file selected." +"""Status shown for a button before its first result.""" + + +class BreathingBackgroundModelLoop(IModelLoop[tuple[SessionDesc, torch.device | str]]): + """Generate a black-to-gray breathing background for the file-picker demo.""" + + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + """Return one background frame along a 3s black-gray cycle.""" + del events + desc, device = self.state + phase = (step_index / desc.frames_per_second_for_step) * ( + 2.0 * math.pi / _BREATHE_PERIOD_S + ) + mix = 0.5 - 0.5 * math.cos(phase) + level = _BREATHE_BLACK + (_BREATHE_GRAY - _BREATHE_BLACK) * mix + return [ + StepResult( + step_index=step_index, + output=torch.full( + (1, 3, desc.video_height, desc.video_width), + level, + dtype=torch.float32, + device=device, + ), + frame_count=1, + output_layout=desc.output_layout, + ) + ] + + def reset(self) -> None: + return + + +@dataclass(slots=True) +class FilePickerState: + """Last file-selector result for each Open button.""" + + statuses: dict[str, str] = field(default_factory=dict) + """Status text by ``request_id``. A missing id has no result yet.""" + + +def _status_text(event: SelectedFilesUserInputEvent) -> str: + """Return the overlay text for one selector result.""" + if event.status is not SelectedFilesStatus.OK: + return _FILE_SELECTION_STATUS_TEXT[event.status] + return ", ".join( + f"{chosen.name} ({len(chosen.data)} bytes)" for chosen in event.files + ) + + +class FilePickerImGuiUILoop(ImGuiUILoop[FilePickerState]): + """Request ``.bin`` / ``.raw`` picks and show each button's own result.""" + + def step_ui( + self, + imgui: Any, + step_index: int, + events: UserInputEvents, + ) -> Tensor | None: + """Draw the Open buttons and one status line per button.""" + del step_index + for event in events.get_events(): + if isinstance(event, SelectedFilesUserInputEvent): + self.state.statuses[event.request_id] = _status_text(event) + imgui.set_next_window_pos(imgui.ImVec2(16.0, 16.0), imgui.Cond_.once) + imgui.set_next_window_size(imgui.ImVec2(520.0, 300.0), imgui.Cond_.once) + imgui.begin("File picker") + try: + for label, request_id, multiple in _FILE_SELECTOR_SLOTS: + if imgui.button(label): + self.request_file_selector( + request_id, + accept=(".bin", ".raw"), + max_file_bytes=MAX_SELECTED_FILE_BYTES, + multiple=multiple, + ) + for label, request_id, _ in _FILE_SELECTOR_SLOTS: + status = self.state.statuses.get(request_id, _NO_FILE_SELECTED) + line = f"{label}: {status}" + if status not in _FILE_SELECTION_STATUS_TEXT.values(): + imgui.text(line) + continue + imgui.push_style_color( + imgui.Col_.text, + imgui.ImVec4(*_FILE_SELECTION_ERROR_COLOR), + ) + try: + imgui.text_wrapped(line) + finally: + imgui.pop_style_color() + finally: + imgui.end() + return self.presented_model_frame() + + def reset(self) -> None: + """Clear every button's result for a new generation.""" + self.state.statuses.clear() + super().reset() + + +class FilePickerSession(ISession): + """Run an ImGui file picker over a generated background.""" + + def __init__( + self, + session_desc: SessionDesc, + *, + device: torch.device | str = "cuda", + ) -> None: + """Configure one file-picker session. + + Args: + session_desc: Output dimensions and loop frequencies. + device: Device used for the background model frame. + """ + if session_desc.output_layout is not VideoTensorLayout.tchw: + raise ValueError( + "The file-picker demo requires tchw output, got " + f"{session_desc.output_layout.value}." + ) + self._session_desc = session_desc + self._device = device + + @property + def session_desc(self) -> SessionDesc: + """Return the resolved session description.""" + return self._session_desc + + def init(self) -> None: + """Register the file-picker UI and background model loops.""" + self.register_ui_loop( + FilePickerImGuiUILoop, + state=FilePickerState(), + width=self._session_desc.video_width, + height=self._session_desc.video_height, + ) + self.register_model_loop( + BreathingBackgroundModelLoop, + state=(self._session_desc, self._device), + ) + + +class FilePickerApplication(IApplication): + """Create ImGui UI file-picker sessions.""" + + def init(self, commandline_args: Sequence[str]) -> None: + """Reject application-specific arguments.""" + if commandline_args: + raise ValueError("The file-picker demo takes no application arguments.") + + def session_desc(self) -> SessionDesc: + """Return the demo's established dimensions and rates.""" + return SessionDesc(video_width=640, video_height=480) + + def create_session(self, session_desc: SessionDesc) -> ISession: + """Create one uninitialized file-picker session.""" + return FilePickerSession(session_desc) + + +def create_app() -> IApplication: + """Return a new file-picker application.""" + return FilePickerApplication() diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py index bc31faadf..ec0b2f5c1 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py @@ -9,10 +9,29 @@ from unittest.mock import Mock import pytest +from imgui_ui_demo.file_picker_app import ( + _FILE_SELECTION_ERROR_COLOR, + _FILE_SELECTION_STATUS_TEXT, + _OPEN_FILE_A_LABEL, + _OPEN_FILE_A_REQUEST_ID, + _OPEN_FILE_B_LABEL, + _OPEN_FILE_B_REQUEST_ID, + _OPEN_FILES_LABEL, + _OPEN_FILES_REQUEST_ID, + FilePickerImGuiUILoop, + FilePickerState, +) from imgui_ui_demo.text_input_app import TextInputImGuiUILoop, TextInputState +from numpy import uint64 from flashdreams.runtime_v2.presentation_manager import PresentationManager +from flashdreams.runtime_v2.selected_file import ( + MAX_SELECTED_FILE_BYTES, + SelectedFile, + SelectedFilesStatus, +) from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent from flashdreams.runtime_v2.user_input_events import UserInputEvents pytestmark = pytest.mark.ci_cpu @@ -47,3 +66,184 @@ def test_text_input_updates_ui_owned_state() -> None: assert state.text == "hello world" imgui.text.assert_called_with("Value: hello world") imgui.end.assert_called_once_with() + + +def _file_picker_loop() -> tuple[FilePickerState, FilePickerImGuiUILoop]: + state = FilePickerState() + loop = FilePickerImGuiUILoop(renderer=Mock()) + loop.register_session_loop_objects( + state=state, + frequency=60, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(), + presentation_manager=PresentationManager(), + ) + return state, loop + + +def _file_picker_imgui(*, pressed: str | tuple[str, ...] = ()) -> SimpleNamespace: + labels = (pressed,) if isinstance(pressed, str) else pressed + return SimpleNamespace( + ImVec2=lambda x, y: (x, y), + ImVec4=lambda x, y, z, w: (x, y, z, w), + Col_=SimpleNamespace(text="text"), + Cond_=SimpleNamespace(once="once"), + set_next_window_pos=Mock(), + set_next_window_size=Mock(), + begin=Mock(), + end=Mock(), + text=Mock(), + text_wrapped=Mock(), + push_style_color=Mock(), + pop_style_color=Mock(), + button=Mock(side_effect=lambda label: label in labels), + ) + + +@pytest.mark.parametrize( + ("label", "request_id"), + [ + (_OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID), + (_OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID), + ], +) +def test_open_file_button_requests_a_client_file(label: str, request_id: str) -> None: + _state, loop = _file_picker_loop() + imgui = _file_picker_imgui(pressed=label) + + loop.step_ui(imgui, 0, UserInputEvents([])) + requests = loop.flush_ui_loop_requests() + + assert requests is not None + [selection] = requests.file_selections + assert selection.request_id == request_id + assert selection.accept == (".bin", ".raw") + assert selection.max_bytes == MAX_SELECTED_FILE_BYTES + assert selection.multiple is False + + +def test_open_files_button_requests_multiple_client_files() -> None: + _state, loop = _file_picker_loop() + imgui = _file_picker_imgui(pressed=_OPEN_FILES_LABEL) + + loop.step_ui(imgui, 0, UserInputEvents([])) + requests = loop.flush_ui_loop_requests() + + assert requests is not None + [selection] = requests.file_selections + assert selection.request_id == _OPEN_FILES_REQUEST_ID + assert selection.accept == (".bin", ".raw") + assert selection.max_bytes == MAX_SELECTED_FILE_BYTES + assert selection.multiple is True + + +def test_selected_files_event_updates_file_picker_status() -> None: + state, loop = _file_picker_loop() + imgui = _file_picker_imgui() + + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILE_A_REQUEST_ID, + status=SelectedFilesStatus.OK, + files=(SelectedFile(name="seed.png", data=b"xx"),), + ) + ] + ) + loop.step_ui(imgui, 0, loop.user_events) + + assert state.statuses == {_OPEN_FILE_A_REQUEST_ID: "seed.png (2 bytes)"} + assert [call.args[0] for call in imgui.text.call_args_list] == [ + f"{_OPEN_FILE_A_LABEL}: seed.png (2 bytes)", + f"{_OPEN_FILE_B_LABEL}: No file selected.", + f"{_OPEN_FILES_LABEL}: No file selected.", + ] + imgui.text_wrapped.assert_not_called() + imgui.push_style_color.assert_not_called() + + +def test_each_button_shows_only_its_own_result() -> None: + state, loop = _file_picker_loop() + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILE_B_REQUEST_ID, + status=SelectedFilesStatus.OK, + files=(SelectedFile(name="b.bin", data=b"b"),), + ), + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILES_REQUEST_ID, + status=SelectedFilesStatus.UNAVAILABLE, + ), + ] + ) + loop.step_ui(_file_picker_imgui(), 0, loop.user_events) + + assert state.statuses == { + _OPEN_FILE_B_REQUEST_ID: "b.bin (1 bytes)", + _OPEN_FILES_REQUEST_ID: _FILE_SELECTION_STATUS_TEXT[ + SelectedFilesStatus.UNAVAILABLE + ], + } + + +def test_selected_files_event_lists_every_chosen_file() -> None: + state, loop = _file_picker_loop() + imgui = _file_picker_imgui() + + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILES_REQUEST_ID, + status=SelectedFilesStatus.OK, + files=( + SelectedFile(name="a.bin", data=b"aa"), + SelectedFile(name="b.raw", data=b"bbb"), + ), + ) + ] + ) + loop.step_ui(imgui, 0, loop.user_events) + + assert state.statuses == { + _OPEN_FILES_REQUEST_ID: "a.bin (2 bytes), b.raw (3 bytes)" + } + imgui.text.assert_called_with( + f"{_OPEN_FILES_LABEL}: a.bin (2 bytes), b.raw (3 bytes)" + ) + imgui.text_wrapped.assert_not_called() + imgui.push_style_color.assert_not_called() + + +def test_file_picker_shows_error_status_in_red() -> None: + assert set(_FILE_SELECTION_STATUS_TEXT) == { + status for status in SelectedFilesStatus if status is not SelectedFilesStatus.OK + } + state, loop = _file_picker_loop() + imgui = _file_picker_imgui() + cancelled = SelectedFilesStatus.CANCELLED + + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILE_A_REQUEST_ID, + status=cancelled, + ) + ] + ) + loop.step_ui(imgui, 0, loop.user_events) + + message = _FILE_SELECTION_STATUS_TEXT[cancelled] + assert state.statuses == {_OPEN_FILE_A_REQUEST_ID: message} + imgui.push_style_color.assert_called_once_with("text", _FILE_SELECTION_ERROR_COLOR) + imgui.text_wrapped.assert_called_once_with(f"{_OPEN_FILE_A_LABEL}: {message}") + imgui.pop_style_color.assert_called_once_with() + assert imgui.text.call_count == 2 diff --git a/integrations_v2/imgui_ui_demo/pyproject.toml b/integrations_v2/imgui_ui_demo/pyproject.toml index 980ec0071..af0552b7c 100644 --- a/integrations_v2/imgui_ui_demo/pyproject.toml +++ b/integrations_v2/imgui_ui_demo/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ ] [project.entry-points."flashdreams.applications_v2"] +"imgui-ui-file-picker" = "imgui_ui_demo.file_picker_app:create_app" "imgui-ui-query-string" = "imgui_ui_demo.query_string_app:create_app" "imgui-ui-text-input" = "imgui_ui_demo.text_input_app:create_app" "imgui-ui-window-size" = "imgui_ui_demo.window_size_app:create_app" diff --git a/integrations_v2/red_screen/red_screen/tests/test_red_screen.py b/integrations_v2/red_screen/red_screen/tests/test_red_screen.py index 4dafd5952..382546640 100644 --- a/integrations_v2/red_screen/red_screen/tests/test_red_screen.py +++ b/integrations_v2/red_screen/red_screen/tests/test_red_screen.py @@ -12,6 +12,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import ( BackpressureMode, PresentationMode, @@ -62,6 +63,9 @@ def get_user_input_events(self) -> UserInputEvents: return self._batches.pop(0) return UserInputEvents([]) + def _put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc self._is_open = True diff --git a/tests/test_step_agreement.py b/tests/test_step_agreement.py index 7cf1bf365..64ed228e1 100644 --- a/tests/test_step_agreement.py +++ b/tests/test_step_agreement.py @@ -164,6 +164,9 @@ def _main_thread(self): assert threading.get_ident() == self.session.main_thread assert self.session.ctx.is_main + def _put_user_input_event(self, event): + del event + def open(self, desc): self._main_thread() From d2626fa1b0ebaa14c706fe800eb67ca39030434c Mon Sep 17 00:00:00 2001 From: Set Paing Date: Thu, 15 Oct 2026 17:29:44 -0700 Subject: [PATCH 02/12] refactor(api_v2): apply file-selection results on the UI thread Remove the file-selector locks: IClientWindow and the WebRTC server's upload tokens are each touched by one thread. The native file dialog runs in a child process polled by the UI thread, so a dialog left open when its session ends is closed. Trim the file-selection docstrings. Signed-off-by: Set Paing --- flashdreams/flashdreams/api_v2/README.md | 3 +- .../flashdreams/api_v2/client_window.py | 138 +++++--------- flashdreams/flashdreams/api_v2/loop.py | 56 ++---- flashdreams/flashdreams/runtime_v2/README.md | 8 +- .../runtime_v2/mp4_client_window.py | 4 +- .../runtime_v2/native_window_client_window.py | 177 ++++++++++-------- .../runtime_v2/null_client_window.py | 4 +- .../runtime_v2/serving/webrtc_server.py | 40 ++-- .../runtime_v2/webrtc_client_window.py | 23 ++- .../test_native_window_client_window.py | 82 +++----- .../test_v2/test_webrtc_client_window.py | 29 ++- 11 files changed, 252 insertions(+), 312 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/README.md b/flashdreams/flashdreams/api_v2/README.md index cb46848bb..f50ff82d2 100644 --- a/flashdreams/flashdreams/api_v2/README.md +++ b/flashdreams/flashdreams/api_v2/README.md @@ -251,8 +251,7 @@ still apply to the replacement. To let the user pick a client file, call The result arrives on a later tick as one `SelectedFilesUserInputEvent` carrying that `request_id`; read its `status` before its `files`. One picker is open at a time, and a call made while one is open is answered `BUSY`. The -`request_file_selector` docstring is the full contract: timing, every status, -size and type limits, and what WebRTC shows. +`request_file_selector` docstring lists every status. For SlangPy's smaller retained widget API, subclass `SlangPyUILoop` from `flashdreams.runtime_v2.slangpy_ui_loop`. The diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index a483363a4..25decdb63 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -5,7 +5,6 @@ import logging import math -import threading import time from abc import ABC, abstractmethod from collections.abc import Callable @@ -106,43 +105,17 @@ class IClientWindow(InputSource, OutputSink, ABC): A window does not describe the output shape. The session does, and the window is given that description in :meth:`OutputSink.open`. - The runtime makes every call on a window from one thread. A window still - needs locking where its backend or file picker delivers input from another - thread, such as :meth:`_put_user_input_event`. - - File selection lifecycle. This class holds the rules; a window only shows - a picker and reports what it returned. - - - **Request.** The runtime calls :meth:`request_file_selector` on the UI - thread, once for each call an app made, except in a tick that also - replaces the session (that session gets no results). The first request - opens a picker - with a new internal ``selector_id``. A request made while a picker is - open is answered at once as ``BUSY``. - - **Showing.** :meth:`_request_file_selector` runs on the UI thread and - must not block. The window calls :meth:`_file_selector_opened` from any - thread once the picker is on screen, which ends its deadline. - - **Deadline.** A picker not shown within - :data:`FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS` completes as - ``UNAVAILABLE``. :meth:`_expire_file_selectors` checks it; a window with a - picker calls it each time its input is read, or that answer waits for the - next request. - - **Result.** The window calls :meth:`_complete_file_selector` from any - thread. This class checks the result against the request (non-``OK`` - results carry no files; ``OK`` needs files that pass ``accept``, the - clamped size limits, and ``multiple``) and queues exactly one event - through :meth:`_put_user_input_event`. A result for a ``selector_id`` - that is no longer active is dropped. - - **Withdrawal.** :meth:`_cancel_active_file_selectors` runs on the UI - thread after a timeout or a flush, so the window can close what it shows. - - **Session end.** When a session ends, for replacement or shutdown, the - runtime calls :meth:`flush_file_selectors`: the open picker is withdrawn - and its result is dropped. :meth:`OutputSink.open` drops events still - queued for the previous session. An in-session reset - (``ResetUserInputEvent``) does not flush; the picker stays open and its - result arrives after the reset. - - **Wrappers.** A window that wraps another forwards - :meth:`request_file_selector` and :meth:`flush_file_selectors` to it. + The runtime makes every call on a window from the UI thread. A window still + needs locking where its backend delivers input from another thread. + + File selection: this class owns the rules (one picker at a time, the + presentation deadline, result checks, stale results). A window with a + picker implements :meth:`_request_file_selector` and + :meth:`_cancel_active_file_selectors`, and calls + :meth:`_expire_file_selectors` when input is read. Its reports must reach + this class on the UI thread. :meth:`OutputSink.open` drops events queued + for the previous session. A window wrapping another forwards + :meth:`request_file_selector` and :meth:`flush_file_selectors`. Created by the runtime, never by an application. """ @@ -186,25 +159,23 @@ def request_file_selector(self, request: FileSelectionRequest) -> None: The result is one ``SelectedFilesUserInputEvent`` from :meth:`InputSource.get_user_input_events` carrying ``request.request_id``. A request made while a picker is open is - answered as ``BUSY``. See the class docstring for the lifecycle. + answered as ``BUSY``. Args: request: Selector request queued by the UI loop. """ self._expire_file_selectors() - with _FILE_SELECTOR_LOCK: - if self._active_file_selector is not None: - self._put_user_input_event( - _selected_files_event(request, SelectedFilesStatus.BUSY) - ) - return - selector_id = self._last_file_selector_id + 1 - self._last_file_selector_id = selector_id - self._active_file_selector = (selector_id, request) - self._file_selector_deadline = ( - self._file_selector_clock() - + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS + if self._active_file_selector is not None: + self._put_user_input_event( + _selected_files_event(request, SelectedFilesStatus.BUSY) ) + return + selector_id = self._last_file_selector_id + 1 + self._last_file_selector_id = selector_id + self._active_file_selector = (selector_id, request) + self._file_selector_deadline = ( + self._file_selector_clock() + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS + ) try: self._request_file_selector(selector_id, request) except Exception: @@ -217,10 +188,9 @@ def flush_file_selectors(self) -> None: The window withdraws a picker still on screen where it can; whatever that picker returns is dropped. """ - with _FILE_SELECTOR_LOCK: - active = self._active_file_selector - self._active_file_selector = None - self._file_selector_deadline = None + active = self._active_file_selector + self._active_file_selector = None + self._file_selector_deadline = None if active is not None: self._cancel_active_file_selectors() @@ -232,8 +202,8 @@ def _request_file_selector( Call :meth:`_file_selector_opened` once it is shown, then :meth:`_complete_file_selector` with ``selector_id`` when the user - finishes, from any thread. The default has no picker and completes as - ``UNAVAILABLE``. + finishes, both on the UI thread. The default has no picker and + completes as ``UNAVAILABLE``. """ del request self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) @@ -248,18 +218,16 @@ def _put_user_input_event(self, event: UserInputEvent) -> None: """Queue ``event`` for :meth:`InputSource.get_user_input_events`. Required even for a window without a picker, which answers every - request through it. Only this class calls it. It may run on any thread - and runs under the file-selector lock, so it must not call back into - file selection. + request through it. Only this class calls it, on the UI thread, and it + must not call back into file selection. """ @final def _file_selector_opened(self, selector_id: int) -> None: """Stop the deadline because the picker for ``selector_id`` is shown.""" - with _FILE_SELECTOR_LOCK: - active = self._active_file_selector - if active is not None and active[0] == selector_id: - self._file_selector_deadline = None + active = self._active_file_selector + if active is not None and active[0] == selector_id: + self._file_selector_deadline = None @final def _complete_file_selector( @@ -270,20 +238,17 @@ def _complete_file_selector( ) -> None: """Report the result for ``selector_id``; a stale id is dropped. - A result that breaks the request's rules is reported by the status it - earns instead; see the class docstring. + A result that breaks the request's rules (type, size, count) is + reported by the status it earns instead. """ - with _FILE_SELECTOR_LOCK: - active = self._active_file_selector - if active is None or active[0] != selector_id: - return - self._active_file_selector = None - self._file_selector_deadline = None - request = active[1] - status, files = _checked_result(request, status, files) - # Queued under the lock so flush_file_selectors, which runs before - # the next session's open(), cannot slip in before the put. - self._put_user_input_event(_selected_files_event(request, status, files)) + active = self._active_file_selector + if active is None or active[0] != selector_id: + return + self._active_file_selector = None + self._file_selector_deadline = None + request = active[1] + status, files = _checked_result(request, status, files) + self._put_user_input_event(_selected_files_event(request, status, files)) @final def _expire_file_selectors(self) -> None: @@ -291,15 +256,10 @@ def _expire_file_selectors(self) -> None: Windows with a picker call this each time input is read. """ - with _FILE_SELECTOR_LOCK: - active = self._active_file_selector - deadline = self._file_selector_deadline - if ( - active is None - or deadline is None - or self._file_selector_clock() < deadline - ): - return + active = self._active_file_selector + deadline = self._file_selector_deadline + if active is None or deadline is None or self._file_selector_clock() < deadline: + return _LOGGER.warning( "File selector %r was not shown within %g seconds.", active[1].request_id, @@ -351,9 +311,3 @@ def _base_name(name: str) -> str: """Return ``name`` without any POSIX or Windows directory part.""" base = name.replace("\\", "/").rsplit("/", 1)[-1] return "upload" if base in ("", ".", "..") else base - - -# ponytail: one lock for every window keeps subclasses constructor-free; it only -# guards a few assignments and one event put. Upgrade path: a per-window lock -# set in ``__init__``. -_FILE_SELECTOR_LOCK = threading.Lock() diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index c1545015e..8b55b5dfd 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -633,52 +633,24 @@ def request_file_selector( ) -> None: """Ask the client to show a file selector. - The call only queues the request and returns nothing. The window sees - it after this step, so the result is never in :attr:`user_events` - during the step that asked. It arrives on a later tick as one + The result arrives on a later tick, never during the step that asked, + as one :class:`~flashdreams.runtime_v2.user_input_event.SelectedFilesUserInputEvent` - carrying ``request_id``. The model loop's ``step`` receives the same - event. + carrying ``request_id``. One selector is open at a time; a call made + while one is open is answered ``BUSY``. A selector stays open across a + session reset; a replaced or ended session gets no result. - Every call gets exactly one event, unless the session ends first: a - session replaced or shut down before the answer gets none. One - selector is open at a time, so a call made while one is open, including - a second call in the same tick, is answered ``BUSY``. Use one stable - ``request_id`` per control and reuse it. A control that is already - waiting for its own result can get a ``BUSY`` for a repeat click - before that result. + Read ``status`` first; ``files`` is empty unless it is ``OK``: - For the full lifecycle, including session reset versus replacement, - see :class:`~flashdreams.api_v2.client_window.IClientWindow`. - - Read ``status`` first. ``files`` is empty unless it is ``OK``, and one - rejected file fails the whole set: - - - ``OK``: ``files`` holds each chosen file as its base ``name`` and its - ``data`` bytes. There is no host path. Empty files are allowed. + - ``OK``: each chosen file as its base ``name`` and ``data`` bytes. - ``CANCELLED``: the user dismissed the selector. - - ``BUSY``: another selector was open, so this one was not shown. - - ``DISALLOWED_TYPE``: a file did not match ``accept``. Suffixes - compare case-insensitively, and this wins over ``TOO_LARGE``. - - ``TOO_LARGE``: a file exceeded ``max_file_bytes``, or the set - exceeded - :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_BATCH_BYTES` - or :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_COUNT`. - - ``UNAVAILABLE``: the window has no picker (MP4, null), the picker - was not shown within - :data:`~flashdreams.runtime_v2.selected_file.FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS`, - the viewer left, the chooser failed, or a file could not be read. - - ``accept`` checks filename suffixes only, never contents, so validate - and decode ``data`` as untrusted input. - - Once the picker is shown there is no deadline. On WebRTC, shown means - the browser's "The app requested a file" prompt; its Select… button - opens the browser's file chooser. The native window uses a Tk dialog, - which needs a Python with Tk and a display; without them the result is - ``UNAVAILABLE``. Tk cannot close its dialog from another thread, so one - left open when its session ends stays on screen, and whatever it - returns is dropped. + - ``BUSY``: another selector was open. + - ``DISALLOWED_TYPE``: a file did not match ``accept``. + - ``TOO_LARGE``: a file or the set exceeded the size or count limits. + - ``UNAVAILABLE``: the window cannot show a selector, or it failed. + + ``accept`` checks filename suffixes only, so treat ``data`` as + untrusted input. Example: diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 3fcd0554a..4579d9947 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -374,11 +374,9 @@ chooses its UI independently of the client window. Browser keyboard, mouse, focus, and file-selector events arrive on the server's own thread, so it queues them and hands them over in batches when the session asks. -File selection rules live in `IClientWindow`, and its class docstring is the -full lifecycle: requests, the 30 second deadline, result checks, withdrawal, -session end versus in-session reset, threads, and wrappers. A window only shows -a picker and reports what it returned. Every window implements the abstract -`_put_user_input_event`; a window with a picker also implements +File selection rules live in `IClientWindow`; a window only shows a picker and +reports what it returned, on the UI thread. Every window implements the +abstract `_put_user_input_event`; a window with a picker also implements `_request_file_selector` and calls `_expire_file_selectors()` each time its input is read. diff --git a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py index 40439ffda..ac7c2ab7c 100644 --- a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py @@ -14,7 +14,7 @@ class Mp4ClientWindow(IClientWindow): - """Write UI frames to an MP4 file and report no input except file results. + """Write UI frames to an MP4 file. It has no user input of its own. The session must finish on its own because this window never sends a close event. Use ``BackpressureMode.BLOCK`` with @@ -36,7 +36,7 @@ def path(self) -> Path: return self._path def get_user_input_events(self) -> UserInputEvents: - """Return the file-selection results queued since the last call.""" + """Return the events queued since the last call.""" events, self._input_events = self._input_events, [] return UserInputEvents(events) diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index 77a42ff8f..df7a0e2e7 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -5,9 +5,12 @@ from __future__ import annotations +import json import logging import queue import stat +import subprocess +import sys import threading import time import webbrowser @@ -51,55 +54,37 @@ _LOGGER = logging.getLogger(__name__) -def _ask_open_filename( - *, - initial_dir: str, - accept: tuple[str, ...] = (), - multiple: bool = False, - on_presented: Callable[[], None] | None = None, -) -> tuple[str, ...]: - """Return paths from the Tk file dialog, or an empty tuple when cancelled. - - Tk shows Explorer on Windows, the macOS panel, and its own dialog on Linux. - ``on_presented`` runs just before the dialog opens. It does not mean a - remote host made that dialog visible. - - Raises: - ImportError: This Python was built without Tk. - tkinter.TclError: Tk cannot open a dialog, such as without a display. - """ - # ponytail: this runs on a worker thread, which macOS may abort for Tk. - # Upgrade path: launch ``osascript`` "choose file" as a separate process. - import tkinter as tk - from tkinter import filedialog - - root = tk.Tk() - root.withdraw() - try: - root.attributes("-topmost", True) - except tk.TclError: - pass - # ponytail: calling the Tk dialog is the presentation signal. Tk does not - # report when the dialog is actually visible. Upgrade path: - # IFileDialogEvents on Windows or NSOpenPanel visibility on macOS. - if on_presented is not None: - on_presented() - try: - kwargs: dict[str, Any] = { - "initialdir": initial_dir, - "title": "Select a file", - } - if accept: - kwargs["filetypes"] = [ - ("Accepted", " ".join(f"*{suffix}" for suffix in accept)) - ] - if multiple: - selected = filedialog.askopenfilenames(**kwargs) - return tuple(str(path) for path in selected if path) - selected = filedialog.askopenfilename(**kwargs) - finally: - root.destroy() - return (str(selected),) if selected else () +_FILE_DIALOG_SCRIPT = """\ +import json +import sys +import tkinter as tk +from tkinter import filedialog + +options = json.loads(sys.argv[1]) +root = tk.Tk() +root.withdraw() +try: + root.attributes("-topmost", True) +except tk.TclError: + pass +kwargs = {"initialdir": options["initial_dir"], "title": "Select a file"} +if options["accept"]: + kwargs["filetypes"] = [ + ("Accepted", " ".join("*" + suffix for suffix in options["accept"])) + ] +if options["multiple"]: + paths = [str(path) for path in filedialog.askopenfilenames(**kwargs) if path] +else: + path = filedialog.askopenfilename(**kwargs) + paths = [str(path)] if path else [] +root.destroy() +print(json.dumps(paths)) +""" +"""Child-process script printing the paths chosen in the Tk dialog as JSON. + +Tk shows Explorer on Windows, the macOS panel, and its own dialog on Linux. The +script exits nonzero when Tk is missing or cannot open a display. +""" def _selected_files_from_paths( @@ -254,6 +239,9 @@ def __init__( self._lock_cursor_to_window = False self._links: tuple[ClientWindowLink, ...] = () self._pressed_link: ClientWindowLink | None = None + self._file_dialog: ( + tuple[int, FileSelectionRequest, subprocess.Popen[str]] | None + ) = None def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace the clickable regions over the native window output.""" @@ -301,43 +289,64 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: def _request_file_selector( self, selector_id: int, request: FileSelectionRequest ) -> None: - """Open the Tk file dialog on a worker thread so the window keeps presenting.""" - # ponytail: Tk cannot close this dialog from another thread, so one - # left open after its session ended can sit beside a newer one; its - # result is dropped. Upgrade path: run the dialog in a child process. - threading.Thread( - target=self._complete_file_selection, - args=(selector_id, request), - name="flashdreams-file-picker", - daemon=True, - ).start() - - def _complete_file_selection( - self, selector_id: int, request: FileSelectionRequest - ) -> None: - """Run the file dialog, then report it opened and its result.""" - files: tuple[SelectedFile, ...] = () - status = SelectedFilesStatus.UNAVAILABLE + """Open the Tk file dialog in a child process that input reads check.""" + options = { + "initial_dir": request.initial_path or str(Path.home()), + "accept": list(request.accept), + "multiple": request.multiple, + } + self._file_dialog = ( + selector_id, + request, + subprocess.Popen( + [sys.executable, "-I", "-c", _FILE_DIALOG_SCRIPT, json.dumps(options)], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + text=True, + ), + ) + # ponytail: a started child is the presentation signal; Tk does not + # report when its dialog is visible. Upgrade path: have the child print + # a line once Tk maps the dialog. + self._file_selector_opened(selector_id) + + def _cancel_active_file_selectors(self) -> None: + """Close the dialog of a picker that was flushed or timed out.""" + dialog = self._file_dialog + self._file_dialog = None + if dialog is not None: + dialog[2].kill() + dialog[2].communicate() + + def _poll_file_dialog(self) -> None: + """Report the dialog's result once its child process has exited.""" + dialog = self._file_dialog + if dialog is None or dialog[2].poll() is None: + return + self._file_dialog = None + selector_id, request, process = dialog + output, _ = process.communicate() try: - selected = _ask_open_filename( - initial_dir=request.initial_path or str(Path.home()), - accept=request.accept, - multiple=request.multiple, - on_presented=lambda: self._file_selector_opened(selector_id), - ) - except Exception: - _LOGGER.exception( + paths = json.loads(output) if process.returncode == 0 else None + except ValueError: + paths = None + files: tuple[SelectedFile, ...] = () + if not isinstance(paths, list) or not all(isinstance(p, str) for p in paths): + _LOGGER.warning( "Native file picker failed for request id %r.", request.request_id ) + status = SelectedFilesStatus.UNAVAILABLE + elif not paths: + status = SelectedFilesStatus.CANCELLED else: - if selected: - files, status = _selected_files_from_paths( - selected if request.multiple else selected[:1], - accept=request.accept, - max_bytes=clamp_selected_file_max_bytes(request.max_bytes), - ) - else: - status = SelectedFilesStatus.CANCELLED + # ponytail: files are read on the UI thread, up to the batch limit, + # which can stall presentation briefly. Upgrade path: have the + # child stream the bytes back. + files, status = _selected_files_from_paths( + paths if request.multiple else paths[:1], + accept=request.accept, + max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + ) self._complete_file_selector(selector_id, status, files) def open(self, session_desc: SessionDesc) -> None: @@ -404,6 +413,7 @@ def open(self, session_desc: SessionDesc) -> None: def get_user_input_events(self) -> UserInputEvents: """Pump GLFW and return native input events not yet read.""" + self._poll_file_dialog() self._expire_file_selectors() presenter = self._presenter if presenter is not None: @@ -475,6 +485,7 @@ def write(self, result: StepResult) -> None: def close(self) -> None: """Release SlangPy and the GLFW window on the runtime's UI thread.""" + self._cancel_active_file_selectors() presenter = self._presenter self._presenter = None self._session_started_ns = None @@ -608,7 +619,7 @@ def _on_window_closed(self) -> None: self._put_input(CloseUserInputEvent(timestamp=uint64(0))) def _put_user_input_event(self, event: UserInputEvent) -> None: - """Queue one event from a picker thread, bypassing the poll-time buffer.""" + """Queue a file-selection event, bypassing the poll-time buffer.""" started_ns = self._session_started_ns elapsed_ns = 0 if started_ns is None else max(0, self._clock_ns() - started_ns) self._input_events.put(replace(event, timestamp=uint64(elapsed_ns // 1_000))) diff --git a/flashdreams/flashdreams/runtime_v2/null_client_window.py b/flashdreams/flashdreams/runtime_v2/null_client_window.py index 377889abe..001f026e5 100644 --- a/flashdreams/flashdreams/runtime_v2/null_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/null_client_window.py @@ -11,13 +11,13 @@ class NullClientWindow(IClientWindow): - """Discard output and report no input except file-selection results.""" + """Discard output. It has no user input of its own.""" def __init__(self) -> None: self._input_events: list[UserInputEvent] = [] def get_user_input_events(self) -> UserInputEvents: - """Return the file-selection results queued since the last call.""" + """Return the events queued since the last call.""" events, self._input_events = self._input_events, [] return UserInputEvents(events) diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 4a6dea337..4f1033f6e 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -380,6 +380,7 @@ def __init__( self._client_connected = False self._hide_cursor = False self._lock_cursor_to_window = False + # Upload tokens, touched only on the event loop once it runs. self._pending_file_requests: dict[str, _PendingFileSelection] = {} self._uploading_file_requests: dict[str, _PendingFileSelection] = {} self._file_selector_callbacks: ( @@ -389,7 +390,6 @@ def __init__( ] | None ) = None - self._file_selector_lock = threading.Lock() self._thread = threading.Thread( target=self._run_server, name="flashdreams-webrtc", @@ -565,16 +565,14 @@ def cancel_file_selectors(self) -> None: """ loop = self._loop if loop is None: - with self._file_selector_lock: - self._pending_file_requests.clear() + self._pending_file_requests.clear() return # On the event loop, after any arm still queued for a forgotten selector. loop.call_soon_threadsafe(self._withdraw_file_selectors) def _withdraw_file_selectors(self) -> None: """Forget armed selectors and tell the viewer to close its prompt.""" - with self._file_selector_lock: - self._pending_file_requests.clear() + self._pending_file_requests.clear() channel = self._control_channel if channel is not None and channel.readyState == "open": channel.send(json.dumps({"type": "file_selector_cancel"})) @@ -592,8 +590,7 @@ def _arm_file_selector(self, policy: _PendingFileSelection) -> None: ) return token = secrets.token_urlsafe(16) - with self._file_selector_lock: - self._pending_file_requests[token] = policy + self._pending_file_requests[token] = policy channel.send( json.dumps( { @@ -608,18 +605,16 @@ def _arm_file_selector(self, policy: _PendingFileSelection) -> None: def _take_pending_file_request(self, token: str) -> _PendingFileSelection | None: """Return and consume the policy for ``token``, if it was pending.""" - with self._file_selector_lock: - return self._pending_file_requests.pop(token, None) + return self._pending_file_requests.pop(token, None) def _complete_abandoned_file_selectors(self) -> None: """Answer every armed pick as ``UNAVAILABLE`` because the viewer left.""" - with self._file_selector_lock: - abandoned = ( - *self._pending_file_requests.values(), - *self._uploading_file_requests.values(), - ) - self._pending_file_requests.clear() - self._uploading_file_requests.clear() + abandoned = ( + *self._pending_file_requests.values(), + *self._uploading_file_requests.values(), + ) + self._pending_file_requests.clear() + self._uploading_file_requests.clear() for policy in abandoned: self._emit_selected_files( policy.selector_id, status=SelectedFilesStatus.UNAVAILABLE @@ -688,8 +683,10 @@ def open(self, session_desc: SessionDesc) -> None: # Keep timestamps comparable across sessions so events buffered # during a handoff retain their real order. self._event_origin_ns = time.monotonic_ns() - with self._file_selector_lock: + if loop is None: self._pending_file_requests.clear() + else: + loop.call_soon_threadsafe(self._pending_file_requests.clear) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the dimensions used to present subsequent UI frames. @@ -987,8 +984,7 @@ async def _upload_selected_file(self, request: web.Request) -> web.Response: if policy is None: raise web.HTTPConflict(reason="No pending file selection for this id.") # Tracked until it ends so a viewer that leaves mid-upload is answered. - with self._file_selector_lock: - self._uploading_file_requests[token] = policy + self._uploading_file_requests[token] = policy emitted = False @@ -1083,8 +1079,7 @@ def emit( emit(SelectedFilesStatus.TOO_LARGE) raise finally: - with self._file_selector_lock: - self._uploading_file_requests.pop(token, None) + self._uploading_file_requests.pop(token, None) if not emitted: emit(SelectedFilesStatus.UNAVAILABLE) @@ -1443,8 +1438,7 @@ def _buffer_browser_message( raise ValueError( "File selector acknowledgement requires a non-empty id." ) - with self._file_selector_lock: - policy = self._pending_file_requests.get(token) + policy = self._pending_file_requests.get(token) if policy is None: return self._file_selector_callbacks_or_raise()[0](policy.selector_id) diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 5331b3372..6a9adf85f 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -5,6 +5,7 @@ import threading from collections import deque +from collections.abc import Callable from dataclasses import replace from numpy import uint64 @@ -24,7 +25,8 @@ class WebRTCClientWindow(IClientWindow): A thin pairing of the :class:`IClientWindow` protocol with :class:`WebRTCServer`, which does the serving. Browser events arrive on the server's own thread, so they are queued here and handed over in batches when - the session asks, as the protocol requires. + the session asks, as the protocol requires. File-selector reports are + queued the same way and applied on the UI thread when input is read. The server timestamps arrivals against one stable clock. This window clears input buffered during a session handoff, then rebases later events to the @@ -54,6 +56,7 @@ def __init__( startup_timeout_seconds: Maximum time to wait for server startup. """ self._input_events: deque[UserInputEvent] = deque() + self._file_selector_reports: deque[Callable[[], None]] = deque() self._hide_cursor = False self._lock_cursor_to_window = False self._input_lock = threading.Lock() @@ -71,11 +74,18 @@ def handle_input(event: UserInputEvent) -> None: with self._input_lock: self._input_events.append(event) + def report_file_selector(apply: Callable[[], None]) -> None: + """Hand a server-thread picker report to the UI thread.""" + with self._input_lock: + self._file_selector_reports.append(apply) + self.server.register_input_callback(handle_input) self.server.register_file_selector_callbacks( - opened=lambda selector_id: self._file_selector_opened(int(selector_id)), - completed=lambda selector_id, status, files: self._complete_file_selector( - int(selector_id), status, files + opened=lambda selector_id: report_file_selector( + lambda: self._file_selector_opened(int(selector_id)) + ), + completed=lambda selector_id, status, files: report_file_selector( + lambda: self._complete_file_selector(int(selector_id), status, files) ), ) @@ -145,6 +155,11 @@ def get_user_input_events(self) -> UserInputEvents: Returns: Buffered browser events in timestamp order, each returned once. """ + with self._input_lock: + reports = list(self._file_selector_reports) + self._file_selector_reports.clear() + for apply in reports: + apply() self._expire_file_selectors() with self._input_lock: events = list(self._input_events) diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index ed6e376e5..8c1557a30 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json import queue import sys import threading @@ -65,8 +66,13 @@ def _file_request( ) +def _fake_dialog(monkeypatch: pytest.MonkeyPatch, script: str) -> None: + """Run ``script`` in the dialog's child process instead of Tk.""" + monkeypatch.setattr(native_window_module, "_FILE_DIALOG_SCRIPT", script) + + def _wait_for_input_events(window: NativeWindowClientWindow) -> list[object]: - for _ in range(100): + for _ in range(500): events = window.get_user_input_events().get_events() if events: return list(events) @@ -1004,13 +1010,7 @@ def test_native_window_reports_a_selected_file( ) -> None: chosen = tmp_path / "seed.png" chosen.write_bytes(b"png-bytes") - monkeypatch.setattr( - native_window_module, - "_ask_open_filename", - lambda *, initial_dir, accept=(), multiple=False, on_presented=None: ( - str(chosen), - ), - ) + _fake_dialog(monkeypatch, f"print({json.dumps([str(chosen)])!r})") presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1032,11 +1032,7 @@ def test_native_window_reports_a_selected_file( def test_native_window_reports_cancelled_file_selection( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - monkeypatch.setattr( - native_window_module, - "_ask_open_filename", - lambda *, initial_dir, accept=(), multiple=False, on_presented=None: (), - ) + _fake_dialog(monkeypatch, "print('[]')") presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1057,17 +1053,7 @@ def test_native_window_reports_cancelled_file_selection( def test_native_window_reports_unavailable_when_picker_raises( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - def fail_picker( - *, - initial_dir: str, - accept: tuple[str, ...] = (), - multiple: bool = False, - on_presented: object = None, - ) -> tuple[str, ...]: - del initial_dir, accept, multiple, on_presented - raise RuntimeError("Native file picker needs a desktop chooser.") - - monkeypatch.setattr(native_window_module, "_ask_open_filename", fail_picker) + _fake_dialog(monkeypatch, "raise SystemExit('Tk cannot open a display.')") presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1132,34 +1118,15 @@ def test_selected_files_from_paths_rejects_a_non_regular_file(tmp_path: Path) -> def test_native_window_shown_picker_has_no_deadline( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - shown = threading.Event() - release = threading.Event() - - def picker( - *, - initial_dir: str, - accept: tuple[str, ...] = (), - multiple: bool = False, - on_presented: Callable[[], None] | None = None, - ) -> tuple[str, ...]: - del initial_dir, accept, multiple - assert on_presented is not None - on_presented() - shown.set() - release.wait(5) - return () - - monkeypatch.setattr(native_window_module, "_ask_open_filename", picker) + _fake_dialog(monkeypatch, "import time; time.sleep(0.5); print('[]')") now = [0.0] presenter = _Presenter() window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) window._file_selector_clock = lambda: now[0] window.open(_session_desc()) window.request_file_selector(_file_request("open-1", str(tmp_path))) - assert shown.wait(5) now[0] = 1_000.0 assert window.get_user_input_events().get_events() == [] - release.set() events = _wait_for_input_events(window) window.close() @@ -1176,14 +1143,7 @@ def test_native_window_reports_multiple_selected_files( second = tmp_path / "b.bin" first.write_bytes(b"one") second.write_bytes(b"two") - monkeypatch.setattr( - native_window_module, - "_ask_open_filename", - lambda *, initial_dir, accept=(), multiple=False, on_presented=None: ( - str(first), - str(second), - ), - ) + _fake_dialog(monkeypatch, f"print({json.dumps([str(first), str(second)])!r})") presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1201,6 +1161,24 @@ def test_native_window_reports_multiple_selected_files( assert [chosen.data for chosen in event.files] == [b"one", b"two"] +def test_native_window_flush_closes_the_open_dialog( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_dialog(monkeypatch, "import time; time.sleep(60)") + presenter = _Presenter() + window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + dialog = window._file_dialog + assert dialog is not None + + window.flush_file_selectors() + + assert dialog[2].returncode is not None + assert window.get_user_input_events().get_events() == [] + window.close() + + def test_native_window_reports_standard_gamepad_events() -> None: presenter = _Presenter() clock_values = iter((1_000_000, 1_001_000, 1_002_000, 1_003_000)) diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 0a04d8eab..137394d90 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -6,11 +6,13 @@ # ruff: noqa: E402 - optional WebRTC imports must follow importorskip. import asyncio +import concurrent.futures import json import threading import time +from collections.abc import Callable from dataclasses import replace -from typing import Any, cast +from typing import Any, TypeVar, cast from unittest.mock import AsyncMock, Mock import pytest @@ -18,6 +20,8 @@ pytestmark = pytest.mark.ci_cpu +T = TypeVar("T") + pytest.importorskip("aiohttp") pytest.importorskip("aiortc") @@ -89,18 +93,31 @@ async def _wait_file_armed(window: WebRTCClientWindow) -> str: raise AssertionError("file selection was not armed") +def _on_server_loop(window: WebRTCClientWindow, call: Callable[[], T]) -> T: + """Run ``call`` on the server's event loop, which owns its upload tokens.""" + loop = window.server._loop + assert loop is not None + future: concurrent.futures.Future[T] = concurrent.futures.Future() + loop.call_soon_threadsafe(lambda: future.set_result(call())) + return future.result(timeout=5) + + def _arm(window: WebRTCClientWindow, request: FileSelectionRequest) -> str: """Arm a selector as if the browser had been sent ``request``; return its token.""" selector_id = window._last_file_selector_id + 1 window._last_file_selector_id = selector_id window._active_file_selector = (selector_id, request) wire_id = f"token-{selector_id}" - window.server._pending_file_requests[wire_id] = webrtc_server._PendingFileSelection( + policy = webrtc_server._PendingFileSelection( selector_id=str(selector_id), accept=request.accept, max_bytes=clamp_selected_file_max_bytes(request.max_bytes), multiple=request.multiple, ) + _on_server_loop( + window, + lambda: window.server._pending_file_requests.__setitem__(wire_id, policy), + ) return wire_id @@ -551,11 +568,11 @@ async def test_browser_ack_reports_the_file_selector_opened() -> None: channel.send(json.dumps({"type": "file_selector_ack", "id": "other"})) channel.send(json.dumps({"type": "file_selector_ack", "id": wire_id})) for _ in range(100): + assert window.get_user_input_events().get_events() == [] if window._file_selector_deadline is None: break await asyncio.sleep(0.01) assert window._file_selector_deadline is None - assert window.get_user_input_events().get_events() == [] channel.send( json.dumps( { @@ -673,7 +690,10 @@ async def test_session_handoff_clears_pending_file_selection() -> None: try: wire_id = _arm(window, _file_request("open-1")) window.open(_session_desc()) - assert window.server._pending_file_requests == {} + assert ( + _on_server_loop(window, lambda: dict(window.server._pending_file_requests)) + == {} + ) assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 409 finally: window.close() @@ -958,7 +978,6 @@ def _link_server() -> tuple[webrtc_server.WebRTCServer, Mock, Mock]: server._link_lock = threading.Lock() server._links = () server._pressed_link = None - server._file_selector_lock = threading.Lock() server._pending_file_requests = {} server._uploading_file_requests = {} channel = Mock(readyState="open") From 62c53578a7424d7ddf90bc54db8f55f6c97d7fff Mon Sep 17 00:00:00 2001 From: Set Paing Date: Thu, 15 Oct 2026 17:40:32 -0700 Subject: [PATCH 03/12] fix(runtime_v2): keep file pickers from getting stuck The native dialog writes its paths to a temporary file, so a long path list cannot fill a pipe and block the child. The WebRTC page trusts the server's one-selector rule and replaces a finished picker with a new one. Signed-off-by: Set Paing --- .../runtime_v2/native_window_client_window.py | 33 +++++++++++-------- .../flashdreams/runtime_v2/serving/web/app.js | 7 ++-- .../test_native_window_client_window.py | 19 +++++++++++ 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index df7a0e2e7..8a8997bcd 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -11,6 +11,7 @@ import stat import subprocess import sys +import tempfile import threading import time import webbrowser @@ -18,7 +19,7 @@ from collections.abc import Callable, Sequence from dataclasses import replace from pathlib import Path -from typing import TYPE_CHECKING, Any, cast +from typing import IO, TYPE_CHECKING, Any, cast import numpy as np import torch @@ -240,7 +241,7 @@ def __init__( self._links: tuple[ClientWindowLink, ...] = () self._pressed_link: ClientWindowLink | None = None self._file_dialog: ( - tuple[int, FileSelectionRequest, subprocess.Popen[str]] | None + tuple[int, FileSelectionRequest, subprocess.Popen[bytes], IO[str]] | None ) = None def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: @@ -295,16 +296,19 @@ def _request_file_selector( "accept": list(request.accept), "multiple": request.multiple, } - self._file_dialog = ( - selector_id, - request, - subprocess.Popen( + # A file, not a pipe: a long path list would fill a pipe and block the + # child before it exits. + output = tempfile.TemporaryFile("w+") + try: + process = subprocess.Popen( [sys.executable, "-I", "-c", _FILE_DIALOG_SCRIPT, json.dumps(options)], stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - text=True, - ), - ) + stdout=output, + ) + except BaseException: + output.close() + raise + self._file_dialog = (selector_id, request, process, output) # ponytail: a started child is the presentation signal; Tk does not # report when its dialog is visible. Upgrade path: have the child print # a line once Tk maps the dialog. @@ -316,7 +320,8 @@ def _cancel_active_file_selectors(self) -> None: self._file_dialog = None if dialog is not None: dialog[2].kill() - dialog[2].communicate() + dialog[2].wait() + dialog[3].close() def _poll_file_dialog(self) -> None: """Report the dialog's result once its child process has exited.""" @@ -324,8 +329,10 @@ def _poll_file_dialog(self) -> None: if dialog is None or dialog[2].poll() is None: return self._file_dialog = None - selector_id, request, process = dialog - output, _ = process.communicate() + selector_id, request, process, output_file = dialog + with output_file: + output_file.seek(0) + output = output_file.read() try: paths = json.loads(output) if process.returncode == 0 else None except ValueError: diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 4f6740425..af6600110 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -336,10 +336,11 @@ const openFileSelector = options => { return; } if (pendingFileRequest !== null) { - if (pendingFileRequest.id !== requestId) { - completeFileSelection(requestId, "unavailable"); + if (pendingFileRequest.id === requestId) { + return; } - return; + // The server opens one selector at a time, so a new one means the old one finished. + hideFilePicker(); } const accept = Array.isArray(options.accept) ? options.accept : []; const maxBytes = Number(options.max_bytes); diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index 8c1557a30..af6ca23c3 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -1161,6 +1161,25 @@ def test_native_window_reports_multiple_selected_files( assert [chosen.data for chosen in event.files] == [b"one", b"two"] +def test_native_window_reads_a_path_list_larger_than_a_pipe( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_dialog( + monkeypatch, "import json; print(json.dumps(['/' + 'x' * 1000] * 200))" + ) + presenter = _Presenter() + window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path), multiple=True)) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.status is SelectedFilesStatus.UNAVAILABLE + + def test_native_window_flush_closes_the_open_dialog( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 6394075bc285114e32dcb41af65ef1b19c33bbd1 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Fri, 16 Oct 2026 17:07:38 -0700 Subject: [PATCH 04/12] docs(api_v2): trim the IClientWindow file-selection notes Signed-off-by: Set Paing --- flashdreams/flashdreams/api_v2/client_window.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 25decdb63..b82ea3b64 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -105,17 +105,12 @@ class IClientWindow(InputSource, OutputSink, ABC): A window does not describe the output shape. The session does, and the window is given that description in :meth:`OutputSink.open`. - The runtime makes every call on a window from the UI thread. A window still - needs locking where its backend delivers input from another thread. - - File selection: this class owns the rules (one picker at a time, the - presentation deadline, result checks, stale results). A window with a - picker implements :meth:`_request_file_selector` and - :meth:`_cancel_active_file_selectors`, and calls - :meth:`_expire_file_selectors` when input is read. Its reports must reach - this class on the UI thread. :meth:`OutputSink.open` drops events queued - for the previous session. A window wrapping another forwards - :meth:`request_file_selector` and :meth:`flush_file_selectors`. + One thread makes every call on a window, so an implementation needs no + locking except when its backend delivers input from another thread. + + A window with a file picker implements :meth:`_request_file_selector` and + :meth:`_cancel_active_file_selectors`, reports results on the UI thread, + and calls :meth:`_expire_file_selectors` when input is read. Created by the runtime, never by an application. """ From 3786d0a928aa35cb8055037a8ebf987cf7c46e52 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Mon, 19 Oct 2026 10:28:13 -0700 Subject: [PATCH 05/12] refactor(runtime_v2): address file-selection review - Run the native Tk dialog on a worker thread instead of a child process. A dialog left open at session end stays until the user closes it; its result is dropped by the selector id, and new requests get BUSY meanwhile. - Drop the 30-second presentation deadline and the browser ack it needed. - Rename _checked_result to _check_file_selection_result, mark request_file_selector and flush_file_selectors @final, and use Path(name).name for basenames. - Add a delayed "Restart session" button to the file-picker demo. Signed-off-by: Set Paing --- .../flashdreams/api_v2/client_window.py | 79 ++------ flashdreams/flashdreams/runtime_v2/README.md | 3 +- .../runtime_v2/native_window_client_window.py | 185 ++++++++---------- .../flashdreams/runtime_v2/selected_file.py | 7 - .../flashdreams/runtime_v2/serving/web/app.js | 2 - .../runtime_v2/serving/webrtc_server.py | 49 ++--- .../runtime_v2/webrtc_client_window.py | 35 ++-- .../test_native_window_client_window.py | 87 ++++---- flashdreams/test_v2/test_selected_files.py | 44 +---- flashdreams/test_v2/test_session_runner.py | 10 +- .../test_v2/test_webrtc_client_window.py | 36 ---- integrations_v2/imgui_ui_demo/README.md | 5 +- .../imgui_ui_demo/file_picker_app.py | 18 +- 13 files changed, 190 insertions(+), 370 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index b82ea3b64..e925cbeea 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -5,17 +5,15 @@ import logging import math -import time from abc import ABC, abstractmethod -from collections.abc import Callable from dataclasses import dataclass, replace +from pathlib import Path from typing import final from urllib.parse import urlsplit from numpy import uint64 from flashdreams.runtime_v2.selected_file import ( - FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS, FileSelectionRequest, SelectedFile, SelectedFilesStatus, @@ -109,8 +107,8 @@ class IClientWindow(InputSource, OutputSink, ABC): locking except when its backend delivers input from another thread. A window with a file picker implements :meth:`_request_file_selector` and - :meth:`_cancel_active_file_selectors`, reports results on the UI thread, - and calls :meth:`_expire_file_selectors` when input is read. + :meth:`_cancel_active_file_selectors`, and reports results on the UI + thread. Created by the runtime, never by an application. """ @@ -141,25 +139,22 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """ pass - # Plain class defaults so subclasses need no constructor call; assigning - # one on an instance shadows it for that window only. + # File Selector Methods & Members _active_file_selector: tuple[int, FileSelectionRequest] | None = None - _file_selector_deadline: float | None = None _last_file_selector_id = 0 - _file_selector_clock: Callable[[], float] = staticmethod(time.monotonic) + @final def request_file_selector(self, request: FileSelectionRequest) -> None: - """Show a file selector; its result arrives later as an input event. + """Request for IClientWindow to open a file selector; the result of the + user interaction arrives later as a ``SelectedFilesUserInputEvent`` via + :meth:`InputSource.get_user_input_events`. - The result is one ``SelectedFilesUserInputEvent`` from - :meth:`InputSource.get_user_input_events` carrying - ``request.request_id``. A request made while a picker is open is - answered as ``BUSY``. + A request made while a picker is open is answered with a status of + ``SelectedFilesStatus.BUSY``. Args: request: Selector request queued by the UI loop. """ - self._expire_file_selectors() if self._active_file_selector is not None: self._put_user_input_event( _selected_files_event(request, SelectedFilesStatus.BUSY) @@ -168,15 +163,13 @@ def request_file_selector(self, request: FileSelectionRequest) -> None: selector_id = self._last_file_selector_id + 1 self._last_file_selector_id = selector_id self._active_file_selector = (selector_id, request) - self._file_selector_deadline = ( - self._file_selector_clock() + FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS - ) try: self._request_file_selector(selector_id, request) except Exception: _LOGGER.exception("File selector %r failed to start.", request.request_id) self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + @final def flush_file_selectors(self) -> None: """Forget the open picker without a result, for a session that ended. @@ -185,7 +178,6 @@ def flush_file_selectors(self) -> None: """ active = self._active_file_selector self._active_file_selector = None - self._file_selector_deadline = None if active is not None: self._cancel_active_file_selectors() @@ -195,17 +187,16 @@ def _request_file_selector( ) -> None: """Show a picker for ``request`` without blocking. - Call :meth:`_file_selector_opened` once it is shown, then - :meth:`_complete_file_selector` with ``selector_id`` when the user - finishes, both on the UI thread. The default has no picker and - completes as ``UNAVAILABLE``. + Call :meth:`_complete_file_selector` with ``selector_id`` on the UI + thread when the user finishes. The default has no picker and completes + as ``UNAVAILABLE``. """ del request self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) # Optional to implement def _cancel_active_file_selectors(self) -> None: - """Withdraw the picker that was flushed or that timed out.""" + """Withdraw the picker that was flushed.""" pass @abstractmethod @@ -217,13 +208,6 @@ def _put_user_input_event(self, event: UserInputEvent) -> None: must not call back into file selection. """ - @final - def _file_selector_opened(self, selector_id: int) -> None: - """Stop the deadline because the picker for ``selector_id`` is shown.""" - active = self._active_file_selector - if active is not None and active[0] == selector_id: - self._file_selector_deadline = None - @final def _complete_file_selector( self, @@ -240,29 +224,10 @@ def _complete_file_selector( if active is None or active[0] != selector_id: return self._active_file_selector = None - self._file_selector_deadline = None request = active[1] - status, files = _checked_result(request, status, files) + status, files = _check_file_selection_result(request, status, files) self._put_user_input_event(_selected_files_event(request, status, files)) - @final - def _expire_file_selectors(self) -> None: - """Complete a picker not shown by its deadline as ``UNAVAILABLE``. - - Windows with a picker call this each time input is read. - """ - active = self._active_file_selector - deadline = self._file_selector_deadline - if active is None or deadline is None or self._file_selector_clock() < deadline: - return - _LOGGER.warning( - "File selector %r was not shown within %g seconds.", - active[1].request_id, - FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS, - ) - self._complete_file_selector(active[0], SelectedFilesStatus.UNAVAILABLE) - self._cancel_active_file_selectors() - def _selected_files_event( request: FileSelectionRequest, @@ -278,20 +243,18 @@ def _selected_files_event( ) -def _checked_result( +def _check_file_selection_result( request: FileSelectionRequest, status: SelectedFilesStatus, files: tuple[SelectedFile, ...], ) -> tuple[SelectedFilesStatus, tuple[SelectedFile, ...]]: """Return the status and files a window's result earns under ``request``.""" - if status is SelectedFilesStatus.BUSY: - status = SelectedFilesStatus.UNAVAILABLE if status is not SelectedFilesStatus.OK: return status, () if not files or (len(files) > 1 and not request.multiple): _LOGGER.warning("File selector %r returned no valid set.", request.request_id) return SelectedFilesStatus.UNAVAILABLE, () - files = tuple(replace(chosen, name=_base_name(chosen.name)) for chosen in files) + files = tuple(replace(chosen, name=Path(chosen.name).name) for chosen in files) rejected = selected_files_policy_status( [(chosen.name, len(chosen.data)) for chosen in files], accept=request.accept, @@ -300,9 +263,3 @@ def _checked_result( if rejected is not None: return rejected, () return status, files - - -def _base_name(name: str) -> str: - """Return ``name`` without any POSIX or Windows directory part.""" - base = name.replace("\\", "/").rsplit("/", 1)[-1] - return "upload" if base in ("", ".", "..") else base diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 4579d9947..163c9ad43 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -377,8 +377,7 @@ them and hands them over in batches when the session asks. File selection rules live in `IClientWindow`; a window only shows a picker and reports what it returned, on the UI thread. Every window implements the abstract `_put_user_input_event`; a window with a picker also implements -`_request_file_selector` and calls `_expire_file_selectors()` each time its -input is read. +`_request_file_selector`. A UI loop may call `request_new_session(session_desc)` during its step with a fully resolved replacement description. `run_session` stops and cleans the diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index 8a8997bcd..9a90c519c 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -5,13 +5,9 @@ from __future__ import annotations -import json import logging import queue import stat -import subprocess -import sys -import tempfile import threading import time import webbrowser @@ -19,7 +15,7 @@ from collections.abc import Callable, Sequence from dataclasses import replace from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, cast import numpy as np import torch @@ -55,37 +51,37 @@ _LOGGER = logging.getLogger(__name__) -_FILE_DIALOG_SCRIPT = """\ -import json -import sys -import tkinter as tk -from tkinter import filedialog - -options = json.loads(sys.argv[1]) -root = tk.Tk() -root.withdraw() -try: - root.attributes("-topmost", True) -except tk.TclError: - pass -kwargs = {"initialdir": options["initial_dir"], "title": "Select a file"} -if options["accept"]: - kwargs["filetypes"] = [ - ("Accepted", " ".join("*" + suffix for suffix in options["accept"])) - ] -if options["multiple"]: - paths = [str(path) for path in filedialog.askopenfilenames(**kwargs) if path] -else: - path = filedialog.askopenfilename(**kwargs) - paths = [str(path)] if path else [] -root.destroy() -print(json.dumps(paths)) -""" -"""Child-process script printing the paths chosen in the Tk dialog as JSON. - -Tk shows Explorer on Windows, the macOS panel, and its own dialog on Linux. The -script exits nonzero when Tk is missing or cannot open a display. -""" +def _choose_paths(request: FileSelectionRequest) -> list[str]: + """Block in the Tk file dialog and return the chosen paths. + + Tk shows Explorer on Windows and its own dialog on Linux. Raises when Tk is + missing or cannot open a display. + """ + import tkinter as tk + from tkinter import filedialog + + root = tk.Tk() + try: + root.withdraw() + try: + root.attributes("-topmost", True) + except tk.TclError: + pass + kwargs: dict[str, Any] = { + "parent": root, + "initialdir": request.initial_path or str(Path.home()), + "title": "Select a file", + } + if request.accept: + kwargs["filetypes"] = [ + ("Accepted", " ".join("*" + suffix for suffix in request.accept)) + ] + if request.multiple: + return [str(path) for path in filedialog.askopenfilenames(**kwargs) if path] + path = filedialog.askopenfilename(**kwargs) + return [str(path)] if path else [] + finally: + root.destroy() def _selected_files_from_paths( @@ -240,9 +236,10 @@ def __init__( self._lock_cursor_to_window = False self._links: tuple[ClientWindowLink, ...] = () self._pressed_link: ClientWindowLink | None = None - self._file_dialog: ( - tuple[int, FileSelectionRequest, subprocess.Popen[bytes], IO[str]] | None - ) = None + self._file_dialog_thread: threading.Thread | None = None + self._file_dialog_results: queue.SimpleQueue[ + tuple[int, FileSelectionRequest, list[str] | None] + ] = queue.SimpleQueue() def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace the clickable regions over the native window output.""" @@ -290,71 +287,53 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: def _request_file_selector( self, selector_id: int, request: FileSelectionRequest ) -> None: - """Open the Tk file dialog in a child process that input reads check.""" - options = { - "initial_dir": request.initial_path or str(Path.home()), - "accept": list(request.accept), - "multiple": request.multiple, - } - # A file, not a pipe: a long path list would fill a pipe and block the - # child before it exits. - output = tempfile.TemporaryFile("w+") - try: - process = subprocess.Popen( - [sys.executable, "-I", "-c", _FILE_DIALOG_SCRIPT, json.dumps(options)], - stdin=subprocess.DEVNULL, - stdout=output, - ) - except BaseException: - output.close() - raise - self._file_dialog = (selector_id, request, process, output) - # ponytail: a started child is the presentation signal; Tk does not - # report when its dialog is visible. Upgrade path: have the child print - # a line once Tk maps the dialog. - self._file_selector_opened(selector_id) - - def _cancel_active_file_selectors(self) -> None: - """Close the dialog of a picker that was flushed or timed out.""" - dialog = self._file_dialog - self._file_dialog = None - if dialog is not None: - dialog[2].kill() - dialog[2].wait() - dialog[3].close() + """Show the Tk file dialog on a worker thread that queues its result.""" + thread = self._file_dialog_thread + if thread is not None and thread.is_alive(): + # A flushed dialog stays open until the user closes it, and two Tk + # dialogs on separate threads are not safe. + self._complete_file_selector(selector_id, SelectedFilesStatus.BUSY) + return + results = self._file_dialog_results + + def show_dialog() -> None: + try: + paths: list[str] | None = _choose_paths(request) + except Exception: + _LOGGER.exception( + "Native file picker failed for request id %r.", request.request_id + ) + paths = None + results.put((selector_id, request, paths)) + + thread = threading.Thread( + target=show_dialog, name="flashdreams-file-dialog", daemon=True + ) + thread.start() + self._file_dialog_thread = thread def _poll_file_dialog(self) -> None: - """Report the dialog's result once its child process has exited.""" - dialog = self._file_dialog - if dialog is None or dialog[2].poll() is None: - return - self._file_dialog = None - selector_id, request, process, output_file = dialog - with output_file: - output_file.seek(0) - output = output_file.read() - try: - paths = json.loads(output) if process.returncode == 0 else None - except ValueError: - paths = None - files: tuple[SelectedFile, ...] = () - if not isinstance(paths, list) or not all(isinstance(p, str) for p in paths): - _LOGGER.warning( - "Native file picker failed for request id %r.", request.request_id - ) - status = SelectedFilesStatus.UNAVAILABLE - elif not paths: - status = SelectedFilesStatus.CANCELLED - else: - # ponytail: files are read on the UI thread, up to the batch limit, - # which can stall presentation briefly. Upgrade path: have the - # child stream the bytes back. - files, status = _selected_files_from_paths( - paths if request.multiple else paths[:1], - accept=request.accept, - max_bytes=clamp_selected_file_max_bytes(request.max_bytes), - ) - self._complete_file_selector(selector_id, status, files) + """Report the results the dialog thread queued; stale ones are dropped.""" + while True: + try: + selector_id, request, paths = self._file_dialog_results.get_nowait() + except queue.Empty: + return + files: tuple[SelectedFile, ...] = () + if paths is None: + status = SelectedFilesStatus.UNAVAILABLE + elif not paths: + status = SelectedFilesStatus.CANCELLED + else: + # ponytail: files are read on the UI thread, up to the batch + # limit, which can stall presentation briefly. Upgrade path: + # read them on the dialog thread. + files, status = _selected_files_from_paths( + paths if request.multiple else paths[:1], + accept=request.accept, + max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + ) + self._complete_file_selector(selector_id, status, files) def open(self, session_desc: SessionDesc) -> None: """Open a session in the GLFW window on the runtime's UI thread. @@ -421,7 +400,6 @@ def open(self, session_desc: SessionDesc) -> None: def get_user_input_events(self) -> UserInputEvents: """Pump GLFW and return native input events not yet read.""" self._poll_file_dialog() - self._expire_file_selectors() presenter = self._presenter if presenter is not None: pending_before_poll = tuple(self._pending_printable_keys) @@ -492,7 +470,6 @@ def write(self, result: StepResult) -> None: def close(self) -> None: """Release SlangPy and the GLFW window on the runtime's UI thread.""" - self._cancel_active_file_selectors() presenter = self._presenter self._presenter = None self._session_started_ns = None diff --git a/flashdreams/flashdreams/runtime_v2/selected_file.py b/flashdreams/flashdreams/runtime_v2/selected_file.py index ce100e183..37c74b5b6 100644 --- a/flashdreams/flashdreams/runtime_v2/selected_file.py +++ b/flashdreams/flashdreams/runtime_v2/selected_file.py @@ -18,13 +18,6 @@ MAX_SELECTED_FILE_COUNT = 256 """Hard ceiling for the number of files in one selected-files request.""" -FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS = 30.0 -"""Seconds to wait for a client window to show a picker before ``UNAVAILABLE``. - -The deadline starts when a request is handed to the window. Once the window -reports the picker is shown, the user may take as long as they want. -""" - def clamp_selected_file_max_bytes(max_bytes: int | None) -> int: """Return a file-size budget that cannot exceed ``MAX_SELECTED_FILE_BYTES``. diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index af6600110..49a17ed37 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -383,9 +383,7 @@ const openFileSelector = options => { if (promptDialog === null || !promptDialog.open) { completeFileSelection(requestId, "unavailable"); hideFilePicker(); - return; } - send({type: "file_selector_ack", id: requestId}); }; window.addEventListener("focus", updateCursorVisibility); diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 4f1033f6e..bddb1f83b 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -383,12 +383,8 @@ def __init__( # Upload tokens, touched only on the event loop once it runs. self._pending_file_requests: dict[str, _PendingFileSelection] = {} self._uploading_file_requests: dict[str, _PendingFileSelection] = {} - self._file_selector_callbacks: ( - tuple[ - Callable[[str], None], - Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], - ] - | None + self._file_selector_callback: ( + Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None] | None ) = None self._thread = threading.Thread( target=self._run_server, @@ -743,21 +739,17 @@ def register_input_callback( raise RuntimeError("An input callback is already registered.") self._input_callback = callback - def register_file_selector_callbacks( + def register_file_selector_callback( self, - *, - opened: Callable[[str], None], - completed: Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], + callback: Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], ) -> None: """Register how armed file selectors report back, by selector id. - Both run on the server thread. - Args: - opened: Called when the browser shows the selector. - completed: Called once with the selector's status and files. + callback: Called once per selector on the server thread, with its + status and files. """ - self._file_selector_callbacks = (opened, completed) + self._file_selector_callback = callback def write(self, result: StepResult) -> None: """Materialize and admit one generated result to the sender mailbox. @@ -1432,17 +1424,6 @@ def _buffer_browser_message( elif event_type == "close": self._cancel_link_press() event = CloseUserInputEvent(timestamp=timestamp_us) - elif event_type == "file_selector_ack": - token = payload.get("id") - if not isinstance(token, str) or not token: - raise ValueError( - "File selector acknowledgement requires a non-empty id." - ) - policy = self._pending_file_requests.get(token) - if policy is None: - return - self._file_selector_callbacks_or_raise()[0](policy.selector_id) - return elif event_type == "file_selector_result": token = payload.get("id") if not isinstance(token, str) or not token: @@ -1480,18 +1461,10 @@ def _emit_selected_files( files: tuple[SelectedFile, ...] = (), ) -> None: """Report the result of one armed file selector to the window.""" - self._file_selector_callbacks_or_raise()[1](selector_id, status, files) - - def _file_selector_callbacks_or_raise( - self, - ) -> tuple[ - Callable[[str], None], - Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], - ]: - callbacks = self._file_selector_callbacks - if callbacks is None: - raise RuntimeError("WebRTC file selector callbacks are not registered.") - return callbacks + callback = self._file_selector_callback + if callback is None: + raise RuntimeError("WebRTC file selector callback is not registered.") + callback(selector_id, status, files) async def _release_peer_connection( self, diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 6a9adf85f..23eef0111 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -5,13 +5,16 @@ import threading from collections import deque -from collections.abc import Callable from dataclasses import replace from numpy import uint64 from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow -from flashdreams.runtime_v2.selected_file import FileSelectionRequest +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, +) from flashdreams.runtime_v2.serving.webrtc_server import WebRTCServer from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -56,7 +59,9 @@ def __init__( startup_timeout_seconds: Maximum time to wait for server startup. """ self._input_events: deque[UserInputEvent] = deque() - self._file_selector_reports: deque[Callable[[], None]] = deque() + self._file_selector_reports: deque[ + tuple[int, SelectedFilesStatus, tuple[SelectedFile, ...]] + ] = deque() self._hide_cursor = False self._lock_cursor_to_window = False self._input_lock = threading.Lock() @@ -74,20 +79,17 @@ def handle_input(event: UserInputEvent) -> None: with self._input_lock: self._input_events.append(event) - def report_file_selector(apply: Callable[[], None]) -> None: - """Hand a server-thread picker report to the UI thread.""" + def report_file_selector( + selector_id: str, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], + ) -> None: + """Hand a server-thread picker result to the UI thread.""" with self._input_lock: - self._file_selector_reports.append(apply) + self._file_selector_reports.append((int(selector_id), status, files)) self.server.register_input_callback(handle_input) - self.server.register_file_selector_callbacks( - opened=lambda selector_id: report_file_selector( - lambda: self._file_selector_opened(int(selector_id)) - ), - completed=lambda selector_id, status, files: report_file_selector( - lambda: self._complete_file_selector(int(selector_id), status, files) - ), - ) + self.server.register_file_selector_callback(report_file_selector) def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace links opened locally by the browser viewing this window.""" @@ -158,9 +160,8 @@ def get_user_input_events(self) -> UserInputEvents: with self._input_lock: reports = list(self._file_selector_reports) self._file_selector_reports.clear() - for apply in reports: - apply() - self._expire_file_selectors() + for selector_id, status, files in reports: + self._complete_file_selector(selector_id, status, files) with self._input_lock: events = list(self._input_events) self._input_events.clear() diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index af6ca23c3..0862c2743 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -5,7 +5,6 @@ from __future__ import annotations -import json import queue import sys import threading @@ -66,9 +65,12 @@ def _file_request( ) -def _fake_dialog(monkeypatch: pytest.MonkeyPatch, script: str) -> None: - """Run ``script`` in the dialog's child process instead of Tk.""" - monkeypatch.setattr(native_window_module, "_FILE_DIALOG_SCRIPT", script) +def _fake_dialog( + monkeypatch: pytest.MonkeyPatch, + choose: Callable[[FileSelectionRequest], list[str]], +) -> None: + """Run ``choose`` on the dialog thread instead of Tk.""" + monkeypatch.setattr(native_window_module, "_choose_paths", choose) def _wait_for_input_events(window: NativeWindowClientWindow) -> list[object]: @@ -1010,7 +1012,7 @@ def test_native_window_reports_a_selected_file( ) -> None: chosen = tmp_path / "seed.png" chosen.write_bytes(b"png-bytes") - _fake_dialog(monkeypatch, f"print({json.dumps([str(chosen)])!r})") + _fake_dialog(monkeypatch, lambda _: [str(chosen)]) presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1032,7 +1034,7 @@ def test_native_window_reports_a_selected_file( def test_native_window_reports_cancelled_file_selection( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - _fake_dialog(monkeypatch, "print('[]')") + _fake_dialog(monkeypatch, lambda _: []) presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1053,7 +1055,10 @@ def test_native_window_reports_cancelled_file_selection( def test_native_window_reports_unavailable_when_picker_raises( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - _fake_dialog(monkeypatch, "raise SystemExit('Tk cannot open a display.')") + def no_display(_: FileSelectionRequest) -> list[str]: + raise RuntimeError("Tk cannot open a display.") + + _fake_dialog(monkeypatch, no_display) presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1115,27 +1120,6 @@ def test_selected_files_from_paths_rejects_a_non_regular_file(tmp_path: Path) -> assert status is SelectedFilesStatus.UNAVAILABLE -def test_native_window_shown_picker_has_no_deadline( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - _fake_dialog(monkeypatch, "import time; time.sleep(0.5); print('[]')") - now = [0.0] - presenter = _Presenter() - window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) - window._file_selector_clock = lambda: now[0] - window.open(_session_desc()) - window.request_file_selector(_file_request("open-1", str(tmp_path))) - now[0] = 1_000.0 - assert window.get_user_input_events().get_events() == [] - events = _wait_for_input_events(window) - window.close() - - assert len(events) == 1 - event = events[0] - assert isinstance(event, SelectedFilesUserInputEvent) - assert event.status is SelectedFilesStatus.CANCELLED - - def test_native_window_reports_multiple_selected_files( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -1143,7 +1127,7 @@ def test_native_window_reports_multiple_selected_files( second = tmp_path / "b.bin" first.write_bytes(b"one") second.write_bytes(b"two") - _fake_dialog(monkeypatch, f"print({json.dumps([str(first), str(second)])!r})") + _fake_dialog(monkeypatch, lambda _: [str(first), str(second)]) presenter = _Presenter() window = NativeWindowClientWindow( presenter_factory=_presenter_factory(presenter), @@ -1161,42 +1145,39 @@ def test_native_window_reports_multiple_selected_files( assert [chosen.data for chosen in event.files] == [b"one", b"two"] -def test_native_window_reads_a_path_list_larger_than_a_pipe( +def test_native_window_drops_a_flushed_dialog_and_is_busy_until_it_closes( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - _fake_dialog( - monkeypatch, "import json; print(json.dumps(['/' + 'x' * 1000] * 200))" - ) - presenter = _Presenter() - window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) - window.open(_session_desc()) - window.request_file_selector(_file_request("open-1", str(tmp_path), multiple=True)) - events = _wait_for_input_events(window) - window.close() - - assert len(events) == 1 - event = events[0] - assert isinstance(event, SelectedFilesUserInputEvent) - assert event.status is SelectedFilesStatus.UNAVAILABLE + chosen = tmp_path / "stale.png" + chosen.write_bytes(b"png") + user_closes = threading.Event() + def choose(_: FileSelectionRequest) -> list[str]: + user_closes.wait(5) + return [str(chosen)] -def test_native_window_flush_closes_the_open_dialog( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - _fake_dialog(monkeypatch, "import time; time.sleep(60)") + _fake_dialog(monkeypatch, choose) presenter = _Presenter() window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) window.open(_session_desc()) window.request_file_selector(_file_request("open-1", str(tmp_path))) - dialog = window._file_dialog - assert dialog is not None - window.flush_file_selectors() - assert dialog[2].returncode is not None - assert window.get_user_input_events().get_events() == [] + window.request_file_selector(_file_request("open-2", str(tmp_path))) + busy = window.get_user_input_events().get_events() + user_closes.set() + thread = window._file_dialog_thread + assert thread is not None + thread.join(5) + stale = window.get_user_input_events().get_events() window.close() + assert len(busy) == 1 + assert isinstance(busy[0], SelectedFilesUserInputEvent) + assert busy[0].request_id == "open-2" + assert busy[0].status is SelectedFilesStatus.BUSY + assert stale == [] + def test_native_window_reports_standard_gamepad_events() -> None: presenter = _Presenter() diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index 46c24186d..1ba06e517 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -16,7 +16,6 @@ from flashdreams.runtime_v2.null_client_window import NullClientWindow from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.selected_file import ( - FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS, MAX_SELECTED_FILE_BATCH_BYTES, MAX_SELECTED_FILE_BYTES, MAX_SELECTED_FILE_COUNT, @@ -125,8 +124,6 @@ def test_ui_loop_keeps_file_results_until_a_step_runs() -> None: class _PickerWindow(NullClientWindow): def __init__(self) -> None: super().__init__() - self.now = 0.0 - self._file_selector_clock = lambda: self.now self.started: list[tuple[int, str]] = [] self.cancelled = 0 @@ -205,7 +202,7 @@ def complete( SelectedFilesStatus.TOO_LARGE, ), (png, SelectedFilesStatus.CANCELLED, (a,), SelectedFilesStatus.CANCELLED), - (png, SelectedFilesStatus.BUSY, (), SelectedFilesStatus.UNAVAILABLE), + (png, SelectedFilesStatus.BUSY, (a,), SelectedFilesStatus.BUSY), ] for request, status, files, expected in cases: [event] = complete(request, status, files) @@ -217,15 +214,10 @@ def complete( pathy = ( SelectedFile(name="../outside.png", data=b"1"), - SelectedFile(name="C:\\dir\\in.png", data=b"2"), - SelectedFile(name="..", data=b"3"), + SelectedFile(name="/home/user/in.png", data=b"2"), ) [event] = complete(many, SelectedFilesStatus.OK, pathy) - assert [chosen.name for chosen in event.files] == [ - "outside.png", - "in.png", - "upload", - ] + assert [chosen.name for chosen in event.files] == ["outside.png", "in.png"] def test_window_without_put_user_input_event_cannot_be_created() -> None: @@ -246,18 +238,6 @@ def close(self) -> None: _Incomplete() # type: ignore[abstract] -def test_window_times_out_unshown_picker_and_drops_its_late_result() -> None: - window = _PickerWindow() - window.request_file_selector(_request("open-1")) - window.now = FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS - window._expire_file_selectors() - assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] - assert window.cancelled == 1 - - window._complete_file_selector(window.started[0][0], SelectedFilesStatus.CANCELLED) - assert _statuses(window) == [] - - def test_null_window_open_drops_a_result_left_by_the_previous_session() -> None: window = NullClientWindow() window.request_file_selector(_request("open-1")) @@ -265,24 +245,6 @@ def test_null_window_open_drops_a_result_left_by_the_previous_session() -> None: assert _statuses(window) == [] -def test_window_expires_a_stuck_picker_when_the_next_request_arrives() -> None: - window = _PickerWindow() - window.request_file_selector(_request("open-1")) - window.now = FILE_SELECTION_PRESENTATION_TIMEOUT_SECONDS - window.request_file_selector(_request("open-2")) - assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] - assert [request_id for _, request_id in window.started] == ["open-1", "open-2"] - - -def test_window_shown_picker_waits_for_the_user_without_deadline() -> None: - window = _PickerWindow() - window.request_file_selector(_request("open-1")) - window._file_selector_opened(window.started[0][0]) - window.now = 1_000.0 - window._expire_file_selectors() - assert _statuses(window) == [] - - def test_flush_forgets_the_open_picker_without_a_result() -> None: window = _PickerWindow() window.request_file_selector(_request("open-1")) diff --git a/flashdreams/test_v2/test_session_runner.py b/flashdreams/test_v2/test_session_runner.py index 990ca76b6..3d9f5beb6 100644 --- a/flashdreams/test_v2/test_session_runner.py +++ b/flashdreams/test_v2/test_session_runner.py @@ -682,11 +682,13 @@ def request_lock_cursor_to_window(self, lock_cursor_to_window: bool) -> None: self._log.record("window.request_lock_cursor_to_window") self.cursor_requests.append(("lock", lock_cursor_to_window)) - def request_file_selector(self, request: FileSelectionRequest) -> None: + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: """Record one file-selector request and answer with the default.""" - self._log.record("window.request_file_selector") + self._log.record("window._request_file_selector") self.file_selection_requests.append(request) - super().request_file_selector(request) + super()._request_file_selector(selector_id, request) def _put_user_input_event(self, event: UserInputEvent) -> None: with self._lock: @@ -1419,7 +1421,7 @@ def open_selector(_: object) -> None: max_bytes=1024, ) ] - assert "window.request_file_selector" in log.calls + assert "window._request_file_selector" in log.calls def test_window_requests_skip_a_file_selector_for_a_replaced_session() -> None: diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 137394d90..b13948fcc 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -556,41 +556,6 @@ async def test_file_upload_prefers_disallowed_suffix_over_body_limit() -> None: window.close() -@pytest.mark.asyncio -async def test_browser_ack_reports_the_file_selector_opened() -> None: - window = WebRTCClientWindow() - window.open(_session_desc()) - peer: RTCPeerConnection | None = None - try: - peer, channel, _ = await _connect_browser(window) - window.request_file_selector(_file_request("open-1")) - wire_id = await _wait_file_armed(window) - channel.send(json.dumps({"type": "file_selector_ack", "id": "other"})) - channel.send(json.dumps({"type": "file_selector_ack", "id": wire_id})) - for _ in range(100): - assert window.get_user_input_events().get_events() == [] - if window._file_selector_deadline is None: - break - await asyncio.sleep(0.01) - assert window._file_selector_deadline is None - channel.send( - json.dumps( - { - "type": "file_selector_result", - "id": wire_id, - "status": "cancelled", - } - ) - ) - event = await _wait_selected_files_event(window) - assert event.status is SelectedFilesStatus.CANCELLED - assert event.files == () - finally: - if peer is not None: - await peer.close() - window.close() - - @pytest.mark.asyncio async def test_browser_file_selector_result_reports_status() -> None: window = WebRTCClientWindow() @@ -676,7 +641,6 @@ async def test_browser_script_checks_type_and_size_before_upload() -> None: assert '"disallowed_type"' in script assert "fetch(`/api/files?id=${encodeURIComponent(requestId)}`" in script assert "input.multiple" in script - assert "file_selector_ack" in script assert "event.stopImmediatePropagation();" in script assert "(promptDialog !== null && promptDialog.open)" in script finally: diff --git a/integrations_v2/imgui_ui_demo/README.md b/integrations_v2/imgui_ui_demo/README.md index 3ca20bdb9..7f1a7a27c 100644 --- a/integrations_v2/imgui_ui_demo/README.md +++ b/integrations_v2/imgui_ui_demo/README.md @@ -12,9 +12,8 @@ Small ImGui applications for the FlashDreams v2 loop runtime: name and size on one status line per button. Each Open button calls `request_file_selector` with its own `request_id`, and the result event carrying that id updates the button's line. One picker is open at a time; - a click on any button while it is open shows `BUSY` on that button's line. Dropping the WebRTC tab completes in-flight picks as unavailable. A picker - the host never shows within 30 seconds also completes as unavailable; a - picker that is shown stays open as long as the user wants. + a click on any button while it is open shows `BUSY` on that button's line. + Dropping the WebRTC tab completes in-flight picks as unavailable. - `imgui-ui-query-string` sets background from a `?(r,g,b)` browser URL. - `imgui-ui-window-size` resizes native window or UI render target without resetting session. diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py index c39767795..35a3a4bb6 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py @@ -4,6 +4,7 @@ """ImGui file-picker application for the v2 loop runtime.""" import math +import time from collections.abc import Sequence from dataclasses import dataclass, field from typing import Any @@ -51,8 +52,7 @@ ), SelectedFilesStatus.UNAVAILABLE: ( "SelectedFilesStatus.UNAVAILABLE: picker could not complete " - "(no or gone client, picker not shown in time, failed chooser, or " - "unreadable file)." + "(no or gone client, failed chooser, or unreadable file)." ), } _FILE_SELECTION_ERROR_COLOR = (1.0, 0.25, 0.25, 1.0) @@ -86,6 +86,9 @@ _NO_FILE_SELECTED = "No file selected." """Status shown for a button before its first result.""" +_RESTART_DELAY_S = 5.0 +"""Seconds before a requested restart, long enough to open a picker first.""" + class BreathingBackgroundModelLoop(IModelLoop[tuple[SessionDesc, torch.device | str]]): """Generate a black-to-gray breathing background for the file-picker demo.""" @@ -124,6 +127,9 @@ class FilePickerState: statuses: dict[str, str] = field(default_factory=dict) """Status text by ``request_id``. A missing id has no result yet.""" + restart_at: float | None = None + """``time.monotonic()`` deadline for a requested session restart.""" + def _status_text(event: SelectedFilesUserInputEvent) -> str: """Return the overlay text for one selector result.""" @@ -160,6 +166,14 @@ def step_ui( max_file_bytes=MAX_SELECTED_FILE_BYTES, multiple=multiple, ) + if self.state.restart_at is None: + if imgui.button(f"Restart session in {_RESTART_DELAY_S:.0f}s"): + self.state.restart_at = time.monotonic() + _RESTART_DELAY_S + else: + remaining = self.state.restart_at - time.monotonic() + if remaining <= 0: + self.request_new_session(self.session_desc) + imgui.text(f"Restarting in {max(remaining, 0.0):.1f}s...") for label, request_id, _ in _FILE_SELECTOR_SLOTS: status = self.state.statuses.get(request_id, _NO_FILE_SELECTED) line = f"{label}: {status}" From 5ca507ef848776ce01961684f32b3c1c9cd4ca26 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Mon, 19 Oct 2026 12:39:59 -0700 Subject: [PATCH 06/12] refactor(api_v2): drop the file-result carryover in _finish_run Restore main's behavior: input read by a run with no step is dropped. Signed-off-by: Set Paing --- flashdreams/flashdreams/api_v2/loop.py | 13 +++---------- flashdreams/test_v2/test_selected_files.py | 18 ------------------ 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 8b55b5dfd..257e865f8 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -33,7 +33,6 @@ from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( CloseUserInputEvent, - SelectedFilesUserInputEvent, UserInputEvent, ) from flashdreams.runtime_v2.user_input_events import UserInputEvents @@ -265,16 +264,10 @@ def _finish_run( """ if step_completed: self.latest_result = result - self._pending_user_events.clear() - else: - # Input that reaches a run with no step is dropped, except a file - # result: it answers a request and arrives once. - self._pending_user_events = [ - event - for event in self._pending_user_events - if isinstance(event, SelectedFilesUserInputEvent) - ] self._step_index += 1 + # Input read by a run with no step is dropped, such as an ON_DEMAND UI + # tick without a new frame. + self._pending_user_events.clear() @final def _shutdown(self) -> None: diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index 1ba06e517..f1899943b 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -103,24 +103,6 @@ def test_ui_loop_forwards_every_file_selection_of_a_tick() -> None: ] -def test_ui_loop_keeps_file_results_until_a_step_runs() -> None: - loop = _ui_loop() - picked = SelectedFilesUserInputEvent( - timestamp=uint64(0), - request_id="open-1", - status=SelectedFilesStatus.CANCELLED, - ) - - loop._begin_run(UserInputEvents([picked]), generation=0) - loop._finish_run(None, step_completed=False) - loop._begin_run(UserInputEvents([]), generation=0) - assert loop.user_events.get_events() == [picked] - - loop._finish_run([], step_completed=True) - loop._begin_run(UserInputEvents([]), generation=0) - assert loop.user_events.get_events() == [] - - class _PickerWindow(NullClientWindow): def __init__(self) -> None: super().__init__() From dd3932c9d3756cafa5a039b9bdc4b1985e50b007 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Mon, 19 Oct 2026 17:30:23 -0700 Subject: [PATCH 07/12] Make file selection limits app-declared defaults request_file_selector takes max_file_bytes, max_total_bytes and max_file_count, defaulting to 1 GiB, 1 GiB and 256, with no hard ceiling. max_file_bytes must not exceed max_total_bytes. The WebRTC server enforces the app's limits on its side and bounds the request body up front. Signed-off-by: Set Paing --- .../flashdreams/api_v2/client_window.py | 10 +- flashdreams/flashdreams/api_v2/loop.py | 52 +++++--- .../runtime_v2/native_window_client_window.py | 37 ++---- .../flashdreams/runtime_v2/selected_file.py | 123 ++++++++---------- .../flashdreams/runtime_v2/serving/web/app.js | 2 +- .../runtime_v2/serving/webrtc_server.py | 80 ++++++------ .../test_native_window_client_window.py | 29 +++-- flashdreams/test_v2/test_selected_files.py | 121 +++++++++-------- flashdreams/test_v2/test_session_runner.py | 2 +- .../test_v2/test_webrtc_client_window.py | 96 ++++++++------ integrations_v2/imgui_ui_demo/README.md | 4 +- .../imgui_ui_demo/file_picker_app.py | 20 +-- .../imgui_ui_demo/tests/test_imgui_ui_demo.py | 9 +- 13 files changed, 299 insertions(+), 286 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index e925cbeea..88efa2d4c 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -17,7 +17,6 @@ FileSelectionRequest, SelectedFile, SelectedFilesStatus, - clamp_selected_file_max_bytes, selected_files_policy_status, ) from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent @@ -203,9 +202,8 @@ def _cancel_active_file_selectors(self) -> None: def _put_user_input_event(self, event: UserInputEvent) -> None: """Queue ``event`` for :meth:`InputSource.get_user_input_events`. - Required even for a window without a picker, which answers every - request through it. Only this class calls it, on the UI thread, and it - must not call back into file selection. + Each window decides how an event from outside its own input polling + joins its input stream. Only this class calls it, on the UI thread. """ @final @@ -256,9 +254,7 @@ def _check_file_selection_result( return SelectedFilesStatus.UNAVAILABLE, () files = tuple(replace(chosen, name=Path(chosen.name).name) for chosen in files) rejected = selected_files_policy_status( - [(chosen.name, len(chosen.data)) for chosen in files], - accept=request.accept, - max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + [(chosen.name, len(chosen.data)) for chosen in files], request ) if rejected is not None: return rejected, () diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 257e865f8..4564d3f80 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -27,6 +27,9 @@ unbind_inference_profiler, ) from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_FILE_COUNT, + DEFAULT_MAX_TOTAL_BYTES, FileSelectionRequest, normalize_selected_file_accept, ) @@ -621,7 +624,9 @@ def request_file_selector( initial_path: str | None = None, *, accept: Sequence[str] = (), - max_file_bytes: int | None = None, + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, + max_file_count: int = DEFAULT_MAX_FILE_COUNT, multiple: bool = False, ) -> None: """Ask the client to show a file selector. @@ -639,7 +644,8 @@ def request_file_selector( - ``CANCELLED``: the user dismissed the selector. - ``BUSY``: another selector was open. - ``DISALLOWED_TYPE``: a file did not match ``accept``. - - ``TOO_LARGE``: a file or the set exceeded the size or count limits. + - ``TOO_LARGE``: the set broke ``max_file_bytes``, ``max_total_bytes``, + or ``max_file_count``. - ``UNAVAILABLE``: the window cannot show a selector, or it failed. ``accept`` checks filename suffixes only, so treat ``data`` as @@ -672,20 +678,20 @@ def request_file_selector( initial_path: Directory hint for the native selector; ``None`` uses the host user's home directory. The browser ignores it. accept: Filename suffixes such as ``.png``. Empty allows any type. - max_file_bytes: Maximum size of each file in bytes, or ``None`` for - the ceiling. Larger values are clamped to - :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_BYTES` - without notice. - multiple: Whether the selector may return more than one file. The - whole set is also capped by - :data:`~flashdreams.runtime_v2.selected_file.MAX_SELECTED_FILE_BATCH_BYTES`. + max_file_bytes: Largest single file in bytes. Must not exceed + ``max_total_bytes``. + max_total_bytes: Most bytes all chosen files may add up to. Has no + effect unless ``multiple`` is set. + max_file_count: Most files the selector may return. Has no effect + unless ``multiple`` is set. + multiple: Whether the selector may return more than one file. Raises: TypeError: ``request_id`` or ``initial_path`` is not a string, or - ``accept`` / ``max_file_bytes`` / ``multiple`` has the wrong - type. - ValueError: ``request_id`` or ``initial_path`` is empty, or - ``accept`` / ``max_file_bytes`` is invalid. + ``accept``, a limit, or ``multiple`` has the wrong type. + ValueError: ``request_id`` or ``initial_path`` is empty, ``accept`` + is invalid, a limit is not positive, or ``max_file_bytes`` + exceeds ``max_total_bytes``. """ if not isinstance(request_id, str): raise TypeError("request_id must be a string.") @@ -700,11 +706,17 @@ def request_file_selector( raise ValueError("initial_path must be a non-empty string.") resolved_initial_path = initial_path suffixes = normalize_selected_file_accept(accept) - if max_file_bytes is not None: - if isinstance(max_file_bytes, bool) or not isinstance(max_file_bytes, int): - raise TypeError("max_file_bytes must be an integer.") - if max_file_bytes <= 0: - raise ValueError("max_file_bytes must be > 0.") + for name, limit in ( + ("max_file_bytes", max_file_bytes), + ("max_total_bytes", max_total_bytes), + ("max_file_count", max_file_count), + ): + if isinstance(limit, bool) or not isinstance(limit, int): + raise TypeError(f"{name} must be an integer.") + if limit <= 0: + raise ValueError(f"{name} must be > 0.") + if max_file_bytes > max_total_bytes: + raise ValueError("max_file_bytes must not exceed max_total_bytes.") if not isinstance(multiple, bool): raise TypeError("multiple must be a bool.") self.get_or_create_ui_loop_requests().file_selections.append( @@ -712,7 +724,9 @@ def request_file_selector( request_id=request_id, initial_path=resolved_initial_path, accept=suffixes, - max_bytes=max_file_bytes, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + max_file_count=max_file_count, multiple=multiple, ) ) diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index 9a90c519c..ebceede33 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -25,11 +25,9 @@ from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.selected_file import ( - MAX_SELECTED_FILE_BYTES, FileSelectionRequest, SelectedFile, SelectedFilesStatus, - clamp_selected_file_max_bytes, selected_files_policy_status, ) from flashdreams.runtime_v2.session_desc import SessionDesc @@ -85,17 +83,11 @@ def _choose_paths(request: FileSelectionRequest) -> list[str]: def _selected_files_from_paths( - paths: Sequence[str], - *, - accept: tuple[str, ...] = (), - max_bytes: int | None = None, + paths: Sequence[str], request: FileSelectionRequest ) -> tuple[tuple[SelectedFile, ...], SelectedFilesStatus]: """Load chosen files, or empty files and a failure status.""" - budget = MAX_SELECTED_FILE_BYTES if max_bytes is None else max_bytes rejected = selected_files_policy_status( - [(Path(path).name, None) for path in paths], - accept=accept, - max_bytes=budget, + [(Path(path).name, None) for path in paths], request ) if rejected is not None: return (), rejected @@ -109,9 +101,7 @@ def _selected_files_from_paths( return (), SelectedFilesStatus.UNAVAILABLE sized.append((path, info.st_size)) rejected = selected_files_policy_status( - [(Path(path).name, size) for path, size in sized], - accept=accept, - max_bytes=budget, + [(Path(path).name, size) for path, size in sized], request ) if rejected is not None: return (), rejected @@ -119,17 +109,12 @@ def _selected_files_from_paths( for path, _size in sized: try: with open(path, "rb") as stream: - data = stream.read(budget + 1) + # Read one byte past the limit so a file over it is + # rejected, not truncated. + data = stream.read(request.max_file_bytes + 1) except OSError: return (), SelectedFilesStatus.UNAVAILABLE files.append(SelectedFile(name=Path(path).name, data=data)) - rejected = selected_files_policy_status( - [(chosen.name, len(chosen.data)) for chosen in files], - accept=accept, - max_bytes=budget, - ) - if rejected is not None: - return (), rejected return tuple(files), SelectedFilesStatus.OK @@ -325,13 +310,11 @@ def _poll_file_dialog(self) -> None: elif not paths: status = SelectedFilesStatus.CANCELLED else: - # ponytail: files are read on the UI thread, up to the batch - # limit, which can stall presentation briefly. Upgrade path: - # read them on the dialog thread. + # ponytail: files are read on the UI thread, up to the + # request's max_total_bytes, which stalls presentation while + # they load. Upgrade path: read them on the dialog thread. files, status = _selected_files_from_paths( - paths if request.multiple else paths[:1], - accept=request.accept, - max_bytes=clamp_selected_file_max_bytes(request.max_bytes), + paths if request.multiple else paths[:1], request ) self._complete_file_selector(selector_id, status, files) diff --git a/flashdreams/flashdreams/runtime_v2/selected_file.py b/flashdreams/flashdreams/runtime_v2/selected_file.py index 37c74b5b6..667636537 100644 --- a/flashdreams/flashdreams/runtime_v2/selected_file.py +++ b/flashdreams/flashdreams/runtime_v2/selected_file.py @@ -7,38 +7,14 @@ from dataclasses import dataclass from enum import Enum -MAX_SELECTED_FILE_BYTES = 32 * 1024 * 1024 -"""Hard ceiling for one selected file. Windows clamp requested budgets to this.""" +DEFAULT_MAX_FILE_BYTES = 1024 * 1024 * 1024 +"""Default ``max_file_bytes`` (1 GiB): the largest single file a request accepts.""" -# ponytail: 8× the per-file ceiling is the batch cap; raise it or -# make it per-request if an app needs a larger set in one pick. -MAX_SELECTED_FILE_BATCH_BYTES = 8 * MAX_SELECTED_FILE_BYTES -"""Hard ceiling for the total bytes of one selected-files request.""" +DEFAULT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024 +"""Default ``max_total_bytes`` (1 GiB): the most bytes all chosen files add up to.""" -MAX_SELECTED_FILE_COUNT = 256 -"""Hard ceiling for the number of files in one selected-files request.""" - - -def clamp_selected_file_max_bytes(max_bytes: int | None) -> int: - """Return a file-size budget that cannot exceed ``MAX_SELECTED_FILE_BYTES``. - - Args: - max_bytes: Requested budget in bytes, or ``None`` for the ceiling. - - Returns: - The requested budget when it is below the ceiling, otherwise the ceiling. - - Raises: - TypeError: ``max_bytes`` is not ``None`` or an integer. - ValueError: ``max_bytes`` is not positive. - """ - if max_bytes is None: - return MAX_SELECTED_FILE_BYTES - if isinstance(max_bytes, bool) or not isinstance(max_bytes, int): - raise TypeError("max_bytes must be an integer.") - if max_bytes <= 0: - raise ValueError("max_bytes must be > 0.") - return min(max_bytes, MAX_SELECTED_FILE_BYTES) +DEFAULT_MAX_FILE_COUNT = 256 +"""Default ``max_file_count``: the most files one request may return.""" def normalize_selected_file_accept(accept: Sequence[str] = ()) -> tuple[str, ...]: @@ -94,8 +70,8 @@ class SelectedFilesStatus(Enum): """The user dismissed the selector.""" TOO_LARGE = "too_large" - """A file exceeded its size budget, or the set exceeded the batch size or - file-count ceiling.""" + """The set broke the request's ``max_file_bytes``, ``max_total_bytes``, or + ``max_file_count``.""" DISALLOWED_TYPE = "disallowed_type" """A chosen file did not match ``accept``. Checked before size.""" @@ -105,44 +81,8 @@ class SelectedFilesStatus(Enum): open selector keeps going and answers its own request.""" UNAVAILABLE = "unavailable" - """No pick was made: the client has no picker or left, the picker was not - shown in time, or a chosen file was unreadable.""" - - -def selected_files_policy_status( - files: Sequence[tuple[str, int | None]], - *, - accept: tuple[str, ...] = (), - max_bytes: int, -) -> SelectedFilesStatus | None: - """Return why a chosen set is rejected, or ``None`` if every file is allowed. - - Type is checked before size and count, across the whole set. - - Args: - files: ``(name, size)`` pairs. ``size`` of ``None`` skips that file's - size check and omits it from the batch total. - accept: Filename suffixes such as ``.png``. Empty allows any type. - max_bytes: Maximum allowed size of each file in bytes. - - Returns: - ``DISALLOWED_TYPE``, ``TOO_LARGE``, or ``None`` when the set passes. - """ - for name, _size in files: - if not selected_file_suffix_allowed(name, accept): - return SelectedFilesStatus.DISALLOWED_TYPE - if len(files) > MAX_SELECTED_FILE_COUNT: - return SelectedFilesStatus.TOO_LARGE - total = 0 - for _name, size in files: - if size is None: - continue - if size > max_bytes: - return SelectedFilesStatus.TOO_LARGE - total += size - if total > MAX_SELECTED_FILE_BATCH_BYTES: - return SelectedFilesStatus.TOO_LARGE - return None + """No pick was made: the client has no picker or left, the picker failed, + or a chosen file was unreadable.""" @dataclass(frozen=True, slots=True) @@ -169,8 +109,47 @@ class FileSelectionRequest: accept: tuple[str, ...] = () """Filename suffixes the client may choose, empty for any type.""" - max_bytes: int | None = None - """Requested size budget per file in bytes, or ``None`` for the ceiling.""" + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES + """Largest single file in bytes.""" + + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES + """Most bytes all chosen files may add up to.""" + + max_file_count: int = DEFAULT_MAX_FILE_COUNT + """Most files the selector may return.""" multiple: bool = False """Whether the selector may return more than one file.""" + + +def selected_files_policy_status( + files: Sequence[tuple[str, int | None]], + request: FileSelectionRequest, +) -> SelectedFilesStatus | None: + """Return why a chosen set breaks ``request``, or ``None`` if it passes. + + Type is checked before size and count, across the whole set. + + Args: + files: ``(name, size)`` pairs. ``size`` of ``None`` skips that file's + size check and omits it from the total. + request: The request whose ``accept`` and limits apply. + + Returns: + ``DISALLOWED_TYPE``, ``TOO_LARGE``, or ``None`` when the set passes. + """ + for name, _size in files: + if not selected_file_suffix_allowed(name, request.accept): + return SelectedFilesStatus.DISALLOWED_TYPE + if len(files) > request.max_file_count: + return SelectedFilesStatus.TOO_LARGE + total = 0 + for _name, size in files: + if size is None: + continue + if size > request.max_file_bytes: + return SelectedFilesStatus.TOO_LARGE + total += size + if total > request.max_total_bytes: + return SelectedFilesStatus.TOO_LARGE + return None diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 49a17ed37..774f1e19e 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -343,7 +343,7 @@ const openFileSelector = options => { hideFilePicker(); } const accept = Array.isArray(options.accept) ? options.accept : []; - const maxBytes = Number(options.max_bytes); + const maxBytes = Number(options.max_file_bytes); pendingFileRequest = { id: requestId, accept, diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index bddb1f83b..fc6cf31a2 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -32,13 +32,9 @@ from flashdreams.api_v2.client_window import ClientWindowLink from flashdreams.runtime_v2.cuda_utils import resolve_cuda_device from flashdreams.runtime_v2.selected_file import ( - MAX_SELECTED_FILE_BATCH_BYTES, - MAX_SELECTED_FILE_COUNT, FileSelectionRequest, SelectedFile, SelectedFilesStatus, - clamp_selected_file_max_bytes, - normalize_selected_file_accept, selected_file_suffix_allowed, selected_files_policy_status, ) @@ -310,19 +306,13 @@ async def _next_queued_frame(self) -> _QueuedRGBFrame: @dataclass(frozen=True, slots=True) class _PendingFileSelection: - """Accept suffixes and size budget for one armed file-selector request.""" + """One armed file-selector request and the upload rules it sets.""" selector_id: str """Window selector this upload answers. Never sent to the browser.""" - accept: tuple[str, ...] - """Filename suffixes the client may upload, empty for any type.""" - - max_bytes: int - """Maximum file bytes, already clamped to the window ceiling.""" - - multiple: bool - """Whether this request may include more than one file.""" + request: FileSelectionRequest + """The app's request, whose ``accept`` and limits the upload must meet.""" class WebRTCServer: @@ -543,12 +533,7 @@ def request_file_selector( selector_id: Window selector reported back with this result. request: File selector to show. """ - policy = _PendingFileSelection( - selector_id=selector_id, - accept=normalize_selected_file_accept(request.accept), - max_bytes=clamp_selected_file_max_bytes(request.max_bytes), - multiple=request.multiple, - ) + policy = _PendingFileSelection(selector_id=selector_id, request=request) loop = self._loop if loop is None: raise RuntimeError("WebRTC server is not running.") @@ -592,9 +577,9 @@ def _arm_file_selector(self, policy: _PendingFileSelection) -> None: { "type": "file_selector", "id": token, - "accept": list(policy.accept), - "max_bytes": policy.max_bytes, - "multiple": policy.multiple, + "accept": list(policy.request.accept), + "max_file_bytes": policy.request.max_file_bytes, + "multiple": policy.request.multiple, } ) ) @@ -918,9 +903,7 @@ def _run_server(self) -> None: async def _start_server(self) -> None: """Create and bind the standalone aiohttp application.""" self._offer_lock = asyncio.Lock() - app = web.Application( - client_max_size=MAX_SELECTED_FILE_BATCH_BYTES + 1024 * 1024 - ) + app = web.Application() app.router.add_get("/", self._serve_browser) app.router.add_get("/app.js", self._serve_browser_script) app.router.add_get("/healthz", self._health) @@ -992,11 +975,21 @@ def emit( if not request.content_type.startswith("multipart/"): emit(SelectedFilesStatus.UNAVAILABLE) raise web.HTTPBadRequest(reason="File upload requires a file.") - try: - reader = await request.multipart() - except web.HTTPRequestEntityTooLarge: + limits = policy.request + # The body bound keeps the whole read finite, even for parts the + # loop below only skips. + body_limit = limits.max_total_bytes + _upload_overhead_bytes( + limits.max_file_count + ) + if request.content_length is None: + emit(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPLengthRequired() + if request.content_length > body_limit: emit(SelectedFilesStatus.TOO_LARGE) - raise + raise web.HTTPRequestEntityTooLarge( + max_size=body_limit, actual_size=request.content_length + ) + reader = await request.multipart() chosen: list[tuple[str, bytes]] = [] file_count = 0 total_bytes = 0 @@ -1010,18 +1003,18 @@ def emit( if part.name != "file": await part.release() continue - if not policy.multiple and (chosen or too_large): + if not limits.multiple and (chosen or too_large): await part.release() continue if not _identity_encoded(part): emit(SelectedFilesStatus.UNAVAILABLE) raise web.HTTPBadRequest(reason="File parts must not be encoded.") file_count += 1 - if file_count > MAX_SELECTED_FILE_COUNT: + if file_count > limits.max_file_count: too_large = True chosen.clear() name = part.filename or "upload" - if not selected_file_suffix_allowed(name, policy.accept): + if not selected_file_suffix_allowed(name, limits.accept): emit(SelectedFilesStatus.DISALLOWED_TYPE) await _drain_multipart_reader(reader, part) return web.Response(status=204) @@ -1029,10 +1022,12 @@ def emit( # Keep reading names so a disallowed type still wins. await part.release() continue - data = await _read_upload_part_bounded(part, max_bytes=policy.max_bytes) + data = await _read_upload_part_bounded( + part, max_bytes=limits.max_file_bytes + ) if data is not None: total_bytes += len(data) - if data is None or total_bytes > MAX_SELECTED_FILE_BATCH_BYTES: + if data is None or total_bytes > limits.max_total_bytes: too_large = True chosen.clear() continue @@ -1040,21 +1035,18 @@ def emit( if too_large: emit(SelectedFilesStatus.TOO_LARGE) raise web.HTTPRequestEntityTooLarge( - max_size=policy.max_bytes, actual_size=policy.max_bytes + 1 + max_size=limits.max_total_bytes, actual_size=total_bytes + 1 ) if not chosen: emit(SelectedFilesStatus.UNAVAILABLE) raise web.HTTPBadRequest(reason="File upload requires a file.") rejected = selected_files_policy_status( - [(name, len(data)) for name, data in chosen], - accept=policy.accept, - max_bytes=policy.max_bytes, + [(name, len(data)) for name, data in chosen], limits ) if rejected is SelectedFilesStatus.TOO_LARGE: emit(rejected) raise web.HTTPRequestEntityTooLarge( - max_size=policy.max_bytes, - actual_size=max(len(data) for _name, data in chosen), + max_size=limits.max_total_bytes, actual_size=total_bytes ) if rejected is not None: emit(rejected) @@ -1566,6 +1558,14 @@ async def _shutdown(self) -> None: """Bytes read per multipart chunk while enforcing ``max_bytes``.""" +def _upload_overhead_bytes(max_file_count: int) -> int: + """Return the multipart framing an upload of ``max_file_count`` files may add.""" + # ponytail: a flat 4 KiB of boundary and headers per file plus 64 KiB; a + # client sending longer headers is answered TOO_LARGE. Upgrade path: bound + # the bytes of skipped parts while streaming instead of the whole body. + return 64 * 1024 + 4 * 1024 * max_file_count + + def _identity_encoded(part: BodyPartReader) -> bool: """Return whether ``part`` carries its bytes as-is, as browsers send them.""" transfer = part.headers.get("Content-Transfer-Encoding", "").lower() diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index 0862c2743..c3dd6023a 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -25,6 +25,7 @@ NativeWindowClientWindow, ) from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, FileSelectionRequest, SelectedFilesStatus, ) @@ -53,14 +54,14 @@ def _file_request( initial_path: str, *, accept: tuple[str, ...] = (), - max_bytes: int | None = None, + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, multiple: bool = False, ) -> FileSelectionRequest: return FileSelectionRequest( request_id=request_id, initial_path=initial_path, accept=accept, - max_bytes=max_bytes, + max_file_bytes=max_file_bytes, multiple=multiple, ) @@ -1079,32 +1080,32 @@ def no_display(_: FileSelectionRequest) -> list[str]: def test_selected_files_from_paths_skips_unreadable_and_oversize( tmp_path: Path, ) -> None: - missing = tmp_path / "gone.png" - files, status = native_window_module._selected_files_from_paths((str(missing),)) + def load( + path: Path, **limits: Any + ) -> tuple[tuple[object, ...], SelectedFilesStatus]: + return native_window_module._selected_files_from_paths( + (str(path),), _file_request("open-1", str(tmp_path), **limits) + ) + + files, status = load(tmp_path / "gone.png") assert files == () assert status is SelectedFilesStatus.UNAVAILABLE too_big = tmp_path / "big.bin" too_big.write_bytes(b"abcd") - files, status = native_window_module._selected_files_from_paths( - (str(too_big),), max_bytes=3 - ) + files, status = load(too_big, max_file_bytes=3) assert files == () assert status is SelectedFilesStatus.TOO_LARGE wrong_type = tmp_path / "notes.txt" wrong_type.write_bytes(b"hi") - files, status = native_window_module._selected_files_from_paths( - (str(wrong_type),), accept=(".png",) - ) + files, status = load(wrong_type, accept=(".png",)) assert files == () assert status is SelectedFilesStatus.DISALLOWED_TYPE too_big_wrong_type = tmp_path / "huge.txt" too_big_wrong_type.write_bytes(b"abcd") - files, status = native_window_module._selected_files_from_paths( - (str(too_big_wrong_type),), accept=(".png",), max_bytes=1 - ) + files, status = load(too_big_wrong_type, accept=(".png",), max_file_bytes=1) assert files == () assert status is SelectedFilesStatus.DISALLOWED_TYPE @@ -1114,7 +1115,7 @@ def test_selected_files_from_paths_rejects_a_non_regular_file(tmp_path: Path) -> endless = tmp_path / "endless.bin" endless.symlink_to("/dev/zero") files, status = native_window_module._selected_files_from_paths( - (str(endless),), accept=(".bin",) + (str(endless),), _file_request("open-1", str(tmp_path), accept=(".bin",)) ) assert files == () assert status is SelectedFilesStatus.UNAVAILABLE diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index f1899943b..08d472d48 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -6,6 +6,7 @@ import queue import threading from pathlib import Path +from typing import Any import pytest from numpy import uint64 @@ -16,13 +17,12 @@ from flashdreams.runtime_v2.null_client_window import NullClientWindow from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.selected_file import ( - MAX_SELECTED_FILE_BATCH_BYTES, - MAX_SELECTED_FILE_BYTES, - MAX_SELECTED_FILE_COUNT, + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_FILE_COUNT, + DEFAULT_MAX_TOTAL_BYTES, FileSelectionRequest, SelectedFile, SelectedFilesStatus, - clamp_selected_file_max_bytes, normalize_selected_file_accept, selected_file_suffix_allowed, selected_files_policy_status, @@ -61,24 +61,61 @@ def _ui_loop() -> _UILoop: return loop -def test_ui_loop_queues_file_selection_policy_without_clamping() -> None: +def test_ui_loop_queues_the_app_declared_limits() -> None: loop = _ui_loop() loop.request_file_selector( "open-1", "/tmp", accept=(".png", ".JPG"), - max_file_bytes=MAX_SELECTED_FILE_BYTES * 2, + max_file_bytes=DEFAULT_MAX_FILE_BYTES * 4, + max_total_bytes=DEFAULT_MAX_TOTAL_BYTES * 4, + max_file_count=DEFAULT_MAX_FILE_COUNT * 4, multiple=True, ) + loop.request_file_selector("open-2", "/tmp") requests = loop.flush_ui_loop_requests() assert requests is not None - [selection] = requests.file_selections - assert selection.request_id == "open-1" - assert selection.initial_path == "/tmp" - assert selection.accept == (".png", ".JPG") - assert selection.max_bytes == MAX_SELECTED_FILE_BYTES * 2 - assert selection.multiple is True + raised, defaults = requests.file_selections + assert raised == FileSelectionRequest( + request_id="open-1", + initial_path="/tmp", + accept=(".png", ".JPG"), + max_file_bytes=DEFAULT_MAX_FILE_BYTES * 4, + max_total_bytes=DEFAULT_MAX_TOTAL_BYTES * 4, + max_file_count=DEFAULT_MAX_FILE_COUNT * 4, + multiple=True, + ) + assert ( + defaults.max_file_bytes, + defaults.max_total_bytes, + defaults.max_file_count, + ) == ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_TOTAL_BYTES, + DEFAULT_MAX_FILE_COUNT, + ) + + +@pytest.mark.parametrize( + "limit", ["max_file_bytes", "max_total_bytes", "max_file_count"] +) +def test_ui_loop_rejects_a_non_positive_limit(limit: str) -> None: + loop = _ui_loop() + zero: dict[str, Any] = {limit: 0} + fractional: dict[str, Any] = {limit: 1.5} + with pytest.raises(ValueError, match=f"{limit} must be > 0"): + loop.request_file_selector("open-1", **zero) + with pytest.raises(TypeError, match=f"{limit} must be an integer"): + loop.request_file_selector("open-1", **fractional) + + +def test_ui_loop_rejects_a_file_limit_over_the_total_limit() -> None: + loop = _ui_loop() + with pytest.raises(ValueError, match="max_file_bytes must not exceed"): + loop.request_file_selector("open-1", max_file_bytes=2, max_total_bytes=1) + with pytest.raises(ValueError, match="max_file_bytes must not exceed"): + loop.request_file_selector("open-1", max_file_bytes=DEFAULT_MAX_TOTAL_BYTES + 1) def test_ui_loop_uses_home_when_initial_path_is_omitted() -> None: @@ -164,7 +201,7 @@ def complete( return selected png = FileSelectionRequest( - request_id="open-1", initial_path="/tmp", accept=(".png",), max_bytes=4 + request_id="open-1", initial_path="/tmp", accept=(".png",), max_file_bytes=4 ) a = SelectedFile(name="a.png", data=b"1") b = SelectedFile(name="b.png", data=b"2") @@ -254,53 +291,27 @@ def _request_file_selector( def test_selected_file_policy_helpers() -> None: - assert clamp_selected_file_max_bytes(None) == MAX_SELECTED_FILE_BYTES - assert clamp_selected_file_max_bytes(16) == 16 - assert clamp_selected_file_max_bytes(MAX_SELECTED_FILE_BYTES * 2) == ( - MAX_SELECTED_FILE_BYTES - ) assert normalize_selected_file_accept([".png", ".jpg"]) == (".png", ".jpg") assert selected_file_suffix_allowed("seed.PNG", (".png",)) assert not selected_file_suffix_allowed("seed.txt", (".png",)) assert selected_file_suffix_allowed("seed.txt", ()) - assert ( - selected_files_policy_status( - [("a.bin", 0)] * (MAX_SELECTED_FILE_COUNT + 1), max_bytes=3 - ) - is SelectedFilesStatus.TOO_LARGE - ) - assert ( - selected_files_policy_status( - [("a.bin", 2), ("b.txt", 2)], - accept=(".bin",), - max_bytes=3, - ) - is SelectedFilesStatus.DISALLOWED_TYPE - ) - assert ( - selected_files_policy_status( - [("a.bin", 2), ("b.bin", 99)], - accept=(".bin",), - max_bytes=3, - ) - is SelectedFilesStatus.TOO_LARGE - ) - assert ( - selected_files_policy_status( - [("a.bin", MAX_SELECTED_FILE_BATCH_BYTES), ("b.bin", 1)], - accept=(".bin",), - max_bytes=MAX_SELECTED_FILE_BYTES, - ) - is SelectedFilesStatus.TOO_LARGE - ) - assert ( - selected_files_policy_status( - [("a.bin", 2), ("b.bin", 2)], - accept=(".bin",), - max_bytes=3, - ) - is None + bins = FileSelectionRequest( + request_id="open-1", + initial_path="/tmp", + accept=(".bin",), + max_file_bytes=3, + max_total_bytes=5, + max_file_count=2, ) + cases = [ + ([("a.bin", 0)] * 3, SelectedFilesStatus.TOO_LARGE), + ([("a.bin", 2), ("b.txt", 2)], SelectedFilesStatus.DISALLOWED_TYPE), + ([("a.bin", 2), ("b.bin", 4)], SelectedFilesStatus.TOO_LARGE), + ([("a.bin", 3), ("b.bin", 3)], SelectedFilesStatus.TOO_LARGE), + ([("a.bin", 2), ("b.bin", 3)], None), + ] + for files, expected in cases: + assert selected_files_policy_status(files, bins) is expected def test_ui_loop_rejects_empty_request_id() -> None: diff --git a/flashdreams/test_v2/test_session_runner.py b/flashdreams/test_v2/test_session_runner.py index 3d9f5beb6..301b3cf16 100644 --- a/flashdreams/test_v2/test_session_runner.py +++ b/flashdreams/test_v2/test_session_runner.py @@ -1418,7 +1418,7 @@ def open_selector(_: object) -> None: request_id="open-1", initial_path="/tmp", accept=(".png",), - max_bytes=1024, + max_file_bytes=1024, ) ] assert "window._request_file_selector" in log.calls diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index b13948fcc..eea66c91e 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -10,7 +10,7 @@ import json import threading import time -from collections.abc import Callable +from collections.abc import AsyncIterator, Callable from dataclasses import replace from typing import Any, TypeVar, cast from unittest.mock import AsyncMock, Mock @@ -38,10 +38,9 @@ from flashdreams.api_v2.client_window import ClientWindowLink from flashdreams.runtime_v2.selected_file import ( - MAX_SELECTED_FILE_BYTES, + DEFAULT_MAX_FILE_BYTES, FileSelectionRequest, SelectedFilesStatus, - clamp_selected_file_max_bytes, ) from flashdreams.runtime_v2.serving import webrtc_server from flashdreams.runtime_v2.serving.webrtc_server import _VideoTrack @@ -64,19 +63,10 @@ def _file_request( - request_id: str, - initial_path: str = "/tmp", - *, - accept: tuple[str, ...] = (), - max_bytes: int | None = None, - multiple: bool = False, + request_id: str, initial_path: str = "/tmp", **fields: Any ) -> FileSelectionRequest: return FileSelectionRequest( - request_id=request_id, - initial_path=initial_path, - accept=accept, - max_bytes=max_bytes, - multiple=multiple, + request_id=request_id, initial_path=initial_path, **fields ) @@ -109,10 +99,7 @@ def _arm(window: WebRTCClientWindow, request: FileSelectionRequest) -> str: window._active_file_selector = (selector_id, request) wire_id = f"token-{selector_id}" policy = webrtc_server._PendingFileSelection( - selector_id=str(selector_id), - accept=request.accept, - max_bytes=clamp_selected_file_max_bytes(request.max_bytes), - multiple=request.multiple, + selector_id=str(selector_id), request=request ) _on_server_loop( window, @@ -369,7 +356,7 @@ async def test_upload_without_the_armed_token_is_rejected() -> None: @pytest.mark.asyncio -async def test_browser_file_selector_sends_clamped_policy_without_host_path() -> None: +async def test_browser_file_selector_sends_the_request_without_host_path() -> None: window = WebRTCClientWindow() window.open(_session_desc()) peer: RTCPeerConnection | None = None @@ -387,7 +374,7 @@ def on_message(message: object) -> None: "open-1", "/home/secret", accept=(".png", ".jpg"), - max_bytes=MAX_SELECTED_FILE_BYTES * 2, + max_file_bytes=DEFAULT_MAX_FILE_BYTES * 2, ) ) for _ in range(100): @@ -401,7 +388,7 @@ def on_message(message: object) -> None: assert payload == { "type": "file_selector", "accept": [".png", ".jpg"], - "max_bytes": MAX_SELECTED_FILE_BYTES, + "max_file_bytes": DEFAULT_MAX_FILE_BYTES * 2, "multiple": False, } finally: @@ -478,7 +465,7 @@ async def test_file_upload_rejects_oversize() -> None: window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm(window, _file_request("open-1", max_bytes=1)) + wire_id = _arm(window, _file_request("open-1", max_file_bytes=1)) status = await _post_selected_file(window, wire_id, b"x" * 8192, "big.bin") assert status == 413 event = await _wait_selected_files_event(window) @@ -490,14 +477,12 @@ async def test_file_upload_rejects_oversize() -> None: @pytest.mark.asyncio -async def test_file_upload_stops_once_the_batch_exceeds_its_cap( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(webrtc_server, "MAX_SELECTED_FILE_BATCH_BYTES", 5) +@pytest.mark.parametrize("limits", [{"max_total_bytes": 5}, {"max_file_count": 2}]) +async def test_file_upload_enforces_the_request_limits(limits: dict[str, Any]) -> None: window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm(window, _file_request("open-1", multiple=True)) + wire_id = _arm(window, _file_request("open-1", multiple=True, **limits)) form = FormData() for name in ("a.bin", "b.bin", "c.bin"): form.add_field( @@ -513,15 +498,51 @@ async def test_file_upload_stops_once_the_batch_exceeds_its_cap( window.close() +async def _chunked_junk() -> AsyncIterator[bytes]: + yield b"junk" + + +@pytest.mark.parametrize( + ("body", "status", "selected_status"), + [ + (b"x" * (256 * 1024), 413, SelectedFilesStatus.TOO_LARGE), + (None, 411, SelectedFilesStatus.UNAVAILABLE), + ], +) @pytest.mark.asyncio -async def test_file_upload_over_the_batch_cap_still_reports_a_disallowed_type( - monkeypatch: pytest.MonkeyPatch, +async def test_file_upload_rejects_an_unbounded_body_before_reading_it( + body: bytes | None, status: int, selected_status: SelectedFilesStatus ) -> None: - monkeypatch.setattr(webrtc_server, "MAX_SELECTED_FILE_BATCH_BYTES", 5) window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm(window, _file_request("open-1", accept=(".bin",), multiple=True)) + wire_id = _arm( + window, _file_request("open-1", max_total_bytes=1, max_file_count=1) + ) + async with ClientSession() as client: + async with client.post( + _files_url(window, wire_id), + data=_chunked_junk() if body is None else body, + headers={"Content-Type": "multipart/form-data; boundary=x"}, + ) as response: + assert response.status == status + event = await _wait_selected_files_event(window) + assert event.status is selected_status + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_over_the_total_limit_still_reports_a_disallowed_type() -> ( + None +): + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm( + window, + _file_request("open-1", accept=(".bin",), max_total_bytes=5, multiple=True), + ) form = FormData() for name in ("a.bin", "b.bin", "c.exe"): form.add_field( @@ -537,16 +558,17 @@ async def test_file_upload_over_the_batch_cap_still_reports_a_disallowed_type( @pytest.mark.asyncio -async def test_file_upload_prefers_disallowed_suffix_over_body_limit() -> None: +async def test_file_upload_over_the_file_limit_still_reports_a_disallowed_type() -> ( + None +): window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm(window, _file_request("open-1", accept=(".png",), max_bytes=1)) + wire_id = _arm( + window, _file_request("open-1", accept=(".png",), max_file_bytes=1) + ) assert ( - await _post_selected_file( - window, wire_id, b"x" * (2 * 1024 * 1024), "hello.txt" - ) - == 204 + await _post_selected_file(window, wire_id, b"x" * 1024, "hello.txt") == 204 ) event = await _wait_selected_files_event(window) assert event.request_id == "open-1" diff --git a/integrations_v2/imgui_ui_demo/README.md b/integrations_v2/imgui_ui_demo/README.md index 7f1a7a27c..b554db29f 100644 --- a/integrations_v2/imgui_ui_demo/README.md +++ b/integrations_v2/imgui_ui_demo/README.md @@ -33,8 +33,8 @@ uv run --no-sync flashdreams-run-v2 imgui-ui-text-input --mode native-window Run the file-picker application. The native window uses the Tk file dialog (on Linux, install Python's Tk package, such as `python3-tk`). WebRTC shows a confirmation dialog in the viewer (the picker must open from a browser click). -The demo asks for `.bin` / `.raw` at the per-file ceiling -(`MAX_SELECTED_FILE_BYTES`). Controls are **Open file (A)**, **Open file (B)**, and **Open files** +The demo asks for `.bin` / `.raw` and overrides the default size limits with +`max_file_bytes` of 32 MiB and `max_total_bytes` of 256 MiB. Controls are **Open file (A)**, **Open file (B)**, and **Open files** (more than one file). Failed picks show in red with the `SelectedFilesStatus` name and why. ```bash diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py index 35a3a4bb6..f80d99378 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py @@ -16,10 +16,7 @@ from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.session import ISession from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop -from flashdreams.runtime_v2.selected_file import ( - MAX_SELECTED_FILE_BYTES, - SelectedFilesStatus, -) +from flashdreams.runtime_v2.selected_file import SelectedFilesStatus from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent @@ -43,9 +40,8 @@ "SelectedFilesStatus.BUSY: another picker was already open." ), SelectedFilesStatus.TOO_LARGE: ( - "SelectedFilesStatus.TOO_LARGE: a file exceeded this request's " - f"max_file_bytes={MAX_SELECTED_FILE_BYTES}, or the set exceeded the " - "batch limits." + "SelectedFilesStatus.TOO_LARGE: the pick broke this request's " + "max_file_bytes, max_total_bytes, or max_file_count." ), SelectedFilesStatus.DISALLOWED_TYPE: ( "SelectedFilesStatus.DISALLOWED_TYPE: suffix not in accept=('.bin', '.raw')." @@ -83,6 +79,13 @@ ) """``(label, request_id, multiple)`` for each Open button, in display order.""" +_MAX_FILE_BYTES = 32 * 1024 * 1024 +"""Each button's ``max_file_bytes`` (32 MiB), lowered from the 1 GiB default.""" + +_MAX_TOTAL_BYTES = 256 * 1024 * 1024 +"""Each button's ``max_total_bytes`` (256 MiB), lowered from the 1 GiB default. +Only Open files can pick enough to reach it.""" + _NO_FILE_SELECTED = "No file selected." """Status shown for a button before its first result.""" @@ -163,7 +166,8 @@ def step_ui( self.request_file_selector( request_id, accept=(".bin", ".raw"), - max_file_bytes=MAX_SELECTED_FILE_BYTES, + max_file_bytes=_MAX_FILE_BYTES, + max_total_bytes=_MAX_TOTAL_BYTES, multiple=multiple, ) if self.state.restart_at is None: diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py index ec0b2f5c1..ceb41ea14 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py @@ -12,6 +12,8 @@ from imgui_ui_demo.file_picker_app import ( _FILE_SELECTION_ERROR_COLOR, _FILE_SELECTION_STATUS_TEXT, + _MAX_FILE_BYTES, + _MAX_TOTAL_BYTES, _OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID, _OPEN_FILE_B_LABEL, @@ -26,7 +28,6 @@ from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.selected_file import ( - MAX_SELECTED_FILE_BYTES, SelectedFile, SelectedFilesStatus, ) @@ -121,7 +122,8 @@ def test_open_file_button_requests_a_client_file(label: str, request_id: str) -> [selection] = requests.file_selections assert selection.request_id == request_id assert selection.accept == (".bin", ".raw") - assert selection.max_bytes == MAX_SELECTED_FILE_BYTES + assert selection.max_file_bytes == _MAX_FILE_BYTES + assert selection.max_total_bytes == _MAX_TOTAL_BYTES assert selection.multiple is False @@ -136,7 +138,8 @@ def test_open_files_button_requests_multiple_client_files() -> None: [selection] = requests.file_selections assert selection.request_id == _OPEN_FILES_REQUEST_ID assert selection.accept == (".bin", ".raw") - assert selection.max_bytes == MAX_SELECTED_FILE_BYTES + assert selection.max_file_bytes == _MAX_FILE_BYTES + assert selection.max_total_bytes == _MAX_TOTAL_BYTES assert selection.multiple is True From 6c1f18d8c09b3303cfeae6f162444b801496cb5a Mon Sep 17 00:00:00 2001 From: Set Paing Date: Tue, 20 Oct 2026 11:28:24 -0700 Subject: [PATCH 08/12] Replace multiple with max_file_count and upload one file per request max_file_count now decides whether a picker may return several files and defaults to 1. The browser uploads each file as its own raw request, so the server holds every byte it reads to max_file_bytes, max_total_bytes and max_file_count, with no separate allowance for multipart framing. Spell out TOO_LARGE and reword _put_user_input_event. Signed-off-by: Set Paing --- .../flashdreams/api_v2/client_window.py | 12 +- flashdreams/flashdreams/api_v2/loop.py | 18 +- .../runtime_v2/native_window_client_window.py | 11 +- .../flashdreams/runtime_v2/selected_file.py | 13 +- .../flashdreams/runtime_v2/serving/web/app.js | 46 +-- .../runtime_v2/serving/webrtc_server.py | 280 +++++++----------- .../runtime_v2/webrtc_client_window.py | 2 +- .../test_native_window_client_window.py | 15 +- flashdreams/test_v2/test_selected_files.py | 14 +- .../test_v2/test_webrtc_client_window.py | 203 +++++++------ integrations_v2/imgui_ui_demo/README.md | 2 +- .../imgui_ui_demo/file_picker_app.py | 19 +- .../imgui_ui_demo/tests/test_imgui_ui_demo.py | 5 +- 13 files changed, 292 insertions(+), 348 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 88efa2d4c..4b7d1498e 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -105,9 +105,9 @@ class IClientWindow(InputSource, OutputSink, ABC): One thread makes every call on a window, so an implementation needs no locking except when its backend delivers input from another thread. - A window with a file picker implements :meth:`_request_file_selector` and - :meth:`_cancel_active_file_selectors`, and reports results on the UI - thread. + A window with a file picker implements :meth:`_request_file_selector`, and + :meth:`_cancel_active_file_selectors` if it can withdraw a shown picker. It + reports results on the UI thread. Created by the runtime, never by an application. """ @@ -202,8 +202,8 @@ def _cancel_active_file_selectors(self) -> None: def _put_user_input_event(self, event: UserInputEvent) -> None: """Queue ``event`` for :meth:`InputSource.get_user_input_events`. - Each window decides how an event from outside its own input polling - joins its input stream. Only this class calls it, on the UI thread. + Lets each window add an event that did not come from its own input + polling. Only this class calls it, on the UI thread. """ @final @@ -249,7 +249,7 @@ def _check_file_selection_result( """Return the status and files a window's result earns under ``request``.""" if status is not SelectedFilesStatus.OK: return status, () - if not files or (len(files) > 1 and not request.multiple): + if not files: _LOGGER.warning("File selector %r returned no valid set.", request.request_id) return SelectedFilesStatus.UNAVAILABLE, () files = tuple(replace(chosen, name=Path(chosen.name).name) for chosen in files) diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 4564d3f80..325206b98 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -627,7 +627,6 @@ def request_file_selector( max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, max_file_count: int = DEFAULT_MAX_FILE_COUNT, - multiple: bool = False, ) -> None: """Ask the client to show a file selector. @@ -644,8 +643,9 @@ def request_file_selector( - ``CANCELLED``: the user dismissed the selector. - ``BUSY``: another selector was open. - ``DISALLOWED_TYPE``: a file did not match ``accept``. - - ``TOO_LARGE``: the set broke ``max_file_bytes``, ``max_total_bytes``, - or ``max_file_count``. + - ``TOO_LARGE``: a file was larger than ``max_file_bytes``, the files + together were larger than ``max_total_bytes``, or more than + ``max_file_count`` files were chosen. - ``UNAVAILABLE``: the window cannot show a selector, or it failed. ``accept`` checks filename suffixes only, so treat ``data`` as @@ -681,14 +681,13 @@ def request_file_selector( max_file_bytes: Largest single file in bytes. Must not exceed ``max_total_bytes``. max_total_bytes: Most bytes all chosen files may add up to. Has no - effect unless ``multiple`` is set. - max_file_count: Most files the selector may return. Has no effect - unless ``multiple`` is set. - multiple: Whether the selector may return more than one file. + effect when ``max_file_count`` is one. + max_file_count: Most files the selector may return. Above one, the + user may pick several. Raises: TypeError: ``request_id`` or ``initial_path`` is not a string, or - ``accept``, a limit, or ``multiple`` has the wrong type. + ``accept`` or a limit has the wrong type. ValueError: ``request_id`` or ``initial_path`` is empty, ``accept`` is invalid, a limit is not positive, or ``max_file_bytes`` exceeds ``max_total_bytes``. @@ -717,8 +716,6 @@ def request_file_selector( raise ValueError(f"{name} must be > 0.") if max_file_bytes > max_total_bytes: raise ValueError("max_file_bytes must not exceed max_total_bytes.") - if not isinstance(multiple, bool): - raise TypeError("multiple must be a bool.") self.get_or_create_ui_loop_requests().file_selections.append( FileSelectionRequest( request_id=request_id, @@ -727,7 +724,6 @@ def request_file_selector( max_file_bytes=max_file_bytes, max_total_bytes=max_total_bytes, max_file_count=max_file_count, - multiple=multiple, ) ) diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index ebceede33..1ea58b397 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -74,7 +74,7 @@ def _choose_paths(request: FileSelectionRequest) -> list[str]: kwargs["filetypes"] = [ ("Accepted", " ".join("*" + suffix for suffix in request.accept)) ] - if request.multiple: + if request.max_file_count > 1: return [str(path) for path in filedialog.askopenfilenames(**kwargs) if path] path = filedialog.askopenfilename(**kwargs) return [str(path)] if path else [] @@ -304,6 +304,9 @@ def _poll_file_dialog(self) -> None: selector_id, request, paths = self._file_dialog_results.get_nowait() except queue.Empty: return + active = self._active_file_selector + if active is None or active[0] != selector_id: + continue files: tuple[SelectedFile, ...] = () if paths is None: status = SelectedFilesStatus.UNAVAILABLE @@ -313,9 +316,7 @@ def _poll_file_dialog(self) -> None: # ponytail: files are read on the UI thread, up to the # request's max_total_bytes, which stalls presentation while # they load. Upgrade path: read them on the dialog thread. - files, status = _selected_files_from_paths( - paths if request.multiple else paths[:1], request - ) + files, status = _selected_files_from_paths(paths, request) self._complete_file_selector(selector_id, status, files) def open(self, session_desc: SessionDesc) -> None: @@ -586,7 +587,7 @@ def _on_window_closed(self) -> None: self._put_input(CloseUserInputEvent(timestamp=uint64(0))) def _put_user_input_event(self, event: UserInputEvent) -> None: - """Queue a file-selection event, bypassing the poll-time buffer.""" + """Queue ``event``, bypassing the poll-time buffer.""" started_ns = self._session_started_ns elapsed_ns = 0 if started_ns is None else max(0, self._clock_ns() - started_ns) self._input_events.put(replace(event, timestamp=uint64(elapsed_ns // 1_000))) diff --git a/flashdreams/flashdreams/runtime_v2/selected_file.py b/flashdreams/flashdreams/runtime_v2/selected_file.py index 667636537..63e450f1c 100644 --- a/flashdreams/flashdreams/runtime_v2/selected_file.py +++ b/flashdreams/flashdreams/runtime_v2/selected_file.py @@ -13,8 +13,8 @@ DEFAULT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024 """Default ``max_total_bytes`` (1 GiB): the most bytes all chosen files add up to.""" -DEFAULT_MAX_FILE_COUNT = 256 -"""Default ``max_file_count``: the most files one request may return.""" +DEFAULT_MAX_FILE_COUNT = 1 +"""Default ``max_file_count``: one file, unless the app asks for more.""" def normalize_selected_file_accept(accept: Sequence[str] = ()) -> tuple[str, ...]: @@ -70,8 +70,8 @@ class SelectedFilesStatus(Enum): """The user dismissed the selector.""" TOO_LARGE = "too_large" - """The set broke the request's ``max_file_bytes``, ``max_total_bytes``, or - ``max_file_count``.""" + """A file was larger than ``max_file_bytes``, the files together were larger + than ``max_total_bytes``, or more than ``max_file_count`` files were chosen.""" DISALLOWED_TYPE = "disallowed_type" """A chosen file did not match ``accept``. Checked before size.""" @@ -116,10 +116,7 @@ class FileSelectionRequest: """Most bytes all chosen files may add up to.""" max_file_count: int = DEFAULT_MAX_FILE_COUNT - """Most files the selector may return.""" - - multiple: bool = False - """Whether the selector may return more than one file.""" + """Most files the selector may return. Above one, the user may pick several.""" def selected_files_policy_status( diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 774f1e19e..e2058987c 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -185,6 +185,10 @@ if (promptDialog !== null) { onCancel?.(); }); promptDialog.addEventListener("close", () => { + // The event is queued, so it can arrive after a newer prompt reopened the dialog. + if (promptDialog.open) { + return; + } setPromptBackdropVisible(false); if (promptSettled) { return; @@ -300,31 +304,33 @@ const startFileInput = pending => { finishFileSelection(rejected); return; } - const body = new FormData(); - for (const file of files) { - body.append("file", file, file.name); - } - fetch(`/api/files?id=${encodeURIComponent(requestId)}`, { - method: "POST", - body, - }) - .then(response => { + // One request per file, in order, so the server reads only file bytes. + const uploadFiles = async () => { + for (const [index, file] of files.entries()) { + const query = new URLSearchParams({ + id: requestId, + name: file.name, + index: String(index), + count: String(files.length), + }); + const response = await fetch(`/api/files?${query}`, {method: "POST", body: file}); if (!response.ok) { // A proxy may refuse the upload before the server can answer it. console.debug("File upload was rejected.", response.status); finishFileSelection("unavailable"); return; } - if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { - return; - } - settled = true; - hideFilePicker(); - }) - .catch(error => { - console.debug("Unable to upload the selected file.", error); - finishFileSelection("unavailable"); - }); + } + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + hideFilePicker(); + }; + uploadFiles().catch(error => { + console.debug("Unable to upload the selected file.", error); + finishFileSelection("unavailable"); + }); }); input.addEventListener("cancel", () => finishFileSelection("cancelled")); input.click(); @@ -348,7 +354,7 @@ const openFileSelector = options => { id: requestId, accept, maxBytes: Number.isFinite(maxBytes) ? maxBytes : 0, - multiple: options.multiple === true, + multiple: Number(options.max_file_count) > 1, }; showPromptDialog({ title: pendingFileRequest.multiple diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index fc6cf31a2..30b258458 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -14,7 +14,7 @@ import time from collections import deque from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from fractions import Fraction from importlib.resources import files from typing import Any, Literal, cast @@ -22,7 +22,6 @@ import numpy as np import torch from aiohttp import web -from aiohttp.multipart import BodyPartReader, MultipartReader from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription from aiortc.mediastreams import MediaStreamError from av import VideoFrame @@ -315,6 +314,26 @@ class _PendingFileSelection: """The app's request, whose ``accept`` and limits the upload must meet.""" +@dataclass(slots=True) +class _FileUpload: + """One pick whose files are arriving, one request per file.""" + + policy: _PendingFileSelection + """The armed request this upload answers.""" + + count: int + """How many files the browser said it will send.""" + + files: list[SelectedFile] = field(default_factory=list) + """Files received so far, in order.""" + + total_bytes: int = 0 + """Bytes received so far across ``files``.""" + + receiving: bool = False + """Whether a request is reading a file now. Another one meanwhile is refused.""" + + class WebRTCServer: """Own the HTTP, signaling, input buffering, and media transport.""" @@ -372,7 +391,7 @@ def __init__( self._lock_cursor_to_window = False # Upload tokens, touched only on the event loop once it runs. self._pending_file_requests: dict[str, _PendingFileSelection] = {} - self._uploading_file_requests: dict[str, _PendingFileSelection] = {} + self._uploading_file_requests: dict[str, _FileUpload] = {} self._file_selector_callback: ( Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None] | None ) = None @@ -554,6 +573,7 @@ def cancel_file_selectors(self) -> None: def _withdraw_file_selectors(self) -> None: """Forget armed selectors and tell the viewer to close its prompt.""" self._pending_file_requests.clear() + self._uploading_file_requests.clear() channel = self._control_channel if channel is not None and channel.readyState == "open": channel.send(json.dumps({"type": "file_selector_cancel"})) @@ -579,7 +599,7 @@ def _arm_file_selector(self, policy: _PendingFileSelection) -> None: "id": token, "accept": list(policy.request.accept), "max_file_bytes": policy.request.max_file_bytes, - "multiple": policy.request.multiple, + "max_file_count": policy.request.max_file_count, } ) ) @@ -592,7 +612,7 @@ def _complete_abandoned_file_selectors(self) -> None: """Answer every armed pick as ``UNAVAILABLE`` because the viewer left.""" abandoned = ( *self._pending_file_requests.values(), - *self._uploading_file_requests.values(), + *(upload.policy for upload in self._uploading_file_requests.values()), ) self._pending_file_requests.clear() self._uploading_file_requests.clear() @@ -947,7 +967,12 @@ async def _health(self, _: web.Request) -> web.Response: ) async def _upload_selected_file(self, request: web.Request) -> web.Response: - """Turn a browser file upload into a selected-files input event.""" + """Receive one chosen file; the last one answers the pick. + + The browser posts each file as its own request, in order, with the raw + file bytes as the body. Every byte read is then a file byte, held to + the request's own limits before it is read. + """ if self._closed: raise web.HTTPServiceUnavailable(reason="WebRTC server is closed.") if self._timestamp_us() is None: @@ -955,117 +980,80 @@ async def _upload_selected_file(self, request: web.Request) -> web.Response: token = request.query.get("id") if not isinstance(token, str) or not token: raise web.HTTPBadRequest(reason="File upload requires an id.") - policy = self._take_pending_file_request(token) - if policy is None: - raise web.HTTPConflict(reason="No pending file selection for this id.") - # Tracked until it ends so a viewer that leaves mid-upload is answered. - self._uploading_file_requests[token] = policy - - emitted = False + count = _query_int(request, "count") + upload = self._uploading_file_requests.get(token) + if upload is None: + policy = self._take_pending_file_request(token) + if policy is None: + raise web.HTTPConflict(reason="No pending file selection for this id.") + # Tracked until it ends so a viewer that leaves mid-upload is answered. + upload = _FileUpload(policy=policy, count=count or 0) + self._uploading_file_requests[token] = upload - def emit( - status: SelectedFilesStatus, - files: tuple[SelectedFile, ...] = (), + def finish( + status: SelectedFilesStatus, files: tuple[SelectedFile, ...] = () ) -> None: - nonlocal emitted - self._emit_selected_files(policy.selector_id, status=status, files=files) - emitted = True - - try: - if not request.content_type.startswith("multipart/"): - emit(SelectedFilesStatus.UNAVAILABLE) - raise web.HTTPBadRequest(reason="File upload requires a file.") - limits = policy.request - # The body bound keeps the whole read finite, even for parts the - # loop below only skips. - body_limit = limits.max_total_bytes + _upload_overhead_bytes( - limits.max_file_count - ) - if request.content_length is None: - emit(SelectedFilesStatus.UNAVAILABLE) - raise web.HTTPLengthRequired() - if request.content_length > body_limit: - emit(SelectedFilesStatus.TOO_LARGE) - raise web.HTTPRequestEntityTooLarge( - max_size=body_limit, actual_size=request.content_length - ) - reader = await request.multipart() - chosen: list[tuple[str, bytes]] = [] - file_count = 0 - total_bytes = 0 - too_large = False - while True: - part = await reader.next() - if part is None: - break - if not isinstance(part, BodyPartReader): - continue - if part.name != "file": - await part.release() - continue - if not limits.multiple and (chosen or too_large): - await part.release() - continue - if not _identity_encoded(part): - emit(SelectedFilesStatus.UNAVAILABLE) - raise web.HTTPBadRequest(reason="File parts must not be encoded.") - file_count += 1 - if file_count > limits.max_file_count: - too_large = True - chosen.clear() - name = part.filename or "upload" - if not selected_file_suffix_allowed(name, limits.accept): - emit(SelectedFilesStatus.DISALLOWED_TYPE) - await _drain_multipart_reader(reader, part) - return web.Response(status=204) - if too_large: - # Keep reading names so a disallowed type still wins. - await part.release() - continue - data = await _read_upload_part_bounded( - part, max_bytes=limits.max_file_bytes - ) - if data is not None: - total_bytes += len(data) - if data is None or total_bytes > limits.max_total_bytes: - too_large = True - chosen.clear() - continue - chosen.append((name, data)) - if too_large: - emit(SelectedFilesStatus.TOO_LARGE) - raise web.HTTPRequestEntityTooLarge( - max_size=limits.max_total_bytes, actual_size=total_bytes + 1 + if self._uploading_file_requests.get(token) is upload: + del self._uploading_file_requests[token] + self._emit_selected_files( + upload.policy.selector_id, status=status, files=files ) - if not chosen: - emit(SelectedFilesStatus.UNAVAILABLE) - raise web.HTTPBadRequest(reason="File upload requires a file.") - rejected = selected_files_policy_status( - [(name, len(data)) for name, data in chosen], limits - ) - if rejected is SelectedFilesStatus.TOO_LARGE: - emit(rejected) - raise web.HTTPRequestEntityTooLarge( - max_size=limits.max_total_bytes, actual_size=total_bytes - ) - if rejected is not None: - emit(rejected) - return web.Response(status=204) - emit( - SelectedFilesStatus.OK, - files=tuple( - SelectedFile(name=name, data=data) for name, data in chosen - ), - ) + + limits = upload.policy.request + name = request.query.get("name", "") + size = request.content_length + if ( + upload.receiving + or upload.count < 1 + or count != upload.count + or _query_int(request, "index") != len(upload.files) + or not name + ): + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File upload is out of order.") + if upload.count > limits.max_file_count: + finish(SelectedFilesStatus.TOO_LARGE) + raise web.HTTPBadRequest(reason="File upload has too many files.") + if not selected_file_suffix_allowed(name, limits.accept): + finish(SelectedFilesStatus.DISALLOWED_TYPE) return web.Response(status=204) - except web.HTTPRequestEntityTooLarge: - if not emitted: - emit(SelectedFilesStatus.TOO_LARGE) + if request.headers.get("Content-Encoding", "identity").lower() != "identity": + # aiohttp would inflate it, past the length checked below. + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File uploads must not be encoded.") + if size is None: + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPLengthRequired() + allowed = min( + limits.max_file_bytes, limits.max_total_bytes - upload.total_bytes + ) + if size > allowed: + finish(SelectedFilesStatus.TOO_LARGE) + raise web.HTTPRequestEntityTooLarge(max_size=allowed, actual_size=size) + upload.receiving = True + try: + data = await request.content.readexactly(size) + except asyncio.IncompleteReadError as error: + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File upload ended early.") from error + except BaseException: + finish(SelectedFilesStatus.UNAVAILABLE) raise finally: - self._uploading_file_requests.pop(token, None) - if not emitted: - emit(SelectedFilesStatus.UNAVAILABLE) + upload.receiving = False + upload.files.append(SelectedFile(name=name, data=data)) + upload.total_bytes += size + if len(upload.files) < upload.count: + return web.Response(status=204) + files = tuple(upload.files) + rejected = selected_files_policy_status( + [(chosen.name, len(chosen.data)) for chosen in files], limits + ) + if rejected is None: + finish(SelectedFilesStatus.OK, files) + else: + finish(rejected) + return web.Response(status=204) async def _offer(self, request: web.Request) -> web.Response: """Negotiate one browser peer connection.""" @@ -1431,7 +1419,10 @@ def _buffer_browser_message( ) policy = self._take_pending_file_request(token) if policy is None: - return + upload = self._uploading_file_requests.pop(token, None) + if upload is None: + return + policy = upload.policy self._emit_selected_files(policy.selector_id, status=status) return else: @@ -1554,65 +1545,12 @@ async def _shutdown(self) -> None: raise primary -_UPLOAD_READ_CHUNK_BYTES = 64 * 1024 -"""Bytes read per multipart chunk while enforcing ``max_bytes``.""" - - -def _upload_overhead_bytes(max_file_count: int) -> int: - """Return the multipart framing an upload of ``max_file_count`` files may add.""" - # ponytail: a flat 4 KiB of boundary and headers per file plus 64 KiB; a - # client sending longer headers is answered TOO_LARGE. Upgrade path: bound - # the bytes of skipped parts while streaming instead of the whole body. - return 64 * 1024 + 4 * 1024 * max_file_count - - -def _identity_encoded(part: BodyPartReader) -> bool: - """Return whether ``part`` carries its bytes as-is, as browsers send them.""" - transfer = part.headers.get("Content-Transfer-Encoding", "").lower() - content = part.headers.get("Content-Encoding", "").lower() - return transfer in ("", "binary", "7bit", "8bit") and content in ("", "identity") - - -async def _read_upload_part_bounded( - part: BodyPartReader, *, max_bytes: int -) -> bytes | None: - """Return the decoded part body, or ``None`` when it exceeds ``max_bytes``. - - Args: - part: Multipart file part whose headers have already been read. - max_bytes: Maximum allowed size of the decoded file in bytes. - - Returns: - The decoded bytes, or ``None`` when they exceed ``max_bytes``. - """ - chunks: list[bytes] = [] - total = 0 - while True: - chunk = await part.read_chunk(_UPLOAD_READ_CHUNK_BYTES) - if not chunk: - break - async for decoded in part.decode_iter(chunk): - piece = bytes(decoded) - total += len(piece) - if total > max_bytes: - await part.release() - return None - chunks.append(piece) - return b"".join(chunks) - - -async def _drain_multipart_reader( - reader: MultipartReader, part: BodyPartReader | None = None -) -> None: - """Consume leftover multipart parts so the request can finish.""" - if part is not None: - await part.release() - while True: - leftover = await reader.next() - if leftover is None: - break - if isinstance(leftover, BodyPartReader): - await leftover.release() +def _query_int(request: web.Request, key: str) -> int | None: + """Return the query value at ``key`` as an integer, or ``None``.""" + try: + return int(request.query[key]) + except (KeyError, ValueError): + return None def _same_stream_format(first: SessionDesc, second: SessionDesc) -> bool: diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 23eef0111..cd73ca561 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -134,7 +134,7 @@ def _cancel_active_file_selectors(self) -> None: self.server.cancel_file_selectors() def _put_user_input_event(self, event: UserInputEvent) -> None: - """Queue a file-selection event on the server's clock, like browser input.""" + """Queue ``event`` on the server's clock, like browser input.""" event = replace(event, timestamp=self.server.event_timestamp_us()) with self._input_lock: self._input_events.append(event) diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index c3dd6023a..a73f3dbd3 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -55,14 +55,14 @@ def _file_request( *, accept: tuple[str, ...] = (), max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, - multiple: bool = False, + max_file_count: int = 1, ) -> FileSelectionRequest: return FileSelectionRequest( request_id=request_id, initial_path=initial_path, accept=accept, max_file_bytes=max_file_bytes, - multiple=multiple, + max_file_count=max_file_count, ) @@ -1134,7 +1134,9 @@ def test_native_window_reports_multiple_selected_files( presenter_factory=_presenter_factory(presenter), ) window.open(_session_desc()) - window.request_file_selector(_file_request("open-1", str(tmp_path), multiple=True)) + window.request_file_selector( + _file_request("open-1", str(tmp_path), max_file_count=2) + ) events = _wait_for_input_events(window) window.close() @@ -1158,6 +1160,12 @@ def choose(_: FileSelectionRequest) -> list[str]: return [str(chosen)] _fake_dialog(monkeypatch, choose) + read: list[object] = [] + monkeypatch.setattr( + native_window_module, + "_selected_files_from_paths", + lambda paths, request: read.append(paths) or ((), SelectedFilesStatus.OK), + ) presenter = _Presenter() window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) window.open(_session_desc()) @@ -1178,6 +1186,7 @@ def choose(_: FileSelectionRequest) -> list[str]: assert busy[0].request_id == "open-2" assert busy[0].status is SelectedFilesStatus.BUSY assert stale == [] + assert read == [], "a flushed dialog's files are never read" def test_native_window_reports_standard_gamepad_events() -> None: diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index 08d472d48..cd76b722d 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -70,7 +70,6 @@ def test_ui_loop_queues_the_app_declared_limits() -> None: max_file_bytes=DEFAULT_MAX_FILE_BYTES * 4, max_total_bytes=DEFAULT_MAX_TOTAL_BYTES * 4, max_file_count=DEFAULT_MAX_FILE_COUNT * 4, - multiple=True, ) loop.request_file_selector("open-2", "/tmp") @@ -84,7 +83,6 @@ def test_ui_loop_queues_the_app_declared_limits() -> None: max_file_bytes=DEFAULT_MAX_FILE_BYTES * 4, max_total_bytes=DEFAULT_MAX_TOTAL_BYTES * 4, max_file_count=DEFAULT_MAX_FILE_COUNT * 4, - multiple=True, ) assert ( defaults.max_file_bytes, @@ -207,7 +205,7 @@ def complete( b = SelectedFile(name="b.png", data=b"2") cases = [ (png, SelectedFilesStatus.OK, (), SelectedFilesStatus.UNAVAILABLE), - (png, SelectedFilesStatus.OK, (a, b), SelectedFilesStatus.UNAVAILABLE), + (png, SelectedFilesStatus.OK, (a, b), SelectedFilesStatus.TOO_LARGE), ( png, SelectedFilesStatus.OK, @@ -227,7 +225,9 @@ def complete( [event] = complete(request, status, files) assert (event.status, event.files) == (expected, ()) - many = FileSelectionRequest(request_id="open-1", initial_path="/tmp", multiple=True) + many = FileSelectionRequest( + request_id="open-1", initial_path="/tmp", max_file_count=2 + ) [event] = complete(many, SelectedFilesStatus.OK, (a, b)) assert (event.status, event.files) == (SelectedFilesStatus.OK, (a, b)) @@ -326,12 +326,6 @@ def test_ui_loop_rejects_invalid_accept() -> None: loop.request_file_selector("open-1", accept=("png",)) -def test_ui_loop_rejects_invalid_multiple() -> None: - loop = _ui_loop() - with pytest.raises(TypeError, match="multiple must be a bool"): - loop.request_file_selector("open-1", multiple=1) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - - def test_selected_files_event_rejects_inconsistent_status() -> None: with pytest.raises(ValueError, match="must include files"): SelectedFilesUserInputEvent( diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index eea66c91e..10ac019cc 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -25,7 +25,8 @@ pytest.importorskip("aiohttp") pytest.importorskip("aiortc") -from aiohttp import ClientSession, FormData, MultipartWriter, web +from aiohttp import ClientSession +from aiohttp.streams import StreamReader from aiortc import ( MediaStreamTrack, RTCDataChannel, @@ -108,22 +109,33 @@ def _arm(window: WebRTCClientWindow, request: FileSelectionRequest) -> str: return wire_id +async def _post_files( + window: WebRTCClientWindow, + token: str, + files: list[tuple[str, bytes]], +) -> list[int]: + """Upload ``files`` as the browser does, one request each; stop at a refusal.""" + statuses: list[int] = [] + async with ClientSession() as client: + for index, (name, data) in enumerate(files): + query = {"name": name, "index": str(index), "count": str(len(files))} + async with client.post( + _files_url(window, token), params=query, data=data + ) as response: + statuses.append(response.status) + if response.status != 204: + break + return statuses + + async def _post_selected_file( window: WebRTCClientWindow, - request_id: str, + token: str, data: bytes, filename: str, ) -> int: - form = FormData() - form.add_field( - "file", - data, - filename=filename, - content_type="application/octet-stream", - ) - async with ClientSession() as client: - async with client.post(_files_url(window, request_id), data=form) as response: - return response.status + [status] = await _post_files(window, token, [(filename, data)]) + return status @pytest.mark.asyncio @@ -132,24 +144,12 @@ async def test_file_upload_accepts_multiple_files() -> None: window.open(_session_desc()) peer, _, _ = await _connect_browser(window) try: - window.request_file_selector(_file_request("open-1", multiple=True)) + window.request_file_selector(_file_request("open-1", max_file_count=2)) wire_id = await _wait_file_armed(window) - form = FormData() - form.add_field( - "file", - b"one", - filename="a.bin", - content_type="application/octet-stream", + statuses = await _post_files( + window, wire_id, [("a.bin", b"one"), ("b.bin", b"two")] ) - form.add_field( - "file", - b"two", - filename="b.bin", - content_type="application/octet-stream", - ) - async with ClientSession() as client: - async with client.post(_files_url(window, wire_id), data=form) as response: - assert response.status == 204 + assert statuses == [204, 204] event = await _wait_selected_files_event(window) assert event.status is SelectedFilesStatus.OK assert [chosen.name for chosen in event.files] == ["a.bin", "b.bin"] @@ -297,19 +297,19 @@ async def test_in_flight_upload_is_unavailable_after_peer_loss( peer, _, _ = await _connect_browser(window) window.request_file_selector(_file_request("open-1")) wire_id = await _wait_file_armed(window) - original_multipart = web.Request.multipart + original_readexactly = StreamReader.readexactly disconnected = False - async def multipart_after_disconnect(request: web.Request) -> Any: + async def readexactly_after_disconnect(self: StreamReader, n: int) -> bytes: nonlocal disconnected if not disconnected: disconnected = True peer_connection = window.server._peer_connection if peer_connection is not None: await window.server._release_peer_connection(peer_connection) - return await original_multipart(request) + return await original_readexactly(self, n) - monkeypatch.setattr(web.Request, "multipart", multipart_after_disconnect) + monkeypatch.setattr(StreamReader, "readexactly", readexactly_after_disconnect) assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 204 event = await _wait_selected_files_event(window) assert event.request_id == "open-1" @@ -322,25 +322,6 @@ async def multipart_after_disconnect(request: web.Request) -> Any: window.close() -@pytest.mark.asyncio -async def test_encoded_file_part_is_unavailable() -> None: - window = WebRTCClientWindow() - window.open(_session_desc()) - try: - wire_id = _arm(window, _file_request("open-1")) - with MultipartWriter("form-data") as body: - part = body.append(b"abc=\n") - part.set_content_disposition("form-data", name="file", filename="a.bin") - part.headers["Content-Transfer-Encoding"] = "quoted-printable" - async with ClientSession() as client: - async with client.post(_files_url(window, wire_id), data=body) as response: - assert response.status == 400 - event = await _wait_selected_files_event(window) - assert event.status is SelectedFilesStatus.UNAVAILABLE - finally: - window.close() - - @pytest.mark.asyncio async def test_upload_without_the_armed_token_is_rejected() -> None: window = WebRTCClientWindow() @@ -389,7 +370,7 @@ def on_message(message: object) -> None: "type": "file_selector", "accept": [".png", ".jpg"], "max_file_bytes": DEFAULT_MAX_FILE_BYTES * 2, - "multiple": False, + "max_file_count": 1, } finally: if peer is not None: @@ -442,15 +423,23 @@ async def _wait_selected_files_event( @pytest.mark.asyncio -async def test_file_upload_without_file_is_unavailable() -> None: +@pytest.mark.parametrize( + "query", + [ + {}, + {"name": "a.bin", "index": "1", "count": "2"}, + {"name": "", "index": "0", "count": "1"}, + ], +) +async def test_file_upload_out_of_order_is_unavailable(query: dict[str, str]) -> None: window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm(window, _file_request("open-1")) - form = FormData() - form.add_field("not_file", "hello") + wire_id = _arm(window, _file_request("open-1", max_file_count=2)) async with ClientSession() as client: - async with client.post(_files_url(window, wire_id), data=form) as response: + async with client.post( + _files_url(window, wire_id), params=query, data=b"hello" + ) as response: assert response.status == 400 event = await _wait_selected_files_event(window) assert event.request_id == "open-1" @@ -477,20 +466,22 @@ async def test_file_upload_rejects_oversize() -> None: @pytest.mark.asyncio -@pytest.mark.parametrize("limits", [{"max_total_bytes": 5}, {"max_file_count": 2}]) +@pytest.mark.parametrize( + "limits", + [ + {"max_file_bytes": 5, "max_total_bytes": 5, "max_file_count": 3}, + {"max_file_count": 2}, + {}, + ], +) async def test_file_upload_enforces_the_request_limits(limits: dict[str, Any]) -> None: window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm(window, _file_request("open-1", multiple=True, **limits)) - form = FormData() - for name in ("a.bin", "b.bin", "c.bin"): - form.add_field( - "file", b"xyz", filename=name, content_type="application/octet-stream" - ) - async with ClientSession() as client: - async with client.post(_files_url(window, wire_id), data=form) as response: - assert response.status == 413 + wire_id = _arm(window, _file_request("open-1", **limits)) + files = [(name, b"xyz") for name in ("a.bin", "b.bin", "c.bin")] + statuses = await _post_files(window, wire_id, files) + assert statuses[-1] in (400, 413) event = await _wait_selected_files_event(window) assert event.status is SelectedFilesStatus.TOO_LARGE assert event.files == () @@ -503,27 +494,30 @@ async def _chunked_junk() -> AsyncIterator[bytes]: @pytest.mark.parametrize( - ("body", "status", "selected_status"), + ("body", "headers", "status", "selected_status"), [ - (b"x" * (256 * 1024), 413, SelectedFilesStatus.TOO_LARGE), - (None, 411, SelectedFilesStatus.UNAVAILABLE), + (b"x" * 1024, {}, 413, SelectedFilesStatus.TOO_LARGE), + (None, {}, 411, SelectedFilesStatus.UNAVAILABLE), + (b"x", {"Content-Encoding": "gzip"}, 400, SelectedFilesStatus.UNAVAILABLE), ], ) @pytest.mark.asyncio -async def test_file_upload_rejects_an_unbounded_body_before_reading_it( - body: bytes | None, status: int, selected_status: SelectedFilesStatus +async def test_file_upload_is_refused_before_its_body_is_read( + body: bytes | None, + headers: dict[str, str], + status: int, + selected_status: SelectedFilesStatus, ) -> None: window = WebRTCClientWindow() window.open(_session_desc()) try: - wire_id = _arm( - window, _file_request("open-1", max_total_bytes=1, max_file_count=1) - ) + wire_id = _arm(window, _file_request("open-1", max_file_bytes=16)) async with ClientSession() as client: async with client.post( _files_url(window, wire_id), + params={"name": "a.bin", "index": "0", "count": "1"}, data=_chunked_junk() if body is None else body, - headers={"Content-Type": "multipart/form-data; boundary=x"}, + headers=headers, ) as response: assert response.status == status event = await _wait_selected_files_event(window) @@ -532,31 +526,6 @@ async def test_file_upload_rejects_an_unbounded_body_before_reading_it( window.close() -@pytest.mark.asyncio -async def test_file_upload_over_the_total_limit_still_reports_a_disallowed_type() -> ( - None -): - window = WebRTCClientWindow() - window.open(_session_desc()) - try: - wire_id = _arm( - window, - _file_request("open-1", accept=(".bin",), max_total_bytes=5, multiple=True), - ) - form = FormData() - for name in ("a.bin", "b.bin", "c.exe"): - form.add_field( - "file", b"xyz", filename=name, content_type="application/octet-stream" - ) - async with ClientSession() as client: - async with client.post(_files_url(window, wire_id), data=form) as response: - assert response.status == 204 - event = await _wait_selected_files_event(window) - assert event.status is SelectedFilesStatus.DISALLOWED_TYPE - finally: - window.close() - - @pytest.mark.asyncio async def test_file_upload_over_the_file_limit_still_reports_a_disallowed_type() -> ( None @@ -606,6 +575,36 @@ async def test_browser_file_selector_result_reports_status() -> None: window.close() +@pytest.mark.asyncio +async def test_browser_result_ends_an_upload_left_partway() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1", max_file_count=2)) + wire_id = await _wait_file_armed(window) + async with ClientSession() as client: + async with client.post( + _files_url(window, wire_id), + params={"name": "a.bin", "index": "0", "count": "2"}, + data=b"one", + ) as response: + assert response.status == 204 + channel.send( + json.dumps( + {"type": "file_selector_result", "id": wire_id, "status": "unavailable"} + ) + ) + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert await _post_selected_file(window, wire_id, b"two", "b.bin") == 409 + finally: + if peer is not None: + await peer.close() + window.close() + + @pytest.mark.asyncio async def test_browser_file_selector_result_cannot_claim_ok() -> None: window = WebRTCClientWindow() @@ -661,7 +660,7 @@ async def test_browser_script_checks_type_and_size_before_upload() -> None: assert "file.size" in script assert '"too_large"' in script assert '"disallowed_type"' in script - assert "fetch(`/api/files?id=${encodeURIComponent(requestId)}`" in script + assert "fetch(`/api/files?${query}`" in script assert "input.multiple" in script assert "event.stopImmediatePropagation();" in script assert "(promptDialog !== null && promptDialog.open)" in script diff --git a/integrations_v2/imgui_ui_demo/README.md b/integrations_v2/imgui_ui_demo/README.md index b554db29f..a991b0cd9 100644 --- a/integrations_v2/imgui_ui_demo/README.md +++ b/integrations_v2/imgui_ui_demo/README.md @@ -35,7 +35,7 @@ Run the file-picker application. The native window uses the Tk file dialog confirmation dialog in the viewer (the picker must open from a browser click). The demo asks for `.bin` / `.raw` and overrides the default size limits with `max_file_bytes` of 32 MiB and `max_total_bytes` of 256 MiB. Controls are **Open file (A)**, **Open file (B)**, and **Open files** -(more than one file). Failed picks show in red with the +(up to 8 files, through `max_file_count`). Failed picks show in red with the `SelectedFilesStatus` name and why. ```bash uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode native-window diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py index f80d99378..62a5e002f 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py @@ -40,8 +40,8 @@ "SelectedFilesStatus.BUSY: another picker was already open." ), SelectedFilesStatus.TOO_LARGE: ( - "SelectedFilesStatus.TOO_LARGE: the pick broke this request's " - "max_file_bytes, max_total_bytes, or max_file_count." + "SelectedFilesStatus.TOO_LARGE: a file was over 32 MiB, the files " + "together were over 256 MiB, or too many files were chosen." ), SelectedFilesStatus.DISALLOWED_TYPE: ( "SelectedFilesStatus.DISALLOWED_TYPE: suffix not in accept=('.bin', '.raw')." @@ -72,12 +72,15 @@ _OPEN_FILES_LABEL = "Open files" """ImGui label for a multi-file selector slot.""" +_OPEN_FILES_MAX_FILE_COUNT = 8 +"""Open files' ``max_file_count``, raised from the default of one.""" + _FILE_SELECTOR_SLOTS = ( - (_OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID, False), - (_OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID, False), - (_OPEN_FILES_LABEL, _OPEN_FILES_REQUEST_ID, True), + (_OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID, 1), + (_OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID, 1), + (_OPEN_FILES_LABEL, _OPEN_FILES_REQUEST_ID, _OPEN_FILES_MAX_FILE_COUNT), ) -"""``(label, request_id, multiple)`` for each Open button, in display order.""" +"""``(label, request_id, max_file_count)`` for each Open button, in display order.""" _MAX_FILE_BYTES = 32 * 1024 * 1024 """Each button's ``max_file_bytes`` (32 MiB), lowered from the 1 GiB default.""" @@ -161,14 +164,14 @@ def step_ui( imgui.set_next_window_size(imgui.ImVec2(520.0, 300.0), imgui.Cond_.once) imgui.begin("File picker") try: - for label, request_id, multiple in _FILE_SELECTOR_SLOTS: + for label, request_id, max_file_count in _FILE_SELECTOR_SLOTS: if imgui.button(label): self.request_file_selector( request_id, accept=(".bin", ".raw"), max_file_bytes=_MAX_FILE_BYTES, max_total_bytes=_MAX_TOTAL_BYTES, - multiple=multiple, + max_file_count=max_file_count, ) if self.state.restart_at is None: if imgui.button(f"Restart session in {_RESTART_DELAY_S:.0f}s"): diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py index ceb41ea14..cb409bb93 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py @@ -19,6 +19,7 @@ _OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID, _OPEN_FILES_LABEL, + _OPEN_FILES_MAX_FILE_COUNT, _OPEN_FILES_REQUEST_ID, FilePickerImGuiUILoop, FilePickerState, @@ -124,7 +125,7 @@ def test_open_file_button_requests_a_client_file(label: str, request_id: str) -> assert selection.accept == (".bin", ".raw") assert selection.max_file_bytes == _MAX_FILE_BYTES assert selection.max_total_bytes == _MAX_TOTAL_BYTES - assert selection.multiple is False + assert selection.max_file_count == 1 def test_open_files_button_requests_multiple_client_files() -> None: @@ -140,7 +141,7 @@ def test_open_files_button_requests_multiple_client_files() -> None: assert selection.accept == (".bin", ".raw") assert selection.max_file_bytes == _MAX_FILE_BYTES assert selection.max_total_bytes == _MAX_TOTAL_BYTES - assert selection.multiple is True + assert selection.max_file_count == _OPEN_FILES_MAX_FILE_COUNT def test_selected_files_event_updates_file_picker_status() -> None: From 7f10c0b629e3c6eb8be973e30309aeac91b8f0ed Mon Sep 17 00:00:00 2001 From: Set Paing Date: Tue, 20 Oct 2026 12:31:02 -0700 Subject: [PATCH 09/12] Release pointer lock when the browser prompt opens Signed-off-by: Set Paing --- .../flashdreams/api_v2/client_window.py | 3 +-- .../flashdreams/runtime_v2/serving/web/app.js | 3 +++ .../runtime_v2/serving/webrtc_server.py | 20 +++++++++---------- .../test_v2/test_webrtc_client_window.py | 6 +++--- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 4b7d1498e..f5e5eebdb 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -202,8 +202,7 @@ def _cancel_active_file_selectors(self) -> None: def _put_user_input_event(self, event: UserInputEvent) -> None: """Queue ``event`` for :meth:`InputSource.get_user_input_events`. - Lets each window add an event that did not come from its own input - polling. Only this class calls it, on the UI thread. + Canonical method for how user-input-events are added to a window. """ @final diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index e2058987c..4e94d4670 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -143,6 +143,9 @@ const showPromptDialog = ({ ) { return; } + if (document.pointerLockElement === video) { + document.exitPointerLock(); + } if (promptDialog.open) { promptSettled = true; promptOnConfirm = null; diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 30b258458..82aaaa18e 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -305,7 +305,7 @@ async def _next_queued_frame(self) -> _QueuedRGBFrame: @dataclass(frozen=True, slots=True) class _PendingFileSelection: - """One armed file-selector request and the upload rules it sets.""" + """One open file-selector request and the upload rules it sets.""" selector_id: str """Window selector this upload answers. Never sent to the browser.""" @@ -319,7 +319,7 @@ class _FileUpload: """One pick whose files are arriving, one request per file.""" policy: _PendingFileSelection - """The armed request this upload answers.""" + """The open request this upload answers.""" count: int """How many files the browser said it will send.""" @@ -556,10 +556,10 @@ def request_file_selector( loop = self._loop if loop is None: raise RuntimeError("WebRTC server is not running.") - loop.call_soon_threadsafe(self._arm_file_selector, policy) + loop.call_soon_threadsafe(self._open_client_side_file_selector, policy) def cancel_file_selectors(self) -> None: - """Forget every armed selector and close the viewer's prompt. + """Forget every open selector and close the viewer's prompt. A later upload for a forgotten selector is refused. """ @@ -567,18 +567,18 @@ def cancel_file_selectors(self) -> None: if loop is None: self._pending_file_requests.clear() return - # On the event loop, after any arm still queued for a forgotten selector. + # On the event loop, after any opening still queued for a forgotten selector. loop.call_soon_threadsafe(self._withdraw_file_selectors) def _withdraw_file_selectors(self) -> None: - """Forget armed selectors and tell the viewer to close its prompt.""" + """Forget open selectors and tell the viewer to close its prompt.""" self._pending_file_requests.clear() self._uploading_file_requests.clear() channel = self._control_channel if channel is not None and channel.readyState == "open": channel.send(json.dumps({"type": "file_selector_cancel"})) - def _arm_file_selector(self, policy: _PendingFileSelection) -> None: + def _open_client_side_file_selector(self, policy: _PendingFileSelection) -> None: """Send the selector to the browser and remember its upload policy. The browser gets an unguessable token instead of the selector id, so a @@ -609,7 +609,7 @@ def _take_pending_file_request(self, token: str) -> _PendingFileSelection | None return self._pending_file_requests.pop(token, None) def _complete_abandoned_file_selectors(self) -> None: - """Answer every armed pick as ``UNAVAILABLE`` because the viewer left.""" + """Answer every open pick as ``UNAVAILABLE`` because the viewer left.""" abandoned = ( *self._pending_file_requests.values(), *(upload.policy for upload in self._uploading_file_requests.values()), @@ -748,7 +748,7 @@ def register_file_selector_callback( self, callback: Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], ) -> None: - """Register how armed file selectors report back, by selector id. + """Register how open file selectors report back, by selector id. Args: callback: Called once per selector on the server thread, with its @@ -1443,7 +1443,7 @@ def _emit_selected_files( status: SelectedFilesStatus, files: tuple[SelectedFile, ...] = (), ) -> None: - """Report the result of one armed file selector to the window.""" + """Report the result of one open file selector to the window.""" callback = self._file_selector_callback if callback is None: raise RuntimeError("WebRTC file selector callback is not registered.") diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 10ac019cc..93095ed85 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -76,12 +76,12 @@ def _files_url(window: WebRTCClientWindow, token: str) -> str: async def _wait_file_armed(window: WebRTCClientWindow) -> str: - """Return the token the browser was sent once a selector is armed.""" + """Return the token the browser was sent once a selector is open.""" for _ in range(100): if window.server._pending_file_requests: return next(iter(window.server._pending_file_requests)) await asyncio.sleep(0.01) - raise AssertionError("file selection was not armed") + raise AssertionError("file selection was not opened") def _on_server_loop(window: WebRTCClientWindow, call: Callable[[], T]) -> T: @@ -94,7 +94,7 @@ def _on_server_loop(window: WebRTCClientWindow, call: Callable[[], T]) -> T: def _arm(window: WebRTCClientWindow, request: FileSelectionRequest) -> str: - """Arm a selector as if the browser had been sent ``request``; return its token.""" + """Open a selector as if the browser had been sent ``request``; return its token.""" selector_id = window._last_file_selector_id + 1 window._last_file_selector_id = selector_id window._active_file_selector = (selector_id, request) From e135fc32197735600fec07d1d7ac392bca0b38a6 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Thu, 22 Oct 2026 13:05:04 -0700 Subject: [PATCH 10/12] Stop the read of a file upload whose selector was withdrawn Signed-off-by: Set Paing --- .../runtime_v2/serving/webrtc_server.py | 20 ++++++--- .../test_v2/test_webrtc_client_window.py | 42 ++++++++++++++++++- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 82aaaa18e..8920873d5 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -330,8 +330,13 @@ class _FileUpload: total_bytes: int = 0 """Bytes received so far across ``files``.""" - receiving: bool = False - """Whether a request is reading a file now. Another one meanwhile is refused.""" + reader: asyncio.Task[Any] | None = None + """The request reading a file now, if any. Another one meanwhile is refused.""" + + def stop(self) -> None: + """Stop the read in progress, so a forgotten upload frees its bytes now.""" + if self.reader is not None: + self.reader.cancel() class WebRTCServer: @@ -573,6 +578,8 @@ def cancel_file_selectors(self) -> None: def _withdraw_file_selectors(self) -> None: """Forget open selectors and tell the viewer to close its prompt.""" self._pending_file_requests.clear() + for upload in self._uploading_file_requests.values(): + upload.stop() self._uploading_file_requests.clear() channel = self._control_channel if channel is not None and channel.readyState == "open": @@ -615,6 +622,8 @@ def _complete_abandoned_file_selectors(self) -> None: *(upload.policy for upload in self._uploading_file_requests.values()), ) self._pending_file_requests.clear() + for upload in self._uploading_file_requests.values(): + upload.stop() self._uploading_file_requests.clear() for policy in abandoned: self._emit_selected_files( @@ -1003,7 +1012,7 @@ def finish( name = request.query.get("name", "") size = request.content_length if ( - upload.receiving + upload.reader is not None or upload.count < 1 or count != upload.count or _query_int(request, "index") != len(upload.files) @@ -1030,7 +1039,7 @@ def finish( if size > allowed: finish(SelectedFilesStatus.TOO_LARGE) raise web.HTTPRequestEntityTooLarge(max_size=allowed, actual_size=size) - upload.receiving = True + upload.reader = asyncio.current_task() try: data = await request.content.readexactly(size) except asyncio.IncompleteReadError as error: @@ -1040,7 +1049,7 @@ def finish( finish(SelectedFilesStatus.UNAVAILABLE) raise finally: - upload.receiving = False + upload.reader = None upload.files.append(SelectedFile(name=name, data=data)) upload.total_bytes += size if len(upload.files) < upload.count: @@ -1422,6 +1431,7 @@ def _buffer_browser_message( upload = self._uploading_file_requests.pop(token, None) if upload is None: return + upload.stop() policy = upload.policy self._emit_selected_files(policy.selector_id, status=status) return diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 93095ed85..ed4e1fc88 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -25,7 +25,7 @@ pytest.importorskip("aiohttp") pytest.importorskip("aiortc") -from aiohttp import ClientSession +from aiohttp import ClientError, ClientSession from aiohttp.streams import StreamReader from aiortc import ( MediaStreamTrack, @@ -310,7 +310,8 @@ async def readexactly_after_disconnect(self: StreamReader, n: int) -> bytes: return await original_readexactly(self, n) monkeypatch.setattr(StreamReader, "readexactly", readexactly_after_disconnect) - assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 204 + with pytest.raises(ClientError): + await _post_selected_file(window, wire_id, b"hello", "hello.txt") event = await _wait_selected_files_event(window) assert event.request_id == "open-1" assert event.status is SelectedFilesStatus.UNAVAILABLE @@ -605,6 +606,43 @@ async def test_browser_result_ends_an_upload_left_partway() -> None: window.close() +@pytest.mark.asyncio +async def test_withdrawn_selector_stops_its_upload_read() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + writer: asyncio.StreamWriter | None = None + try: + wire_id = _arm(window, _file_request("open-1")) + url = URL(_files_url(window, wire_id)) + assert url.host is not None and url.port is not None + reader, writer = await asyncio.open_connection(url.host, url.port) + target = url.with_query( + id=wire_id, name="slow.bin", index="0", count="1" + ).raw_path_qs + writer.write( + f"POST {target} HTTP/1.1\r\nHost: {url.host}\r\n" + "Content-Length: 10\r\n\r\nx".encode() + ) + await writer.drain() + for _ in range(100): + uploads = window.server._uploading_file_requests + if wire_id in uploads and uploads[wire_id].reader is not None: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("upload read did not start") + + window.server.cancel_file_selectors() + + await asyncio.wait_for(reader.read(), timeout=5) + assert window.server._uploading_file_requests == {} + assert window.get_user_input_events().get_events() == [] + finally: + if writer is not None: + writer.close() + window.close() + + @pytest.mark.asyncio async def test_browser_file_selector_result_cannot_claim_ok() -> None: window = WebRTCClientWindow() From 5087b23659fc1543c0a5d9fb40ee69215009a40b Mon Sep 17 00:00:00 2001 From: Set Paing Date: Thu, 22 Oct 2026 13:09:43 -0700 Subject: [PATCH 11/12] Forward queued input events through the handdreams controller window Signed-off-by: Set Paing --- .../controller_source_v2_window.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py index 78c8668c0..ee89d1d8b 100644 --- a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py +++ b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py @@ -26,6 +26,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -243,6 +244,10 @@ def open(self, session_desc: SessionDesc) -> None: self._disconnect_emitted = False self._last_status = None + def _put_user_input_event(self, event: UserInputEvent) -> None: + """Forward event to the presentation backend, merged with its input.""" + self.output_window._put_user_input_event(event) + def get_user_input_events(self) -> UserInputEvents: """Merge source-clock controller history with buffered window input.""" # The source owns XR input. Operator B only signals its headset worker; From 8be97afc52f550ce9f63311a5f13205bc93e8099 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Thu, 22 Oct 2026 16:43:44 -0700 Subject: [PATCH 12/12] Make put_user_input_event public on InputSource Signed-off-by: Set Paing --- apps/action2v/tests/test_application.py | 2 +- .../controller_source_v2_window.py | 4 ++-- apps/t2v/t2v/testing.py | 2 +- apps/t2v/tests/test_cli.py | 2 +- flashdreams/flashdreams/api_v2/client_window.py | 12 ++---------- flashdreams/flashdreams/api_v2/input_source.py | 11 +++++++++++ flashdreams/flashdreams/runtime_v2/README.md | 4 ++-- .../flashdreams/runtime_v2/mp4_client_window.py | 2 +- .../runtime_v2/native_window_client_window.py | 2 +- .../flashdreams/runtime_v2/null_client_window.py | 2 +- .../flashdreams/runtime_v2/webrtc_client_window.py | 2 +- flashdreams/test_v2/test_application_runner.py | 2 +- flashdreams/test_v2/test_cli.py | 2 +- flashdreams/test_v2/test_client_window.py | 2 +- flashdreams/test_v2/test_selected_files.py | 2 +- flashdreams/test_v2/test_session_runner.py | 2 +- flashdreams/tests/test_profiler.py | 2 +- .../red_screen/red_screen/tests/test_red_screen.py | 2 +- tests/test_step_agreement.py | 2 +- 19 files changed, 32 insertions(+), 29 deletions(-) diff --git a/apps/action2v/tests/test_application.py b/apps/action2v/tests/test_application.py index e1dbf23dc..fbfc71bb7 100644 --- a/apps/action2v/tests/test_application.py +++ b/apps/action2v/tests/test_application.py @@ -139,7 +139,7 @@ def __init__(self) -> None: self.opened = False self.cursor_requests: list[tuple[str, bool]] = [] - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py index ee89d1d8b..8e8aed730 100644 --- a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py +++ b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py @@ -244,9 +244,9 @@ def open(self, session_desc: SessionDesc) -> None: self._disconnect_emitted = False self._last_status = None - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: """Forward event to the presentation backend, merged with its input.""" - self.output_window._put_user_input_event(event) + self.output_window.put_user_input_event(event) def get_user_input_events(self) -> UserInputEvents: """Merge source-clock controller history with buffered window input.""" diff --git a/apps/t2v/t2v/testing.py b/apps/t2v/t2v/testing.py index a20337d9c..ca78800a4 100644 --- a/apps/t2v/t2v/testing.py +++ b/apps/t2v/t2v/testing.py @@ -411,7 +411,7 @@ class _DiscardingClientWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/apps/t2v/tests/test_cli.py b/apps/t2v/tests/test_cli.py index 0cadccec3..62e2261f7 100644 --- a/apps/t2v/tests/test_cli.py +++ b/apps/t2v/tests/test_cli.py @@ -161,7 +161,7 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index f5e5eebdb..840875df2 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -23,7 +23,6 @@ from .input_source import InputSource from .output_sink import OutputSink -from .user_input_event import UserInputEvent _LOGGER = logging.getLogger(__name__) @@ -155,7 +154,7 @@ def request_file_selector(self, request: FileSelectionRequest) -> None: request: Selector request queued by the UI loop. """ if self._active_file_selector is not None: - self._put_user_input_event( + self.put_user_input_event( _selected_files_event(request, SelectedFilesStatus.BUSY) ) return @@ -198,13 +197,6 @@ def _cancel_active_file_selectors(self) -> None: """Withdraw the picker that was flushed.""" pass - @abstractmethod - def _put_user_input_event(self, event: UserInputEvent) -> None: - """Queue ``event`` for :meth:`InputSource.get_user_input_events`. - - Canonical method for how user-input-events are added to a window. - """ - @final def _complete_file_selector( self, @@ -223,7 +215,7 @@ def _complete_file_selector( self._active_file_selector = None request = active[1] status, files = _check_file_selection_result(request, status, files) - self._put_user_input_event(_selected_files_event(request, status, files)) + self.put_user_input_event(_selected_files_event(request, status, files)) def _selected_files_event( diff --git a/flashdreams/flashdreams/api_v2/input_source.py b/flashdreams/flashdreams/api_v2/input_source.py index d64a4df88..dcbf8a572 100644 --- a/flashdreams/flashdreams/api_v2/input_source.py +++ b/flashdreams/flashdreams/api_v2/input_source.py @@ -6,6 +6,7 @@ from abc import abstractmethod from typing import Protocol, runtime_checkable +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.user_input_events import UserInputEvents @@ -31,3 +32,13 @@ def get_user_input_events(self) -> UserInputEvents: Events in timestamp order, empty when nothing arrived. """ ... + + @abstractmethod + def put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event`` for :meth:`get_user_input_events`. + + Canonical method for how user-input-events are added to an input source. + Called on the UI thread, by ``IClientWindow`` and by windows that wrap + another window. + """ + ... diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 163c9ad43..ba4df4118 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -375,8 +375,8 @@ focus, and file-selector events arrive on the server's own thread, so it queues them and hands them over in batches when the session asks. File selection rules live in `IClientWindow`; a window only shows a picker and -reports what it returned, on the UI thread. Every window implements the -abstract `_put_user_input_event`; a window with a picker also implements +reports what it returned, on the UI thread. Every window implements +`InputSource.put_user_input_event`; a window with a picker also implements `_request_file_selector`. A UI loop may call `request_new_session(session_desc)` during its step with a diff --git a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py index ac7c2ab7c..300b926a3 100644 --- a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py @@ -40,7 +40,7 @@ def get_user_input_events(self) -> UserInputEvents: events, self._input_events = self._input_events, [] return UserInputEvents(events) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: self._input_events.append(event) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index 1ea58b397..2ad057613 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -586,7 +586,7 @@ def _on_window_closed(self) -> None: self._close_event_enqueued = True self._put_input(CloseUserInputEvent(timestamp=uint64(0))) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: """Queue ``event``, bypassing the poll-time buffer.""" started_ns = self._session_started_ns elapsed_ns = 0 if started_ns is None else max(0, self._clock_ns() - started_ns) diff --git a/flashdreams/flashdreams/runtime_v2/null_client_window.py b/flashdreams/flashdreams/runtime_v2/null_client_window.py index 001f026e5..8255d7841 100644 --- a/flashdreams/flashdreams/runtime_v2/null_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/null_client_window.py @@ -21,7 +21,7 @@ def get_user_input_events(self) -> UserInputEvents: events, self._input_events = self._input_events, [] return UserInputEvents(events) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: self._input_events.append(event) def open(self, session_desc: SessionDesc) -> None: diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index cd73ca561..0e15ebe07 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -133,7 +133,7 @@ def _cancel_active_file_selectors(self) -> None: """Refuse a later upload for the selector that was forgotten.""" self.server.cancel_file_selectors() - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: """Queue ``event`` on the server's clock, like browser input.""" event = replace(event, timestamp=self.server.event_timestamp_us()) with self._input_lock: diff --git a/flashdreams/test_v2/test_application_runner.py b/flashdreams/test_v2/test_application_runner.py index 89329e07a..b51be79a0 100644 --- a/flashdreams/test_v2/test_application_runner.py +++ b/flashdreams/test_v2/test_application_runner.py @@ -159,7 +159,7 @@ def get_user_input_events(self) -> UserInputEvents: ) return UserInputEvents([]) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/flashdreams/test_v2/test_cli.py b/flashdreams/test_v2/test_cli.py index 1baf6864c..1c20a8dd2 100644 --- a/flashdreams/test_v2/test_cli.py +++ b/flashdreams/test_v2/test_cli.py @@ -43,7 +43,7 @@ class _StubWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/flashdreams/test_v2/test_client_window.py b/flashdreams/test_v2/test_client_window.py index 7e66a592f..207f11716 100644 --- a/flashdreams/test_v2/test_client_window.py +++ b/flashdreams/test_v2/test_client_window.py @@ -37,7 +37,7 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return self._input - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index cd76b722d..a98fd80b0 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -253,7 +253,7 @@ def write(self, result: StepResult) -> None: def close(self) -> None: return - with pytest.raises(TypeError, match="_put_user_input_event"): + with pytest.raises(TypeError, match="put_user_input_event"): _Incomplete() # type: ignore[abstract] diff --git a/flashdreams/test_v2/test_session_runner.py b/flashdreams/test_v2/test_session_runner.py index 301b3cf16..4eb8b336e 100644 --- a/flashdreams/test_v2/test_session_runner.py +++ b/flashdreams/test_v2/test_session_runner.py @@ -690,7 +690,7 @@ def _request_file_selector( self.file_selection_requests.append(request) super()._request_file_selector(selector_id, request) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: with self._lock: self._file_events.append(event) diff --git a/flashdreams/tests/test_profiler.py b/flashdreams/tests/test_profiler.py index 935494a13..409dcd77e 100644 --- a/flashdreams/tests/test_profiler.py +++ b/flashdreams/tests/test_profiler.py @@ -351,7 +351,7 @@ def get_user_input_events(self) -> UserInputEvents: self.batches += 1 return UserInputEvents([_key_press()]) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/integrations_v2/red_screen/red_screen/tests/test_red_screen.py b/integrations_v2/red_screen/red_screen/tests/test_red_screen.py index 382546640..f0801ff2f 100644 --- a/integrations_v2/red_screen/red_screen/tests/test_red_screen.py +++ b/integrations_v2/red_screen/red_screen/tests/test_red_screen.py @@ -63,7 +63,7 @@ def get_user_input_events(self) -> UserInputEvents: return self._batches.pop(0) return UserInputEvents([]) - def _put_user_input_event(self, event: UserInputEvent) -> None: + def put_user_input_event(self, event: UserInputEvent) -> None: del event def open(self, session_desc: SessionDesc) -> None: diff --git a/tests/test_step_agreement.py b/tests/test_step_agreement.py index 64ed228e1..5a962f3b9 100644 --- a/tests/test_step_agreement.py +++ b/tests/test_step_agreement.py @@ -164,7 +164,7 @@ def _main_thread(self): assert threading.get_ident() == self.session.main_thread assert self.session.ctx.is_main - def _put_user_input_event(self, event): + def put_user_input_event(self, event): del event def open(self, desc):