diff --git a/apps/action2v/tests/test_application.py b/apps/action2v/tests/test_application.py index c767d5950..fbfc71bb7 100644 --- a/apps/action2v/tests/test_application.py +++ b/apps/action2v/tests/test_application.py @@ -22,6 +22,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.session_runner import run_session from flashdreams.runtime_v2.step_result import StepResult @@ -138,6 +139,9 @@ def __init__(self) -> None: self.opened = False self.cursor_requests: list[tuple[str, bool]] = [] + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc self.opened = True diff --git a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py index 78c8668c0..8e8aed730 100644 --- a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py +++ b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py @@ -26,6 +26,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -243,6 +244,10 @@ def open(self, session_desc: SessionDesc) -> None: self._disconnect_emitted = False self._last_status = None + def put_user_input_event(self, event: UserInputEvent) -> None: + """Forward event to the presentation backend, merged with its input.""" + self.output_window.put_user_input_event(event) + def get_user_input_events(self) -> UserInputEvents: """Merge source-clock controller history with buffered window input.""" # The source owns XR input. Operator B only signals its headset worker; diff --git a/apps/t2v/t2v/testing.py b/apps/t2v/t2v/testing.py index 648639a68..ca78800a4 100644 --- a/apps/t2v/t2v/testing.py +++ b/apps/t2v/t2v/testing.py @@ -21,6 +21,7 @@ import torch from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.blit_model_output_to_screen_loop import ( BlitModelOutputToScreenLoop, ) @@ -410,6 +411,9 @@ class _DiscardingClientWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc diff --git a/apps/t2v/tests/test_cli.py b/apps/t2v/tests/test_cli.py index 8521ddeea..62e2261f7 100644 --- a/apps/t2v/tests/test_cli.py +++ b/apps/t2v/tests/test_cli.py @@ -27,6 +27,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2 import cli from flashdreams.runtime_v2.application_registry import ( APPLICATION_ENTRY_POINT_GROUP, @@ -160,6 +161,9 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc diff --git a/flashdreams/flashdreams/api_v2/README.md b/flashdreams/flashdreams/api_v2/README.md index 4bdad8a4b..f50ff82d2 100644 --- a/flashdreams/flashdreams/api_v2/README.md +++ b/flashdreams/flashdreams/api_v2/README.md @@ -245,7 +245,13 @@ UI control that needs a fresh application session calls `request_new_session(session_desc)` with a fully resolved replacement description; the runtime cleans the current session and passes that description to `ApplicationRunner` unchanged. Remaining timeout and remaining model steps -still apply to the replacement. +still apply to the replacement. To let the user pick a client file, call +`request_file_selector(request_id, ...)` when they ask (for ImGui, when +`imgui.button(...)` returns `True`), with one stable `request_id` per control. +The result arrives on a later tick as one `SelectedFilesUserInputEvent` +carrying that `request_id`; read its `status` before its `files`. One picker +is open at a time, and a call made while one is open is answered `BUSY`. The +`request_file_selector` docstring lists every status. For SlangPy's smaller retained widget API, subclass `SlangPyUILoop` from `flashdreams.runtime_v2.slangpy_ui_loop`. The diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 25b55131c..840875df2 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -3,14 +3,29 @@ """Client window abstract interface.""" +import logging import math -from abc import ABC -from dataclasses import dataclass +from abc import ABC, abstractmethod +from dataclasses import dataclass, replace +from pathlib import Path +from typing import final from urllib.parse import urlsplit +from numpy import uint64 + +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + selected_files_policy_status, +) +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent + from .input_source import InputSource from .output_sink import OutputSink +_LOGGER = logging.getLogger(__name__) + @dataclass(frozen=True, slots=True) class ClientWindowLink: @@ -89,6 +104,10 @@ class IClientWindow(InputSource, OutputSink, ABC): One thread makes every call on a window, so an implementation needs no locking except when its backend delivers input from another thread. + A window with a file picker implements :meth:`_request_file_selector`, and + :meth:`_cancel_active_file_selectors` if it can withdraw a shown picker. It + reports results on the UI thread. + Created by the runtime, never by an application. """ @@ -117,3 +136,117 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: new_window_size: Requested ``(width, height)`` in pixels. """ pass + + # File Selector Methods & Members + _active_file_selector: tuple[int, FileSelectionRequest] | None = None + _last_file_selector_id = 0 + + @final + def request_file_selector(self, request: FileSelectionRequest) -> None: + """Request for IClientWindow to open a file selector; the result of the + user interaction arrives later as a ``SelectedFilesUserInputEvent`` via + :meth:`InputSource.get_user_input_events`. + + A request made while a picker is open is answered with a status of + ``SelectedFilesStatus.BUSY``. + + Args: + request: Selector request queued by the UI loop. + """ + if self._active_file_selector is not None: + self.put_user_input_event( + _selected_files_event(request, SelectedFilesStatus.BUSY) + ) + return + selector_id = self._last_file_selector_id + 1 + self._last_file_selector_id = selector_id + self._active_file_selector = (selector_id, request) + try: + self._request_file_selector(selector_id, request) + except Exception: + _LOGGER.exception("File selector %r failed to start.", request.request_id) + self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + + @final + def flush_file_selectors(self) -> None: + """Forget the open picker without a result, for a session that ended. + + The window withdraws a picker still on screen where it can; whatever + that picker returns is dropped. + """ + active = self._active_file_selector + self._active_file_selector = None + if active is not None: + self._cancel_active_file_selectors() + + # Optional to implement + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Show a picker for ``request`` without blocking. + + Call :meth:`_complete_file_selector` with ``selector_id`` on the UI + thread when the user finishes. The default has no picker and completes + as ``UNAVAILABLE``. + """ + del request + self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + + # Optional to implement + def _cancel_active_file_selectors(self) -> None: + """Withdraw the picker that was flushed.""" + pass + + @final + def _complete_file_selector( + self, + selector_id: int, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), + ) -> None: + """Report the result for ``selector_id``; a stale id is dropped. + + A result that breaks the request's rules (type, size, count) is + reported by the status it earns instead. + """ + active = self._active_file_selector + if active is None or active[0] != selector_id: + return + self._active_file_selector = None + request = active[1] + status, files = _check_file_selection_result(request, status, files) + self.put_user_input_event(_selected_files_event(request, status, files)) + + +def _selected_files_event( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), +) -> SelectedFilesUserInputEvent: + """Return the event answering ``request``.""" + return SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=request.request_id, + status=status, + files=files, + ) + + +def _check_file_selection_result( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], +) -> tuple[SelectedFilesStatus, tuple[SelectedFile, ...]]: + """Return the status and files a window's result earns under ``request``.""" + if status is not SelectedFilesStatus.OK: + return status, () + if not files: + _LOGGER.warning("File selector %r returned no valid set.", request.request_id) + return SelectedFilesStatus.UNAVAILABLE, () + files = tuple(replace(chosen, name=Path(chosen.name).name) for chosen in files) + rejected = selected_files_policy_status( + [(chosen.name, len(chosen.data)) for chosen in files], request + ) + if rejected is not None: + return rejected, () + return status, files diff --git a/flashdreams/flashdreams/api_v2/input_source.py b/flashdreams/flashdreams/api_v2/input_source.py index d64a4df88..dcbf8a572 100644 --- a/flashdreams/flashdreams/api_v2/input_source.py +++ b/flashdreams/flashdreams/api_v2/input_source.py @@ -6,6 +6,7 @@ from abc import abstractmethod from typing import Protocol, runtime_checkable +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.user_input_events import UserInputEvents @@ -31,3 +32,13 @@ def get_user_input_events(self) -> UserInputEvents: Events in timestamp order, empty when nothing arrived. """ ... + + @abstractmethod + def put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event`` for :meth:`get_user_input_events`. + + Canonical method for how user-input-events are added to an input source. + Called on the UI thread, by ``IClientWindow`` and by windows that wrap + another window. + """ + ... diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 11078776b..325206b98 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -9,9 +9,10 @@ import threading import time from abc import ABC, abstractmethod -from collections.abc import Callable -from dataclasses import dataclass +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field from enum import Enum +from pathlib import Path from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar, final import torch @@ -25,6 +26,13 @@ bind_inference_profiler, unbind_inference_profiler, ) +from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_FILE_COUNT, + DEFAULT_MAX_TOTAL_BYTES, + FileSelectionRequest, + normalize_selected_file_accept, +) from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( CloseUserInputEvent, @@ -101,6 +109,9 @@ class UILoopRequests: links: tuple[ClientWindowLink, ...] | None = None """Replacement clickable regions, or ``None`` to leave them unchanged.""" + file_selections: list[FileSelectionRequest] = field(default_factory=list) + """File-selector requests in call order. The window answers each one.""" + class ILoop(ABC, Generic[StateT]): """Shared state, messaging, and lifecycle for a session loop. @@ -257,6 +268,8 @@ def _finish_run( if step_completed: self.latest_result = result self._step_index += 1 + # Input read by a run with no step is dropped, such as an ON_DEMAND UI + # tick without a new frame. self._pending_user_events.clear() @final @@ -604,6 +617,116 @@ def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: raise ValueError("At most sixteen client-window links are supported.") self.get_or_create_ui_loop_requests().links = links + @final + def request_file_selector( + self, + request_id: str, + initial_path: str | None = None, + *, + accept: Sequence[str] = (), + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, + max_file_count: int = DEFAULT_MAX_FILE_COUNT, + ) -> None: + """Ask the client to show a file selector. + + The result arrives on a later tick, never during the step that asked, + as one + :class:`~flashdreams.runtime_v2.user_input_event.SelectedFilesUserInputEvent` + carrying ``request_id``. One selector is open at a time; a call made + while one is open is answered ``BUSY``. A selector stays open across a + session reset; a replaced or ended session gets no result. + + Read ``status`` first; ``files`` is empty unless it is ``OK``: + + - ``OK``: each chosen file as its base ``name`` and ``data`` bytes. + - ``CANCELLED``: the user dismissed the selector. + - ``BUSY``: another selector was open. + - ``DISALLOWED_TYPE``: a file did not match ``accept``. + - ``TOO_LARGE``: a file was larger than ``max_file_bytes``, the files + together were larger than ``max_total_bytes``, or more than + ``max_file_count`` files were chosen. + - ``UNAVAILABLE``: the window cannot show a selector, or it failed. + + ``accept`` checks filename suffixes only, so treat ``data`` as + untrusted input. + + Example: + + .. code-block:: python + + from flashdreams.runtime_v2.selected_file import SelectedFilesStatus + from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, + ) + + if imgui.button("Open image"): + self.request_file_selector("open-image", accept=(".png",)) + for event in self.user_events.get_events(): + if not isinstance(event, SelectedFilesUserInputEvent): + continue + if event.request_id != "open-image": + continue + if event.status is SelectedFilesStatus.OK: + image_bytes = event.files[0].data + else: + self.state.message = f"No image: {event.status.value}" + + Args: + request_id: Names the control that asked, so its result can be + told apart from another control's. Echoed on the result. + initial_path: Directory hint for the native selector; ``None`` + uses the host user's home directory. The browser ignores it. + accept: Filename suffixes such as ``.png``. Empty allows any type. + max_file_bytes: Largest single file in bytes. Must not exceed + ``max_total_bytes``. + max_total_bytes: Most bytes all chosen files may add up to. Has no + effect when ``max_file_count`` is one. + max_file_count: Most files the selector may return. Above one, the + user may pick several. + + Raises: + TypeError: ``request_id`` or ``initial_path`` is not a string, or + ``accept`` or a limit has the wrong type. + ValueError: ``request_id`` or ``initial_path`` is empty, ``accept`` + is invalid, a limit is not positive, or ``max_file_bytes`` + exceeds ``max_total_bytes``. + """ + if not isinstance(request_id, str): + raise TypeError("request_id must be a string.") + if not request_id: + raise ValueError("request_id must be a non-empty string.") + if initial_path is None: + resolved_initial_path = str(Path.home()) + else: + if not isinstance(initial_path, str): + raise TypeError("initial_path must be a string.") + if not initial_path: + raise ValueError("initial_path must be a non-empty string.") + resolved_initial_path = initial_path + suffixes = normalize_selected_file_accept(accept) + for name, limit in ( + ("max_file_bytes", max_file_bytes), + ("max_total_bytes", max_total_bytes), + ("max_file_count", max_file_count), + ): + if isinstance(limit, bool) or not isinstance(limit, int): + raise TypeError(f"{name} must be an integer.") + if limit <= 0: + raise ValueError(f"{name} must be > 0.") + if max_file_bytes > max_total_bytes: + raise ValueError("max_file_bytes must not exceed max_total_bytes.") + self.get_or_create_ui_loop_requests().file_selections.append( + FileSelectionRequest( + request_id=request_id, + initial_path=resolved_initial_path, + accept=suffixes, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + max_file_count=max_file_count, + ) + ) + def get_or_create_ui_loop_requests(self) -> UILoopRequests: if self._ui_loop_requests is None: self._ui_loop_requests = UILoopRequests() @@ -699,6 +822,7 @@ def invoke_async(loop: ILoop[StateT], operation: Callable[[StateT], None]) -> No __all__ = [ + "FileSelectionRequest", "ILoop", "IModelLoop", "IUILoop", diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 6f2cd1398..ba4df4118 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -370,16 +370,22 @@ delegates over the thing that does the work — `Mp4ClientWindow` over the output shape, because the session already did. They differ in what they can do rather than in how they are driven. A session -chooses its UI independently of the client window. Browser keyboard, mouse and -focus events arrive on the server's own thread, so it queues them and hands them -over in batches when the session asks. +chooses its UI independently of the client window. Browser keyboard, mouse, +focus, and file-selector events arrive on the server's own thread, so it queues +them and hands them over in batches when the session asks. + +File selection rules live in `IClientWindow`; a window only shows a picker and +reports what it returned, on the UI thread. Every window implements +`InputSource.put_user_input_event`; a window with a picker also implements +`_request_file_selector`. A UI loop may call `request_new_session(session_desc)` during its step with a fully resolved replacement description. `run_session` stops and cleans the current session, leaves the interactive window open, and returns that description unchanged. `ApplicationRunner` creates the replacement from it. A WebRTC browser disconnect releases only its peer connection, so refreshing the -page does not stop the session, server, or application. +page does not stop the session, server, or application. Leftover and in-flight +file picks complete as `unavailable`; a replacement viewer does not inherit them. The UI thread owns WebRTC cadence. Each `write` synchronously materializes one owned video frame and admits it to a two-frame FIFO of unsent frames. The WebRTC diff --git a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py index 3e2c72ac6..300b926a3 100644 --- a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py @@ -6,6 +6,7 @@ from pathlib import Path from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.mp4_output_sink import Mp4OutputSink from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -13,7 +14,7 @@ class Mp4ClientWindow(IClientWindow): - """Write UI frames to an MP4 file and report no input. + """Write UI frames to an MP4 file. It has no user input of its own. The session must finish on its own because this window never sends a close event. Use ``BackpressureMode.BLOCK`` with @@ -27,6 +28,7 @@ def __init__(self, path: str | Path) -> None: """ self._path = Path(path) self._video_sink = Mp4OutputSink(path) + self._input_events: list[UserInputEvent] = [] @property def path(self) -> Path: @@ -34,8 +36,12 @@ def path(self) -> Path: return self._path def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return the events queued since the last call.""" + events, self._input_events = self._input_events, [] + return UserInputEvents(events) + + def put_user_input_event(self, event: UserInputEvent) -> None: + self._input_events.append(event) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the MP4 presentation dimensions without replacing the session. @@ -52,6 +58,7 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: def open(self, session_desc: SessionDesc) -> None: """Prepare to write a session's output.""" + self._input_events = [] self._video_sink.open(session_desc) def write(self, result: StepResult) -> None: diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index e2a8c2af1..2ad057613 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -7,12 +7,14 @@ import logging import queue +import stat import threading import time import webbrowser from collections import deque -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import replace +from pathlib import Path from typing import TYPE_CHECKING, Any, cast import numpy as np @@ -22,6 +24,12 @@ from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow from flashdreams.api_v2.user_input_event import UserInputEvent +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + selected_files_policy_status, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -40,6 +48,76 @@ _LOGGER = logging.getLogger(__name__) + +def _choose_paths(request: FileSelectionRequest) -> list[str]: + """Block in the Tk file dialog and return the chosen paths. + + Tk shows Explorer on Windows and its own dialog on Linux. Raises when Tk is + missing or cannot open a display. + """ + import tkinter as tk + from tkinter import filedialog + + root = tk.Tk() + try: + root.withdraw() + try: + root.attributes("-topmost", True) + except tk.TclError: + pass + kwargs: dict[str, Any] = { + "parent": root, + "initialdir": request.initial_path or str(Path.home()), + "title": "Select a file", + } + if request.accept: + kwargs["filetypes"] = [ + ("Accepted", " ".join("*" + suffix for suffix in request.accept)) + ] + if request.max_file_count > 1: + return [str(path) for path in filedialog.askopenfilenames(**kwargs) if path] + path = filedialog.askopenfilename(**kwargs) + return [str(path)] if path else [] + finally: + root.destroy() + + +def _selected_files_from_paths( + paths: Sequence[str], request: FileSelectionRequest +) -> tuple[tuple[SelectedFile, ...], SelectedFilesStatus]: + """Load chosen files, or empty files and a failure status.""" + rejected = selected_files_policy_status( + [(Path(path).name, None) for path in paths], request + ) + if rejected is not None: + return (), rejected + sized: list[tuple[str, int]] = [] + for path in paths: + try: + info = Path(path).stat() + except OSError: + return (), SelectedFilesStatus.UNAVAILABLE + if not stat.S_ISREG(info.st_mode): + return (), SelectedFilesStatus.UNAVAILABLE + sized.append((path, info.st_size)) + rejected = selected_files_policy_status( + [(Path(path).name, size) for path, size in sized], request + ) + if rejected is not None: + return (), rejected + files: list[SelectedFile] = [] + for path, _size in sized: + try: + with open(path, "rb") as stream: + # Read one byte past the limit so a file over it is + # rejected, not truncated. + data = stream.read(request.max_file_bytes + 1) + except OSError: + return (), SelectedFilesStatus.UNAVAILABLE + files.append(SelectedFile(name=Path(path).name, data=data)) + return tuple(files), SelectedFilesStatus.OK + + _PRINTABLE_KEY_NAMES = { "space": " ", "apostrophe": "'", @@ -143,6 +221,10 @@ def __init__( self._lock_cursor_to_window = False self._links: tuple[ClientWindowLink, ...] = () self._pressed_link: ClientWindowLink | None = None + self._file_dialog_thread: threading.Thread | None = None + self._file_dialog_results: queue.SimpleQueue[ + tuple[int, FileSelectionRequest, list[str] | None] + ] = queue.SimpleQueue() def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace the clickable regions over the native window output.""" @@ -187,6 +269,56 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: ) self._window_size = presenter.resize(*new_window_size) + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Show the Tk file dialog on a worker thread that queues its result.""" + thread = self._file_dialog_thread + if thread is not None and thread.is_alive(): + # A flushed dialog stays open until the user closes it, and two Tk + # dialogs on separate threads are not safe. + self._complete_file_selector(selector_id, SelectedFilesStatus.BUSY) + return + results = self._file_dialog_results + + def show_dialog() -> None: + try: + paths: list[str] | None = _choose_paths(request) + except Exception: + _LOGGER.exception( + "Native file picker failed for request id %r.", request.request_id + ) + paths = None + results.put((selector_id, request, paths)) + + thread = threading.Thread( + target=show_dialog, name="flashdreams-file-dialog", daemon=True + ) + thread.start() + self._file_dialog_thread = thread + + def _poll_file_dialog(self) -> None: + """Report the results the dialog thread queued; stale ones are dropped.""" + while True: + try: + selector_id, request, paths = self._file_dialog_results.get_nowait() + except queue.Empty: + return + active = self._active_file_selector + if active is None or active[0] != selector_id: + continue + files: tuple[SelectedFile, ...] = () + if paths is None: + status = SelectedFilesStatus.UNAVAILABLE + elif not paths: + status = SelectedFilesStatus.CANCELLED + else: + # ponytail: files are read on the UI thread, up to the + # request's max_total_bytes, which stalls presentation while + # they load. Upgrade path: read them on the dialog thread. + files, status = _selected_files_from_paths(paths, request) + self._complete_file_selector(selector_id, status, files) + def open(self, session_desc: SessionDesc) -> None: """Open a session in the GLFW window on the runtime's UI thread. @@ -251,6 +383,7 @@ def open(self, session_desc: SessionDesc) -> None: def get_user_input_events(self) -> UserInputEvents: """Pump GLFW and return native input events not yet read.""" + self._poll_file_dialog() presenter = self._presenter if presenter is not None: pending_before_poll = tuple(self._pending_printable_keys) @@ -453,6 +586,12 @@ def _on_window_closed(self) -> None: self._close_event_enqueued = True self._put_input(CloseUserInputEvent(timestamp=uint64(0))) + def put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event``, bypassing the poll-time buffer.""" + started_ns = self._session_started_ns + elapsed_ns = 0 if started_ns is None else max(0, self._clock_ns() - started_ns) + self._input_events.put(replace(event, timestamp=uint64(elapsed_ns // 1_000))) + class _SlangPyNativeWindowPresenter: """Own the SlangPy window and forward frames to its presentation context.""" diff --git a/flashdreams/flashdreams/runtime_v2/null_client_window.py b/flashdreams/flashdreams/runtime_v2/null_client_window.py index 76ff107fa..8255d7841 100644 --- a/flashdreams/flashdreams/runtime_v2/null_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/null_client_window.py @@ -4,21 +4,30 @@ """Client window that discards output and reports no input.""" from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_events import UserInputEvents class NullClientWindow(IClientWindow): - """Discard output and report no input.""" + """Discard output. It has no user input of its own.""" + + def __init__(self) -> None: + self._input_events: list[UserInputEvent] = [] def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return the events queued since the last call.""" + events, self._input_events = self._input_events, [] + return UserInputEvents(events) + + def put_user_input_event(self, event: UserInputEvent) -> None: + self._input_events.append(event) def open(self, session_desc: SessionDesc) -> None: """Accept a session without opening an output.""" del session_desc + self._input_events = [] def write(self, result: StepResult) -> None: """Discard one result.""" diff --git a/flashdreams/flashdreams/runtime_v2/selected_file.py b/flashdreams/flashdreams/runtime_v2/selected_file.py new file mode 100644 index 000000000..63e450f1c --- /dev/null +++ b/flashdreams/flashdreams/runtime_v2/selected_file.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Selected-file types, request policy, and size/type checks.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum + +DEFAULT_MAX_FILE_BYTES = 1024 * 1024 * 1024 +"""Default ``max_file_bytes`` (1 GiB): the largest single file a request accepts.""" + +DEFAULT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024 +"""Default ``max_total_bytes`` (1 GiB): the most bytes all chosen files add up to.""" + +DEFAULT_MAX_FILE_COUNT = 1 +"""Default ``max_file_count``: one file, unless the app asks for more.""" + + +def normalize_selected_file_accept(accept: Sequence[str] = ()) -> tuple[str, ...]: + """Return accepted filename suffixes such as ``.png``. + + Args: + accept: Suffixes the client may choose. Empty means any type. + + Returns: + The suffixes as a tuple, unchanged except for the sequence type. + + Raises: + TypeError: ``accept`` is not a sequence of strings. + ValueError: A suffix is missing the leading ``.`` or contains a path. + """ + if isinstance(accept, str) or not isinstance(accept, Sequence): + raise TypeError("accept must be a sequence of suffixes.") + suffixes: list[str] = [] + for suffix in accept: + if not isinstance(suffix, str): + raise TypeError("accept suffixes must be strings.") + if ( + not suffix.startswith(".") + or suffix == "." + or "/" in suffix + or "\\" in suffix + ): + raise ValueError( + "accept suffixes must look like ``.png`` and cannot contain a path." + ) + suffixes.append(suffix) + return tuple(suffixes) + + +def selected_file_suffix_allowed(name: str, accept: tuple[str, ...]) -> bool: + """Return whether ``name`` matches ``accept``. + + Empty ``accept`` allows any name. Comparison is case-insensitive. + """ + if not accept: + return True + lower = name.lower() + return any(lower.endswith(suffix.lower()) for suffix in accept) + + +class SelectedFilesStatus(Enum): + """Outcome of one client file-selector request.""" + + OK = "ok" + """The client chose files that passed type and size checks.""" + + CANCELLED = "cancelled" + """The user dismissed the selector.""" + + TOO_LARGE = "too_large" + """A file was larger than ``max_file_bytes``, the files together were larger + than ``max_total_bytes``, or more than ``max_file_count`` files were chosen.""" + + DISALLOWED_TYPE = "disallowed_type" + """A chosen file did not match ``accept``. Checked before size.""" + + BUSY = "busy" + """Another selector was already open, so this request was not shown. The + open selector keeps going and answers its own request.""" + + UNAVAILABLE = "unavailable" + """No pick was made: the client has no picker or left, the picker failed, + or a chosen file was unreadable.""" + + +@dataclass(frozen=True, slots=True) +class SelectedFile: + """One file chosen by a client file selector.""" + + name: str + """Base name of the chosen file, without any host path.""" + + data: bytes + """File contents.""" + + +@dataclass(frozen=True, slots=True) +class FileSelectionRequest: + """One client file-selector request queued by a UI loop.""" + + request_id: str + """Stable id for which button this request is for. Echoed on the matching event.""" + + initial_path: str + """Directory hint for the selector.""" + + accept: tuple[str, ...] = () + """Filename suffixes the client may choose, empty for any type.""" + + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES + """Largest single file in bytes.""" + + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES + """Most bytes all chosen files may add up to.""" + + max_file_count: int = DEFAULT_MAX_FILE_COUNT + """Most files the selector may return. Above one, the user may pick several.""" + + +def selected_files_policy_status( + files: Sequence[tuple[str, int | None]], + request: FileSelectionRequest, +) -> SelectedFilesStatus | None: + """Return why a chosen set breaks ``request``, or ``None`` if it passes. + + Type is checked before size and count, across the whole set. + + Args: + files: ``(name, size)`` pairs. ``size`` of ``None`` skips that file's + size check and omits it from the total. + request: The request whose ``accept`` and limits apply. + + Returns: + ``DISALLOWED_TYPE``, ``TOO_LARGE``, or ``None`` when the set passes. + """ + for name, _size in files: + if not selected_file_suffix_allowed(name, request.accept): + return SelectedFilesStatus.DISALLOWED_TYPE + if len(files) > request.max_file_count: + return SelectedFilesStatus.TOO_LARGE + total = 0 + for _name, size in files: + if size is None: + continue + if size > request.max_file_bytes: + return SelectedFilesStatus.TOO_LARGE + total += size + if total > request.max_total_bytes: + return SelectedFilesStatus.TOO_LARGE + return None diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 57a659621..4e94d4670 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -7,6 +7,12 @@ const pointerControls = peer.createDataChannel("pointer-controls"); peer.addTransceiver("video", {direction: "recvonly"}); const video = document.getElementById("video"); const status = document.getElementById("status"); +const promptDialog = document.getElementById("prompt-dialog"); +const promptBackdrop = document.getElementById("prompt-backdrop"); +const promptDialogTitle = document.getElementById("prompt-dialog-title"); +const promptDialogDetails = document.getElementById("prompt-dialog-details"); +const promptDialogConfirm = document.getElementById("prompt-dialog-confirm"); +const promptDialogCancel = document.getElementById("prompt-dialog-cancel"); const pressedKeys = new Map(); const pressedButtons = new Set(); const gamepadSnapshots = new Map(); @@ -74,6 +80,321 @@ const applyCursorOptions = options => { } }; +let promptSettled = false; +let promptOnConfirm = null; +let promptOnCancel = null; + +const setPromptBackdropVisible = visible => { + if (promptBackdrop !== null) { + promptBackdrop.hidden = !visible; + } +}; + +const closePromptDialog = () => { + setPromptBackdropVisible(false); + if (promptDialog !== null && promptDialog.open) { + promptDialog.close(); + } +}; + +const fillPromptDetails = details => { + if (promptDialogDetails === null) { + return; + } + promptDialogDetails.replaceChildren(); + if (!Array.isArray(details) || details.length === 0) { + promptDialogDetails.hidden = true; + return; + } + for (const section of details) { + const row = document.createElement("div"); + row.className = "prompt-dialog-row"; + const label = document.createElement("div"); + label.className = "prompt-dialog-detail-label"; + label.textContent = section.label; + const value = document.createElement("div"); + value.className = "prompt-dialog-row-value"; + const values = Array.isArray(section.items) ? section.items : []; + for (const item of values) { + const chip = document.createElement("span"); + chip.className = "prompt-dialog-chip"; + chip.textContent = item; + value.append(chip); + } + row.append(label, value); + promptDialogDetails.append(row); + } + promptDialogDetails.hidden = false; +}; + +const showPromptDialog = ({ + title, + details = [], + confirmLabel = "OK", + cancelLabel = "Cancel", + onConfirm = null, + onCancel = null, +} = {}) => { + if ( + promptDialog === null + || promptDialogTitle === null + || promptDialogConfirm === null + || promptDialogCancel === null + ) { + return; + } + if (document.pointerLockElement === video) { + document.exitPointerLock(); + } + if (promptDialog.open) { + promptSettled = true; + promptOnConfirm = null; + promptOnCancel = null; + promptDialog.close(); + } + promptDialogTitle.textContent = title; + fillPromptDetails(details); + promptDialogConfirm.textContent = confirmLabel; + promptDialogCancel.textContent = cancelLabel; + promptOnConfirm = onConfirm; + promptOnCancel = onCancel; + promptSettled = false; + setPromptBackdropVisible(true); + promptDialog.show(); + promptDialogConfirm.focus(); +}; + +if (promptDialog !== null) { + promptDialogConfirm?.addEventListener("click", () => { + if (promptSettled) { + return; + } + promptSettled = true; + const onConfirm = promptOnConfirm; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + onConfirm?.(); + }); + promptDialogCancel?.addEventListener("click", () => { + if (promptSettled) { + return; + } + promptSettled = true; + const onCancel = promptOnCancel; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + onCancel?.(); + }); + promptDialog.addEventListener("close", () => { + // The event is queued, so it can arrive after a newer prompt reopened the dialog. + if (promptDialog.open) { + return; + } + setPromptBackdropVisible(false); + if (promptSettled) { + return; + } + promptSettled = true; + const onCancel = promptOnCancel; + promptOnConfirm = null; + promptOnCancel = null; + onCancel?.(); + }); +} + +window.addEventListener("keydown", event => { + if (event.key !== "Escape" || promptDialog === null || !promptDialog.open) { + return; + } + event.preventDefault(); + event.stopImmediatePropagation(); + promptDialogCancel?.click(); +}); + +if (promptBackdrop !== null) { + promptBackdrop.addEventListener("click", () => { + promptDialogCancel?.click(); + }); +} + +let pendingFileRequest = null; + +const hideFilePicker = () => { + pendingFileRequest = null; + if (promptDialog !== null && promptDialog.open) { + promptSettled = true; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + } +}; + +const completeFileSelection = (requestId, status) => { + send({type: "file_selector_result", id: requestId, status}); +}; + +const formatByteBudget = bytes => { + if (!Number.isFinite(bytes) || bytes <= 0) { + return "the allowed size"; + } + const mib = bytes / (1024 * 1024); + if (mib >= 1) { + return Number.isInteger(mib) ? `${mib} MiB` : `${mib.toFixed(1)} MiB`; + } + const kib = bytes / 1024; + if (kib >= 1) { + return Number.isInteger(kib) ? `${kib} KiB` : `${kib.toFixed(1)} KiB`; + } + return `${bytes} bytes`; +}; + +const fileNameMatchesAccept = (name, accept) => { + if (!Array.isArray(accept) || accept.length === 0) { + return true; + } + const lower = name.toLowerCase(); + return accept.some( + suffix => typeof suffix === "string" && lower.endsWith(suffix.toLowerCase()), + ); +}; + +const startFileInput = pending => { + const requestId = pending.id; + const input = document.createElement("input"); + input.type = "file"; + if (pending.accept.length > 0) { + input.accept = pending.accept.join(","); + } + input.multiple = pending.multiple === true; + let settled = false; + const finishFileSelection = status => { + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + completeFileSelection(requestId, status); + hideFilePicker(); + }; + input.addEventListener("change", () => { + if (pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + const files = pending.multiple + ? Array.from(input.files || []) + : Array.from(input.files || []).slice(0, 1); + if (files.length === 0) { + finishFileSelection("cancelled"); + return; + } + let rejected = null; + for (const file of files) { + if (!fileNameMatchesAccept(file.name || "", pending.accept)) { + rejected = "disallowed_type"; + break; + } + } + if (rejected === null) { + for (const file of files) { + if (file.size > pending.maxBytes) { + rejected = "too_large"; + break; + } + } + } + if (rejected !== null) { + finishFileSelection(rejected); + return; + } + // One request per file, in order, so the server reads only file bytes. + const uploadFiles = async () => { + for (const [index, file] of files.entries()) { + const query = new URLSearchParams({ + id: requestId, + name: file.name, + index: String(index), + count: String(files.length), + }); + const response = await fetch(`/api/files?${query}`, {method: "POST", body: file}); + if (!response.ok) { + // A proxy may refuse the upload before the server can answer it. + console.debug("File upload was rejected.", response.status); + finishFileSelection("unavailable"); + return; + } + } + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + hideFilePicker(); + }; + uploadFiles().catch(error => { + console.debug("Unable to upload the selected file.", error); + finishFileSelection("unavailable"); + }); + }); + input.addEventListener("cancel", () => finishFileSelection("cancelled")); + input.click(); +}; + +const openFileSelector = options => { + const requestId = options?.id; + if (typeof requestId !== "string" || !requestId) { + return; + } + if (pendingFileRequest !== null) { + if (pendingFileRequest.id === requestId) { + return; + } + // The server opens one selector at a time, so a new one means the old one finished. + hideFilePicker(); + } + const accept = Array.isArray(options.accept) ? options.accept : []; + const maxBytes = Number(options.max_file_bytes); + pendingFileRequest = { + id: requestId, + accept, + maxBytes: Number.isFinite(maxBytes) ? maxBytes : 0, + multiple: Number(options.max_file_count) > 1, + }; + showPromptDialog({ + title: pendingFileRequest.multiple + ? "The app requested files" + : "The app requested a file", + details: [ + { + label: "Accepted types", + items: accept.length > 0 ? accept : ["Any"], + }, + { + label: "Max size per file", + items: [formatByteBudget(pendingFileRequest.maxBytes)], + }, + ], + confirmLabel: "Select…", + onConfirm: () => { + const pending = pendingFileRequest; + if (pending === null || pending.id !== requestId) { + return; + } + startFileInput(pending); + }, + onCancel: () => { + if (pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + completeFileSelection(requestId, "cancelled"); + hideFilePicker(); + }, + }); + if (promptDialog === null || !promptDialog.open) { + completeFileSelection(requestId, "unavailable"); + hideFilePicker(); + } +}; + window.addEventListener("focus", updateCursorVisibility); window.addEventListener("blur", updateCursorVisibility); @@ -91,6 +412,14 @@ controls.addEventListener("message", event => { openUrl(payload.url); return; } + if (payload?.type === "file_selector") { + openFileSelector(payload); + return; + } + if (payload?.type === "file_selector_cancel") { + hideFilePicker(); + return; + } if (payload?.type === "error") { console.warn(`WebRTC server: ${payload.message ?? "unknown error"}`); } @@ -124,7 +453,7 @@ peer.addEventListener("connectionstatechange", () => { window.addEventListener("keydown", event => { const keyId = event.code || event.key; - if (pressedKeys.has(keyId)) { + if (pressedKeys.has(keyId) || (promptDialog !== null && promptDialog.open)) { return; } const wasSent = sendInput({ diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/index.html b/flashdreams/flashdreams/runtime_v2/serving/web/index.html index 1b73e1dc8..eb0d4ad2c 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/index.html +++ b/flashdreams/flashdreams/runtime_v2/serving/web/index.html @@ -48,11 +48,120 @@ color: #ff8a8a; } + #prompt-backdrop { + position: fixed; + inset: 0; + z-index: 2; + background: rgb(0 0 0 / 28%); + } + + #prompt-backdrop[hidden] { + display: none; + } + + #prompt-dialog { + position: fixed; + inset: 0; + z-index: 3; + margin: auto; + height: fit-content; + color-scheme: light dark; + max-width: 26rem; + width: min(26rem, calc(100% - 2rem)); + padding: 1.25rem 1.35rem 1rem; + border: 1px solid color-mix(in srgb, CanvasText 18%, Canvas); + border-radius: 12px; + background: Canvas; + color: CanvasText; + font: inherit; + box-shadow: 0 18px 40px rgb(0 0 0 / 45%); + } + + #prompt-dialog-title { + margin: 0 0 0.9rem; + font-size: 1.1rem; + font-weight: 650; + } + + #prompt-dialog-details { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + align-items: baseline; + column-gap: 0.85rem; + row-gap: 0.4rem; + margin: 0 0 1.15rem; + } + + #prompt-dialog-details[hidden] { + display: none; + } + + .prompt-dialog-row { + display: contents; + } + + .prompt-dialog-detail-label { + margin: 0; + font-size: 0.85rem; + color: color-mix(in srgb, CanvasText 62%, Canvas); + text-align: end; + white-space: nowrap; + } + + .prompt-dialog-detail-label::after { + content: ":"; + } + + .prompt-dialog-row-value { + display: flex; + flex-wrap: wrap; + column-gap: 0.55rem; + row-gap: 0.15rem; + } + + .prompt-dialog-chip { + font-size: 0.95rem; + } + + .prompt-dialog-actions { + display: flex; + justify-content: flex-end; + gap: 0.55rem; + } + + .prompt-dialog-actions button { + font: inherit; + padding: 0.45rem 0.95rem; + border-radius: 0.45rem; + cursor: pointer; + } + + #prompt-dialog-cancel { + border: 1px solid color-mix(in srgb, CanvasText 28%, Canvas); + background: transparent; + color: CanvasText; + } + + #prompt-dialog-confirm { + border: 0; + background: CanvasText; + color: Canvas; + } +
Waiting for the server…
+ + +

+ +
+ + +
+
diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 8550aed5d..8920873d5 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -8,12 +8,13 @@ import asyncio import json import math +import secrets import socket import threading import time from collections import deque from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from fractions import Fraction from importlib.resources import files from typing import Any, Literal, cast @@ -29,6 +30,13 @@ from flashdreams.api_v2.client_window import ClientWindowLink from flashdreams.runtime_v2.cuda_utils import resolve_cuda_device +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + selected_file_suffix_allowed, + selected_files_policy_status, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -295,6 +303,42 @@ async def _next_queued_frame(self) -> _QueuedRGBFrame: await self._frame_available.wait() +@dataclass(frozen=True, slots=True) +class _PendingFileSelection: + """One open file-selector request and the upload rules it sets.""" + + selector_id: str + """Window selector this upload answers. Never sent to the browser.""" + + request: FileSelectionRequest + """The app's request, whose ``accept`` and limits the upload must meet.""" + + +@dataclass(slots=True) +class _FileUpload: + """One pick whose files are arriving, one request per file.""" + + policy: _PendingFileSelection + """The open request this upload answers.""" + + count: int + """How many files the browser said it will send.""" + + files: list[SelectedFile] = field(default_factory=list) + """Files received so far, in order.""" + + total_bytes: int = 0 + """Bytes received so far across ``files``.""" + + reader: asyncio.Task[Any] | None = None + """The request reading a file now, if any. Another one meanwhile is refused.""" + + def stop(self) -> None: + """Stop the read in progress, so a forgotten upload frees its bytes now.""" + if self.reader is not None: + self.reader.cancel() + + class WebRTCServer: """Own the HTTP, signaling, input buffering, and media transport.""" @@ -350,6 +394,12 @@ def __init__( self._client_connected = False self._hide_cursor = False self._lock_cursor_to_window = False + # Upload tokens, touched only on the event loop once it runs. + self._pending_file_requests: dict[str, _PendingFileSelection] = {} + self._uploading_file_requests: dict[str, _FileUpload] = {} + self._file_selector_callback: ( + Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None] | None + ) = None self._thread = threading.Thread( target=self._run_server, name="flashdreams-webrtc", @@ -495,6 +545,91 @@ def _send_cursor_options(self) -> None: self._sent_cursor_options = cursor_options + def request_file_selector( + self, selector_id: str, request: FileSelectionRequest + ) -> None: + """Ask the connected browser to open a file selector. + + The browser cannot use a host path, so ``initial_path`` is ignored. + Without a connected viewer the request completes as ``UNAVAILABLE``. + + Args: + selector_id: Window selector reported back with this result. + request: File selector to show. + """ + policy = _PendingFileSelection(selector_id=selector_id, request=request) + loop = self._loop + if loop is None: + raise RuntimeError("WebRTC server is not running.") + loop.call_soon_threadsafe(self._open_client_side_file_selector, policy) + + def cancel_file_selectors(self) -> None: + """Forget every open selector and close the viewer's prompt. + + A later upload for a forgotten selector is refused. + """ + loop = self._loop + if loop is None: + self._pending_file_requests.clear() + return + # On the event loop, after any opening still queued for a forgotten selector. + loop.call_soon_threadsafe(self._withdraw_file_selectors) + + def _withdraw_file_selectors(self) -> None: + """Forget open selectors and tell the viewer to close its prompt.""" + self._pending_file_requests.clear() + for upload in self._uploading_file_requests.values(): + upload.stop() + self._uploading_file_requests.clear() + channel = self._control_channel + if channel is not None and channel.readyState == "open": + channel.send(json.dumps({"type": "file_selector_cancel"})) + + def _open_client_side_file_selector(self, policy: _PendingFileSelection) -> None: + """Send the selector to the browser and remember its upload policy. + + The browser gets an unguessable token instead of the selector id, so a + page that cannot read the data channel cannot answer the selector. + """ + channel = self._control_channel + if channel is None or channel.readyState != "open": + self._emit_selected_files( + policy.selector_id, status=SelectedFilesStatus.UNAVAILABLE + ) + return + token = secrets.token_urlsafe(16) + self._pending_file_requests[token] = policy + channel.send( + json.dumps( + { + "type": "file_selector", + "id": token, + "accept": list(policy.request.accept), + "max_file_bytes": policy.request.max_file_bytes, + "max_file_count": policy.request.max_file_count, + } + ) + ) + + def _take_pending_file_request(self, token: str) -> _PendingFileSelection | None: + """Return and consume the policy for ``token``, if it was pending.""" + return self._pending_file_requests.pop(token, None) + + def _complete_abandoned_file_selectors(self) -> None: + """Answer every open pick as ``UNAVAILABLE`` because the viewer left.""" + abandoned = ( + *self._pending_file_requests.values(), + *(upload.policy for upload in self._uploading_file_requests.values()), + ) + self._pending_file_requests.clear() + for upload in self._uploading_file_requests.values(): + upload.stop() + self._uploading_file_requests.clear() + for policy in abandoned: + self._emit_selected_files( + policy.selector_id, status=SelectedFilesStatus.UNAVAILABLE + ) + def event_timestamp_us(self) -> np.uint64: """Return the current timestamp relative to this server's first session. @@ -558,6 +693,10 @@ def open(self, session_desc: SessionDesc) -> None: # Keep timestamps comparable across sessions so events buffered # during a handoff retain their real order. self._event_origin_ns = time.monotonic_ns() + if loop is None: + self._pending_file_requests.clear() + else: + loop.call_soon_threadsafe(self._pending_file_requests.clear) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the dimensions used to present subsequent UI frames. @@ -614,6 +753,18 @@ def register_input_callback( raise RuntimeError("An input callback is already registered.") self._input_callback = callback + def register_file_selector_callback( + self, + callback: Callable[[str, SelectedFilesStatus, tuple[SelectedFile, ...]], None], + ) -> None: + """Register how open file selectors report back, by selector id. + + Args: + callback: Called once per selector on the server thread, with its + status and files. + """ + self._file_selector_callback = callback + def write(self, result: StepResult) -> None: """Materialize and admit one generated result to the sender mailbox. @@ -786,6 +937,7 @@ async def _start_server(self) -> None: app.router.add_get("/app.js", self._serve_browser_script) app.router.add_get("/healthz", self._health) app.router.add_post("/api/webrtc/offer", self._offer) + app.router.add_post("/api/files", self._upload_selected_file) runner = web.AppRunner(app) await runner.setup() address_family = socket.AF_INET6 if ":" in self._host else socket.AF_INET @@ -823,6 +975,95 @@ async def _health(self, _: web.Request) -> web.Response: } ) + async def _upload_selected_file(self, request: web.Request) -> web.Response: + """Receive one chosen file; the last one answers the pick. + + The browser posts each file as its own request, in order, with the raw + file bytes as the body. Every byte read is then a file byte, held to + the request's own limits before it is read. + """ + if self._closed: + raise web.HTTPServiceUnavailable(reason="WebRTC server is closed.") + if self._timestamp_us() is None: + raise web.HTTPConflict(reason="WebRTC server is not open.") + token = request.query.get("id") + if not isinstance(token, str) or not token: + raise web.HTTPBadRequest(reason="File upload requires an id.") + count = _query_int(request, "count") + upload = self._uploading_file_requests.get(token) + if upload is None: + policy = self._take_pending_file_request(token) + if policy is None: + raise web.HTTPConflict(reason="No pending file selection for this id.") + # Tracked until it ends so a viewer that leaves mid-upload is answered. + upload = _FileUpload(policy=policy, count=count or 0) + self._uploading_file_requests[token] = upload + + def finish( + status: SelectedFilesStatus, files: tuple[SelectedFile, ...] = () + ) -> None: + if self._uploading_file_requests.get(token) is upload: + del self._uploading_file_requests[token] + self._emit_selected_files( + upload.policy.selector_id, status=status, files=files + ) + + limits = upload.policy.request + name = request.query.get("name", "") + size = request.content_length + if ( + upload.reader is not None + or upload.count < 1 + or count != upload.count + or _query_int(request, "index") != len(upload.files) + or not name + ): + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File upload is out of order.") + if upload.count > limits.max_file_count: + finish(SelectedFilesStatus.TOO_LARGE) + raise web.HTTPBadRequest(reason="File upload has too many files.") + if not selected_file_suffix_allowed(name, limits.accept): + finish(SelectedFilesStatus.DISALLOWED_TYPE) + return web.Response(status=204) + if request.headers.get("Content-Encoding", "identity").lower() != "identity": + # aiohttp would inflate it, past the length checked below. + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File uploads must not be encoded.") + if size is None: + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPLengthRequired() + allowed = min( + limits.max_file_bytes, limits.max_total_bytes - upload.total_bytes + ) + if size > allowed: + finish(SelectedFilesStatus.TOO_LARGE) + raise web.HTTPRequestEntityTooLarge(max_size=allowed, actual_size=size) + upload.reader = asyncio.current_task() + try: + data = await request.content.readexactly(size) + except asyncio.IncompleteReadError as error: + finish(SelectedFilesStatus.UNAVAILABLE) + raise web.HTTPBadRequest(reason="File upload ended early.") from error + except BaseException: + finish(SelectedFilesStatus.UNAVAILABLE) + raise + finally: + upload.reader = None + upload.files.append(SelectedFile(name=name, data=data)) + upload.total_bytes += size + if len(upload.files) < upload.count: + return web.Response(status=204) + files = tuple(upload.files) + rejected = selected_files_policy_status( + [(chosen.name, len(chosen.data)) for chosen in files], limits + ) + if rejected is None: + finish(SelectedFilesStatus.OK, files) + else: + finish(rejected) + return web.Response(status=204) + async def _offer(self, request: web.Request) -> web.Response: """Negotiate one browser peer connection.""" if self._closed: @@ -1172,6 +1413,28 @@ def _buffer_browser_message( elif event_type == "close": self._cancel_link_press() event = CloseUserInputEvent(timestamp=timestamp_us) + elif event_type == "file_selector_result": + token = payload.get("id") + if not isinstance(token, str) or not token: + raise ValueError("File selector result requires a non-empty id.") + try: + status = SelectedFilesStatus(payload.get("status")) + except ValueError as error: + raise ValueError("File selector result status is invalid.") from error + if status is SelectedFilesStatus.OK: + raise ValueError( + "File selector result must not set status ok; send files " + "with POST /api/files." + ) + policy = self._take_pending_file_request(token) + if policy is None: + upload = self._uploading_file_requests.pop(token, None) + if upload is None: + return + upload.stop() + policy = upload.policy + self._emit_selected_files(policy.selector_id, status=status) + return else: raise ValueError("Unsupported browser event type.") self._append_event(event) @@ -1183,6 +1446,19 @@ def _append_event(self, event: UserInputEvent) -> None: raise RuntimeError("WebRTC input callback is not registered.") callback(event) + def _emit_selected_files( + self, + selector_id: str, + *, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), + ) -> None: + """Report the result of one open file selector to the window.""" + callback = self._file_selector_callback + if callback is None: + raise RuntimeError("WebRTC file selector callback is not registered.") + callback(selector_id, status, files) + async def _release_peer_connection( self, peer_connection: RTCPeerConnection, @@ -1202,6 +1478,7 @@ async def _release_peer_connection( self._video_track = None self._sent_cursor_options = None self._cancel_link_press() + self._complete_abandoned_file_selectors() else: track = detached_video_track failures: list[BaseException] = [] @@ -1278,6 +1555,14 @@ async def _shutdown(self) -> None: raise primary +def _query_int(request: web.Request, key: str) -> int | None: + """Return the query value at ``key`` as an integer, or ``None``.""" + try: + return int(request.query[key]) + except (KeyError, ValueError): + return None + + def _same_stream_format(first: SessionDesc, second: SessionDesc) -> bool: """Return whether two sessions can share one negotiated video track.""" return ( diff --git a/flashdreams/flashdreams/runtime_v2/session_runner.py b/flashdreams/flashdreams/runtime_v2/session_runner.py index b306ac05e..b6a051646 100644 --- a/flashdreams/flashdreams/runtime_v2/session_runner.py +++ b/flashdreams/flashdreams/runtime_v2/session_runner.py @@ -399,6 +399,8 @@ def should_end_ui() -> bool: cleanup(sink.close) # Both loops have stopped, so nothing will close an input wait still open. cleanup(profiler.reset_counts) + if window is not None: + cleanup(window.flush_file_selectors) if next_session_desc is None and window is not None: cleanup(window.close) cleanup(session.close) @@ -455,6 +457,10 @@ def _apply_window_requests(window: IClientWindow, request: UILoopRequests) -> No window.request_new_window_size(request.new_window_size) if request.links is not None: window.request_links(request.links) + # A replaced session gets no results, so its requests are not shown. + if request.new_session is None: + for file_selection in request.file_selections: + window.request_file_selector(file_selection) def _open_chunk_trace(path_value: object) -> _ChunkTraceLog: diff --git a/flashdreams/flashdreams/runtime_v2/user_input_event.py b/flashdreams/flashdreams/runtime_v2/user_input_event.py index 6f9c4ca50..7c3e2bacf 100644 --- a/flashdreams/flashdreams/runtime_v2/user_input_event.py +++ b/flashdreams/flashdreams/runtime_v2/user_input_event.py @@ -8,6 +8,7 @@ from typing import Literal from flashdreams.api_v2.user_input_event import UserInputEvent +from flashdreams.runtime_v2.selected_file import SelectedFile, SelectedFilesStatus class KeyboardInputState(Enum): @@ -245,6 +246,40 @@ def get_type_name(cls) -> str: """Optional controller quaternion in client XR space.""" +@dataclass(frozen=True, slots=True, eq=False) +class SelectedFilesUserInputEvent(UserInputEvent): + """Result of one file-selector request. + + The answer to :meth:`IUILoop.request_file_selector`, read from + ``user_events`` like any other input. + ``OK`` is the only status with files; every other status has empty ``files``. + """ + + @classmethod + def get_type_name(cls) -> str: + """Return the event type name.""" + return "selected_files" + + request_id: str + """``request_id`` of the control that asked for this result.""" + + status: SelectedFilesStatus + """Why this request completed. ``OK`` is the only value with files.""" + + files: tuple[SelectedFile, ...] = () + """Chosen files. Empty unless ``status`` is ``OK``.""" + + def __post_init__(self) -> None: + """Reject an ``ok`` event without files, or files on a failed pick.""" + has_files = bool(self.files) + if self.status is SelectedFilesStatus.OK: + if not has_files: + raise ValueError("ok selected-files events must include files.") + return + if has_files: + raise ValueError("failed selected-files events must not include files.") + + @dataclass(frozen=True, slots=True, eq=False) class UnknownUserInputEvent(UserInputEvent): """User input event for an unknown input modality.""" diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 9eb4ba640..0e15ebe07 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -10,6 +10,11 @@ from numpy import uint64 from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, +) from flashdreams.runtime_v2.serving.webrtc_server import WebRTCServer from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -23,7 +28,8 @@ class WebRTCClientWindow(IClientWindow): A thin pairing of the :class:`IClientWindow` protocol with :class:`WebRTCServer`, which does the serving. Browser events arrive on the server's own thread, so they are queued here and handed over in batches when - the session asks, as the protocol requires. + the session asks, as the protocol requires. File-selector reports are + queued the same way and applied on the UI thread when input is read. The server timestamps arrivals against one stable clock. This window clears input buffered during a session handoff, then rebases later events to the @@ -53,6 +59,9 @@ def __init__( startup_timeout_seconds: Maximum time to wait for server startup. """ self._input_events: deque[UserInputEvent] = deque() + self._file_selector_reports: deque[ + tuple[int, SelectedFilesStatus, tuple[SelectedFile, ...]] + ] = deque() self._hide_cursor = False self._lock_cursor_to_window = False self._input_lock = threading.Lock() @@ -70,7 +79,17 @@ def handle_input(event: UserInputEvent) -> None: with self._input_lock: self._input_events.append(event) + def report_file_selector( + selector_id: str, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], + ) -> None: + """Hand a server-thread picker result to the UI thread.""" + with self._input_lock: + self._file_selector_reports.append((int(selector_id), status, files)) + self.server.register_input_callback(handle_input) + self.server.register_file_selector_callback(report_file_selector) def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace links opened locally by the browser viewing this window.""" @@ -104,6 +123,22 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """ self.server.request_new_window_size(new_window_size) + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Ask the connected browser to open a file selector.""" + self.server.request_file_selector(str(selector_id), request) + + def _cancel_active_file_selectors(self) -> None: + """Refuse a later upload for the selector that was forgotten.""" + self.server.cancel_file_selectors() + + def put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event`` on the server's clock, like browser input.""" + event = replace(event, timestamp=self.server.event_timestamp_us()) + with self._input_lock: + self._input_events.append(event) + def open(self, session_desc: SessionDesc) -> None: """Implement ``OutputSink.open`` by configuring WebRTC output. @@ -122,6 +157,11 @@ def get_user_input_events(self) -> UserInputEvents: Returns: Buffered browser events in timestamp order, each returned once. """ + with self._input_lock: + reports = list(self._file_selector_reports) + self._file_selector_reports.clear() + for selector_id, status, files in reports: + self._complete_file_selector(selector_id, status, files) with self._input_lock: events = list(self._input_events) self._input_events.clear() diff --git a/flashdreams/test_v2/test_application_runner.py b/flashdreams/test_v2/test_application_runner.py index 606c3899d..b51be79a0 100644 --- a/flashdreams/test_v2/test_application_runner.py +++ b/flashdreams/test_v2/test_application_runner.py @@ -16,6 +16,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop, IUILoop from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.core.distributed.parallel import ParallelContext from flashdreams.runtime_v2.application_runner import ApplicationRunner from flashdreams.runtime_v2.metrics_output_sink import MetricsOutputSink @@ -158,6 +159,9 @@ def get_user_input_events(self) -> UserInputEvents: ) return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc self._calls.append("window.open") diff --git a/flashdreams/test_v2/test_cli.py b/flashdreams/test_v2/test_cli.py index a9ecd7d97..1c20a8dd2 100644 --- a/flashdreams/test_v2/test_cli.py +++ b/flashdreams/test_v2/test_cli.py @@ -18,6 +18,7 @@ from flashdreams.api_v2.application import IApplication from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2 import cli, client_window_factory from flashdreams.runtime_v2.application_runner import ApplicationRunner, Unbound from flashdreams.runtime_v2.client_window_factory import ClientWindowMode @@ -42,6 +43,9 @@ class _StubWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc diff --git a/flashdreams/test_v2/test_client_window.py b/flashdreams/test_v2/test_client_window.py index 720ab4acc..207f11716 100644 --- a/flashdreams/test_v2/test_client_window.py +++ b/flashdreams/test_v2/test_client_window.py @@ -11,6 +11,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.input_source import InputSource from flashdreams.api_v2.output_sink import OutputSink +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -36,6 +37,9 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return self._input + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc self._is_open = True diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index a36f00c23..a73f3dbd3 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -8,8 +8,10 @@ import queue import sys import threading +import time import webbrowser from collections.abc import Callable, Sequence +from pathlib import Path from types import SimpleNamespace from typing import TYPE_CHECKING, Any, cast from unittest.mock import Mock, call @@ -22,6 +24,11 @@ from flashdreams.runtime_v2.native_window_client_window import ( NativeWindowClientWindow, ) +from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, + FileSelectionRequest, + SelectedFilesStatus, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -31,6 +38,7 @@ KeyboardInputState, KeyboardUserInputEvent, MouseUserInputEvent, + SelectedFilesUserInputEvent, ) from flashdreams.runtime_v2.video_encoder import result_to_rgb24_tensor from flashdreams.runtime_v2.video_tensor import VideoTensorLayout @@ -41,6 +49,40 @@ pytestmark = pytest.mark.ci_cpu +def _file_request( + request_id: str, + initial_path: str, + *, + accept: tuple[str, ...] = (), + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + max_file_count: int = 1, +) -> FileSelectionRequest: + return FileSelectionRequest( + request_id=request_id, + initial_path=initial_path, + accept=accept, + max_file_bytes=max_file_bytes, + max_file_count=max_file_count, + ) + + +def _fake_dialog( + monkeypatch: pytest.MonkeyPatch, + choose: Callable[[FileSelectionRequest], list[str]], +) -> None: + """Run ``choose`` on the dialog thread instead of Tk.""" + monkeypatch.setattr(native_window_module, "_choose_paths", choose) + + +def _wait_for_input_events(window: NativeWindowClientWindow) -> list[object]: + for _ in range(500): + events = window.get_user_input_events().get_events() + if events: + return list(events) + time.sleep(0.01) + return [] + + @pytest.mark.parametrize("fullscreen", [False, True]) def test_fullscreen_creation_reports_size_on_first_poll(fullscreen: bool) -> None: presenter = _Presenter() @@ -966,6 +1008,187 @@ def test_native_link_warns_when_browser_is_unavailable( window.close() +def test_native_window_reports_a_selected_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + chosen = tmp_path / "seed.png" + chosen.write_bytes(b"png-bytes") + _fake_dialog(monkeypatch, lambda _: [str(chosen)]) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.OK + assert event.files[0].name == "seed.png" + assert event.files[0].data == b"png-bytes" + + +def test_native_window_reports_cancelled_file_selection( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_dialog(monkeypatch, lambda _: []) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.CANCELLED + assert event.files == () + + +def test_native_window_reports_unavailable_when_picker_raises( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + def no_display(_: FileSelectionRequest) -> list[str]: + raise RuntimeError("Tk cannot open a display.") + + _fake_dialog(monkeypatch, no_display) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert event.files == () + + +def test_selected_files_from_paths_skips_unreadable_and_oversize( + tmp_path: Path, +) -> None: + def load( + path: Path, **limits: Any + ) -> tuple[tuple[object, ...], SelectedFilesStatus]: + return native_window_module._selected_files_from_paths( + (str(path),), _file_request("open-1", str(tmp_path), **limits) + ) + + files, status = load(tmp_path / "gone.png") + assert files == () + assert status is SelectedFilesStatus.UNAVAILABLE + + too_big = tmp_path / "big.bin" + too_big.write_bytes(b"abcd") + files, status = load(too_big, max_file_bytes=3) + assert files == () + assert status is SelectedFilesStatus.TOO_LARGE + + wrong_type = tmp_path / "notes.txt" + wrong_type.write_bytes(b"hi") + files, status = load(wrong_type, accept=(".png",)) + assert files == () + assert status is SelectedFilesStatus.DISALLOWED_TYPE + + too_big_wrong_type = tmp_path / "huge.txt" + too_big_wrong_type.write_bytes(b"abcd") + files, status = load(too_big_wrong_type, accept=(".png",), max_file_bytes=1) + assert files == () + assert status is SelectedFilesStatus.DISALLOWED_TYPE + + +@pytest.mark.skipif(not Path("/dev/zero").exists(), reason="needs /dev/zero") +def test_selected_files_from_paths_rejects_a_non_regular_file(tmp_path: Path) -> None: + endless = tmp_path / "endless.bin" + endless.symlink_to("/dev/zero") + files, status = native_window_module._selected_files_from_paths( + (str(endless),), _file_request("open-1", str(tmp_path), accept=(".bin",)) + ) + assert files == () + assert status is SelectedFilesStatus.UNAVAILABLE + + +def test_native_window_reports_multiple_selected_files( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + first = tmp_path / "a.bin" + second = tmp_path / "b.bin" + first.write_bytes(b"one") + second.write_bytes(b"two") + _fake_dialog(monkeypatch, lambda _: [str(first), str(second)]) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_file_selector( + _file_request("open-1", str(tmp_path), max_file_count=2) + ) + events = _wait_for_input_events(window) + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.status is SelectedFilesStatus.OK + assert [chosen.name for chosen in event.files] == ["a.bin", "b.bin"] + assert [chosen.data for chosen in event.files] == [b"one", b"two"] + + +def test_native_window_drops_a_flushed_dialog_and_is_busy_until_it_closes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + chosen = tmp_path / "stale.png" + chosen.write_bytes(b"png") + user_closes = threading.Event() + + def choose(_: FileSelectionRequest) -> list[str]: + user_closes.wait(5) + return [str(chosen)] + + _fake_dialog(monkeypatch, choose) + read: list[object] = [] + monkeypatch.setattr( + native_window_module, + "_selected_files_from_paths", + lambda paths, request: read.append(paths) or ((), SelectedFilesStatus.OK), + ) + presenter = _Presenter() + window = NativeWindowClientWindow(presenter_factory=_presenter_factory(presenter)) + window.open(_session_desc()) + window.request_file_selector(_file_request("open-1", str(tmp_path))) + window.flush_file_selectors() + + window.request_file_selector(_file_request("open-2", str(tmp_path))) + busy = window.get_user_input_events().get_events() + user_closes.set() + thread = window._file_dialog_thread + assert thread is not None + thread.join(5) + stale = window.get_user_input_events().get_events() + window.close() + + assert len(busy) == 1 + assert isinstance(busy[0], SelectedFilesUserInputEvent) + assert busy[0].request_id == "open-2" + assert busy[0].status is SelectedFilesStatus.BUSY + assert stale == [] + assert read == [], "a flushed dialog's files are never read" + + def test_native_window_reports_standard_gamepad_events() -> None: presenter = _Presenter() clock_values = iter((1_000_000, 1_001_000, 1_002_000, 1_003_000)) diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py new file mode 100644 index 000000000..a98fd80b0 --- /dev/null +++ b/flashdreams/test_v2/test_selected_files.py @@ -0,0 +1,356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for client file selection through the v2 window contract.""" + +import queue +import threading +from pathlib import Path +from typing import Any + +import pytest +from numpy import uint64 + +from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.loop import IUILoop +from flashdreams.runtime_v2.mp4_client_window import Mp4ClientWindow +from flashdreams.runtime_v2.null_client_window import NullClientWindow +from flashdreams.runtime_v2.presentation_manager import PresentationManager +from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_FILE_COUNT, + DEFAULT_MAX_TOTAL_BYTES, + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + normalize_selected_file_accept, + selected_file_suffix_allowed, + selected_files_policy_status, +) +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents + +pytestmark = pytest.mark.ci_cpu + + +class _UILoop(IUILoop[None]): + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + del step_index, events + return [] + + def reset(self) -> None: + return + + +def _ui_loop() -> _UILoop: + loop = _UILoop() + loop.register_session_loop_objects( + state=None, + frequency=0, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(), + presentation_manager=PresentationManager(), + ) + return loop + + +def test_ui_loop_queues_the_app_declared_limits() -> None: + loop = _ui_loop() + loop.request_file_selector( + "open-1", + "/tmp", + accept=(".png", ".JPG"), + max_file_bytes=DEFAULT_MAX_FILE_BYTES * 4, + max_total_bytes=DEFAULT_MAX_TOTAL_BYTES * 4, + max_file_count=DEFAULT_MAX_FILE_COUNT * 4, + ) + loop.request_file_selector("open-2", "/tmp") + + requests = loop.flush_ui_loop_requests() + assert requests is not None + raised, defaults = requests.file_selections + assert raised == FileSelectionRequest( + request_id="open-1", + initial_path="/tmp", + accept=(".png", ".JPG"), + max_file_bytes=DEFAULT_MAX_FILE_BYTES * 4, + max_total_bytes=DEFAULT_MAX_TOTAL_BYTES * 4, + max_file_count=DEFAULT_MAX_FILE_COUNT * 4, + ) + assert ( + defaults.max_file_bytes, + defaults.max_total_bytes, + defaults.max_file_count, + ) == ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_TOTAL_BYTES, + DEFAULT_MAX_FILE_COUNT, + ) + + +@pytest.mark.parametrize( + "limit", ["max_file_bytes", "max_total_bytes", "max_file_count"] +) +def test_ui_loop_rejects_a_non_positive_limit(limit: str) -> None: + loop = _ui_loop() + zero: dict[str, Any] = {limit: 0} + fractional: dict[str, Any] = {limit: 1.5} + with pytest.raises(ValueError, match=f"{limit} must be > 0"): + loop.request_file_selector("open-1", **zero) + with pytest.raises(TypeError, match=f"{limit} must be an integer"): + loop.request_file_selector("open-1", **fractional) + + +def test_ui_loop_rejects_a_file_limit_over_the_total_limit() -> None: + loop = _ui_loop() + with pytest.raises(ValueError, match="max_file_bytes must not exceed"): + loop.request_file_selector("open-1", max_file_bytes=2, max_total_bytes=1) + with pytest.raises(ValueError, match="max_file_bytes must not exceed"): + loop.request_file_selector("open-1", max_file_bytes=DEFAULT_MAX_TOTAL_BYTES + 1) + + +def test_ui_loop_uses_home_when_initial_path_is_omitted() -> None: + loop = _ui_loop() + loop.request_file_selector("open-1") + + requests = loop.flush_ui_loop_requests() + assert requests is not None + assert requests.file_selections[0].initial_path == str(Path.home()) + + +def test_ui_loop_forwards_every_file_selection_of_a_tick() -> None: + loop = _ui_loop() + loop.request_file_selector("open-1", "/tmp") + loop.request_file_selector("open-2", "/var") + + requests = loop.flush_ui_loop_requests() + assert requests is not None + assert [item.request_id for item in requests.file_selections] == [ + "open-1", + "open-2", + ] + + +class _PickerWindow(NullClientWindow): + def __init__(self) -> None: + super().__init__() + self.started: list[tuple[int, str]] = [] + self.cancelled = 0 + + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + self.started.append((selector_id, request.request_id)) + + def _cancel_active_file_selectors(self) -> None: + self.cancelled += 1 + + +def _request(request_id: str) -> FileSelectionRequest: + return FileSelectionRequest(request_id=request_id, initial_path="/tmp") + + +def _statuses(window: IClientWindow) -> list[tuple[str, SelectedFilesStatus]]: + return [ + (event.request_id, event.status) + for event in window.get_user_input_events().get_events() + if isinstance(event, SelectedFilesUserInputEvent) + ] + + +def test_window_shows_one_picker_at_a_time() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window.request_file_selector(_request("open-1")) + window.request_file_selector(_request("open-2")) + assert [request_id for _, request_id in window.started] == ["open-1"] + assert _statuses(window) == [ + ("open-1", SelectedFilesStatus.BUSY), + ("open-2", SelectedFilesStatus.BUSY), + ] + + selector_id = window.started[0][0] + window._complete_file_selector(selector_id, SelectedFilesStatus.CANCELLED) + window._complete_file_selector(selector_id, SelectedFilesStatus.CANCELLED) + assert _statuses(window) == [("open-1", SelectedFilesStatus.CANCELLED)] + window.request_file_selector(_request("open-2")) + assert [request_id for _, request_id in window.started] == ["open-1", "open-2"] + + +def test_window_checks_a_result_against_its_request() -> None: + def complete( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], + ) -> list[SelectedFilesUserInputEvent]: + window = _PickerWindow() + window.request_file_selector(request) + window._complete_file_selector(window.started[0][0], status, files) + events = window.get_user_input_events().get_events() + selected = [e for e in events if isinstance(e, SelectedFilesUserInputEvent)] + assert selected == events + return selected + + png = FileSelectionRequest( + request_id="open-1", initial_path="/tmp", accept=(".png",), max_file_bytes=4 + ) + a = SelectedFile(name="a.png", data=b"1") + b = SelectedFile(name="b.png", data=b"2") + cases = [ + (png, SelectedFilesStatus.OK, (), SelectedFilesStatus.UNAVAILABLE), + (png, SelectedFilesStatus.OK, (a, b), SelectedFilesStatus.TOO_LARGE), + ( + png, + SelectedFilesStatus.OK, + (SelectedFile(name="a.txt", data=b"1"),), + SelectedFilesStatus.DISALLOWED_TYPE, + ), + ( + png, + SelectedFilesStatus.OK, + (SelectedFile(name="a.png", data=b"12345"),), + SelectedFilesStatus.TOO_LARGE, + ), + (png, SelectedFilesStatus.CANCELLED, (a,), SelectedFilesStatus.CANCELLED), + (png, SelectedFilesStatus.BUSY, (a,), SelectedFilesStatus.BUSY), + ] + for request, status, files, expected in cases: + [event] = complete(request, status, files) + assert (event.status, event.files) == (expected, ()) + + many = FileSelectionRequest( + request_id="open-1", initial_path="/tmp", max_file_count=2 + ) + [event] = complete(many, SelectedFilesStatus.OK, (a, b)) + assert (event.status, event.files) == (SelectedFilesStatus.OK, (a, b)) + + pathy = ( + SelectedFile(name="../outside.png", data=b"1"), + SelectedFile(name="/home/user/in.png", data=b"2"), + ) + [event] = complete(many, SelectedFilesStatus.OK, pathy) + assert [chosen.name for chosen in event.files] == ["outside.png", "in.png"] + + +def test_window_without_put_user_input_event_cannot_be_created() -> None: + class _Incomplete(IClientWindow): + def get_user_input_events(self) -> UserInputEvents: + return UserInputEvents([]) + + def open(self, session_desc: SessionDesc) -> None: + del session_desc + + def write(self, result: StepResult) -> None: + del result + + def close(self) -> None: + return + + with pytest.raises(TypeError, match="put_user_input_event"): + _Incomplete() # type: ignore[abstract] + + +def test_null_window_open_drops_a_result_left_by_the_previous_session() -> None: + window = NullClientWindow() + window.request_file_selector(_request("open-1")) + window.open(SessionDesc()) + assert _statuses(window) == [] + + +def test_flush_forgets_the_open_picker_without_a_result() -> None: + window = _PickerWindow() + window.request_file_selector(_request("open-1")) + window.flush_file_selectors() + window._complete_file_selector(window.started[0][0], SelectedFilesStatus.CANCELLED) + assert _statuses(window) == [] + assert window.cancelled == 1 + + window.flush_file_selectors() + assert window.cancelled == 1 + window.request_file_selector(_request("open-1")) + assert len(window.started) == 2 + + +def test_window_answers_unavailable_when_its_picker_fails_to_start() -> None: + class _BrokenWindow(NullClientWindow): + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + raise RuntimeError("no picker") + + window = _BrokenWindow() + window.request_file_selector(_request("open-1")) + assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] + + +def test_selected_file_policy_helpers() -> None: + assert normalize_selected_file_accept([".png", ".jpg"]) == (".png", ".jpg") + assert selected_file_suffix_allowed("seed.PNG", (".png",)) + assert not selected_file_suffix_allowed("seed.txt", (".png",)) + assert selected_file_suffix_allowed("seed.txt", ()) + bins = FileSelectionRequest( + request_id="open-1", + initial_path="/tmp", + accept=(".bin",), + max_file_bytes=3, + max_total_bytes=5, + max_file_count=2, + ) + cases = [ + ([("a.bin", 0)] * 3, SelectedFilesStatus.TOO_LARGE), + ([("a.bin", 2), ("b.txt", 2)], SelectedFilesStatus.DISALLOWED_TYPE), + ([("a.bin", 2), ("b.bin", 4)], SelectedFilesStatus.TOO_LARGE), + ([("a.bin", 3), ("b.bin", 3)], SelectedFilesStatus.TOO_LARGE), + ([("a.bin", 2), ("b.bin", 3)], None), + ] + for files, expected in cases: + assert selected_files_policy_status(files, bins) is expected + + +def test_ui_loop_rejects_empty_request_id() -> None: + loop = _ui_loop() + with pytest.raises(ValueError, match="non-empty string"): + loop.request_file_selector("") + + +def test_ui_loop_rejects_invalid_accept() -> None: + loop = _ui_loop() + with pytest.raises(ValueError, match="accept suffixes"): + loop.request_file_selector("open-1", accept=("png",)) + + +def test_selected_files_event_rejects_inconsistent_status() -> None: + with pytest.raises(ValueError, match="must include files"): + SelectedFilesUserInputEvent( + timestamp=uint64(1), + request_id="open-1", + status=SelectedFilesStatus.OK, + ) + with pytest.raises(ValueError, match="must not include files"): + SelectedFilesUserInputEvent( + timestamp=uint64(1), + request_id="open-1", + status=SelectedFilesStatus.CANCELLED, + files=(SelectedFile(name="seed.png", data=b"png"),), + ) + + +@pytest.mark.parametrize("kind", ["mp4", "null"]) +def test_window_without_a_selector_answers_unavailable( + kind: str, tmp_path: Path +) -> None: + window = ( + Mp4ClientWindow(tmp_path / "clip.mp4") if kind == "mp4" else NullClientWindow() + ) + window.request_file_selector( + FileSelectionRequest(request_id="open-1", initial_path="/tmp") + ) + + assert _statuses(window) == [("open-1", SelectedFilesStatus.UNAVAILABLE)] diff --git a/flashdreams/test_v2/test_session_runner.py b/flashdreams/test_v2/test_session_runner.py index 359ef819b..4eb8b336e 100644 --- a/flashdreams/test_v2/test_session_runner.py +++ b/flashdreams/test_v2/test_session_runner.py @@ -21,6 +21,7 @@ IModelLoop, IUILoop, ModelInferenceState, + UILoopRequests, invoke_async, ) from flashdreams.api_v2.session import ISession @@ -35,18 +36,23 @@ PresentationManager, _PresentationClock, ) +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFilesStatus, +) from flashdreams.runtime_v2.session_desc import ( BackpressureMode, PresentationMode, SessionDesc, ) -from flashdreams.runtime_v2.session_runner import run_session +from flashdreams.runtime_v2.session_runner import _apply_window_requests, run_session from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( CloseUserInputEvent, KeyboardInputState, KeyboardUserInputEvent, ResetUserInputEvent, + SelectedFilesUserInputEvent, ) from flashdreams.runtime_v2.user_input_events import UserInputEvents from flashdreams.runtime_v2.video_tensor import VideoTensorLayout @@ -663,6 +669,8 @@ def __init__( self.session_desc: SessionDesc | None = None self.results: list[StepResult] = [] self.cursor_requests: list[tuple[str, bool]] = [] + self.file_selection_requests: list[FileSelectionRequest] = [] + self._file_events: list[UserInputEvent] = [] def request_hide_cursor(self, hide_cursor: bool) -> None: """Record one cursor visibility request.""" @@ -674,12 +682,27 @@ def request_lock_cursor_to_window(self, lock_cursor_to_window: bool) -> None: self._log.record("window.request_lock_cursor_to_window") self.cursor_requests.append(("lock", lock_cursor_to_window)) + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Record one file-selector request and answer with the default.""" + self._log.record("window._request_file_selector") + self.file_selection_requests.append(request) + super()._request_file_selector(selector_id, request) + + def put_user_input_event(self, event: UserInputEvent) -> None: + with self._lock: + self._file_events.append(event) + def get_user_input_events(self) -> UserInputEvents: self._log.record("window.get_user_input_events") with self._lock: + file_events, self._file_events = self._file_events, [] if self._scripted: - return self._scripted.pop(0) - return UserInputEvents([]) + return UserInputEvents( + [*file_events, *self._scripted.pop(0).get_events()] + ) + return UserInputEvents(file_events) def open(self, session_desc: SessionDesc) -> None: self._log.record("window.open") @@ -1370,6 +1393,77 @@ def init(self) -> None: assert "window.close" not in log.calls +def test_run_session_forwards_file_selection_requests_to_the_window() -> None: + log = CallLog() + + class RequestingSession(FakeSession): + def init(self) -> None: + super().init() + + def open_selector(_: object) -> None: + self.ui_loop.request_file_selector( + "open-1", + "/tmp", + accept=(".png",), + max_file_bytes=1024, + ) + + invoke_async(self.ui_loop, open_selector) + + window = RecordingClientWindow(log) + run_session(RequestingSession(_session_desc(), log), window, steps=1) + + assert window.file_selection_requests == [ + FileSelectionRequest( + request_id="open-1", + initial_path="/tmp", + accept=(".png",), + max_file_bytes=1024, + ) + ] + assert "window._request_file_selector" in log.calls + + +def test_window_requests_skip_a_file_selector_for_a_replaced_session() -> None: + window = RecordingClientWindow(CallLog()) + _apply_window_requests( + window, + UILoopRequests( + new_session=_session_desc(), + file_selections=[ + FileSelectionRequest(request_id="open-1", initial_path="/tmp") + ], + ), + ) + assert window.file_selection_requests == [] + + +def test_run_session_delivers_an_immediate_file_selection_result() -> None: + log = CallLog() + picks: list[SelectedFilesUserInputEvent] = [] + + class PickingSession(FakeSession): + def run_ui(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + if self.ui_steps == 0: + self.ui_loop.request_file_selector("open-1") + self.ui_steps += 1 + picks.extend( + event + for event in self.ui_loop.user_events.get_events() + if isinstance(event, SelectedFilesUserInputEvent) + ) + return super().run_ui(step_index, events) + + session_desc = replace( + _session_desc(), presentation_mode=PresentationMode.CONTINUOUS + ) + run_session(PickingSession(session_desc, log), RecordingClientWindow(log), steps=4) + + assert [(pick.request_id, pick.status) for pick in picks] == [ + ("open-1", SelectedFilesStatus.UNAVAILABLE) + ] + + def test_timeout_wins_over_a_ui_requested_replacement() -> None: log = CallLog() resolved = _session_desc() diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 8300f4f97..ed4e1fc88 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -6,11 +6,13 @@ # ruff: noqa: E402 - optional WebRTC imports must follow importorskip. import asyncio +import concurrent.futures import json import threading import time +from collections.abc import AsyncIterator, Callable from dataclasses import replace -from typing import Any, cast +from typing import Any, TypeVar, cast from unittest.mock import AsyncMock, Mock import pytest @@ -18,10 +20,13 @@ pytestmark = pytest.mark.ci_cpu +T = TypeVar("T") + pytest.importorskip("aiohttp") pytest.importorskip("aiortc") -from aiohttp import ClientSession +from aiohttp import ClientError, ClientSession +from aiohttp.streams import StreamReader from aiortc import ( MediaStreamTrack, RTCDataChannel, @@ -33,6 +38,11 @@ from yarl import URL from flashdreams.api_v2.client_window import ClientWindowLink +from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, + FileSelectionRequest, + SelectedFilesStatus, +) from flashdreams.runtime_v2.serving import webrtc_server from flashdreams.runtime_v2.serving.webrtc_server import _VideoTrack from flashdreams.runtime_v2.session_desc import PresentationMode, SessionDesc @@ -45,6 +55,7 @@ KeyboardUserInputEvent, MouseUserInputEvent, QueryStringUserInputEvent, + SelectedFilesUserInputEvent, TouchUserInputEvent, XRControllerUserInputEvent, ) @@ -52,6 +63,102 @@ from flashdreams.runtime_v2.webrtc_client_window import WebRTCClientWindow +def _file_request( + request_id: str, initial_path: str = "/tmp", **fields: Any +) -> FileSelectionRequest: + return FileSelectionRequest( + request_id=request_id, initial_path=initial_path, **fields + ) + + +def _files_url(window: WebRTCClientWindow, token: str) -> str: + return f"{window.server.url}api/files?id={token}" + + +async def _wait_file_armed(window: WebRTCClientWindow) -> str: + """Return the token the browser was sent once a selector is open.""" + for _ in range(100): + if window.server._pending_file_requests: + return next(iter(window.server._pending_file_requests)) + await asyncio.sleep(0.01) + raise AssertionError("file selection was not opened") + + +def _on_server_loop(window: WebRTCClientWindow, call: Callable[[], T]) -> T: + """Run ``call`` on the server's event loop, which owns its upload tokens.""" + loop = window.server._loop + assert loop is not None + future: concurrent.futures.Future[T] = concurrent.futures.Future() + loop.call_soon_threadsafe(lambda: future.set_result(call())) + return future.result(timeout=5) + + +def _arm(window: WebRTCClientWindow, request: FileSelectionRequest) -> str: + """Open a selector as if the browser had been sent ``request``; return its token.""" + selector_id = window._last_file_selector_id + 1 + window._last_file_selector_id = selector_id + window._active_file_selector = (selector_id, request) + wire_id = f"token-{selector_id}" + policy = webrtc_server._PendingFileSelection( + selector_id=str(selector_id), request=request + ) + _on_server_loop( + window, + lambda: window.server._pending_file_requests.__setitem__(wire_id, policy), + ) + return wire_id + + +async def _post_files( + window: WebRTCClientWindow, + token: str, + files: list[tuple[str, bytes]], +) -> list[int]: + """Upload ``files`` as the browser does, one request each; stop at a refusal.""" + statuses: list[int] = [] + async with ClientSession() as client: + for index, (name, data) in enumerate(files): + query = {"name": name, "index": str(index), "count": str(len(files))} + async with client.post( + _files_url(window, token), params=query, data=data + ) as response: + statuses.append(response.status) + if response.status != 204: + break + return statuses + + +async def _post_selected_file( + window: WebRTCClientWindow, + token: str, + data: bytes, + filename: str, +) -> int: + [status] = await _post_files(window, token, [(filename, data)]) + return status + + +@pytest.mark.asyncio +async def test_file_upload_accepts_multiple_files() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer, _, _ = await _connect_browser(window) + try: + window.request_file_selector(_file_request("open-1", max_file_count=2)) + wire_id = await _wait_file_armed(window) + statuses = await _post_files( + window, wire_id, [("a.bin", b"one"), ("b.bin", b"two")] + ) + assert statuses == [204, 204] + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.OK + assert [chosen.name for chosen in event.files] == ["a.bin", "b.bin"] + assert [chosen.data for chosen in event.files] == [b"one", b"two"] + finally: + await peer.close() + window.close() + + def _session_desc( presentation_mode: PresentationMode = PresentationMode.ON_DEMAND, ) -> SessionDesc: @@ -143,6 +250,478 @@ async def test_browser_query_string_reaches_the_input_stream() -> None: window.close() +@pytest.mark.asyncio +async def test_file_selector_without_a_viewer_is_unavailable() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + window.request_file_selector(_file_request("open-1")) + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert window.server._pending_file_requests == {} + finally: + window.close() + + +@pytest.mark.asyncio +async def test_peer_loss_completes_armed_file_selections_as_unavailable() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, _, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + await _wait_file_armed(window) + await peer.close() + peer = None + + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert window.server._pending_file_requests == {} + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_in_flight_upload_is_unavailable_after_peer_loss( + monkeypatch: pytest.MonkeyPatch, +) -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, _, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + original_readexactly = StreamReader.readexactly + disconnected = False + + async def readexactly_after_disconnect(self: StreamReader, n: int) -> bytes: + nonlocal disconnected + if not disconnected: + disconnected = True + peer_connection = window.server._peer_connection + if peer_connection is not None: + await window.server._release_peer_connection(peer_connection) + return await original_readexactly(self, n) + + monkeypatch.setattr(StreamReader, "readexactly", readexactly_after_disconnect) + with pytest.raises(ClientError): + await _post_selected_file(window, wire_id, b"hello", "hello.txt") + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + await asyncio.sleep(0.05) + assert window.get_user_input_events().get_events() == [] + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_upload_without_the_armed_token_is_rejected() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + assert await _post_selected_file(window, "1", b"hello", "hello.txt") == 409 + _arm(window, _file_request("open-1")) + assert await _post_selected_file(window, "1", b"hello", "hello.txt") == 409 + assert window.get_user_input_events().get_events() == [] + assert len(window.server._pending_file_requests) == 1 + finally: + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_sends_the_request_without_host_path() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + messages: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + messages.append(json.loads(message)) + + window.request_file_selector( + _file_request( + "open-1", + "/home/secret", + accept=(".png", ".jpg"), + max_file_bytes=DEFAULT_MAX_FILE_BYTES * 2, + ) + ) + for _ in range(100): + if any(item.get("type") == "file_selector" for item in messages): + break + await asyncio.sleep(0.01) + payload = next(item for item in messages if item.get("type") == "file_selector") + token = payload.pop("id") + assert isinstance(token, str) and len(token) >= 16 + assert token in window.server._pending_file_requests + assert payload == { + "type": "file_selector", + "accept": [".png", ".jpg"], + "max_file_bytes": DEFAULT_MAX_FILE_BYTES * 2, + "max_file_count": 1, + } + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_flushed_file_selector_closes_the_browser_prompt() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + messages: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + messages.append(json.loads(message)) + + window.request_file_selector(_file_request("open-1")) + await _wait_file_armed(window) + window.flush_file_selectors() + for _ in range(100): + if any(item.get("type") == "file_selector_cancel" for item in messages): + break + await asyncio.sleep(0.01) + assert {"type": "file_selector_cancel"} in messages + assert window.server._pending_file_requests == {} + finally: + if peer is not None: + await peer.close() + window.close() + + +async def _wait_selected_files_event( + window: WebRTCClientWindow, +) -> SelectedFilesUserInputEvent: + events: list[object] = [] + for _ in range(100): + events.extend(window.get_user_input_events().get_events()) + if events: + break + await asyncio.sleep(0.01) + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + return event + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "query", + [ + {}, + {"name": "a.bin", "index": "1", "count": "2"}, + {"name": "", "index": "0", "count": "1"}, + ], +) +async def test_file_upload_out_of_order_is_unavailable(query: dict[str, str]) -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", max_file_count=2)) + async with ClientSession() as client: + async with client.post( + _files_url(window, wire_id), params=query, data=b"hello" + ) as response: + assert response.status == 400 + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_rejects_oversize() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", max_file_bytes=1)) + status = await _post_selected_file(window, wire_id, b"x" * 8192, "big.bin") + assert status == 413 + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.TOO_LARGE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "limits", + [ + {"max_file_bytes": 5, "max_total_bytes": 5, "max_file_count": 3}, + {"max_file_count": 2}, + {}, + ], +) +async def test_file_upload_enforces_the_request_limits(limits: dict[str, Any]) -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", **limits)) + files = [(name, b"xyz") for name in ("a.bin", "b.bin", "c.bin")] + statuses = await _post_files(window, wire_id, files) + assert statuses[-1] in (400, 413) + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.TOO_LARGE + assert event.files == () + finally: + window.close() + + +async def _chunked_junk() -> AsyncIterator[bytes]: + yield b"junk" + + +@pytest.mark.parametrize( + ("body", "headers", "status", "selected_status"), + [ + (b"x" * 1024, {}, 413, SelectedFilesStatus.TOO_LARGE), + (None, {}, 411, SelectedFilesStatus.UNAVAILABLE), + (b"x", {"Content-Encoding": "gzip"}, 400, SelectedFilesStatus.UNAVAILABLE), + ], +) +@pytest.mark.asyncio +async def test_file_upload_is_refused_before_its_body_is_read( + body: bytes | None, + headers: dict[str, str], + status: int, + selected_status: SelectedFilesStatus, +) -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1", max_file_bytes=16)) + async with ClientSession() as client: + async with client.post( + _files_url(window, wire_id), + params={"name": "a.bin", "index": "0", "count": "1"}, + data=_chunked_junk() if body is None else body, + headers=headers, + ) as response: + assert response.status == status + event = await _wait_selected_files_event(window) + assert event.status is selected_status + finally: + window.close() + + +@pytest.mark.asyncio +async def test_file_upload_over_the_file_limit_still_reports_a_disallowed_type() -> ( + None +): + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm( + window, _file_request("open-1", accept=(".png",), max_file_bytes=1) + ) + assert ( + await _post_selected_file(window, wire_id, b"x" * 1024, "hello.txt") == 204 + ) + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.DISALLOWED_TYPE + assert event.files == () + finally: + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_result_reports_status() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + channel.send( + json.dumps( + { + "type": "file_selector_result", + "id": wire_id, + "status": "cancelled", + } + ) + ) + event = await _wait_selected_files_event(window) + assert event.request_id == "open-1" + assert event.status is SelectedFilesStatus.CANCELLED + assert event.files == () + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_browser_result_ends_an_upload_left_partway() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + window.request_file_selector(_file_request("open-1", max_file_count=2)) + wire_id = await _wait_file_armed(window) + async with ClientSession() as client: + async with client.post( + _files_url(window, wire_id), + params={"name": "a.bin", "index": "0", "count": "2"}, + data=b"one", + ) as response: + assert response.status == 204 + channel.send( + json.dumps( + {"type": "file_selector_result", "id": wire_id, "status": "unavailable"} + ) + ) + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.UNAVAILABLE + assert await _post_selected_file(window, wire_id, b"two", "b.bin") == 409 + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_withdrawn_selector_stops_its_upload_read() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + writer: asyncio.StreamWriter | None = None + try: + wire_id = _arm(window, _file_request("open-1")) + url = URL(_files_url(window, wire_id)) + assert url.host is not None and url.port is not None + reader, writer = await asyncio.open_connection(url.host, url.port) + target = url.with_query( + id=wire_id, name="slow.bin", index="0", count="1" + ).raw_path_qs + writer.write( + f"POST {target} HTTP/1.1\r\nHost: {url.host}\r\n" + "Content-Length: 10\r\n\r\nx".encode() + ) + await writer.drain() + for _ in range(100): + uploads = window.server._uploading_file_requests + if wire_id in uploads and uploads[wire_id].reader is not None: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("upload read did not start") + + window.server.cancel_file_selectors() + + await asyncio.wait_for(reader.read(), timeout=5) + assert window.server._uploading_file_requests == {} + assert window.get_user_input_events().get_events() == [] + finally: + if writer is not None: + writer.close() + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_result_cannot_claim_ok() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + errors: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + payload = json.loads(message) + if payload.get("type") == "error": + errors.append(payload) + + window.request_file_selector(_file_request("open-1")) + wire_id = await _wait_file_armed(window) + channel.send( + json.dumps( + { + "type": "file_selector_result", + "id": wire_id, + "status": "ok", + } + ) + ) + for _ in range(100): + if errors: + break + await asyncio.sleep(0.01) + assert errors + assert "POST /api/files" in str(errors[0].get("message")) + assert window.get_user_input_events().get_events() == [] + assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 204 + event = await _wait_selected_files_event(window) + assert event.status is SelectedFilesStatus.OK + assert event.files[0].data == b"hello" + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_browser_script_checks_type_and_size_before_upload() -> None: + window = WebRTCClientWindow() + try: + async with ClientSession() as client: + async with client.get(f"{window.server.url}app.js") as response: + script = await response.text() + assert "fileNameMatchesAccept" in script + assert "file.size" in script + assert '"too_large"' in script + assert '"disallowed_type"' in script + assert "fetch(`/api/files?${query}`" in script + assert "input.multiple" in script + assert "event.stopImmediatePropagation();" in script + assert "(promptDialog !== null && promptDialog.open)" in script + finally: + window.close() + + +@pytest.mark.asyncio +async def test_session_handoff_clears_pending_file_selection() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + wire_id = _arm(window, _file_request("open-1")) + window.open(_session_desc()) + assert ( + _on_server_loop(window, lambda: dict(window.server._pending_file_requests)) + == {} + ) + assert await _post_selected_file(window, wire_id, b"hello", "hello.txt") == 409 + finally: + window.close() + + @pytest.mark.asyncio async def test_window_buffers_browser_events_until_drained() -> None: window = WebRTCClientWindow() @@ -422,6 +1001,8 @@ def _link_server() -> tuple[webrtc_server.WebRTCServer, Mock, Mock]: server._link_lock = threading.Lock() server._links = () server._pressed_link = None + server._pending_file_requests = {} + server._uploading_file_requests = {} channel = Mock(readyState="open") server._control_channel = channel server._loop = Mock(spec=asyncio.AbstractEventLoop) diff --git a/flashdreams/tests/test_profiler.py b/flashdreams/tests/test_profiler.py index ffad1ff1e..409dcd77e 100644 --- a/flashdreams/tests/test_profiler.py +++ b/flashdreams/tests/test_profiler.py @@ -35,6 +35,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.profiler import IProfiler +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.infra import profiler as event_timer_module from flashdreams.runtime_v2 import profiler as profiler_module from flashdreams.runtime_v2.event_buffer import EventBuffer @@ -350,6 +351,9 @@ def get_user_input_events(self) -> UserInputEvents: self.batches += 1 return UserInputEvents([_key_press()]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: pass diff --git a/integrations_v2/imgui_ui_demo/README.md b/integrations_v2/imgui_ui_demo/README.md index dcca7516a..a991b0cd9 100644 --- a/integrations_v2/imgui_ui_demo/README.md +++ b/integrations_v2/imgui_ui_demo/README.md @@ -8,6 +8,12 @@ SPDX-License-Identifier: Apache-2.0 Small ImGui applications for the FlashDreams v2 loop runtime: - `imgui-ui-text-input` renders an editable text field over model output. +- `imgui-ui-file-picker` opens the client file selector and shows the chosen + name and size on one status line per button. Each Open button calls + `request_file_selector` with its own `request_id`, and the result event + carrying that id updates the button's line. One picker is open at a time; + a click on any button while it is open shows `BUSY` on that button's line. + Dropping the WebRTC tab completes in-flight picks as unavailable. - `imgui-ui-query-string` sets background from a `?(r,g,b)` browser URL. - `imgui-ui-window-size` resizes native window or UI render target without resetting session. @@ -24,6 +30,19 @@ Run the text-input application: uv run --no-sync flashdreams-run-v2 imgui-ui-text-input --mode native-window ``` +Run the file-picker application. The native window uses the Tk file dialog +(on Linux, install Python's Tk package, such as `python3-tk`). WebRTC shows a +confirmation dialog in the viewer (the picker must open from a browser click). +The demo asks for `.bin` / `.raw` and overrides the default size limits with +`max_file_bytes` of 32 MiB and `max_total_bytes` of 256 MiB. Controls are **Open file (A)**, **Open file (B)**, and **Open files** +(up to 8 files, through `max_file_count`). Failed picks show in red with the +`SelectedFilesStatus` name and why. +```bash +uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode native-window +uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode webrtc \ + --host 127.0.0.1 --port 8080 +``` + Run the query-string application: ```bash uv run --no-sync flashdreams-run-v2 imgui-ui-query-string \ diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py index b3fcbf18f..fa3acf920 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py @@ -3,10 +3,13 @@ """Interactive ImGui UI applications for the v2 loop runtime.""" +from .file_picker_app import FilePickerApplication, FilePickerSession from .text_input_app import TextInputApplication, TextInputSession, create_app from .window_size_app import WindowSizeApplication, WindowSizeSession __all__ = [ + "FilePickerApplication", + "FilePickerSession", "TextInputApplication", "TextInputSession", "WindowSizeApplication", diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py new file mode 100644 index 000000000..62a5e002f --- /dev/null +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ImGui file-picker application for the v2 loop runtime.""" + +import math +import time +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any + +import torch +from torch import Tensor + +from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop +from flashdreams.runtime_v2.selected_file import SelectedFilesStatus +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout + +_BREATHE_PERIOD_S = 3.0 +"""Seconds for one black → gray → black cycle in the demo background.""" + +_BREATHE_BLACK = -1.0 +"""Model-frame value for black in ``[-1, 1]``.""" + +_BREATHE_GRAY = -0.25 +"""Brightest gray in the cycle. Stops short of white.""" + +_FILE_SELECTION_STATUS_TEXT = { + SelectedFilesStatus.CANCELLED: ( + "SelectedFilesStatus.CANCELLED: user dismissed the selector." + ), + SelectedFilesStatus.BUSY: ( + "SelectedFilesStatus.BUSY: another picker was already open." + ), + SelectedFilesStatus.TOO_LARGE: ( + "SelectedFilesStatus.TOO_LARGE: a file was over 32 MiB, the files " + "together were over 256 MiB, or too many files were chosen." + ), + SelectedFilesStatus.DISALLOWED_TYPE: ( + "SelectedFilesStatus.DISALLOWED_TYPE: suffix not in accept=('.bin', '.raw')." + ), + SelectedFilesStatus.UNAVAILABLE: ( + "SelectedFilesStatus.UNAVAILABLE: picker could not complete " + "(no or gone client, failed chooser, or unreadable file)." + ), +} +_FILE_SELECTION_ERROR_COLOR = (1.0, 0.25, 0.25, 1.0) +"""RGBA used for cancelled and other failed selector results.""" + +_OPEN_FILE_A_REQUEST_ID = "open-file-a" +"""Selector slot for Open file (A).""" + +_OPEN_FILE_B_REQUEST_ID = "open-file-b" +"""Selector slot for Open file (B).""" + +_OPEN_FILE_A_LABEL = "Open file (A)" +"""ImGui label for selector slot A.""" + +_OPEN_FILE_B_LABEL = "Open file (B)" +"""ImGui label for selector slot B.""" + +_OPEN_FILES_REQUEST_ID = "open-files" +"""Selector slot for Open files. Asks for more than one file.""" + +_OPEN_FILES_LABEL = "Open files" +"""ImGui label for a multi-file selector slot.""" + +_OPEN_FILES_MAX_FILE_COUNT = 8 +"""Open files' ``max_file_count``, raised from the default of one.""" + +_FILE_SELECTOR_SLOTS = ( + (_OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID, 1), + (_OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID, 1), + (_OPEN_FILES_LABEL, _OPEN_FILES_REQUEST_ID, _OPEN_FILES_MAX_FILE_COUNT), +) +"""``(label, request_id, max_file_count)`` for each Open button, in display order.""" + +_MAX_FILE_BYTES = 32 * 1024 * 1024 +"""Each button's ``max_file_bytes`` (32 MiB), lowered from the 1 GiB default.""" + +_MAX_TOTAL_BYTES = 256 * 1024 * 1024 +"""Each button's ``max_total_bytes`` (256 MiB), lowered from the 1 GiB default. +Only Open files can pick enough to reach it.""" + +_NO_FILE_SELECTED = "No file selected." +"""Status shown for a button before its first result.""" + +_RESTART_DELAY_S = 5.0 +"""Seconds before a requested restart, long enough to open a picker first.""" + + +class BreathingBackgroundModelLoop(IModelLoop[tuple[SessionDesc, torch.device | str]]): + """Generate a black-to-gray breathing background for the file-picker demo.""" + + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + """Return one background frame along a 3s black-gray cycle.""" + del events + desc, device = self.state + phase = (step_index / desc.frames_per_second_for_step) * ( + 2.0 * math.pi / _BREATHE_PERIOD_S + ) + mix = 0.5 - 0.5 * math.cos(phase) + level = _BREATHE_BLACK + (_BREATHE_GRAY - _BREATHE_BLACK) * mix + return [ + StepResult( + step_index=step_index, + output=torch.full( + (1, 3, desc.video_height, desc.video_width), + level, + dtype=torch.float32, + device=device, + ), + frame_count=1, + output_layout=desc.output_layout, + ) + ] + + def reset(self) -> None: + return + + +@dataclass(slots=True) +class FilePickerState: + """Last file-selector result for each Open button.""" + + statuses: dict[str, str] = field(default_factory=dict) + """Status text by ``request_id``. A missing id has no result yet.""" + + restart_at: float | None = None + """``time.monotonic()`` deadline for a requested session restart.""" + + +def _status_text(event: SelectedFilesUserInputEvent) -> str: + """Return the overlay text for one selector result.""" + if event.status is not SelectedFilesStatus.OK: + return _FILE_SELECTION_STATUS_TEXT[event.status] + return ", ".join( + f"{chosen.name} ({len(chosen.data)} bytes)" for chosen in event.files + ) + + +class FilePickerImGuiUILoop(ImGuiUILoop[FilePickerState]): + """Request ``.bin`` / ``.raw`` picks and show each button's own result.""" + + def step_ui( + self, + imgui: Any, + step_index: int, + events: UserInputEvents, + ) -> Tensor | None: + """Draw the Open buttons and one status line per button.""" + del step_index + for event in events.get_events(): + if isinstance(event, SelectedFilesUserInputEvent): + self.state.statuses[event.request_id] = _status_text(event) + imgui.set_next_window_pos(imgui.ImVec2(16.0, 16.0), imgui.Cond_.once) + imgui.set_next_window_size(imgui.ImVec2(520.0, 300.0), imgui.Cond_.once) + imgui.begin("File picker") + try: + for label, request_id, max_file_count in _FILE_SELECTOR_SLOTS: + if imgui.button(label): + self.request_file_selector( + request_id, + accept=(".bin", ".raw"), + max_file_bytes=_MAX_FILE_BYTES, + max_total_bytes=_MAX_TOTAL_BYTES, + max_file_count=max_file_count, + ) + if self.state.restart_at is None: + if imgui.button(f"Restart session in {_RESTART_DELAY_S:.0f}s"): + self.state.restart_at = time.monotonic() + _RESTART_DELAY_S + else: + remaining = self.state.restart_at - time.monotonic() + if remaining <= 0: + self.request_new_session(self.session_desc) + imgui.text(f"Restarting in {max(remaining, 0.0):.1f}s...") + for label, request_id, _ in _FILE_SELECTOR_SLOTS: + status = self.state.statuses.get(request_id, _NO_FILE_SELECTED) + line = f"{label}: {status}" + if status not in _FILE_SELECTION_STATUS_TEXT.values(): + imgui.text(line) + continue + imgui.push_style_color( + imgui.Col_.text, + imgui.ImVec4(*_FILE_SELECTION_ERROR_COLOR), + ) + try: + imgui.text_wrapped(line) + finally: + imgui.pop_style_color() + finally: + imgui.end() + return self.presented_model_frame() + + def reset(self) -> None: + """Clear every button's result for a new generation.""" + self.state.statuses.clear() + super().reset() + + +class FilePickerSession(ISession): + """Run an ImGui file picker over a generated background.""" + + def __init__( + self, + session_desc: SessionDesc, + *, + device: torch.device | str = "cuda", + ) -> None: + """Configure one file-picker session. + + Args: + session_desc: Output dimensions and loop frequencies. + device: Device used for the background model frame. + """ + if session_desc.output_layout is not VideoTensorLayout.tchw: + raise ValueError( + "The file-picker demo requires tchw output, got " + f"{session_desc.output_layout.value}." + ) + self._session_desc = session_desc + self._device = device + + @property + def session_desc(self) -> SessionDesc: + """Return the resolved session description.""" + return self._session_desc + + def init(self) -> None: + """Register the file-picker UI and background model loops.""" + self.register_ui_loop( + FilePickerImGuiUILoop, + state=FilePickerState(), + width=self._session_desc.video_width, + height=self._session_desc.video_height, + ) + self.register_model_loop( + BreathingBackgroundModelLoop, + state=(self._session_desc, self._device), + ) + + +class FilePickerApplication(IApplication): + """Create ImGui UI file-picker sessions.""" + + def init(self, commandline_args: Sequence[str]) -> None: + """Reject application-specific arguments.""" + if commandline_args: + raise ValueError("The file-picker demo takes no application arguments.") + + def session_desc(self) -> SessionDesc: + """Return the demo's established dimensions and rates.""" + return SessionDesc(video_width=640, video_height=480) + + def create_session(self, session_desc: SessionDesc) -> ISession: + """Create one uninitialized file-picker session.""" + return FilePickerSession(session_desc) + + +def create_app() -> IApplication: + """Return a new file-picker application.""" + return FilePickerApplication() diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py index bc31faadf..cb409bb93 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py @@ -9,10 +9,31 @@ from unittest.mock import Mock import pytest +from imgui_ui_demo.file_picker_app import ( + _FILE_SELECTION_ERROR_COLOR, + _FILE_SELECTION_STATUS_TEXT, + _MAX_FILE_BYTES, + _MAX_TOTAL_BYTES, + _OPEN_FILE_A_LABEL, + _OPEN_FILE_A_REQUEST_ID, + _OPEN_FILE_B_LABEL, + _OPEN_FILE_B_REQUEST_ID, + _OPEN_FILES_LABEL, + _OPEN_FILES_MAX_FILE_COUNT, + _OPEN_FILES_REQUEST_ID, + FilePickerImGuiUILoop, + FilePickerState, +) from imgui_ui_demo.text_input_app import TextInputImGuiUILoop, TextInputState +from numpy import uint64 from flashdreams.runtime_v2.presentation_manager import PresentationManager +from flashdreams.runtime_v2.selected_file import ( + SelectedFile, + SelectedFilesStatus, +) from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent from flashdreams.runtime_v2.user_input_events import UserInputEvents pytestmark = pytest.mark.ci_cpu @@ -47,3 +68,186 @@ def test_text_input_updates_ui_owned_state() -> None: assert state.text == "hello world" imgui.text.assert_called_with("Value: hello world") imgui.end.assert_called_once_with() + + +def _file_picker_loop() -> tuple[FilePickerState, FilePickerImGuiUILoop]: + state = FilePickerState() + loop = FilePickerImGuiUILoop(renderer=Mock()) + loop.register_session_loop_objects( + state=state, + frequency=60, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(), + presentation_manager=PresentationManager(), + ) + return state, loop + + +def _file_picker_imgui(*, pressed: str | tuple[str, ...] = ()) -> SimpleNamespace: + labels = (pressed,) if isinstance(pressed, str) else pressed + return SimpleNamespace( + ImVec2=lambda x, y: (x, y), + ImVec4=lambda x, y, z, w: (x, y, z, w), + Col_=SimpleNamespace(text="text"), + Cond_=SimpleNamespace(once="once"), + set_next_window_pos=Mock(), + set_next_window_size=Mock(), + begin=Mock(), + end=Mock(), + text=Mock(), + text_wrapped=Mock(), + push_style_color=Mock(), + pop_style_color=Mock(), + button=Mock(side_effect=lambda label: label in labels), + ) + + +@pytest.mark.parametrize( + ("label", "request_id"), + [ + (_OPEN_FILE_A_LABEL, _OPEN_FILE_A_REQUEST_ID), + (_OPEN_FILE_B_LABEL, _OPEN_FILE_B_REQUEST_ID), + ], +) +def test_open_file_button_requests_a_client_file(label: str, request_id: str) -> None: + _state, loop = _file_picker_loop() + imgui = _file_picker_imgui(pressed=label) + + loop.step_ui(imgui, 0, UserInputEvents([])) + requests = loop.flush_ui_loop_requests() + + assert requests is not None + [selection] = requests.file_selections + assert selection.request_id == request_id + assert selection.accept == (".bin", ".raw") + assert selection.max_file_bytes == _MAX_FILE_BYTES + assert selection.max_total_bytes == _MAX_TOTAL_BYTES + assert selection.max_file_count == 1 + + +def test_open_files_button_requests_multiple_client_files() -> None: + _state, loop = _file_picker_loop() + imgui = _file_picker_imgui(pressed=_OPEN_FILES_LABEL) + + loop.step_ui(imgui, 0, UserInputEvents([])) + requests = loop.flush_ui_loop_requests() + + assert requests is not None + [selection] = requests.file_selections + assert selection.request_id == _OPEN_FILES_REQUEST_ID + assert selection.accept == (".bin", ".raw") + assert selection.max_file_bytes == _MAX_FILE_BYTES + assert selection.max_total_bytes == _MAX_TOTAL_BYTES + assert selection.max_file_count == _OPEN_FILES_MAX_FILE_COUNT + + +def test_selected_files_event_updates_file_picker_status() -> None: + state, loop = _file_picker_loop() + imgui = _file_picker_imgui() + + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILE_A_REQUEST_ID, + status=SelectedFilesStatus.OK, + files=(SelectedFile(name="seed.png", data=b"xx"),), + ) + ] + ) + loop.step_ui(imgui, 0, loop.user_events) + + assert state.statuses == {_OPEN_FILE_A_REQUEST_ID: "seed.png (2 bytes)"} + assert [call.args[0] for call in imgui.text.call_args_list] == [ + f"{_OPEN_FILE_A_LABEL}: seed.png (2 bytes)", + f"{_OPEN_FILE_B_LABEL}: No file selected.", + f"{_OPEN_FILES_LABEL}: No file selected.", + ] + imgui.text_wrapped.assert_not_called() + imgui.push_style_color.assert_not_called() + + +def test_each_button_shows_only_its_own_result() -> None: + state, loop = _file_picker_loop() + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILE_B_REQUEST_ID, + status=SelectedFilesStatus.OK, + files=(SelectedFile(name="b.bin", data=b"b"),), + ), + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILES_REQUEST_ID, + status=SelectedFilesStatus.UNAVAILABLE, + ), + ] + ) + loop.step_ui(_file_picker_imgui(), 0, loop.user_events) + + assert state.statuses == { + _OPEN_FILE_B_REQUEST_ID: "b.bin (1 bytes)", + _OPEN_FILES_REQUEST_ID: _FILE_SELECTION_STATUS_TEXT[ + SelectedFilesStatus.UNAVAILABLE + ], + } + + +def test_selected_files_event_lists_every_chosen_file() -> None: + state, loop = _file_picker_loop() + imgui = _file_picker_imgui() + + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILES_REQUEST_ID, + status=SelectedFilesStatus.OK, + files=( + SelectedFile(name="a.bin", data=b"aa"), + SelectedFile(name="b.raw", data=b"bbb"), + ), + ) + ] + ) + loop.step_ui(imgui, 0, loop.user_events) + + assert state.statuses == { + _OPEN_FILES_REQUEST_ID: "a.bin (2 bytes), b.raw (3 bytes)" + } + imgui.text.assert_called_with( + f"{_OPEN_FILES_LABEL}: a.bin (2 bytes), b.raw (3 bytes)" + ) + imgui.text_wrapped.assert_not_called() + imgui.push_style_color.assert_not_called() + + +def test_file_picker_shows_error_status_in_red() -> None: + assert set(_FILE_SELECTION_STATUS_TEXT) == { + status for status in SelectedFilesStatus if status is not SelectedFilesStatus.OK + } + state, loop = _file_picker_loop() + imgui = _file_picker_imgui() + cancelled = SelectedFilesStatus.CANCELLED + + loop.user_events = UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=_OPEN_FILE_A_REQUEST_ID, + status=cancelled, + ) + ] + ) + loop.step_ui(imgui, 0, loop.user_events) + + message = _FILE_SELECTION_STATUS_TEXT[cancelled] + assert state.statuses == {_OPEN_FILE_A_REQUEST_ID: message} + imgui.push_style_color.assert_called_once_with("text", _FILE_SELECTION_ERROR_COLOR) + imgui.text_wrapped.assert_called_once_with(f"{_OPEN_FILE_A_LABEL}: {message}") + imgui.pop_style_color.assert_called_once_with() + assert imgui.text.call_count == 2 diff --git a/integrations_v2/imgui_ui_demo/pyproject.toml b/integrations_v2/imgui_ui_demo/pyproject.toml index 980ec0071..af0552b7c 100644 --- a/integrations_v2/imgui_ui_demo/pyproject.toml +++ b/integrations_v2/imgui_ui_demo/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ ] [project.entry-points."flashdreams.applications_v2"] +"imgui-ui-file-picker" = "imgui_ui_demo.file_picker_app:create_app" "imgui-ui-query-string" = "imgui_ui_demo.query_string_app:create_app" "imgui-ui-text-input" = "imgui_ui_demo.text_input_app:create_app" "imgui-ui-window-size" = "imgui_ui_demo.window_size_app:create_app" diff --git a/integrations_v2/red_screen/red_screen/tests/test_red_screen.py b/integrations_v2/red_screen/red_screen/tests/test_red_screen.py index 4dafd5952..f0801ff2f 100644 --- a/integrations_v2/red_screen/red_screen/tests/test_red_screen.py +++ b/integrations_v2/red_screen/red_screen/tests/test_red_screen.py @@ -12,6 +12,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import ( BackpressureMode, PresentationMode, @@ -62,6 +63,9 @@ def get_user_input_events(self) -> UserInputEvents: return self._batches.pop(0) return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc self._is_open = True diff --git a/tests/test_step_agreement.py b/tests/test_step_agreement.py index 7cf1bf365..5a962f3b9 100644 --- a/tests/test_step_agreement.py +++ b/tests/test_step_agreement.py @@ -164,6 +164,9 @@ def _main_thread(self): assert threading.get_ident() == self.session.main_thread assert self.session.ctx.is_main + def put_user_input_event(self, event): + del event + def open(self, desc): self._main_thread()