From 317313a08d215feedab6f36ac2f22c03d703ecc9 Mon Sep 17 00:00:00 2001 From: Set Paing Date: Wed, 30 Sep 2026 14:19:07 -0700 Subject: [PATCH 01/43] [API] Add client file selection on IClientWindow Apps request a file through the UI loop and read bytes from SelectedFilesUserInputEvent, so native, WebRTC, and MP4 share one contract instead of a host-path picker. Signed-off-by: Set Paing --- flashdreams/flashdreams/api_v2/README.md | 6 +- .../flashdreams/api_v2/client_window.py | 16 +++ flashdreams/flashdreams/api_v2/loop.py | 61 +++++++- .../runtime_v2/mp4_client_window.py | 36 ++++- .../runtime_v2/native_window_client_window.py | 108 +++++++++++++++ .../flashdreams/runtime_v2/serving/web/app.js | 27 ++++ .../runtime_v2/serving/webrtc_server.py | 101 +++++++++++++- .../flashdreams/runtime_v2/session_runner.py | 4 + .../runtime_v2/user_input_event.py | 34 +++++ .../runtime_v2/webrtc_client_window.py | 26 +++- .../test_native_window_client_window.py | 68 +++++++++ flashdreams/test_v2/test_selected_files.py | 88 ++++++++++++ flashdreams/test_v2/test_session_runner.py | 26 ++++ .../test_v2/test_webrtc_client_window.py | 47 ++++++- integrations_v2/imgui_ui_demo/README.md | 11 ++ .../imgui_ui_demo/imgui_ui_demo/__init__.py | 3 + .../imgui_ui_demo/file_picker_app.py | 131 ++++++++++++++++++ .../imgui_ui_demo/tests/test_imgui_ui_demo.py | 74 ++++++++++ integrations_v2/imgui_ui_demo/pyproject.toml | 1 + 19 files changed, 860 insertions(+), 8 deletions(-) create mode 100644 flashdreams/test_v2/test_selected_files.py create mode 100644 integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py diff --git a/flashdreams/flashdreams/api_v2/README.md b/flashdreams/flashdreams/api_v2/README.md index 5670024b1..f8b8a91de 100644 --- a/flashdreams/flashdreams/api_v2/README.md +++ b/flashdreams/flashdreams/api_v2/README.md @@ -210,7 +210,11 @@ exposes `imgui_bundle.imgui` and an image-like pixel upload convenience form. A UI control that needs a fresh application session calls `request_new_session(session_desc)` with a fully resolved replacement description; the runtime cleans the current session and passes that description -to `ApplicationRunner` unchanged. +to `ApplicationRunner` unchanged. A control that needs a file from the client +calls `request_selected_files(request_id)`; the window later reports +`SelectedFilesUserInputEvent` through the same input stream as keyboard and +mouse. The application reads file bytes from that event and does not open a +host path. For SlangPy's smaller retained widget API, subclass `SlangPyUILoop` from `flashdreams.runtime_v2.slangpy_ui_loop`. The [`slangpy_ui_demo` integration](../../../integrations_v2/slangpy_ui_demo/README.md) diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 1a2dd1754..936f4a7cf 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -46,3 +46,19 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: new_window_size: Requested ``(width, height)`` in pixels. """ pass + + # Optional to implement + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Ask this client to pick files and report them as input events. + + The chosen files arrive later through + :meth:`InputSource.get_user_input_events`. + + Args: + request_id: Correlation token for the later input event. + initial_path: Directory the selector should start in; ``None`` lets + the window choose a default. + """ + pass diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 56740c09c..916b32f5a 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -10,8 +10,9 @@ import time from abc import ABC, abstractmethod from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum +from pathlib import Path from typing import TYPE_CHECKING, Any, Generic, TypeVar, final from torch import Tensor @@ -61,6 +62,17 @@ class _Message(Generic[StateT]): """Operation to run before the loop's next step.""" +@dataclass(frozen=True, slots=True) +class FileSelectionRequest: + """One client file-selector request queued by a UI loop.""" + + request_id: str + """Correlation token for the later selected-files input event.""" + + initial_path: str + """Directory the selector should start in.""" + + @dataclass(slots=True) class UILoopRequests: """Changes requested by a UI loop for the session runtime to apply. @@ -80,6 +92,9 @@ class UILoopRequests: new_window_size: tuple[int, int] | None = None """Requested downstream window size, or ``None`` to leave it unchanged.""" + file_selections: list[FileSelectionRequest] = field(default_factory=list) + """File-selector requests to apply, in the order they were queued.""" + class ILoop(ABC, Generic[StateT]): """Shared state, messaging, and lifecycle for a session loop. @@ -466,6 +481,49 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: raise ValueError("new_window_size dimensions must be > 0.") self.get_or_create_ui_loop_requests().new_window_size = new_window_size + @final + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Ask the client window to open a file selector. + + The window reports the result later as + :class:`~flashdreams.runtime_v2.user_input_event.SelectedFilesUserInputEvent`. + + Args: + request_id: Correlation token for the later input event. + initial_path: Directory the selector should start in; ``None`` uses + the current user's home directory. + + Raises: + TypeError: ``request_id`` or ``initial_path`` is not a string. + ValueError: ``request_id`` is empty, ``initial_path`` is empty, or + ``request_id`` is already queued in this UI step. + """ + if not isinstance(request_id, str): + raise TypeError("request_id must be a string.") + if not request_id: + raise ValueError("request_id must be a non-empty string.") + if initial_path is None: + resolved_initial_path = str(Path.home()) + else: + if not isinstance(initial_path, str): + raise TypeError("initial_path must be a string.") + if not initial_path: + raise ValueError("initial_path must be a non-empty string.") + resolved_initial_path = initial_path + requests = self.get_or_create_ui_loop_requests() + if any( + selection.request_id == request_id for selection in requests.file_selections + ): + raise ValueError(f"Duplicate file-selection request id {request_id!r}.") + requests.file_selections.append( + FileSelectionRequest( + request_id=request_id, + initial_path=resolved_initial_path, + ) + ) + def get_or_create_ui_loop_requests(self) -> UILoopRequests: if self._ui_loop_requests is None: self._ui_loop_requests = UILoopRequests() @@ -561,6 +619,7 @@ def invoke_async(loop: ILoop[StateT], operation: Callable[[StateT], None]) -> No __all__ = [ + "FileSelectionRequest", "ILoop", "IModelLoop", "IUILoop", diff --git a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py index 3e2c72ac6..8784d02f5 100644 --- a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py @@ -5,19 +5,28 @@ from pathlib import Path +from numpy import uint64 + from flashdreams.api_v2.client_window import IClientWindow from flashdreams.runtime_v2.mp4_output_sink import Mp4OutputSink from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, + UserInputEvent, +) from flashdreams.runtime_v2.user_input_events import UserInputEvents class Mp4ClientWindow(IClientWindow): - """Write UI frames to an MP4 file and report no input. + """Write UI frames to an MP4 file. The session must finish on its own because this window never sends a close event. Use ``BackpressureMode.BLOCK`` with ``PresentationMode.ON_DEMAND`` to write every frame once. + + File-selector requests complete immediately with no files; there is no + client to pick one. """ def __init__(self, path: str | Path) -> None: @@ -27,6 +36,7 @@ def __init__(self, path: str | Path) -> None: """ self._path = Path(path) self._video_sink = Mp4OutputSink(path) + self._pending_events: list[UserInputEvent] = [] @property def path(self) -> Path: @@ -34,8 +44,28 @@ def path(self) -> Path: return self._path def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return file-selector results queued since the previous poll.""" + events = self._pending_events + self._pending_events = [] + return UserInputEvents(events) + + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Complete a file-selector request with no files. + + Args: + request_id: Correlation token from the UI loop. + initial_path: Ignored; an MP4 has no selector. + """ + del initial_path + self._pending_events.append( + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=request_id, + files=(), + ) + ) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the MP4 presentation dimensions without replacing the session. diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index 3c73881d2..b83e37108 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -7,11 +7,15 @@ import logging import queue +import shutil +import subprocess +import sys import threading import time from collections import deque from collections.abc import Callable from dataclasses import replace +from pathlib import Path from typing import TYPE_CHECKING, Any, cast import numpy as np @@ -29,6 +33,8 @@ KeyboardInputState, KeyboardUserInputEvent, MouseUserInputEvent, + SelectedFile, + SelectedFilesUserInputEvent, ) from flashdreams.runtime_v2.user_input_events import UserInputEvents from flashdreams.runtime_v2.video_encoder import result_to_rgb24_tensor @@ -38,6 +44,69 @@ _LOGGER = logging.getLogger(__name__) + +def _ask_open_filename(*, initial_dir: str) -> str: + """Return a path from the OS file picker, or ``""`` when cancelled.""" + if sys.platform.startswith("linux"): + return _ask_open_filename_linux(initial_dir=initial_dir) + return _ask_open_filename_tkinter(initial_dir=initial_dir) + + +def _ask_open_filename_linux(*, initial_dir: str) -> str: + """Open the desktop file picker via ``zenity`` or ``kdialog``. + + ``zenity --file-selection`` uses ``GtkFileChooserNative``, which is the + xdg-desktop-portal chooser. The helper is a separate process so the + GLFW window does not host GTK. + """ + directory = str(Path(initial_dir)) + zenity = shutil.which("zenity") + if zenity is not None: + argv = [ + zenity, + "--file-selection", + f"--filename={directory}/", + "--title=Select a file", + ] + else: + kdialog = shutil.which("kdialog") + if kdialog is None: + raise RuntimeError( + "Native file picker needs a desktop chooser (zenity or kdialog)." + ) + argv = [kdialog, "--getopenfilename", directory] + completed = subprocess.run( + argv, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + return "" + return completed.stdout.strip() + + +def _ask_open_filename_tkinter(*, initial_dir: str) -> str: + """Open the Tk file dialog (native Explorer / macOS panel).""" + import tkinter as tk + from tkinter import filedialog + + root = tk.Tk() + root.withdraw() + try: + root.attributes("-topmost", True) + except tk.TclError: + pass + try: + selected = filedialog.askopenfilename( + initialdir=initial_dir, + title="Select a file", + ) + finally: + root.destroy() + return str(selected or "") + + _PRINTABLE_KEY_NAMES = { "space": " ", "apostrophe": "'", @@ -126,6 +195,7 @@ def __init__( self._pressed_key_values: dict[str, str] = {} self._hide_cursor = False self._lock_cursor_to_window = False + self._in_flight_file_requests: set[str] = set() def request_hide_cursor(self, hide_cursor: bool) -> None: """Show or hide the cursor in the native window.""" @@ -164,6 +234,44 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: ) self._window_size = presenter.resize(*new_window_size) + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Open a native OS file selector and enqueue the chosen file. + + Args: + request_id: Correlation token for the later input event. + initial_path: Directory the selector should start in; ``None`` uses + the current user's home directory. + + Raises: + ValueError: ``request_id`` is already waiting for a selector. + """ + if request_id in self._in_flight_file_requests: + raise ValueError(f"Duplicate file-selection request id {request_id!r}.") + self._in_flight_file_requests.add(request_id) + try: + selected = _ask_open_filename( + initial_dir=initial_path or str(Path.home()), + ) + files: tuple[SelectedFile, ...] = () + if selected: + path = Path(selected) + try: + data = path.read_bytes() + except OSError: + data = b"" + files = (SelectedFile(name=path.name, path=str(path), data=data),) + self._put_input( + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=request_id, + files=files, + ) + ) + finally: + self._in_flight_file_requests.discard(request_id) + def open(self, session_desc: SessionDesc) -> None: """Create the GLFW window on the runtime's UI thread. diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index d8e10c3e5..998b403a1 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -74,6 +74,29 @@ const applyCursorOptions = options => { } }; +const openFileSelector = options => { + const requestId = options?.id; + if (typeof requestId !== "string" || !requestId) { + return; + } + const input = document.createElement("input"); + input.type = "file"; + input.addEventListener("change", () => { + const file = input.files?.[0]; + if (!file) { + send({type: "file_selector_result", id: requestId, cancelled: true}); + return; + } + const body = new FormData(); + body.append("request_id", requestId); + body.append("file", file, file.name); + fetch("/api/files", {method: "POST", body}).catch(error => { + console.debug("Unable to upload the selected file.", error); + }); + }); + input.click(); +}; + window.addEventListener("focus", updateCursorVisibility); window.addEventListener("blur", updateCursorVisibility); @@ -87,6 +110,10 @@ controls.addEventListener("message", event => { applyCursorOptions(payload); return; } + if (payload?.type === "file_selector") { + openFileSelector(payload); + return; + } if (payload?.type === "error") { console.warn(`WebRTC server: ${payload.message ?? "unknown error"}`); } diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 721959f85..fd23e24e4 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -16,11 +16,13 @@ from dataclasses import dataclass from fractions import Fraction from importlib.resources import files +from pathlib import Path from typing import Any, Literal, cast import numpy as np import torch from aiohttp import web +from aiohttp.web_request import FileField from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription from aiortc.mediastreams import MediaStreamError from av import VideoFrame @@ -40,6 +42,8 @@ MouseUserInputEvent, QueryStringUserInputEvent, ResetUserInputEvent, + SelectedFile, + SelectedFilesUserInputEvent, TouchUserInputEvent, UserInputEvent, XRControllerUserInputEvent, @@ -53,6 +57,9 @@ _INTERACTIVE_FRAME_QUEUE_SIZE = 2 """Send-ready frames retained while the media sender is temporarily busy.""" +_MAX_SELECTED_FILE_BYTES = 32 * 1024 * 1024 +"""Maximum file the browser may upload for one selected-files event.""" + _TRANSFER_STREAM_PRIORITY = -1 """Portable high-priority CUDA stream request for interactive output copies.""" @@ -346,6 +353,7 @@ def __init__( self._client_connected = False self._hide_cursor = False self._lock_cursor_to_window = False + self._queued_file_selectors: list[dict[str, str]] = [] self._thread = threading.Thread( target=self._run_server, name="flashdreams-webrtc", @@ -436,6 +444,49 @@ def _send_cursor_options(self) -> None: self._sent_cursor_options = cursor_options + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Ask the connected browser to open a file selector. + + Args: + request_id: Correlation token for the later input event. + initial_path: Directory hint for the selector; ``None`` uses the + current user's home directory. + """ + if not isinstance(request_id, str) or not request_id: + raise ValueError("request_id must be a non-empty string.") + if initial_path is None: + resolved_initial_path = str(Path.home()) + elif not isinstance(initial_path, str) or not initial_path: + raise ValueError("initial_path must be a non-empty string.") + else: + resolved_initial_path = initial_path + payload = { + "type": "file_selector", + "id": request_id, + "initial_path": resolved_initial_path, + } + loop = self._loop + if loop is None: + raise RuntimeError("WebRTC server is not running.") + loop.call_soon_threadsafe(self._send_file_selector, payload) + + def _send_file_selector(self, payload: dict[str, str]) -> None: + """Send one file-selector request on the WebRTC event-loop thread.""" + channel = self._control_channel + if channel is None or channel.readyState != "open": + self._queued_file_selectors.append(payload) + return + channel.send(json.dumps(payload)) + + def _flush_queued_file_selectors(self) -> None: + """Send file-selector requests that arrived before the datachannel opened.""" + queued = self._queued_file_selectors + self._queued_file_selectors = [] + for payload in queued: + self._send_file_selector(payload) + def event_timestamp_us(self) -> np.uint64: """Return the current timestamp relative to this server's first session. @@ -720,11 +771,12 @@ def _run_server(self) -> None: async def _start_server(self) -> None: """Create and bind the standalone aiohttp application.""" self._offer_lock = asyncio.Lock() - app = web.Application() + app = web.Application(client_max_size=_MAX_SELECTED_FILE_BYTES + 1024 * 1024) app.router.add_get("/", self._serve_browser) app.router.add_get("/app.js", self._serve_browser_script) app.router.add_get("/healthz", self._health) app.router.add_post("/api/webrtc/offer", self._offer) + app.router.add_post("/api/files", self._upload_selected_file) runner = web.AppRunner(app) await runner.setup() address_family = socket.AF_INET6 if ":" in self._host else socket.AF_INET @@ -762,6 +814,36 @@ async def _health(self, _: web.Request) -> web.Response: } ) + async def _upload_selected_file(self, request: web.Request) -> web.Response: + """Turn a browser file upload into a selected-files input event.""" + if self._closed: + raise web.HTTPServiceUnavailable(reason="WebRTC server is closed.") + timestamp_us = self._timestamp_us() + if timestamp_us is None: + raise web.HTTPConflict(reason="WebRTC server is not open.") + post = await request.post() + request_id = post.get("request_id") + if not isinstance(request_id, str) or not request_id: + raise web.HTTPBadRequest(reason="File upload requires request_id.") + upload = post.get("file") + if not isinstance(upload, FileField): + raise web.HTTPBadRequest(reason="File upload requires a file.") + data = upload.file.read() + if len(data) > _MAX_SELECTED_FILE_BYTES: + raise web.HTTPRequestEntityTooLarge( + max_size=_MAX_SELECTED_FILE_BYTES, + actual_size=len(data), + ) + name = upload.filename or "upload" + self._append_event( + SelectedFilesUserInputEvent( + timestamp=timestamp_us, + request_id=request_id, + files=(SelectedFile(name=name, path=name, data=data),), + ) + ) + return web.Response(status=204) + async def _offer(self, request: web.Request) -> web.Response: """Negotiate one browser peer connection.""" if self._closed: @@ -815,12 +897,14 @@ def on_datachannel(channel: Any) -> None: def on_open() -> None: if self._peer_connection is peer_connection: self._send_cursor_options() + self._flush_queued_file_selectors() if ( self._peer_connection is peer_connection and channel.readyState == "open" ): self._send_cursor_options() + self._flush_queued_file_selectors() @channel.on("message") def on_message(message: Any) -> None: @@ -903,6 +987,7 @@ async def on_connectionstatechange() -> None: self._sent_cursor_options = None if control_channel is not None and control_channel.readyState == "open": self._send_cursor_options() + self._flush_queued_file_selectors() query_string = request.rel_url.raw_query_string if query_string: timestamp_us = self._timestamp_us() @@ -1080,6 +1165,20 @@ def _buffer_browser_message( event = ResetUserInputEvent(timestamp=timestamp_us) elif event_type == "close": event = CloseUserInputEvent(timestamp=timestamp_us) + elif event_type == "file_selector_result": + request_id = payload.get("id") + if not isinstance(request_id, str) or not request_id: + raise ValueError("File selector result requires a non-empty id.") + if payload.get("cancelled") is not True: + raise ValueError( + "File selector result must set cancelled true; send files " + "with POST /api/files." + ) + event = SelectedFilesUserInputEvent( + timestamp=timestamp_us, + request_id=request_id, + files=(), + ) else: raise ValueError("Unsupported browser event type.") self._append_event(event) diff --git a/flashdreams/flashdreams/runtime_v2/session_runner.py b/flashdreams/flashdreams/runtime_v2/session_runner.py index afab47e7f..30a2a2fd5 100644 --- a/flashdreams/flashdreams/runtime_v2/session_runner.py +++ b/flashdreams/flashdreams/runtime_v2/session_runner.py @@ -143,6 +143,10 @@ def run_ui_once(*, step_requested: bool = True) -> None: ) if request.new_window_size is not None: window.request_new_window_size(request.new_window_size) + for selection in request.file_selections: + window.request_selected_files( + selection.request_id, selection.initial_path + ) if request.new_session is not None: next_session_desc = request.new_session stop.set() diff --git a/flashdreams/flashdreams/runtime_v2/user_input_event.py b/flashdreams/flashdreams/runtime_v2/user_input_event.py index 4190efae1..13674d5c5 100644 --- a/flashdreams/flashdreams/runtime_v2/user_input_event.py +++ b/flashdreams/flashdreams/runtime_v2/user_input_event.py @@ -230,6 +230,40 @@ def get_type_name(cls) -> str: """Optional controller quaternion in client XR space.""" +@dataclass(frozen=True, slots=True) +class SelectedFile: + """One file chosen by a client file selector.""" + + name: str + """File name shown to the application, without requiring a host path.""" + + path: str + """Native host path, or the file name when the client has no path.""" + + data: bytes + """File contents. Applications should use this rather than opening ``path``.""" + + +@dataclass(frozen=True, slots=True, eq=False) +class SelectedFilesUserInputEvent(UserInputEvent): + """Files chosen for one :meth:`IClientWindow.request_selected_files` call. + + An empty ``files`` tuple is a cancelled or unavailable selector, including + ``--mode mp4`` where no client can pick a file. + """ + + @classmethod + def get_type_name(cls) -> str: + """Return the event type name.""" + return "selected_files" + + request_id: str + """Identifier supplied with the matching file-selection request.""" + + files: tuple[SelectedFile, ...] = () + """Chosen files, empty when the selector was cancelled or unavailable.""" + + @dataclass(frozen=True, slots=True, eq=False) class UnknownUserInputEvent(UserInputEvent): """User input event for an unknown input modality.""" diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 8eaa9b809..320df157c 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -13,7 +13,10 @@ from flashdreams.runtime_v2.serving.webrtc_server import WebRTCServer from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult -from flashdreams.runtime_v2.user_input_event import UserInputEvent +from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, + UserInputEvent, +) from flashdreams.runtime_v2.user_input_events import UserInputEvents @@ -55,6 +58,7 @@ def __init__( self._input_events: deque[UserInputEvent] = deque() self._hide_cursor = False self._lock_cursor_to_window = False + self._in_flight_file_requests: set[str] = set() self._input_lock = threading.Lock() # Offset from the server's stable clock to the current session's clock. self._session_event_offset_us = uint64(0) @@ -68,6 +72,8 @@ def handle_input(event: UserInputEvent) -> None: """Buffer one backend event for the ``InputSource`` protocol.""" # TODO: do we really need to buffer all events? Some mouse moves may be superseded by later ones. with self._input_lock: + if isinstance(event, SelectedFilesUserInputEvent): + self._in_flight_file_requests.discard(event.request_id) self._input_events.append(event) self.server.register_input_callback(handle_input) @@ -100,6 +106,24 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """ self.server.request_new_window_size(new_window_size) + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Ask the connected browser to open a file selector. + + Args: + request_id: Correlation token for the later input event. + initial_path: Directory hint for the selector; browsers may ignore + it. ``None`` uses the current user's home directory. + + Raises: + ValueError: ``request_id`` is already waiting for a selector. + """ + if request_id in self._in_flight_file_requests: + raise ValueError(f"Duplicate file-selection request id {request_id!r}.") + self._in_flight_file_requests.add(request_id) + self.server.request_selected_files(request_id, initial_path=initial_path) + def open(self, session_desc: SessionDesc) -> None: """Implement ``OutputSink.open`` by configuring WebRTC output. diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index 4f327fb01..c039ed928 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -8,6 +8,7 @@ import queue import threading from collections.abc import Callable, Sequence +from pathlib import Path from types import SimpleNamespace from typing import TYPE_CHECKING, Any, cast from unittest.mock import Mock @@ -27,6 +28,7 @@ KeyboardInputState, KeyboardUserInputEvent, MouseUserInputEvent, + SelectedFilesUserInputEvent, ) from flashdreams.runtime_v2.video_encoder import result_to_rgb24_tensor from flashdreams.runtime_v2.video_tensor import VideoTensorLayout @@ -585,6 +587,72 @@ def create_presenter(**kwargs: object) -> _Presenter: assert (unlocked_mouse.x, unlocked_mouse.y) == (1.0, 0.0) +def test_native_window_reports_a_selected_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + chosen = tmp_path / "seed.png" + chosen.write_bytes(b"png-bytes") + monkeypatch.setattr( + native_window_module, + "_ask_open_filename", + lambda *, initial_dir: str(chosen), + ) + presenter = _Presenter() + window = NativeWindowClientWindow( + presenter_factory=_presenter_factory(presenter), + ) + window.open(_session_desc()) + window.request_selected_files("open-1", str(tmp_path)) + events = window.get_user_input_events().get_events() + window.close() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.files[0].name == "seed.png" + assert event.files[0].path == str(chosen) + assert event.files[0].data == b"png-bytes" + + +def test_ask_open_filename_linux_uses_zenity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + native_window_module.shutil, + "which", + lambda name: "/usr/bin/zenity" if name == "zenity" else None, + ) + + def fake_run(argv: list[str], **kwargs: object) -> object: + assert argv[0] == "/usr/bin/zenity" + assert "--file-selection" in argv + assert any(arg.startswith("--filename=/tmp/") for arg in argv) + return SimpleNamespace(returncode=0, stdout="/tmp/seed.png\n") + + monkeypatch.setattr(native_window_module.subprocess, "run", fake_run) + assert ( + native_window_module._ask_open_filename_linux(initial_dir="/tmp") + == "/tmp/seed.png" + ) + + +def test_ask_open_filename_linux_cancel_is_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + native_window_module.shutil, + "which", + lambda name: "/usr/bin/zenity" if name == "zenity" else None, + ) + monkeypatch.setattr( + native_window_module.subprocess, + "run", + lambda argv, **kwargs: SimpleNamespace(returncode=1, stdout=""), + ) + assert native_window_module._ask_open_filename_linux(initial_dir="/tmp") == "" + + def test_native_window_reports_standard_gamepad_events() -> None: presenter = _Presenter() clock_values = iter((1_000_000, 1_001_000, 1_002_000, 1_003_000)) diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py new file mode 100644 index 000000000..2e156b2de --- /dev/null +++ b/flashdreams/test_v2/test_selected_files.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for client file selection through the v2 window contract.""" + +import queue +import threading +from pathlib import Path + +import pytest +from numpy import uint64 + +from flashdreams.api_v2.loop import IUILoop +from flashdreams.runtime_v2.mp4_client_window import Mp4ClientWindow +from flashdreams.runtime_v2.presentation_manager import PresentationManager +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import ( + SelectedFile, + SelectedFilesUserInputEvent, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents + +pytestmark = pytest.mark.ci_cpu + + +class _UILoop(IUILoop[None]): + def step(self, step_index: int, events: UserInputEvents) -> StepResult | None: + del step_index, events + return None + + def reset(self) -> None: + return + + +def _ui_loop() -> _UILoop: + loop = _UILoop() + loop.register_session_loop_objects( + state=None, + frequency=0, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(), + presentation_manager=PresentationManager(), + ) + return loop + + +def test_selected_files_event_carries_bytes_and_request_id() -> None: + event = SelectedFilesUserInputEvent( + timestamp=uint64(1), + request_id="open-1", + files=(SelectedFile(name="seed.png", path="/tmp/seed.png", data=b"png"),), + ) + + assert event.get_type_name() == "selected_files" + assert event.request_id == "open-1" + assert event.files[0].data == b"png" + + +def test_ui_loop_queues_file_selection_and_rejects_duplicate_ids() -> None: + loop = _ui_loop() + loop.request_selected_files("open-1", "/tmp") + + with pytest.raises(ValueError, match="Duplicate file-selection request id"): + loop.request_selected_files("open-1", "/var") + + requests = loop.flush_ui_loop_requests() + assert requests is not None + assert len(requests.file_selections) == 1 + assert requests.file_selections[0].request_id == "open-1" + assert requests.file_selections[0].initial_path == "/tmp" + + +def test_mp4_window_completes_file_selection_with_no_files(tmp_path: Path) -> None: + window = Mp4ClientWindow(tmp_path / "clip.mp4") + window.request_selected_files("open-1", "/tmp") + + events = window.get_user_input_events().get_events() + + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.files == () + assert window.get_user_input_events().get_events() == [] diff --git a/flashdreams/test_v2/test_session_runner.py b/flashdreams/test_v2/test_session_runner.py index 90fe8751b..43b673119 100644 --- a/flashdreams/test_v2/test_session_runner.py +++ b/flashdreams/test_v2/test_session_runner.py @@ -659,6 +659,7 @@ def __init__( self.session_desc: SessionDesc | None = None self.results: list[StepResult] = [] self.cursor_requests: list[tuple[str, bool]] = [] + self.file_selection_requests: list[tuple[str, str | None]] = [] def request_hide_cursor(self, hide_cursor: bool) -> None: """Record one cursor visibility request.""" @@ -670,6 +671,13 @@ def request_lock_cursor_to_window(self, lock_cursor_to_window: bool) -> None: self._log.record("window.request_lock_cursor_to_window") self.cursor_requests.append(("lock", lock_cursor_to_window)) + def request_selected_files( + self, request_id: str, initial_path: str | None = None + ) -> None: + """Record one file-selector request.""" + self._log.record("window.request_selected_files") + self.file_selection_requests.append((request_id, initial_path)) + def get_user_input_events(self) -> UserInputEvents: self._log.record("window.get_user_input_events") with self._lock: @@ -1364,6 +1372,24 @@ def init(self) -> None: assert "window.close" not in log.calls +def test_run_session_forwards_file_selection_requests_to_the_window() -> None: + log = CallLog() + + class RequestingSession(FakeSession): + def init(self) -> None: + super().init() + invoke_async( + self.ui_loop, + lambda _: self.ui_loop.request_selected_files("open-1", "/tmp"), + ) + + window = RecordingClientWindow(log) + run_session(RequestingSession(_session_desc(), log), window, steps=1) + + assert window.file_selection_requests == [("open-1", "/tmp")] + assert "window.request_selected_files" in log.calls + + def test_interactive_ui_can_replace_an_already_finished_session() -> None: log = CallLog() diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index fb3435860..812f4875e 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -20,7 +20,7 @@ pytest.importorskip("aiohttp") pytest.importorskip("aiortc") -from aiohttp import ClientSession +from aiohttp import ClientSession, FormData from aiortc import ( MediaStreamTrack, RTCDataChannel, @@ -43,6 +43,7 @@ KeyboardUserInputEvent, MouseUserInputEvent, QueryStringUserInputEvent, + SelectedFilesUserInputEvent, TouchUserInputEvent, XRControllerUserInputEvent, ) @@ -141,6 +142,47 @@ async def test_browser_query_string_reaches_the_input_stream() -> None: window.close() +@pytest.mark.asyncio +async def test_browser_file_upload_reaches_the_input_stream() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + window.request_selected_files("open-1", "/tmp") + with pytest.raises(ValueError, match="Duplicate file-selection request id"): + window.request_selected_files("open-1", "/tmp") + + form = FormData() + form.add_field("request_id", "open-1") + form.add_field( + "file", + b"hello", + filename="hello.txt", + content_type="text/plain", + ) + async with ClientSession() as client: + async with client.post( + f"{window.server.url}api/files", data=form + ) as response: + assert response.status == 204 + + events = [] + for _ in range(100): + events.extend(window.get_user_input_events().get_events()) + if events: + break + await asyncio.sleep(0.01) + assert len(events) == 1 + event = events[0] + assert isinstance(event, SelectedFilesUserInputEvent) + assert event.request_id == "open-1" + assert event.files[0].name == "hello.txt" + assert event.files[0].data == b"hello" + + window.request_selected_files("open-1", "/tmp") + finally: + window.close() + + @pytest.mark.asyncio async def test_window_buffers_browser_events_until_drained() -> None: window = WebRTCClientWindow() @@ -215,6 +257,9 @@ async def test_window_buffers_browser_events_until_drained() -> None: assert '["failed", "closed"]' in connection_handler assert "navigator.getGamepads" in browser_script assert 'type: "touch"' in browser_script + assert "openFileSelector" in browser_script + assert 'type === "file_selector"' in browser_script + assert 'fetch("/api/files"' in browser_script window.request_hide_cursor(True) window.open(_session_desc()) diff --git a/integrations_v2/imgui_ui_demo/README.md b/integrations_v2/imgui_ui_demo/README.md index dcca7516a..50eaa79a9 100644 --- a/integrations_v2/imgui_ui_demo/README.md +++ b/integrations_v2/imgui_ui_demo/README.md @@ -8,6 +8,8 @@ SPDX-License-Identifier: Apache-2.0 Small ImGui applications for the FlashDreams v2 loop runtime: - `imgui-ui-text-input` renders an editable text field over model output. +- `imgui-ui-file-picker` opens the client file selector and shows the chosen + name and size (the file bytes are on the event; apps should use those). - `imgui-ui-query-string` sets background from a `?(r,g,b)` browser URL. - `imgui-ui-window-size` resizes native window or UI render target without resetting session. @@ -24,6 +26,15 @@ Run the text-input application: uv run --no-sync flashdreams-run-v2 imgui-ui-text-input --mode native-window ``` +Run the file-picker application. Native Linux needs `zenity` or `kdialog` +(the desktop portal picker). Windows and macOS use Tk. WebRTC uses the +browser picker; uploads larger than 32 MiB are rejected. +```bash +uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode native-window +uv run --no-sync flashdreams-run-v2 imgui-ui-file-picker --mode webrtc \ + --host 127.0.0.1 --port 8080 +``` + Run the query-string application: ```bash uv run --no-sync flashdreams-run-v2 imgui-ui-query-string \ diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py index b3fcbf18f..fa3acf920 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/__init__.py @@ -3,10 +3,13 @@ """Interactive ImGui UI applications for the v2 loop runtime.""" +from .file_picker_app import FilePickerApplication, FilePickerSession from .text_input_app import TextInputApplication, TextInputSession, create_app from .window_size_app import WindowSizeApplication, WindowSizeSession __all__ = [ + "FilePickerApplication", + "FilePickerSession", "TextInputApplication", "TextInputSession", "WindowSizeApplication", diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py new file mode 100644 index 000000000..91ce96693 --- /dev/null +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/file_picker_app.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ImGui file-picker application for the v2 loop runtime.""" + +import uuid +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +import torch +from torch import Tensor + +from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout + +from .text_input_app import BackgroundModelLoop + + +@dataclass(slots=True) +class FilePickerState: + """Last file-selector result shown by the demo overlay.""" + + status: str = "No file selected." + """Human-readable status of the last selector result.""" + + +class FilePickerImGuiUILoop(ImGuiUILoop[FilePickerState]): + """Request a client file pick and display the chosen name and size.""" + + def step_ui( + self, + imgui: Any, + step_index: int, + events: UserInputEvents, + ) -> Tensor | None: + """Draw the Open-file control and apply any selected-files events.""" + del step_index + for event in events.get_events(): + if not isinstance(event, SelectedFilesUserInputEvent): + continue + if not event.files: + self.state.status = "Cancelled." + continue + chosen = event.files[0] + self.state.status = f"{chosen.name} ({len(chosen.data)} bytes)" + + imgui.set_next_window_pos(imgui.ImVec2(16.0, 16.0), imgui.Cond_.once) + imgui.set_next_window_size(imgui.ImVec2(420.0, 130.0), imgui.Cond_.once) + imgui.begin("File picker") + try: + if imgui.button("Open file"): + self.request_selected_files(uuid.uuid4().hex) + imgui.text(self.state.status) + finally: + imgui.end() + return self.presented_model_frame() + + def reset(self) -> None: + """Clear the last selector result for a new generation.""" + self.state.status = "No file selected." + super().reset() + + +class FilePickerSession(ISession): + """Run an ImGui file picker over a generated background.""" + + def __init__( + self, + session_desc: SessionDesc, + *, + device: torch.device | str = "cuda", + ) -> None: + """Configure one file-picker session. + + Args: + session_desc: Output dimensions and loop frequencies. + device: Device used for the background model frame. + """ + if session_desc.output_layout is not VideoTensorLayout.tchw: + raise ValueError( + "The file-picker demo requires tchw output, got " + f"{session_desc.output_layout.value}." + ) + self._session_desc = session_desc + self._device = device + + @property + def session_desc(self) -> SessionDesc: + """Return the resolved session description.""" + return self._session_desc + + def init(self) -> None: + """Register the file-picker UI and background model loops.""" + self.register_ui_loop( + FilePickerImGuiUILoop, + state=FilePickerState(), + width=self._session_desc.video_width, + height=self._session_desc.video_height, + ) + self.register_model_loop( + BackgroundModelLoop, + state=(self._session_desc, self._device), + ) + + +class FilePickerApplication(IApplication): + """Create ImGui UI file-picker sessions.""" + + def init(self, commandline_args: Sequence[str]) -> None: + """Reject application-specific arguments.""" + if commandline_args: + raise ValueError("The file-picker demo takes no application arguments.") + + def session_desc(self) -> SessionDesc: + """Return the demo's established dimensions and rates.""" + return SessionDesc(video_width=640, video_height=480) + + def create_session(self, session_desc: SessionDesc) -> ISession: + """Create one uninitialized file-picker session.""" + return FilePickerSession(session_desc) + + +def create_app() -> IApplication: + """Return a new file-picker application.""" + return FilePickerApplication() diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py index bc31faadf..9763cda37 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py @@ -9,10 +9,16 @@ from unittest.mock import Mock import pytest +from numpy import uint64 +from imgui_ui_demo.file_picker_app import FilePickerImGuiUILoop, FilePickerState from imgui_ui_demo.text_input_app import TextInputImGuiUILoop, TextInputState from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.user_input_event import ( + SelectedFile, + SelectedFilesUserInputEvent, +) from flashdreams.runtime_v2.user_input_events import UserInputEvents pytestmark = pytest.mark.ci_cpu @@ -47,3 +53,71 @@ def test_text_input_updates_ui_owned_state() -> None: assert state.text == "hello world" imgui.text.assert_called_with("Value: hello world") imgui.end.assert_called_once_with() + + +def _file_picker_loop() -> tuple[FilePickerState, FilePickerImGuiUILoop]: + state = FilePickerState() + loop = FilePickerImGuiUILoop(renderer=Mock()) + loop.register_session_loop_objects( + state=state, + frequency=60, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(), + presentation_manager=PresentationManager(), + ) + return state, loop + + +def test_open_file_button_requests_a_client_file() -> None: + _state, loop = _file_picker_loop() + imgui = SimpleNamespace( + ImVec2=lambda x, y: (x, y), + Cond_=SimpleNamespace(once="once"), + set_next_window_pos=Mock(), + set_next_window_size=Mock(), + begin=Mock(), + end=Mock(), + text=Mock(), + button=Mock(return_value=True), + ) + + loop.step_ui(imgui, 0, UserInputEvents([])) + requests = loop.flush_ui_loop_requests() + + assert requests is not None + assert len(requests.file_selections) == 1 + assert requests.file_selections[0].request_id + + +def test_selected_files_event_updates_file_picker_status() -> None: + state, loop = _file_picker_loop() + imgui = SimpleNamespace( + ImVec2=lambda x, y: (x, y), + Cond_=SimpleNamespace(once="once"), + set_next_window_pos=Mock(), + set_next_window_size=Mock(), + begin=Mock(), + end=Mock(), + text=Mock(), + button=Mock(return_value=False), + ) + + loop.step_ui( + imgui, + 0, + UserInputEvents( + [ + SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id="open-1", + files=(SelectedFile(name="seed.png", path="seed.png", data=b"xx"),), + ) + ] + ), + ) + + assert state.status == "seed.png (2 bytes)" + imgui.text.assert_called_with("seed.png (2 bytes)") diff --git a/integrations_v2/imgui_ui_demo/pyproject.toml b/integrations_v2/imgui_ui_demo/pyproject.toml index 5b1371922..3c1a73e25 100644 --- a/integrations_v2/imgui_ui_demo/pyproject.toml +++ b/integrations_v2/imgui_ui_demo/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ ] [project.entry-points."flashdreams.applications_v2"] +"imgui-ui-file-picker" = "imgui_ui_demo.file_picker_app:create_app" "imgui-ui-query-string" = "imgui_ui_demo.query_string_app:create_app" "imgui-ui-text-input" = "imgui_ui_demo.text_input_app:create_app" "imgui-ui-window-size" = "imgui_ui_demo.window_size_app:create_app" From fba1dc259991a8932814f03b6330b2d39dd8c02a Mon Sep 17 00:00:00 2001 From: Set Paing Date: Wed, 30 Sep 2026 14:31:56 -0700 Subject: [PATCH 02/43] Ignore duplicate in-flight file-selection ids instead of aborting the session A reused request id (for example a second Open click while a picker is still open) should keep the original pick so the chosen file can still arrive. Signed-off-by: Set Paing --- flashdreams/flashdreams/api_v2/loop.py | 14 +++++++++++--- .../runtime_v2/native_window_client_window.py | 9 +++++---- .../flashdreams/runtime_v2/webrtc_client_window.py | 12 ++++++++---- flashdreams/test_v2/test_selected_files.py | 6 ++---- flashdreams/test_v2/test_webrtc_client_window.py | 3 +-- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 916b32f5a..2503615c7 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -5,6 +5,7 @@ from __future__ import annotations +import logging import queue import threading import time @@ -31,6 +32,8 @@ StateT = TypeVar("StateT") +_LOGGER = logging.getLogger(__name__) + class ModelInferenceState(Enum): """Lifecycle state of model inference observed by the UI loop.""" @@ -489,6 +492,8 @@ def request_selected_files( The window reports the result later as :class:`~flashdreams.runtime_v2.user_input_event.SelectedFilesUserInputEvent`. + A second call with the same ``request_id`` while the first is still + queued is ignored so the original picker can still complete. Args: request_id: Correlation token for the later input event. @@ -497,8 +502,7 @@ def request_selected_files( Raises: TypeError: ``request_id`` or ``initial_path`` is not a string. - ValueError: ``request_id`` is empty, ``initial_path`` is empty, or - ``request_id`` is already queued in this UI step. + ValueError: ``request_id`` is empty or ``initial_path`` is empty. """ if not isinstance(request_id, str): raise TypeError("request_id must be a string.") @@ -516,7 +520,11 @@ def request_selected_files( if any( selection.request_id == request_id for selection in requests.file_selections ): - raise ValueError(f"Duplicate file-selection request id {request_id!r}.") + _LOGGER.warning( + "Ignoring duplicate file-selection request id %r.", + request_id, + ) + return requests.file_selections.append( FileSelectionRequest( request_id=request_id, diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index b83e37108..dd01d56a0 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -243,12 +243,13 @@ def request_selected_files( request_id: Correlation token for the later input event. initial_path: Directory the selector should start in; ``None`` uses the current user's home directory. - - Raises: - ValueError: ``request_id`` is already waiting for a selector. """ if request_id in self._in_flight_file_requests: - raise ValueError(f"Duplicate file-selection request id {request_id!r}.") + _LOGGER.warning( + "Ignoring duplicate file-selection request id %r.", + request_id, + ) + return self._in_flight_file_requests.add(request_id) try: selected = _ask_open_filename( diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 320df157c..cf6c7cbc5 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -3,6 +3,7 @@ """WebRTC client window for the v2 runtime.""" +import logging import threading from collections import deque from dataclasses import replace @@ -19,6 +20,8 @@ ) from flashdreams.runtime_v2.user_input_events import UserInputEvents +_LOGGER = logging.getLogger(__name__) + class WebRTCClientWindow(IClientWindow): """Client window streaming a run to a browser. @@ -115,12 +118,13 @@ def request_selected_files( request_id: Correlation token for the later input event. initial_path: Directory hint for the selector; browsers may ignore it. ``None`` uses the current user's home directory. - - Raises: - ValueError: ``request_id`` is already waiting for a selector. """ if request_id in self._in_flight_file_requests: - raise ValueError(f"Duplicate file-selection request id {request_id!r}.") + _LOGGER.warning( + "Ignoring duplicate file-selection request id %r.", + request_id, + ) + return self._in_flight_file_requests.add(request_id) self.server.request_selected_files(request_id, initial_path=initial_path) diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index 2e156b2de..05186d6a1 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -60,12 +60,10 @@ def test_selected_files_event_carries_bytes_and_request_id() -> None: assert event.files[0].data == b"png" -def test_ui_loop_queues_file_selection_and_rejects_duplicate_ids() -> None: +def test_ui_loop_ignores_duplicate_file_selection_ids() -> None: loop = _ui_loop() loop.request_selected_files("open-1", "/tmp") - - with pytest.raises(ValueError, match="Duplicate file-selection request id"): - loop.request_selected_files("open-1", "/var") + loop.request_selected_files("open-1", "/var") requests = loop.flush_ui_loop_requests() assert requests is not None diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 812f4875e..3a3c478db 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -148,8 +148,7 @@ async def test_browser_file_upload_reaches_the_input_stream() -> None: window.open(_session_desc()) try: window.request_selected_files("open-1", "/tmp") - with pytest.raises(ValueError, match="Duplicate file-selection request id"): - window.request_selected_files("open-1", "/tmp") + window.request_selected_files("open-1", "/tmp") form = FormData() form.add_field("request_id", "open-1") From 07a23a26a6348f649eaf9dc27ad41d73b2dac76f Mon Sep 17 00:00:00 2001 From: Set Paing Date: Wed, 30 Sep 2026 15:16:45 -0700 Subject: [PATCH 03/43] Fix CPU ruff isort and leftover review nits CI's ruff --select I hook reordered numpy after the imgui_ui_demo imports. Fold that in with the stale StepResult stub and the file-selector README line. Signed-off-by: Set Paing --- flashdreams/flashdreams/runtime_v2/README.md | 6 +++--- flashdreams/test_v2/test_selected_files.py | 4 ++-- .../imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 267129740..ad400a895 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -246,9 +246,9 @@ delegates over the thing that does the work — `Mp4ClientWindow` over the output shape, because the session already did. They differ in what they can do rather than in how they are driven. A session -chooses its UI independently of the client window. Browser keyboard, mouse and -focus events arrive on the server's own thread, so it queues them and hands them -over in batches when the session asks. +chooses its UI independently of the client window. Browser keyboard, mouse, +focus, and file-selector events arrive on the server's own thread, so it queues +them and hands them over in batches when the session asks. A UI loop may call `request_new_session(session_desc)` during its step with a fully resolved replacement description. `run_session` stops and cleans the diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index 05186d6a1..a29f879ea 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -25,9 +25,9 @@ class _UILoop(IUILoop[None]): - def step(self, step_index: int, events: UserInputEvents) -> StepResult | None: + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: del step_index, events - return None + return [] def reset(self) -> None: return diff --git a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py index 9763cda37..b5dbff118 100644 --- a/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py +++ b/integrations_v2/imgui_ui_demo/imgui_ui_demo/tests/test_imgui_ui_demo.py @@ -9,9 +9,9 @@ from unittest.mock import Mock import pytest -from numpy import uint64 from imgui_ui_demo.file_picker_app import FilePickerImGuiUILoop, FilePickerState from imgui_ui_demo.text_input_app import TextInputImGuiUILoop, TextInputState +from numpy import uint64 from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.session_desc import SessionDesc From 9ee2447f68e6b1fdb3b00bb999f6b1b57aab1b9b Mon Sep 17 00:00:00 2001 From: Set Paing Date: Wed, 30 Sep 2026 16:40:45 -0700 Subject: [PATCH 04/43] Harden client file selection against picker lifecycle and untrusted uploads Drop SelectedFile.path so apps only see name and bytes. WebRTC admits uploads for pending ids, completes cancel and failure, and opens the picker from a viewer click. Native picks off the UI thread with the same size cap. Signed-off-by: Set Paing --- flashdreams/flashdreams/api_v2/README.md | 5 +- .../runtime_v2/native_window_client_window.py | 69 ++++++++---- .../flashdreams/runtime_v2/serving/web/app.js | 59 ++++++++++- .../runtime_v2/serving/web/index.html | 20 ++++ .../runtime_v2/serving/webrtc_server.py | 84 +++++++++++---- .../runtime_v2/user_input_event.py | 14 +-- .../runtime_v2/webrtc_client_window.py | 5 +- .../test_native_window_client_window.py | 25 ++++- flashdreams/test_v2/test_selected_files.py | 2 +- .../test_v2/test_webrtc_client_window.py | 100 ++++++++++++++++++ integrations_v2/imgui_ui_demo/README.md | 5 +- .../imgui_ui_demo/tests/test_imgui_ui_demo.py | 2 +- 12 files changed, 326 insertions(+), 64 deletions(-) diff --git a/flashdreams/flashdreams/api_v2/README.md b/flashdreams/flashdreams/api_v2/README.md index f8b8a91de..08b26231a 100644 --- a/flashdreams/flashdreams/api_v2/README.md +++ b/flashdreams/flashdreams/api_v2/README.md @@ -213,8 +213,9 @@ description; the runtime cleans the current session and passes that description to `ApplicationRunner` unchanged. A control that needs a file from the client calls `request_selected_files(request_id)`; the window later reports `SelectedFilesUserInputEvent` through the same input stream as keyboard and -mouse. The application reads file bytes from that event and does not open a -host path. +mouse. The application reads ``name`` and ``data`` from that event. An empty +``files`` tuple is a cancelled or unavailable pick (including MP4 and files +that exceed the window size cap). For SlangPy's smaller retained widget API, subclass `SlangPyUILoop` from `flashdreams.runtime_v2.slangpy_ui_loop`. The [`slangpy_ui_demo` integration](../../../integrations_v2/slangpy_ui_demo/README.md) diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index dd01d56a0..957bee19c 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -28,6 +28,7 @@ from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( + MAX_SELECTED_FILE_BYTES, CloseUserInputEvent, GamepadUserInputEvent, KeyboardInputState, @@ -107,6 +108,19 @@ def _ask_open_filename_tkinter(*, initial_dir: str) -> str: return str(selected or "") +def _selected_file_from_path(path: Path) -> SelectedFile | None: + """Load one picked file, or ``None`` when it cannot be used.""" + try: + if path.stat().st_size > MAX_SELECTED_FILE_BYTES: + return None + data = path.read_bytes() + except OSError: + return None + if len(data) > MAX_SELECTED_FILE_BYTES: + return None + return SelectedFile(name=path.name, data=data) + + _PRINTABLE_KEY_NAMES = { "space": " ", "apostrophe": "'", @@ -196,6 +210,7 @@ def __init__( self._hide_cursor = False self._lock_cursor_to_window = False self._in_flight_file_requests: set[str] = set() + self._file_request_lock = threading.Lock() def request_hide_cursor(self, hide_cursor: bool) -> None: """Show or hide the cursor in the native window.""" @@ -237,41 +252,55 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: def request_selected_files( self, request_id: str, initial_path: str | None = None ) -> None: - """Open a native OS file selector and enqueue the chosen file. + """Ask the OS file selector and enqueue the chosen file. + + The desktop dialog runs on a worker thread so the UI loop can keep + polling and presenting while the picker is open. Args: request_id: Correlation token for the later input event. initial_path: Directory the selector should start in; ``None`` uses the current user's home directory. """ - if request_id in self._in_flight_file_requests: - _LOGGER.warning( - "Ignoring duplicate file-selection request id %r.", - request_id, - ) - return - self._in_flight_file_requests.add(request_id) + with self._file_request_lock: + if request_id in self._in_flight_file_requests: + _LOGGER.warning( + "Ignoring duplicate file-selection request id %r.", + request_id, + ) + return + self._in_flight_file_requests.add(request_id) + worker = threading.Thread( + target=self._complete_file_selection, + args=(request_id, initial_path or str(Path.home())), + name="flashdreams-file-picker", + daemon=True, + ) + worker.start() + + def _complete_file_selection(self, request_id: str, initial_dir: str) -> None: + """Run the OS picker and enqueue a selected-files event.""" try: - selected = _ask_open_filename( - initial_dir=initial_path or str(Path.home()), - ) + selected = _ask_open_filename(initial_dir=initial_dir) files: tuple[SelectedFile, ...] = () if selected: - path = Path(selected) - try: - data = path.read_bytes() - except OSError: - data = b"" - files = (SelectedFile(name=path.name, path=str(path), data=data),) - self._put_input( + chosen = _selected_file_from_path(Path(selected)) + if chosen is not None: + files = (chosen,) + started_ns = self._session_started_ns + elapsed_ns = ( + 0 if started_ns is None else max(0, self._clock_ns() - started_ns) + ) + self._input_events.put( SelectedFilesUserInputEvent( - timestamp=uint64(0), + timestamp=uint64(elapsed_ns // 1_000), request_id=request_id, files=files, ) ) finally: - self._in_flight_file_requests.discard(request_id) + with self._file_request_lock: + self._in_flight_file_requests.discard(request_id) def open(self, session_desc: SessionDesc) -> None: """Create the GLFW window on the runtime's UI thread. diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 998b403a1..ddfc4acf6 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -7,6 +7,7 @@ const pointerControls = peer.createDataChannel("pointer-controls"); peer.addTransceiver("video", {direction: "recvonly"}); const video = document.getElementById("video"); const status = document.getElementById("status"); +const filePick = document.getElementById("file-pick"); const pressedKeys = new Map(); const pressedButtons = new Set(); const gamepadSnapshots = new Map(); @@ -74,28 +75,76 @@ const applyCursorOptions = options => { } }; +let pendingFileRequestId = null; + +const hideFilePicker = () => { + pendingFileRequestId = null; + if (filePick !== null) { + filePick.hidden = true; + } +}; + +const cancelFileSelection = requestId => { + send({type: "file_selector_result", id: requestId, cancelled: true}); +}; + const openFileSelector = options => { const requestId = options?.id; + if (typeof requestId !== "string" || !requestId || filePick === null) { + return; + } + if (pendingFileRequestId !== null && pendingFileRequestId !== requestId) { + cancelFileSelection(pendingFileRequestId); + } + pendingFileRequestId = requestId; + filePick.hidden = false; +}; + +filePick?.addEventListener("click", () => { + const requestId = pendingFileRequestId; if (typeof requestId !== "string" || !requestId) { return; } const input = document.createElement("input"); input.type = "file"; + let settled = false; + const finishCancelled = () => { + if (settled || pendingFileRequestId !== requestId) { + return; + } + settled = true; + cancelFileSelection(requestId); + hideFilePicker(); + }; input.addEventListener("change", () => { const file = input.files?.[0]; if (!file) { - send({type: "file_selector_result", id: requestId, cancelled: true}); + finishCancelled(); return; } const body = new FormData(); body.append("request_id", requestId); body.append("file", file, file.name); - fetch("/api/files", {method: "POST", body}).catch(error => { - console.debug("Unable to upload the selected file.", error); - }); + fetch("/api/files", {method: "POST", body}) + .then(response => { + if (settled || pendingFileRequestId !== requestId) { + return; + } + if (!response.ok) { + finishCancelled(); + return; + } + settled = true; + hideFilePicker(); + }) + .catch(error => { + console.debug("Unable to upload the selected file.", error); + finishCancelled(); + }); }); + input.addEventListener("cancel", finishCancelled); input.click(); -}; +}); window.addEventListener("focus", updateCursorVisibility); window.addEventListener("blur", updateCursorVisibility); diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/index.html b/flashdreams/flashdreams/runtime_v2/serving/web/index.html index 1b73e1dc8..c5d453f17 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/index.html +++ b/flashdreams/flashdreams/runtime_v2/serving/web/index.html @@ -48,11 +48,31 @@ color: #ff8a8a; } + #file-pick { + position: fixed; + left: 50%; + bottom: 1.5rem; + transform: translateX(-50%); + z-index: 2; + padding: 0.6rem 1.1rem; + border: 0; + border-radius: 0.4rem; + background: #eee; + color: #111; + font: inherit; + cursor: pointer; + } + + #file-pick[hidden] { + display: none; + } +
Waiting for the server…
+ diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index fd23e24e4..ceb6eb5d2 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -16,7 +16,6 @@ from dataclasses import dataclass from fractions import Fraction from importlib.resources import files -from pathlib import Path from typing import Any, Literal, cast import numpy as np @@ -33,6 +32,7 @@ from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( + MAX_SELECTED_FILE_BYTES, CloseUserInputEvent, FocusUserInputEvent, GamepadUserInputEvent, @@ -57,9 +57,6 @@ _INTERACTIVE_FRAME_QUEUE_SIZE = 2 """Send-ready frames retained while the media sender is temporarily busy.""" -_MAX_SELECTED_FILE_BYTES = 32 * 1024 * 1024 -"""Maximum file the browser may upload for one selected-files event.""" - _TRANSFER_STREAM_PRIORITY = -1 """Portable high-priority CUDA stream request for interactive output copies.""" @@ -354,6 +351,8 @@ def __init__( self._hide_cursor = False self._lock_cursor_to_window = False self._queued_file_selectors: list[dict[str, str]] = [] + self._pending_file_request_ids: set[str] = set() + self._file_selector_lock = threading.Lock() self._thread = threading.Thread( target=self._run_server, name="flashdreams-webrtc", @@ -451,26 +450,54 @@ def request_selected_files( Args: request_id: Correlation token for the later input event. - initial_path: Directory hint for the selector; ``None`` uses the - current user's home directory. + initial_path: Ignored; the browser picker cannot use a host path. """ + del initial_path if not isinstance(request_id, str) or not request_id: raise ValueError("request_id must be a non-empty string.") - if initial_path is None: - resolved_initial_path = str(Path.home()) - elif not isinstance(initial_path, str) or not initial_path: - raise ValueError("initial_path must be a non-empty string.") - else: - resolved_initial_path = initial_path - payload = { - "type": "file_selector", - "id": request_id, - "initial_path": resolved_initial_path, - } + payload = {"type": "file_selector", "id": request_id} loop = self._loop if loop is None: raise RuntimeError("WebRTC server is not running.") - loop.call_soon_threadsafe(self._send_file_selector, payload) + loop.call_soon_threadsafe(self._arm_file_selector, payload) + + def _arm_file_selector(self, payload: dict[str, str]) -> None: + """Record a pending id and send or queue the selector request.""" + with self._file_selector_lock: + self._pending_file_request_ids.add(payload["id"]) + self._send_file_selector(payload) + + def _take_pending_file_request(self, request_id: str) -> bool: + """Return whether ``request_id`` was pending and consume it.""" + with self._file_selector_lock: + if request_id not in self._pending_file_request_ids: + return False + self._pending_file_request_ids.discard(request_id) + return True + + def _invalidate_file_selectors(self) -> None: + """Drop queued and pending selector requests without completing them.""" + with self._file_selector_lock: + self._pending_file_request_ids.clear() + self._queued_file_selectors.clear() + + def _complete_abandoned_file_selectors(self) -> None: + """Complete pending selector requests with empty files.""" + timestamp_us = self._timestamp_us() + with self._file_selector_lock: + pending = tuple(self._pending_file_request_ids) + self._pending_file_request_ids.clear() + self._queued_file_selectors.clear() + if timestamp_us is None: + return + for request_id in pending: + self._append_event( + SelectedFilesUserInputEvent( + timestamp=timestamp_us, + request_id=request_id, + files=(), + ) + ) def _send_file_selector(self, payload: dict[str, str]) -> None: """Send one file-selector request on the WebRTC event-loop thread.""" @@ -549,6 +576,7 @@ def open(self, session_desc: SessionDesc) -> None: # Keep timestamps comparable across sessions so events buffered # during a handoff retain their real order. self._event_origin_ns = time.monotonic_ns() + self._invalidate_file_selectors() def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the dimensions used to present subsequent UI frames. @@ -771,7 +799,7 @@ def _run_server(self) -> None: async def _start_server(self) -> None: """Create and bind the standalone aiohttp application.""" self._offer_lock = asyncio.Lock() - app = web.Application(client_max_size=_MAX_SELECTED_FILE_BYTES + 1024 * 1024) + app = web.Application(client_max_size=MAX_SELECTED_FILE_BYTES + 1024 * 1024) app.router.add_get("/", self._serve_browser) app.router.add_get("/app.js", self._serve_browser_script) app.router.add_get("/healthz", self._health) @@ -828,10 +856,19 @@ async def _upload_selected_file(self, request: web.Request) -> web.Response: upload = post.get("file") if not isinstance(upload, FileField): raise web.HTTPBadRequest(reason="File upload requires a file.") + if not self._take_pending_file_request(request_id): + raise web.HTTPConflict(reason="No pending file selection for this request_id.") data = upload.file.read() - if len(data) > _MAX_SELECTED_FILE_BYTES: + if len(data) > MAX_SELECTED_FILE_BYTES: + self._append_event( + SelectedFilesUserInputEvent( + timestamp=timestamp_us, + request_id=request_id, + files=(), + ) + ) raise web.HTTPRequestEntityTooLarge( - max_size=_MAX_SELECTED_FILE_BYTES, + max_size=MAX_SELECTED_FILE_BYTES, actual_size=len(data), ) name = upload.filename or "upload" @@ -839,7 +876,7 @@ async def _upload_selected_file(self, request: web.Request) -> web.Response: SelectedFilesUserInputEvent( timestamp=timestamp_us, request_id=request_id, - files=(SelectedFile(name=name, path=name, data=data),), + files=(SelectedFile(name=name, data=data),), ) ) return web.Response(status=204) @@ -1174,6 +1211,8 @@ def _buffer_browser_message( "File selector result must set cancelled true; send files " "with POST /api/files." ) + if not self._take_pending_file_request(request_id): + return event = SelectedFilesUserInputEvent( timestamp=timestamp_us, request_id=request_id, @@ -1208,6 +1247,7 @@ async def _release_peer_connection( self._control_channel = None self._video_track = None self._sent_cursor_options = None + self._complete_abandoned_file_selectors() else: track = detached_video_track failures: list[BaseException] = [] diff --git a/flashdreams/flashdreams/runtime_v2/user_input_event.py b/flashdreams/flashdreams/runtime_v2/user_input_event.py index 13674d5c5..e820d1a13 100644 --- a/flashdreams/flashdreams/runtime_v2/user_input_event.py +++ b/flashdreams/flashdreams/runtime_v2/user_input_event.py @@ -9,6 +9,9 @@ from flashdreams.api_v2.user_input_event import UserInputEvent +MAX_SELECTED_FILE_BYTES = 32 * 1024 * 1024 +"""Maximum bytes one selected file may carry into the application.""" + class KeyboardInputState(Enum): """State transition reported by a keyboard input event.""" @@ -235,21 +238,18 @@ class SelectedFile: """One file chosen by a client file selector.""" name: str - """File name shown to the application, without requiring a host path.""" - - path: str - """Native host path, or the file name when the client has no path.""" + """File name shown to the application.""" data: bytes - """File contents. Applications should use this rather than opening ``path``.""" + """File contents.""" @dataclass(frozen=True, slots=True, eq=False) class SelectedFilesUserInputEvent(UserInputEvent): """Files chosen for one :meth:`IClientWindow.request_selected_files` call. - An empty ``files`` tuple is a cancelled or unavailable selector, including - ``--mode mp4`` where no client can pick a file. + An empty ``files`` tuple is a cancelled or unavailable selector: dismiss, + oversize, unreadable, a dropped WebRTC peer, or ``--mode mp4``. """ @classmethod diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index cf6c7cbc5..ce399fdbc 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -116,8 +116,8 @@ def request_selected_files( Args: request_id: Correlation token for the later input event. - initial_path: Directory hint for the selector; browsers may ignore - it. ``None`` uses the current user's home directory. + initial_path: Ignored by the browser picker; kept so the window + signature matches :meth:`IClientWindow.request_selected_files`. """ if request_id in self._in_flight_file_requests: _LOGGER.warning( @@ -138,6 +138,7 @@ def open(self, session_desc: SessionDesc) -> None: session_event_offset_us = self.server.event_timestamp_us() with self._input_lock: self._input_events.clear() + self._in_flight_file_requests.clear() self._session_event_offset_us = session_event_offset_us def get_user_input_events(self) -> UserInputEvents: diff --git a/flashdreams/test_v2/test_native_window_client_window.py b/flashdreams/test_v2/test_native_window_client_window.py index c039ed928..d6be159c9 100644 --- a/flashdreams/test_v2/test_native_window_client_window.py +++ b/flashdreams/test_v2/test_native_window_client_window.py @@ -7,6 +7,7 @@ import queue import threading +import time from collections.abc import Callable, Sequence from pathlib import Path from types import SimpleNamespace @@ -39,6 +40,15 @@ pytestmark = pytest.mark.ci_cpu +def _wait_for_input_events(window: NativeWindowClientWindow) -> list[object]: + for _ in range(100): + events = window.get_user_input_events().get_events() + if events: + return list(events) + time.sleep(0.01) + return [] + + def _session_desc() -> SessionDesc: return SessionDesc( output_layout=VideoTensorLayout.tchw, @@ -603,7 +613,7 @@ def test_native_window_reports_a_selected_file( ) window.open(_session_desc()) window.request_selected_files("open-1", str(tmp_path)) - events = window.get_user_input_events().get_events() + events = _wait_for_input_events(window) window.close() assert len(events) == 1 @@ -611,10 +621,21 @@ def test_native_window_reports_a_selected_file( assert isinstance(event, SelectedFilesUserInputEvent) assert event.request_id == "open-1" assert event.files[0].name == "seed.png" - assert event.files[0].path == str(chosen) assert event.files[0].data == b"png-bytes" +def test_selected_file_from_path_skips_unreadable_and_oversize( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + missing = tmp_path / "gone.png" + assert native_window_module._selected_file_from_path(missing) is None + + too_big = tmp_path / "big.bin" + too_big.write_bytes(b"abcd") + monkeypatch.setattr(native_window_module, "MAX_SELECTED_FILE_BYTES", 3) + assert native_window_module._selected_file_from_path(too_big) is None + + def test_ask_open_filename_linux_uses_zenity( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/flashdreams/test_v2/test_selected_files.py b/flashdreams/test_v2/test_selected_files.py index a29f879ea..b169f34d3 100644 --- a/flashdreams/test_v2/test_selected_files.py +++ b/flashdreams/test_v2/test_selected_files.py @@ -52,7 +52,7 @@ def test_selected_files_event_carries_bytes_and_request_id() -> None: event = SelectedFilesUserInputEvent( timestamp=uint64(1), request_id="open-1", - files=(SelectedFile(name="seed.png", path="/tmp/seed.png", data=b"png"),), + files=(SelectedFile(name="seed.png", data=b"png"),), ) assert event.get_type_name() == "selected_files" diff --git a/flashdreams/test_v2/test_webrtc_client_window.py b/flashdreams/test_v2/test_webrtc_client_window.py index 3a3c478db..17b2841b1 100644 --- a/flashdreams/test_v2/test_webrtc_client_window.py +++ b/flashdreams/test_v2/test_webrtc_client_window.py @@ -149,6 +149,12 @@ async def test_browser_file_upload_reaches_the_input_stream() -> None: try: window.request_selected_files("open-1", "/tmp") window.request_selected_files("open-1", "/tmp") + for _ in range(100): + if "open-1" in window.server._pending_file_request_ids: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("file selection was not armed") form = FormData() form.add_field("request_id", "open-1") @@ -182,6 +188,96 @@ async def test_browser_file_upload_reaches_the_input_stream() -> None: window.close() +@pytest.mark.asyncio +async def test_unsolicited_file_upload_is_rejected() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + form = FormData() + form.add_field("request_id", "open-1") + form.add_field( + "file", + b"hello", + filename="hello.txt", + content_type="text/plain", + ) + async with ClientSession() as client: + async with client.post( + f"{window.server.url}api/files", data=form + ) as response: + assert response.status == 409 + assert window.get_user_input_events().get_events() == [] + finally: + window.close() + + +@pytest.mark.asyncio +async def test_browser_file_selector_omits_host_path() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + peer: RTCPeerConnection | None = None + try: + peer, channel, _ = await _connect_browser(window) + messages: list[dict[str, object]] = [] + + @channel.on("message") + def on_message(message: object) -> None: + if isinstance(message, str): + messages.append(json.loads(message)) + + window.request_selected_files("open-1", "/home/secret") + for _ in range(100): + if any(item.get("type") == "file_selector" for item in messages): + break + await asyncio.sleep(0.01) + payload = next( + item for item in messages if item.get("type") == "file_selector" + ) + assert payload == {"type": "file_selector", "id": "open-1"} + finally: + if peer is not None: + await peer.close() + window.close() + + +@pytest.mark.asyncio +async def test_session_handoff_clears_pending_file_selection() -> None: + window = WebRTCClientWindow() + window.open(_session_desc()) + try: + window.request_selected_files("open-1", "/tmp") + for _ in range(100): + if "open-1" in window.server._pending_file_request_ids: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("file selection was not armed") + window.open(_session_desc()) + assert "open-1" not in window.server._pending_file_request_ids + window.request_selected_files("open-1", "/tmp") + for _ in range(100): + if "open-1" in window.server._pending_file_request_ids: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("file selection was not re-armed") + form = FormData() + form.add_field("request_id", "open-1") + form.add_field( + "file", + b"hello", + filename="hello.txt", + content_type="text/plain", + ) + async with ClientSession() as client: + async with client.post( + f"{window.server.url}api/files", data=form + ) as response: + assert response.status == 204 + finally: + window.close() + + @pytest.mark.asyncio async def test_window_buffers_browser_events_until_drained() -> None: window = WebRTCClientWindow() @@ -220,6 +316,7 @@ async def test_window_buffers_browser_events_until_drained() -> None: assert 'id="reset"' not in browser_page assert '