diff --git a/apps/action2v/tests/test_application.py b/apps/action2v/tests/test_application.py index c767d5950..fbfc71bb7 100644 --- a/apps/action2v/tests/test_application.py +++ b/apps/action2v/tests/test_application.py @@ -22,6 +22,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.session_runner import run_session from flashdreams.runtime_v2.step_result import StepResult @@ -138,6 +139,9 @@ def __init__(self) -> None: self.opened = False self.cursor_requests: list[tuple[str, bool]] = [] + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc self.opened = True diff --git a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py index 78c8668c0..8e8aed730 100644 --- a/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py +++ b/apps/handdreams/controller_input/flashdreams_controller_input/controller_source_v2_window.py @@ -26,6 +26,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -243,6 +244,10 @@ def open(self, session_desc: SessionDesc) -> None: self._disconnect_emitted = False self._last_status = None + def put_user_input_event(self, event: UserInputEvent) -> None: + """Forward event to the presentation backend, merged with its input.""" + self.output_window.put_user_input_event(event) + def get_user_input_events(self) -> UserInputEvents: """Merge source-clock controller history with buffered window input.""" # The source owns XR input. Operator B only signals its headset worker; diff --git a/apps/t2v/t2v/testing.py b/apps/t2v/t2v/testing.py index 648639a68..ca78800a4 100644 --- a/apps/t2v/t2v/testing.py +++ b/apps/t2v/t2v/testing.py @@ -21,6 +21,7 @@ import torch from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.blit_model_output_to_screen_loop import ( BlitModelOutputToScreenLoop, ) @@ -410,6 +411,9 @@ class _DiscardingClientWindow(IClientWindow): def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: del session_desc diff --git a/apps/t2v/tests/test_cli.py b/apps/t2v/tests/test_cli.py index 8521ddeea..62e2261f7 100644 --- a/apps/t2v/tests/test_cli.py +++ b/apps/t2v/tests/test_cli.py @@ -27,6 +27,7 @@ from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2 import cli from flashdreams.runtime_v2.application_registry import ( APPLICATION_ENTRY_POINT_GROUP, @@ -160,6 +161,9 @@ def __init__(self) -> None: def get_user_input_events(self) -> UserInputEvents: return UserInputEvents([]) + def put_user_input_event(self, event: UserInputEvent) -> None: + del event + def open(self, session_desc: SessionDesc) -> None: self.session_desc = session_desc diff --git a/flashdreams/flashdreams/api_v2/README.md b/flashdreams/flashdreams/api_v2/README.md index 4bdad8a4b..f50ff82d2 100644 --- a/flashdreams/flashdreams/api_v2/README.md +++ b/flashdreams/flashdreams/api_v2/README.md @@ -245,7 +245,13 @@ UI control that needs a fresh application session calls `request_new_session(session_desc)` with a fully resolved replacement description; the runtime cleans the current session and passes that description to `ApplicationRunner` unchanged. Remaining timeout and remaining model steps -still apply to the replacement. +still apply to the replacement. To let the user pick a client file, call +`request_file_selector(request_id, ...)` when they ask (for ImGui, when +`imgui.button(...)` returns `True`), with one stable `request_id` per control. +The result arrives on a later tick as one `SelectedFilesUserInputEvent` +carrying that `request_id`; read its `status` before its `files`. One picker +is open at a time, and a call made while one is open is answered `BUSY`. The +`request_file_selector` docstring lists every status. For SlangPy's smaller retained widget API, subclass `SlangPyUILoop` from `flashdreams.runtime_v2.slangpy_ui_loop`. The diff --git a/flashdreams/flashdreams/api_v2/client_window.py b/flashdreams/flashdreams/api_v2/client_window.py index 25b55131c..840875df2 100644 --- a/flashdreams/flashdreams/api_v2/client_window.py +++ b/flashdreams/flashdreams/api_v2/client_window.py @@ -3,14 +3,29 @@ """Client window abstract interface.""" +import logging import math -from abc import ABC -from dataclasses import dataclass +from abc import ABC, abstractmethod +from dataclasses import dataclass, replace +from pathlib import Path +from typing import final from urllib.parse import urlsplit +from numpy import uint64 + +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + selected_files_policy_status, +) +from flashdreams.runtime_v2.user_input_event import SelectedFilesUserInputEvent + from .input_source import InputSource from .output_sink import OutputSink +_LOGGER = logging.getLogger(__name__) + @dataclass(frozen=True, slots=True) class ClientWindowLink: @@ -89,6 +104,10 @@ class IClientWindow(InputSource, OutputSink, ABC): One thread makes every call on a window, so an implementation needs no locking except when its backend delivers input from another thread. + A window with a file picker implements :meth:`_request_file_selector`, and + :meth:`_cancel_active_file_selectors` if it can withdraw a shown picker. It + reports results on the UI thread. + Created by the runtime, never by an application. """ @@ -117,3 +136,117 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: new_window_size: Requested ``(width, height)`` in pixels. """ pass + + # File Selector Methods & Members + _active_file_selector: tuple[int, FileSelectionRequest] | None = None + _last_file_selector_id = 0 + + @final + def request_file_selector(self, request: FileSelectionRequest) -> None: + """Request for IClientWindow to open a file selector; the result of the + user interaction arrives later as a ``SelectedFilesUserInputEvent`` via + :meth:`InputSource.get_user_input_events`. + + A request made while a picker is open is answered with a status of + ``SelectedFilesStatus.BUSY``. + + Args: + request: Selector request queued by the UI loop. + """ + if self._active_file_selector is not None: + self.put_user_input_event( + _selected_files_event(request, SelectedFilesStatus.BUSY) + ) + return + selector_id = self._last_file_selector_id + 1 + self._last_file_selector_id = selector_id + self._active_file_selector = (selector_id, request) + try: + self._request_file_selector(selector_id, request) + except Exception: + _LOGGER.exception("File selector %r failed to start.", request.request_id) + self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + + @final + def flush_file_selectors(self) -> None: + """Forget the open picker without a result, for a session that ended. + + The window withdraws a picker still on screen where it can; whatever + that picker returns is dropped. + """ + active = self._active_file_selector + self._active_file_selector = None + if active is not None: + self._cancel_active_file_selectors() + + # Optional to implement + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Show a picker for ``request`` without blocking. + + Call :meth:`_complete_file_selector` with ``selector_id`` on the UI + thread when the user finishes. The default has no picker and completes + as ``UNAVAILABLE``. + """ + del request + self._complete_file_selector(selector_id, SelectedFilesStatus.UNAVAILABLE) + + # Optional to implement + def _cancel_active_file_selectors(self) -> None: + """Withdraw the picker that was flushed.""" + pass + + @final + def _complete_file_selector( + self, + selector_id: int, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), + ) -> None: + """Report the result for ``selector_id``; a stale id is dropped. + + A result that breaks the request's rules (type, size, count) is + reported by the status it earns instead. + """ + active = self._active_file_selector + if active is None or active[0] != selector_id: + return + self._active_file_selector = None + request = active[1] + status, files = _check_file_selection_result(request, status, files) + self.put_user_input_event(_selected_files_event(request, status, files)) + + +def _selected_files_event( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...] = (), +) -> SelectedFilesUserInputEvent: + """Return the event answering ``request``.""" + return SelectedFilesUserInputEvent( + timestamp=uint64(0), + request_id=request.request_id, + status=status, + files=files, + ) + + +def _check_file_selection_result( + request: FileSelectionRequest, + status: SelectedFilesStatus, + files: tuple[SelectedFile, ...], +) -> tuple[SelectedFilesStatus, tuple[SelectedFile, ...]]: + """Return the status and files a window's result earns under ``request``.""" + if status is not SelectedFilesStatus.OK: + return status, () + if not files: + _LOGGER.warning("File selector %r returned no valid set.", request.request_id) + return SelectedFilesStatus.UNAVAILABLE, () + files = tuple(replace(chosen, name=Path(chosen.name).name) for chosen in files) + rejected = selected_files_policy_status( + [(chosen.name, len(chosen.data)) for chosen in files], request + ) + if rejected is not None: + return rejected, () + return status, files diff --git a/flashdreams/flashdreams/api_v2/input_source.py b/flashdreams/flashdreams/api_v2/input_source.py index d64a4df88..dcbf8a572 100644 --- a/flashdreams/flashdreams/api_v2/input_source.py +++ b/flashdreams/flashdreams/api_v2/input_source.py @@ -6,6 +6,7 @@ from abc import abstractmethod from typing import Protocol, runtime_checkable +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.user_input_events import UserInputEvents @@ -31,3 +32,13 @@ def get_user_input_events(self) -> UserInputEvents: Events in timestamp order, empty when nothing arrived. """ ... + + @abstractmethod + def put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event`` for :meth:`get_user_input_events`. + + Canonical method for how user-input-events are added to an input source. + Called on the UI thread, by ``IClientWindow`` and by windows that wrap + another window. + """ + ... diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 11078776b..325206b98 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -9,9 +9,10 @@ import threading import time from abc import ABC, abstractmethod -from collections.abc import Callable -from dataclasses import dataclass +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field from enum import Enum +from pathlib import Path from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar, final import torch @@ -25,6 +26,13 @@ bind_inference_profiler, unbind_inference_profiler, ) +from flashdreams.runtime_v2.selected_file import ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_FILE_COUNT, + DEFAULT_MAX_TOTAL_BYTES, + FileSelectionRequest, + normalize_selected_file_accept, +) from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( CloseUserInputEvent, @@ -101,6 +109,9 @@ class UILoopRequests: links: tuple[ClientWindowLink, ...] | None = None """Replacement clickable regions, or ``None`` to leave them unchanged.""" + file_selections: list[FileSelectionRequest] = field(default_factory=list) + """File-selector requests in call order. The window answers each one.""" + class ILoop(ABC, Generic[StateT]): """Shared state, messaging, and lifecycle for a session loop. @@ -257,6 +268,8 @@ def _finish_run( if step_completed: self.latest_result = result self._step_index += 1 + # Input read by a run with no step is dropped, such as an ON_DEMAND UI + # tick without a new frame. self._pending_user_events.clear() @final @@ -604,6 +617,116 @@ def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: raise ValueError("At most sixteen client-window links are supported.") self.get_or_create_ui_loop_requests().links = links + @final + def request_file_selector( + self, + request_id: str, + initial_path: str | None = None, + *, + accept: Sequence[str] = (), + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, + max_file_count: int = DEFAULT_MAX_FILE_COUNT, + ) -> None: + """Ask the client to show a file selector. + + The result arrives on a later tick, never during the step that asked, + as one + :class:`~flashdreams.runtime_v2.user_input_event.SelectedFilesUserInputEvent` + carrying ``request_id``. One selector is open at a time; a call made + while one is open is answered ``BUSY``. A selector stays open across a + session reset; a replaced or ended session gets no result. + + Read ``status`` first; ``files`` is empty unless it is ``OK``: + + - ``OK``: each chosen file as its base ``name`` and ``data`` bytes. + - ``CANCELLED``: the user dismissed the selector. + - ``BUSY``: another selector was open. + - ``DISALLOWED_TYPE``: a file did not match ``accept``. + - ``TOO_LARGE``: a file was larger than ``max_file_bytes``, the files + together were larger than ``max_total_bytes``, or more than + ``max_file_count`` files were chosen. + - ``UNAVAILABLE``: the window cannot show a selector, or it failed. + + ``accept`` checks filename suffixes only, so treat ``data`` as + untrusted input. + + Example: + + .. code-block:: python + + from flashdreams.runtime_v2.selected_file import SelectedFilesStatus + from flashdreams.runtime_v2.user_input_event import ( + SelectedFilesUserInputEvent, + ) + + if imgui.button("Open image"): + self.request_file_selector("open-image", accept=(".png",)) + for event in self.user_events.get_events(): + if not isinstance(event, SelectedFilesUserInputEvent): + continue + if event.request_id != "open-image": + continue + if event.status is SelectedFilesStatus.OK: + image_bytes = event.files[0].data + else: + self.state.message = f"No image: {event.status.value}" + + Args: + request_id: Names the control that asked, so its result can be + told apart from another control's. Echoed on the result. + initial_path: Directory hint for the native selector; ``None`` + uses the host user's home directory. The browser ignores it. + accept: Filename suffixes such as ``.png``. Empty allows any type. + max_file_bytes: Largest single file in bytes. Must not exceed + ``max_total_bytes``. + max_total_bytes: Most bytes all chosen files may add up to. Has no + effect when ``max_file_count`` is one. + max_file_count: Most files the selector may return. Above one, the + user may pick several. + + Raises: + TypeError: ``request_id`` or ``initial_path`` is not a string, or + ``accept`` or a limit has the wrong type. + ValueError: ``request_id`` or ``initial_path`` is empty, ``accept`` + is invalid, a limit is not positive, or ``max_file_bytes`` + exceeds ``max_total_bytes``. + """ + if not isinstance(request_id, str): + raise TypeError("request_id must be a string.") + if not request_id: + raise ValueError("request_id must be a non-empty string.") + if initial_path is None: + resolved_initial_path = str(Path.home()) + else: + if not isinstance(initial_path, str): + raise TypeError("initial_path must be a string.") + if not initial_path: + raise ValueError("initial_path must be a non-empty string.") + resolved_initial_path = initial_path + suffixes = normalize_selected_file_accept(accept) + for name, limit in ( + ("max_file_bytes", max_file_bytes), + ("max_total_bytes", max_total_bytes), + ("max_file_count", max_file_count), + ): + if isinstance(limit, bool) or not isinstance(limit, int): + raise TypeError(f"{name} must be an integer.") + if limit <= 0: + raise ValueError(f"{name} must be > 0.") + if max_file_bytes > max_total_bytes: + raise ValueError("max_file_bytes must not exceed max_total_bytes.") + self.get_or_create_ui_loop_requests().file_selections.append( + FileSelectionRequest( + request_id=request_id, + initial_path=resolved_initial_path, + accept=suffixes, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + max_file_count=max_file_count, + ) + ) + def get_or_create_ui_loop_requests(self) -> UILoopRequests: if self._ui_loop_requests is None: self._ui_loop_requests = UILoopRequests() @@ -699,6 +822,7 @@ def invoke_async(loop: ILoop[StateT], operation: Callable[[StateT], None]) -> No __all__ = [ + "FileSelectionRequest", "ILoop", "IModelLoop", "IUILoop", diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 6f2cd1398..ba4df4118 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -370,16 +370,22 @@ delegates over the thing that does the work — `Mp4ClientWindow` over the output shape, because the session already did. They differ in what they can do rather than in how they are driven. A session -chooses its UI independently of the client window. Browser keyboard, mouse and -focus events arrive on the server's own thread, so it queues them and hands them -over in batches when the session asks. +chooses its UI independently of the client window. Browser keyboard, mouse, +focus, and file-selector events arrive on the server's own thread, so it queues +them and hands them over in batches when the session asks. + +File selection rules live in `IClientWindow`; a window only shows a picker and +reports what it returned, on the UI thread. Every window implements +`InputSource.put_user_input_event`; a window with a picker also implements +`_request_file_selector`. A UI loop may call `request_new_session(session_desc)` during its step with a fully resolved replacement description. `run_session` stops and cleans the current session, leaves the interactive window open, and returns that description unchanged. `ApplicationRunner` creates the replacement from it. A WebRTC browser disconnect releases only its peer connection, so refreshing the -page does not stop the session, server, or application. +page does not stop the session, server, or application. Leftover and in-flight +file picks complete as `unavailable`; a replacement viewer does not inherit them. The UI thread owns WebRTC cadence. Each `write` synchronously materializes one owned video frame and admits it to a two-frame FIFO of unsent frames. The WebRTC diff --git a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py index 3e2c72ac6..300b926a3 100644 --- a/flashdreams/flashdreams/runtime_v2/mp4_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/mp4_client_window.py @@ -6,6 +6,7 @@ from pathlib import Path from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.mp4_output_sink import Mp4OutputSink from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -13,7 +14,7 @@ class Mp4ClientWindow(IClientWindow): - """Write UI frames to an MP4 file and report no input. + """Write UI frames to an MP4 file. It has no user input of its own. The session must finish on its own because this window never sends a close event. Use ``BackpressureMode.BLOCK`` with @@ -27,6 +28,7 @@ def __init__(self, path: str | Path) -> None: """ self._path = Path(path) self._video_sink = Mp4OutputSink(path) + self._input_events: list[UserInputEvent] = [] @property def path(self) -> Path: @@ -34,8 +36,12 @@ def path(self) -> Path: return self._path def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return the events queued since the last call.""" + events, self._input_events = self._input_events, [] + return UserInputEvents(events) + + def put_user_input_event(self, event: UserInputEvent) -> None: + self._input_events.append(event) def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: """Change the MP4 presentation dimensions without replacing the session. @@ -52,6 +58,7 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: def open(self, session_desc: SessionDesc) -> None: """Prepare to write a session's output.""" + self._input_events = [] self._video_sink.open(session_desc) def write(self, result: StepResult) -> None: diff --git a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py index e2a8c2af1..2ad057613 100644 --- a/flashdreams/flashdreams/runtime_v2/native_window_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/native_window_client_window.py @@ -7,12 +7,14 @@ import logging import queue +import stat import threading import time import webbrowser from collections import deque -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import replace +from pathlib import Path from typing import TYPE_CHECKING, Any, cast import numpy as np @@ -22,6 +24,12 @@ from flashdreams.api_v2.client_window import ClientWindowLink, IClientWindow from flashdreams.api_v2.user_input_event import UserInputEvent +from flashdreams.runtime_v2.selected_file import ( + FileSelectionRequest, + SelectedFile, + SelectedFilesStatus, + selected_files_policy_status, +) from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_event import ( @@ -40,6 +48,76 @@ _LOGGER = logging.getLogger(__name__) + +def _choose_paths(request: FileSelectionRequest) -> list[str]: + """Block in the Tk file dialog and return the chosen paths. + + Tk shows Explorer on Windows and its own dialog on Linux. Raises when Tk is + missing or cannot open a display. + """ + import tkinter as tk + from tkinter import filedialog + + root = tk.Tk() + try: + root.withdraw() + try: + root.attributes("-topmost", True) + except tk.TclError: + pass + kwargs: dict[str, Any] = { + "parent": root, + "initialdir": request.initial_path or str(Path.home()), + "title": "Select a file", + } + if request.accept: + kwargs["filetypes"] = [ + ("Accepted", " ".join("*" + suffix for suffix in request.accept)) + ] + if request.max_file_count > 1: + return [str(path) for path in filedialog.askopenfilenames(**kwargs) if path] + path = filedialog.askopenfilename(**kwargs) + return [str(path)] if path else [] + finally: + root.destroy() + + +def _selected_files_from_paths( + paths: Sequence[str], request: FileSelectionRequest +) -> tuple[tuple[SelectedFile, ...], SelectedFilesStatus]: + """Load chosen files, or empty files and a failure status.""" + rejected = selected_files_policy_status( + [(Path(path).name, None) for path in paths], request + ) + if rejected is not None: + return (), rejected + sized: list[tuple[str, int]] = [] + for path in paths: + try: + info = Path(path).stat() + except OSError: + return (), SelectedFilesStatus.UNAVAILABLE + if not stat.S_ISREG(info.st_mode): + return (), SelectedFilesStatus.UNAVAILABLE + sized.append((path, info.st_size)) + rejected = selected_files_policy_status( + [(Path(path).name, size) for path, size in sized], request + ) + if rejected is not None: + return (), rejected + files: list[SelectedFile] = [] + for path, _size in sized: + try: + with open(path, "rb") as stream: + # Read one byte past the limit so a file over it is + # rejected, not truncated. + data = stream.read(request.max_file_bytes + 1) + except OSError: + return (), SelectedFilesStatus.UNAVAILABLE + files.append(SelectedFile(name=Path(path).name, data=data)) + return tuple(files), SelectedFilesStatus.OK + + _PRINTABLE_KEY_NAMES = { "space": " ", "apostrophe": "'", @@ -143,6 +221,10 @@ def __init__( self._lock_cursor_to_window = False self._links: tuple[ClientWindowLink, ...] = () self._pressed_link: ClientWindowLink | None = None + self._file_dialog_thread: threading.Thread | None = None + self._file_dialog_results: queue.SimpleQueue[ + tuple[int, FileSelectionRequest, list[str] | None] + ] = queue.SimpleQueue() def request_links(self, links: tuple[ClientWindowLink, ...]) -> None: """Replace the clickable regions over the native window output.""" @@ -187,6 +269,56 @@ def request_new_window_size(self, new_window_size: tuple[int, int]) -> None: ) self._window_size = presenter.resize(*new_window_size) + def _request_file_selector( + self, selector_id: int, request: FileSelectionRequest + ) -> None: + """Show the Tk file dialog on a worker thread that queues its result.""" + thread = self._file_dialog_thread + if thread is not None and thread.is_alive(): + # A flushed dialog stays open until the user closes it, and two Tk + # dialogs on separate threads are not safe. + self._complete_file_selector(selector_id, SelectedFilesStatus.BUSY) + return + results = self._file_dialog_results + + def show_dialog() -> None: + try: + paths: list[str] | None = _choose_paths(request) + except Exception: + _LOGGER.exception( + "Native file picker failed for request id %r.", request.request_id + ) + paths = None + results.put((selector_id, request, paths)) + + thread = threading.Thread( + target=show_dialog, name="flashdreams-file-dialog", daemon=True + ) + thread.start() + self._file_dialog_thread = thread + + def _poll_file_dialog(self) -> None: + """Report the results the dialog thread queued; stale ones are dropped.""" + while True: + try: + selector_id, request, paths = self._file_dialog_results.get_nowait() + except queue.Empty: + return + active = self._active_file_selector + if active is None or active[0] != selector_id: + continue + files: tuple[SelectedFile, ...] = () + if paths is None: + status = SelectedFilesStatus.UNAVAILABLE + elif not paths: + status = SelectedFilesStatus.CANCELLED + else: + # ponytail: files are read on the UI thread, up to the + # request's max_total_bytes, which stalls presentation while + # they load. Upgrade path: read them on the dialog thread. + files, status = _selected_files_from_paths(paths, request) + self._complete_file_selector(selector_id, status, files) + def open(self, session_desc: SessionDesc) -> None: """Open a session in the GLFW window on the runtime's UI thread. @@ -251,6 +383,7 @@ def open(self, session_desc: SessionDesc) -> None: def get_user_input_events(self) -> UserInputEvents: """Pump GLFW and return native input events not yet read.""" + self._poll_file_dialog() presenter = self._presenter if presenter is not None: pending_before_poll = tuple(self._pending_printable_keys) @@ -453,6 +586,12 @@ def _on_window_closed(self) -> None: self._close_event_enqueued = True self._put_input(CloseUserInputEvent(timestamp=uint64(0))) + def put_user_input_event(self, event: UserInputEvent) -> None: + """Queue ``event``, bypassing the poll-time buffer.""" + started_ns = self._session_started_ns + elapsed_ns = 0 if started_ns is None else max(0, self._clock_ns() - started_ns) + self._input_events.put(replace(event, timestamp=uint64(elapsed_ns // 1_000))) + class _SlangPyNativeWindowPresenter: """Own the SlangPy window and forward frames to its presentation context.""" diff --git a/flashdreams/flashdreams/runtime_v2/null_client_window.py b/flashdreams/flashdreams/runtime_v2/null_client_window.py index 76ff107fa..8255d7841 100644 --- a/flashdreams/flashdreams/runtime_v2/null_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/null_client_window.py @@ -4,21 +4,30 @@ """Client window that discards output and reports no input.""" from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult from flashdreams.runtime_v2.user_input_events import UserInputEvents class NullClientWindow(IClientWindow): - """Discard output and report no input.""" + """Discard output. It has no user input of its own.""" + + def __init__(self) -> None: + self._input_events: list[UserInputEvent] = [] def get_user_input_events(self) -> UserInputEvents: - """Return an empty input batch.""" - return UserInputEvents([]) + """Return the events queued since the last call.""" + events, self._input_events = self._input_events, [] + return UserInputEvents(events) + + def put_user_input_event(self, event: UserInputEvent) -> None: + self._input_events.append(event) def open(self, session_desc: SessionDesc) -> None: """Accept a session without opening an output.""" del session_desc + self._input_events = [] def write(self, result: StepResult) -> None: """Discard one result.""" diff --git a/flashdreams/flashdreams/runtime_v2/selected_file.py b/flashdreams/flashdreams/runtime_v2/selected_file.py new file mode 100644 index 000000000..63e450f1c --- /dev/null +++ b/flashdreams/flashdreams/runtime_v2/selected_file.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Selected-file types, request policy, and size/type checks.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum + +DEFAULT_MAX_FILE_BYTES = 1024 * 1024 * 1024 +"""Default ``max_file_bytes`` (1 GiB): the largest single file a request accepts.""" + +DEFAULT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024 +"""Default ``max_total_bytes`` (1 GiB): the most bytes all chosen files add up to.""" + +DEFAULT_MAX_FILE_COUNT = 1 +"""Default ``max_file_count``: one file, unless the app asks for more.""" + + +def normalize_selected_file_accept(accept: Sequence[str] = ()) -> tuple[str, ...]: + """Return accepted filename suffixes such as ``.png``. + + Args: + accept: Suffixes the client may choose. Empty means any type. + + Returns: + The suffixes as a tuple, unchanged except for the sequence type. + + Raises: + TypeError: ``accept`` is not a sequence of strings. + ValueError: A suffix is missing the leading ``.`` or contains a path. + """ + if isinstance(accept, str) or not isinstance(accept, Sequence): + raise TypeError("accept must be a sequence of suffixes.") + suffixes: list[str] = [] + for suffix in accept: + if not isinstance(suffix, str): + raise TypeError("accept suffixes must be strings.") + if ( + not suffix.startswith(".") + or suffix == "." + or "/" in suffix + or "\\" in suffix + ): + raise ValueError( + "accept suffixes must look like ``.png`` and cannot contain a path." + ) + suffixes.append(suffix) + return tuple(suffixes) + + +def selected_file_suffix_allowed(name: str, accept: tuple[str, ...]) -> bool: + """Return whether ``name`` matches ``accept``. + + Empty ``accept`` allows any name. Comparison is case-insensitive. + """ + if not accept: + return True + lower = name.lower() + return any(lower.endswith(suffix.lower()) for suffix in accept) + + +class SelectedFilesStatus(Enum): + """Outcome of one client file-selector request.""" + + OK = "ok" + """The client chose files that passed type and size checks.""" + + CANCELLED = "cancelled" + """The user dismissed the selector.""" + + TOO_LARGE = "too_large" + """A file was larger than ``max_file_bytes``, the files together were larger + than ``max_total_bytes``, or more than ``max_file_count`` files were chosen.""" + + DISALLOWED_TYPE = "disallowed_type" + """A chosen file did not match ``accept``. Checked before size.""" + + BUSY = "busy" + """Another selector was already open, so this request was not shown. The + open selector keeps going and answers its own request.""" + + UNAVAILABLE = "unavailable" + """No pick was made: the client has no picker or left, the picker failed, + or a chosen file was unreadable.""" + + +@dataclass(frozen=True, slots=True) +class SelectedFile: + """One file chosen by a client file selector.""" + + name: str + """Base name of the chosen file, without any host path.""" + + data: bytes + """File contents.""" + + +@dataclass(frozen=True, slots=True) +class FileSelectionRequest: + """One client file-selector request queued by a UI loop.""" + + request_id: str + """Stable id for which button this request is for. Echoed on the matching event.""" + + initial_path: str + """Directory hint for the selector.""" + + accept: tuple[str, ...] = () + """Filename suffixes the client may choose, empty for any type.""" + + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES + """Largest single file in bytes.""" + + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES + """Most bytes all chosen files may add up to.""" + + max_file_count: int = DEFAULT_MAX_FILE_COUNT + """Most files the selector may return. Above one, the user may pick several.""" + + +def selected_files_policy_status( + files: Sequence[tuple[str, int | None]], + request: FileSelectionRequest, +) -> SelectedFilesStatus | None: + """Return why a chosen set breaks ``request``, or ``None`` if it passes. + + Type is checked before size and count, across the whole set. + + Args: + files: ``(name, size)`` pairs. ``size`` of ``None`` skips that file's + size check and omits it from the total. + request: The request whose ``accept`` and limits apply. + + Returns: + ``DISALLOWED_TYPE``, ``TOO_LARGE``, or ``None`` when the set passes. + """ + for name, _size in files: + if not selected_file_suffix_allowed(name, request.accept): + return SelectedFilesStatus.DISALLOWED_TYPE + if len(files) > request.max_file_count: + return SelectedFilesStatus.TOO_LARGE + total = 0 + for _name, size in files: + if size is None: + continue + if size > request.max_file_bytes: + return SelectedFilesStatus.TOO_LARGE + total += size + if total > request.max_total_bytes: + return SelectedFilesStatus.TOO_LARGE + return None diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/app.js b/flashdreams/flashdreams/runtime_v2/serving/web/app.js index 57a659621..4e94d4670 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/app.js +++ b/flashdreams/flashdreams/runtime_v2/serving/web/app.js @@ -7,6 +7,12 @@ const pointerControls = peer.createDataChannel("pointer-controls"); peer.addTransceiver("video", {direction: "recvonly"}); const video = document.getElementById("video"); const status = document.getElementById("status"); +const promptDialog = document.getElementById("prompt-dialog"); +const promptBackdrop = document.getElementById("prompt-backdrop"); +const promptDialogTitle = document.getElementById("prompt-dialog-title"); +const promptDialogDetails = document.getElementById("prompt-dialog-details"); +const promptDialogConfirm = document.getElementById("prompt-dialog-confirm"); +const promptDialogCancel = document.getElementById("prompt-dialog-cancel"); const pressedKeys = new Map(); const pressedButtons = new Set(); const gamepadSnapshots = new Map(); @@ -74,6 +80,321 @@ const applyCursorOptions = options => { } }; +let promptSettled = false; +let promptOnConfirm = null; +let promptOnCancel = null; + +const setPromptBackdropVisible = visible => { + if (promptBackdrop !== null) { + promptBackdrop.hidden = !visible; + } +}; + +const closePromptDialog = () => { + setPromptBackdropVisible(false); + if (promptDialog !== null && promptDialog.open) { + promptDialog.close(); + } +}; + +const fillPromptDetails = details => { + if (promptDialogDetails === null) { + return; + } + promptDialogDetails.replaceChildren(); + if (!Array.isArray(details) || details.length === 0) { + promptDialogDetails.hidden = true; + return; + } + for (const section of details) { + const row = document.createElement("div"); + row.className = "prompt-dialog-row"; + const label = document.createElement("div"); + label.className = "prompt-dialog-detail-label"; + label.textContent = section.label; + const value = document.createElement("div"); + value.className = "prompt-dialog-row-value"; + const values = Array.isArray(section.items) ? section.items : []; + for (const item of values) { + const chip = document.createElement("span"); + chip.className = "prompt-dialog-chip"; + chip.textContent = item; + value.append(chip); + } + row.append(label, value); + promptDialogDetails.append(row); + } + promptDialogDetails.hidden = false; +}; + +const showPromptDialog = ({ + title, + details = [], + confirmLabel = "OK", + cancelLabel = "Cancel", + onConfirm = null, + onCancel = null, +} = {}) => { + if ( + promptDialog === null + || promptDialogTitle === null + || promptDialogConfirm === null + || promptDialogCancel === null + ) { + return; + } + if (document.pointerLockElement === video) { + document.exitPointerLock(); + } + if (promptDialog.open) { + promptSettled = true; + promptOnConfirm = null; + promptOnCancel = null; + promptDialog.close(); + } + promptDialogTitle.textContent = title; + fillPromptDetails(details); + promptDialogConfirm.textContent = confirmLabel; + promptDialogCancel.textContent = cancelLabel; + promptOnConfirm = onConfirm; + promptOnCancel = onCancel; + promptSettled = false; + setPromptBackdropVisible(true); + promptDialog.show(); + promptDialogConfirm.focus(); +}; + +if (promptDialog !== null) { + promptDialogConfirm?.addEventListener("click", () => { + if (promptSettled) { + return; + } + promptSettled = true; + const onConfirm = promptOnConfirm; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + onConfirm?.(); + }); + promptDialogCancel?.addEventListener("click", () => { + if (promptSettled) { + return; + } + promptSettled = true; + const onCancel = promptOnCancel; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + onCancel?.(); + }); + promptDialog.addEventListener("close", () => { + // The event is queued, so it can arrive after a newer prompt reopened the dialog. + if (promptDialog.open) { + return; + } + setPromptBackdropVisible(false); + if (promptSettled) { + return; + } + promptSettled = true; + const onCancel = promptOnCancel; + promptOnConfirm = null; + promptOnCancel = null; + onCancel?.(); + }); +} + +window.addEventListener("keydown", event => { + if (event.key !== "Escape" || promptDialog === null || !promptDialog.open) { + return; + } + event.preventDefault(); + event.stopImmediatePropagation(); + promptDialogCancel?.click(); +}); + +if (promptBackdrop !== null) { + promptBackdrop.addEventListener("click", () => { + promptDialogCancel?.click(); + }); +} + +let pendingFileRequest = null; + +const hideFilePicker = () => { + pendingFileRequest = null; + if (promptDialog !== null && promptDialog.open) { + promptSettled = true; + promptOnConfirm = null; + promptOnCancel = null; + closePromptDialog(); + } +}; + +const completeFileSelection = (requestId, status) => { + send({type: "file_selector_result", id: requestId, status}); +}; + +const formatByteBudget = bytes => { + if (!Number.isFinite(bytes) || bytes <= 0) { + return "the allowed size"; + } + const mib = bytes / (1024 * 1024); + if (mib >= 1) { + return Number.isInteger(mib) ? `${mib} MiB` : `${mib.toFixed(1)} MiB`; + } + const kib = bytes / 1024; + if (kib >= 1) { + return Number.isInteger(kib) ? `${kib} KiB` : `${kib.toFixed(1)} KiB`; + } + return `${bytes} bytes`; +}; + +const fileNameMatchesAccept = (name, accept) => { + if (!Array.isArray(accept) || accept.length === 0) { + return true; + } + const lower = name.toLowerCase(); + return accept.some( + suffix => typeof suffix === "string" && lower.endsWith(suffix.toLowerCase()), + ); +}; + +const startFileInput = pending => { + const requestId = pending.id; + const input = document.createElement("input"); + input.type = "file"; + if (pending.accept.length > 0) { + input.accept = pending.accept.join(","); + } + input.multiple = pending.multiple === true; + let settled = false; + const finishFileSelection = status => { + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + completeFileSelection(requestId, status); + hideFilePicker(); + }; + input.addEventListener("change", () => { + if (pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + const files = pending.multiple + ? Array.from(input.files || []) + : Array.from(input.files || []).slice(0, 1); + if (files.length === 0) { + finishFileSelection("cancelled"); + return; + } + let rejected = null; + for (const file of files) { + if (!fileNameMatchesAccept(file.name || "", pending.accept)) { + rejected = "disallowed_type"; + break; + } + } + if (rejected === null) { + for (const file of files) { + if (file.size > pending.maxBytes) { + rejected = "too_large"; + break; + } + } + } + if (rejected !== null) { + finishFileSelection(rejected); + return; + } + // One request per file, in order, so the server reads only file bytes. + const uploadFiles = async () => { + for (const [index, file] of files.entries()) { + const query = new URLSearchParams({ + id: requestId, + name: file.name, + index: String(index), + count: String(files.length), + }); + const response = await fetch(`/api/files?${query}`, {method: "POST", body: file}); + if (!response.ok) { + // A proxy may refuse the upload before the server can answer it. + console.debug("File upload was rejected.", response.status); + finishFileSelection("unavailable"); + return; + } + } + if (settled || pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + settled = true; + hideFilePicker(); + }; + uploadFiles().catch(error => { + console.debug("Unable to upload the selected file.", error); + finishFileSelection("unavailable"); + }); + }); + input.addEventListener("cancel", () => finishFileSelection("cancelled")); + input.click(); +}; + +const openFileSelector = options => { + const requestId = options?.id; + if (typeof requestId !== "string" || !requestId) { + return; + } + if (pendingFileRequest !== null) { + if (pendingFileRequest.id === requestId) { + return; + } + // The server opens one selector at a time, so a new one means the old one finished. + hideFilePicker(); + } + const accept = Array.isArray(options.accept) ? options.accept : []; + const maxBytes = Number(options.max_file_bytes); + pendingFileRequest = { + id: requestId, + accept, + maxBytes: Number.isFinite(maxBytes) ? maxBytes : 0, + multiple: Number(options.max_file_count) > 1, + }; + showPromptDialog({ + title: pendingFileRequest.multiple + ? "The app requested files" + : "The app requested a file", + details: [ + { + label: "Accepted types", + items: accept.length > 0 ? accept : ["Any"], + }, + { + label: "Max size per file", + items: [formatByteBudget(pendingFileRequest.maxBytes)], + }, + ], + confirmLabel: "Select…", + onConfirm: () => { + const pending = pendingFileRequest; + if (pending === null || pending.id !== requestId) { + return; + } + startFileInput(pending); + }, + onCancel: () => { + if (pendingFileRequest === null || pendingFileRequest.id !== requestId) { + return; + } + completeFileSelection(requestId, "cancelled"); + hideFilePicker(); + }, + }); + if (promptDialog === null || !promptDialog.open) { + completeFileSelection(requestId, "unavailable"); + hideFilePicker(); + } +}; + window.addEventListener("focus", updateCursorVisibility); window.addEventListener("blur", updateCursorVisibility); @@ -91,6 +412,14 @@ controls.addEventListener("message", event => { openUrl(payload.url); return; } + if (payload?.type === "file_selector") { + openFileSelector(payload); + return; + } + if (payload?.type === "file_selector_cancel") { + hideFilePicker(); + return; + } if (payload?.type === "error") { console.warn(`WebRTC server: ${payload.message ?? "unknown error"}`); } @@ -124,7 +453,7 @@ peer.addEventListener("connectionstatechange", () => { window.addEventListener("keydown", event => { const keyId = event.code || event.key; - if (pressedKeys.has(keyId)) { + if (pressedKeys.has(keyId) || (promptDialog !== null && promptDialog.open)) { return; } const wasSent = sendInput({ diff --git a/flashdreams/flashdreams/runtime_v2/serving/web/index.html b/flashdreams/flashdreams/runtime_v2/serving/web/index.html index 1b73e1dc8..eb0d4ad2c 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/web/index.html +++ b/flashdreams/flashdreams/runtime_v2/serving/web/index.html @@ -48,11 +48,120 @@ color: #ff8a8a; } + #prompt-backdrop { + position: fixed; + inset: 0; + z-index: 2; + background: rgb(0 0 0 / 28%); + } + + #prompt-backdrop[hidden] { + display: none; + } + + #prompt-dialog { + position: fixed; + inset: 0; + z-index: 3; + margin: auto; + height: fit-content; + color-scheme: light dark; + max-width: 26rem; + width: min(26rem, calc(100% - 2rem)); + padding: 1.25rem 1.35rem 1rem; + border: 1px solid color-mix(in srgb, CanvasText 18%, Canvas); + border-radius: 12px; + background: Canvas; + color: CanvasText; + font: inherit; + box-shadow: 0 18px 40px rgb(0 0 0 / 45%); + } + + #prompt-dialog-title { + margin: 0 0 0.9rem; + font-size: 1.1rem; + font-weight: 650; + } + + #prompt-dialog-details { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + align-items: baseline; + column-gap: 0.85rem; + row-gap: 0.4rem; + margin: 0 0 1.15rem; + } + + #prompt-dialog-details[hidden] { + display: none; + } + + .prompt-dialog-row { + display: contents; + } + + .prompt-dialog-detail-label { + margin: 0; + font-size: 0.85rem; + color: color-mix(in srgb, CanvasText 62%, Canvas); + text-align: end; + white-space: nowrap; + } + + .prompt-dialog-detail-label::after { + content: ":"; + } + + .prompt-dialog-row-value { + display: flex; + flex-wrap: wrap; + column-gap: 0.55rem; + row-gap: 0.15rem; + } + + .prompt-dialog-chip { + font-size: 0.95rem; + } + + .prompt-dialog-actions { + display: flex; + justify-content: flex-end; + gap: 0.55rem; + } + + .prompt-dialog-actions button { + font: inherit; + padding: 0.45rem 0.95rem; + border-radius: 0.45rem; + cursor: pointer; + } + + #prompt-dialog-cancel { + border: 1px solid color-mix(in srgb, CanvasText 28%, Canvas); + background: transparent; + color: CanvasText; + } + + #prompt-dialog-confirm { + border: 0; + background: CanvasText; + color: Canvas; + } +