Restyle the panel and make renders faster and smoother

Installer/install_eternal2x.py

@@ -95,8 +95,6 @@ def _pick_python(repo_root: Path) -> str:
     "output_dir": "",
     "upscale": "true",
     "interpolate": "true",
-    "grain": "false",
-    "base_hold": "0",
     "threshold": "5",
     "format": "png",
 }

Pipeline/dedupe.py

@@ -49,7 +49,7 @@ class DedupePlan:
     # One source frame index per intermediate frame, in order.
     output_indices: List[int] = field(default_factory=list)
     # Every frame came out new, but the scores look like grain on held
-    # frames: worth turning on Grainy source and analysing again.
+    # frames, so the drawings are best marked by hand.
     grain_suspected: bool = False
     # Made from the frames the editor marked, not from detection. Every frame
     # between two marks is then an in-between, however long the gap.

Pipeline/render.py

@@ -15,8 +15,10 @@
 
 from __future__ import annotations
 
+import os
 import sys
 from collections import deque
+from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Deque, Iterator, List, Optional, Sequence, Tuple
@@ -367,6 +369,11 @@ def _iter_plan_frames(cap: cv2.VideoCapture, indices: Sequence[int]) -> Iterator
 # How many in-betweens are drawn before they are written.
 BATCH = 8
 
+# Threads that encode and write PNG frames. A 4K PNG takes about as long to
+# encode as the GPU takes to draw and upscale a frame, and OpenCV lets go of
+# Python while it encodes, so the two overlap. One core is left for Resolve.
+WRITERS = max(1, min(3, (os.cpu_count() or 2) - 1))
+
 # Encoders to try per container, best quality first.
 _ENCODERS = {
     ".mkv": ("FFV1",),             # lossless; Resolve refuses FFV1 in AVI
@@ -462,6 +469,24 @@ def render_plan(
         buckets[i].append(p - i)
 
     cap = _open(source, frames, progress)
+    pool = ThreadPoolExecutor(max_workers=WRITERS) if is_sequence else None
+    # Frames still being written, oldest first. A few at most, so a fast GPU
+    # can't fill memory with 4K frames waiting for the disk.
+    pending: Deque[Future] = deque()
+
+    def write_png(frame: np.ndarray, target: Path, number: int) -> None:
+        # Encode in memory and let Python write the file. cv2.imwrite
+        # cannot open a non-ASCII path on Windows, so a clip in a folder
+        # called Épisode 01 would fail on its first frame.
+        encoded, data = cv2.imencode(".png", frame)
+        if not encoded:
+            raise SourceError(f"Could not encode frame {number}.")
+        try:
+            target.write_bytes(data.tobytes())
+        except OSError as exc:
+            raise SourceError(f"Could not write {target}: {exc}. Check the disk "
+                              "has space and the folder is writable.") from exc
+
     state = {"writer": None, "written": 0, "width": 0, "height": 0, "upscaler": upscaler,
              "upscale_note": upscale_note, "last_in": None, "last_out": None}
 
@@ -493,17 +518,10 @@ def emit(frame: np.ndarray) -> None:
         state["height"], state["width"] = frame.shape[:2]
         if is_sequence:
             target = output / f"frame_{state['written']:06d}.png"
-            # Encode in memory and let Python write the file. cv2.imwrite
-            # cannot open a non-ASCII path on Windows, so a clip in a folder
-            # called Épisode 01 would fail on its first frame.
-            encoded, data = cv2.imencode(".png", frame)
-            if not encoded:
-                raise SourceError(f"Could not encode frame {state['written']}.")
-            try:
-                target.write_bytes(data.tobytes())
-            except OSError as exc:
-                raise SourceError(f"Could not write {target}: {exc}. Check the disk "
-                                  "has space and the folder is writable.") from exc
+            # A write that failed raises here, on the next frame.
+            while len(pending) >= 2 * WRITERS:
+                pending.popleft().result()
+            pending.append(pool.submit(write_png, frame, target, state["written"]))
         else:
             if state["writer"] is None:
                 # Every writer here rounds an odd size down to even without a
@@ -522,8 +540,15 @@ def emit(frame: np.ndarray) -> None:
             # A GPU that gave out mid-render leaves the rest to the CPU, which
             # takes seconds a frame. A new stage name restarts the time left.
             on_cpu = not getattr(state["upscaler"], "on_gpu", True)
-            progress("Rendering, upscaling on the CPU" if on_cpu else "Rendering",
-                     min(1.0, state["written"] / float(len(positions))))
+            # The panel shows this as it goes, so it names the work being done.
+            upscaling = "upscaling on the CPU" if on_cpu else "upscaling"
+            if state["upscaler"] is None and factor == 1:
+                stage = "Interpolating" if pair is not None else "Rendering"
+            elif pair is not None:
+                stage = "Interpolating and " + upscaling
+            else:
+                stage = upscaling.capitalize()
+            progress(stage, min(1.0, state["written"] / float(len(positions))))
 
     try:
         indices = plan.output_indices
@@ -592,10 +617,15 @@ def change_from(slot: int) -> Optional[float]:
         prev = frame_at(index)
         while state["written"] < len(positions):
             emit(prev)
+        while pending:
+            pending.popleft().result()
     finally:
         cap.release()
         if state["writer"] is not None:
             state["writer"].release()
+        if pool is not None:
+            # Even on the way out of an error, no write is left going.
+            pool.shutdown(wait=True)
 
     return RenderResult(
         output=output,

Pipeline/rife.py

@@ -41,6 +41,17 @@
 # multiple of this. The frame is padded by repeating its edge, then cropped.
 _PAD = 64
 
+# Asked for a frame close to a drawing, the network hands back the drawing
+# itself: on Hero's walk, every t up to 0.1 came back as the first drawing
+# exactly and every t from 0.9 as the second, and the motion stayed uneven to
+# about 0.2 and from 0.8. Slowed three times, that held each drawing for about
+# six extra frames, a pause at every drawing. Between 0.25 and 0.75 it is even,
+# so it is only ever asked for those. A frame nearer a drawing is drawn between
+# the drawing and the halfway frame, and so on until it falls inside.
+_EVEN_FROM = 0.25
+# A millionth of a gap: closer to a drawing than any real frame can be.
+_ROUNDING = 1e-6
+
 ProgressFn = Callable[[str, float], None]
 
 
@@ -83,7 +94,13 @@ def _pip(cmd: List[str], progress: Optional[ProgressFn]) -> subprocess.Completed
             return subprocess.CompletedProcess(cmd, proc.returncode, out, err)
         except subprocess.TimeoutExpired:
             if progress:
-                progress("Installing AI runtime", 0.0)
+                try:
+                    progress("Installing AI runtime", 0.0)
+                except BaseException:
+                    # Stopped. Cut short, pip could leave this Python half
+                    # installed, so it finishes before the run ends.
+                    proc.communicate()
+                    raise
 
 
 def ensure_runtime(progress: Optional[ProgressFn] = None):
@@ -164,6 +181,12 @@ def ensure_model(progress: Optional[ProgressFn] = None, path: Path = MODEL_PATH,
 class Rife:
     """One loaded model, reused for every pair of drawings in a render."""
 
+    # The pair being drawn and the halfway frames drawn for it, by where they
+    # sit between its drawings. A render asks for a pair's in-betweens a few
+    # at a time, so they are kept until the next pair.
+    _pair = None
+    _known = None
+
     def __init__(self, progress: Optional[ProgressFn] = None):
         ort = ensure_runtime(progress)
         model = ensure_model(progress)
@@ -256,10 +279,32 @@ def interpolate_pair(self, a: np.ndarray, b: np.ndarray, times: List[float],
         out: List[np.ndarray] = []
         for t in times:
             t = float(min(1.0, max(0.0, t)))
-            if t <= 0.0:
+            # This close to a drawing is rounding in the time map, not a frame.
+            if t <= _ROUNDING:
                 out.append(a.copy())
-            elif t >= 1.0:
+            elif t >= 1.0 - _ROUNDING:
                 out.append(b.copy())
             else:
-                out.append(self._run(a, b, t))
+                out.append(self._between(a, b, t))
         return out
+
+    def _between(self, a: np.ndarray, b: np.ndarray, t: float) -> np.ndarray:
+        """The frame at t between a and b, asking the network only for 0.25 to 0.75."""
+        if self._pair is None or self._pair[0] is not a or self._pair[1] is not b:
+            self._pair, self._known = (a, b), {0.0: a, 1.0: b}
+        known = self._known
+        lo, hi = 0.0, 1.0
+        while True:
+            local = (t - lo) / (hi - lo)
+            # Every t strictly inside lands here: each halving doubles its
+            # distance from the nearer end. A gap of 5000 frames slowed three
+            # times needs 12 halvings and keeps 23 halfway frames.
+            if _EVEN_FROM <= local <= 1.0 - _EVEN_FROM:
+                return self._run(known[lo], known[hi], local)
+            if hi - lo < 2.0 ** -30:
+                # Only rounding gets here, where both ends are one picture.
+                return (known[lo] if local < 0.5 else known[hi]).copy()
+            mid = (lo + hi) / 2.0
+            if mid not in known:
+                known[mid] = self._run(known[lo], known[hi], 0.5)
+            lo, hi = (lo, mid) if local < 0.5 else (mid, hi)

README.md

@@ -35,7 +35,7 @@ holds for three times its length.
 
 - Finds duplicated frames and works out whether the clip is on 1s, 2s or 3s.
   It counts a change of colour, a slow fade or a slow pan as motion rather
-  than a hold. With **Grainy source** ticked, it sees through film grain.
+  than a hold.
 - Draws the in-between frames that were never drawn with the RIFE network,
   run on your own machine, on the GPU on Windows wherever DirectML loads. Moving characters stay in
   one piece.
@@ -109,7 +109,8 @@ move it.
 
 1. Move the playhead over a clip on your timeline. Resolve does not tell
    scripts which clip is selected, so Upflow2x works on the clip under the
-   playhead, read again each time you press a button.
+   playhead. The panel follows the playhead, and in a gap it keeps the
+   last clip.
 2. Open **Workspace → Scripts → Upflow2x**.
 3. Click **Analyse**. It reads the clip, tells you what it found, for
    example `1438 frames, 719 unique drawings (719 held, 50%). Animated on 2s.`,
@@ -134,7 +135,7 @@ rendered from the drawings Upflow2x finds on its own.
 ![The Upflow2x panel in DaVinci Resolve Studio after a render. The rebuilt clip sits on V2, lined up above the original on V1.](https://upflow2x.com/assets/demo/resolve.jpg)
 
 The render runs in the background and the panel shows progress and the time
-left, so Resolve stays usable while it works. When it finishes, the rebuilt
+left, so Resolve stays usable while it works. **Stop** ends it early. When it finishes, the rebuilt
 clip is imported and placed on the track above the original, starting on the
 same frame. With Interpolate on it runs three times as long, up to the last
 marker if the clip has any. It goes on the lowest
@@ -161,14 +162,15 @@ Whole file when Resolve does not report one.
 
 ### Settings
 
+They are all on the panel, and it remembers them between sessions. Rest the
+pointer on one to see what it does.
+
 | Setting | What it does |
 |---|---|
 | **Threshold** | How big a change Analyse counts as a new drawing, 1 to 9. Lower catches small moves, like only a mouth or a slow drift. Higher counts only big ones, like the whole shot moving. It does nothing to frames you have marked. |
-| **Hold pattern** | Leave on `Auto detect`. Force `On 2s` or `On 3s` if a clip has unusual timing that trips the detector. |
 | **Output** | `Image sequence` is lossless and the most reliable to import. `MP4` is a single compact file but lossy. `MKV` is a lossless single file. All three are 8-bit RGB: transparency and 10-bit colour in the source are not kept. MP4 and MKV need an even width and height. |
-| **Upscale 2x** | Draws every frame at 2x with Real-ESRGAN's anime video model, on the GPU through DirectML, about 0.14 s a 1080p frame on a Radeon. The model ships with Upflow2x. Without a GPU, on a Mac for one, Resolve's Super Scale is applied to the imported clip instead. Either way it only shows on a timeline bigger than the clip, such as 3840x2160 for 1080p footage. A 2x render has four times the pixels. As an image sequence, a 4K frame of anime is about 6 to 7 MB, four to five times its 1080p size. |
-| **Interpolate** | On, the RIFE network draws every missing in-between. On Windows it runs on the GPU through DirectML, about 0.13 s an in-between at 2K on a Radeon, and falls back to the CPU, about ten times slower. If its Python already has ONNX Runtime without DirectML, that one is used, on the CPU. Its first run installs ONNX Runtime and downloads the model, about 50 MB, so it needs a connection once; without one, every drawing is kept as drawn and the panel says why. On, the render is three times longer, so it takes three times as long to render and an image sequence takes three times the disk. Off skips the in-betweens, so every drawing comes back on its original frame and the render keeps the clip's length. Use it with Upscale 2x to upscale only. |
-| **Grainy source** | For film grain, which makes every held frame differ a little, so they all look new. It raises the threshold above the clip's own grain. Leave it off for clean animation, where it can merge fine real changes. Analyse suggests it when every frame differs in a way that looks like grain. |
+| **Upscaling** | Draws every frame at 2x with Real-ESRGAN's anime video model, on the GPU through DirectML, about 0.14 s a 1080p frame on a Radeon. The model ships with Upflow2x. Without a GPU, on a Mac for one, Resolve's Super Scale is applied to the imported clip instead. Either way it only shows on a timeline bigger than the clip, such as 3840x2160 for 1080p footage. A 2x render has four times the pixels. As an image sequence, a 4K frame of anime is about 6 to 7 MB, four to five times its 1080p size. |
+| **Interpolate** | On, the RIFE network draws every missing in-between. On Windows it runs on the GPU through DirectML, about 0.13 s an in-between at 2K on a Radeon, and falls back to the CPU, about ten times slower. If its Python already has ONNX Runtime without DirectML, that one is used, on the CPU. Its first run installs ONNX Runtime and downloads the model, about 50 MB, so it needs a connection once; without one, every drawing is kept as drawn and the panel says why. On, the render is three times longer, so it takes three times as long to render and an image sequence takes three times the disk. Off skips the in-betweens, so every drawing comes back on its original frame and the render keeps the clip's length. With Upscaling on, that only upscales. |
 
 Settings are remembered between sessions.
 
@@ -220,9 +222,9 @@ afterwards.
 | Resolve says you need Studio | Since 19.1 the free edition blocks script panels. Upflow2x needs Studio. |
 | Clicking Upflow2x does nothing, or "Plugin folder not configured" | The plugin folder moved after installing, and Workspace → Console says `UI script not found`. Re-run `python Installer/install_eternal2x.py` from where it is now. |
 | "Could not reach Resolve" | Open the panel from Workspace → Scripts, not from a terminal. |
-| "No clip under the playhead" | Move the playhead over the clip, then press **Use Clip at Playhead**. |
+| "No clip under the playhead" | Move the playhead over the clip, and the panel picks it up. |
 | "No duplicate frames. This clip is already on 1s." | The clip has no duplicated frames. It is live action, or already on 1s. |
-| "Every frame differs, but the differences look like grain" | Tick **Grainy source** and analyse again. Heavy grain on a clip with long holds can still hide the duplicates. |
+| "Every frame differs, but the differences look like grain" | Mark the drawings with M, then press **Render**. From the command line, `--grain` raises the threshold above the grain. |
 | "...has a variable frame rate" | Render the clip to a constant frame rate first, from Resolve or with ffmpeg, as the message says. |
 | Progress seems stuck | Press **Refresh Progress**. If Python stopped, the panel says so and shows its last error. If a run was interrupted, press **Reset**. |
 | Resolve would not import the render | The render still finished. The panel prints where it is, so drag it into your media pool. Switching Output to `Image sequence` usually avoids this. |