From 1a66c2228585c11edbfd4c98d8b673351784929a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Devernay?= Date: Fri, 9 Jun 2023 14:54:50 -0700 Subject: [PATCH 1/2] fix "ns-render interpolate", so that it can render the input view, add crop box Previously, ns-render interpolate: - ignored distortion parameters - used an interpolated focal length, but principal point was from the first camera - rendered all images with the same size, even if input views were of different sizes - forgot to render the last frame - There was no default support for crop box and background color, except when rendering from a camera path, where these would exclusively be fetched from the camera_path.json Now: - There is an command-line option `--fixed-intrinsics` to keep fixed intrinsics through the whole sequence, but they are all interpolated by default - Everything is interpolated, even size, so that `ns-render interpolate -step 1` should render exactly the input images - All render modes have the options `--use-bounding-box` `--background-color` `--bounding-box-min` `--bouding-box-max` (same syntax as ns-export), which can override the json settings when rendering a camera path. Works great with the gray colormap from https://github.com/nerfstudio-project/nerfstudio/pull/2058 to render object masks. --- nerfstudio/cameras/camera_paths.py | 42 +++++++--- nerfstudio/cameras/camera_utils.py | 125 +++++++++++++++++++++++++---- nerfstudio/scripts/render.py | 63 ++++++++++++--- 3 files changed, 193 insertions(+), 37 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index 50d417f928..07af54b894 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -27,28 +27,50 @@ from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length -def get_interpolated_camera_path(cameras: Cameras, steps: int, order_poses: bool) -> Cameras: +def get_interpolated_camera_path(cameras: Cameras, steps: int, order_poses: bool, fixed_intrinsics: bool) -> Cameras: """Generate a camera path between two cameras. Uses the camera type of the first camera Args: cameras: Cameras object containing intrinsics of all cameras. steps: The number of steps to interpolate between the two cameras. + fixed_intrinsics: Use intrinsics from the first camera for the whole sequence. Returns: A new set of cameras along a path. """ Ks = cameras.get_intrinsics_matrices() poses = cameras.camera_to_worlds - poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps, order_poses=order_poses) - - cameras = Cameras( - fx=Ks[:, 0, 0], - fy=Ks[:, 1, 1], - cx=Ks[0, 0, 2], - cy=Ks[0, 1, 2], - camera_type=cameras.camera_type[0], - camera_to_worlds=poses, + widths = cameras.width + heights = cameras.height + dist_coeffs = cameras.distortion_params + poses, Ks, widths, heights, dist_coeffs = get_interpolated_poses_many( + poses, Ks, widths, heights, dist_coeffs, steps_per_transition=steps, order_poses=order_poses ) + + if fixed_intrinsics: + cameras = Cameras( + fx=Ks[0, 0, 0], + fy=Ks[0, 1, 1], + cx=Ks[0, 0, 2], + cy=Ks[0, 1, 2], + width=widths[0], + height=heights[0], + distortion_params=dist_coeffs[0] if dist_coeffs is not None else None, + camera_type=cameras.camera_type[0], + camera_to_worlds=poses, + ) + else: + cameras = Cameras( + fx=Ks[:, 0, 0], + fy=Ks[:, 1, 1], + cx=Ks[:, 0, 2], + cy=Ks[:, 1, 2], + width=widths, + height=heights, + distortion_params=dist_coeffs, + camera_type=cameras.camera_type[0], + camera_to_worlds=poses, + ) return cameras diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index 2c0b746d1a..cb282d0192 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -21,7 +21,7 @@ import numpy as np import torch -from jaxtyping import Float +from jaxtyping import Float, Shaped from numpy.typing import NDArray from torch import Tensor @@ -208,10 +208,53 @@ def get_interpolated_k( return Ks -def get_ordered_poses_and_k( +def get_interpolated_dist_coeffs( + dist_coeffs_a: Float[Tensor, "6"], dist_coeffs_b: Float[Tensor, "6"], steps: int = 10 +) -> List[Float[Tensor, "6"]]: + """ + Returns interpolated distortion coefficients with specified number of steps. + + Args: + dist_coeffs_a: distortion coefficients 1 + dist_coeffs_b: distortion coefficients 2 + steps: number of steps the interpolated distortion coefficients should contain + + Returns: + List of interpolated distortion coefficients + """ + dist_coeffs: List[Float[Tensor, "6"]] = [] + ts = np.linspace(0, 1, steps) + for t in ts: + new_dist_coeffs = dist_coeffs_a * (1.0 - t) + dist_coeffs_b * t + dist_coeffs.append(new_dist_coeffs) + return dist_coeffs + + +def get_interpolated_sizes(size_a: int, size_b: int, steps: int = 10) -> List[int]: + """ + Returns interpolated sizes with specified number of steps. + + Args: + sizes_a: sizes 1 + sizes_b: sizes 2 + steps: number of steps the output sizes should contain + + Returns: + List of interpolated sizes + """ + if size_a == size_b: + return [size_a] * steps + sizes: List[int] = [] + ts = np.linspace(0, 1, steps) + for t in ts: + new_size = size_a * (1.0 - t) + size_b * t + sizes.append(int(new_size)) + return sizes + + +def get_ordered_pose_indices( poses: Float[Tensor, "num_poses 3 4"], - Ks: Float[Tensor, "num_poses 3 3"], -) -> Tuple[Float[Tensor, "num_poses 3 4"], Float[Tensor, "num_poses 3 3"]]: +) -> List[int]: """ Returns ordered poses and intrinsics by euclidian distance between poses. @@ -220,41 +263,54 @@ def get_ordered_poses_and_k( Ks: list of camera intrinsics Returns: - tuple of ordered poses and intrinsics + indices of ordered poses """ poses_num = len(poses) ordered_poses = torch.unsqueeze(poses[0], 0) - ordered_ks = torch.unsqueeze(Ks[0], 0) + ordered_idx = [0] # remove the first pose from poses poses = poses[1:] - Ks = Ks[1:] + poses_indices = list(range(1, poses_num)) for _ in range(poses_num - 1): distances = torch.norm(ordered_poses[-1][:, 3] - poses[:, :, 3], dim=1) idx = torch.argmin(distances) ordered_poses = torch.cat((ordered_poses, torch.unsqueeze(poses[idx], 0)), dim=0) - ordered_ks = torch.cat((ordered_ks, torch.unsqueeze(Ks[idx], 0)), dim=0) + ordered_idx.append(poses_indices[idx]) poses = torch.cat((poses[0:idx], poses[idx + 1 :]), dim=0) - Ks = torch.cat((Ks[0:idx], Ks[idx + 1 :]), dim=0) + poses_indices = poses_indices[0:idx] + poses_indices[idx + 1 :] - return ordered_poses, ordered_ks + assert len(ordered_idx) == poses_num + return ordered_idx def get_interpolated_poses_many( poses: Float[Tensor, "num_poses 3 4"], Ks: Float[Tensor, "num_poses 3 3"], + widths: Shaped[Tensor, "num_poses"], + heights: Shaped[Tensor, "num_poses"], + dist_coeffs: Optional[Float[Tensor, "num_poses 6"]] = None, steps_per_transition: int = 10, order_poses: bool = False, -) -> Tuple[Float[Tensor, "num_poses 3 4"], Float[Tensor, "num_poses 3 3"]]: +) -> Tuple[ + Float[Tensor, "num_poses 3 4"], + Float[Tensor, "num_poses 3 3"], + Shaped[Tensor, "num_poses"], + Shaped[Tensor, "num_poses"], + Optional[Float[Tensor, "num_poses 6"]], +]: """Return interpolated poses for many camera poses. Args: poses: list of camera poses Ks: list of camera intrinsics + widths: list of image widths + heights: list of image heights + dist_coeffs: list of distortion coefficients steps_per_transition: number of steps per transition order_poses: whether to order poses by euclidian distance @@ -263,9 +319,18 @@ def get_interpolated_poses_many( """ traj = [] k_interp = [] + widths_interp = [] + heights_interp = [] + dist_coeffs_interp = [] if order_poses: - poses, Ks = get_ordered_poses_and_k(poses, Ks) + pose_indices = get_ordered_pose_indices(poses) + poses = poses[pose_indices] + Ks = Ks[pose_indices] + widths = widths[pose_indices] + heights = heights[pose_indices] + if dist_coeffs is not None: + dist_coeffs = dist_coeffs[pose_indices] for idx in range(poses.shape[0] - 1): pose_a = poses[idx].cpu().numpy() @@ -273,11 +338,39 @@ def get_interpolated_poses_many( poses_ab = get_interpolated_poses(pose_a, pose_b, steps=steps_per_transition) traj += poses_ab k_interp += get_interpolated_k(Ks[idx], Ks[idx + 1], steps=steps_per_transition) - - traj = np.stack(traj, axis=0) + widths_interp += get_interpolated_sizes( + int(widths[idx].item()), int(widths[idx + 1].item()), steps=steps_per_transition + ) + heights_interp += get_interpolated_sizes( + int(heights[idx].item()), int(heights[idx + 1].item()), steps=steps_per_transition + ) + if dist_coeffs is not None: + dist_coeffs_interp += get_interpolated_dist_coeffs( + dist_coeffs[idx], dist_coeffs[idx + 1], steps=steps_per_transition + ) + + # add last pose + traj.append(poses[-1]) + k_interp.append(Ks[-1]) + widths_interp.append(widths[-1].item()) + heights_interp.append(heights[-1].item()) + if dist_coeffs is not None: + dist_coeffs_interp.append(dist_coeffs[-1]) + + traj = torch.tensor(np.stack(traj, axis=0), dtype=torch.float32) k_interp = torch.stack(k_interp, dim=0) - - return torch.tensor(traj, dtype=torch.float32), torch.tensor(k_interp, dtype=torch.float32) + if dist_coeffs is not None: + dist_coeffs_interp = torch.stack(dist_coeffs_interp, dim=0) + else: + dist_coeffs_interp = None + + return ( + traj, + k_interp, + torch.tensor(widths_interp, dtype=torch.int32), + torch.tensor(heights_interp, dtype=torch.int32), + dist_coeffs_interp, + ) def normalize(x: torch.Tensor) -> Float[Tensor, "*batch"]: diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index aef5d4909d..16d6ed0790 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -26,7 +26,7 @@ from contextlib import ExitStack from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union, Tuple import mediapy as media import numpy as np @@ -125,8 +125,8 @@ def _render_trajectory_video( for camera_idx in progress.track(range(cameras.size), description=""): aabb_box = None if crop_data is not None: - bounding_box_min = crop_data.center - crop_data.scale / 2.0 - bounding_box_max = crop_data.center + crop_data.scale / 2.0 + bounding_box_min = crop_data.bounding_box_min + bounding_box_max = crop_data.bounding_box_max aabb_box = SceneBox(torch.stack([bounding_box_min, bounding_box_max]).to(pipeline.device)) camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx, aabb_box=aabb_box) @@ -271,10 +271,10 @@ class CropData: background_color: Float[Tensor, "3"] = torch.Tensor([0.0, 0.0, 0.0]) """background color""" - center: Float[Tensor, "3"] = torch.Tensor([0.0, 0.0, 0.0]) - """center of the crop""" - scale: Float[Tensor, "3"] = torch.Tensor([2.0, 2.0, 2.0]) - """scale of the crop""" + bounding_box_min: Float[Tensor, "3"] = torch.Tensor([-1.0, -1.0, -1.0]) + """Minimum of the crop box.""" + bounding_box_max: Float[Tensor, "3"] = torch.Tensor([1.0, 1.0, 1.0]) + """Minimum of the crop box.""" def get_crop_from_json(camera_json: Dict[str, Any]) -> Optional[CropData]: @@ -289,11 +289,12 @@ def get_crop_from_json(camera_json: Dict[str, Any]) -> Optional[CropData]: return None bg_color = camera_json["crop"]["crop_bg_color"] - + center = torch.Tensor(camera_json["crop"]["crop_center"]) + scale = torch.Tensor(camera_json["crop"]["crop_scale"]) return CropData( background_color=torch.Tensor([bg_color["r"] / 255.0, bg_color["g"] / 255.0, bg_color["b"] / 255.0]), - center=torch.Tensor(camera_json["crop"]["crop_center"]), - scale=torch.Tensor(camera_json["crop"]["crop_scale"]), + bounding_box_min=center - scale / 2.0, + bounding_box_max=center + scale / 2.0, ) @@ -315,6 +316,14 @@ class BaseRender: """Specifies number of rays per chunk during eval. If None, use the value in the config file.""" colormap_options: colormaps.ColormapOptions = colormaps.ColormapOptions() """Colormap options.""" + use_bounding_box: bool = False + """Only render points within the bounding box (overriding crop data from the camera path).""" + background_color: Tuple[float, float, float] = (0, 0, 0) + """Background color for the cropped render, used if use_bounding_box is True.""" + bounding_box_min: Tuple[float, float, float] = (-1, -1, -1) + """Minimum of the bounding box, used if use_bounding_box is True.""" + bounding_box_max: Tuple[float, float, float] = (1, 1, 1) + """Minimum of the bounding box, used if use_bounding_box is True.""" @dataclass @@ -341,7 +350,14 @@ def main(self) -> None: with open(self.camera_path_filename, "r", encoding="utf-8") as f: camera_path = json.load(f) seconds = camera_path["seconds"] - crop_data = get_crop_from_json(camera_path) + if self.use_bounding_box: + crop_data = CropData( + background_color=torch.Tensor(self.background_color), + bounding_box_min=torch.Tensor(self.bounding_box_min), + bounding_box_max=torch.Tensor(self.bounding_box_max), + ) + else: + crop_data = get_crop_from_json(camera_path) camera_path = get_path_from_json(camera_path) if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value: @@ -432,6 +448,8 @@ class RenderInterpolated(BaseRender): """Number of interpolation steps between eval dataset cameras.""" order_poses: bool = False """Whether to order camera poses by proximity.""" + fixed_intrinsics: bool = False + """Use intrinsics from the first camera for the whole sequence.""" frame_rate: int = 24 """Frame rate of the output video.""" output_format: Literal["images", "video"] = "video" @@ -459,7 +477,16 @@ def main(self) -> None: cameras=cameras, steps=self.interpolation_steps, order_poses=self.order_poses, + fixed_intrinsics=self.fixed_intrinsics, ) + if self.use_bounding_box: + crop_data = CropData( + background_color=torch.Tensor(self.background_color), + bounding_box_min=torch.Tensor(self.bounding_box_min), + bounding_box_max=torch.Tensor(self.bounding_box_max), + ) + else: + crop_data = None _render_trajectory_video( pipeline, @@ -467,8 +494,11 @@ def main(self) -> None: output_filename=self.output_path, rendered_output_names=self.rendered_output_names, rendered_resolution_scaling_factor=1.0 / self.downscale_factor, + crop_data=crop_data, seconds=seconds, output_format=self.output_format, + image_format=self.image_format, + jpeg_quality=self.jpeg_quality, colormap_options=self.colormap_options, ) @@ -502,6 +532,14 @@ def main(self) -> None: steps = int(self.frame_rate * self.seconds) camera_start = pipeline.datamanager.eval_dataloader.get_camera(image_idx=0).flatten() camera_path = get_spiral_path(camera_start, steps=steps, radius=self.radius) + if self.use_bounding_box: + crop_data = CropData( + background_color=torch.Tensor(self.background_color), + bounding_box_min=torch.Tensor(self.bounding_box_min), + bounding_box_max=torch.Tensor(self.bounding_box_max), + ) + else: + crop_data = None _render_trajectory_video( pipeline, @@ -509,8 +547,11 @@ def main(self) -> None: output_filename=self.output_path, rendered_output_names=self.rendered_output_names, rendered_resolution_scaling_factor=1.0 / self.downscale_factor, + crop_data=crop_data, seconds=self.seconds, output_format=self.output_format, + image_format=self.image_format, + jpeg_quality=self.jpeg_quality, colormap_options=self.colormap_options, ) From 10747bec68112aa67454f039955838983e63e36c Mon Sep 17 00:00:00 2001 From: Frederic Devernay Date: Tue, 28 Jul 2026 19:16:08 +0200 Subject: [PATCH 2/2] fix(render): expose and pass fixed_intrinsics in RenderInterpolated Completes the #2059 integration: get_interpolated_camera_path requires fixed_intrinsics, but RenderInterpolated neither exposed the flag nor passed it, which would crash 'ns-render interpolate'. --- nerfstudio/scripts/render.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index fc035c4a9f..4f46bb1fa2 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -598,6 +598,8 @@ class RenderInterpolated(BaseRender): """Number of interpolation steps between eval dataset cameras.""" order_poses: bool = False """Whether to order camera poses by proximity.""" + fixed_intrinsics: bool = False + """Use intrinsics from the first camera for the whole interpolated sequence.""" frame_rate: int = 24 """Frame rate of the output video.""" output_format: Literal["images", "video"] = "video" @@ -625,6 +627,7 @@ def main(self) -> None: cameras=cameras, steps=self.interpolation_steps, order_poses=self.order_poses, + fixed_intrinsics=self.fixed_intrinsics, ) if self.camera_idx is not None: