From abbad03508b81dc24e1b53dad00d985cdc73c320 Mon Sep 17 00:00:00 2001 From: Ollie Boyne Date: Thu, 29 Oct 2026 10:04:45 -0700 Subject: [PATCH] Split camera_conversions into Numpy and TensorFlow modules PiperOrigin-RevId: 1009770425 --- gnm/shape/visualization/camera_conversions.py | 122 ------------ .../visualization/camera_conversions_test.py | 87 +-------- .../visualization/camera_conversions_tf.py | 158 +++++++++++++++ .../camera_conversions_tf_test.py | 183 ++++++++++++++++++ gnm/shape/visualization/render_common.py | 3 +- 5 files changed, 343 insertions(+), 210 deletions(-) create mode 100644 gnm/shape/visualization/camera_conversions_tf.py create mode 100644 gnm/shape/visualization/camera_conversions_tf_test.py diff --git a/gnm/shape/visualization/camera_conversions.py b/gnm/shape/visualization/camera_conversions.py index b557062c..4cd075b2 100644 --- a/gnm/shape/visualization/camera_conversions.py +++ b/gnm/shape/visualization/camera_conversions.py @@ -21,7 +21,6 @@ """ import numpy as np -import tensorflow as tf # Rotate OpenCV's camera coordinate system 180 degrees around the X-axis to # align it with the OpenGL coordinate frame. @@ -200,124 +199,3 @@ def opencv_intrinsics_matrix_to_opengl_view_matrix( far, skew_coefficient=skew_coefficient[..., np.newaxis], ) - - -# TensorFlow implementations -def opencv_extrinsics_to_opengl_tf(extrinsics: tf.Tensor) -> tf.Tensor: - """Converts OpenCV extrinsics to OpenGL using TensorFlow. - - Args: - extrinsics: The camera extrinsics or world-to-camera transformation matrix, - (A1, A2, ..., An, 4, 4). - - Returns: - Camera extrinsics in OpenGL coordinate system, with the same shape as input. - """ - return tf.einsum( - 'mk,...kn->...mn', - tf.convert_to_tensor(OPENCV_TO_OPENGL, dtype=tf.float32), - extrinsics, - ) - - -def opencv_intrinsics_matrix_to_opengl_view_matrix_tf( - camera_to_image: tf.Tensor, - width: int, - height: int, - near: float, - far: float, -) -> tf.Tensor: - """Converts OpenCV camera intrinsics to OpenGL view matrix. - - Args: - camera_to_image: Camera intrinsics matrix, (A1, ..., An, 3, 3). - width: Width of the image, in pixels. - height: Height of the image, in pixels. - near: Depth of the near clipping plane in meters (m). - far: Depth of the far clipping plane in meters (m). - - Returns: - A projection matrix, with shape (A1, ..., An, 4, 4), that follows the - OpenGL convention. - """ - fx = camera_to_image[..., 0, 0] - fy = camera_to_image[..., 1, 1] - focal_length = tf.stack([fx, fy], axis=-1) - principal_point = camera_to_image[..., :2, 2] - skew_coefficient = camera_to_image[..., 0, 1:2] - return opencv_intrinsics_to_opengl_view_matrix_tf( - focal_length, - principal_point, - width, - height, - near, - far, - skew_coefficient=skew_coefficient, - ) - - -def opencv_intrinsics_to_opengl_view_matrix_tf( - focal_length: tf.Tensor, - principal_point: tf.Tensor, - width: int, - height: int, - near: float, - far: float, - skew_coefficient: tf.Tensor | None = None, -) -> tf.Tensor: - """Converts OpenCV camera intrinsics to OpenGL view matrix using TensorFlow. - - See here for more information: - https://ksimek.github.io/2013/06/03/calibrated_cameras_in_opengl/ - - Args: - focal_length: Camera focal length, (A1, ...., An, 2). - principal_point: Camera center, (A1, ...., An, 2). - width: Width of the image. - height: Height of the image. - near: Depth of the near clipping plane. - far: Depth of the far clipping plane. - skew_coefficient: Optional skew coefficient, (A1, ...., An, 1). If not - given, it will be set to 0. - - Returns: - A projection matrix, with shape (A1, ..., An, 4, 4), that follows the - OpenGL convention. - """ - fx, fy = focal_length[..., :1], focal_length[..., 1:] - cx, cy = principal_point[..., :1], principal_point[..., 1:] - - zero = tf.zeros_like(fx) - ones = tf.ones_like(fx) - - if skew_coefficient is None: - skew_coefficient = zero - - near = ones * near - far = ones * far - near_minus_far = near - far - - # The next two rows apply the camera intrinsics to the XY coordinates and map - # the result to [-1, 1]. - row1 = [ - 2.0 / width * ones * fx, - -2.0 / width * skew_coefficient, - -(2 * cx / width - ones), - zero, - ] - row2 = [zero, 2.0 / height * ones * fy, (2 * cy / height - ones), zero] - - # Maps z coordinate values from (-near, -far) to (-1, 1). - row3 = [ - zero, - zero, - (far + near) / near_minus_far, - 2.0 * far * near / near_minus_far, - ] - row4 = [zero, zero, -ones, zero] - - view_matrix = tf.concat(row1 + row2 + row3 + row4, axis=-1) - - output_shape = tf.concat([focal_length.shape[:-1], [4, 4]], axis=0) - view_matrix = tf.reshape(view_matrix, output_shape) - return view_matrix diff --git a/gnm/shape/visualization/camera_conversions_test.py b/gnm/shape/visualization/camera_conversions_test.py index de95c072..dc492448 100644 --- a/gnm/shape/visualization/camera_conversions_test.py +++ b/gnm/shape/visualization/camera_conversions_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for camera_conversions.""" +"""Tests for NumPy OpenCV and OpenGL camera conversion functions.""" import tempfile from typing import TypeAlias @@ -25,7 +25,6 @@ import mediapy as media import numpy as np from scipy.spatial import transform -import tensorflow as tf _OPENCV_TO_OPENGL = np.diag([1.0, -1.0, -1.0, 1.0]).astype(dtype=np.float32) @@ -34,17 +33,6 @@ camera_conversions.opencv_intrinsics_to_opengl_view_matrix ) -_opencv_intrinsics_to_opengl_view_matrix_tf = ( - camera_conversions.opencv_intrinsics_to_opengl_view_matrix_tf -) - - -_opencv_extrinsics_to_opengl = camera_conversions.opencv_extrinsics_to_opengl - -_opencv_extrinsics_to_opengl_tf = ( - camera_conversions.opencv_extrinsics_to_opengl_tf -) - _Rotation: TypeAlias = transform.Rotation @@ -338,79 +326,6 @@ def test_opengl_to_opencv_intrinsics_consistency(self): np.testing.assert_allclose(as_opencv, opencv_camera_matrix) - # TF Parity Tests - - @parameterized.parameters([ - ((1,),), - ((5,),), - ((5, 4),), - ((5, 4, 3),), - ((5, 4, 3, 2),), - ((5, 4, 3, 2, 1),), - ]) - def test_opencv_to_opengl_intrinsics_tf_parity( - self, batch_dims: tuple[int, ...] - ): - """Tests that the TensorFlow implementation matches the NumPy one.""" - height, width = 256, 256 - - focal_length = np.full(batch_dims + (2,), height, dtype=np.float32) - principal_point = np.full( - batch_dims + (2,), [width * 0.5, height * 0.5], dtype=np.float32 - ) - - opengl_view_matrix_np = _opencv_intrinsics_to_opengl_view_matrix( - focal_length, - principal_point, - width=width, - height=height, - near=self.near, - far=self.far, - ) - opengl_view_matrix_tf = _opencv_intrinsics_to_opengl_view_matrix_tf( - tf.convert_to_tensor(focal_length, dtype=tf.float32), - tf.convert_to_tensor(principal_point, dtype=tf.float32), - width=width, - height=height, - near=self.near, - far=self.far, - ) - - np.testing.assert_allclose( - opengl_view_matrix_np, opengl_view_matrix_tf.numpy() - ) - - @parameterized.parameters([ - ((1,),), - ((5,),), - ((5, 4),), - ((5, 4, 3),), - ((5, 4, 3, 2),), - ((5, 4, 3, 2, 1),), - ]) - def test_opencv_to_opengl_extrinsics_tf_parity( - self, batch_dims: tuple[int, ...] - ): - """Tests that the TensorFlow implementation matches the NumPy one.""" - rotations = _Rotation.random(int(np.prod(batch_dims)), self.rng).as_matrix() - rotations = np.reshape(rotations, batch_dims + (3, 3)) - - translation = self.rng.uniform(-2, 2, size=batch_dims + (3,)) - - extrinsics = np.expand_dims(np.eye(4), tuple(range(len(batch_dims)))) - extrinsics = np.tile(extrinsics, batch_dims + (1, 1)) - extrinsics[..., :3, :3] = rotations[..., :3, :3] - extrinsics[..., 3, :3] = translation - - opengl_extrinsics_np = _opencv_extrinsics_to_opengl(extrinsics) - opengl_extrinsics_tf = _opencv_extrinsics_to_opengl_tf( - tf.convert_to_tensor(extrinsics, dtype=tf.float32) - ) - - np.testing.assert_allclose( - opengl_extrinsics_np, opengl_extrinsics_tf.numpy() - ) - def _rasterize_rectangle( self, rectangle_3d: np.ndarray, diff --git a/gnm/shape/visualization/camera_conversions_tf.py b/gnm/shape/visualization/camera_conversions_tf.py new file mode 100644 index 00000000..b0c4eb18 --- /dev/null +++ b/gnm/shape/visualization/camera_conversions_tf.py @@ -0,0 +1,158 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TensorFlow implementations of camera conversion functions.""" + +from gnm.shape.visualization import camera_conversions +import tensorflow as tf + +_OPENCV_TO_OPENGL = camera_conversions.OPENCV_TO_OPENGL +_OPENGL_TO_OPENCV = camera_conversions.OPENGL_TO_OPENCV + + +def opencv_extrinsics_to_opengl_tf(extrinsics: tf.Tensor) -> tf.Tensor: + """Converts OpenCV extrinsics to OpenGL using TensorFlow. + + Args: + extrinsics: The camera extrinsics or world-to-camera transformation matrix, + (A1, A2, ..., An, 4, 4). + + Returns: + Camera extrinsics in OpenGL coordinate system, with the same shape as input. + """ + return tf.einsum( + 'mk,...kn->...mn', + tf.convert_to_tensor(_OPENCV_TO_OPENGL, dtype=tf.float32), + extrinsics, + ) + + +def opengl_extrinsics_to_opencv_tf(extrinsics: tf.Tensor) -> tf.Tensor: + """Converts OpenGL extrinsics to OpenCV using TensorFlow. + + Args: + extrinsics: The camera extrinsics or world-to-camera transformation matrix, + (A1, A2, ..., An, 4, 4). + + Returns: + Camera extrinsics in OpenCV coordinate system, with the same shape as input. + """ + return tf.einsum( + 'mk,...kn->...mn', + tf.convert_to_tensor(_OPENGL_TO_OPENCV, dtype=tf.float32), + extrinsics, + ) + + +def opencv_intrinsics_matrix_to_opengl_view_matrix_tf( + camera_to_image: tf.Tensor, + width: int, + height: int, + near: float, + far: float, +) -> tf.Tensor: + """Converts OpenCV camera intrinsics to OpenGL view matrix. + + Args: + camera_to_image: Camera intrinsics matrix, (A1, ..., An, 3, 3). + width: Width of the image, in pixels. + height: Height of the image, in pixels. + near: Depth of the near clipping plane in meters (m). + far: Depth of the far clipping plane in meters (m). + + Returns: + A projection matrix, with shape (A1, ..., An, 4, 4), that follows the + OpenGL convention. + """ + fx = camera_to_image[..., 0, 0] + fy = camera_to_image[..., 1, 1] + focal_length = tf.stack([fx, fy], axis=-1) + principal_point = camera_to_image[..., :2, 2] + skew_coefficient = camera_to_image[..., 0, 1:2] + return opencv_intrinsics_to_opengl_view_matrix_tf( + focal_length, + principal_point, + width, + height, + near, + far, + skew_coefficient=skew_coefficient, + ) + + +def opencv_intrinsics_to_opengl_view_matrix_tf( + focal_length: tf.Tensor, + principal_point: tf.Tensor, + width: int, + height: int, + near: float, + far: float, + skew_coefficient: tf.Tensor | None = None, +) -> tf.Tensor: + """Converts OpenCV camera intrinsics to OpenGL view matrix using TensorFlow. + + See here for more information: + https://ksimek.github.io/2013/06/03/calibrated_cameras_in_opengl/ + + Args: + focal_length: Camera focal length, (A1, ...., An, 2). + principal_point: Camera center, (A1, ...., An, 2). + width: Width of the image. + height: Height of the image. + near: Depth of the near clipping plane. + far: Depth of the far clipping plane. + skew_coefficient: Optional skew coefficient, (A1, ...., An, 1). If not + given, it will be set to 0. + + Returns: + A projection matrix, with shape (A1, ..., An, 4, 4), that follows the + OpenGL convention. + """ + fx, fy = focal_length[..., :1], focal_length[..., 1:] + cx, cy = principal_point[..., :1], principal_point[..., 1:] + + zero = tf.zeros_like(fx) + ones = tf.ones_like(fx) + + if skew_coefficient is None: + skew_coefficient = zero + + near = ones * near + far = ones * far + near_minus_far = near - far + + # The next two rows apply the camera intrinsics to the XY coordinates and map + # the result to [-1, 1]. + row1 = [ + 2.0 / width * ones * fx, + -2.0 / width * skew_coefficient, + -(2 * cx / width - ones), + zero, + ] + row2 = [zero, 2.0 / height * ones * fy, (2 * cy / height - ones), zero] + + # Maps z coordinate values from (-near, -far) to (-1, 1). + row3 = [ + zero, + zero, + (far + near) / near_minus_far, + 2.0 * far * near / near_minus_far, + ] + row4 = [zero, zero, -ones, zero] + + view_matrix = tf.concat(row1 + row2 + row3 + row4, axis=-1) + + output_shape = tf.concat([focal_length.shape[:-1], [4, 4]], axis=0) + view_matrix = tf.reshape(view_matrix, output_shape) + return view_matrix diff --git a/gnm/shape/visualization/camera_conversions_tf_test.py b/gnm/shape/visualization/camera_conversions_tf_test.py new file mode 100644 index 00000000..53da0b5d --- /dev/null +++ b/gnm/shape/visualization/camera_conversions_tf_test.py @@ -0,0 +1,183 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for TensorFlow OpenCV and OpenGL camera conversion functions.""" + +from typing import TypeAlias + +from absl.testing import absltest +from absl.testing import parameterized +from gnm.shape.visualization import camera_conversions +from gnm.shape.visualization import camera_conversions_tf +import numpy as np +from scipy.spatial import transform +import tensorflow as tf + + +_opencv_intrinsics_to_opengl_view_matrix = ( + camera_conversions.opencv_intrinsics_to_opengl_view_matrix +) +_opencv_intrinsics_to_opengl_view_matrix_tf = ( + camera_conversions_tf.opencv_intrinsics_to_opengl_view_matrix_tf +) + +_opencv_intrinsics_matrix_to_opengl_view_matrix = ( + camera_conversions.opencv_intrinsics_matrix_to_opengl_view_matrix +) +_opencv_intrinsics_matrix_to_opengl_view_matrix_tf = ( + camera_conversions_tf.opencv_intrinsics_matrix_to_opengl_view_matrix_tf +) + +_opencv_extrinsics_to_opengl = camera_conversions.opencv_extrinsics_to_opengl +_opencv_extrinsics_to_opengl_tf = ( + camera_conversions_tf.opencv_extrinsics_to_opengl_tf +) + +_opengl_extrinsics_to_opencv = camera_conversions.opengl_extrinsics_to_opencv +_opengl_extrinsics_to_opencv_tf = ( + camera_conversions_tf.opengl_extrinsics_to_opencv_tf +) + +_Rotation: TypeAlias = transform.Rotation + +_BATCH_DIMS = ( + (1,), + (5,), + (5, 4), + (5, 4, 3), + (5, 4, 3, 2), + (5, 4, 3, 2, 1), +) + + +class CameraConversionsTfTest(parameterized.TestCase): + + def setUp(self): + super().setUp() + self.rng = np.random.default_rng(43) + self.near = 0.01 + self.far = 20 + + def _random_extrinsics(self, batch_dims: tuple[int, ...]) -> np.ndarray: + rotations = _Rotation.random(int(np.prod(batch_dims)), self.rng).as_matrix() + rotations = np.reshape(rotations, batch_dims + (3, 3)) + translation = self.rng.uniform(-2, 2, size=batch_dims + (3,)) + + extrinsics = np.expand_dims(np.eye(4), tuple(range(len(batch_dims)))) + extrinsics = np.tile(extrinsics, batch_dims + (1, 1)) + extrinsics[..., :3, :3] = rotations[..., :3, :3] + extrinsics[..., :3, 3] = translation + return extrinsics.astype(np.float32) + + @parameterized.parameters([(dims,) for dims in _BATCH_DIMS]) + def test_opencv_to_opengl_intrinsics_tf_parity( + self, batch_dims: tuple[int, ...] + ): + """Tests that the TensorFlow implementation matches the NumPy one.""" + height, width = 256, 256 + + focal_length = np.full(batch_dims + (2,), height, dtype=np.float32) + principal_point = np.full( + batch_dims + (2,), [width * 0.5, height * 0.5], dtype=np.float32 + ) + + opengl_view_matrix_np = _opencv_intrinsics_to_opengl_view_matrix( + focal_length, + principal_point, + width=width, + height=height, + near=self.near, + far=self.far, + ) + opengl_view_matrix_tf = _opencv_intrinsics_to_opengl_view_matrix_tf( + tf.convert_to_tensor(focal_length, dtype=tf.float32), + tf.convert_to_tensor(principal_point, dtype=tf.float32), + width=width, + height=height, + near=self.near, + far=self.far, + ) + + np.testing.assert_allclose( + opengl_view_matrix_np, opengl_view_matrix_tf.numpy() + ) + + @parameterized.parameters([(dims,) for dims in _BATCH_DIMS]) + def test_opencv_intrinsics_matrix_to_opengl_view_matrix_tf_parity( + self, batch_dims: tuple[int, ...] + ): + """Tests that the matrix intrinsics TF helper matches the NumPy one.""" + height, width = 256, 256 + camera_to_image = np.broadcast_to( + np.array( + [[256.0, 12.5, 128.0], [0.0, 256.0, 128.0], [0.0, 0.0, 1.0]], + dtype=np.float32, + ), + batch_dims + (3, 3), + ) + + opengl_view_matrix_np = _opencv_intrinsics_matrix_to_opengl_view_matrix( + camera_to_image, + width=width, + height=height, + near=self.near, + far=self.far, + ) + opengl_view_matrix_tf = _opencv_intrinsics_matrix_to_opengl_view_matrix_tf( + tf.convert_to_tensor(camera_to_image, dtype=tf.float32), + width=width, + height=height, + near=self.near, + far=self.far, + ) + + np.testing.assert_allclose( + opengl_view_matrix_np, opengl_view_matrix_tf.numpy() + ) + + @parameterized.parameters([(dims,) for dims in _BATCH_DIMS]) + def test_opencv_to_opengl_extrinsics_tf_parity( + self, batch_dims: tuple[int, ...] + ): + """Tests that the TensorFlow implementation matches the NumPy one.""" + extrinsics = self._random_extrinsics(batch_dims) + + opengl_extrinsics_np = _opencv_extrinsics_to_opengl(extrinsics) + opengl_extrinsics_tf = _opencv_extrinsics_to_opengl_tf( + tf.convert_to_tensor(extrinsics, dtype=tf.float32) + ) + + np.testing.assert_allclose( + opengl_extrinsics_np, opengl_extrinsics_tf.numpy() + ) + + @parameterized.parameters([(dims,) for dims in _BATCH_DIMS]) + def test_opengl_to_opencv_extrinsics_tf_parity( + self, batch_dims: tuple[int, ...] + ): + """Tests that OpenGL-to-OpenCV extrinsics in TF match NumPy.""" + extrinsics = self._random_extrinsics(batch_dims) + + opencv_extrinsics_np = _opengl_extrinsics_to_opencv(extrinsics) + opencv_extrinsics_tf = _opengl_extrinsics_to_opencv_tf( + tf.convert_to_tensor(extrinsics, dtype=tf.float32) + ) + + np.testing.assert_allclose( + opencv_extrinsics_np, opencv_extrinsics_tf.numpy() + ) + + +if __name__ == "__main__": + absltest.main() diff --git a/gnm/shape/visualization/render_common.py b/gnm/shape/visualization/render_common.py index 69c27417..39b80b13 100644 --- a/gnm/shape/visualization/render_common.py +++ b/gnm/shape/visualization/render_common.py @@ -800,8 +800,7 @@ def load_edgeflow_texture(texture_path: epath.Path | None) -> FloatArray | None: """ if texture_path is None: return None - with texture_path.open('rb') as f: - image = imageio.imread(f).astype(np.float32) + image = imageio.imread(texture_path.read_bytes()).astype(np.float32) image = (image / np.iinfo(np.uint8).max) * 0.5 + 0.5 image.flags.writeable = False return image