Skip to content

Split camera_conversions into Numpy and TensorFlow modules #187

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Split camera_conversions into Numpy and TensorFlow modules #187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 0 additions & 122 deletions gnm/shape/visualization/camera_conversions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
"""

import numpy as np
import tensorflow as tf

# Rotate OpenCV's camera coordinate system 180 degrees around the X-axis to
# align it with the OpenGL coordinate frame.
Expand Down Expand Up @@ -200,124 +199,3 @@ def opencv_intrinsics_matrix_to_opengl_view_matrix(
far,
skew_coefficient=skew_coefficient[..., np.newaxis],
)


# TensorFlow implementations
def opencv_extrinsics_to_opengl_tf(extrinsics: tf.Tensor) -> tf.Tensor:
"""Converts OpenCV extrinsics to OpenGL using TensorFlow.

Args:
extrinsics: The camera extrinsics or world-to-camera transformation matrix,
(A1, A2, ..., An, 4, 4).

Returns:
Camera extrinsics in OpenGL coordinate system, with the same shape as input.
"""
return tf.einsum(
'mk,...kn->...mn',
tf.convert_to_tensor(OPENCV_TO_OPENGL, dtype=tf.float32),
extrinsics,
)


def opencv_intrinsics_matrix_to_opengl_view_matrix_tf(
camera_to_image: tf.Tensor,
width: int,
height: int,
near: float,
far: float,
) -> tf.Tensor:
"""Converts OpenCV camera intrinsics to OpenGL view matrix.

Args:
camera_to_image: Camera intrinsics matrix, (A1, ..., An, 3, 3).
width: Width of the image, in pixels.
height: Height of the image, in pixels.
near: Depth of the near clipping plane in meters (m).
far: Depth of the far clipping plane in meters (m).

Returns:
A projection matrix, with shape (A1, ..., An, 4, 4), that follows the
OpenGL convention.
"""
fx = camera_to_image[..., 0, 0]
fy = camera_to_image[..., 1, 1]
focal_length = tf.stack([fx, fy], axis=-1)
principal_point = camera_to_image[..., :2, 2]
skew_coefficient = camera_to_image[..., 0, 1:2]
return opencv_intrinsics_to_opengl_view_matrix_tf(
focal_length,
principal_point,
width,
height,
near,
far,
skew_coefficient=skew_coefficient,
)


def opencv_intrinsics_to_opengl_view_matrix_tf(
focal_length: tf.Tensor,
principal_point: tf.Tensor,
width: int,
height: int,
near: float,
far: float,
skew_coefficient: tf.Tensor | None = None,
) -> tf.Tensor:
"""Converts OpenCV camera intrinsics to OpenGL view matrix using TensorFlow.

See here for more information:
https://ksimek.github.io/2013/06/03/calibrated_cameras_in_opengl/

Args:
focal_length: Camera focal length, (A1, ...., An, 2).
principal_point: Camera center, (A1, ...., An, 2).
width: Width of the image.
height: Height of the image.
near: Depth of the near clipping plane.
far: Depth of the far clipping plane.
skew_coefficient: Optional skew coefficient, (A1, ...., An, 1). If not
given, it will be set to 0.

Returns:
A projection matrix, with shape (A1, ..., An, 4, 4), that follows the
OpenGL convention.
"""
fx, fy = focal_length[..., :1], focal_length[..., 1:]
cx, cy = principal_point[..., :1], principal_point[..., 1:]

zero = tf.zeros_like(fx)
ones = tf.ones_like(fx)

if skew_coefficient is None:
skew_coefficient = zero

near = ones * near
far = ones * far
near_minus_far = near - far

# The next two rows apply the camera intrinsics to the XY coordinates and map
# the result to [-1, 1].
row1 = [
2.0 / width * ones * fx,
-2.0 / width * skew_coefficient,
-(2 * cx / width - ones),
zero,
]
row2 = [zero, 2.0 / height * ones * fy, (2 * cy / height - ones), zero]

# Maps z coordinate values from (-near, -far) to (-1, 1).
row3 = [
zero,
zero,
(far + near) / near_minus_far,
2.0 * far * near / near_minus_far,
]
row4 = [zero, zero, -ones, zero]

view_matrix = tf.concat(row1 + row2 + row3 + row4, axis=-1)

output_shape = tf.concat([focal_length.shape[:-1], [4, 4]], axis=0)
view_matrix = tf.reshape(view_matrix, output_shape)
return view_matrix
87 changes: 1 addition & 86 deletions gnm/shape/visualization/camera_conversions_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for camera_conversions."""
"""Tests for NumPy OpenCV and OpenGL camera conversion functions."""

import tempfile
from typing import TypeAlias
Expand All @@ -25,7 +25,6 @@
import mediapy as media
import numpy as np
from scipy.spatial import transform
import tensorflow as tf


_OPENCV_TO_OPENGL = np.diag([1.0, -1.0, -1.0, 1.0]).astype(dtype=np.float32)
Expand All @@ -34,17 +33,6 @@
camera_conversions.opencv_intrinsics_to_opengl_view_matrix
)

_opencv_intrinsics_to_opengl_view_matrix_tf = (
camera_conversions.opencv_intrinsics_to_opengl_view_matrix_tf
)


_opencv_extrinsics_to_opengl = camera_conversions.opencv_extrinsics_to_opengl

_opencv_extrinsics_to_opengl_tf = (
camera_conversions.opencv_extrinsics_to_opengl_tf
)


_Rotation: TypeAlias = transform.Rotation

Expand Down Expand Up @@ -338,79 +326,6 @@ def test_opengl_to_opencv_intrinsics_consistency(self):

np.testing.assert_allclose(as_opencv, opencv_camera_matrix)

# TF Parity Tests

@parameterized.parameters([
((1,),),
((5,),),
((5, 4),),
((5, 4, 3),),
((5, 4, 3, 2),),
((5, 4, 3, 2, 1),),
])
def test_opencv_to_opengl_intrinsics_tf_parity(
self, batch_dims: tuple[int, ...]
):
"""Tests that the TensorFlow implementation matches the NumPy one."""
height, width = 256, 256

focal_length = np.full(batch_dims + (2,), height, dtype=np.float32)
principal_point = np.full(
batch_dims + (2,), [width * 0.5, height * 0.5], dtype=np.float32
)

opengl_view_matrix_np = _opencv_intrinsics_to_opengl_view_matrix(
focal_length,
principal_point,
width=width,
height=height,
near=self.near,
far=self.far,
)
opengl_view_matrix_tf = _opencv_intrinsics_to_opengl_view_matrix_tf(
tf.convert_to_tensor(focal_length, dtype=tf.float32),
tf.convert_to_tensor(principal_point, dtype=tf.float32),
width=width,
height=height,
near=self.near,
far=self.far,
)

np.testing.assert_allclose(
opengl_view_matrix_np, opengl_view_matrix_tf.numpy()
)

@parameterized.parameters([
((1,),),
((5,),),
((5, 4),),
((5, 4, 3),),
((5, 4, 3, 2),),
((5, 4, 3, 2, 1),),
])
def test_opencv_to_opengl_extrinsics_tf_parity(
self, batch_dims: tuple[int, ...]
):
"""Tests that the TensorFlow implementation matches the NumPy one."""
rotations = _Rotation.random(int(np.prod(batch_dims)), self.rng).as_matrix()
rotations = np.reshape(rotations, batch_dims + (3, 3))

translation = self.rng.uniform(-2, 2, size=batch_dims + (3,))

extrinsics = np.expand_dims(np.eye(4), tuple(range(len(batch_dims))))
extrinsics = np.tile(extrinsics, batch_dims + (1, 1))
extrinsics[..., :3, :3] = rotations[..., :3, :3]
extrinsics[..., 3, :3] = translation

opengl_extrinsics_np = _opencv_extrinsics_to_opengl(extrinsics)
opengl_extrinsics_tf = _opencv_extrinsics_to_opengl_tf(
tf.convert_to_tensor(extrinsics, dtype=tf.float32)
)

np.testing.assert_allclose(
opengl_extrinsics_np, opengl_extrinsics_tf.numpy()
)

def _rasterize_rectangle(
self,
rectangle_3d: np.ndarray,
Expand Down
Loading
Loading