Epic 106: add video recognition substrate

CHANGELOG.md

@@ -21,6 +21,12 @@ removed no sooner than the next major (see `docs/API_STABILITY.md`).
 
 ### Added
 
+- **Video recognition substrate (Epic 106)**: dense integer block flow,
+  adaptive Gaussian foreground segmentation, deterministic same-class IoU
+  tracking, and timestamped pull-based video source contracts. Codec/camera
+  adapters remain feature-gated; Python dense flow, OpenCV Farneback comparison,
+  and QQVGA/QVGA Criterion workloads cover the portable core.
+
 - **Camera calibration contracts (Epic 105)**: robust pinhole intrinsics,
   Kannala–Brandt4 fisheye fitting, supplied-rotation stereo and hand-eye
   translation solves, and fixed-camera sparse point bundle adjustment. All

bench/opencv_comparison/manifest.json

@@ -25,6 +25,9 @@
       { "id": "stereo_bm", "domain": "calib3d", "modes": ["allocate"] },
       { "id": "pinhole_calibration", "domain": "calib3d", "modes": ["allocate"] },
       { "id": "fisheye_calibration", "domain": "calib3d", "modes": ["allocate"] },
+      { "id": "dense_optical_flow", "domain": "video", "modes": ["allocate"] },
+      { "id": "background_model", "domain": "video", "modes": ["streaming"] },
+      { "id": "multi_object_tracking", "domain": "video", "modes": ["streaming"] },
     { "id": "depth_to_xyz", "domain": "rgbd", "modes": ["allocate", "reuse"] },
     { "id": "rgbd_to_point_cloud", "domain": "spatial-e2e", "modes": ["allocate"] },
     { "id": "ai_preprocess", "domain": "dnn-adapter", "modes": ["allocate", "reuse"] },

bench/opencv_comparison/run.py

@@ -19,6 +19,7 @@
     / "opencv_vision_comparison"
     / "performance.py",
     "rgbd": ROOT / "bench" / "opencv_rgbd_comparison" / "run.py",
+    "video": ROOT / "bench" / "opencv_video_comparison" / "run.py",
 }
 
 

bench/opencv_video_comparison/README.md

@@ -0,0 +1,9 @@
+# OpenCV video comparison
+
+This suite compares SpatialRust dense integer block flow with OpenCV Farneback
+flow on a deterministic translated texture:
+
+```bash
+python bench/opencv_video_comparison/run.py \
+  --output target/opencv-comparison/video.json
+```

bench/opencv_video_comparison/run.py

@@ -0,0 +1,81 @@
+"""Dense-flow correctness comparison for Epic 106 video recognition."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import cv2
+import numpy as np
+import spatialrust as sr
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from opencv_comparison.report import emit_report, environment, make_report
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=Path)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    height, width = 96, 128
+    dx, dy = 3.0, 2.0
+    yy, xx = np.indices((height, width), dtype=np.int32)
+    previous = ((xx * 17 + yy * 29 + xx * yy * 3) % 251).astype(np.uint8)
+    transform = np.array([[1.0, 0.0, dx], [0.0, 1.0, dy]], dtype=np.float32)
+    next_frame = cv2.warpAffine(
+        previous,
+        transform,
+        (width, height),
+        flags=cv2.INTER_NEAREST,
+        borderMode=cv2.BORDER_CONSTANT,
+    )
+    spatialrust_flow = sr.dense_flow_image(previous, next_frame, search_radius=4)
+    opencv_flow = cv2.calcOpticalFlowFarneback(
+        previous,
+        next_frame,
+        None,
+        pyr_scale=0.5,
+        levels=3,
+        winsize=15,
+        iterations=5,
+        poly_n=5,
+        poly_sigma=1.1,
+        flags=0,
+    )
+    region = np.s_[20:-20, 20:-20]
+    spatialrust_median = np.nanmedian(spatialrust_flow[region], axis=(0, 1))
+    opencv_median = np.median(opencv_flow[region], axis=(0, 1))
+    expected = np.array([dx, dy])
+    spatialrust_error = float(np.max(np.abs(spatialrust_median - expected)))
+    opencv_error = float(np.max(np.abs(opencv_median - expected)))
+    cross_error = float(np.max(np.abs(spatialrust_median - opencv_median)))
+    status = "pass" if spatialrust_error <= 0.01 and cross_error <= 0.5 else "fail"
+    report = make_report(
+        suite="opencv-video",
+        kind="correctness",
+        status=status,
+        environment_receipt=environment(
+            opencv_version=cv2.__version__, spatialrust_version=sr.__version__
+        ),
+        results={
+            "translation": [dx, dy],
+            "spatialrust_median_flow": spatialrust_median.tolist(),
+            "opencv_median_flow": opencv_median.tolist(),
+            "spatialrust_max_error": spatialrust_error,
+            "opencv_max_error": opencv_error,
+            "cross_max_error": cross_error,
+            "thresholds": {"spatialrust_max_error": 0.01, "cross_max_error": 0.5},
+        },
+    )
+    emit_report(report, args.output)
+    if status != "pass":
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

crates/spatialrust-platform/src/stability.rs

@@ -118,6 +118,7 @@ impl StabilityRegistry {
             "spatialrust-vision::geometry",
             "spatialrust-vision::stereo",
             "spatialrust-vision::optical-flow",
+            "spatialrust-vision::video",
             "spatialrust-vision::ai-adapters",
             "spatialrust-gpu::GpuImage",
         ];

crates/spatialrust-py/spatialrust.pyi

@@ -28,7 +28,7 @@ __all__: list[str] = [
     "apply_transform", "recenter", "scale", "normalize_unit_sphere", "merge",
     "centroid", "bounding_box", "oriented_bounding_box", "voxelize", "range_image",
     "rgbd_to_point_cloud", "depth_to_xyz", "calibrate_pinhole_camera",
-    "calibrate_fisheye_angles", "filter2d_image", "gaussian_blur_image",
+    "calibrate_fisheye_angles", "dense_flow_image", "filter2d_image", "gaussian_blur_image",
     "median_blur_image", "bilateral_filter_image", "sobel_image", "scharr_image",
     "laplacian_image", "pyr_down_image", "pyr_up_image", "morphology_image",
     "threshold_image", "otsu_threshold_image", "adaptive_threshold_image",
@@ -129,6 +129,12 @@ def calibrate_pinhole_camera(
 def calibrate_fisheye_angles(
     theta: _F64Array, distorted_radius: _F64Array
 ) -> tuple[float, float, float, float, float]: ...
+def dense_flow_image(
+    previous: _U8Array,
+    next: _U8Array,
+    block_radius: int = ...,
+    search_radius: int = ...,
+) -> _F32Array: ...
 
 def rgbd_to_point_cloud(
     depth: _F32Array,

crates/spatialrust-py/src/lib.rs

@@ -95,6 +95,7 @@ use spatialrust::vision::{
     OrbScoreType, PointCorrespondence2, PointMap, RleOrder, RobustEstimationOptions,
     ShiTomasiOptions, SoftNmsMethod, StereoBmOptions, StructuringElement, ThresholdType,
 };
+use spatialrust::vision::{dense_flow_block_match as dense_flow_native, DenseFlowOptions};
 use spatialrust::voxelize::{
     range_image as range_image_proj, voxelize as voxelize_grid, RangeImageConfig, VoxelFill,
     VoxelGridConfig,
@@ -760,6 +761,32 @@ fn calibrate_fisheye_angles(
     Ok((model.k1, model.k2, model.k3, model.k4, report.rms_residual))
 }
 
+/// Computes dense integer grayscale flow as an `(H, W, 2)` float32 array.
+#[pyfunction]
+#[pyo3(signature = (previous, next, block_radius=2, search_radius=4))]
+fn dense_flow_image<'py>(
+    py: Python<'py>,
+    previous: PyReadonlyArray2<'_, u8>,
+    next: PyReadonlyArray2<'_, u8>,
+    block_radius: usize,
+    search_radius: usize,
+) -> PyResult<Bound<'py, PyArray3<f32>>> {
+    let previous = gray_u8_image_from_numpy(previous)?;
+    let next = gray_u8_image_from_numpy(next)?;
+    let flow = dense_flow_native(
+        previous.view(),
+        next.view(),
+        DenseFlowOptions { block_radius, search_radius, minimum_improvement: 1 },
+    )
+    .map_err(to_py_err)?;
+    let array = Array3::from_shape_vec(
+        (previous.height(), previous.width(), 2),
+        flow.image().as_slice().to_vec(),
+    )
+    .map_err(to_py_err)?;
+    Ok(array.into_pyarray_bound(py))
+}
+
 fn cloud_from_xyz(arr: PyReadonlyArray2<'_, f32>) -> PyResult<PointCloud> {
     let view = arr.as_array();
     let shape = view.shape();
@@ -3142,6 +3169,7 @@ fn spatialrust_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(rgbd_to_point_cloud, m)?)?;
     m.add_function(wrap_pyfunction!(calibrate_pinhole_camera, m)?)?;
     m.add_function(wrap_pyfunction!(calibrate_fisheye_angles, m)?)?;
+    m.add_function(wrap_pyfunction!(dense_flow_image, m)?)?;
     m.add_function(wrap_pyfunction!(filter2d_image, m)?)?;
     m.add_function(wrap_pyfunction!(gaussian_blur_image, m)?)?;
     m.add_function(wrap_pyfunction!(median_blur_image, m)?)?;

crates/spatialrust-py/tests/test_bindings.py

@@ -771,3 +771,15 @@ def test_calibration_bindings_recover_intrinsics_and_fisheye():
     fitted = sr.calibrate_fisheye_angles(theta, radius)
     np.testing.assert_allclose(fitted[:4], expected, atol=1e-9)
     assert fitted[4] < 1e-12
+
+
+def test_dense_flow_binding_recovers_translation_and_marks_border_invalid():
+    height, width = 32, 40
+    yy, xx = np.indices((height, width), dtype=np.int32)
+    previous = ((xx * 17 + yy * 29 + xx * yy * 3) % 251).astype(np.uint8)
+    next_frame = np.zeros_like(previous)
+    next_frame[1:, 2:] = previous[:-1, :-2]
+    flow = sr.dense_flow_image(previous, next_frame)
+    assert flow.shape == (height, width, 2)
+    np.testing.assert_array_equal(flow[16, 20], [2.0, 1.0])
+    assert np.isnan(flow[0, 0]).all()

crates/spatialrust-vision/Cargo.toml

@@ -23,7 +23,9 @@ detection = []
 dense = ["detection"]
 spatial = ["dense", "dep:spatialrust-core", "dep:spatialrust-camera"]
 ai-adapters = ["preprocess", "dense", "detection", "dep:spatialrust-tensor", "dep:bytemuck", "spatialrust-tensor/image"]
-full = ["preprocess", "warp", "imgproc-filter", "imgproc-morphology", "imgproc-analysis", "imgproc-canny", "feature2d", "geometry", "detection", "dense", "spatial", "ai-adapters"]
+video = ["dense", "detection"]
+video-adapters = ["video"]
+full = ["preprocess", "warp", "imgproc-filter", "imgproc-morphology", "imgproc-analysis", "imgproc-canny", "feature2d", "geometry", "detection", "dense", "spatial", "ai-adapters", "video-adapters"]
 
 [dependencies]
 spatialrust-image.workspace = true
@@ -77,3 +79,8 @@ required-features = ["feature2d"]
 name = "geometry"
 harness = false
 required-features = ["geometry"]
+
+[[bench]]
+name = "video"
+harness = false
+required-features = ["video"]

crates/spatialrust-vision/benches/video.rs

@@ -0,0 +1,32 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use spatialrust_image::Image;
+use spatialrust_vision::{dense_flow_block_match, DenseFlowOptions};
+
+fn benchmark_video(c: &mut Criterion) {
+    for &(name, width, height) in &[("qqvga", 160, 120), ("qvga", 320, 240)] {
+        let previous = Image::<u8, 1>::try_new(
+            width,
+            height,
+            (0..width * height).map(|index| ((index * 37) % 251) as u8).collect(),
+        )
+        .unwrap();
+        let next = previous.clone();
+        let mut group = c.benchmark_group("dense_flow_block_match");
+        group.sample_size(10);
+        group.throughput(Throughput::Elements((width * height) as u64));
+        group.bench_function(BenchmarkId::from_parameter(name), |b| {
+            b.iter(|| {
+                dense_flow_block_match(
+                    black_box(previous.view()),
+                    black_box(next.view()),
+                    DenseFlowOptions::default(),
+                )
+                .unwrap()
+            });
+        });
+        group.finish();
+    }
+}
+
+criterion_group!(benches, benchmark_video);
+criterion_main!(benches);

crates/spatialrust-vision/src/lib.rs

@@ -53,6 +53,8 @@ mod spatial;
 mod stereo;
 #[cfg(feature = "warp")]
 mod warp;
+#[cfg(feature = "video")]
+mod video;
 
 pub use border::BorderMode;
 pub use error::{VisionError, VisionResult};
@@ -101,3 +103,5 @@ pub use spatial::*;
 pub use stereo::*;
 #[cfg(feature = "warp")]
 pub use warp::*;
+#[cfg(feature = "video")]
+pub use video::*;