fix(reader): decode fastlanes.delta into the arena, not four heap long[]

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- A malformed `fastlanes.delta` column no longer fails with a raw JDK exception: a row window running past the elements the chunks reconstruct threw `ArrayIndexOutOfBoundsException`, and an absurd or negative declared element count sized a heap array before anything checked it (`NegativeArraySizeException`, or `OutOfMemoryError`). All now fail as `VortexException`. ([#338](https://github.com/dfa1/vortex-java/issues/338))
+- A `fastlanes.delta` column no longer routes its decode through four row-scaled heap `long[]` arrays, every value widened to 8 bytes whatever the column's width; values are reconstructed into a single arena segment at the ptype's real width, and only the chunks overlapping the requested rows are reconstructed at all. ([#338](https://github.com/dfa1/vortex-java/issues/338))
+
 - A run-end-encoded Utf8/Binary column (`vortex.runend`) no longer expands every run into a fully materialized buffer on decode; rows now resolve through the runs lazily, removing an unbounded `sum(runLength * valueLength)` allocation that a crafted file could drive to `OutOfMemoryError`. ([#334](https://github.com/dfa1/vortex-java/issues/334))
 - A `vortex.sequence` column no longer materializes `base + i * multiplier` into a full buffer on decode; rows are computed on access, so the encoding allocates nothing regardless of row count — closing an `OutOfMemoryError` risk from a metadata-only encoding whose row count no buffer bounds. ([#335](https://github.com/dfa1/vortex-java/issues/335))
 - A primitive `vortex.dict` column decoded through the encoding path no longer expands its codes into an `n * elemSize` buffer; it now returns the same lazy `DictXxxArray` carriers the layout path already used, so a dict column keeps the dictionary's memory benefit however it is reached. ([#336](https://github.com/dfa1/vortex-java/issues/336))

docs/compatibility.md

@@ -159,7 +159,7 @@ decoder falls into one of three shapes:
 | `vortex.datetimeparts`      | Lazy          | Lazy          | `LazyDateTimePartsLongArray` — reassembles parts on access               |
 | `vortex.pco`                | Materialized  | Materialized  | range-encoded decompression                                              |
 | `fastlanes.bitpacked`       | Materialized  | Materialized  | window unpacks bits                                                      |
-| `fastlanes.delta`           | Materialized  | Materialized  | cumulative sum requires sequential decode                                |
+| `fastlanes.delta`           | Materialized  | Materialized  | cumulative sum requires sequential decode; output is one arena segment at the ptype's width, and only the chunks the row window touches are reconstructed |
 | `fastlanes.for`             | Lazy          | Lazy          | `LazyForXxxArray` (I8/U8/I16/U16/I32/U32/I64/U64), ADR 0010 + 0015      |
 | `fastlanes.rle`             | Lazy          | Lazy          | `LazyRleXxxArray`; validity → `OffsetBoolArray`; empty → `LazyConstantXxxArray`, ADR 0015 |
 | `vortex.patched`            | Materialized  | Materialized  | inner is full base + chunked patches (1024-elem blocks, lane-window-sorted); per-row access requires 2 laneOffsets reads + binary search inside the chunk window, so eager scatter wins for full scans |

integration/src/test/java/io/github/dfa1/vortex/integration/JavaRoundTripIntegrationTest.java

@@ -1,34 +1,61 @@
 package io.github.dfa1.vortex.integration;
 
+import io.github.dfa1.vortex.core.compute.FastLanes;
 import io.github.dfa1.vortex.core.model.ColumnName;
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.model.Editions;
+import io.github.dfa1.vortex.core.model.PType;
+import io.github.dfa1.vortex.core.model.EncodingId;
+import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata;
+import io.github.dfa1.vortex.inspect.InspectorTree;
 import io.github.dfa1.vortex.reader.ReadRegistry;
 import io.github.dfa1.vortex.reader.ScanOptions;
 import io.github.dfa1.vortex.reader.VortexReader;
+import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.ByteArray;
 import io.github.dfa1.vortex.reader.array.IntArray;
+import io.github.dfa1.vortex.reader.array.LongArray;
+import io.github.dfa1.vortex.reader.array.ShortArray;
+import io.github.dfa1.vortex.reader.decode.ArrayNode;
+import io.github.dfa1.vortex.reader.decode.DecodeContext;
+import io.github.dfa1.vortex.reader.decode.DeltaEncodingDecoder;
 import io.github.dfa1.vortex.writer.VortexWriter;
 import io.github.dfa1.vortex.writer.WriteOptions;
+import io.github.dfa1.vortex.writer.WriteRegistry;
+import io.github.dfa1.vortex.writer.encode.EncodeContext;
+import io.github.dfa1.vortex.writer.encode.EncodeResult;
+import io.github.dfa1.vortex.writer.encode.DeltaEncodingEncoder;
 import io.github.dfa1.vortex.writer.encode.PatchedEncodingEncoder;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
 
 import java.io.IOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
 import java.nio.channels.FileChannel;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-/// Java writer → Java reader round-trips for encodings the bundled `vortex-jni` build cannot read
-/// back, so they have no Java→Rust coverage. This is still a real cross-module integration test: it
-/// drives the writer's encode, the on-disk file format, and the reader's decode end to end.
+/// Java writer → Java reader round-trips for encodings whose Java *decode* has no other end-to-end
+/// cover. This is a real cross-module integration test either way: it drives the writer's encode,
+/// the on-disk file format, and the reader's decode end to end.
 ///
-/// `vortex.patched` is the case here — the JNI reader rejects a standalone patched array with
-/// "Unknown encoding: vortex.patched", so the round-trip is asserted on the Java side instead.
+/// Two reasons land a case here:
+/// - the bundled `vortex-jni` build cannot read the encoding back, so there is no Java→Rust test.
+///   `vortex.patched` is this case — the JNI reader rejects a standalone patched array with
+///   "Unknown encoding: vortex.patched".
+/// - Java→Rust cover exists but only exercises the *encoder*. `fastlanes.delta` is this case:
+///   `JavaWritesRustReadsIntegrationTest#javaWriter_rustReader_delta_i64` proves what Java writes
+///   is readable, and says nothing about `DeltaEncodingDecoder`.
 class JavaRoundTripIntegrationTest {
 
     private static final DType.Struct I32_SCHEMA = new DType.Struct(
@@ -62,6 +89,152 @@ void patched_i32_javaWriteJavaRead(@TempDir Path tmp) throws IOException {
         assertThat(decoded).containsExactly(data);
     }
 
+    /// `fastlanes.delta` decode across every width it accepts, over three FastLanes chunks.
+    ///
+    /// The unit tests reach the decoder only with I64 and single-element (constant) children, so
+    /// nothing covered the per-width read and write paths, and nothing covered more than one
+    /// chunk — which is where the chunk-window arithmetic lives. Values are full-width random
+    /// bit patterns, not a monotonic ramp: the high bit is exactly where a read that
+    /// sign-extends and one that zero-extends diverge, and delta round-trips any values at all
+    /// since encode and decode both wrap modulo the type width.
+    @ParameterizedTest
+    @EnumSource(value = PType.class, names = {"I8", "I16", "I32", "I64", "U8", "U16", "U32", "U64"})
+    void delta_javaWriteJavaRead(PType ptype, @TempDir Path tmp) throws IOException {
+        // Given — 2500 rows is three 1024-element chunks, the last one padded.
+        long mask = FastLanes.lowMask(ptype.bits());
+        Random rng = new Random(338);
+        long[] expected = new long[2500];
+        for (int i = 0; i < expected.length; i++) {
+            expected[i] = rng.nextLong() & mask;
+        }
+        DType.Struct schema = new DType.Struct(List.of(ColumnName.of("v")),
+                List.of(new DType.Primitive(ptype, false)), false);
+        Path file = tmp.resolve("java_delta_" + ptype + ".vtx");
+
+        // When
+        try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+             var sut = VortexWriter.create(ch, schema,
+                     WriteOptions.defaults().withEdition(Editions.UNSTABLE_2025_05_0),
+                     List.of(new DeltaEncodingEncoder()))) {
+            sut.writeChunk(Map.of(ColumnName.of("v"), narrow(expected, ptype)));
+        }
+
+        // Then — the encoding is asserted too, so a writer that quietly stopped choosing delta
+        // would fail here rather than leave the decoder untested
+        try (var reader = VortexReader.open(file, ReadRegistry.loadAll())) {
+            assertThat(InspectorTree.build(reader).usedEncodings()).contains("fastlanes.delta");
+        }
+        // compared as stored bit patterns, so signed and unsigned widths assert alike
+        assertThat(readColumnBits(file, "v", mask)).containsExactly(expected);
+    }
+
+    /// `fastlanes.delta`'s `offset` metadata — which makes a decode start partway into the
+    /// reconstructed elements — has no round-trip cover, because the Java writer always emits 0;
+    /// a non-zero offset only ever arrives on a Rust-written sliced array. So this drives the
+    /// decoder directly over encoder-produced children instead of through a file, and asserts
+    /// the window is exactly the corresponding slice of the full decode. The window arithmetic
+    /// (which chunks to reconstruct, and where each lands in the output) is the part of decode
+    /// that only a non-zero offset reaches.
+    @Test
+    void delta_offsetWindowIsTheSliceOfTheFullDecode() {
+        // Given — 2500 rows, so the encoder pads to three chunks
+        DType dtype = new DType.Primitive(PType.I64, false);
+        Random rng = new Random(3381);
+        long[] data = new long[2500];
+        for (int i = 0; i < data.length; i++) {
+            data[i] = rng.nextLong();
+        }
+        try (Arena arena = Arena.ofConfined()) {
+            EncodeResult encoded = new DeltaEncodingEncoder().encode(dtype, data,
+                    EncodeContext.of(arena, WriteRegistry.builder().registerDefaults().build()));
+            long padded = 3L * FastLanes.CHUNK;
+            long[] full = decodeDelta(encoded, dtype, padded, 0, padded, arena);
+
+            // When — a window opening inside chunk 0 and closing inside chunk 2
+            long[] result = decodeDelta(encoded, dtype, padded, 700, 1500, arena);
+
+            // Then
+            assertThat(result).containsExactly(Arrays.copyOfRange(full, 700, 2200));
+        }
+    }
+
+    /// Decodes `encoded` as a `fastlanes.delta` array over the given window, bypassing the file
+    /// format so the `offset` the writer never emits can be set.
+    ///
+    /// @param encoded   the encoder's output (bases buffer, deltas buffer)
+    /// @param dtype     logical element type
+    /// @param deltasLen number of reconstructed elements the chunks cover
+    /// @param offset    absolute index the first returned row maps to
+    /// @param rowCount  number of rows to decode
+    /// @param arena     allocator for the decoded segment
+    /// @return the decoded values
+    private static long[] decodeDelta(EncodeResult encoded, DType dtype, long deltasLen,
+            int offset, long rowCount, Arena arena) {
+        MemorySegment meta = MemorySegment.ofArray(new ProtoDeltaMetadata(deltasLen, offset).encode());
+        ArrayNode bases = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0});
+        ArrayNode deltas = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1});
+        ArrayNode node = new ArrayNode(EncodingId.FASTLANES_DELTA, meta,
+                new ArrayNode[]{bases, deltas}, new int[0]);
+        DecodeContext ctx = new DecodeContext(node, dtype, rowCount,
+                encoded.buffers().toArray(new MemorySegment[0]), ReadRegistry.loadAll(), arena);
+        LongArray decoded = (LongArray) new DeltaEncodingDecoder().decode(ctx);
+        long[] out = new long[(int) decoded.length()];
+        for (int i = 0; i < out.length; i++) {
+            out[i] = decoded.getLong(i);
+        }
+        return out;
+    }
+
+    /// Narrows logical values to the Java array type the writer expects for `ptype`.
+    private static Object narrow(long[] values, PType ptype) {
+        return switch (ptype) {
+            case I8, U8 -> {
+                byte[] out = new byte[values.length];
+                for (int i = 0; i < values.length; i++) {
+                    out[i] = (byte) values[i];
+                }
+                yield out;
+            }
+            case I16, U16 -> {
+                short[] out = new short[values.length];
+                for (int i = 0; i < values.length; i++) {
+                    out[i] = (short) values[i];
+                }
+                yield out;
+            }
+            case I32, U32 -> {
+                int[] out = new int[values.length];
+                for (int i = 0; i < values.length; i++) {
+                    out[i] = (int) values[i];
+                }
+                yield out;
+            }
+            default -> values.clone();
+        };
+    }
+
+    /// Reads a primitive column back as raw bit patterns, masked to the type's width so a
+    /// sign-extending accessor and a zero-extending one compare equal.
+    private static long[] readColumnBits(Path file, String column, long mask) throws IOException {
+        var out = new ArrayList<Long>();
+        try (var vf = VortexReader.open(file, ReadRegistry.loadAll());
+             var iter = vf.scan(ScanOptions.columns(column))) {
+            iter.forEachRemaining(c -> {
+                Array arr = c.column(column);
+                for (long i = 0; i < arr.length(); i++) {
+                    out.add(switch (arr) {
+                        case ByteArray a -> a.getByte(i) & mask;
+                        case ShortArray a -> a.getShort(i) & mask;
+                        case IntArray a -> a.getInt(i) & mask;
+                        case LongArray a -> a.getLong(i) & mask;
+                        default -> throw new IllegalStateException("unexpected array " + arr.getClass());
+                    });
+                }
+            });
+        }
+        return out.stream().mapToLong(Long::longValue).toArray();
+    }
+
     @SuppressWarnings("SameParameterValue")
     private static int[] readIntColumn(Path file, String column) throws IOException {
         try (var vf = VortexReader.open(file, ReadRegistry.loadAll());