fix: Saturate NativeBuffer growth instead of overflowing past half of int.MaxValue

Grow doubled the current length in a checked context without saturating, so once a buffer passed half
of the addressable maximum its next grow threw OverflowException however small the requested increase
and even though the requested size still fit. For a byte buffer that was a hard ceiling near 1 GiB.
The TODO those lines carried described exactly this.

Growth now saturates at the largest addressable element count, keeping it amortised right up to the
ceiling. A request that genuinely cannot be addressed still fails at the byte-size calculation, as
before, so behaviour is unchanged for anything that could not have worked.

The count arithmetic is extracted so it can be tested at the boundary without allocating more than a
gigabyte in a unit test, including the per-element-size ceiling: the limit is a byte count, so a
wider element type saturates at proportionally fewer elements.

Closes #418.

Author: CurtHagenlocher

src/Apache.Arrow/Memory/NativeBuffer.cs

@@ -90,11 +90,8 @@ public void Grow(int newElementCount, bool zeroFill = true)
             if (newElementCount <= Length)
                 return;
 
-            // Exponential growth (2x) to amortise repeated grows
-            // TODO: There might be a size that's big enough to work for this case but not too big to overflow.
-            // We could use that instead of blindly doubling.
-            int newCount = Math.Max(newElementCount, checked(Length * 2));
             int elementSize = Unsafe.SizeOf<TItem>();
+            int newCount = ComputeGrowCount(Length, newElementCount, elementSize);
             int needed = checked(newCount * elementSize);
 
             var owner = _owner ?? throw new ObjectDisposedException(nameof(NativeBuffer<TItem, TTracker>));
@@ -109,6 +106,24 @@ public void Grow(int newElementCount, bool zeroFill = true)
             Length = newCount;
         }
 
+        /// <summary>
+        /// The element count to grow to: double the current length to amortise repeated grows, but never
+        /// past the largest buffer that can be addressed, and never below what the caller asked for.
+        /// </summary>
+        /// <remarks>
+        /// Doubling used to be unconditional and checked, so once the buffer passed half of the maximum
+        /// its next grow threw <see cref="OverflowException"/> however little was asked for, even though
+        /// the requested size still fit. Saturating instead keeps growth amortised right up to the
+        /// ceiling; a request that genuinely cannot be addressed still overflows at the byte-size
+        /// calculation in <see cref="Grow"/>, as before.
+        /// </remarks>
+        internal static int ComputeGrowCount(int length, int newElementCount, int elementSize)
+        {
+            int maxCount = int.MaxValue / elementSize;
+            long doubled = (long)length * 2;
+            return (int)Math.Max(newElementCount, Math.Min(doubled, maxCount));
+        }
+
         public void Dispose()
         {
             IDisposable disposable = _owner;

test/Apache.Arrow.Tests/NativeBufferTests.cs

@@ -84,6 +84,69 @@ public void GrowWithSmallerOrEqualCountIsNoOp()
             Assert.Equal(42, buf.Span[0]);
         }
 
+        // Growth doubles to stay amortised, but must saturate rather than overflow. Doubling used to be
+        // unconditional and checked, so a buffer past half the maximum threw OverflowException on its
+        // next grow however little was asked for — a byte buffer could not grow beyond about 1 GiB.
+        //
+        // The arithmetic is tested directly: reproducing it through Grow would mean allocating more than
+        // a gigabyte, which is not something to put in a unit test.
+        [Theory]
+        // length, requested, elementSize, expected
+        [InlineData(0, 1, 1, 1)]                    // nothing to double yet
+        [InlineData(3, 10, 4, 10)]                  // request exceeds the doubling
+        [InlineData(8, 10, 4, 16)]                  // doubling exceeds the request
+        [InlineData(5, 5, 4, 10)]                   // equal: doubling still wins
+        public void ComputeGrowCountDoublesWhileItFits(
+            int length, int requested, int elementSize, int expected)
+        {
+            Assert.Equal(
+                expected,
+                NativeBuffer<byte, NoOpAllocationTracker>.ComputeGrowCount(length, requested, elementSize));
+        }
+
+        [Fact]
+        public void ComputeGrowCountSaturatesInsteadOfOverflowing()
+        {
+            // Past half the maximum, doubling would overflow. The result saturates at the largest
+            // addressable count and still covers the request.
+            const int elementSize = 1;
+            int overHalf = (int.MaxValue / 2) + 1000;
+
+            int grown = NativeBuffer<byte, NoOpAllocationTracker>.ComputeGrowCount(
+                overHalf, overHalf + 1, elementSize);
+
+            Assert.Equal(int.MaxValue, grown);
+            Assert.True(grown >= overHalf + 1);
+        }
+
+        [Fact]
+        public void ComputeGrowCountSaturatesPerElementSize()
+        {
+            // The ceiling is a byte count, so a wider element saturates at proportionally fewer of them.
+            const int elementSize = 8;
+            int maxCount = int.MaxValue / elementSize;
+            int overHalf = (maxCount / 2) + 1000;
+
+            int grown = NativeBuffer<long, NoOpAllocationTracker>.ComputeGrowCount(
+                overHalf, overHalf + 1, elementSize);
+
+            Assert.Equal(maxCount, grown);
+            Assert.True((long)grown * elementSize <= int.MaxValue);
+        }
+
+        [Fact]
+        public void ComputeGrowCountNeverReturnsLessThanRequested()
+        {
+            // A request larger than the ceiling is not silently truncated; Grow still refuses it when it
+            // works out the byte size.
+            const int elementSize = 8;
+            int beyond = (int.MaxValue / elementSize) + 1;
+
+            Assert.Equal(
+                beyond,
+                NativeBuffer<long, NoOpAllocationTracker>.ComputeGrowCount(0, beyond, elementSize));
+        }
+
         [Fact]
         public void BuildTransfersOwnershipToArrowBuffer()
         {