From a0dd202326ca59f4567c7dd9b27e9acc3a041a07 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 14:28:38 +0100 Subject: [PATCH 01/64] test(qwp): pin slot retention until worker quiescence --- ...CursorSendEngineSlotReacquisitionTest.java | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java new file mode 100644 index 00000000..2b2ce724 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -0,0 +1,218 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Engine-level regression for the shutdown hazard where + * {@link CursorSendEngine#close()} released the slot lock, closed the ring + * and watermark, and unlinked segment files while the shared + * {@link SegmentManager} worker was still mid service pass for the engine's + * ring. A replacement engine could acquire the same slot the moment the + * lock was released, after which the stale worker's abandon/trim path could + * unlink a segment path the replacement was actively writing through — + * store-and-forward data loss after restart. + *

+ * The fix makes {@code close()} run a quiescence barrier + * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and + * refuse to release any worker-reachable resource (ring, watermark, segment + * files, slot lock) until the barrier confirms the worker cannot touch the + * slot again. On barrier timeout the engine deliberately leaks and a later + * {@code close()} retries the cleanup. + */ +public class CursorSendEngineSlotReacquisitionTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-slot-reacq-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + + /** + * The structural guarantee: while the manager worker is provably still + * inside a service pass for the engine's ring, {@code close()} must NOT + * hand the slot to anyone else. With the quiescence barrier reverted, + * close() releases the slot lock immediately and the mid-test + * {@code SlotLock.acquire} probe succeeds — failing the test. + */ + @Test(timeout = 30_000L) + public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/slot"; + // 60 s poll: the worker only acts when explicitly woken, so the + // single pass we park below is the only pass in flight. + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + CursorSendEngine engine = null; + try { + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + + // Shared manager: ownsManager=false, so engine close() cannot + // fall back on manager.close()'s join — the per-ring barrier + // is the only protection, which is exactly what we pin here. + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached the install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Barrier must time out fast: the worker is parked inside the + // service pass for this engine's ring. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + + // The slot must still be locked: a replacement engine (or raw + // SlotLock) acquiring it now would race the stale worker. + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while the manager " + + "worker was still mid service pass for its ring — a " + + "replacement engine could acquire the slot and have its " + + "segment files unlinked by the stale worker"); + } catch (Exception expected) { + // good — slot retained. + } + + // Let the worker finish its pass (it abandons the spare: the + // ring was deregistered by the close attempt above). + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + + // Retry close(): the barrier now succeeds and the full cleanup + // (ring, watermark, unlink, slot lock) must complete. + engine.close(); + engine = null; + + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("retried close() did not release the slot lock", e); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (engine != null) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + if (!managerClosed) { + manager.close(); + } + } + }); + } + + /** + * Plain-positive path: after a normal close (worker quiesces promptly), + * a second engine must be able to acquire and use the same slot. + */ + @Test(timeout = 30_000L) + public void testSameSlotReacquirableAfterNormalClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/slot"; + CursorSendEngine first = new CursorSendEngine(slot, 4L * 1024 * 1024); + first.close(); + CursorSendEngine second = new CursorSendEngine(slot, 4L * 1024 * 1024); + try { + Assert.assertFalse("fully-drained close must leave no segments to recover", + second.wasRecoveredFromDisk()); + } finally { + second.close(); + } + }); + } + + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } +} From 8f0dd45b72cb651c46ff7af4c1e65354c77e0233 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 12:17:55 +0100 Subject: [PATCH 02/64] fix(qwp): make engine close a worker-quiescence barrier before releasing slot resources SegmentManager.close() gives up after a bounded join, but CursorSendEngine.close() could not observe the incomplete shutdown: it closed the ring and watermark, unlinked segment files and released the slot flock while the worker could still be mid service pass - able to unlink a spare/trim path inside a slot directory that a replacement engine had already re-acquired (SF data loss after restart). - serviceRing now claims the entry as in-service under the manager lock and skips entries deregistered before the pass starts - new SegmentManager.awaitRingQuiescence(ring): bounded, interrupt-preserving barrier that confirms the worker can never touch the ring/slot again - CursorSendEngine.close() releases ring/watermark/segment files/slot lock only after confirmed quiescence (or a reaped owned worker); otherwise it leaks them deliberately, logs, and allows close() to be retried - a timed-out SegmentManager.close() hands pathScratch ownership to the worker, which frees it on exit - no permanent native leak when the worker outlives the join - regression: CursorSendEngineSlotReacquisitionTest (slot retained while worker mid-pass + retry completes cleanup; same-slot reacquisition after normal close) and an awaitRingQuiescence contract test --- .../client/sf/cursor/CursorSendEngine.java | 69 +++++- .../qwp/client/sf/cursor/SegmentManager.java | 200 +++++++++++++++--- .../cursor/SegmentManagerCloseRaceTest.java | 75 +++++++ 3 files changed, 309 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..379b5dbe 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -114,6 +114,12 @@ public final class CursorSendEngine implements QuietCloseable { // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. private volatile boolean closed; + // True once close() has run its full cleanup sequence. Stays false when + // a close attempt could not confirm manager-worker quiescence and had to + // leak the ring/watermark/slot lock — in that case a later close() call + // retries the cleanup (the worker may have exited by then). Guarded by + // the synchronized close() method; never read elsewhere. + private boolean closeCompleted; // Producer-thread-only: timestamp of the last "we're backpressured" log // line, used to throttle. Plain long is fine. private long lastBackpressureLogNs; @@ -480,7 +486,7 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos @Override public synchronized void close() { - if (closed) return; + if (closed && closeCompleted) return; closed = true; // Capture drain state BEFORE closing the ring — once the ring is // closed, its accessors aren't safe to read. The active segment is @@ -492,10 +498,14 @@ public synchronized void close() { // the server has no dedup state for those messageSequences. // Memory mode has no files to unlink. // The whole close sequence runs under try/finally so the slot lock - // is ALWAYS released, even if manager/ring close or unlink throws — - // otherwise a kernel-held flock outlives the engine and the next - // sender for the same slot collides on a lock the dead engine - // never released. + // is released whenever it is safe to do so, even if manager/ring + // close or unlink throws — otherwise a kernel-held flock outlives + // the engine and the next sender for the same slot collides on a + // lock the dead engine never released. The one deliberate exception + // is a manager worker that failed to quiesce: releasing the lock + // then would let a replacement engine acquire the slot while the + // stale worker can still create/unlink segment paths inside it. + boolean workerQuiescent = false; try { // "Fully drained" includes BOTH the obvious case (every published // FSN has been acked) AND the never-published case (publishedFsn @@ -504,9 +514,15 @@ public synchronized void close() { // recreates a fresh sf-initial.sfa — would otherwise leave that // fresh empty file behind, the next scanner finds it, adopts the // slot again, and the cycle repeats forever (M6). - boolean fullyDrained = sfDir != null - && (ring.publishedFsn() < 0 - || ring.ackedFsn() >= ring.publishedFsn()); + // Own try/catch so sabotaged/broken ring state cannot skip the + // quiescence barrier below or the slot-lock release in finally. + boolean fullyDrained = false; + try { + fullyDrained = sfDir != null + && (ring.publishedFsn() < 0 + || ring.ackedFsn() >= ring.publishedFsn()); + } catch (Throwable ignored) { + } // Each cleanup step in its own try/catch so a single failure // doesn't strand later cleanups — mirrors the constructor's // catch block. Without this, a throw from manager.deregister @@ -517,11 +533,41 @@ public synchronized void close() { manager.deregister(ring); } catch (Throwable ignored) { } + // Quiescence barrier. deregister alone only removes the entry + // from the registry — the worker may still be mid service pass + // for this ring (creating a spare file, trimming, unlinking). + // Releasing the ring, watermark, segment files, or the slot lock + // while that pass is in flight lets the stale worker unlink a + // path that a replacement engine — which can acquire the slot + // the moment the lock is released — is actively writing through: + // store-and-forward data loss after restart. Wait for confirmed + // quiescence before touching anything the worker can reach. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { + } if (ownsManager) { try { manager.close(); } catch (Throwable ignored) { } + if (!workerQuiescent) { + // manager.close() joins the worker; a reaped (or + // never-started) worker is an even stronger barrier + // than per-ring quiescence. + try { + workerQuiescent = manager.isWorkerReaped(); + } catch (Throwable ignored) { + } + } + } + if (!workerQuiescent) { + LOG.error("SF manager worker did not quiesce during engine close; leaking the " + + "ring, watermark and slot lock so a stale worker cannot corrupt a " + + "future engine on slot {}. The kernel releases the slot flock on " + + "process exit; close() may be invoked again to retry cleanup once " + + "the worker has exited.", sfDir == null ? "" : sfDir); + return; } try { ring.close(); @@ -551,8 +597,13 @@ public synchronized void close() { } catch (Throwable ignored) { } } + closeCompleted = true; } finally { - if (slotLock != null) { + // Gate on quiescence: releasing the flock while a stale worker + // can still touch this slot directory hands the slot to a new + // engine that the worker may then corrupt. Leaking the fd is + // the safe failure mode — the kernel drops it on process exit. + if (workerQuiescent && slotLock != null) { try { slotLock.close(); } catch (Throwable ignored) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 3b7cff60..6548fdef 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -100,15 +100,34 @@ public final class SegmentManager implements QuietCloseable { // registered flag check inside the trim block closes for watermark writes // and totalBytes accounting. private volatile Runnable beforeTrimSyncHook; + // Entry currently being serviced by the worker thread, or null when the + // worker is between service passes (or not running). Guarded by + // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can + // block until an in-flight pass for a just-deregistered ring finishes. + private RingEntry inService; private long lastDiskFullLogNs; private volatile boolean running; + // pathScratch free-exactly-once coordination between a timed-out close() + // and the worker's exit path. All three are guarded by {@link #lock}. + // When close() gives up on the join while the worker loop has not yet + // exited, it hands scratch ownership to the worker + // (scratchHandedToWorker=true) and the worker frees the buffer in its + // exit block; in every other case close() frees it. Without the handoff, + // a worker that outlives the bounded join leaks the native scratch + // buffer forever, because nobody retries manager cleanup after close() + // returns. + private boolean scratchFreed; + private boolean scratchHandedToWorker; + private boolean workerLoopExited; // Total bytes currently allocated across every segment owned by every // registered ring (active + sealed + hot-spare). Mutated by the manager // thread on provision/trim and by register/deregister callers under // {@link #lock}; the lock covers both paths so the counter stays // consistent across registration boundaries. private long totalBytes; - private long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; + // volatile: read by awaitRingQuiescence() from arbitrary caller threads + // while the @TestOnly setter may run on another. + private volatile long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; // volatile because wakeWorker() reads workerThread without holding the // monitor; the synchronized start()/close() pair handles the // start-vs-close ordering. @@ -194,18 +213,100 @@ public synchronized void close() { } } if (t.isAlive()) { - LOG.warn("SegmentManager worker did not stop before close wait completed; " - + "leaving worker-owned resources allocated"); - return; + synchronized (lock) { + if (!workerLoopExited) { + // Hand pathScratch ownership to the worker: its exit + // block frees the buffer under the same lock, so the + // native allocation is reclaimed even though this + // close() could not confirm termination. workerThread + // stays set so isWorkerReaped() reports the incomplete + // shutdown and a later close() can retry the join. + scratchHandedToWorker = true; + LOG.warn("SegmentManager worker did not stop before close wait completed; " + + "worker frees its native scratch buffer on exit"); + return; + } + } + // The worker loop has already run its exit block; the thread + // is at most a few instructions from terminating and can no + // longer touch manager state. Fall through and reap it. } workerThread = null; } // Free the rotation-path native scratch buffer only after worker - // termination has been observed. The worker is the only thread that - // touches the buffer, but close() uses a bounded join; if the worker is - // still alive, leaking this one scratch allocation is safer than freeing - // native memory it may still read or write. - pathScratch.close(); + // termination (or worker-loop exit) has been observed. The worker is + // the only thread that touches the buffer; the scratchFreed flag + // (shared with the worker's exit block) makes the free exactly-once + // no matter which side runs last. + synchronized (lock) { + if (!scratchFreed) { + scratchFreed = true; + pathScratch.close(); + } + } + } + + /** + * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until + * the worker thread is provably no longer executing a service pass for + * {@code ring}, or the worker-join timeout elapses. After this returns + * {@code true}, the worker will never again touch the ring, its + * watermark, or path names under its slot directory: deregister has + * removed the entry from the registry, a stale snapshot entry that has + * not started its pass is skipped by the registration check at the top + * of {@link #serviceRing(RingEntry)}, and this method has observed the + * end of any in-flight pass. Only then may the caller release dependent + * resources (ring, watermark, segment files, slot lock). + *

+ * Returns {@code true} immediately when no worker is running or when + * called from the worker thread itself (test hooks inject deregister + * calls there; waiting would self-deadlock). Returns {@code false} when + * the in-flight pass did not finish within the timeout — the caller must + * treat the worker as still live and leak rather than release. + *

+ * A pending caller interrupt is preserved but does not abort the wait, + * mirroring {@link #close()}. + */ + public boolean awaitRingQuiescence(SegmentRing ring) { + Thread t = workerThread; + if (t == null || t == Thread.currentThread()) { + return true; + } + long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L; + boolean interrupted = Thread.interrupted(); + try { + synchronized (lock) { + while (inService != null && inService.ring == ring) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + try { + // Round up so a sub-millisecond remainder still waits + // instead of spinning through wait(0) == wait-forever. + lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } + return true; + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * True when no manager worker thread can be running: either + * {@link #start()} was never called, or a {@link #close()} confirmed + * worker termination and reaped the thread. Owners use this as a + * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)} + * times out but a subsequent {@code close()} join succeeded. + */ + public synchronized boolean isWorkerReaped() { + return workerThread == null; } /** @@ -213,6 +314,13 @@ public synchronized void close() { * created after this returns, but already-installed spares stay with * the ring (the ring closes them on its own {@link SegmentRing#close}). * Idempotent; safe to call from any thread. + *

+ * Non-blocking: a worker service pass already in flight for this ring + * may still be running when this returns. Callers about to release + * resources the worker can reach (the ring itself, its watermark, its + * segment files, or the slot lock guarding its directory) MUST follow + * up with {@link #awaitRingQuiescence(SegmentRing)} and only release on + * a {@code true} result. */ public void deregister(SegmentRing ring) { synchronized (lock) { @@ -412,6 +520,32 @@ private String nextSparePath(String dir) { } private void serviceRing(RingEntry e) { + // Claim the entry as in-service so deregister-side quiescence + // barriers (awaitRingQuiescence) can wait for this pass to finish. + // A stale snapshot entry deregistered before the pass starts is + // skipped entirely: the deregistering thread may already be + // releasing the ring / watermark / slot resources, so the worker + // must not touch them at all. The registered check and the + // in-service claim are atomic under `lock` — deregister flips + // `registered` under the same lock, so it either prevents this + // pass or the barrier observes it via `inService`. + synchronized (lock) { + if (!e.registered) { + return; + } + inService = e; + } + try { + serviceRing0(e); + } finally { + synchronized (lock) { + inService = null; + lock.notifyAll(); + } + } + } + + private void serviceRing0(RingEntry e) { // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) @@ -571,26 +705,40 @@ private void serviceRing(RingEntry e) { } private void workerLoop() { - while (running) { - // Snapshot the rings under the lock so we don't hold it through the - // (potentially slow) syscalls during creation/unlink. ringSnapshot - // is a thread-confined field — no per-tick allocation. - ringSnapshot.clear(); - synchronized (lock) { - for (int i = 0, n = rings.size(); i < n; i++) { - ringSnapshot.add(rings.getQuick(i)); + try { + while (running) { + // Snapshot the rings under the lock so we don't hold it through the + // (potentially slow) syscalls during creation/unlink. ringSnapshot + // is a thread-confined field — no per-tick allocation. + ringSnapshot.clear(); + synchronized (lock) { + for (int i = 0, n = rings.size(); i < n; i++) { + ringSnapshot.add(rings.getQuick(i)); + } } - } - for (int i = 0, n = ringSnapshot.size(); i < n; i++) { + for (int i = 0, n = ringSnapshot.size(); i < n; i++) { + if (!running) break; + serviceRing(ringSnapshot.getQuick(i)); + } + // Drop strong refs so a deregistered ring becomes collectable + // before the next tick (otherwise the snapshot pins it for up + // to pollNanos after deregister). + ringSnapshot.clear(); if (!running) break; - serviceRing(ringSnapshot.getQuick(i)); + LockSupport.parkNanos(pollNanos); + } + } finally { + // If a timed-out close() abandoned the reap, it handed + // pathScratch ownership to this thread (see close()). Freeing it + // here reclaims the native buffer even when the worker outlives + // every close() attempt — nobody else retries manager cleanup. + synchronized (lock) { + workerLoopExited = true; + if (scratchHandedToWorker && !scratchFreed) { + scratchFreed = true; + pathScratch.close(); + } } - // Drop strong refs so a deregistered ring becomes collectable - // before the next tick (otherwise the snapshot pins it for up - // to pollNanos after deregister). - ringSnapshot.clear(); - if (!running) break; - LockSupport.parkNanos(pollNanos); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 3d7c6a7c..4e62f8ca 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,81 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the {@link SegmentManager#awaitRingQuiescence} contract that + * {@code CursorSendEngine.close()} depends on: + *

+ */ + @Test(timeout = 15_000L) + public void testAwaitRingQuiescenceBlocksWhileServicePassInFlight() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/quiesce-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + manager.deregister(ring); + manager.setWorkerJoinTimeoutMillis(50L); + Thread.currentThread().interrupt(); + Assert.assertFalse( + "awaitRingQuiescence returned true while the worker was parked " + + "inside the service pass for this ring", + manager.awaitRingQuiescence(ring)); + Assert.assertTrue("awaitRingQuiescence must preserve the caller's interrupt", + Thread.interrupted()); + + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + Assert.assertTrue( + "awaitRingQuiescence must return true once the in-flight pass finished", + manager.awaitRingQuiescence(ring)); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + @Test(timeout = 15_000L) public void testInterruptedCallerDoesNotAbandonReapableWorker() throws Exception { TestUtils.assertMemoryLeak(() -> { From d976fd5db4176df99db78e5ea8668e49917103ec Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 15:16:00 +0100 Subject: [PATCH 03/64] fix(qwp): preserve slot ownership after incomplete close --- .../qwp/client/QwpWebSocketSender.java | 57 ++++--- .../client/sf/cursor/CursorSendEngine.java | 39 +++-- .../qwp/client/sf/cursor/SegmentManager.java | 13 ++ .../io/questdb/client/impl/SenderPool.java | 35 ++--- .../client/SlotLockReleasedContractTest.java | 143 +++++++++++++++++- ...CursorSendEngineSlotReacquisitionTest.java | 36 +++++ 6 files changed, 271 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..c0998bcf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -278,10 +278,10 @@ public class QwpWebSocketSender implements Sender { private boolean ownsCursorEngine; private long pendingBytes; // Set true by close() once the SF slot flock has been released (the normal - // teardown path). Stays false if close() bailed early with the I/O thread - // still running -- then cursorEngine.close() never ran and the flock is - // still held, so the owning pool MUST keep the slot reserved rather than - // hand the still-locked dir to the next borrow ("sf slot already in use"). + // teardown path). Stays false if an I/O or manager worker did not stop and + // cursorEngine retained the flock, so the owning pool MUST keep the slot + // reserved rather than hand the still-locked dir to the next borrow + // ("sf slot already in use"). private boolean slotLockReleased; private int pendingRowCount; private SenderProgressDispatcher progressDispatcher; @@ -1038,7 +1038,10 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { * replaying frames get a 2.5s grace window plus a 0.5s stop window * — worst case ~3s when a drainer sits in a blocking native * connect (15s background deadline) and must be abandoned to exit - * on its own. + * on its own; + *
  • SF manager stop: normally immediate, bounded by 5s when its worker + * is stuck in a filesystem operation. On timeout the slot remains + * locked rather than being exposed to a stale worker.
    • * */ @Override @@ -1189,15 +1192,23 @@ public void close() { // the failed close() and now — then closing here is safe. if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null && !cursorSendLoop.delegateEngineClose()) { + CursorSendEngine engine = cursorEngine; try { - cursorEngine.close(); + engine.close(); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); } - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + // Preserve the engine and report the retained flock. + // Sender.close() is idempotent, so this is a deliberate + // safe leak until process exit rather than a retry path. + slotLockReleased = false; + } } rethrowTerminal(terminalError); return; @@ -1231,19 +1242,27 @@ public void close() { } if (ownsCursorEngine && cursorEngine != null) { + CursorSendEngine engine = cursorEngine; try { - cursorEngine.close(); + engine.close(); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); } - cursorEngine = null; - ownsCursorEngine = false; + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + // The manager worker did not quiesce. Preserve ownership + // and report the retained flock so pools retire this slot. + // Repeated Sender.close() calls remain no-ops by contract. + slotLockReleased = false; + } + } else { + // This sender owns no cursor engine holding an SF flock. + slotLockReleased = true; } - // Past the ioThreadStopped guard => cursorEngine.close() ran and - // released the SF flock in its finally (or this sender owned no - // engine holding one). Signal the pool it may reuse the slot. - slotLockReleased = true; // Shutdown order: dispatcher last, after the I/O loop has stopped // producing into it. close() drains pending entries with a short @@ -1288,9 +1307,9 @@ public void close() { /** * True once {@link #close()} has released the store-and-forward slot - * flock. False means close() leaked the still-running I/O thread (and its - * resources), so the flock is still held; the owning pool must keep the - * slot index reserved instead of reusing the still-locked slot dir. + * flock. False means an I/O or manager worker did not stop and close() + * retained the lock and worker-reachable resources; the owning pool must + * keep the slot index reserved instead of reusing the still-locked dir. */ public boolean isSlotLockReleased() { return slotLockReleased; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 379b5dbe..54f675f9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -540,25 +540,26 @@ public synchronized void close() { // while that pass is in flight lets the stale worker unlink a // path that a replacement engine — which can acquire the slot // the moment the lock is released — is actively writing through: - // store-and-forward data loss after restart. Wait for confirmed - // quiescence before touching anything the worker can reach. - try { - workerQuiescent = manager.awaitRingQuiescence(ring); - } catch (Throwable ignored) { - } + // store-and-forward data loss after restart. if (ownsManager) { + // Stopping and reaping a private manager is a stronger barrier + // than waiting for this ring alone. Do it directly so a stuck + // worker consumes at most one workerJoinTimeoutMillis budget, + // rather than one here and a second one in manager.close(). try { manager.close(); } catch (Throwable ignored) { } - if (!workerQuiescent) { - // manager.close() joins the worker; a reaped (or - // never-started) worker is an even stronger barrier - // than per-ring quiescence. - try { - workerQuiescent = manager.isWorkerReaped(); - } catch (Throwable ignored) { - } + try { + workerQuiescent = manager.isWorkerReaped(); + } catch (Throwable ignored) { + } + } else { + // A shared manager must keep serving its other rings, so wait + // only for the deregistered ring's current pass to finish. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { } } if (!workerQuiescent) { @@ -613,6 +614,16 @@ public synchronized void close() { } } + /** + * Whether {@link #close()} completed all cleanup, including releasing the + * SF slot lock. A false value after close means manager-worker quiescence + * could not be confirmed and the worker-reachable resources were retained + * deliberately. Owners must not report or reuse the slot in that case. + */ + public synchronized boolean isCloseCompleted() { + return closeCompleted; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 6548fdef..06e77371 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -93,6 +93,10 @@ public final class SegmentManager implements QuietCloseable { // but before ownership/accounting commit. Callers may inject a deregister // or hold this stale worker snapshot while caller-side cleanup runs. private volatile Runnable beforeInstallSyncHook; + // Test seam: records entry into the per-ring quiescence wait. Null in + // production; owned-engine close tests use it to prove they take only the + // stronger whole-manager join path, not two sequential timeout budgets. + private volatile Runnable beforeRingQuiescenceAwaitHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically @@ -268,6 +272,10 @@ public synchronized void close() { * mirroring {@link #close()}. */ public boolean awaitRingQuiescence(SegmentRing ring) { + Runnable hook = beforeRingQuiescenceAwaitHook; + if (hook != null) { + hook.run(); + } Thread t = workerThread; if (t == null || t == Thread.currentThread()) { return true; @@ -410,6 +418,11 @@ public void setBeforeInstallSyncHook(Runnable hook) { this.beforeInstallSyncHook = hook; } + @TestOnly + public void setBeforeRingQuiescenceAwaitHook(Runnable hook) { + this.beforeRingQuiescenceAwaitHook = hook; + } + @TestOnly public void setBeforeTrimSyncHook(Runnable hook) { this.beforeTrimSyncHook = hook; diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1cc214b4..31e50f31 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -171,7 +171,7 @@ public final class SenderPool implements AutoCloseable { // down on another thread. Guarded by lock. private int pendingLeaseTeardowns; // Slots whose delegate close() returned with the SF flock still held - // (the I/O thread refused to stop). Permanently consumed: the index is + // because an I/O or manager worker did not stop. Permanently consumed: // never freed and never reused, so no borrow ever hands out a still- // locked slot dir. Counted in the cap check so the lost capacity is // accounted for. Guarded by lock; only ever ticks for SF slots. @@ -523,15 +523,15 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { // leakedSlots (retire) or the freed index carries the cap math. recoveringSlots--; if (flockHeld[0]) { - // close() bailed early with the I/O thread still running and - // the flock still held. Retire the slot permanently (mirror + // close() retained the flock because an I/O or manager + // worker did not stop. Retire the slot permanently (mirror // discardBroken/reapIdle): keep slotInUse[i] set and count it // in leakedSlots so the borrow() cap math accounts for the // lost capacity and no later borrow ever reuses the // still-locked dir. leakedSlots++; LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with " - + "the flock still held (I/O thread refused to stop); pool capacity reduced by 1, " + + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " + "now {} of {} usable [leakedSlots={}]", i, maxSize - leakedSlots, maxSize, leakedSlots); } else { @@ -580,12 +580,13 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (flockHeld[0]) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this - // dir, so a still-held flock only leaks this recoverer's I/O - // thread (a best-effort teardown loss, logged). Crucially we do + // dir, so a still-held flock only leaks this recoverer's + // worker-reachable resources (a best-effort teardown loss, + // logged). Crucially we do // NOT touch leakedSlots -- that would wrongly shrink the // in-range pool capacity. LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held " - + "(I/O thread refused to stop); its data is durable on disk for a later attempt", + + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); } if (stopScan) { @@ -609,7 +610,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { * * @param flockHeld single-element out-param set to {@code true} iff a * recoverer was built and its {@code close()} returned with - * the flock still held (the I/O thread refused to stop) + * the flock still held because a worker did not stop * @return {@code true} if a build/drain failure occurred that will very * likely repeat for every remaining slot, so the caller should stop scanning */ @@ -618,8 +619,8 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, flockHeld[0] = false; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and - // delegate().close() can early-return with the I/O thread still running - // (flock still held). + // delegate().close() can retain it when an I/O or manager worker does + // not stop. SenderSlot recoverer = null; boolean stopScan = false; try { @@ -1067,7 +1068,7 @@ public void reapIdle() { } // Return reserved SF slot indices to the free set -- but only for // slots whose delegate confirmed the flock was released. A slot - // left locked (I/O thread refused to stop) is retired permanently. + // left locked because a worker did not stop is retired permanently. if (storeAndForward) { lock.lock(); try { @@ -1107,8 +1108,8 @@ public int totalSize() { /** * Snapshot of the number of SF slots permanently retired because a - * delegate {@code close()} returned with the slot flock still held (the - * I/O thread refused to stop). Each leaked slot permanently lowers the + * delegate {@code close()} returned with the slot flock still held after + * an I/O or manager worker did not stop. Each leaked slot permanently lowers the * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A * non-zero, growing value explains a pool that has started timing out * every {@code borrow()}. For metrics and tests. @@ -1271,7 +1272,7 @@ private void freeSlotIndex(int idx) { * non-QWP delegate never holds an SF flock, so it is always treated as * released. A {@link QwpWebSocketSender} reports it via * {@link QwpWebSocketSender#isSlotLockReleased()} -- false means close() - * bailed early with the I/O thread still running and the flock still held. + * retained the flock because an I/O or manager worker did not stop. */ private static boolean flockReleased(SenderSlot s) { Sender d = s.delegate(); @@ -1281,8 +1282,8 @@ private static boolean flockReleased(SenderSlot s) { /** * Reclaims one SF slot after its delegate's {@code close()} has been * attempted. When the flock was released the index returns to the free - * set; when {@code close()} returned with the flock still held (the I/O - * thread refused to stop) the slot is retired permanently -- + * set; when {@code close()} returned with the flock still held because an + * I/O or manager worker did not stop, the slot is retired permanently -- * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap * math accounts for the lost capacity and no later borrow ever reuses the * still-locked dir. Either way {@code closingSlots} is decremented. @@ -1304,7 +1305,7 @@ private boolean reclaimSlot(SenderSlot s, String context) { } leakedSlots++; LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " + - "(I/O thread refused to stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots); return false; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 85522e9d..97ae9dbe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -26,7 +26,12 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -36,8 +41,12 @@ import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; /** @@ -46,8 +55,8 @@ *
      *
    • Happy path — a clean {@code close()} that winds the I/O thread * down reports {@code isSlotLockReleased() == true}.
      • - *
    • Leak path — a {@code close()} that early-returns because the I/O - * thread refused to stop ({@code ioThreadStopped == false}) reports + *
    • Leak path — a {@code close()} that retains the lock because an + * I/O or manager worker did not stop reports * {@code isSlotLockReleased() == false}.
      • *
    * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held, @@ -178,6 +187,114 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception }); } + /** + * Manager-worker leak path: engine close retains the slot while a shared + * manager is still inside a service pass. The sender must expose that + * retained flock to SenderPool. Repeated sender close calls remain no-ops; + * the test cleans the deliberately retained engine up directly. + */ + @Test(timeout = 30_000L) + public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-manager-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + CursorSendEngine engine = null; + QwpWebSocketSender wss = null; + boolean managerClosed = false; + try { + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + manager.setWorkerJoinTimeoutMillis(50L); + wss.close(); + + Assert.assertFalse("sender reported a retained flock as released", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must remain incomplete while worker is in service", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while manager worker was still in service"); + } catch (Exception expected) { + // good — the incomplete close retained the flock. + } + + // Sender.close() is idempotent: a second call must not retry + // or change ownership while the manager worker is still live. + wss.close(); + Assert.assertFalse("repeated close changed retained-flock reporting", + wss.isSlotLockReleased()); + Assert.assertFalse("repeated close unexpectedly retried engine cleanup", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + // Test-only cleanup through the retained local reference. The + // sender conservatively keeps reporting false, as required by + // a pool that may already have retired this slot. + engine.close(); + Assert.assertTrue("direct cleanup did not complete after worker exit", + engine.isCloseCompleted()); + Assert.assertFalse("sender must not revise the result after close returned", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after direct cleanup", probe); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (wss != null) { + try { + wss.close(); + } catch (Throwable ignored) { + } + } + if (engine != null && !engine.isCloseCompleted()) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + if (!managerClosed) { + manager.close(); + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -192,6 +309,28 @@ private static void freeFieldQuietly(Object target, String name) { } } + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + private static T readField(Object target, String name, Class type) throws Exception { Class cls = target.getClass(); while (cls != null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 2b2ce724..5ab33da7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -35,6 +35,7 @@ import org.junit.Before; import org.junit.Test; +import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -123,6 +124,8 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // service pass for this engine's ring. manager.setWorkerJoinTimeoutMillis(50L); engine.close(); + Assert.assertFalse("incomplete close must remain observable to the owner", + engine.isCloseCompleted()); // The slot must still be locked: a replacement engine (or raw // SlotLock) acquiring it now would race the stale worker. @@ -145,6 +148,8 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // Retry close(): the barrier now succeeds and the full cleanup // (ring, watermark, unlink, slot lock) must complete. engine.close(); + Assert.assertTrue("retried close must report complete cleanup", + engine.isCloseCompleted()); engine = null; try (SlotLock probe = SlotLock.acquire(slot)) { @@ -174,6 +179,31 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { }); } + /** + * An engine that owns its manager must use the whole-manager stop/join as + * its only quiescence barrier. Calling the per-ring barrier first would + * give a stuck worker two independent timeout budgets. + */ + @Test(timeout = 30_000L) + public void testOwnedManagerCloseSkipsPerRingQuiescenceWait() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/owned-slot"; + CursorSendEngine engine = new CursorSendEngine(slot, 4L * 1024 * 1024); + SegmentManager manager = readManager(engine); + AtomicBoolean perRingAwaited = new AtomicBoolean(); + try { + manager.setBeforeRingQuiescenceAwaitHook(() -> perRingAwaited.set(true)); + engine.close(); + Assert.assertTrue("owned engine close did not complete", engine.isCloseCompleted()); + Assert.assertFalse("owned engine close spent a separate per-ring wait budget", + perRingAwaited.get()); + } finally { + manager.setBeforeRingQuiescenceAwaitHook(null); + engine.close(); + } + }); + } + /** * Plain-positive path: after a normal close (worker quiesces promptly), * a second engine must be able to acquire and use the same slot. @@ -194,6 +224,12 @@ public void testSameSlotReacquirableAfterNormalClose() throws Exception { }); } + private static SegmentManager readManager(CursorSendEngine engine) throws Exception { + Field field = CursorSendEngine.class.getDeclaredField("manager"); + field.setAccessible(true); + return (SegmentManager) field.get(engine); + } + private static void rmDirRecursive(String dir) { if (!Files.exists(dir)) return; long find = Files.findFirst(dir); From d3b61478ddb57256ffe70c209fab03de19eaaebf Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 17:08:44 +0100 Subject: [PATCH 04/64] test(qwp): pin slot retention on the owned-manager close path Every production CursorSendEngine (Sender.build, BackgroundDrainer, QwpWebSocketSender.connect) owns its SegmentManager, so close() takes the manager.close() + isWorkerReaped() branch - yet all deterministic retention tests exercised the test-only shared-manager branch (awaitRingQuiescence). A regression confined to the owned path - reporting quiescence unconditionally, or isWorkerReaped() returning true while the worker is alive - would have gone green through the whole suite and silently reintroduced the SF-data-loss hazard on the only path production runs. testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass builds the production shape (2-arg ctor, private owned manager), waits for the initial hot-spare install so the park hook can neither be missed nor fire early, rotates onto the spare to force the worker back into an install pass, parks it there, and drives close() with a 50 ms join budget. It asserts the incomplete close stays observable (isCloseCompleted() == false), the slot flock is retained (SlotLock.acquire throws), and a retried close after worker release completes the full cleanup and frees the slot. Mutation-verified red on all three reverts: - owned branch forcing workerQuiescent = true (only this test catches it) - finally gate reverted to unconditional slotLock.close() - SegmentManager.isWorkerReaped() returning true while the worker is alive (previously zero coverage anywhere) --- ...CursorSendEngineSlotReacquisitionTest.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 5ab33da7..9afb86eb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -27,8 +27,11 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Assert; @@ -179,6 +182,120 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { }); } + /** + * Owned-manager twin of {@link #testCloseRetainsSlotWhileWorkerIsMidServicePass}: + * the ONLY construction shape production uses (Sender.build, BackgroundDrainer, + * QwpWebSocketSender.connect all own their manager). The owned close path does + * not run the per-ring barrier at all — it relies on {@code manager.close()}'s + * bounded join and the {@code isWorkerReaped()} check. If that check regressed + * to report quiescence unconditionally (or {@code isWorkerReaped()} itself + * returned true while the worker is alive), close() would release the slot + * lock mid service pass and the shared-manager tests would stay green — this + * test is the red gate for the production path. + *

    + * Determinism: the owned manager starts inside the engine ctor (1 ms poll), + * so its first spare-install pass races test setup. We first wait until the + * initial hot spare is installed — after that the worker cannot enter another + * install pass until a rotation consumes the spare, so the park hook installed + * afterwards can neither be missed nor fire early. Two appends then fill the + * active segment and rotate onto the spare; the worker's next poll tick + * re-enters the install pass and parks in the hook. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-parked-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: let the worker finish the initial spare install so + // the hook below can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: one frame fills the active segment exactly; the + // second forces rotation onto the spare. needsHotSpare() is + // true again, so the worker's next tick parks in the hook. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid service + // pass. manager.close()'s 50 ms join times out, the worker is + // not reaped, and close() must retain every worker-reachable + // resource — above all the slot flock. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("incomplete owned close must remain observable to the owner", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("owned engine.close() released the slot lock while its manager " + + "worker was still mid service pass — a replacement engine could " + + "acquire the slot and have its segment files unlinked by the " + + "stale worker (the production SF-data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained. + } + + // Phase 4: release the worker (it abandons the spare: the ring + // was deregistered by the close attempt) and retry. The join + // now reaps the worker and the full cleanup must complete. + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried owned close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("retried owned close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would @@ -230,6 +347,12 @@ private static SegmentManager readManager(CursorSendEngine engine) throws Except return (SegmentManager) field.get(engine); } + private static SegmentRing readRing(CursorSendEngine engine) throws Exception { + Field field = CursorSendEngine.class.getDeclaredField("ring"); + field.setAccessible(true); + return (SegmentRing) field.get(engine); + } + private static void rmDirRecursive(String dir) { if (!Files.exists(dir)) return; long find = Files.findFirst(dir); From 7cf96122fb3ad39f1cf532b1ad255ee9f780d0db Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 19:19:22 +0100 Subject: [PATCH 05/64] perf(qwp): wake quiescence waiters only when one is parked serviceRing's per-pass finally called lock.notifyAll() unconditionally - with the default 1 ms poll that is ~1000 wakeups/sec per registered ring on the production worker, paid for a barrier (awaitRingQuiescence) that production never takes: all three production constructions own their manager, so close() goes through manager.close()+isWorkerReaped() and never parks on the lock. - new quiescenceWaiters count, incremented/decremented around the awaitRingQuiescence wait loop under the same lock as the worker's check - no lost-wakeup window, and the timed wait remains a fallback - the per-pass finally notifies only when quiescenceWaiters > 0; in steady state it never fires - notifyAll (not notify) retained when a waiter exists: with a shared manager, distinct waiters may await different rings --- .../qwp/client/sf/cursor/SegmentManager.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 06e77371..726374d6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -109,6 +109,14 @@ public final class SegmentManager implements QuietCloseable { // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can // block until an in-flight pass for a just-deregistered ring finishes. private RingEntry inService; + // Number of threads currently parked in awaitRingQuiescence()'s wait + // loop. Guarded by {@link #lock}. The worker's per-pass finally block + // only calls lock.notifyAll() when this is > 0, so the steady-state + // production path (no quiescence barrier in flight) never notifies. + // Both sides mutate/check under the same lock, so there is no lost + // wakeup: a waiter that increments after the worker's check re-reads + // inService before waiting and sees the pass already finished. + private int quiescenceWaiters; private long lastDiskFullLogNs; private volatile boolean running; // pathScratch free-exactly-once coordination between a timed-out close() @@ -284,18 +292,23 @@ public boolean awaitRingQuiescence(SegmentRing ring) { boolean interrupted = Thread.interrupted(); try { synchronized (lock) { - while (inService != null && inService.ring == ring) { - long remainingNanos = deadlineNanos - System.nanoTime(); - if (remainingNanos <= 0) { - return false; - } - try { - // Round up so a sub-millisecond remainder still waits - // instead of spinning through wait(0) == wait-forever. - lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); - } catch (InterruptedException ignored) { - interrupted = true; + quiescenceWaiters++; + try { + while (inService != null && inService.ring == ring) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + try { + // Round up so a sub-millisecond remainder still waits + // instead of spinning through wait(0) == wait-forever. + lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); + } catch (InterruptedException ignored) { + interrupted = true; + } } + } finally { + quiescenceWaiters--; } } return true; @@ -553,7 +566,14 @@ private void serviceRing(RingEntry e) { } finally { synchronized (lock) { inService = null; - lock.notifyAll(); + // Wake quiescence barriers only when one is actually parked. + // In production no engine ever waits here (owned-manager + // close joins the worker thread instead), so the per-tick + // pass must not pay an unconditional notifyAll. notifyAll, + // not notify: distinct waiters may await different rings. + if (quiescenceWaiters > 0) { + lock.notifyAll(); + } } } } From 930175fce064feaa8c98b4d915dc4386696a3ffc Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 19:19:57 +0100 Subject: [PATCH 06/64] fix(qwp): transfer slot cleanup to the worker's exit path and recover retired pool slots When an owned manager's bounded join timed out during engine close, the engine leaked ring + watermark mmaps and the slot flock until process exit, the sender latched slotLockReleased=false forever, and SenderPool retired the slot permanently (leakedSlots++) - a transiently-slow SF filesystem op at close time (> workerJoinTimeoutMillis, default 5 s) permanently ratcheted pool capacity down, even though the worker often exits moments later. - new SegmentManager.deferUntilWorkerExit(cleanup): hands an action to the worker-loop exit block, which runs strictly after the final service pass - the last point the worker can touch any slot path. Registration and the exit block's workerLoopExited flip share the manager lock, so the handoff is exactly-once: accepted while the worker is live, rejected (caller cleans up inline) once it exited - CursorSendEngine.close(): on an owned-manager join timeout, terminal cleanup (ring, watermark, drained-file unlink, flock release) now transfers to the worker's exit path instead of leaking; the quiescence gate is unchanged - the slot stays locked until the pass provably ends - exactly-once via a terminalCleanupClaimed CAS, deliberately not the engine monitor: a retried close() holds the monitor while joining the worker, and the worker cannot die until its cleanup returns - monitor-based exclusion would stall that close() for its full join budget. With the CAS the worker never blocks and the join returns as soon as the pass ends - QwpWebSocketSender.isSlotLockReleased() is now monotonic, not frozen: it re-probes the retained engine (volatile reads only, safe under the pool lock) and flips true once the deferred cleanup - worker exit path or delegated I/O-thread close - releases the flock - SenderPool re-probes retired slots (housekeeper reapIdle tick and capacity-starved borrows just before parking) and returns a recovered index to the free set: leakedSlots goes back down and a would-be borrow timeout becomes an immediate creation. A persistent non-zero leakedSlotCount() now means a genuinely wedged worker - shared-manager engines (test-only construction) keep the old leak-and-retry-close contract: their worker serves other rings and has no exit to defer to; startup-recovery retirements also stay permanent - regression: deferUntilWorkerExit contract test, owned-close handoff test (slot reacquirable after worker exit with NO close() retry), pool recovery via reapIdle and via capacity-starved borrow; the SlotLockReleasedContractTest leak-path pin updated to the new monotonic-getter contract --- .../qwp/client/QwpWebSocketSender.java | 88 +++++-- .../client/sf/cursor/CursorSendEngine.java | 241 ++++++++++++------ .../qwp/client/sf/cursor/SegmentManager.java | 60 +++++ .../io/questdb/client/impl/SenderPool.java | 77 +++++- .../client/SlotLockReleasedContractTest.java | 20 +- ...CursorSendEngineSlotReacquisitionTest.java | 117 ++++++++- .../cursor/SegmentManagerCloseRaceTest.java | 92 +++++++ .../client/test/impl/SenderPoolSfTest.java | 113 ++++++++ 8 files changed, 697 insertions(+), 111 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c0998bcf..f1e3533c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -281,8 +281,17 @@ public class QwpWebSocketSender implements Sender { // teardown path). Stays false if an I/O or manager worker did not stop and // cursorEngine retained the flock, so the owning pool MUST keep the slot // reserved rather than hand the still-locked dir to the next borrow - // ("sf slot already in use"). - private boolean slotLockReleased; + // ("sf slot already in use"). May flip to true LATER via the getter's + // re-probe of retainedEngine, once the deferred cleanup (manager-worker + // exit path or delegated I/O-thread close) releases the flock — pools + // re-probe retired slots to recover their capacity. volatile: written on + // the closing thread, read by pool threads. + private volatile boolean slotLockReleased; + // Engine whose close() could not complete during sender close() — its + // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() + // re-probes it so a late flock release becomes visible to the owning pool. + // Only ever set inside close(); null for a sender that closed cleanly. + private volatile CursorSendEngine retainedEngine; private int pendingRowCount; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a @@ -1190,24 +1199,33 @@ public void close() { // close actually runs, so the pool must not reuse the slot // meanwhile. A false return means the thread exited between // the failed close() and now — then closing here is safe. - if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null - && !cursorSendLoop.delegateEngineClose()) { - CursorSendEngine engine = cursorEngine; - try { - engine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - if (engine.isCloseCompleted()) { - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; + if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null) { + if (cursorSendLoop.delegateEngineClose()) { + // The I/O thread adopted the engine close and runs it on + // its exit path. Keep the engine visible for re-probing: + // isSlotLockReleased() flips true once that close + // completes, letting a pool recover the retired slot. + retainedEngine = cursorEngine; } else { - // Preserve the engine and report the retained flock. - // Sender.close() is idempotent, so this is a deliberate - // safe leak until process exit rather than a retry path. - slotLockReleased = false; + CursorSendEngine engine = cursorEngine; + try { + engine.close(); + } catch (Throwable t) { + LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + // Engine cleanup is pending on its manager worker's + // exit path (or leaked, for a shared manager). Report + // the retained flock now; the getter re-probes the + // retained engine so a late release is not lost. + slotLockReleased = false; + retainedEngine = engine; + } } } rethrowTerminal(terminalError); @@ -1257,7 +1275,12 @@ public void close() { // The manager worker did not quiesce. Preserve ownership // and report the retained flock so pools retire this slot. // Repeated Sender.close() calls remain no-ops by contract. + // Engine cleanup was handed to the worker's exit path + // (owned manager); the getter re-probes the retained + // engine so the pool can reclaim the slot once cleanup + // actually completes. slotLockReleased = false; + retainedEngine = engine; } } else { // This sender owns no cursor engine holding an SF flock. @@ -1306,13 +1329,30 @@ public void close() { } /** - * True once {@link #close()} has released the store-and-forward slot - * flock. False means an I/O or manager worker did not stop and close() - * retained the lock and worker-reachable resources; the owning pool must - * keep the slot index reserved instead of reusing the still-locked dir. + * True once the store-and-forward slot flock has been released. False + * means an I/O or manager worker did not stop and close() retained the + * lock and worker-reachable resources; the owning pool must keep the slot + * index reserved instead of reusing the still-locked dir. + *

    + * Not a one-shot snapshot: when close() left engine cleanup pending on a + * worker/I/O-thread exit path, this re-probes the retained engine and + * latches true the moment that cleanup completes — pools re-probe retired + * slots through this getter to recover their capacity. Monotonic: + * false→true only, never back. Lock-free (volatile reads) so pools may + * call it under their capacity lock. */ public boolean isSlotLockReleased() { - return slotLockReleased; + if (slotLockReleased) { + return true; + } + CursorSendEngine engine = retainedEngine; + if (engine != null && engine.isCloseCompleted()) { + // Benign latch race: concurrent callers may both observe the + // completed cleanup and both write true. + slotLockReleased = true; + return true; + } + return false; } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 54f675f9..9cd52169 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -29,6 +29,7 @@ import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; /** @@ -119,7 +120,20 @@ public final class CursorSendEngine implements QuietCloseable { // leak the ring/watermark/slot lock — in that case a later close() call // retries the cleanup (the worker may have exited by then). Guarded by // the synchronized close() method; never read elsewhere. - private boolean closeCompleted; + // volatile: latched by finishClose(), but read lock-free by + // isCloseCompleted() from pool threads re-probing a retired slot (see the + // getter for why it must not synchronize). + private volatile boolean closeCompleted; + // Exactly-once claim on the terminal cleanup (finishClose). Contended by + // close() and a worker-exit handoff (completeDeferredClose); whoever wins + // the CAS runs the cleanup, the loser never touches ring/watermark/flock. + // Deliberately NOT the engine monitor: a retried close() holds the + // monitor while joining the manager worker, and the worker cannot die + // until its exit-path cleanup finishes — monitor-based exclusion would + // stall that close() for its full join budget (test-visible livelock). + // With the CAS the worker's cleanup never blocks, so the join returns as + // soon as the pass ends. + private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean(); // Producer-thread-only: timestamp of the last "we're backpressured" log // line, used to throttle. Plain long is fine. private long lastBackpressureLogNs; @@ -497,79 +511,136 @@ public synchronized void close() { // against potentially-fresh server state — duplicate writes when // the server has no dedup state for those messageSequences. // Memory mode has no files to unlink. - // The whole close sequence runs under try/finally so the slot lock - // is released whenever it is safe to do so, even if manager/ring - // close or unlink throws — otherwise a kernel-held flock outlives - // the engine and the next sender for the same slot collides on a - // lock the dead engine never released. The one deliberate exception - // is a manager worker that failed to quiesce: releasing the lock - // then would let a replacement engine acquire the slot while the - // stale worker can still create/unlink segment paths inside it. - boolean workerQuiescent = false; + // + // Cleanup is gated on worker quiescence: releasing the ring, + // watermark, segment files or the slot lock while the manager worker + // is still mid service pass would let a replacement engine acquire + // the slot and have its files unlinked by the stale worker — + // store-and-forward data loss after restart. When the bounded worker + // join times out on an owned manager, cleanup OWNERSHIP TRANSFERS to + // the worker's exit path (deferUntilWorkerExit): the worker is + // provably the last thread able to touch the slot directory, so it + // releases everything the moment its in-flight pass finishes instead + // of leaking the slot until process exit. + // + // "Fully drained" includes BOTH the obvious case (every published + // FSN has been acked) AND the never-published case (publishedFsn + // < 0). The latter matters because a drainer adopting an empty + // orphan slot — segments filtered as empty by recovery, engine + // recreates a fresh sf-initial.sfa — would otherwise leave that + // fresh empty file behind, the next scanner finds it, adopts the + // slot again, and the cycle repeats forever (M6). + // Own try/catch so sabotaged/broken ring state cannot skip the + // quiescence barrier below or the slot-lock release in finishClose. + boolean drained = false; + try { + drained = sfDir != null + && (ring.publishedFsn() < 0 + || ring.ackedFsn() >= ring.publishedFsn()); + } catch (Throwable ignored) { + } + final boolean fullyDrained = drained; + // Each cleanup step in its own try/catch so a single failure + // doesn't strand later cleanups — mirrors the constructor's + // catch block. Without this, a throw from manager.deregister + // or manager.close() would leave the ring mmap'd and any + // residual .sfa files on disk, where the next sender can + // adopt them and replay already-acked data. try { - // "Fully drained" includes BOTH the obvious case (every published - // FSN has been acked) AND the never-published case (publishedFsn - // < 0). The latter matters because a drainer adopting an empty - // orphan slot — segments filtered as empty by recovery, engine - // recreates a fresh sf-initial.sfa — would otherwise leave that - // fresh empty file behind, the next scanner finds it, adopts the - // slot again, and the cycle repeats forever (M6). - // Own try/catch so sabotaged/broken ring state cannot skip the - // quiescence barrier below or the slot-lock release in finally. - boolean fullyDrained = false; + manager.deregister(ring); + } catch (Throwable ignored) { + } + // Quiescence barrier. deregister alone only removes the entry + // from the registry — the worker may still be mid service pass + // for this ring (creating a spare file, trimming, unlinking). + boolean workerQuiescent = false; + if (ownsManager) { + // Stopping and reaping a private manager is a stronger barrier + // than waiting for this ring alone. Do it directly so a stuck + // worker consumes at most one workerJoinTimeoutMillis budget, + // rather than one here and a second one in manager.close(). try { - fullyDrained = sfDir != null - && (ring.publishedFsn() < 0 - || ring.ackedFsn() >= ring.publishedFsn()); + manager.close(); } catch (Throwable ignored) { } - // Each cleanup step in its own try/catch so a single failure - // doesn't strand later cleanups — mirrors the constructor's - // catch block. Without this, a throw from manager.deregister - // or manager.close() would leave the ring mmap'd and any - // residual .sfa files on disk, where the next sender can - // adopt them and replay already-acked data. try { - manager.deregister(ring); + workerQuiescent = manager.isWorkerReaped(); } catch (Throwable ignored) { } - // Quiescence barrier. deregister alone only removes the entry - // from the registry — the worker may still be mid service pass - // for this ring (creating a spare file, trimming, unlinking). - // Releasing the ring, watermark, segment files, or the slot lock - // while that pass is in flight lets the stale worker unlink a - // path that a replacement engine — which can acquire the slot - // the moment the lock is released — is actively writing through: - // store-and-forward data loss after restart. - if (ownsManager) { - // Stopping and reaping a private manager is a stronger barrier - // than waiting for this ring alone. Do it directly so a stuck - // worker consumes at most one workerJoinTimeoutMillis budget, - // rather than one here and a second one in manager.close(). - try { - manager.close(); - } catch (Throwable ignored) { - } - try { - workerQuiescent = manager.isWorkerReaped(); - } catch (Throwable ignored) { - } - } else { - // A shared manager must keep serving its other rings, so wait - // only for the deregistered ring's current pass to finish. - try { - workerQuiescent = manager.awaitRingQuiescence(ring); - } catch (Throwable ignored) { - } + } else { + // A shared manager must keep serving its other rings, so wait + // only for the deregistered ring's current pass to finish. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { + } + } + if (!workerQuiescent && ownsManager) { + // Ownership handoff: manager.close() already stopped the worker + // loop (running=false), so the worker exits as soon as its + // in-flight pass finishes — it is merely slow, or wedged in a + // syscall. Either way it is the last thread able to touch the + // slot, so hand it the terminal cleanup instead of leaking the + // slot until process exit. completeDeferredClose and a retried + // close() race through the terminalCleanupClaimed CAS, so the + // cleanup runs exactly once and the worker never blocks on the + // engine monitor a retried close() holds while joining it. + boolean handedOff = false; + try { + handedOff = manager.deferUntilWorkerExit(() -> completeDeferredClose(fullyDrained)); + } catch (Throwable ignored) { } - if (!workerQuiescent) { - LOG.error("SF manager worker did not quiesce during engine close; leaking the " - + "ring, watermark and slot lock so a stale worker cannot corrupt a " - + "future engine on slot {}. The kernel releases the slot flock on " - + "process exit; close() may be invoked again to retry cleanup once " - + "the worker has exited.", sfDir == null ? "" : sfDir); + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the worker's exit path and run the " + + "moment its in-flight service pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can race " + + "the stale worker on slot {}", sfDir == null ? "" : sfDir); return; } + // Handoff rejected: the worker loop exited between the failed + // bounded join and the registration attempt (both sides share the + // manager's lock, so this observation is exact). A worker past + // its loop can never touch slot paths again — inline cleanup is + // as safe as a reaped worker. + workerQuiescent = true; + } + if (!workerQuiescent) { + // Shared (caller-owned) manager: its worker keeps serving other + // rings indefinitely, so there is no exit path to hand cleanup + // to — leak and let a retried close() reclaim once the pass ends. + LOG.error("SF manager worker did not quiesce during engine close; leaking the " + + "ring, watermark and slot lock so a stale worker cannot corrupt a " + + "future engine on slot {}. The kernel releases the slot flock on " + + "process exit; close() may be invoked again to retry cleanup once " + + "the worker has exited.", sfDir == null ? "" : sfDir); + return; + } + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A worker-exit handoff from an earlier close() attempt owns the + // terminal cleanup: it has run or is finishing right now on the + // exiting worker. closeCompleted flips the moment it is done — + // owners observe it via isCloseCompleted(), never by re-running + // the cleanup here (that would double-release ring/flock). + return; + } + finishClose(fullyDrained); + } + + /** + * Terminal cleanup: closes the ring and watermark, unlinks drained + * segment files, releases the slot flock, and latches + * {@link #closeCompleted}. The caller must hold the engine monitor and + * must have established that the manager worker can no longer touch the + * slot directory (reaped, provably exited, or running this on its own + * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the + * claim token, not the engine monitor, is the exclusion between a + * worker-exit handoff and a retried close(), so ring/watermark/flock are + * never double-released and the worker's cleanup can never deadlock + * against a close() that holds the monitor while joining the worker. + */ + private void finishClose(boolean fullyDrained) { + try { try { ring.close(); } catch (Throwable ignored) { @@ -600,11 +671,10 @@ public synchronized void close() { } closeCompleted = true; } finally { - // Gate on quiescence: releasing the flock while a stale worker - // can still touch this slot directory hands the slot to a new - // engine that the worker may then corrupt. Leaking the fd is - // the safe failure mode — the kernel drops it on process exit. - if (workerQuiescent && slotLock != null) { + // Reaching finishClose at all requires established quiescence, so + // releasing the flock is safe even if a step above threw. Leaking + // it would strand the slot until process exit for no reason. + if (slotLock != null) { try { slotLock.close(); } catch (Throwable ignored) { @@ -614,13 +684,42 @@ public synchronized void close() { } } + /** + * Runs on the manager worker's exit path when {@link #close()} handed + * cleanup ownership to the worker (see {@code deferUntilWorkerExit}). + * Deliberately does NOT take the engine monitor: a retried close() can + * hold it while joining this very worker, and the thread cannot die until + * this method returns — the {@link #terminalCleanupClaimed} CAS provides + * the exactly-once exclusion instead, so the worker always exits promptly + * and the racing close() converges via {@code isWorkerReaped()}. + */ + private void completeDeferredClose(boolean fullyDrained) { + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A retried close() (or an earlier duplicate handoff) already ran + // the terminal cleanup. + return; + } + finishClose(fullyDrained); + LOG.info("deferred SF engine cleanup completed on manager-worker exit; slot released: {}", + sfDir == null ? "" : sfDir); + } + /** * Whether {@link #close()} completed all cleanup, including releasing the * SF slot lock. A false value after close means manager-worker quiescence * could not be confirmed and the worker-reachable resources were retained - * deliberately. Owners must not report or reuse the slot in that case. + * deliberately — either handed to the worker's exit path (owned manager), + * which flips this to true the moment the worker's in-flight pass + * finishes, or leaked until a retried close() (shared manager). Owners + * must not reuse the slot while this is false; pools may re-probe it to + * recover a retired slot's capacity once it flips. + *

    + * Deliberately unsynchronized ({@code closeCompleted} is volatile): pools + * probe this under their own capacity lock, and the deferred cleanup can + * hold the engine monitor through munmap/unlink syscalls — a synchronized + * getter would stall the pool's hot borrow path behind them. */ - public synchronized boolean isCloseCompleted() { + public boolean isCloseCompleted() { return closeCompleted; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 726374d6..5deaf2ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -109,6 +109,14 @@ public final class SegmentManager implements QuietCloseable { // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can // block until an in-flight pass for a just-deregistered ring finishes. private RingEntry inService; + // Cleanup actions handed to the worker's exit block by an owning engine + // whose close() found the worker still mid service pass after the bounded + // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock}; + // consumed exactly once by the worker-loop finally, which runs them + // OUTSIDE `lock`: they perform syscalls (munmap/unlink/flock release), + // and no caller-facing lock may ever be held while running third-party + // cleanup code. + private ObjList exitCleanups; // Number of threads currently parked in awaitRingQuiescence()'s wait // loop. Guarded by {@link #lock}. The worker's per-pass finally block // only calls lock.notifyAll() when this is > 0, so the steady-state @@ -258,6 +266,37 @@ public synchronized void close() { } } + /** + * Hands a cleanup action to the worker thread's exit block, to run + * strictly after the worker loop has finished its final service pass -- + * i.e. after the last point where the worker can create, write or unlink + * anything under a registered ring's slot directory. Returns {@code false} + * when the worker loop has already exited (or the worker never started): + * the caller must run the cleanup itself, which is equally safe for the + * same reason -- no further worker access to the slot is possible. + *

    + * This is the slot-ownership transfer used by an owning engine's close() + * when the bounded worker join timed out: instead of retiring the slot + * until process exit, ring/watermark/flock release moves to the worker, + * which is provably the last thread able to touch the slot directory. + * The registration here and the exit block's {@code workerLoopExited} + * flip share {@link #lock}, so the cleanup runs exactly once: either it + * is registered before the flip and the worker runs it, or the flip won + * and this method rejects the handoff. + */ + public boolean deferUntilWorkerExit(Runnable cleanup) { + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (exitCleanups == null) { + exitCleanups = new ObjList<>(); + } + exitCleanups.add(cleanup); + return true; + } + } + /** * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until * the worker thread is provably no longer executing a service pass for @@ -765,12 +804,33 @@ private void workerLoop() { // pathScratch ownership to this thread (see close()). Freeing it // here reclaims the native buffer even when the worker outlives // every close() attempt — nobody else retries manager cleanup. + ObjList cleanups; synchronized (lock) { workerLoopExited = true; if (scratchHandedToWorker && !scratchFreed) { scratchFreed = true; pathScratch.close(); } + cleanups = exitCleanups; + exitCleanups = null; + } + // Deferred engine cleanups (see deferUntilWorkerExit) run OUTSIDE + // `lock`: they perform syscalls (munmap, unlink, flock release) + // and must never execute under a lock that close()/register/ + // deregister callers contend on. Running them after the loop body + // is what makes the handoff safe: this thread can no longer touch + // any slot path. They must also never block on a caller-held + // monitor — a retried engine.close() joins this thread while + // holding the engine monitor, which is why the engine side uses a + // lock-free claim (CAS), not synchronization, for exactly-once. + if (cleanups != null) { + for (int i = 0, n = cleanups.size(); i < n; i++) { + try { + cleanups.getQuick(i).run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed on manager-worker exit", t); + } + } } } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 31e50f31..798a931a 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -171,11 +171,21 @@ public final class SenderPool implements AutoCloseable { // down on another thread. Guarded by lock. private int pendingLeaseTeardowns; // Slots whose delegate close() returned with the SF flock still held - // because an I/O or manager worker did not stop. Permanently consumed: + // because an I/O or manager worker did not stop. Consumed while retired: // never freed and never reused, so no borrow ever hands out a still- // locked slot dir. Counted in the cap check so the lost capacity is - // accounted for. Guarded by lock; only ever ticks for SF slots. + // accounted for. NOT necessarily permanent: engine cleanup may be pending + // on a worker/I/O-thread exit path, so reprobeRetiredSlots() re-checks + // retiredSlots and returns any index whose flock has since dropped. + // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; + // The retired slots behind the leakedSlots count (runtime reclaim paths + // only; startup-recovery retirements stay permanent — their transient + // recoverer sender is out of scope by retire time). Re-probed by + // reprobeRetiredSlots() so a late flock release (deferred engine cleanup + // on a worker exit path) restores the pool's capacity instead of + // ratcheting it down until process exit. Guarded by lock. + private final ArrayList retiredSlots = new ArrayList<>(); // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a @@ -773,6 +783,12 @@ public PooledSender borrow() { throw new LineSenderException( "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } + // Capacity-starved: before parking, re-probe retired slots — a + // deferred engine cleanup may have released a flock since the + // retire, and the freed index can admit a creation right now. + if (reprobeRetiredSlots()) { + continue; + } try { remainingNanos = slotReleased.awaitNanos(remainingNanos); } catch (InterruptedException e) { @@ -1029,6 +1045,11 @@ public void reapIdle() { if (closed) { return; } + // Housekeeper tick doubles as the retired-slot recovery driver: + // a slot retired because its worker did not stop is re-probed + // here and returns to the free set once the deferred cleanup + // finally released its flock. + reprobeRetiredSlots(); Iterator it = available.iterator(); while (it.hasNext() && all.size() > minSize) { SenderSlot s = it.next(); @@ -1107,12 +1128,15 @@ public int totalSize() { } /** - * Snapshot of the number of SF slots permanently retired because a + * Snapshot of the number of SF slots currently retired because a * delegate {@code close()} returned with the slot flock still held after - * an I/O or manager worker did not stop. Each leaked slot permanently lowers the - * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A - * non-zero, growing value explains a pool that has started timing out - * every {@code borrow()}. For metrics and tests. + * an I/O or manager worker did not stop. Each leaked slot lowers the + * pool's effective capacity ({@code maxSize - leakedSlotCount()}) while + * retired. Retired slots are re-probed (housekeeper tick and + * capacity-starved borrows) and recovered once the delegate's deferred + * cleanup releases the flock, so the count can go back down; a non-zero, + * persistent value means a worker is still wedged and explains a pool + * that has started timing out {@code borrow()}. For metrics and tests. */ public int leakedSlotCount() { lock.lock(); @@ -1304,9 +1328,44 @@ private boolean reclaimSlot(SenderSlot s, String context) { return true; } leakedSlots++; - LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " + - "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", + retiredSlots.add(s); + LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " + + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " + + "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later", s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots); return false; } + + /** + * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to + * the free set any whose delegate now reports the flock released — the + * deferred engine cleanup (manager-worker or I/O-thread exit path) has + * run since the retire. Restores {@code leakedSlots} capacity and signals + * waiters so a parked borrow can admit a creation immediately. + *

    + * Caller must hold {@code lock}. The probe is lock-free on the delegate + * side ({@code isSlotLockReleased()} reads volatiles only), so holding + * the pool lock across it cannot stall behind delegate teardown. + * + * @return {@code true} if at least one slot's capacity was recovered + */ + private boolean reprobeRetiredSlots() { + boolean recovered = false; + for (int i = retiredSlots.size() - 1; i >= 0; i--) { + SenderSlot s = retiredSlots.get(i); + if (flockReleased(s)) { + retiredSlots.remove(i); + leakedSlots--; + freeSlotIndex(s.slotIndex()); + recovered = true; + LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + + "pool capacity restored, now {} of {} usable [leakedSlots={}]", + s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + } + } + if (recovered) { + slotReleased.signalAll(); + } + return recovered; + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 97ae9dbe..e8ef2670 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -58,6 +58,10 @@ *

  • Leak path — a {@code close()} that retains the lock because an * I/O or manager worker did not stop reports * {@code isSlotLockReleased() == false}.
    • + *
  • Recovery path — the getter is monotonic but not frozen: it + * re-probes the retained engine, so once the deferred cleanup (worker + * exit path or a retried close) releases the flock, the getter flips to + * {@code true} and a pool that retired the slot can recover it.
    • * * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held, * retire the slot permanently". Before this test that contract was driven only @@ -191,7 +195,9 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception * Manager-worker leak path: engine close retains the slot while a shared * manager is still inside a service pass. The sender must expose that * retained flock to SenderPool. Repeated sender close calls remain no-ops; - * the test cleans the deliberately retained engine up directly. + * the test cleans the deliberately retained engine up directly — and once + * that cleanup completes, the sender must expose the released flock so a + * pool that retired the slot can recover its capacity. */ @Test(timeout = 30_000L) public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { @@ -254,13 +260,17 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception releaseWorker.countDown(); manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); - // Test-only cleanup through the retained local reference. The - // sender conservatively keeps reporting false, as required by - // a pool that may already have retired this slot. + // Test-only cleanup through the retained local reference (the + // shared-manager path has no worker-exit handoff to defer to; + // a retried close() is its reclaim driver). engine.close(); Assert.assertTrue("direct cleanup did not complete after worker exit", engine.isCloseCompleted()); - Assert.assertFalse("sender must not revise the result after close returned", + // Recovery contract: the sender re-probes its retained engine, + // so the completed cleanup (and released flock) MUST become + // visible — this is what lets SenderPool recover a slot it + // had already retired as leaked. + Assert.assertTrue("sender must expose the late flock release to the pool", wss.isSlotLockReleased()); try (SlotLock probe = SlotLock.acquire(slot)) { Assert.assertNotNull("slot must be acquirable after direct cleanup", probe); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 9afb86eb..892bdc59 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -59,8 +59,10 @@ * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and * refuse to release any worker-reachable resource (ring, watermark, segment * files, slot lock) until the barrier confirms the worker cannot touch the - * slot again. On barrier timeout the engine deliberately leaks and a later - * {@code close()} retries the cleanup. + * slot again. On barrier timeout an owned-manager engine hands cleanup + * ownership to the worker's exit path (see + * {@code SegmentManager.deferUntilWorkerExit}); a shared-manager engine + * deliberately leaks and a later {@code close()} retries the cleanup. */ public class CursorSendEngineSlotReacquisitionTest { @@ -296,6 +298,117 @@ public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws }); } + /** + * The ownership handoff (owned manager): when close() cannot confirm + * worker quiescence within the bounded join, the terminal cleanup (ring, + * watermark, flock release) transfers to the worker's exit path — the + * worker is provably the last thread able to touch the slot directory. + * Once the parked pass is released the worker must run the cleanup + * itself, WITHOUT any retried {@code close()}: {@code isCloseCompleted()} + * flips true and the slot becomes acquirable again. This is what lets a + * pool recover a retired slot instead of losing its capacity until + * process exit. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-handoff-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid-pass. The + // bounded join times out; cleanup ownership is handed to the + // worker's exit path. Every worker-reachable resource — above + // all the slot flock — must still be retained at this point. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while its manager worker " + + "was still mid service pass — the handoff must not weaken the " + + "quiescence gate"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4 — the contract under test: release the worker and do + // NOT retry close(). The worker finishes its pass, exits, and + // runs the deferred cleanup itself, flipping isCloseCompleted + // and releasing the flock with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred cleanup never ran on manager-worker exit — the slot " + + "would stay retired until process exit"); + } + Thread.sleep(1); + } + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } catch (Exception e) { + throw new AssertionError("worker-exit cleanup did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 4e62f8ca..9f6fc225 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,98 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the {@link SegmentManager#deferUntilWorkerExit} handoff contract + * that {@code CursorSendEngine.close()}'s slot-ownership transfer depends + * on: + *
      + *
    • rejects the handoff ({@code false}) when no worker ever started — + * the caller must clean up inline;
      • + *
    • accepts it ({@code true}) while the worker is live-but-slow + * mid service pass after a timed-out close(), and runs the cleanup + * exactly when the worker exits — never while the pass is still in + * flight;
      • + *
    • rejects it again once the worker loop has exited/been reaped.
      • + *
    + */ + @Test(timeout = 15_000L) + public void testDeferUntilWorkerExitRunsCleanupAfterFinalPass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/defer-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch cleanupRan = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + // Never-started manager: no worker will ever run the cleanup, + // and none can touch a slot — the caller must do it inline. + Assert.assertFalse("never-started manager must reject the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: running=false, worker parked mid-pass — the + // exact state CursorSendEngine.close() hands ownership over in. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Assert.assertFalse("worker must not be reaped while parked mid-pass", + manager.isWorkerReaped()); + + Assert.assertTrue("live-but-slow worker must accept the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + Assert.assertEquals("cleanup must not run while the pass is still in flight", + 1, cleanupRan.getCount()); + + // Release the pass; the worker loop observes running=false, + // exits, and must run the deferred cleanup on its way out. + releaseWorker.countDown(); + Assert.assertTrue("cleanup never ran on worker exit", + cleanupRan.await(10, TimeUnit.SECONDS)); + + // Reap the exited worker, then: no live worker, no handoff. + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + managerClosed = true; + Assert.assertFalse("reaped manager must reject the handoff", + manager.deferUntilWorkerExit(() -> { + })); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + /** * Pins the {@link SegmentManager#awaitRingQuiescence} contract that * {@code CursorSendEngine.close()} depends on: diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index cfef2feb..75aca494 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -658,6 +658,119 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw }); } + @Test + public void testRetiredSlotRecoveredByHousekeeperAfterLateFlockRelease() throws Exception { + // Recovery twin of testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock: + // a slot retired because close() returned with the flock still held is + // NOT lost until process exit. Engine cleanup may complete later on a + // worker/I/O-thread exit path (isSlotLockReleased() re-probes the + // retained engine), and the housekeeper's reapIdle() tick must then + // return the index to the free set: leakedSlots back down, slotInUse + // cleared, full capacity restored. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + // Forge the retire: real teardown first (no native leaks), + // then clear slotLockReleased so discardBroken retires the + // slot as leaked. + Sender delegate = getDelegate(a); + delegate.close(); + setBooleanField(delegate, "slotLockReleased", false); + invokeDiscardBroken(pool, a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // Forge the late release: the deferred cleanup finished and + // the delegate now reports the flock dropped (in production + // this flip comes from isSlotLockReleased() re-probing the + // retained engine after the manager worker exited). + setBooleanField(delegate, "slotLockReleased", true); + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertFalse("recovered slot index 0 must return to the free set", + slotInUse[0]); + + // Full capacity restored: with maxSize=2, two concurrent + // borrows must succeed again (index 0 is reusable — its + // flock is genuinely free). + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + } + } + }); + } + + @Test + public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { + // Borrow-path twin of the housekeeper recovery test: a borrow that + // would otherwise park on the cap check must re-probe retired slots + // before waiting, so a late flock release converts a guaranteed + // borrow timeout into an immediate creation on the recovered index — + // no housekeeper tick required. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + delegate.close(); + setBooleanField(delegate, "slotLockReleased", false); + invokeDiscardBroken(pool, a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // One live borrow + one retired slot = cap reached. + PooledSender b = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-1"))); + try { + // Late release lands while the pool is capacity-starved. + setBooleanField(delegate, "slotLockReleased", true); + + // The next borrow hits the cap check, re-probes, frees + // index 0, and must create on it instead of timing out. + PooledSender c = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + } + } finally { + b.close(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- From eea7f89fb4a799e236f4e82efa2da757680939cb Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 21:09:56 +0100 Subject: [PATCH 07/64] test(qwp): pin slot retention when worker-exit handoff registration fails A throw from deferUntilWorkerExit (allocation failure while building the handoff) carries no worker-liveness information, so close() must retain every worker-reachable resource instead of running terminal cleanup inline under a possibly-live worker. Add a test seam that throws from the registration path while the worker is provably mid service pass and assert the slot flock, ring and watermark are retained, close stays incomplete, and a retried close() after worker exit converges and releases the slot. --- .../qwp/client/sf/cursor/SegmentManager.java | 22 ++++ ...CursorSendEngineSlotReacquisitionTest.java | 114 ++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 5deaf2ca..51bd204e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -97,6 +97,13 @@ public final class SegmentManager implements QuietCloseable { // production; owned-engine close tests use it to prove they take only the // stronger whole-manager join path, not two sequential timeout budgets. private volatile Runnable beforeRingQuiescenceAwaitHook; + // Test seam: runs at the top of deferUntilWorkerExit, before the + // worker-liveness check. Null in production; registration-failure tests + // throw from it to simulate an allocation failure (OOM building the + // cleanup lambda or growing exitCleanups) while the worker is still + // live. Callers must treat such a throw as "worker state unknown", + // never as the exact false return meaning the worker loop has exited. + private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically @@ -283,8 +290,18 @@ public synchronized void close() { * flip share {@link #lock}, so the cleanup runs exactly once: either it * is registered before the flip and the worker runs it, or the flip won * and this method rejects the handoff. + *

    + * May throw on allocation failure (the lambda at the call site, the + * {@code ObjList}, or its growth). A throw carries NO liveness + * information: the worker was never observed, so the caller must treat + * it as "worker possibly still live" and retain resources — never as + * the exact {@code false} return above. */ public boolean deferUntilWorkerExit(Runnable cleanup) { + Runnable hook = beforeExitCleanupRegistrationHook; + if (hook != null) { + hook.run(); + } synchronized (lock) { if (workerLoopExited || workerThread == null) { return false; @@ -465,6 +482,11 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public void setBeforeExitCleanupRegistrationHook(Runnable hook) { + this.beforeExitCleanupRegistrationHook = hook; + } + @TestOnly public void setBeforeInstallSyncHook(Runnable hook) { this.beforeInstallSyncHook = hook; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 892bdc59..ad25ff78 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -409,6 +409,120 @@ public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { }); } + /** + * Registration-failure twin of + * {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: when + * {@code deferUntilWorkerExit} itself throws (allocation failure while + * building the handoff), close() must NOT mistake the swallowed throw + * for "worker already exited" and run the terminal cleanup inline — the + * worker is provably still mid service pass, so releasing the ring, + * watermark or slot flock here is the original stale-worker UAF/data-loss + * hazard. Every worker-reachable resource must be retained and the close + * must stay incomplete; a retried close() after the worker exits + * converges and releases the slot. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-regfail-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: make the handoff registration throw — simulating + // an OutOfMemoryError allocating the cleanup lambda/list — + // with the worker provably still mid service pass. close() + // must retain everything: no inline finishClose, no flock + // release, closeCompleted stays false. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated allocation failure registering exit cleanup"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete when handoff registration fails", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock after a failed handoff " + + "registration while the manager worker was still mid service " + + "pass — the swallowed throw was mistaken for proof the worker " + + "exited (stale-worker UAF/data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4: clear the fault, release the worker (its loop was + // already stopped by the close attempt, so it exits), and + // retry close(). The retry must converge via isWorkerReaped() + // and release the slot. + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the retried close", probe); + } catch (Exception e) { + throw new AssertionError("retried close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would From 940e1307b061fce56e41dc2016cedd444dd8f9fe Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 21:10:08 +0100 Subject: [PATCH 08/64] fix(qwp): publish close completion only after confirmed slot flock release finishClose() wrote closeCompleted=true before slotLock.close(), so a pool thread could observe completion (isSlotLockReleased -> reprobeRetiredSlots), free the slot index, and admit a replacement sender whose SlotLock.acquire collided with the still-open flock fd -- a spurious "sf slot already in use" naming the process's own pid. SlotLock.close() also discarded the Files.close() result, reporting an unconfirmed release as completion. Reorder the terminal cleanup: release the flock first via the new SlotLock.release() (checks the close rc, retains the fd on failure so the lock state is never misreported), and publish closeCompleted only on a confirmed release. An unconfirmed release keeps closeCompleted false, degrading into the existing retired/leaked-slot accounting with the kernel's process-exit backstop. Add a @TestOnly hook between cleanup and release so the otherwise microsecond-wide window is deterministically testable; the new test parks the closer inside it and asserts completion is not observable while the flock is provably still held, then latches once released. --- .../client/sf/cursor/CursorSendEngine.java | 118 ++++++++++-- .../qwp/client/sf/cursor/SlotLock.java | 42 ++++- ...gineClosePublishAfterFlockReleaseTest.java | 176 ++++++++++++++++++ .../qwp/client/sf/cursor/SlotLockTest.java | 19 ++ 4 files changed, 334 insertions(+), 21 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 9cd52169..c9b85aea 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -28,6 +28,7 @@ import io.questdb.client.std.Files; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; @@ -115,15 +116,28 @@ public final class CursorSendEngine implements QuietCloseable { // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. private volatile boolean closed; - // True once close() has run its full cleanup sequence. Stays false when - // a close attempt could not confirm manager-worker quiescence and had to - // leak the ring/watermark/slot lock — in that case a later close() call - // retries the cleanup (the worker may have exited by then). Guarded by - // the synchronized close() method; never read elsewhere. + // True once close() has run its full cleanup sequence INCLUDING a + // CONFIRMED slot-flock release — finishClose() publishes this strictly + // after SlotLock.release() reports success, never before. Pool threads + // treat the flip as "the slot dir is reusable" and free the slot index + // the moment they observe it (QwpWebSocketSender.isSlotLockReleased -> + // SenderPool.reprobeRetiredSlots), so publishing before the release + // would let a replacement engine's SlotLock.acquire collide with the + // still-open fd. Stays false when a close attempt could not confirm + // manager-worker quiescence (or the flock release itself failed) and had + // to leak the ring/watermark/slot lock — in that case a later close() + // call retries the cleanup (the worker may have exited by then). // volatile: latched by finishClose(), but read lock-free by // isCloseCompleted() from pool threads re-probing a retired slot (see the // getter for why it must not synchronize). private volatile boolean closeCompleted; + // Test-only hook run by finishClose() between the terminal cleanup and + // the flock release. Lets a test park the releasing thread inside the + // cleanup/release window and assert that closeCompleted stays false — + // i.e. that completion is never observable while the flock is still + // held. volatile: finishClose may run on the manager worker's exit + // thread while the hook is installed from a test thread. + private volatile Runnable beforeFlockReleaseHook; // Exactly-once claim on the terminal cleanup (finishClose). Contended by // close() and a worker-exit handoff (completeDeferredClose); whoever wins // the CAS runs the cleanup, the loser never touches ring/watermark/flock. @@ -586,9 +600,17 @@ public synchronized void close() { // cleanup runs exactly once and the worker never blocks on the // engine monitor a retried close() holds while joining it. boolean handedOff = false; + boolean registrationFailed = false; try { handedOff = manager.deferUntilWorkerExit(() -> completeDeferredClose(fullyDrained)); } catch (Throwable ignored) { + // Allocation failure (OOM building the cleanup lambda or + // growing the manager's exitCleanups list). Unlike the exact + // false return below, a throw carries NO worker-liveness + // information — the two must never be conflated, or the + // inline cleanup below would release worker-reachable + // resources under a possibly-live worker. + registrationFailed = true; } if (handedOff) { LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " @@ -598,6 +620,23 @@ public synchronized void close() { + "the stale worker on slot {}", sfDir == null ? "" : sfDir); return; } + if (registrationFailed) { + // The handoff never registered and the worker was never + // observed — it must be presumed live and mid service pass. + // Retain every worker-reachable resource (ring, watermark, + // segment files, slot flock) and leave terminalCleanupClaimed + // unclaimed and closeCompleted false, exactly like the + // shared-manager leak branch: manager.close() above already + // stopped the worker loop, so a retried close() converges via + // isWorkerReaped() once the in-flight pass ends. The kernel + // releases the slot flock on process exit regardless. + LOG.error("SF worker-exit handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live " + + "worker cannot corrupt a future engine on slot {}. close() may be " + + "invoked again to retry cleanup once the worker has exited.", + sfDir == null ? "" : sfDir); + return; + } // Handoff rejected: the worker loop exited between the failed // bounded join and the registration attempt (both sides share the // manager's lock, so this observation is exact). A worker past @@ -629,8 +668,13 @@ public synchronized void close() { /** * Terminal cleanup: closes the ring and watermark, unlinks drained - * segment files, releases the slot flock, and latches - * {@link #closeCompleted}. The caller must hold the engine monitor and + * segment files, releases the slot flock, and — only once the release + * is confirmed — latches {@link #closeCompleted}. Publish order + * is load-bearing: pools free the slot index the instant they observe + * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must + * never be visible while the flock fd is still open, or a replacement + * engine races the release and fails acquisition on a live slot. + * The caller must hold the engine monitor and * must have established that the manager worker can no longer touch the * slot directory (reaped, provably exited, or running this on its own * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the @@ -669,21 +713,64 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } - closeCompleted = true; } finally { // Reaching finishClose at all requires established quiescence, so // releasing the flock is safe even if a step above threw. Leaking // it would strand the slot until process exit for no reason. + // + // ORDER MATTERS: release the flock FIRST, verify it, and only + // then publish closeCompleted. Pools read isCloseCompleted() as + // "the slot dir is reusable" and free the slot index the moment + // it flips; publishing before Files.close(fd) completes would + // open a window where a replacement engine's SlotLock.acquire + // collides with the still-open fd and fails with a spurious + // "slot already in use". + Runnable hook = beforeFlockReleaseHook; + if (hook != null) { + try { + hook.run(); + } catch (Throwable ignored) { + // test-only; must never block the release + } + } + boolean released; if (slotLock != null) { try { - slotLock.close(); + released = slotLock.release(); } catch (Throwable ignored) { - // best-effort; flock is also released by kernel on process exit + released = false; } + } else { + released = true; + } + if (released) { + closeCompleted = true; + } else { + // Unconfirmed release: the flock may still be held, so keep + // closeCompleted false — the owning pool keeps the slot + // retired (leakedSlots accounting) instead of reusing a + // possibly-locked dir. The kernel releases the flock on + // process exit regardless. + LOG.error("SF slot flock release failed during engine close; keeping " + + "closeCompleted=false so the pool cannot reuse a possibly " + + "still-locked slot dir; the kernel releases the flock on " + + "process exit [slot={}]", sfDir == null ? "" : sfDir); } } } + /** + * Installs a hook that {@link #finishClose} runs between the terminal + * cleanup and the slot-flock release. Test-only: makes the otherwise + * microsecond-wide cleanup/release window deterministic so tests can + * assert {@link #isCloseCompleted()} stays false until the release is + * confirmed. + */ + @TestOnly + public void setBeforeFlockReleaseHook(Runnable hook) { + this.beforeFlockReleaseHook = hook; + } + /** * Runs on the manager worker's exit path when {@link #close()} handed * cleanup ownership to the worker (see {@code deferUntilWorkerExit}). @@ -705,10 +792,13 @@ private void completeDeferredClose(boolean fullyDrained) { } /** - * Whether {@link #close()} completed all cleanup, including releasing the - * SF slot lock. A false value after close means manager-worker quiescence - * could not be confirmed and the worker-reachable resources were retained - * deliberately — either handed to the worker's exit path (owned manager), + * Whether {@link #close()} completed all cleanup, including a + * confirmed release of the SF slot lock — the flip is published + * strictly after the flock fd is closed, so observing {@code true} + * guarantees the slot dir is acquirable by a replacement engine. A false + * value after close means manager-worker quiescence could not be + * confirmed (or the flock release itself failed) and the + * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager), * which flips this to true the moment the worker's in-flight pass * finishes, or leaked until a retried close() (shared manager). Owners * must not reuse the slot while this is false; pools may re-probe it to diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0b8379de..1b72d157 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -116,15 +116,43 @@ public String slotDir() { return slotDir; } - @Override - public void close() { - // Closing the fd releases the lock. We do NOT remove the .lock - // file or the .lock.pid sidecar — a stale PID is harmless (next - // acquirer overwrites .lock.pid on success). - if (fd >= 0) { - Files.close(fd); + /** + * Releases the flock by closing the lock fd and reports whether the + * release was confirmed. Closing the fd releases the lock. We do + * NOT remove the {@code .lock} file or the {@code .lock.pid} sidecar — + * a stale PID is harmless (next acquirer overwrites {@code .lock.pid} + * on success). + *

    + * On close failure the fd is retained: the flock may still be + * held by this process, so forgetting the fd would misreport the lock + * state and forfeit any chance of a later retry. Idempotent — once + * released, subsequent calls return {@code true}. + *

    + * Owners that gate a "slot dir is reusable" signal on the release + * (e.g. {@code CursorSendEngine.finishClose} publishing + * {@code closeCompleted}) must call this and check the result rather + * than {@link #close()}, which is best-effort by contract. + * + * @return {@code true} if the fd was closed (or was already released), + * {@code false} if the OS reported a close failure and the + * flock may still be held + */ + public boolean release() { + if (fd < 0) { + return true; + } + if (Files.close(fd) == 0) { fd = -1; + return true; } + return false; + } + + @Override + public void close() { + // QuietCloseable contract: best-effort, no signal. Callers that + // must confirm the release use release() and check the result. + release(); } private static String readHolder(String pidPath) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java new file mode 100644 index 00000000..cd7fdff0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -0,0 +1,176 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Regression test for the completion/release ordering in + * {@code CursorSendEngine.finishClose()}: {@code closeCompleted} must be + * published strictly AFTER the slot flock release is confirmed, never + * before. + * + *

    The bug being pinned: {@code closeCompleted = true} used to be written + * before {@code slotLock} was closed. A pool thread could observe completion + * through {@code QwpWebSocketSender.isSlotLockReleased()}, free the slot + * index in {@code SenderPool.reprobeRetiredSlots()}, and admit a replacement + * sender whose {@code SlotLock.acquire} then collided with the still-open + * flock fd — a spurious "sf slot already in use" construction failure naming + * the process's own pid as the holder. + * + *

    The window between the publish and the {@code Files.close(fd)} is + * microseconds wide in the wild; {@code setBeforeFlockReleaseHook} makes it + * deterministic. The closing thread is parked between terminal cleanup and + * the flock release, and the test asserts from outside the window's two + * halves of the contract: + *

      + *
    • inside the window: {@code isCloseCompleted()} is still false AND a + * fresh {@code SlotLock.acquire} on the slot fails (proving the flock + * is genuinely held — i.e. reporting completion here would have been + * a lie);
      • + *
    • after the window: {@code isCloseCompleted()} is true AND a fresh + * {@code SlotLock.acquire} succeeds (completion still implies + * reusability — the reorder did not break the happy path).
      • + *
    + */ +public class EngineClosePublishAfterFlockReleaseTest { + + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-publish-order-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir == null) return; + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + @Test(timeout = 30_000L) + public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + CountDownLatch inWindow = new CountDownLatch(1); + CountDownLatch proceed = new CountDownLatch(1); + engine.setBeforeFlockReleaseHook(() -> { + inWindow.countDown(); + try { + // Bounded so a failed assertion on the main thread can + // never wedge the closer past the test timeout. + proceed.await(20, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + AtomicReference closerError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + engine.close(); + } catch (Throwable t) { + closerError.set(t); + } + }, "engine-closer"); + closer.start(); + + try { + assertTrue("closer thread never reached the cleanup/release window", + inWindow.await(10, TimeUnit.SECONDS)); + + // Inside the window: terminal cleanup has run, the flock has + // NOT been released. Completion must not be observable yet — + // this is the exact read a pool thread performs before + // freeing the slot index. + assertFalse("closeCompleted was published before the flock release; " + + "a pool observing this would free the slot index and a " + + "replacement sender would collide with the still-held flock", + engine.isCloseCompleted()); + + // Prove the window is real: the flock is genuinely still + // held, so acquisition by a "replacement engine" fails. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("scaffolding error: expected the slot flock to still be " + + "held inside the pre-release window, but a fresh " + + "SlotLock.acquire succeeded"); + } catch (IllegalStateException expected) { + // good — slot is locked, which is why completion must + // not have been published yet. + } + } finally { + proceed.countDown(); + closer.join(10_000L); + } + assertFalse("closer thread did not finish", closer.isAlive()); + assertNull("engine.close() threw", closerError.get()); + + // After the window: the release is confirmed, so completion must + // now be latched, and completion must still imply reusability. + assertTrue("closeCompleted must latch once the flock release is confirmed", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — completion implies the slot dir is acquirable. + } catch (IllegalStateException stillHeld) { + fail("closeCompleted reported true but the slot flock is still held: " + + stillHeld.getMessage()); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644a7463..0d4b9f43 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -103,6 +103,25 @@ public void testCloseReleasesLock() throws Exception { }); } + @Test + public void testReleaseConfirmsAndIsIdempotent() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/verified-release"; + SlotLock lock = SlotLock.acquire(slot); + assertTrue("first release must confirm success", lock.release()); + // Idempotent: an already-released lock keeps reporting true — + // callers gating a "slot reusable" signal on it must never see + // a spurious false after a confirmed release. + assertTrue("repeat release must stay true", lock.release()); + // close() after release() is a safe no-op (QuietCloseable path). + lock.close(); + // Confirmed release means the slot is genuinely acquirable. + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { From 7b9924c98d1ef3d1722b95833fd0160e8282cd1d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 21:40:41 +0100 Subject: [PATCH 09/64] fix(qwp): retain startup-retired recoverers so a late flock release restores pool capacity isSlotLockReleased() is no longer a one-shot snapshot: deferred engine cleanup on a worker/I/O-thread exit path can release the SF slot flock after close() returned. The runtime reclaim paths (discardBroken/reapIdle via reclaimSlot) already keep such slots in retiredSlots and re-probe them, but the in-range startup-recovery pass only ticked leakedSlots and dropped the recoverer, making the retirement permanent even after the release -- fatal at maxSize=1, where every later borrow timed out until process restart. Hand the retained recoverer out of drainCandidateSlotForRecovery (retainedOut replaces the flockHeld boolean) and add it to retiredSlots alongside the leakedSlots tick, so the existing reprobeRetiredSlots() drivers (capacity-starved borrow, housekeeper tick) recover the capacity once the worker finally exits. Out-of-range recoverers stay excluded: they carry no leakedSlots tick and freeSlotIndex(idx) would index past the slotInUse array. --- .../io/questdb/client/impl/SenderPool.java | 62 ++++++---- .../client/test/impl/SenderPoolSfTest.java | 106 +++++++++++++++++- 2 files changed, 142 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 798a931a..4d56a6af 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -179,12 +179,15 @@ public final class SenderPool implements AutoCloseable { // retiredSlots and returns any index whose flock has since dropped. // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; - // The retired slots behind the leakedSlots count (runtime reclaim paths - // only; startup-recovery retirements stay permanent — their transient - // recoverer sender is out of scope by retire time). Re-probed by - // reprobeRetiredSlots() so a late flock release (deferred engine cleanup - // on a worker exit path) restores the pool's capacity instead of - // ratcheting it down until process exit. Guarded by lock. + // The retired slots behind the leakedSlots count: runtime reclaim paths + // (discardBroken/reapIdle via reclaimSlot) and the in-range startup- + // recovery pass (recoverOneSlotStep, which retains the recoverer slot for + // exactly this purpose). Re-probed by reprobeRetiredSlots() so a late + // flock release (deferred engine cleanup on a worker exit path) restores + // the pool's capacity instead of ratcheting it down until process exit. + // Out-of-range startup recoverers are NEVER added: they carry no + // leakedSlots tick and their index has no slotInUse entry to free. + // Guarded by lock. private final ArrayList retiredSlots = new ArrayList<>(); // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the @@ -480,7 +483,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryComplete = true; return false; } - final boolean[] flockHeld = new boolean[1]; + final SenderSlot[] retained = new SenderSlot[1]; // Pass 1: in-range managed slots [0, maxSize). Skip live and empty slots // cheaply; spend the step on the first slot that actually holds data. @@ -526,23 +529,29 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { // A real candidate -> spend the step on it. Advance the cursor first // so a resume never reprocesses this index. recoveryInRangeNext++; - boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, flockHeld); + boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { // Release the recovery reservation accounting; from here either // leakedSlots (retire) or the freed index carries the cap math. recoveringSlots--; - if (flockHeld[0]) { + if (retained[0] != null) { // close() retained the flock because an I/O or manager - // worker did not stop. Retire the slot permanently (mirror + // worker did not stop. Retire the slot (mirror // discardBroken/reapIdle): keep slotInUse[i] set and count it // in leakedSlots so the borrow() cap math accounts for the // lost capacity and no later borrow ever reuses the - // still-locked dir. + // still-locked dir. Keep the recoverer in retiredSlots so + // reprobeRetiredSlots() restores the capacity once the + // deferred engine cleanup releases the flock — without it + // the retirement would be permanent even after the release + // (fatal at maxSize=1: every later borrow would time out). leakedSlots++; - LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with " + retiredSlots.add(retained[0]); + LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with " + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " - + "now {} of {} usable [leakedSlots={}]", + + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered " + + "if the worker releases the flock later", i, maxSize - leakedSlots, maxSize, leakedSlots); } else { slotInUse[i] = false; @@ -586,15 +595,17 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (!OrphanScanner.isCandidateOrphan(slotPath)) { continue; } - boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, flockHeld); - if (flockHeld[0]) { + boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); + if (retained[0] != null) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this // dir, so a still-held flock only leaks this recoverer's // worker-reachable resources (a best-effort teardown loss, // logged). Crucially we do // NOT touch leakedSlots -- that would wrongly shrink the - // in-range pool capacity. + // in-range pool capacity -- and we do NOT add to retiredSlots: + // there is no capacity to recover, and freeSlotIndex(idx) + // would index past the slotInUse array (sized maxSize). LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held " + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); @@ -616,17 +627,20 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { * (whose {@link #defaultSender} derives the dir {@code -slotIndex}), * drains its unacked data, and closes the delegate. Shared by both recovery * passes -- the in-range pass and the out-of-range pass -- which differ only - * in their slot bookkeeping, handled by the caller via {@code flockHeld}. + * in their slot bookkeeping, handled by the caller via {@code retainedOut}. * - * @param flockHeld single-element out-param set to {@code true} iff a - * recoverer was built and its {@code close()} returned with - * the flock still held because a worker did not stop + * @param retainedOut single-element out-param set to the recoverer iff one + * was built and its {@code close()} returned with the + * flock still held because a worker did not stop; the + * in-range caller keeps it in {@link #retiredSlots} so a + * late flock release can be re-probed. {@code null} when + * the flock was released (or no recoverer was built). * @return {@code true} if a build/drain failure occurred that will very * likely repeat for every remaining slot, so the caller should stop scanning */ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, - long remainingMillis, boolean[] flockHeld) { - flockHeld[0] = false; + long remainingMillis, SenderSlot[] retainedOut) { + retainedOut[0] = null; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and // delegate().close() can retain it when an I/O or manager worker does @@ -678,8 +692,8 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, LOG.warn("startup SF recovery: scan failed for slot {} ({})", slotPath, scanErr.toString()); } - if (recoverer != null) { - flockHeld[0] = !flockReleased(recoverer); + if (recoverer != null && !flockReleased(recoverer)) { + retainedOut[0] = recoverer; } return stopScan; } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 75aca494..60519571 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -875,7 +875,7 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th // C1 regression: the startup recovery loop MUST mirror discardBroken / // reapIdle. When a recoverer's delegate close() returns with the SF // flock still held (the I/O thread refused to stop), the recovered slot - // index must be retired permanently (leakedSlots++, slotInUse stays + // index must be retired (leakedSlots++, slotInUse stays // set) -- NOT freed. Freeing it would let a later borrow re-pick the // still-locked dir and resurrect "sf slot already in use", the exact // failure class this PR exists to kill. Pre-fix the recovery finally @@ -951,7 +951,7 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th try { Assert.assertTrue("borrow must use a fresh slot dir", Files.exists(slot("default-1"))); - // Capacity is permanently reduced by the leaked slot: + // Capacity stays reduced while the flock is held: // max=2, one leaked + one live => the next borrow times // out rather than colliding on the locked dir. try { @@ -979,6 +979,108 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th }); } + @Test + public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Exception { + // Recovery twin of + // testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld: a + // slot retired by STARTUP recovery (recoverer close() returned with + // the flock still held) must not stay lost until process exit. + // isSlotLockReleased() is no longer a one-shot snapshot -- deferred + // engine cleanup on a worker exit path can release the flock later -- + // so the pool must keep the recoverer in retiredSlots and re-probe it. + // Pre-fix, startup recovery only ticked leakedSlots and dropped the + // recoverer: at maxSize=1 every later borrow timed out forever even + // after the flock dropped. This test is RED until the recoverer is + // retained. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1 -- the worst case, where the single slot's + // retirement starves the whole pool. Forge the retention exactly + // like the retire test (closed=true makes the recovery drain and + // close a no-op, so the real flock stays held and slotLockReleased + // stays false), but only for the FIRST build of slot 0: the + // post-recovery borrow below must get a working delegate on the + // recovered index. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 500, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // While the flock is genuinely held, borrows time out: the + // cap-check re-probe finds the flock still reported held. + try { + pool.borrow(); + Assert.fail("borrow must time out while the slot is retired"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + } + + // The late release: the "wedged worker" finishes and the + // flock genuinely drops (un-forge and close the recoverer + // for real; in production this flip comes from + // isSlotLockReleased() re-probing the retained engine after + // the worker exited). + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + // The capacity-starved borrow must re-probe the startup- + // retired slot, recover its capacity, and create on the + // freed index -- no housekeeper tick required. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the startup-retired slot's capacity", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertTrue("recovered index 0 must carry the new borrow", slotInUse[0]); + Assert.assertEquals(1, countSlotDirs()); + } finally { + b.close(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Concurrency stress: borrow/return churn must never collide on a slot. // ---------------------------------------------------------------------- From d990470aaeb239bfa83b362c9be1661aed41122d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 23:55:08 +0100 Subject: [PATCH 10/64] fix(qwp): give capacity-starved borrows a final retired-slot probe before the timeout check borrow() ran the terminal timeout check before reprobeRetiredSlots(), so a zero-timeout (try-once) borrow threw without its one probe, and a borrower whose awaitNanos budget expired mid-wait timed out on capacity that a deferred engine cleanup had already returned (the delegate-side flock release never signals slotReleased). Hoist the probe above the timeout check so both paths recover the capacity instead of failing. Also pre-size retiredSlots to maxSize: every entry keeps a distinct in-range slot index reserved, so add() can never grow the backing array -- a retire (leakedSlots++ then add, under lock) can no longer fail on allocation and strand a counted-but-untracked slot that reprobeRetiredSlots() could never recover. Tests: - testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing (red pre-fix) - testParkedBorrowerGetsFinalProbeAfterBudgetExpiry (red pre-fix) - testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge: full-stack retire/recover cycle with no forged flags -- real wedged manager worker, real timed-out close handoff, real flock release, and a re-borrow on the recovered index proving the slot dir is genuinely reusable --- .../io/questdb/client/impl/SenderPool.java | 29 +- .../client/test/impl/SenderPoolSfTest.java | 308 ++++++++++++++++++ 2 files changed, 329 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 4d56a6af..34b59191 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -187,8 +187,12 @@ public final class SenderPool implements AutoCloseable { // the pool's capacity instead of ratcheting it down until process exit. // Out-of-range startup recoverers are NEVER added: they carry no // leakedSlots tick and their index has no slotInUse entry to free. - // Guarded by lock. - private final ArrayList retiredSlots = new ArrayList<>(); + // Pre-sized to maxSize (every entry keeps a distinct in-range slot index + // reserved, so size can never exceed maxSize): add() never grows the + // backing array, so a retire (leakedSlots++ then add, under lock) cannot + // fail on allocation and strand a counted-but-untracked slot that + // reprobeRetiredSlots() could never recover. Guarded by lock. + private final ArrayList retiredSlots; // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a @@ -306,6 +310,7 @@ public SenderPool( this.maxLifetimeMillis = maxLifetimeMillis; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.retiredSlots = new ArrayList<>(maxSize); this.slotReleased = lock.newCondition(); // Probe the config once, up front: this validates it eagerly (so a // bad config fails at construction even when minSize == 0) and tells @@ -793,16 +798,24 @@ public PooledSender borrow() { created.bumpGeneration(); return new PooledSender(created, created.generation()); } + // Capacity-starved: re-probe retired slots BEFORE the terminal + // timeout check — a deferred engine cleanup may have released a + // flock since the retire, and the freed index can admit a + // creation right now. The release itself never signals + // slotReleased (it happens in the delegate on a worker/I/O-thread + // exit path, volatile writes only), so this poll is the only way + // a borrower learns of it. Ordering matters twice over: a + // zero-timeout (try-once) borrow must get its one probe before + // throwing, and a borrower whose awaitNanos budget just expired + // must get a final probe on its wake-up pass instead of timing + // out on capacity that has already come back. + if (reprobeRetiredSlots()) { + continue; + } if (remainingNanos <= 0) { throw new LineSenderException( "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } - // Capacity-starved: before parking, re-probe retired slots — a - // deferred engine cleanup may have released a flock since the - // retire, and the freed index can admit a creation right now. - if (reprobeRetiredSlots()) { - continue; - } try { remainingNanos = slotReleased.awaitNanos(remainingNanos); } catch (InterruptedException e) { diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 60519571..abe95d56 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -30,7 +30,9 @@ import ch.qos.logback.core.read.ListAppender; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -53,6 +55,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -771,6 +774,116 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { }); } + @Test + public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Exception { + // Full-stack twin of the two forged-flag recovery tests above: no + // reflection-forged slotLockReleased anywhere. The REAL mechanism is + // driven end to end — the delegate's owned SegmentManager worker is + // wedged mid service pass (test hook), the delegate's close() takes + // the real timed-out-join → worker-exit handoff path, the pool + // retires the slot off the delegate's genuine isSlotLockReleased() + // report, the worker's deferred cleanup releases the real flock, and + // the housekeeper re-probe restores capacity — proving the recovered + // index is genuinely reusable by borrowing on it again (a forged flag + // would pass the accounting asserts but collide on the flock here). + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + Assert.assertNotNull("SF delegate must own a cursor engine", engine); + SegmentManager manager = (SegmentManager) getField(engine, "manager"); + + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + // Park the manager worker inside a service pass for + // the delegate's ring. The trim-sync point is reached + // on every ~1ms tick, so this wedges deterministically. + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + Assert.assertTrue("manager worker never entered a service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Real teardown against the wedged worker: the + // delegate's engine close times out its bounded join, + // hands cleanup to the worker's exit path, and reports + // the retained flock; the pool must retire the slot. + manager.setWorkerJoinTimeoutMillis(50L); + invokeDiscardBroken(pool, a); + Assert.assertEquals( + "pool must retire the slot while the delegate's manager " + + "worker holds the deferred cleanup", + 1, pool.leakedSlotCount()); + Assert.assertFalse("engine cleanup must still be pending", + engine.isCloseCompleted()); + + // Un-wedge: the worker finishes its pass, exits (its + // loop was stopped by the close), and runs the + // deferred cleanup that releases the real flock. + releaseWorker.countDown(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError( + "deferred engine cleanup never ran on manager-worker exit"); + } + Thread.sleep(1); + } + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertFalse("recovered slot index 0 must return to the free set", + slotInUse[0]); + + // The proof a forged flag cannot fake: both indices — + // including the recovered one, whose flock was really + // dropped by the worker-exit cleanup — admit live + // senders again. + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + if (hookErr.get() != null) { + throw new AssertionError("trim hook failed", hookErr.get()); + } + } finally { + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- @@ -1081,6 +1194,201 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except }); } + @Test + public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Exception { + // Boundary twin of testStartupRetiredSlotRecoveredAfterLateFlockRelease: + // acquireTimeoutMillis=0 is a valid try-once borrow (builder rejects only + // < 0). Pre-fix, borrow() ran the terminal timeout check BEFORE + // reprobeRetiredSlots(), so a zero-budget borrow threw "timed out" + // without its one probe -- even when the retired slot's flock had + // already dropped and a probe would have restored capacity and admitted + // a creation. Deterministic: no housekeeper runs in this test, so + // borrow() is the only reprobe driver. Recovery is driven manually via + // the deferred-pool step helper because the inline path reuses + // acquireTimeoutMillis as its recovery budget -- 0 would skip recovery + // outright. This test is RED until the probe is hoisted above the + // timeout check. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1, zero acquire budget, deferred recovery driven + // step-by-step (the housekeeper's per-tick unit, budgeted by + // RECOVERY_DRAIN_BUDGET_MILLIS, not the zero acquire budget). Forge + // the retirement (closed=true makes the recovery drain and close a + // no-op, so the real flock stays held), then release the flock for + // real BEFORE borrowing: the recovery is discoverable, but only via + // a probe. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newDeferredPoolWithFactory(cfg2, 0, 1, 0, factory)) { + //noinspection StatementWithEmptyBody + while (invokeRunStartupRecoveryStep(pool)) { + // drive the whole backlog, one housekeeper-tick unit at a time + } + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // The late release: un-forge and close the recoverer for + // real. The flock genuinely drops, but nothing signals the + // pool -- the release happens in the delegate, volatile + // writes only. + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + // Try-once borrow: its single pass must probe, recover the + // capacity, and create on the freed index. Pre-fix this + // threw "timed out" without ever probing. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("zero-timeout borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } + } + } + }); + } + + @Test + public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception { + // Release-during-wait twin of the zero-timeout test. A borrower parks + // in awaitNanos while the retired slot's flock is genuinely held; the + // deferred cleanup then releases the flock mid-wait. Nothing signals + // slotReleased (the release is delegate-side, volatile writes only), so + // the borrower sleeps out its full budget. Pre-fix its wake-up pass hit + // the terminal timeout check before reprobeRetiredSlots() and threw -- + // missing capacity that had already come back. Post-fix the wake-up + // pass probes first, recovers the index, and the creation is admitted. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1, generous budget so the mid-wait release lands + // comfortably inside it. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 2_000, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // Borrower parks: its pre-park probe correctly fails while + // the flock is genuinely held. + AtomicReference failure = new AtomicReference<>(); + AtomicReference borrowed = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + borrowed.set(pool.borrow()); + } catch (Throwable e) { + failure.set(e); + } + }); + borrower.start(); + + // Let the borrower enter borrow() and park. If a CI stall + // delays it past the release below, the pre-park probe + // recovers instead and the test degrades to a pass -- never + // a flaky failure. + Thread.sleep(300); + + // Mid-wait release: flock drops, no signal reaches the pool. + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + borrower.join(10_000); + Assert.assertFalse("borrower must have finished", borrower.isAlive()); + if (failure.get() != null) { + throw new AssertionError( + "borrower must recover the retired slot on its wake-up pass, not time out", + failure.get()); + } + PooledSender b = borrowed.get(); + Assert.assertNotNull("borrower must have obtained a sender", b); + try { + Assert.assertEquals("wake-up probe must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Concurrency stress: borrow/return churn must never collide on a slot. // ---------------------------------------------------------------------- From 7df62d5d6925eedee0a6461c5e18bd73b70d0450 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 23:55:27 +0100 Subject: [PATCH 11/64] test(qwp): close the remaining engine/manager shutdown coverage gaps Deterministic regression tests for the shutdown paths the coverage review flagged as untested: - SegmentManagerCloseRaceTest#testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim: a snapshot entry deregistered before the worker claims it must be skipped at claim time (serviced rings recorded from the worker's own trim-sync point via inService, so the assertion is exact -- no sleeps). - SegmentManagerCloseRaceTest#testWorkerAloneFreesPathScratchAfterTimedOutClose: after a timed-out close hands pathScratch to the worker, the worker's exit block alone must free it -- no retried close() runs in production, so the sibling test's retry-then-assert shape masked a regression here. - CursorSendEngineSlotReacquisitionTest#testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff: a retried close() racing the worker parked mid-finishClose must lose the terminalCleanupClaimed CAS -- no double cleanup, no premature completion, flock untouched until the worker's release. - CursorSendEngineSlotReacquisitionTest#testMemoryModeOwnedCloseHandsCleanupToWorkerExit: memory-mode (null sfDir/slotLock/watermark) timed-out close must take the same worker-exit handoff without NPE and free the ring's native memory. - EngineClosePublishAfterFlockReleaseTest#testUnconfirmedFlockReleaseKeepsCloseIncomplete: a failed flock release must never publish closeCompleted, and a retried close() must neither throw nor fabricate completion. - SlotLockTest#testFailedCloseRetainsFdAndReportsFalse: release()==false retains the fd for retry and keeps reporting false while the failure persists. - SlotLockReleasedContractTest#testDelegatedIoThreadEngineCloseFlipsSlotLockReleased: the delegated-I/O-close branch (delegateEngineClose()==true) must retain the engine so isSlotLockReleased() flips true once the I/O thread's exit path releases the flock; the existing forged I/O-refusal test throws from delegateEngineClose() before the retained-engine assignment and can never reach this branch. Mutation-verified: dropping the retainedEngine assignment fails the test with the pinned message. --- .../client/SlotLockReleasedContractTest.java | 181 +++++++++++++ ...CursorSendEngineSlotReacquisitionTest.java | 250 ++++++++++++++++++ ...gineClosePublishAfterFlockReleaseTest.java | 66 +++++ .../cursor/SegmentManagerCloseRaceTest.java | 236 +++++++++++++++++ .../qwp/client/sf/cursor/SlotLockTest.java | 49 ++++ 5 files changed, 782 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index e8ef2670..0a6c0895 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -24,10 +24,13 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; @@ -305,6 +308,162 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception }); } + /** + * Delegated-I/O-close recovery path: when {@code close()} bails early + * because the I/O thread refuses to stop, the engine close is delegated + * to that thread's exit path ({@code delegateEngineClose()} returned + * true) and the sender must retain the engine for re-probing — + * {@code isSlotLockReleased()} stays false while the thread lives, then + * flips true the moment the thread's exit path completes the engine + * close and releases the flock. Pre-fix, this branch discarded the + * engine reference, so the late release was permanently invisible to a + * pool that had retired the slot. The existing forged I/O-refusal test + * can never reach this branch: its sabotaged loop throws from + * {@code delegateEngineClose()} itself (nulled latch), before the + * retained-engine assignment. + *

    + * The wedge is the same deterministic one CursorWebSocketSendLoop's C5 + * test uses: the I/O thread sits in a blocking "connect" (a + * ReconnectFactory immune to unpark, never interrupted by loop.close()), + * and the closer thread's pre-set interrupt flag makes + * {@code shutdownLatch.await()} throw immediately — the real production + * path to {@code ioThreadStopped = false}. + */ + @Test(timeout = 30_000L) + public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-delegated-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + final CountDownLatch enteredConnect = new CountDownLatch(1); + final CountDownLatch releaseConnect = new CountDownLatch(1); + final AtomicReference ioThreadRef = new AtomicReference<>(); + final StubWebSocketClient stubClient = new StubWebSocketClient(); + // Healthy owned manager: once the wedged I/O thread is released, + // its exit-path engine.close() completes normally and releases + // the flock — the flip this test pins. + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + QwpWebSocketSender wss = null; + try { + // Stand-in for a blocking native connect(2): entered by the + // loop's I/O thread, immune to unpark, never interrupted by + // loop.close(). + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return stubClient; + }; + loop = new CursorWebSocketSendLoop( + null /* async-initial-connect: the I/O thread drives the connect */, + engine, 0L, 1_000L, + stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + setField(wss, "cursorSendLoop", loop); + + // Drive the real early-bail close() on a thread whose pending + // interrupt lands in loop.close()'s shutdownLatch.await(). + AtomicReference closeFailure = new AtomicReference<>(); + QwpWebSocketSender wssRef = wss; + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + wssRef.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "delegated-close-closer"); + closer.setDaemon(true); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + // The I/O thread is still wedged: the flock is retained and + // must be reported retained. + Assert.assertFalse( + "isSlotLockReleased() must be false while the delegated engine " + + "close is pending on the wedged I/O thread", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must not have run yet", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while the delegated engine close " + + "was still pending"); + } catch (Exception expected) { + // good — the flock is genuinely held. + } + + // Un-wedge the connect. The I/O thread exits; its exit path + // runs the delegated engine.close(), which releases the flock. + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after the connect returned", + ioThread.isAlive()); + + // The recovery contract under test: the getter re-probes the + // retained engine, so the late release MUST become visible — + // this is what lets SenderPool recover the retired slot. + Assert.assertTrue("delegated engine close must have completed on the " + + "I/O thread's exit path", + engine.isCloseCompleted()); + Assert.assertTrue( + "isSlotLockReleased() must flip true once the delegated engine " + + "close released the flock — otherwise the pool retires the " + + "slot's capacity until process exit", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the delegated close", probe); + } + } finally { + releaseConnect.countDown(); + // Reap the loop's bookkeeping now that the I/O thread is gone + // (close() threw mid-teardown, so ioThread was left set). + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + if (engine != null && !engine.isCloseCompleted()) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + // The early-return close() deliberately leaked the resources + // the (then-running) I/O thread might touch; free the same set + // the post-guard tail would have freed. + if (wss != null) { + freeFieldQuietly(wss, "buffer0"); + freeFieldQuietly(wss, "buffer1"); + freeFieldQuietly(wss, "client"); + freeFieldQuietly(wss, "errorDispatcher"); + freeFieldQuietly(wss, "progressDispatcher"); + freeFieldQuietly(wss, "connectionDispatcher"); + } + stubClient.close(); + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -370,6 +529,28 @@ private static void setField(Object target, String name, Object value) throws Ex throw new NoSuchFieldException(name); } + /** + * Minimal concrete {@link WebSocketClient} — never performs I/O. Handed + * to the loop by the stuck-connect factory; the loop's exit path closes + * it (close is idempotent via the superclass). + */ + private static final class StubWebSocketClient extends WebSocketClient { + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + /** ACKs every binary frame with a running sequence so flush/close drain cleanly. */ private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler { private final AtomicLong nextSeq = new AtomicLong(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index ad25ff78..38cd1bcf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -43,6 +43,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -523,6 +524,255 @@ public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws }); } + /** + * Exactly-once contention on the terminal-cleanup claim + * ({@code terminalCleanupClaimed} CAS): after a timed-out owned close + * handed cleanup to the worker's exit path, a retried {@code close()} + * that races the worker MID-{@code finishClose} must neither re-run the + * terminal cleanup (double munmap / double flock release) nor block on + * the worker, nor publish completion on the worker's behalf. The race + * window is made deterministic by parking the worker inside + * {@code finishClose} (via {@code beforeFlockReleaseHook}) while the + * retried close() converges through {@code isWorkerReaped()} and loses + * the CAS. + */ + @Test(timeout = 30_000L) + public void testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/cas-contention-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch inFinishClose = new CountDownLatch(1); + CountDownLatch releaseFinishClose = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger finishCloseRuns = new AtomicInteger(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Counts terminal-cleanup executions and parks the FIRST one + // (the worker's deferred cleanup) mid-finishClose, before the + // flock release — the exact window a retried close() races. + engine.setBeforeFlockReleaseHook(() -> { + if (finishCloseRuns.incrementAndGet() == 1) { + inFinishClose.countDown(); + try { + if (!releaseFinishClose.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release finishClose")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + } + }); + + // Phase 2: rotate onto the spare so the worker parks in the + // next install pass. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out close — cleanup ownership transfers to + // the worker's exit path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + + // Phase 4: release the pass. The worker exits its loop, wins + // the cleanup CAS, enters finishClose and parks in the hook — + // mid-cleanup, flock still held, completion unpublished. + releaseWorker.countDown(); + Assert.assertTrue("worker never entered the deferred finishClose", + inFinishClose.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, finishCloseRuns.get()); + Assert.assertFalse("completion must not be observable mid-finishClose", + engine.isCloseCompleted()); + + // Phase 5 — the contention under test: a retried close() while + // the worker is parked INSIDE finishClose. The worker loop has + // already exited (workerLoopExited=true precedes the deferred + // cleanups), so the short bounded join reaps the manager state + // and close() converges to the CAS — which it must LOSE, + // returning promptly without touching ring/watermark/flock. + engine.close(); + Assert.assertEquals( + "retried close() re-ran the terminal cleanup while the worker's " + + "deferred cleanup was mid-flight — ring/watermark/flock would " + + "be double-released", + 1, finishCloseRuns.get()); + Assert.assertFalse("retried close() must not publish completion on the worker's behalf", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot lock observable as released while the worker was still " + + "parked before its flock release"); + } catch (Exception expected) { + // good — flock still held by the parked cleanup. + } + + // Phase 6: let the worker finish. Completion publishes, the + // slot becomes acquirable, and the cleanup count stays at 1. + releaseFinishClose.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError("deferred cleanup never completed after release"); + } + Thread.sleep(1); + } + Assert.assertEquals(1, finishCloseRuns.get()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } + // A final close() takes the fast no-op path. + engine.close(); + Assert.assertEquals("post-completion close() must be a no-op", + 1, finishCloseRuns.get()); + if (hookErr.get() != null) { + throw new AssertionError("hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + engine.setBeforeFlockReleaseHook(null); + releaseWorker.countDown(); + releaseFinishClose.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Memory-mode twin of {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: + * {@code sfDir == null}, so there is no slot lock, no watermark and no + * segment files — but the ring's malloc'd native segments are still + * worker-reachable, so the timed-out close must take the same handoff + * path with every SF-only resource null. Pins that (a) the handoff + * branch tolerates null slotLock/watermark/sfDir without NPE, (b) the + * close stays incomplete while the worker can still touch the ring, and + * (c) the worker-exit cleanup completes the close and frees the ring's + * native memory (assertMemoryLeak is the leak oracle here). + */ + @Test(timeout = 30_000L) + public void testMemoryModeOwnedCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Memory mode: null sfDir, private owned manager — the exact + // shape non-SF async ingest uses. + CursorSendEngine engine = new CursorSendEngine(null, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: rotate onto the spare; the worker's next tick + // re-enters the (in-memory) install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out memory-mode close. The worker can still + // touch the ring's native memory, so the close must hand off + // and stay incomplete — releasing the ring here would be a + // use-after-free on the worker's install path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse( + "memory-mode close must stay incomplete while the worker is mid service pass", + engine.isCloseCompleted()); + + // Phase 4: release the worker; its exit path must run the + // deferred cleanup (null slotLock/watermark/sfDir) and flip + // completion with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred memory-mode cleanup never ran on manager-worker exit — " + + "the ring's native segments would leak for the process lifetime"); + } + Thread.sleep(1); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index cd7fdff0..5dcf6def 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -32,12 +32,14 @@ import org.junit.Before; import org.junit.Test; +import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -173,4 +175,68 @@ public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws E } }); } + + /** + * The error half of the publish-after-release contract: when + * {@code SlotLock.release()} reports failure (the OS refused the fd + * close, so the flock may still be held), {@code closeCompleted} must + * NEVER be published — not by the failing close(), and not by a retried + * close() either (the terminal-cleanup claim is already consumed; the + * design deliberately leaves an unconfirmed release to the kernel's + * process-exit cleanup rather than risking a double release). A pool + * observing {@code isCloseCompleted() == false} keeps the slot retired, + * which is exactly right: the flock genuinely is still held. + *

    + * {@code Files.close} cannot be made to fail through the public API, so + * the lock's fd is swapped to a known-bad descriptor for the close and + * restored (and released for real) afterwards. + */ + @Test(timeout = 30_000L) + public void testUnconfirmedFlockReleaseKeepsCloseIncomplete() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); + slotLockField.setAccessible(true); + SlotLock slotLock = (SlotLock) slotLockField.get(engine); + assertNotNull("disk-mode engine must hold a slot lock", slotLock); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + assertTrue("precondition: live flock fd", realFd >= 0); + try { + // A non-negative fd no process has open: close(2) fails EBADF, + // so finishClose's release() confirmation fails. + fdField.setInt(slotLock, 1_000_000_000); + engine.close(); + assertFalse( + "closeCompleted was published despite an unconfirmed flock release; " + + "a pool observing this would free the slot index while the " + + "flock fd is still open", + engine.isCloseCompleted()); + // The REAL flock is still held — the incomplete report is true. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — incomplete close really means "still locked". + } + // A retried close() must neither throw nor re-run the terminal + // cleanup (the claim is consumed) nor suddenly report success. + engine.close(); + assertFalse("retried close() must not fabricate completion after a failed " + + "flock release — the claim is consumed and the kernel owns " + + "the eventual cleanup", + engine.isCloseCompleted()); + } finally { + // Undo the fault and drop the real flock so the test leaves no + // open fd behind (in production the kernel does this at exit). + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — with the flock genuinely dropped, the slot is reusable. + } + }); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 9f6fc225..cee8e41d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,224 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the claim-time registration gate at the top of + * {@code SegmentManager.serviceRing}: a snapshot entry whose ring was + * deregistered BEFORE the worker claims it must be skipped entirely — + * never claimed as {@code inService}, never handed to + * {@code serviceRing0}. This is the exact guarantee + * {@code CursorSendEngine.close()} relies on when it releases the ring, + * watermark and slot flock right after + * {@code awaitRingQuiescence(ring) == true}: the deregistering thread may + * already be freeing those resources, so a stale snapshot entry must not + * be touched at all (spare install, watermark write, drainTrimmable, or + * path building under the slot dir). + *

    + * Deterministic shape: three rings A, B, C registered before start, so + * the worker's first snapshot is exactly [A, B, C]. The worker parks in + * A's spare-install pass; while it is parked, B is deregistered (and a + * quiescence barrier for B must pass immediately — no pass for B is in + * flight). After release the worker walks the rest of its stale + * snapshot: it must skip B and service C. Every serviced ring is + * recorded from inside the worker's own pass (via the trim-sync hook + * reading {@code inService}), so the assertion is exact — no timing + * grace, no sleeps. + */ + @Test(timeout = 15_000L) + public void testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + SegmentRing[] rings = new SegmentRing[3]; + String[] slots = new String[3]; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Rings the worker actually claimed and serviced, recorded from + // the worker thread itself at the trim-sync point every service + // pass reaches (spare needed or not). + java.util.Set serviced = + java.util.Collections.synchronizedSet( + java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>())); + boolean managerClosed = false; + try { + for (int i = 0; i < 3; i++) { + slots[i] = tmpDir + "/claim-skip-slot-" + i; + Assert.assertEquals(0, Files.mkdir(slots[i], Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create( + slots[i] + "/sf-initial.sfa", 0L, segSize); + rings[i] = new SegmentRing(initial, segSize); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.setBeforeTrimSyncHook(() -> { + try { + Object ring = readInServiceRing(manager); + if (ring != null) { + serviced.add(ring); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Register all three BEFORE start: the worker's first snapshot + // is [A, B, C] and every fresh ring wants a hot spare, so the + // install hook parks the worker inside A's pass. + manager.register(rings[0], slots[0]); + manager.register(rings[1], slots[1]); + manager.register(rings[2], slots[2]); + manager.start(); + Assert.assertTrue("worker did not reach A's install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // B is deregistered while its snapshot entry is still ahead of + // the worker's cursor. The quiescence barrier must pass at + // once: the in-flight pass is A's, not B's — this is the state + // in which an engine owner frees B's resources. + manager.deregister(rings[1]); + Assert.assertTrue("no pass for B is in flight — the barrier must pass immediately", + manager.awaitRingQuiescence(rings[1])); + + releaseWorker.countDown(); + + // Positive marker that the worker walked PAST B's snapshot + // slot: C sits after B in the same snapshot, so once C has + // been serviced, B's claim-or-skip decision has been made. + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!serviced.contains(rings[2])) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("worker never serviced ring C after release"); + } + Thread.sleep(1); + } + + Assert.assertTrue("ring A must have been serviced", serviced.contains(rings[0])); + Assert.assertFalse( + "worker claimed and serviced a snapshot entry that was deregistered " + + "before its pass started — the deregistering thread may already " + + "be releasing the ring/watermark/slot lock, so a stale snapshot " + + "entry must be skipped at claim time", + serviced.contains(rings[1])); + // Defense in depth: nothing may have been installed into the + // deregistered ring either. + Assert.assertNull("no hot spare may be installed into a deregistered ring", + readHotSpare(rings[1])); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("worker-side hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + manager.close(); + } + for (SegmentRing ring : rings) { + if (ring != null) { + ring.close(); + } + } + } + }); + } + + /** + * Pins the scratch-handoff half of the timed-out-close contract in + * isolation: after {@code close()} gives up on the bounded join and hands + * {@code pathScratch} ownership to the worker, the WORKER's exit block + * alone must free the native buffer — with no retried {@code close()} + * ever running. The sibling test + * ({@link #testCloseDoesNotFreePathScratchWhenWorkerStillAlive}) retries + * {@code close()} before asserting the free, so a regression that dropped + * the worker-side free (leaving reclaim to a retry nobody is required to + * make) would stay green there: production owners do NOT retry — a + * timed-out close returns to the pool and the worker is the only thread + * left that can reclaim the allocation. + */ + @Test(timeout = 15_000L) + public void testWorkerAloneFreesPathScratchAfterTimedOutClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/worker-frees-scratch-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: hands scratch ownership to the worker and + // returns. This is the last close() call this test makes. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Thread worker = readWorkerThread(manager); + Assert.assertTrue("worker must still be live after the timed-out close", + worker != null && worker.isAlive()); + Assert.assertTrue("scratch must still be allocated while the worker may use it", + readPathScratchImpl(manager) != 0L); + + // Release the pass. running=false already, so the worker + // finishes the pass and exits — its exit block must free the + // handed-over scratch buffer without ANY further caller action. + releaseWorker.countDown(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("worker never exited after release", worker.isAlive()); + Assert.assertEquals( + "worker exit block must free the handed-over path scratch — no retried " + + "close() runs in production after a timed-out close, so leaving " + + "the free to a retry leaks the native buffer for the process " + + "lifetime", + 0L, readPathScratchImpl(manager)); + + // Reap the dead thread for tidiness; must not double-free. + manager.close(); + Assert.assertNull("retried close must reap the exited worker", + readWorkerThread(manager)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.close(); + ring.close(); + } + }); + } + /** * Pins the {@link SegmentManager#deferUntilWorkerExit} handoff contract * that {@code CursorSendEngine.close()}'s slot-ownership transfer depends @@ -473,6 +691,24 @@ private static void cleanupRecursively(String dir) { } } + private static Object readHotSpare(SegmentRing ring) throws Exception { + Field f = SegmentRing.class.getDeclaredField("hotSpare"); + f.setAccessible(true); + return f.get(ring); + } + + private static Object readInServiceRing(SegmentManager manager) throws Exception { + Field inServiceF = SegmentManager.class.getDeclaredField("inService"); + inServiceF.setAccessible(true); + Object entry = inServiceF.get(manager); + if (entry == null) { + return null; + } + Field ringF = entry.getClass().getDeclaredField("ring"); + ringF.setAccessible(true); + return ringF.get(entry); + } + private static long readPathScratchImpl(SegmentManager manager) throws Exception { Field pathScratchF = SegmentManager.class.getDeclaredField("pathScratch"); pathScratchF.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 0d4b9f43..015cb0d8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -34,6 +34,7 @@ import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -122,6 +123,54 @@ public void testReleaseConfirmsAndIsIdempotent() throws Exception { }); } + /** + * The {@code release() == false} branch: when the OS reports a close + * failure, release must (a) return {@code false} so owners gating a + * "slot reusable" signal never see a lie, (b) RETAIN the fd — forgetting + * it would misreport the lock state and forfeit any later retry — and + * (c) keep returning {@code false} on repeat attempts while the failure + * persists. Once the close succeeds, release confirms and stays + * confirmed. {@code Files.close} cannot be made to fail through the + * public API, so the fd is swapped to a known-bad descriptor and + * restored afterwards. + */ + @Test + public void testFailedCloseRetainsFdAndReportsFalse() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-release"; + SlotLock lock = SlotLock.acquire(slot); + java.lang.reflect.Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(lock); + assertTrue("precondition: acquire must hold a live fd", realFd >= 0); + try { + // A non-negative fd no process has open: close(2) fails EBADF. + fdField.setInt(lock, 1_000_000_000); + assertFalse("release must report false when the OS close fails", + lock.release()); + assertEquals("failed release must retain the fd for a retry — " + + "dropping it would misreport the flock as released", + 1_000_000_000, fdField.getInt(lock)); + assertFalse("repeat release must keep reporting false while the failure persists", + lock.release()); + // While the release is unconfirmed the REAL flock is still + // held — a second acquire on the slot must fail. + try (SlotLock ignored = SlotLock.acquire(slot)) { + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — unconfirmed release really means "still locked". + } + } finally { + fdField.setInt(lock, realFd); + } + assertTrue("release must confirm once the close succeeds", lock.release()); + assertTrue("confirmed release must stay confirmed", lock.release()); + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { From 79e08475ae6418c6cf56df9fedc2f06d90a1dbb4 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 02:15:55 +0100 Subject: [PATCH 12/64] Make final-probe regression deterministic Release the retired slot from a test-only hook after the positive borrow wait has actually exhausted its budget. This removes the scheduler-dependent sleep and guarantees the test reaches the final post-wait probe. --- .../io/questdb/client/impl/SenderPool.java | 16 ++++ .../client/test/impl/SenderPoolSfTest.java | 77 ++++++++----------- 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 34b59191..f11a5912 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -148,6 +148,11 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Test seam: runs after a capacity-starved borrow's condition wait has + // exhausted its positive timeout, before the loop's terminal pass. Null in + // production; regression tests release a retired slot here to prove that + // the terminal pass re-probes returned capacity before throwing. + private volatile Runnable borrowWaitExpiredHook; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the // flock drops, so the cap check and the slot allocator stay consistent @@ -818,6 +823,12 @@ public PooledSender borrow() { } try { remainingNanos = slotReleased.awaitNanos(remainingNanos); + if (remainingNanos <= 0) { + Runnable hook = borrowWaitExpiredHook; + if (hook != null) { + hook.run(); + } + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new LineSenderException("interrupted while waiting for a Sender from the pool"); @@ -830,6 +841,11 @@ public PooledSender borrow() { } } + @TestOnly + public void setBorrowWaitExpiredHook(Runnable hook) { + this.borrowWaitExpiredHook = hook; + } + /** * Raises the shutdown signal early -- without tearing down live delegates -- * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index abe95d56..a268b14a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -1290,14 +1290,13 @@ public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Except @Test public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception { - // Release-during-wait twin of the zero-timeout test. A borrower parks - // in awaitNanos while the retired slot's flock is genuinely held; the - // deferred cleanup then releases the flock mid-wait. Nothing signals - // slotReleased (the release is delegate-side, volatile writes only), so - // the borrower sleeps out its full budget. Pre-fix its wake-up pass hit - // the terminal timeout check before reprobeRetiredSlots() and threw -- - // missing capacity that had already come back. Post-fix the wake-up - // pass probes first, recovers the index, and the creation is admitted. + // Positive-timeout twin of the zero-timeout test. A borrower parks in + // awaitNanos while the retired slot's flock is genuinely held and + // sleeps out its full budget. A test hook releases the flock after the + // wait reports expiry but before the terminal loop pass. Pre-fix that + // pass hit the timeout check before reprobeRetiredSlots() and threw -- + // missing capacity that had already come back. Post-fix the terminal + // pass probes first, recovers the index, and admits the creation. TestUtils.assertMemoryLeak(() -> { // Phase 1: strand unacked data under default-0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { @@ -1318,8 +1317,9 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception Assert.assertTrue("unacked data must persist under default-0", hasSegmentFile(slot("default-0"))); - // Phase 2: maxSize=1, generous budget so the mid-wait release lands - // comfortably inside it. + // Phase 2: maxSize=1 and a positive acquire budget. A test hook + // releases the flock only after awaitNanos() has returned with that + // budget exhausted, so the terminal wake-up pass is deterministic. CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { int ackPort = ack.getPort(); @@ -1340,49 +1340,38 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception return real; }; - try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 2_000, factory)) { + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 100, factory)) { Assert.assertNotNull("recovery must have built slot 0", forged.get()); Assert.assertEquals("precondition: startup recovery must retire the slot", 1, pool.leakedSlotCount()); - // Borrower parks: its pre-park probe correctly fails while - // the flock is genuinely held. - AtomicReference failure = new AtomicReference<>(); - AtomicReference borrowed = new AtomicReference<>(); - Thread borrower = new Thread(() -> { + AtomicBoolean waitExpired = new AtomicBoolean(); + pool.setBorrowWaitExpiredHook(() -> { + Assert.assertTrue("expired-wait hook must run exactly once", + waitExpired.compareAndSet(false, true)); + Sender recoverer = forged.get(); try { - borrowed.set(pool.borrow()); - } catch (Throwable e) { - failure.set(e); + setBooleanField(recoverer, "closed", false); + } catch (Exception e) { + throw new RuntimeException(e); } + // The flock drops only after the positive awaitNanos() + // budget is exhausted. This delegate-side release does + // not signal slotReleased. + recoverer.close(); }); - borrower.start(); - - // Let the borrower enter borrow() and park. If a CI stall - // delays it past the release below, the pre-park probe - // recovers instead and the test degrades to a pass -- never - // a flaky failure. - Thread.sleep(300); - - // Mid-wait release: flock drops, no signal reaches the pool. - Sender recoverer = forged.get(); - setBooleanField(recoverer, "closed", false); - recoverer.close(); - - borrower.join(10_000); - Assert.assertFalse("borrower must have finished", borrower.isAlive()); - if (failure.get() != null) { - throw new AssertionError( - "borrower must recover the retired slot on its wake-up pass, not time out", - failure.get()); - } - PooledSender b = borrowed.get(); - Assert.assertNotNull("borrower must have obtained a sender", b); try { - Assert.assertEquals("wake-up probe must recover the retired slot's capacity", - 0, pool.leakedSlotCount()); + PooledSender b = pool.borrow(); + try { + Assert.assertTrue("borrow must exhaust its positive wait budget", + waitExpired.get()); + Assert.assertEquals("wake-up probe must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } } finally { - b.close(); + pool.setBorrowWaitExpiredHook(null); } } } From dd0c5478a27a0fd9c6a1b3f333371a46eec66d86 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 03:32:28 +0100 Subject: [PATCH 13/64] docs(qwp): fix load-bearing concurrency comments that contradict the code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five doc sites documented behavior the code deliberately does not have; each invited a future 'fix' that would reintroduce the hazard the quiescence work eliminated. Comment-only change. - CursorSendEngine.closeCompleted field doc: carve the failed-flock- release case out of the retry sentence. A retried close() exits at the consumed terminalCleanupClaimed CAS and never calls SlotLock.release() again — deliberate, pinned by testUnconfirmedFlockReleaseKeepsCloseIncomplete. - CursorSendEngine.finishClose javadoc: 'must hold the engine monitor' was false for completeDeferredClose (deliberately monitor-free to avoid the join livelock). State the real contract: monitor (close path) OR worker exit path; in all cases the CAS must be won. - CursorSendEngine.isCloseCompleted javadoc: admit the third, unrecoverable state — failed flock release never flips; only process exit frees the flock. - SegmentManager.isWorkerReaped javadoc: the fall-through reap nulls workerThread while the thread may still be running deferred engine cleanups; the engine-side CAS, not this predicate, is the exclusion. - SegmentManager.serviceRing0 trim comment: narrow 'stale snapshots' to mid-pass-deregistered entries — the claim gate makes the trim block unreachable for pre-pass-deregistered entries. - SenderPool reclaimSlot/retireLease: 'retired permanently' contradicted retiredSlots.add + reprobeRetiredSlots recovery three lines down. - QwpWebSocketSender post-guard comment: the incomplete-close branch is not only the owned-manager handoff; document the no-handoff cases (shared manager, failed handoff registration, failed flock release) where the re-probe never flips. --- .../qwp/client/QwpWebSocketSender.java | 22 ++++++++----- .../client/sf/cursor/CursorSendEngine.java | 31 +++++++++++++------ .../qwp/client/sf/cursor/SegmentManager.java | 18 ++++++++--- .../io/questdb/client/impl/SenderPool.java | 15 ++++++--- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index f1e3533c..268728cc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1272,13 +1272,21 @@ public void close() { ownsCursorEngine = false; slotLockReleased = true; } else { - // The manager worker did not quiesce. Preserve ownership - // and report the retained flock so pools retire this slot. - // Repeated Sender.close() calls remain no-ops by contract. - // Engine cleanup was handed to the worker's exit path - // (owned manager); the getter re-probes the retained - // engine so the pool can reclaim the slot once cleanup - // actually completes. + // Engine close() could not confirm a released flock. + // Preserve ownership and report the retained flock so + // pools retire this slot. Repeated Sender.close() calls + // remain no-ops by contract. Recovery depends on WHY the + // close is incomplete: when cleanup was handed to the + // worker's exit path (owned manager — the only shipping + // configuration), isSlotLockReleased()'s re-probe of the + // retained engine flips once that cleanup completes. + // When there was no handoff, the re-probe never flips + // and the slot stays retired until process exit: a + // shared-manager engine or a failed handoff registration + // needs a retried engine.close() that this one-shot + // sender never issues, and a failed flock release has + // consumed the engine's cleanup claim, so nothing can + // re-run the release. slotLockReleased = false; retainedEngine = engine; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index c9b85aea..42ca4d17 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -124,9 +124,16 @@ public final class CursorSendEngine implements QuietCloseable { // SenderPool.reprobeRetiredSlots), so publishing before the release // would let a replacement engine's SlotLock.acquire collide with the // still-open fd. Stays false when a close attempt could not confirm - // manager-worker quiescence (or the flock release itself failed) and had - // to leak the ring/watermark/slot lock — in that case a later close() - // call retries the cleanup (the worker may have exited by then). + // manager-worker quiescence and had to leak the ring/watermark/slot + // lock — in that case a later close() call retries the cleanup (the + // worker may have exited by then). Also stays false — permanently — + // when finishClose() ran but the flock release itself failed: the + // terminalCleanupClaimed CAS is consumed, so a retried close() exits at + // the CAS without re-running the cleanup or calling SlotLock.release() + // again. That non-retry is deliberate (a retry would double-close the + // ring/watermark and double-release the flock; pinned by + // testUnconfirmedFlockReleaseKeepsCloseIncomplete) — the kernel drops + // the flock at process exit. // volatile: latched by finishClose(), but read lock-free by // isCloseCompleted() from pool threads re-probing a retired slot (see the // getter for why it must not synchronize). @@ -674,7 +681,10 @@ public synchronized void close() { * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must * never be visible while the flock fd is still open, or a replacement * engine races the release and fails acquisition on a live slot. - * The caller must hold the engine monitor and + * The caller must either hold the engine monitor (the close() path) or + * run on the manager worker's exit path ({@link #completeDeferredClose}, + * which deliberately takes no monitor — adding one would recreate the + * join livelock the CAS exists to avoid); in all cases the caller * must have established that the manager worker can no longer touch the * slot directory (reaped, provably exited, or running this on its own * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the @@ -796,11 +806,14 @@ private void completeDeferredClose(boolean fullyDrained) { * confirmed release of the SF slot lock — the flip is published * strictly after the flock fd is closed, so observing {@code true} * guarantees the slot dir is acquirable by a replacement engine. A false - * value after close means manager-worker quiescence could not be - * confirmed (or the flock release itself failed) and the - * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager), - * which flips this to true the moment the worker's in-flight pass - * finishes, or leaked until a retried close() (shared manager). Owners + * value after close means the worker-reachable resources were retained + * deliberately, for one of three reasons: cleanup was handed to the + * worker's exit path (owned manager), which flips this to true the + * moment the worker's in-flight pass finishes; cleanup was leaked until + * a retried close() (shared manager, or a failed handoff registration); + * or the flock release itself failed — that last case never flips: the + * {@link #terminalCleanupClaimed} CAS is consumed, so no retry re-runs + * the release, and only process exit frees the flock. Owners * must not reuse the slot while this is false; pools may re-probe it to * recover a retired slot's capacity once it flips. *

    diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 51bd204e..24a9353a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -376,9 +376,16 @@ public boolean awaitRingQuiescence(SegmentRing ring) { } /** - * True when no manager worker thread can be running: either - * {@link #start()} was never called, or a {@link #close()} confirmed - * worker termination and reaped the thread. Owners use this as a + * True when the manager worker can no longer run a service pass: either + * {@link #start()} was never called, or a {@link #close()} reaped the + * thread — including the fall-through reap that observes + * {@code workerLoopExited} while the thread is still alive. In that case + * the worker may still be RUNNING its deferred engine cleanups + * (see {@link #deferUntilWorkerExit}: munmap/unlink/flock-release on the + * exit path) when this flips true, so this predicate is NOT the + * exclusion between a retried engine close() and the worker's deferred + * cleanup — the engine-side {@code terminalCleanupClaimed} CAS is. + * Owners use this as a * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)} * times out but a subsequent {@code close()} join succeeded. */ @@ -742,7 +749,10 @@ private void serviceRing0(RingEntry e) { // The watermark write and totalBytes commit are registration-gated // under `lock` so stale worker snapshots cannot touch the // engine-owned watermark or mutate accounting after deregister() - // returns. drainTrimmable still runs for stale snapshots: it + // returns. drainTrimmable still runs for entries deregistered + // MID-pass (entries deregistered before the pass started never + // get here — the registration/in-service claim at the top of + // serviceRing skips them entirely): it // transfers ownership of fully-acked sealed segments to this // worker, preserving the old close + unlink behavior. // diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index f11a5912..9e7ecd35 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1062,7 +1062,9 @@ private void retireLease(PooledSender ps, String context) { pendingLeaseTeardowns--; if (reserved) { // Free the index only when the flock was released; a slot - // left locked is retired permanently. + // left locked is retired into retiredSlots, recoverable + // by reprobeRetiredSlots() if the deferred cleanup drops + // the flock later. reclaimSlot(s, context); } slotReleased.signalAll(); @@ -1350,10 +1352,13 @@ private static boolean flockReleased(SenderSlot s) { * Reclaims one SF slot after its delegate's {@code close()} has been * attempted. When the flock was released the index returns to the free * set; when {@code close()} returned with the flock still held because an - * I/O or manager worker did not stop, the slot is retired permanently -- - * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap - * math accounts for the lost capacity and no later borrow ever reuses the - * still-locked dir. Either way {@code closingSlots} is decremented. + * I/O or manager worker did not stop, the slot is retired -- + * {@code leakedSlots++}, {@code slotInUse[idx]} stays set, and the sender + * joins {@code retiredSlots} -- so the cap math accounts for the lost + * capacity and no borrow reuses the still-locked dir unless + * {@link #reprobeRetiredSlots} later observes the deferred cleanup's + * release and recovers the index. Either way {@code closingSlots} is + * decremented. *

    * Caller must hold {@code lock} and is responsible for signalling waiters * (only the free path admits a new creation). Shared by From b89384fef4a28bb34a8d3c1e36dba5d8ef60a4a9 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 14:53:55 +0100 Subject: [PATCH 14/64] fix(qwp): recover SF slots after deferred cleanup --- core/src/main/c/share/files.c | 12 + core/src/main/c/windows/files.c | 14 + .../qwp/client/QwpWebSocketSender.java | 42 +- .../client/sf/cursor/CursorSendEngine.java | 231 +++++--- .../qwp/client/sf/cursor/SegmentManager.java | 251 +++++++-- .../qwp/client/sf/cursor/SlotLock.java | 37 +- .../io/questdb/client/impl/SenderPool.java | 24 +- .../cutlass/qwp/client/CloseDrainTest.java | 52 ++ ...ocketSenderCursorEngineAttachmentTest.java | 141 +++++ .../client/SlotLockReleasedContractTest.java | 27 +- ...CursorSendEngineSlotReacquisitionTest.java | 23 +- .../sf/cursor/CursorSendEngineTest.java | 19 + ...gineClosePublishAfterFlockReleaseTest.java | 48 +- .../SegmentManagerPassBarrierBenchmark.java | 131 +++++ .../qwp/client/sf/cursor/SlotLockTest.java | 36 +- .../client/test/impl/SenderPoolSfTest.java | 533 +++++++++++++++++- 16 files changed, 1387 insertions(+), 234 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 75f228fa..209c4490 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -236,6 +236,18 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return flock((int) fd, LOCK_EX | LOCK_NB); } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + if (flock((int) fd, LOCK_UN) != 0) { + return -1; + } + /* Unlock success confirms that the slot is reusable. close() is one-shot: + * POSIX leaves descriptor state unspecified on EINTR, so retrying its + * numeric value could close an unrelated descriptor after reuse. */ + (void) close((int) fd); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { return mkdir((const char *) (uintptr_t) lpszPath, (mode_t) mode); diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 119f6315..d1d3b413 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -331,6 +331,20 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!UnlockFileEx(FD_TO_HANDLE(fd), 0, MAXDWORD, MAXDWORD, &ov)) { + SaveLastError(); + return -1; + } + /* Unlock success confirms that the slot is reusable. Match POSIX by + * closing once without making handle cleanup part of that signal. */ + (void) CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { (void) mode; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index f1e3533c..3586a4d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -194,6 +194,9 @@ public class QwpWebSocketSender implements Sender { // up to this many millis for ackedFsn to catch up to publishedFsn. private long closeFlushTimeoutMillis = 5_000L; private volatile boolean closed; + // Test-only lifecycle witness. close() invokes and clears it strictly after + // publishing closed=true and before starting any drain or teardown work. + private volatile Runnable closeStartedHook; private boolean connected; private SenderConnectionDispatcher connectionDispatcher; // Async-delivery sink for SenderConnectionEvent notifications. Default @@ -1057,6 +1060,16 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { public void close() { if (!closed) { closed = true; + Runnable hook = closeStartedHook; + closeStartedHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-started test hook: {}", String.valueOf(t)); + } + } boolean ioThreadStopped = true; // Captures the first error from the flush/drain path AND any // secondary errors from cleanup steps (added via addSuppressed). @@ -1275,10 +1288,10 @@ public void close() { // The manager worker did not quiesce. Preserve ownership // and report the retained flock so pools retire this slot. // Repeated Sender.close() calls remain no-ops by contract. - // Engine cleanup was handed to the worker's exit path - // (owned manager); the getter re-probes the retained - // engine so the pool can reclaim the slot once cleanup - // actually completes. + // Engine cleanup was handed to a safe manager-worker path: + // owned-manager exit or shared-manager ring-pass completion. + // The getter re-probes the retained engine so the pool can + // reclaim the slot once cleanup actually completes. slotLockReleased = false; retainedEngine = engine; } @@ -1335,8 +1348,8 @@ public void close() { * index reserved instead of reusing the still-locked dir. *

    * Not a one-shot snapshot: when close() left engine cleanup pending on a - * worker/I/O-thread exit path, this re-probes the retained engine and - * latches true the moment that cleanup completes — pools re-probe retired + * manager-worker quiescence or I/O-thread exit path, this re-probes the + * retained engine and latches true the moment that cleanup completes — pools re-probe retired * slots through this getter to recover their capacity. Monotonic: * false→true only, never back. Lock-free (volatile reads) so pools may * call it under their capacity lock. @@ -2194,6 +2207,16 @@ public void setClientFactoryOverride(java.util.function.Supplier completeDeferredClose(fullyDrained)); + handedOff = manager.deferOwnedEngineCloseUntilWorkerExit(deferredClose); } catch (Throwable ignored) { - // Allocation failure (OOM building the cleanup lambda or - // growing the manager's exitCleanups list). Unlike the exact - // false return below, a throw carries NO worker-liveness - // information — the two must never be conflated, or the - // inline cleanup below would release worker-reachable - // resources under a possibly-live worker. + // Unexpected monitor/VM failure carries no worker-liveness + // information. Ordinary handoff cannot allocate: both the + // callback and the manager's single slot were preallocated. registrationFailed = true; } if (handedOff) { @@ -645,22 +670,41 @@ public synchronized void close() { workerQuiescent = true; } if (!workerQuiescent) { - // Shared (caller-owned) manager: its worker keeps serving other - // rings indefinitely, so there is no exit path to hand cleanup - // to — leak and let a retried close() reclaim once the pass ends. - LOG.error("SF manager worker did not quiesce during engine close; leaking the " - + "ring, watermark and slot lock so a stale worker cannot corrupt a " - + "future engine on slot {}. The kernel releases the slot flock on " - + "process exit; close() may be invoked again to retry cleanup once " - + "the worker has exited.", sfDir == null ? "" : sfDir); - return; + // A shared manager keeps serving sibling rings, so it cannot use + // the whole-worker exit handoff above. Transfer cleanup to this + // ring's current pass instead. Registration and pass completion + // share the manager lock: true means the worker owns cleanup; + // false means the pass already finished and inline cleanup is safe. + boolean handedOff = false; + boolean registrationFailed = false; + try { + handedOff = manager.deferUntilRingQuiescent(ring, deferredClose); + } catch (Throwable ignored) { + registrationFailed = true; + } + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the current ring pass and run the " + + "moment that pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can " + + "race the stale worker on slot {}", + sfDir == null ? "" : sfDir); + return; + } + if (registrationFailed) { + LOG.error("SF ring-pass cleanup handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live worker " + + "cannot corrupt a future engine on slot {}. The kernel releases the " + + "flock on process exit.", sfDir == null ? "" : sfDir); + return; + } + workerQuiescent = true; } if (!terminalCleanupClaimed.compareAndSet(false, true)) { - // A worker-exit handoff from an earlier close() attempt owns the - // terminal cleanup: it has run or is finishing right now on the - // exiting worker. closeCompleted flips the moment it is done — - // owners observe it via isCloseCompleted(), never by re-running - // the cleanup here (that would double-release ring/flock). + // A worker-exit handoff or earlier close owns the one-time + // ring/watermark cleanup. Once that work is published complete, + // this close may still retry an unconfirmed flock release. + retryFlockReleaseIfReady(); return; } finishClose(fullyDrained); @@ -672,7 +716,7 @@ public synchronized void close() { * is confirmed — latches {@link #closeCompleted}. Publish order * is load-bearing: pools free the slot index the instant they observe * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must - * never be visible while the flock fd is still open, or a replacement + * never be visible while the flock is still held, or a replacement * engine races the release and fails acquisition on a live slot. * The caller must hold the engine monitor and * must have established that the manager worker can no longer touch the @@ -718,13 +762,12 @@ private void finishClose(boolean fullyDrained) { // releasing the flock is safe even if a step above threw. Leaking // it would strand the slot until process exit for no reason. // - // ORDER MATTERS: release the flock FIRST, verify it, and only - // then publish closeCompleted. Pools read isCloseCompleted() as - // "the slot dir is reusable" and free the slot index the moment - // it flips; publishing before Files.close(fd) completes would - // open a window where a replacement engine's SlotLock.acquire - // collides with the still-open fd and fails with a spurious - // "slot already in use". + // ORDER MATTERS: explicitly release the flock, verify it, and + // only then publish closeCompleted. Pools read isCloseCompleted() + // as "the slot dir is reusable" and free the slot index the moment + // it flips. SlotLock closes the fd once after confirmed unlock, + // but that close result cannot safely control publication because + // POSIX may consume the fd even when close reports failure. Runnable hook = beforeFlockReleaseHook; if (hook != null) { try { @@ -733,32 +776,21 @@ private void finishClose(boolean fullyDrained) { // test-only; must never block the release } } - boolean released; - if (slotLock != null) { - try { - released = slotLock.release(); - } catch (Throwable ignored) { - released = false; - } - } else { - released = true; - } - if (released) { - closeCompleted = true; - } else { - // Unconfirmed release: the flock may still be held, so keep - // closeCompleted false — the owning pool keeps the slot - // retired (leakedSlots accounting) instead of reusing a - // possibly-locked dir. The kernel releases the flock on - // process exit regardless. - LOG.error("SF slot flock release failed during engine close; keeping " - + "closeCompleted=false so the pool cannot reuse a possibly " - + "still-locked slot dir; the kernel releases the flock on " - + "process exit [slot={}]", sfDir == null ? "" : sfDir); - } + terminalResourcesCleaned = true; + retryFlockReleaseIfReady(); } } + /** + * Installs a constructor fault hook immediately before the bound deferred + * close callback is created. Test-only: proves callback allocation failure + * occurs before an owned manager acquires native resources. + */ + @TestOnly + public static void setBeforeDeferredCloseCreationHook(Runnable hook) { + beforeDeferredCloseCreationHook = hook; + } + /** * Installs a hook that {@link #finishClose} runs between the terminal * cleanup and the slot-flock release. Test-only: makes the otherwise @@ -772,29 +804,94 @@ public void setBeforeFlockReleaseHook(Runnable hook) { } /** - * Runs on the manager worker's exit path when {@link #close()} handed - * cleanup ownership to the worker (see {@code deferUntilWorkerExit}). + * Runs on a safe manager-worker handoff path when {@link #close()} moved + * cleanup ownership to either a shared manager's ring-pass completion or + * an owned manager's worker exit. * Deliberately does NOT take the engine monitor: a retried close() can * hold it while joining this very worker, and the thread cannot die until * this method returns — the {@link #terminalCleanupClaimed} CAS provides * the exactly-once exclusion instead, so the worker always exits promptly * and the racing close() converges via {@code isWorkerReaped()}. */ - private void completeDeferredClose(boolean fullyDrained) { + private void completeDeferredClose() { if (!terminalCleanupClaimed.compareAndSet(false, true)) { // A retried close() (or an earlier duplicate handoff) already ran // the terminal cleanup. return; } - finishClose(fullyDrained); - LOG.info("deferred SF engine cleanup completed on manager-worker exit; slot released: {}", - sfDir == null ? "" : sfDir); + finishClose(fullyDrainedForDeferredClose); + LOG.info("deferred SF engine resource cleanup completed after manager-worker quiescence; " + + "slot release confirmed: {} [slot={}]", + closeCompleted, sfDir == null ? "" : sfDir); + } + + private Runnable createDeferredClose() { + Runnable hook = beforeDeferredCloseCreationHook; + if (hook != null) { + hook.run(); + } + return this::completeDeferredClose; + } + + private boolean retryFlockReleaseIfReady() { + if (closeCompleted || !terminalResourcesCleaned) { + return closeCompleted; + } + boolean released; + if (slotLock != null) { + try { + released = slotLock.release(); + } catch (Throwable ignored) { + released = false; + } + } else { + released = true; + } + if (released) { + closeCompleted = true; + return true; + } + startFlockReleaseRetry(); + return false; + } + + private void startFlockReleaseRetry() { + if (!flockReleaseRetryStarted.compareAndSet(false, true)) { + return; + } + try { + Thread retryThread = new Thread(() -> { + while (!retryFlockReleaseIfReady()) { + try { + Thread.sleep(100L); + } catch (InterruptedException ignored) { + // A failed flock release must remain retryable until + // confirmed or process exit; interruption is not a + // safe reason to abandon the slot permanently. + } + } + }, "qdb-sf-flock-release-retry"); + retryThread.setDaemon(true); + retryThread.start(); + LOG.error("SF slot flock release failed during engine close; keeping " + + "closeCompleted=false and retrying outside the pool lock so " + + "retired capacity recovers after the transient failure [slot={}]", + sfDir == null ? "" : sfDir); + } catch (Throwable t) { + // A later explicit close() can still retry without repeating the + // one-time ring/watermark cleanup. The kernel remains the final + // process-exit backstop if thread creation and all retries fail. + flockReleaseRetryStarted.set(false); + LOG.error("Could not start SF flock-release retry driver; close() may be " + + "invoked again to retry [slot={}, error={}]", + sfDir == null ? "" : sfDir, String.valueOf(t)); + } } /** * Whether {@link #close()} completed all cleanup, including a * confirmed release of the SF slot lock — the flip is published - * strictly after the flock fd is closed, so observing {@code true} + * strictly after explicit unlock succeeds, so observing {@code true} * guarantees the slot dir is acquirable by a replacement engine. A false * value after close means manager-worker quiescence could not be * confirmed (or the flock release itself failed) and the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 51bd204e..15d5c607 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -34,7 +34,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.LockSupport; /** @@ -66,6 +68,14 @@ public final class SegmentManager implements QuietCloseable { public static final long DEFAULT_POLL_NANOS = 1_000_000L; // 1 ms public static final long DISK_FULL_LOG_THROTTLE_NANOS = 30_000_000_000L; // 30 s public static final long UNLIMITED_TOTAL_BYTES = Long.MAX_VALUE; + private static final int ENTRY_DEREGISTERED = 3; + private static final int ENTRY_DEREGISTERED_IN_SERVICE = 2; + private static final int ENTRY_IN_SERVICE = 1; + private static final int ENTRY_REGISTERED = 0; + private static final AtomicReferenceFieldUpdater ENTRY_CLEANUP_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(RingEntry.class, Runnable.class, "quiescenceCleanup"); + private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state"); private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class); private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; @@ -86,6 +96,17 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + // Test seam: runs after a deferred ring-pass cleanup returns. Null in + // production; public sender/pool lifecycle tests use it to observe exact + // callback completion without sleeps or polling. + private volatile Runnable afterRingCleanupHook; + // Test seam: runs at the top of deferUntilWorkerExit, before the + // worker-liveness check. Null in production; registration-failure tests + // throw from it to simulate an allocation failure (OOM building the + // cleanup lambda or growing exitCleanups) while the worker is still + // live. Callers must treat such a throw as "worker state unknown", + // never as the exact false return meaning the worker loop has exited. + private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the install path's // synchronized(lock) entry (the one that performs installHotSpare + the // totalBytes += segmentSize commit). Null in production; tests use it to @@ -97,25 +118,19 @@ public final class SegmentManager implements QuietCloseable { // production; owned-engine close tests use it to prove they take only the // stronger whole-manager join path, not two sequential timeout budgets. private volatile Runnable beforeRingQuiescenceAwaitHook; - // Test seam: runs at the top of deferUntilWorkerExit, before the - // worker-liveness check. Null in production; registration-failure tests - // throw from it to simulate an allocation failure (OOM building the - // cleanup lambda or growing exitCleanups) while the worker is still - // live. Callers must treat such a throw as "worker state unknown", - // never as the exact false return meaning the worker loop has exited. - private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically // inject a deregister(ring) call into the exact race window that the - // registered flag check inside the trim block closes for watermark writes - // and totalBytes accounting. + // entry-state check inside the trim block closes for watermark writes and + // totalBytes accounting. private volatile Runnable beforeTrimSyncHook; - // Entry currently being serviced by the worker thread, or null when the - // worker is between service passes (or not running). Guarded by - // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can - // block until an in-flight pass for a just-deregistered ring finishes. - private RingEntry inService; + // Entry the worker is claiming or servicing, or null between passes. + // Volatile publication lets teardown find the entry after deregister() + // removes it from rings. RingEntry.state is the authoritative barrier: + // the worker publishes this reference before its REGISTERED->IN_SERVICE + // CAS and only touches ring resources after that CAS succeeds. + private volatile RingEntry inService; // Cleanup actions handed to the worker's exit block by an owning engine // whose close() found the worker still mid service pass after the bounded // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock}; @@ -124,14 +139,11 @@ public final class SegmentManager implements QuietCloseable { // and no caller-facing lock may ever be held while running third-party // cleanup code. private ObjList exitCleanups; - // Number of threads currently parked in awaitRingQuiescence()'s wait - // loop. Guarded by {@link #lock}. The worker's per-pass finally block - // only calls lock.notifyAll() when this is > 0, so the steady-state - // production path (no quiescence barrier in flight) never notifies. - // Both sides mutate/check under the same lock, so there is no lost - // wakeup: a waiter that increments after the worker's check re-reads - // inService before waiting and sees the pass already finished. - private int quiescenceWaiters; + // A private manager belongs to exactly one CursorSendEngine. Its callback + // is preallocated by that engine and stored directly here, so the critical + // timed-out-close handoff never allocates an ObjList or grows a backing + // array. Guarded by lock and consumed on worker-loop exit outside lock. + private Runnable ownedEngineExitCleanup; private long lastDiskFullLogNs; private volatile boolean running; // pathScratch free-exactly-once coordination between a timed-out close() @@ -273,6 +285,62 @@ public synchronized void close() { } } + /** + * Hands the single owning engine's preallocated close callback to the + * worker exit block. Registration only assigns a reference under + * {@link #lock}; it cannot allocate in the timed-out teardown path. + * Returns {@code false} only after the worker loop has exited (or when it + * never started), so the caller may then clean up inline safely. + */ + public boolean deferOwnedEngineCloseUntilWorkerExit(Runnable cleanup) { + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (ownedEngineExitCleanup != null && ownedEngineExitCleanup != cleanup) { + throw new IllegalStateException("owned manager already has an engine-exit cleanup"); + } + ownedEngineExitCleanup = cleanup; + return true; + } + } + + /** + * Hands a cleanup action to the current service pass for {@code ring}. + * The worker runs it outside the manager lock immediately after that pass + * finishes. Returns {@code false} when no pass for the ring remains in + * flight, in which case the caller already owns a quiescent ring and must + * run the cleanup itself. + *

    + * Registration and pass completion coordinate through the entry's atomic + * state and callback fields, so there is no gap without an owner: either + * this method attaches the cleanup while the pass remains active, the + * worker claims an attached cleanup while completing, or this method + * observes completed state and rejects the handoff. A repeated registration + * for the same pass returns {@code true} without replacing its existing + * owner. + */ + public boolean deferUntilRingQuiescent(SegmentRing ring, Runnable cleanup) { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return false; + } + Runnable existing = ENTRY_CLEANUP_UPDATER.get(e); + if (existing == null && ENTRY_CLEANUP_UPDATER.compareAndSet(e, null, cleanup)) { + if (e.isInService()) { + return true; + } + // Completion changed the state before observing the callback. + // Remove our callback and clean inline, unless the worker already + // claimed it (in which case that worker remains the owner). + return !ENTRY_CLEANUP_UPDATER.compareAndSet(e, cleanup, null); + } + // A callback already attached to this pass is an owner. The sole + // production caller always supplies the engine's preallocated bound + // callback; duplicates intentionally do not replace it. + return true; + } + /** * Hands a cleanup action to the worker thread's exit block, to run * strictly after the worker loop has finished its final service pass -- @@ -347,10 +415,14 @@ public boolean awaitRingQuiescence(SegmentRing ring) { long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L; boolean interrupted = Thread.interrupted(); try { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return true; + } synchronized (lock) { - quiescenceWaiters++; + e.quiescenceWaiters++; try { - while (inService != null && inService.ring == ring) { + while (e.isInService()) { long remainingNanos = deadlineNanos - System.nanoTime(); if (remainingNanos <= 0) { return false; @@ -364,7 +436,7 @@ public boolean awaitRingQuiescence(SegmentRing ring) { } } } finally { - quiescenceWaiters--; + e.quiescenceWaiters--; } } return true; @@ -411,7 +483,7 @@ public void deregister(SegmentRing ring) { // single subtraction covers both the initial seed // and the net manager activity (provisions minus // trims) for this ring. - e.registered = false; + e.deregister(); totalBytes -= ring.totalSegmentBytes(); rings.remove(i); return; @@ -482,6 +554,11 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public void setAfterRingCleanupHook(Runnable hook) { + this.afterRingCleanupHook = hook; + } + @TestOnly public void setBeforeExitCleanupRegistrationHook(Runnable hook) { this.beforeExitCleanupRegistrationHook = hook; @@ -607,33 +684,42 @@ private String nextSparePath(String dir) { } private void serviceRing(RingEntry e) { - // Claim the entry as in-service so deregister-side quiescence - // barriers (awaitRingQuiescence) can wait for this pass to finish. - // A stale snapshot entry deregistered before the pass starts is - // skipped entirely: the deregistering thread may already be - // releasing the ring / watermark / slot resources, so the worker - // must not touch them at all. The registered check and the - // in-service claim are atomic under `lock` — deregister flips - // `registered` under the same lock, so it either prevents this - // pass or the barrier observes it via `inService`. - synchronized (lock) { - if (!e.registered) { - return; - } - inService = e; + // Publish before the CAS so deregister + await can always find the + // entry. The state CAS is the ownership decision: if deregister won, + // this stale snapshot pass skips the ring without touching it. + inService = e; + if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) { + inService = null; + return; } try { serviceRing0(e); } finally { - synchronized (lock) { - inService = null; - // Wake quiescence barriers only when one is actually parked. - // In production no engine ever waits here (owned-manager - // close joins the worker thread instead), so the per-tick - // pass must not pay an unconditional notifyAll. notifyAll, - // not notify: distinct waiters may await different rings. - if (quiescenceWaiters > 0) { - lock.notifyAll(); + e.finishService(); + Runnable cleanup = e.quiescenceCleanup == null + ? null + : ENTRY_CLEANUP_UPDATER.getAndSet(e, null); + inService = null; + // A normal pass performs only the two state CAS operations above. + // The manager monitor is entered here only for an actual close + // waiter; the recheck under lock prevents lost wakeups. + if (e.quiescenceWaiters > 0) { + synchronized (lock) { + if (e.quiescenceWaiters > 0) { + lock.notifyAll(); + } + } + } + if (cleanup != null) { + try { + cleanup.run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed after manager-worker ring pass", t); + } finally { + Runnable hook = afterRingCleanupHook; + if (hook != null) { + hook.run(); + } } } } @@ -704,7 +790,7 @@ private void serviceRing0(RingEntry e) { // observe the spare in the ring (and subtract it) or // run before installation (so no install happens). synchronized (lock) { - if (e.registered) { + if (e.isRegistered()) { e.ring.installHotSpare(spare); totalBytes += segmentSizeBytes; installed = true; @@ -754,7 +840,7 @@ private void serviceRing0(RingEntry e) { hook.run(); } synchronized (lock) { - boolean registered = e.registered; + boolean registered = e.isRegistered(); // Persist the current ackedFsn watermark BEFORE the trim runs. // On host crash between the persist and the unlinks below, the // segments survive and the watermark is correct. On crash AFTER @@ -827,6 +913,7 @@ private void workerLoop() { // here reclaims the native buffer even when the worker outlives // every close() attempt — nobody else retries manager cleanup. ObjList cleanups; + Runnable ownedEngineCleanup; synchronized (lock) { workerLoopExited = true; if (scratchHandedToWorker && !scratchFreed) { @@ -835,8 +922,10 @@ private void workerLoop() { } cleanups = exitCleanups; exitCleanups = null; + ownedEngineCleanup = ownedEngineExitCleanup; + ownedEngineExitCleanup = null; } - // Deferred engine cleanups (see deferUntilWorkerExit) run OUTSIDE + // Deferred engine cleanups run OUTSIDE // `lock`: they perform syscalls (munmap, unlink, flock release) // and must never execute under a lock that close()/register/ // deregister callers contend on. Running them after the loop body @@ -845,6 +934,13 @@ private void workerLoop() { // monitor — a retried engine.close() joins this thread while // holding the engine monitor, which is why the engine side uses a // lock-free claim (CAS), not synchronization, for exactly-once. + if (ownedEngineCleanup != null) { + try { + ownedEngineCleanup.run(); + } catch (Throwable t) { + LOG.error("deferred owned-engine cleanup failed on manager-worker exit", t); + } + } if (cleanups != null) { for (int i = 0, n = cleanups.size(); i < n; i++) { try { @@ -870,16 +966,57 @@ private static final class RingEntry { // Survives across multiple serviceRing ticks and avoids a // write-storm when ackedFsn is steady. long lastPersistedAck = -1L; - // Guarded by SegmentManager.lock. A worker snapshot may retain this - // entry after deregister() removes it from rings; registered=false is - // the O(1) ownership check that prevents post-deregister writes through - // the engine-owned watermark, hot-spare installs, and accounting. - boolean registered = true; + // Updated lock-free by deferUntilRingQuiescent and pass completion. + // The field updater avoids one AtomicReference allocation per ring. + volatile Runnable quiescenceCleanup; + // Waiters mutate this under SegmentManager.lock; pass completion reads + // it before taking the otherwise-cold notification path. + volatile int quiescenceWaiters; + // REGISTERED -> IN_SERVICE -> REGISTERED on a normal pass. + // Deregistration changes either registered state to its corresponding + // deregistered state; completion then changes DEREGISTERED_IN_SERVICE + // to DEREGISTERED. The field updater avoids per-entry allocation. + volatile int state = ENTRY_REGISTERED; RingEntry(SegmentRing ring, String dir, AckWatermark watermark) { this.ring = ring; this.dir = dir; this.watermark = watermark; } + + void deregister() { + while (true) { + int current = state; + int next = current == ENTRY_IN_SERVICE + ? ENTRY_DEREGISTERED_IN_SERVICE + : ENTRY_DEREGISTERED; + if (current == ENTRY_DEREGISTERED || current == ENTRY_DEREGISTERED_IN_SERVICE + || ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + void finishService() { + while (true) { + int current = state; + int next = current == ENTRY_DEREGISTERED_IN_SERVICE + ? ENTRY_DEREGISTERED + : ENTRY_REGISTERED; + if (ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + boolean isInService() { + int current = state; + return current == ENTRY_IN_SERVICE || current == ENTRY_DEREGISTERED_IN_SERVICE; + } + + boolean isRegistered() { + int current = state; + return current == ENTRY_REGISTERED || current == ENTRY_IN_SERVICE; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 1b72d157..2bbcc109 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -36,9 +36,9 @@ * Advisory exclusive lock for a single SF slot directory. *

    * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx} - * for the entire lifetime of the engine that owns the slot. The lock is - * automatically released when the fd is closed — including on hard process - * exit, since the kernel cleans up file locks for terminated processes. + * for the entire lifetime of the engine that owns the slot. Normal teardown + * explicitly unlocks it before closing the fd; hard process exit remains a + * backstop because the kernel cleans up file locks for terminated processes. *

    * The holder's PID is written to a sibling {@code .lock.pid} file at * acquisition time. A failed acquisition reads it back so the error message @@ -117,31 +117,32 @@ public String slotDir() { } /** - * Releases the flock by closing the lock fd and reports whether the - * release was confirmed. Closing the fd releases the lock. We do - * NOT remove the {@code .lock} file or the {@code .lock.pid} sidecar — - * a stale PID is harmless (next acquirer overwrites {@code .lock.pid} - * on success). + * Explicitly releases the flock and reports whether the release was + * confirmed. After a successful unlock the native primitive closes + * the descriptor once, best-effort, and this object forgets its numeric + * value. It never retries that close: POSIX leaves descriptor state + * unspecified after some close failures (notably {@code EINTR}), so a + * retry could close an unrelated descriptor that reused the same number. + * We do NOT remove the {@code .lock} file or {@code .lock.pid} sidecar; a + * stale PID is harmless because the next acquirer overwrites it. *

    - * On close failure the fd is retained: the flock may still be - * held by this process, so forgetting the fd would misreport the lock - * state and forfeit any chance of a later retry. Idempotent — once - * released, subsequent calls return {@code true}. + * When the explicit unlock itself fails, the fd is retained so a later + * attempt can safely retry the non-consuming unlock operation. Idempotent + * once the unlock has succeeded. *

    * Owners that gate a "slot dir is reusable" signal on the release * (e.g. {@code CursorSendEngine.finishClose} publishing * {@code closeCompleted}) must call this and check the result rather * than {@link #close()}, which is best-effort by contract. * - * @return {@code true} if the fd was closed (or was already released), - * {@code false} if the OS reported a close failure and the - * flock may still be held + * @return {@code true} if the lock was explicitly released (or was already + * released), {@code false} if the OS reported an unlock failure */ - public boolean release() { + public synchronized boolean release() { if (fd < 0) { return true; } - if (Files.close(fd) == 0) { + if (release0(fd) == 0) { fd = -1; return true; } @@ -180,6 +181,8 @@ private static String readHolder(String pidPath) { } } + private static native int release0(int fd); + private static void writePid(String pidPath) { long pid; try { diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index f11a5912..71c773cb 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -148,6 +148,11 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Test seam: runs immediately before a capacity-starved borrow enters its + // condition wait, while it still holds the pool lock. Null in production; + // concurrency tests use a latch here to prove that several borrowers have + // all reached the wait path before recovering retired capacity. + private volatile Runnable beforeBorrowWaitHook; // Test seam: runs after a capacity-starved borrow's condition wait has // exhausted its positive timeout, before the loop's terminal pass. Null in // production; regression tests release a retired slot here to prove that @@ -822,6 +827,10 @@ public PooledSender borrow() { "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } try { + Runnable beforeWaitHook = beforeBorrowWaitHook; + if (beforeWaitHook != null) { + beforeWaitHook.run(); + } remainingNanos = slotReleased.awaitNanos(remainingNanos); if (remainingNanos <= 0) { Runnable hook = borrowWaitExpiredHook; @@ -841,6 +850,11 @@ public PooledSender borrow() { } } + @TestOnly + public void setBeforeBorrowWaitHook(Runnable hook) { + this.beforeBorrowWaitHook = hook; + } + @TestOnly public void setBorrowWaitExpiredHook(Runnable hook) { this.borrowWaitExpiredHook = hook; @@ -1397,7 +1411,15 @@ private boolean reprobeRetiredSlots() { for (int i = retiredSlots.size() - 1; i >= 0; i--) { SenderSlot s = retiredSlots.get(i); if (flockReleased(s)) { - retiredSlots.remove(i); + // Order is irrelevant. Swap with the tail before removing so + // each recovery does O(1) list work instead of shifting the + // remaining retired entries. The tail has already been probed + // by this reverse scan and, if still present, is unreleased. + int last = retiredSlots.size() - 1; + if (i < last) { + retiredSlots.set(i, retiredSlots.get(last)); + } + retiredSlots.remove(last); leakedSlots--; freeSlotIndex(s.slotIndex()); recovered = true; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java index 9f1500f2..59d0f83d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java @@ -35,8 +35,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** * Regression tests for the close() drain semantics. @@ -77,6 +79,56 @@ public void testCloseBlocksUntilAckArrives() throws Exception { } } + @Test + public void testCloseStartedHookRunsAfterClosedStateTransition() throws Exception { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference closerFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseStartedHook(() -> { + try { + try { + sender.table("must_reject_after_close_started"); + Assert.fail("close-started hook ran before closed=true was published"); + } catch (LineSenderException expected) { + // Required lifecycle boundary: public operations already reject. + } + entered.countDown(); + if (!release.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("close-started hook release timed out"); + } + } catch (Throwable t) { + hookFailure.set(t); + entered.countDown(); + } + }); + + Assert.assertEquals("installing the hook must not emit a pre-invocation witness", + 1L, entered.getCount()); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closerFailure.set(t); + } + }, "close-started-hook-test"); + try { + closer.start(); + Assert.assertTrue("close() never reached its internal lifecycle witness", + entered.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close() completed while its internal witness was held", + closer.isAlive()); + } finally { + release.countDown(); + closer.join(10_000L); + sender.close(); + } + Assert.assertFalse("close thread did not finish", closer.isAlive()); + Assert.assertNull("close-started hook failed", hookFailure.get()); + Assert.assertNull("close() failed", closerFailure.get()); + } + @Test public void testCloseFastWhenTimeoutIsZero() throws Exception { // Same delayed-ACK server, but with close_flush_timeout_millis=0 diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java new file mode 100644 index 00000000..5bfb0863 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpWebSocketSenderCursorEngineAttachmentTest { + + private static final long SEGMENT_SIZE = 64 * 1024L; + + @Rule + public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testNullDoesNotDetachAnAttachedEngine() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, true); + assertSecondAttachmentRejected(sender, null, false); + sender.close(); + Assert.assertTrue("sender must retain ownership after rejected detach", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + @Test + public void testOwnedEngineCannotBeReplacedAndItsSlotIsReleased() throws Exception { + assertMemoryLeak(() -> { + File slotDir = new File(sfDir.getRoot(), "owned-slot"); + CursorSendEngine first = null; + CursorSendEngine replacement = null; + CursorSendEngine reacquired = null; + QwpWebSocketSender sender = null; + boolean replacementRejected = false; + try { + first = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + replacement = new CursorSendEngine(null, SEGMENT_SIZE); + sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(first, true); + try { + sender.setCursorEngine(replacement, true); + } catch (LineSenderException e) { + replacementRejected = true; + Assert.assertTrue(e.getMessage().contains("already attached")); + } + + sender.close(); + sender = null; + try { + reacquired = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + } catch (IllegalStateException e) { + throw new AssertionError("sender close did not release the first owned " + + "engine's slot after a second attachment attempt", e); + } + Assert.assertTrue("second attachment must be rejected", replacementRejected); + } finally { + if (sender != null) { + sender.close(); + } + if (reacquired != null) { + reacquired.close(); + } + if (replacement != null) { + replacement.close(); + } + if (first != null) { + first.close(); + } + } + }); + } + + @Test + public void testSameSharedEngineCannotTransferOwnership() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, false); + assertSecondAttachmentRejected(sender, engine, true); + sender.close(); + Assert.assertFalse("rejected attachment must not transfer ownership", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + private static void assertSecondAttachmentRejected( + QwpWebSocketSender sender, + CursorSendEngine engine, + boolean takeOwnership + ) { + try { + sender.setCursorEngine(engine, takeOwnership); + Assert.fail("expected second cursor engine attachment to be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("already attached")); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 0a6c0895..f2c560ea 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -195,12 +195,11 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception } /** - * Manager-worker leak path: engine close retains the slot while a shared + * Manager-worker handoff path: engine close retains the slot while a shared * manager is still inside a service pass. The sender must expose that * retained flock to SenderPool. Repeated sender close calls remain no-ops; - * the test cleans the deliberately retained engine up directly — and once - * that cleanup completes, the sender must expose the released flock so a - * pool that retired the slot can recover its capacity. + * the manager pass owns deferred cleanup and releases the flock when it + * finishes, so the sender can expose recovery without a direct close retry. */ @Test(timeout = 30_000L) public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { @@ -211,6 +210,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception String slot = tmpDir + "/slot"; long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); @@ -219,6 +219,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception QwpWebSocketSender wss = null; boolean managerClosed = false; try { + manager.setAfterRingCleanupHook(cleanupFinished::countDown); manager.setBeforeInstallSyncHook(() -> { if (!fired.compareAndSet(false, true)) return; workerBlocked.countDown(); @@ -262,12 +263,9 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception engine.isCloseCompleted()); releaseWorker.countDown(); - manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); - // Test-only cleanup through the retained local reference (the - // shared-manager path has no worker-exit handoff to defer to; - // a retried close() is its reclaim driver). - engine.close(); - Assert.assertTrue("direct cleanup did not complete after worker exit", + Assert.assertTrue("shared-manager pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("deferred cleanup did not complete after the ring pass", engine.isCloseCompleted()); // Recovery contract: the sender re-probes its retained engine, // so the completed cleanup (and released flock) MUST become @@ -276,7 +274,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception Assert.assertTrue("sender must expose the late flock release to the pool", wss.isSlotLockReleased()); try (SlotLock probe = SlotLock.acquire(slot)) { - Assert.assertNotNull("slot must be acquirable after direct cleanup", probe); + Assert.assertNotNull("slot must be acquirable after deferred cleanup", probe); } manager.close(); @@ -285,6 +283,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception throw new AssertionError("install hook failed", hookErr.get()); } } finally { + manager.setAfterRingCleanupHook(null); manager.setBeforeInstallSyncHook(null); releaseWorker.countDown(); if (wss != null) { @@ -293,12 +292,6 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception } catch (Throwable ignored) { } } - if (engine != null && !engine.isCloseCompleted()) { - try { - engine.close(); - } catch (Throwable ignored) { - } - } if (!managerClosed) { manager.close(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 38cd1bcf..6c08aeff 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -88,7 +88,8 @@ public void tearDown() { * inside a service pass for the engine's ring, {@code close()} must NOT * hand the slot to anyone else. With the quiescence barrier reverted, * close() releases the slot lock immediately and the mid-test - * {@code SlotLock.acquire} probe succeeds — failing the test. + * {@code SlotLock.acquire} probe succeeds. Once the pass finishes, its + * deferred cleanup must release the slot without a direct close retry. */ @Test(timeout = 30_000L) public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { @@ -98,6 +99,7 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // 60 s poll: the worker only acts when explicitly woken, so the // single pass we park below is the only pass in flight. SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); @@ -105,6 +107,7 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { boolean managerClosed = false; CursorSendEngine engine = null; try { + manager.setAfterRingCleanupHook(cleanupFinished::countDown); manager.setBeforeInstallSyncHook(() -> { if (!fired.compareAndSet(false, true)) return; workerBlocked.countDown(); @@ -149,19 +152,16 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // Let the worker finish its pass (it abandons the spare: the // ring was deregistered by the close attempt above). releaseWorker.countDown(); - manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); - - // Retry close(): the barrier now succeeds and the full cleanup - // (ring, watermark, unlink, slot lock) must complete. - engine.close(); - Assert.assertTrue("retried close must report complete cleanup", + Assert.assertTrue("ring pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("ring-pass cleanup must report complete cleanup", engine.isCloseCompleted()); engine = null; try (SlotLock probe = SlotLock.acquire(slot)) { Assert.assertNotNull("slot must be acquirable after a completed close", probe); } catch (Exception e) { - throw new AssertionError("retried close() did not release the slot lock", e); + throw new AssertionError("ring-pass cleanup did not release the slot lock", e); } manager.close(); @@ -170,14 +170,9 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { throw new AssertionError("install hook failed", hookErr.get()); } } finally { + manager.setAfterRingCleanupHook(null); manager.setBeforeInstallSyncHook(null); releaseWorker.countDown(); - if (engine != null) { - try { - engine.close(); - } catch (Throwable ignored) { - } - } if (!managerClosed) { manager.close(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index f4de1ff2..c11fe1fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -197,6 +197,25 @@ public void testCloseIsIdempotent() throws Exception { }); } + @Test + public void testCallbackCreationFailurePrecedesOwnedManagerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> { + throw new OutOfMemoryError("simulated bound callback allocation failure"); + }); + try { + try { + new CursorSendEngine(tmpDir, 4096); + fail("expected callback allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("simulated bound callback allocation failure", expected.getMessage()); + } + } finally { + CursorSendEngine.setBeforeDeferredCloseCreationHook(null); + } + }); + } + @Test public void testConstructorFailureAfterOwnedManagerStartCleansResources() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index 5dcf6def..c142dd6a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -178,21 +178,18 @@ public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws E /** * The error half of the publish-after-release contract: when - * {@code SlotLock.release()} reports failure (the OS refused the fd - * close, so the flock may still be held), {@code closeCompleted} must - * NEVER be published — not by the failing close(), and not by a retried - * close() either (the terminal-cleanup claim is already consumed; the - * design deliberately leaves an unconfirmed release to the kernel's - * process-exit cleanup rather than risking a double release). A pool - * observing {@code isCloseCompleted() == false} keeps the slot retired, - * which is exactly right: the flock genuinely is still held. + * {@code SlotLock.release()} reports failure (the OS refused the explicit + * unlock, so the flock may still be held), {@code closeCompleted} must + * stay false until a later close confirms release. A pool observing + * {@code isCloseCompleted() == false} keeps the slot retired while the + * flock is genuinely held, then recovers it once the retry completes. *

    - * {@code Files.close} cannot be made to fail through the public API, so - * the lock's fd is swapped to a known-bad descriptor for the close and - * restored (and released for real) afterwards. + * The lock's fd is swapped to a known-bad descriptor for a deterministic + * native unlock failure, then restored before retrying through the engine's + * public close API. */ @Test(timeout = 30_000L) - public void testUnconfirmedFlockReleaseKeepsCloseIncomplete() throws Exception { + public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws Exception { TestUtils.assertMemoryLeak(() -> { CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); @@ -221,21 +218,24 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncomplete() throws Exception { } catch (IllegalStateException expected) { // good — incomplete close really means "still locked". } - // A retried close() must neither throw nor re-run the terminal - // cleanup (the claim is consumed) nor suddenly report success. + + // Remove the injected fault. The retry must only re-attempt + // the retained fd release: ring/watermark cleanup stays + // exactly-once, while completion and slot reusability recover. + fdField.setInt(slotLock, realFd); engine.close(); - assertFalse("retried close() must not fabricate completion after a failed " - + "flock release — the claim is consumed and the kernel owns " - + "the eventual cleanup", + assertTrue("retried close() must complete after the flock release succeeds", engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — eventual completion implies reusable capacity. + } } finally { - // Undo the fault and drop the real flock so the test leaves no - // open fd behind (in production the kernel does this at exit). - fdField.setInt(slotLock, realFd); - assertTrue("restored fd must release cleanly", slotLock.release()); - } - try (SlotLock ignored = SlotLock.acquire(sfDir)) { - // good — with the flock genuinely dropped, the slot is reusable. + // If an assertion failed before the successful retry, restore + // and release the real fd so the test never leaks a flock. + if (!engine.isCloseCompleted()) { + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } } }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java new file mode 100644 index 00000000..1bb6b899 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +/** + * Standalone comparison of the service-pass ownership barriers used by + * SegmentManager before and after its per-entry state change. This deliberately + * has no pass/fail threshold: it reports target-JVM costs while the production + * state machine and lifecycle tests establish correctness. + * + * 

    + * mvn -pl core test-compile
+ * mvn -pl core exec:java -Dexec.classpathScope=test \
+ *   -Dexec.mainClass=io.questdb.client.test.cutlass.qwp.client.sf.cursor.SegmentManagerPassBarrierBenchmark \
+ *   -Dexec.args="--passes=10000000"
+ *
    + */ +public final class SegmentManagerPassBarrierBenchmark { + + private static final Object MONITOR = new Object(); + private static final AtomicIntegerFieldUpdater STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(Entry.class, "state"); + private static volatile long checksum; + private static volatile int serviceProbe; + + public static void main(String[] args) { + int passes = 10_000_000; + for (String arg : args) { + if (arg.startsWith("--passes=")) { + passes = Integer.parseInt(arg.substring("--passes=".length())); + } else { + throw new IllegalArgumentException("unknown argument: " + arg); + } + } + if (passes < 1) { + throw new IllegalArgumentException("passes must be positive"); + } + + int warmup = Math.max(100_000, passes / 10); + for (int rings : new int[]{1, 32, 256}) { + measureMonitor(rings, warmup); + measureAtomic(rings, warmup); + long monitorNanos = measureMonitor(rings, passes); + long atomicNanos = measureAtomic(rings, passes); + System.out.printf( + "rings=%d passes=%d monitor(two enters)=%.2f ns/pass atomic(two CAS)=%.2f ns/pass ratio=%.2f%n", + rings, + passes, + (double) monitorNanos / passes, + (double) atomicNanos / passes, + (double) monitorNanos / atomicNanos); + } + System.out.println("checksum=" + checksum); + } + + private static Entry[] entries(int count) { + Entry[] entries = new Entry[count]; + for (int i = 0; i < count; i++) { + entries[i] = new Entry(); + } + return entries; + } + + private static long measureAtomic(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + if (!STATE_UPDATER.compareAndSet(entry, 0, 1)) { + throw new AssertionError("claim failed"); + } + service(entry); + if (!STATE_UPDATER.compareAndSet(entry, 1, 0)) { + throw new AssertionError("completion failed"); + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static long measureMonitor(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + synchronized (MONITOR) { + entry.state = 1; + } + service(entry); + synchronized (MONITOR) { + entry.state = 0; + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static void service(Entry entry) { + // Models the non-trivial serviceRing0 call between claim and complete + // and prevents the JVM from coarsening both monitor regions into one. + serviceProbe = entry.state; + } + + private static final class Entry { + volatile int state; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 015cb0d8..20930f50 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -124,18 +124,16 @@ public void testReleaseConfirmsAndIsIdempotent() throws Exception { } /** - * The {@code release() == false} branch: when the OS reports a close - * failure, release must (a) return {@code false} so owners gating a - * "slot reusable" signal never see a lie, (b) RETAIN the fd — forgetting - * it would misreport the lock state and forfeit any later retry — and - * (c) keep returning {@code false} on repeat attempts while the failure - * persists. Once the close succeeds, release confirms and stays - * confirmed. {@code Files.close} cannot be made to fail through the - * public API, so the fd is swapped to a known-bad descriptor and - * restored afterwards. + * The {@code release() == false} branch: when the OS reports an explicit + * unlock failure, release must (a) return {@code false} so owners gating a + * "slot reusable" signal never see a lie, (b) retain the fd because the + * non-consuming unlock can safely be retried, and (c) keep returning + * {@code false} while the failure persists. Once unlock succeeds, release + * confirms and stays confirmed. Swapping in a known-bad descriptor gives + * the slot-specific native primitive a deterministic unlock failure. */ @Test - public void testFailedCloseRetainsFdAndReportsFalse() throws Exception { + public void testFailedUnlockRetainsFdAndReportsFalse() throws Exception { TestUtils.assertMemoryLeak(() -> { String slot = parentDir + "/failed-release"; SlotLock lock = SlotLock.acquire(slot); @@ -144,26 +142,26 @@ public void testFailedCloseRetainsFdAndReportsFalse() throws Exception { int realFd = fdField.getInt(lock); assertTrue("precondition: acquire must hold a live fd", realFd >= 0); try { - // A non-negative fd no process has open: close(2) fails EBADF. + // A non-negative descriptor no process has open makes the + // explicit flock/UnlockFileEx operation fail without consuming + // the real descriptor that continues to hold the slot lock. fdField.setInt(lock, 1_000_000_000); - assertFalse("release must report false when the OS close fails", + assertFalse("release must report false when explicit unlock fails", lock.release()); - assertEquals("failed release must retain the fd for a retry — " - + "dropping it would misreport the flock as released", + assertEquals("failed unlock must retain the fd for a safe retry", 1_000_000_000, fdField.getInt(lock)); - assertFalse("repeat release must keep reporting false while the failure persists", + assertFalse("repeat release must stay false while unlock keeps failing", lock.release()); - // While the release is unconfirmed the REAL flock is still - // held — a second acquire on the slot must fail. + // While the release is unconfirmed the real flock remains held. try (SlotLock ignored = SlotLock.acquire(slot)) { fail("slot must not be acquirable while the original flock fd is still open"); } catch (IllegalStateException expected) { - // good — unconfirmed release really means "still locked". + // good - unconfirmed release really means "still locked". } } finally { fdField.setInt(lock, realFd); } - assertTrue("release must confirm once the close succeeds", lock.release()); + assertTrue("release must confirm once explicit unlock succeeds", lock.release()); assertTrue("confirmed release must stay confirmed", lock.release()); try (SlotLock again = SlotLock.acquire(slot)) { assertEquals(slot, again.slotDir()); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index a268b14a..6db9e009 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -30,9 +30,11 @@ import ch.qos.logback.core.read.ListAppender; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -50,6 +52,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -774,6 +777,285 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { }); } + @Test + public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 8, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[8]; + Sender[] delegates = new Sender[8]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + // Release the real native resources, then forge the + // delayed publication which makes the pool retire the + // slot. The idempotent second close in discardBroken() + // leaves the forged state unchanged. + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(8, pool.leakedSlotCount()); + + // Mix completed and incomplete entries throughout the + // retired list. A recovery pass must remove exactly these + // four without skipping a swapped entry or corrupting the + // bitmap/count relationship. + int[] released = {0, 2, 5, 7}; + for (int i = 0; i < released.length; i++) { + setBooleanField(delegates[released[i]], "slotLockReleased", true); + } + pool.reapIdle(); + + Assert.assertEquals(4, pool.leakedSlotCount()); + Assert.assertEquals(4, ((List) getField(pool, "retiredSlots")).size()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + for (int i = 0; i < slotInUse.length; i++) { + boolean mustRemainRetired = i == 1 || i == 3 || i == 4 || i == 6; + Assert.assertEquals("slot reservation mismatch at index " + i, + mustRemainRetired, slotInUse[i]); + } + + // Exactly the four restored reservations are reusable. + PooledSender[] recovered = new PooledSender[4]; + try { + for (int i = 0; i < recovered.length; i++) { + recovered[i] = pool.borrow(); + } + Assert.assertEquals(4, pool.totalSize()); + Assert.assertEquals(4, pool.leakedSlotCount()); + } finally { + for (int i = 0; i < recovered.length; i++) { + if (recovered[i] != null) { + recovered[i].close(); + } + } + } + } + } + }); + } + + @Test(timeout = 30_000) + public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 6, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[6]; + Sender[] delegates = new Sender[6]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals("all capacity must start retired", + 6, pool.leakedSlotCount()); + + final int borrowerCount = 3; + CountDownLatch allAcquired = new CountDownLatch(borrowerCount); + CountDownLatch allDone = new CountDownLatch(borrowerCount); + CountDownLatch firstAcquired = new CountDownLatch(1); + CountDownLatch initialWaiters = new CountDownLatch(borrowerCount); + CountDownLatch releaseBorrowers = new CountDownLatch(1); + CountDownLatch reparkedWaiters = new CountDownLatch(borrowerCount - 1); + AtomicReference failure = new AtomicReference<>(); + ConcurrentHashMap initialWaiterThreads = new ConcurrentHashMap<>(); + ConcurrentHashMap reparkedWaiterThreads = new ConcurrentHashMap<>(); + PooledSender[] recovered = new PooledSender[borrowerCount]; + pool.setBeforeBorrowWaitHook(() -> { + if (initialWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + initialWaiters.countDown(); + } + }); + for (int i = 0; i < borrowerCount; i++) { + final int borrower = i; + Thread thread = new Thread(() -> { + try { + recovered[borrower] = pool.borrow(); + allAcquired.countDown(); + firstAcquired.countDown(); + if (!releaseBorrowers.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release recovered borrower"); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + try { + if (recovered[borrower] != null) { + recovered[borrower].close(); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + allDone.countDown(); + } + } + }, "sender-pool-retired-waiter-" + i); + thread.start(); + } + + try { + Assert.assertTrue("all borrowers must reach the condition wait", + initialWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("three distinct borrowers must reach the wait path", + borrowerCount, initialWaiterThreads.size()); + + // The hook runs while each borrower holds the pool lock, + // immediately before awaitNanos atomically releases that + // lock and enqueues it. Once the last latch count lands, + // the first reapIdle() cannot acquire the lock until all + // three borrowers are definitely condition waiters. + pool.setBeforeBorrowWaitHook(() -> { + if (reparkedWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + reparkedWaiters.countDown(); + } + }); + setBooleanField(delegates[2], "slotLockReleased", true); + pool.reapIdle(); + + // One restored index admits exactly one borrower. The + // other two must consume signalAll(), lose the capacity + // race, and deterministically re-enter the wait path. + Assert.assertTrue("one borrower must take the first restored index", + firstAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("two distinct borrowers must re-park after the first recovery", + reparkedWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("exactly two distinct borrowers must re-enter the wait path", + borrowerCount - 1, reparkedWaiterThreads.size()); + Assert.assertEquals("exactly one borrower should hold restored capacity", + borrowerCount - 1, allAcquired.getCount()); + + // Recover two non-contiguous entries in one reverse / + // swap-remove pass. A single signal() here would wake + // only one of the two proven waiters; signalAll() must + // wake both and let them claim the two restored indices. + pool.setBeforeBorrowWaitHook(null); + setBooleanField(delegates[0], "slotLockReleased", true); + setBooleanField(delegates[5], "slotLockReleased", true); + pool.reapIdle(); + Assert.assertTrue("all waiting borrowers must receive restored capacity", + allAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(3, pool.totalSize()); + Assert.assertEquals(3, pool.leakedSlotCount()); + + boolean[] seen = new boolean[6]; + for (int i = 0; i < recovered.length; i++) { + int slotIndex = getIntField(slotOf(recovered[i]), "slotIndex"); + Assert.assertFalse("borrowers must receive distinct restored indices", + seen[slotIndex]); + seen[slotIndex] = true; + } + Assert.assertTrue("slot 0 must be restored", seen[0]); + Assert.assertTrue("slot 2 must be restored", seen[2]); + Assert.assertTrue("slot 5 must be restored", seen[5]); + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrowers.countDown(); + Assert.assertTrue("borrower threads must finish", + allDone.await(10, TimeUnit.SECONDS)); + } + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } + } + }); + } + + @Test + public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 1, 2_000, + Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Sender delegate = getDelegate(a); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + Assert.assertTrue("precondition: live flock fd", realFd >= 0); + try { + // Inject one persistent explicit-unlock failure. + // Delegate close must retire the only pool slot rather + // than publish a release while the real flock remains held. + synchronized (slotLock) { + fdField.setInt(slotLock, 1_000_000_000); + } + invokeDiscardBroken(pool, a); + Assert.assertEquals("failed release must retire pool capacity", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + // Remove the fault without calling close again: the + // engine's error-path retry driver runs outside the + // pool lock and must eventually publish completion. + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("flock-release retry never completed"); + } + Thread.sleep(1L); + } + + // A capacity-starved public borrow re-probes the + // retained delegate, frees index 0, and proves the + // recovered flock is genuinely acquirable. + PooledSender recovered = pool.borrow(); + try { + Assert.assertEquals("retry must restore pool capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(1, countSlotDirs()); + } finally { + recovered.close(); + } + } finally { + if (!engine.isCloseCompleted()) { + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + Assert.assertTrue("restored fd must release cleanly", + slotLock.release()); + } + } + } + } + } + }); + } + @Test public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Exception { // Full-stack twin of the two forged-flag recovery tests above: no @@ -830,6 +1112,13 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws // delegate's engine close times out its bounded join, // hands cleanup to the worker's exit path, and reports // the retained flock; the pool must retire the slot. + // Fault the old callback-allocation/registration path. + // C3 makes the owned-engine handoff preallocated, so + // this hook must no longer be reached by production + // teardown. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); manager.setWorkerJoinTimeoutMillis(50L); invokeDiscardBroken(pool, a); Assert.assertEquals( @@ -839,18 +1128,15 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Assert.assertFalse("engine cleanup must still be pending", engine.isCloseCompleted()); - // Un-wedge: the worker finishes its pass, exits (its - // loop was stopped by the close), and runs the - // deferred cleanup that releases the real flock. + // Un-wedge and deterministically reap the manager. + // No sender/engine close retry is allowed: worker exit + // itself must own and finish the preallocated handoff. releaseWorker.countDown(); - long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); - while (!engine.isCloseCompleted()) { - if (System.nanoTime() > deadlineNs) { - throw new AssertionError( - "deferred engine cleanup never ran on manager-worker exit"); - } - Thread.sleep(1); - } + manager.close(); + Assert.assertTrue("manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must run deferred cleanup despite the " + + "old allocation fault injection", + engine.isCloseCompleted()); // The housekeeper tick is the recovery driver. pool.reapIdle(); @@ -876,6 +1162,7 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws throw new AssertionError("trim hook failed", hookErr.get()); } } finally { + manager.setBeforeExitCleanupRegistrationHook(null); manager.setBeforeTrimSyncHook(null); releaseWorker.countDown(); } @@ -884,6 +1171,16 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws }); } + @Test + public void testPreallocatedExitHandoffCleansInRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(0, 1); + } + + @Test + public void testPreallocatedExitHandoffCleansOutOfRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(1, 1); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- @@ -983,6 +1280,92 @@ public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { }); } + @Test + public void testSharedManagerPassCompletionRecoversRetiredPoolSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + long segSize = io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.HEADER_SIZE + + io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + AtomicReference engineRef = new AtomicReference<>(); + manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertEquals(0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT)); + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + IntFunction factory = slotIndex -> { + CursorSendEngine engine = new CursorSendEngine( + slot("default-" + slotIndex), segSize, manager); + engineRef.set(engine); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port); + sender.setCursorEngine(engine, true); + return sender; + }; + SenderPool pool = new SenderPool( + config, 0, 1, 500, 0, Long.MAX_VALUE, factory); + try { + PooledSender lease = pool.borrow(); + Assert.assertTrue("shared manager never entered the sender ring's service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + lease.close(); + + manager.setWorkerJoinTimeoutMillis(50L); + pool.reapIdle(); + Assert.assertEquals("pool must retire the slot while its shared-manager pass is live", + 1, pool.leakedSlotCount()); + CursorSendEngine engine = engineRef.get(); + Assert.assertFalse("engine cleanup must remain pending during the live pass", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + Assert.assertTrue("deferred cleanup did not finish with the ring pass", + cleanupFinished.await(5, TimeUnit.SECONDS)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + + pool.reapIdle(); + Assert.assertEquals("pass completion must drive deferred engine cleanup and restore capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("deferred cleanup must publish the released flock", + engine.isCloseCompleted()); + try (PooledSender recovered = pool.borrow()) { + Assert.assertTrue("recovered slot must be reusable", Files.exists(slot("default-0"))); + } + } finally { + releaseWorker.countDown(); + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + pool.close(); + manager.close(); + } + } + }); + } + @Test public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception { // C1 regression: the startup recovery loop MUST mirror discardBroken / @@ -2318,6 +2701,134 @@ private String slot(String name) { return sfDir + "/" + name; } + private void assertPreallocatedExitHandoffCleansStartupRecoverer( + int strandedIndex, int maxSize) throws Exception { + TestUtils.assertMemoryLeak(() -> { + String strandedId = "default-" + strandedIndex; + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=100;"; + Sender sender = Sender.builder(config).senderId(strandedId).build(); + sender.table("recover").longColumn("v", strandedIndex).atNow(); + sender.flush(); + try { + sender.close(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("drain timed out")); + } + } + Assert.assertTrue("startup-recovery fixture must contain an unacked segment", + hasSegmentFile(slot(strandedId))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean instrumented = new AtomicBoolean(); + AtomicReference engineRef = new AtomicReference<>(); + AtomicReference managerRef = new AtomicReference<>(); + AtomicReference hookErr = new AtomicReference<>(); + CountDownLatch releaseWorker = new CountDownLatch(1); + IntFunction factory = idx -> { + Sender sender = Sender.builder(config).senderId("default-" + idx).build(); + if (idx == strandedIndex && instrumented.compareAndSet(false, true)) { + try { + CursorSendEngine engine = (CursorSendEngine) getField(sender, "cursorEngine"); + SegmentManager manager = (SegmentManager) getField(engine, "manager"); + CountDownLatch workerBlocked = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting to release recovery worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + if (!workerBlocked.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("recovery manager never entered a service pass"); + } + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engineRef.set(engine); + managerRef.set(manager); + } catch (Throwable t) { + try { + sender.close(); + } catch (Throwable ignored) { + } + throw new RuntimeException(t); + } + } + return sender; + }; + + SenderPool pool = null; + try { + pool = newPoolWithFactory(config, 0, maxSize, 2_000, factory); + CursorSendEngine engine = engineRef.get(); + SegmentManager manager = managerRef.get(); + Assert.assertNotNull("startup recovery must build the stranded slot", engine); + if (strandedIndex < maxSize) { + Assert.assertEquals("in-range recoverer must remain retired while worker is live", + 1, pool.leakedSlotCount()); + } else { + Assert.assertEquals("out-of-range recovery must not consume pool capacity", + 0, pool.leakedSlotCount()); + } + Assert.assertFalse("cleanup must remain pending while the worker is live", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + manager.close(); + Assert.assertTrue("recovery manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must complete startup-recoverer cleanup without " + + "a sender or engine close retry [index=" + strandedIndex + "]", + engine.isCloseCompleted()); + if (hookErr.get() != null) { + throw new AssertionError("recovery worker hook failed", hookErr.get()); + } + if (strandedIndex < maxSize) { + pool.reapIdle(); + Assert.assertEquals("late cleanup must restore in-range pool capacity", + 0, pool.leakedSlotCount()); + } + try (SlotLock ignored = SlotLock.acquire(slot(strandedId))) { + // Completion must mean the real slot flock is reusable. + } + } finally { + releaseWorker.countDown(); + SegmentManager manager = managerRef.get(); + if (manager != null) { + manager.setBeforeExitCleanupRegistrationHook(null); + manager.setBeforeTrimSyncHook(null); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + } + CursorSendEngine engine = engineRef.get(); + if (engine != null && !engine.isCloseCompleted()) { + engine.close(); + } + if (pool != null) { + pool.close(); + } + } + } + }); + } + private int countSlotDirs() { if (!Files.exists(sfDir)) { return 0; From a0865758215b20aecceb963ae78d2f3826add3a8 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 21:29:48 +0100 Subject: [PATCH 15/64] fix(qwp): harden cleanup and recovery lifecycle --- core/src/main/c/share/net.c | 5 + core/src/main/c/share/net.h | 8 + core/src/main/c/windows/net.c | 9 + .../cutlass/http/client/WebSocketClient.java | 9 + .../qwp/client/QwpWebSocketSender.java | 48 ++ .../client/sf/cursor/CursorSendEngine.java | 145 +++++- .../sf/cursor/CursorWebSocketSendLoop.java | 26 +- .../qwp/client/sf/cursor/SegmentManager.java | 107 ++-- .../qwp/client/sf/cursor/SegmentRing.java | 31 ++ .../io/questdb/client/impl/SenderPool.java | 71 ++- .../client/network/JavaTlsClientSocket.java | 8 + .../java/io/questdb/client/network/Net.java | 2 + .../questdb/client/network/NetworkFacade.java | 9 + .../client/network/NetworkFacadeImpl.java | 5 + .../questdb/client/network/PlainSocket.java | 13 +- .../io/questdb/client/network/Socket.java | 11 + .../cutlass/qwp/client/CloseDrainTest.java | 77 +++ ...WebSocketSendLoopBlockedSendCloseTest.java | 256 ++++++++++ .../cursor/FlockReleaseRetryDriverTest.java | 247 ++++++++++ .../SegmentManagerUnlinkFailureTest.java | 350 +++++++++++++ .../qwp/websocket/TestWebSocketServer.java | 15 + .../client/test/impl/SenderPoolSfTest.java | 172 +++++++ .../network/SocketTrafficShutdownTest.java | 463 ++++++++++++++++++ 23 files changed, 1987 insertions(+), 100 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java create mode 100644 core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 3b0162fc..9b58fd65 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -128,6 +128,11 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send return com_questdb_network_Net_EOTHERDISCONNECT; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + return shutdown((int) fd, SHUT_RDWR); +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong ptr, jint len) { ssize_t n; diff --git a/core/src/main/c/share/net.h b/core/src/main/c/share/net.h index 27143639..d80617e2 100644 --- a/core/src/main/c/share/net.h +++ b/core/src/main/c/share/net.h @@ -80,6 +80,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send (JNIEnv *, jclass, jint, jlong, jint); +/* + * Class: io_questdb_client_network_Net + * Method: shutdown + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *, jclass, jint); + /* * Class: io_questdb_client_network_Net * Method: getSndBuf diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index fd290629..9cc98000 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -230,6 +230,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_configureNonBlocking return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + const int result = shutdown((SOCKET) fd, SD_BOTH); + if (result == SOCKET_ERROR) { + SaveLastError(); + } + return result; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong addr, jint len) { const int n = recv((SOCKET) fd, (char *) addr, len, 0); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..94018fff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -248,6 +248,15 @@ public void close() { } } + /** + * Shuts down socket traffic without releasing the descriptor or freeing + * buffers that a concurrent I/O worker may still access. The owner must + * call {@link #close()} after joining the worker to complete cleanup. + */ + public void closeTraffic() { + socket.closeTraffic(); + } + /** * Connects to a WebSocket server. * diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3586a4d2..9f454c9c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -189,6 +189,9 @@ public class QwpWebSocketSender implements Sender { // Null in production; set reflectively by tests. @TestOnly private volatile java.util.function.Supplier clientFactoryOverride; + // Test-only lifecycle witness. drainOnClose() invokes and clears it after + // confirming a real unacknowledged close target, immediately before waiting. + private volatile Runnable closeDrainWaitingHook; // close() drain timeout in millis. Default applied at construction. // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. @@ -290,6 +293,9 @@ public class QwpWebSocketSender implements Sender { // re-probe retired slots to recover their capacity. volatile: written on // the closing thread, read by pool threads. private volatile boolean slotLockReleased; + // Optional owning-pool notification, relayed after the engine confirms the + // flock release and slotLockReleased is visible to pool re-probes. + private volatile Runnable slotLockReleaseListener; // Engine whose close() could not complete during sender close() — its // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() // re-probes it so a late flock release becomes visible to the owning pool. @@ -1368,6 +1374,17 @@ public boolean isSlotLockReleased() { return false; } + /** + * Registers a callback for confirmed SF slot-lock release. Pools use this + * to wake borrowers without waiting for a timeout or housekeeper tick. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && slotLockReleased) { + listener.run(); + } + } + @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { checkNotClosed(); @@ -2207,6 +2224,16 @@ public void setClientFactoryOverride(java.util.function.Supplier= target) { return; } + Runnable hook = closeDrainWaitingHook; + closeDrainWaitingHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); + } + } long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; while (cursorEngine.ackedFsn() < target) { // Stop on a latched terminal (acks will never reach target); @@ -3632,6 +3672,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm resetTableBuffersAfterFlush(keys); } + private void onSlotLockReleased() { + slotLockReleased = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { + listener.run(); + } + } + private void resetTableBuffersAfterFlush(ObjList keys) { for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 25699dcd..95addd39 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -30,6 +30,8 @@ import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; +import java.util.ArrayDeque; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; @@ -65,7 +67,15 @@ public final class CursorSendEngine implements QuietCloseable { public static final long DEFAULT_APPEND_DEADLINE_NANOS = 30_000_000_000L; private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class); + private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY = + runnable -> new Thread(runnable, "qdb-sf-flock-release-retry"); + private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object(); + private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>(); + private static volatile Runnable afterFlockReleaseRetryFailureHook; private static volatile Runnable beforeDeferredCloseCreationHook; + private static Thread flockReleaseRetryThread; + private static volatile ThreadFactory flockReleaseRetryThreadFactory = + DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY; private final long appendDeadlineNanos; // Number of times appendBlocking observed BACKPRESSURE_NO_SPARE on its first // ring.appendOrFsn attempt. One increment per blocking-call that had to wait @@ -136,6 +146,9 @@ public final class CursorSendEngine implements QuietCloseable { // isCloseCompleted() from pool threads re-probing a retired slot (see the // getter for why it must not synchronize). private volatile boolean closeCompleted; + // Invoked after closeCompleted publishes a confirmed flock release. Used + // by an owning sender pool to wake capacity-starved borrowers immediately. + private volatile Runnable slotLockReleaseListener; // Test-only hook run by finishClose() between the terminal cleanup and // the flock release. Lets a test park the releasing thread inside the // cleanup/release window and assert that closeCompleted stays false — @@ -143,8 +156,8 @@ public final class CursorSendEngine implements QuietCloseable { // held. volatile: finishClose may run on the manager worker's exit // thread while the hook is installed from a test thread. private volatile Runnable beforeFlockReleaseHook; - // Exactly one daemon drives a failed flock release to completion. The - // error path only: ordinary closes never allocate or start a thread. + // Ensures this engine has at most one entry in the shared flock-release + // retry driver. The error path only: ordinary closes never enqueue work. private final AtomicBoolean flockReleaseRetryStarted = new AtomicBoolean(); // Published before deferredClose is registered. The manager lock provides // the callback handoff fence; volatile also covers a direct test/retry read. @@ -781,6 +794,15 @@ private void finishClose(boolean fullyDrained) { } } + /** + * Installs a hook invoked after each failed shared-driver flock release. + * Test-only: makes persistent retry progress observable without sleeps. + */ + @TestOnly + public static void setAfterFlockReleaseRetryFailureHook(Runnable hook) { + afterFlockReleaseRetryFailureHook = hook; + } + /** * Installs a constructor fault hook immediately before the bound deferred * close callback is created. Test-only: proves callback allocation failure @@ -791,6 +813,23 @@ public static void setBeforeDeferredCloseCreationHook(Runnable hook) { beforeDeferredCloseCreationHook = hook; } + /** + * Overrides creation of the single shared flock-release retry thread. + * Test-only: makes thread creation/start failure and persistent retry + * scalability deterministic without relying on scheduler timing. + */ + @TestOnly + public static void setFlockReleaseRetryThreadFactory(ThreadFactory factory) { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + if (flockReleaseRetryThread != null || !FLOCK_RELEASE_RETRY_QUEUE.isEmpty()) { + throw new IllegalStateException("flock-release retry driver is active"); + } + flockReleaseRetryThreadFactory = factory == null + ? DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY + : factory; + } + } + /** * Installs a hook that {@link #finishClose} runs between the terminal * cleanup and the slot-flock release. Test-only: makes the otherwise @@ -849,42 +888,97 @@ private boolean retryFlockReleaseIfReady() { } if (released) { closeCompleted = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { + try { + listener.run(); + } catch (Throwable ignored) { + // A notification failure must not invalidate a confirmed release. + } + } return true; } startFlockReleaseRetry(); return false; } + private static void runFlockReleaseRetryDriver() { + while (true) { + final int roundSize; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + roundSize = FLOCK_RELEASE_RETRY_QUEUE.size(); + if (roundSize == 0) { + flockReleaseRetryThread = null; + return; + } + } + boolean hasFailures = false; + for (int i = 0; i < roundSize; i++) { + final CursorSendEngine engine; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + engine = FLOCK_RELEASE_RETRY_QUEUE.pollFirst(); + } + if (engine.retryFlockReleaseIfReady()) { + engine.flockReleaseRetryStarted.set(false); + } else { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(engine); + } + hasFailures = true; + Runnable hook = afterFlockReleaseRetryFailureHook; + if (hook != null) { + hook.run(); + } + } + } + if (hasFailures) { + // Interruption must not abandon a retained flock, but clear + // the flag so subsequent parks still throttle retries. + Thread.interrupted(); + LockSupport.parkNanos(100_000_000L); + } + } + } + private void startFlockReleaseRetry() { if (!flockReleaseRetryStarted.compareAndSet(false, true)) { return; } - try { - Thread retryThread = new Thread(() -> { - while (!retryFlockReleaseIfReady()) { - try { - Thread.sleep(100L); - } catch (InterruptedException ignored) { - // A failed flock release must remain retryable until - // confirmed or process exit; interruption is not a - // safe reason to abandon the slot permanently. + Throwable startFailure = null; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(this); + if (flockReleaseRetryThread == null) { + try { + Thread retryThread = flockReleaseRetryThreadFactory.newThread( + CursorSendEngine::runFlockReleaseRetryDriver); + if (retryThread == null) { + throw new IllegalStateException("retry thread factory returned null"); + } + retryThread.setDaemon(true); + flockReleaseRetryThread = retryThread; + retryThread.start(); + } catch (Throwable t) { + startFailure = t; + flockReleaseRetryThread = null; + CursorSendEngine queued; + while ((queued = FLOCK_RELEASE_RETRY_QUEUE.pollFirst()) != null) { + queued.flockReleaseRetryStarted.set(false); } } - }, "qdb-sf-flock-release-retry"); - retryThread.setDaemon(true); - retryThread.start(); + } + } + if (startFailure == null) { LOG.error("SF slot flock release failed during engine close; keeping " - + "closeCompleted=false and retrying outside the pool lock so " + + "closeCompleted=false and retrying on the shared driver so " + "retired capacity recovers after the transient failure [slot={}]", sfDir == null ? "" : sfDir); - } catch (Throwable t) { + } else { // A later explicit close() can still retry without repeating the - // one-time ring/watermark cleanup. The kernel remains the final - // process-exit backstop if thread creation and all retries fail. - flockReleaseRetryStarted.set(false); + // one-time ring/watermark cleanup. The failed queue is cleared so + // the driver does not retain engines it cannot service. LOG.error("Could not start SF flock-release retry driver; close() may be " + "invoked again to retry [slot={}, error={}]", - sfDir == null ? "" : sfDir, String.valueOf(t)); + sfDir == null ? "" : sfDir, String.valueOf(startFailure)); } } @@ -910,6 +1004,17 @@ public boolean isCloseCompleted() { return closeCompleted; } + /** + * Registers a callback for confirmed SF slot-lock release. If release + * already completed, invokes the callback before returning. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && closeCompleted) { + listener.run(); + } + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..32ec4b30 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -200,7 +200,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); - private WebSocketClient client; + private volatile WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's // handler. Null disables async delivery entirely; the producer-side @@ -763,6 +763,25 @@ public synchronized void close() { // would block forever. isAlive()==false also covers the normal // post-exit case where the latch is already counted down. if (t.isAlive()) { + // Break a native send/receive before joining. Full client close + // must remain after the worker exit because it frees buffers the + // worker may still access. + WebSocketClient activeClient = client; + if (activeClient != null) { + try { + activeClient.closeTraffic(); + } catch (Throwable e) { + // A custom transport that cannot safely cancel traffic + // must not be destructively closed while the worker is + // live. Fail without joining indefinitely; the worker's + // exit path retains ownership of final cleanup. + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + } try { shutdownLatch.await(); } catch (InterruptedException e) { @@ -1114,7 +1133,10 @@ private void clearDurableAckTracking() { * before the first connect attempt — see {@link #failPaced}. */ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptMillis) { - if (reconnectFactory == null || !running) { + if (!running) { + return; + } + if (reconnectFactory == null) { recordFatal(initial); return; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 15d5c607..578c4f07 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -80,6 +80,7 @@ public final class SegmentManager implements QuietCloseable { private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; private final AtomicLong fileGeneration = new AtomicLong(); + private final FilesFacade filesFacade; private final Object lock = new Object(); private final long maxTotalBytes; // Reused by the manager worker thread to build spare-segment paths @@ -173,11 +174,11 @@ public final class SegmentManager implements QuietCloseable { private volatile Thread workerThread; public SegmentManager(long segmentSizeBytes) { - this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); } public SegmentManager(long segmentSizeBytes, long pollNanos) { - this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); } /** @@ -203,6 +204,11 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) { * hold an initial active plus one hot spare. */ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE); + } + + @TestOnly + public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) { // The pathScratch field initializer has already allocated its native // buffer by the time this body runs, so a validation throw must free // it or every failed construction leaks 256 bytes of native memory @@ -217,6 +223,7 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) "maxTotalBytes (" + maxTotalBytes + ") must allow at least one segment of " + segmentSizeBytes + " bytes"); } + this.filesFacade = filesFacade; this.segmentSizeBytes = segmentSizeBytes; this.pollNanos = pollNanos; this.maxTotalBytes = maxTotalBytes; @@ -613,21 +620,19 @@ public void wakeWorker() { * files in {@code dir}, or {@code -1} if none exist. Skips files that * don't match the pattern (e.g. the legacy {@code sf-initial.sfa}). */ - private static long scanMaxGeneration(String dir) { + private long scanMaxGeneration(String dir) { long max = -1L; - if (!Files.exists(dir)) return max; - long find = Files.findFirst(dir); + if (!filesFacade.exists(dir)) return max; + long find = filesFacade.findFirst(dir); if (find < 0) { - LOG.warn("scanMaxGeneration could not enumerate {}; " - + "next spare may collide with an existing on-disk segment", dir); - return max; + throw new IllegalStateException("could not enumerate SF segment directory " + dir); } if (find == 0) return max; try { int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - rc = Files.findNext(find); + String name = Files.utf8ToString(filesFacade.findName(find)); + rc = filesFacade.findNext(find); if (name == null || !name.startsWith("sf-") || !name.endsWith(".sfa")) { continue; } @@ -640,8 +645,11 @@ private static long scanMaxGeneration(String dir) { // sf-initial.sfa or non-hex — skip } } + if (rc < 0) { + throw new IllegalStateException("could not fully enumerate SF segment directory " + dir); + } } finally { - Files.findClose(find); + filesFacade.findClose(find); } return max; } @@ -769,7 +777,7 @@ private void serviceRing0(RingEntry e) { // via its long-ptr overload, bypassing the byte[] + native // malloc that the String overload would incur on every // rotation. - spare = MmapSegment.create(FilesFacade.INSTANCE, + spare = MmapSegment.create(filesFacade, pathScratch.ptr(), path, e.ring.nextSeqHint(), segmentSizeBytes); } @@ -815,7 +823,7 @@ private void serviceRing0(RingEntry e) { // on disk, this is the second-line defense. Repeated // unlink on an already-removed path is a harmless no-op. if (path != null) { - Files.remove(path); + filesFacade.remove(path); } } } @@ -828,59 +836,54 @@ private void serviceRing0(RingEntry e) { // The watermark write and totalBytes commit are registration-gated // under `lock` so stale worker snapshots cannot touch the // engine-owned watermark or mutate accounting after deregister() - // returns. drainTrimmable still runs for stale snapshots: it - // transfers ownership of fully-acked sealed segments to this - // worker, preserving the old close + unlink behavior. + // returns. Trimming still runs for stale snapshots: a successful + // close + unlink transfers the fully-acked segment out of the ring, + // preserving cleanup without touching deregistered accounting. // // munmap + unlink stay outside the lock — they can be slow // and shouldn't block register/deregister or sibling rings. - ObjList trim; Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } + boolean trimFailed = false; + while (true) { + MmapSegment segment = e.ring.firstTrimmable(); + if (segment == null) { + break; + } + String path = segment.path(); + try { + segment.close(); + if (path != null && !filesFacade.remove(path)) { + trimFailed = true; + LOG.warn("Failed to unlink trimmed segment {}", path); + break; + } + synchronized (lock) { + if (e.ring.removeTrimmable(segment) && e.isRegistered()) { + totalBytes -= segment.sizeBytes(); + } + } + } catch (Throwable t) { + trimFailed = true; + LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); + break; + } + } + // An unlink failure leaves the acknowledged segment in the ring and + // on disk. Do not advance the persisted watermark past it: recovery + // must continue to see the failed path as live bookkeeping rather + // than infer that trim committed. A later successful retry persists + // the current cumulative ACK normally. synchronized (lock) { - boolean registered = e.isRegistered(); - // Persist the current ackedFsn watermark BEFORE the trim runs. - // On host crash between the persist and the unlinks below, the - // segments survive and the watermark is correct. On crash AFTER - // the unlinks but before next tick, the segments are gone and - // the watermark is stale, but recovery clamps with - // max(lowestSurvivingBaseSeq - 1, watermark) so either ordering - // is safe. Memory-mode rings (and callers that didn't supply a - // watermark) skip the write. - // Persist only on advance to avoid pointless mmap stores when - // ackedFsn is steady. The store is a single 8-byte put against - // an already-mapped region -- no syscall, no allocation -- but - // the gate keeps the dirty-page footprint minimal under - // steady-state load with no new acks arriving. - if (registered && e.watermark != null) { + if (!trimFailed && e.isRegistered() && e.watermark != null) { long currentAck = e.ring.ackedFsn(); if (currentAck > e.lastPersistedAck) { e.watermark.write(currentAck); e.lastPersistedAck = currentAck; } } - trim = e.ring.drainTrimmable(); - if (registered && trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - totalBytes -= trim.get(i).sizeBytes(); - } - } - } - if (trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - MmapSegment s = trim.get(i); - String path = s.path(); - try { - s.close(); - if (path != null && !Files.remove(path)) { - LOG.warn("Failed to unlink trimmed segment {}", path); - } - } catch (Throwable t) { - LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); - } - } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c8516c4c..8d929ae1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -516,6 +516,21 @@ public synchronized MmapSegment firstSealed() { return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; } + /** + * Returns the oldest fully acknowledged sealed segment without removing + * it. The segment manager keeps it owned by the ring until close + unlink + * succeeds, so a failed unlink cannot make the path disappear from live + * bookkeeping or allow its identifier to be reused. + */ + public synchronized MmapSegment firstTrimmable() { + if (sealedSegments.size() == 0) { + return null; + } + MmapSegment segment = sealedSegments.get(0); + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + return lastSeq <= ackedFsn ? segment : null; + } + /** Active segment -- exposed for the I/O thread's "send next batch" path. */ /** * Walks every published frame in the ring (sealed segments plus the active @@ -634,6 +649,22 @@ public long publishedFsn() { return publishedFsn; } + /** + * Commits removal of the segment returned by {@link #firstTrimmable()}. + * Returns false if concurrent lifecycle activity changed the head. + */ + public synchronized boolean removeTrimmable(MmapSegment segment) { + if (sealedSegments.size() == 0 || sealedSegments.get(0) != segment) { + return false; + } + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + if (lastSeq > ackedFsn) { + return false; + } + sealedSegments.remove(0); + return true; + } + /** * Registers a wakeup callback that the producer thread will invoke when * a hot spare is needed -- either right after a rotation has consumed the diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index d2cae5ad..940e6049 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -220,8 +220,9 @@ public final class SenderPool implements AutoCloseable { // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) // for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily // built pass-2 work list (same-base slots at index >= maxSize) and its - // cursor; recoveryComplete latches true when the whole scan finishes or is - // aborted, making runStartupRecoveryStep()/...ToCompletion() idempotent. + // cursor; recoveryComplete latches true only when the whole scan finishes. + // A transient build failure or drain timeout leaves the current candidate + // pending so a later tick or explicit drive can retry it on the same pool. private int recoveryInRangeNext; private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; @@ -427,8 +428,9 @@ void runStartupRecoveryToCompletion() { * No-op (returns {@code false}) when SF is off, the pool is shutting down, or * recovery has already finished. * - * @return {@code true} if recovery has more work (call again), {@code false} - * when recovery is complete or the pool is shutting down + * @return {@code true} if recovery has more work immediately; {@code false} + * when recovery is complete, the pool is shutting down, or a transient + * failure deferred the current candidate until a later tick */ boolean runStartupRecoveryStep() { if (!storeAndForward || closed || recoveryComplete) { @@ -469,9 +471,10 @@ boolean runStartupRecoveryStep() { * {@code slotInUse} entry and are never allocated by borrow(). *

    * Best-effort throughout: a build/close Error or a slow drain is logged and - * never propagates, since the data stays durable on disk for a later attempt; - * the first build failure or drain timeout latches {@code recoveryComplete} - * (the failure will very likely repeat for every remaining slot). + * never propagates, since the data stays durable on disk for a later attempt. + * A build failure or drain timeout stops the current drive but leaves its + * candidate pending so a later housekeeper tick can retry after a transient + * condition clears; it does not poison recovery for the life of the pool. *

    * Boundedness / residual window. Recovery is driven on the * PoolHousekeeper thread, and {@code close()} relies on a step finishing @@ -541,9 +544,8 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryInRangeNext++; continue; } - // A real candidate -> spend the step on it. Advance the cursor first - // so a resume never reprocesses this index. - recoveryInRangeNext++; + // A real candidate -> spend the step on it. Advance the cursor only + // after success so a transient build/drain failure remains retryable. boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { @@ -579,12 +581,12 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { lock.unlock(); } if (stopScan) { - // A build failure or drain timeout that will very likely repeat - // for every remaining slot -- abort the scan; the data stays - // durable on disk for a later attempt. Do not start pass 2. - recoveryComplete = true; + // Stop this drive without advancing the cursor. The same live + // pool retries this candidate after the transient condition is + // removed instead of requiring pool recreation. return false; } + recoveryInRangeNext++; return true; } @@ -605,9 +607,10 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (closed) { return false; } - int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext++); + int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext); String slotPath = sfDir + "/" + slotBaseId + "-" + idx; if (!OrphanScanner.isCandidateOrphan(slotPath)) { + recoveryOutOfRangeNext++; continue; } boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); @@ -626,9 +629,12 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { slotPath); } if (stopScan) { - recoveryComplete = true; + // Keep the out-of-range cursor on this candidate. In particular, + // a transient flock/build collision must be retried after the + // primary sender returns; no capacity bookkeeping is involved. return false; } + recoveryOutOfRangeNext++; return true; } @@ -678,7 +684,7 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // likely repeat for every remaining slot, so stop here rather // than pay a connect timeout per slot. LOG.warn("startup SF recovery: could not open slot {} ({}); " - + "skipping remaining slots", slotPath, buildErr.toString()); + + "deferring this and remaining slots", slotPath, buildErr.toString()); return true; } try { @@ -688,7 +694,7 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // same reasoning as the build-failure case above. if (!recoverer.delegate().drain(remainingMillis)) { LOG.warn("startup SF recovery: drain did not ack slot {} " - + "within {}ms; skipping remaining slots", + + "within {}ms; deferring this and remaining slots", slotPath, remainingMillis); stopScan = true; } @@ -811,10 +817,10 @@ public PooledSender borrow() { // Capacity-starved: re-probe retired slots BEFORE the terminal // timeout check — a deferred engine cleanup may have released a // flock since the retire, and the freed index can admit a - // creation right now. The release itself never signals - // slotReleased (it happens in the delegate on a worker/I/O-thread - // exit path, volatile writes only), so this poll is the only way - // a borrower learns of it. Ordering matters twice over: a + // creation right now. The delegate normally signals this pool + // after deferred release, while this probe also covers delegates + // that do not expose that notification and release/listener races. + // Ordering matters twice over: a // zero-timeout (try-once) borrow must get its one probe before // throwing, and a borrower whose awaitNanos budget just expired // must get a final probe on its wake-up pass instead of timing @@ -1207,7 +1213,11 @@ public int leakedSlotCount() { } private SenderSlot createUnlocked(int slotIndex) { - return new SenderSlot(senderFactory.apply(slotIndex), this, slotIndex); + Sender delegate = senderFactory.apply(slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + } + return new SenderSlot(delegate, this, slotIndex); } /** @@ -1218,7 +1228,11 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { - return new SenderSlot(recoverySenderFactory.apply(slotIndex), this, slotIndex); + Sender delegate = recoverySenderFactory.apply(slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + } + return new SenderSlot(delegate, this, slotIndex); } private Sender defaultSender(int slotIndex) { @@ -1398,6 +1412,15 @@ private boolean reclaimSlot(SenderSlot s, String context) { return false; } + private void recoverReleasedSlots() { + lock.lock(); + try { + reprobeRetiredSlots(); + } finally { + lock.unlock(); + } + } + /** * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to * the free set any whose delegate now reports the flock released — the diff --git a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java index c1b1eec7..b5e43a35 100644 --- a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java +++ b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java @@ -149,6 +149,14 @@ public void close() { } } + @Override + public void closeTraffic() { + // A concurrent TLS send may still use the SSLEngine and direct buffers. + // Shut down only the delegate traffic path here; close() releases the + // retained fd and frees TLS state after the owning worker has stopped. + delegate.closeTraffic(); + } + @Override public int getFd() { return delegate.getFd(); diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index f649d330..b9e669d0 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -151,6 +151,8 @@ public static void init() { public static native int setTcpNoDelay(int fd, boolean noDelay); + public static native int shutdown(int fd); + public static long sockaddr(int ipv4address, int port) { SOCK_ADDR_COUNTER.incrementAndGet(); return sockaddr0(ipv4address, port); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java index d23824a5..4c66f0df 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java @@ -78,6 +78,15 @@ public interface NetworkFacade { int setTcpNoDelay(int fd, boolean noDelay); + /** + * Shuts down traffic on a descriptor owned by this facade without releasing + * the descriptor. Custom facades must override this method rather than let + * native code operate on a synthetic or remapped descriptor. + */ + default int shutdown(int fd) { + throw new UnsupportedOperationException("traffic shutdown is not supported by this network facade"); + } + long sockaddr(int address, int port); int socketTcp(boolean blocking); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java index 64ea0dc7..1d1d7e20 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java @@ -132,6 +132,11 @@ public int setTcpNoDelay(int fd, boolean noDelay) { return Net.setTcpNoDelay(fd, noDelay); } + @Override + public int shutdown(int fd) { + return Net.shutdown(fd); + } + @Override public long sockaddr(int address, int port) { return Net.sockaddr(address, port); diff --git a/core/src/main/java/io/questdb/client/network/PlainSocket.java b/core/src/main/java/io/questdb/client/network/PlainSocket.java index 555affd2..cc175f51 100644 --- a/core/src/main/java/io/questdb/client/network/PlainSocket.java +++ b/core/src/main/java/io/questdb/client/network/PlainSocket.java @@ -29,7 +29,7 @@ public class PlainSocket implements Socket { private final Logger log; private final NetworkFacade nf; - private int fd = -1; + private volatile int fd = -1; public PlainSocket(NetworkFacade nf, Logger log) { this.nf = nf; @@ -37,13 +37,22 @@ public PlainSocket(NetworkFacade nf, Logger log) { } @Override - public void close() { + public synchronized void close() { if (fd != -1) { nf.close(fd, log); fd = -1; } } + @Override + public synchronized void closeTraffic() { + if (fd != -1 && nf.shutdown(fd) != 0) { + throw new IllegalStateException( + "could not shut down socket traffic [fd=" + fd + ", errno=" + nf.errno() + ']' + ); + } + } + @Override public int getFd() { return fd; diff --git a/core/src/main/java/io/questdb/client/network/Socket.java b/core/src/main/java/io/questdb/client/network/Socket.java index 0cdce517..6fceb30b 100644 --- a/core/src/main/java/io/questdb/client/network/Socket.java +++ b/core/src/main/java/io/questdb/client/network/Socket.java @@ -37,6 +37,17 @@ public interface Socket extends QuietCloseable { int WRITE_FLAG = 1; + /** + * Closes only the network traffic path so a concurrent blocking send or + * receive returns. Implementations must retain any I/O buffers until the + * owning worker has stopped and {@link #close()} performs full cleanup. + * The compatibility default fails without touching implementation-owned + * resources; custom sockets must override this method to opt in. + */ + default void closeTraffic() { + throw new UnsupportedOperationException("traffic shutdown is not supported by this socket"); + } + /** * @return file descriptor associated with the socket. */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java index 59d0f83d..e4fb86c5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java @@ -51,6 +51,83 @@ */ public class CloseDrainTest { + @Test(timeout = 30_000L) + public void testCloseBlocksAcrossAllReplicaWindowUntilPromotion() throws Exception { + DelayingAckHandler handler = new DelayingAckHandler(0); + try (TestWebSocketServer server = new TestWebSocketServer(handler, false, "PRIMARY")) { + server.setRejectWithRole("REPLICA"); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + server.getPort() + + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=100" + + ";close_flush_timeout_millis=10000;"; + QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg); + CountDownLatch closeDrainWaiting = new CountDownLatch(1); + CountDownLatch releaseCloseDrain = new CountDownLatch(1); + AtomicReference closeFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseDrainWaitingHook(() -> { + closeDrainWaiting.countDown(); + try { + if (!releaseCloseDrain.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("promotion did not release the close-drain witness"); + } + } catch (Throwable t) { + hookFailure.set(t); + } + }); + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "all-replica-close-drain"); + try { + closer.start(); + Assert.assertTrue("server never produced the all-replica role rejection", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + Assert.assertTrue("close never observed its real unacknowledged drain target", + closeDrainWaiting.await(5, TimeUnit.SECONDS)); + + Assert.assertTrue("pre-promotion witness must include a role rejection", + server.roleRejectCount() >= 1); + Assert.assertEquals("pre-promotion data must remain unacknowledged", + -1L, sender.getAckedFsn()); + Assert.assertTrue("close must remain pending for the whole all-replica window", + closer.isAlive()); + Assert.assertEquals("promotion must not have delivered or replayed the frame yet", + 0L, handler.nextSeq.get()); + + // The close-drain barrier has held continuously since it observed + // targetFsn > ackedFsn. Promote first, then release the barrier: + // completion therefore cannot precede the deterministic recovery event. + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + + Assert.assertFalse("close did not complete after promotion", closer.isAlive()); + Assert.assertNull("close-drain witness failed", hookFailure.get()); + Assert.assertNull("close failed after promotion", closeFailure.get()); + Assert.assertEquals("the unacknowledged frame must be delivered exactly once", + 1L, handler.nextSeq.get()); + } finally { + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + sender.close(); + } + } + } + @Test public void testCloseBlocksUntilAckArrives() throws Exception { // Server delays every ACK by 800ms. With the default diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java new file mode 100644 index 00000000..0a53e28e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java @@ -0,0 +1,256 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSendLoopBlockedSendCloseTest { + + @Test(timeout = 30_000L) + public void testCloseBreaksBlockedSendBeforeJoiningWorker() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(true); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "blocked-send-closer"); + closer.start(); + + Assert.assertTrue("close did not break the traffic path before joining", + client.trafficClosed.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("traffic path must close exactly once", 1, client.trafficCloseCount.get()); + Assert.assertEquals("the closer must break traffic, not the I/O worker", + closer, client.trafficCloseThread.get()); + Assert.assertTrue("blocked send did not observe traffic-path closure", + client.sendExited.await(5, TimeUnit.SECONDS)); + + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("close did not join the I/O worker", closer.isAlive()); + Assert.assertNull("close failed", closeFailure.get()); + Assert.assertNull("ordinary close must not manufacture a terminal error", loop.getTerminalError()); + + Thread ioThread = client.sendThread.get(); + Assert.assertNotNull(ioThread); + Assert.assertFalse("I/O worker must be dead when close returns", ioThread.isAlive()); + Assert.assertEquals("full client cleanup must run exactly once", 1, client.cleanupCount.get()); + Assert.assertEquals("the I/O worker must own cleanup before publishing exit", + ioThread, client.cleanupThread.get()); + Assert.assertFalse("close must not manufacture caller interruption", + closer.isInterrupted()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test(timeout = 30_000L) + public void testUnsupportedCustomTransportFailsWithoutDestroyingWorkerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(false); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + CountDownLatch closeEntered = new CountDownLatch(1); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + closeEntered.countDown(); + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "unsupported-transport-closer"); + closer.start(); + + Assert.assertTrue("close did not start", closeEntered.await(5, TimeUnit.SECONDS)); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("unsupported transport made close join indefinitely", closer.isAlive()); + Assert.assertTrue(closeFailure.get() instanceof LineSenderException); + Assert.assertTrue(closeFailure.get().getCause() instanceof UnsupportedOperationException); + Assert.assertEquals("unsupported cancellation must not release the blocked send", + 1L, client.sendExited.getCount()); + Assert.assertEquals("unsupported cancellation must not perform full cleanup", 0, client.cleanupCount.get()); + Assert.assertTrue("worker must retain resource ownership", client.sendThread.get().isAlive()); + + client.releaseSend.countDown(); + Assert.assertTrue("released send did not exit", client.sendExited.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("worker did not complete delegated cleanup", client.cleanupDone.await(5, TimeUnit.SECONDS)); + client.sendThread.get().join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker lingered after the custom transport was released", client.sendThread.get().isAlive()); + Assert.assertEquals(1, client.cleanupCount.get()); + Assert.assertNull("ordinary worker exit must remain non-terminal", loop.getTerminalError()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + Thread ioThread = client.sendThread.get(); + if (ioThread != null) { + ioThread.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static final class BlockingSendClient extends WebSocketClient { + private final AtomicBoolean cleanupClaimed = new AtomicBoolean(); + private final AtomicInteger cleanupCount = new AtomicInteger(); + private final CountDownLatch cleanupDone = new CountDownLatch(1); + private final AtomicReference cleanupThread = new AtomicReference<>(); + private final boolean trafficShutdownSupported; + private final CountDownLatch releaseSend = new CountDownLatch(1); + private final CountDownLatch sendEntered = new CountDownLatch(1); + private final CountDownLatch sendExited = new CountDownLatch(1); + private final AtomicReference sendThread = new AtomicReference<>(); + private final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final AtomicReference trafficCloseThread = new AtomicReference<>(); + private final CountDownLatch trafficClosed = new CountDownLatch(1); + + private BlockingSendClient(boolean trafficShutdownSupported) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.trafficShutdownSupported = trafficShutdownSupported; + } + + @Override + public void close() { + if (cleanupClaimed.compareAndSet(false, true)) { + cleanupCount.incrementAndGet(); + cleanupThread.set(Thread.currentThread()); + cleanupDone.countDown(); + } + super.close(); + } + + @Override + public void closeTraffic() { + if (!trafficShutdownSupported) { + throw new UnsupportedOperationException("custom transport has no safe cancellation capability"); + } + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficClosed.countDown(); + releaseSend.countDown(); + } + + @Override + public void sendBinary(long dataPtr, int length, int timeout) { + sendThread.set(Thread.currentThread()); + sendEntered.countDown(); + try { + if (!releaseSend.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("traffic path was not closed"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("I/O worker was interrupted instead of traffic being closed", e); + } finally { + sendExited.countDown(); + } + throw new LineSenderException("traffic path closed"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java new file mode 100644 index 00000000..e6c0e143 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -0,0 +1,247 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FlockReleaseRetryDriverTest { + + private final List sfDirs = new ArrayList<>(); + + @After + public void tearDown() { + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + for (String sfDir : sfDirs) { + removeDir(sfDir); + } + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresShareOneRetryThread() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int engineCount = 32; + AtomicInteger retryFailures = new AtomicInteger(); + AtomicInteger threadsCreated = new AtomicInteger(); + AtomicReference retryThreadRef = new AtomicReference<>(); + CountDownLatch retryFailuresObserved = new CountDownLatch(engineCount * 2); + CountDownLatch retryThreadStarted = new CountDownLatch(1); + CountDownLatch runRetryDriver = new CountDownLatch(1); + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(() -> { + retryFailures.incrementAndGet(); + retryFailuresObserved.countDown(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + threadsCreated.incrementAndGet(); + Thread thread = new Thread(() -> { + retryThreadStarted.countDown(); + try { + runRetryDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + task.run(); + }, "test-shared-flock-release-retry"); + retryThreadRef.set(thread); + return thread; + }); + + List engines = new ArrayList<>(); + List slotLocks = new ArrayList<>(); + List realFds = new ArrayList<>(); + boolean fdsRestored = false; + try { + for (int i = 0; i < engineCount; i++) { + String sfDir = newSfDir("persistent-" + i); + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + engines.add(engine); + slotLocks.add(slotLock); + realFds.add(realFd); + setFd(slotLock, 1_000_000_000); + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + if (i == 0) { + assertTrue("retry thread was not started", + retryThreadStarted.await(10, TimeUnit.SECONDS)); + } + } + + assertEquals("persistent failures must share one retry thread", + 1, threadsCreated.get()); + runRetryDriver.countDown(); + assertTrue("driver did not perform two failed rounds", + retryFailuresObserved.await(10, TimeUnit.SECONDS)); + assertTrue("driver did not retain persistent failures", + retryFailures.get() >= engineCount * 2); + for (CursorSendEngine engine : engines) { + assertFalse("failed releases must remain unpublished", + engine.isCloseCompleted()); + } + + restoreFds(slotLocks, realFds); + fdsRestored = true; + Thread retryThread = retryThreadRef.get(); + retryThread.join(10_000L); + assertFalse("shared retry thread retained lifecycle resources after drain", + retryThread.isAlive()); + for (CursorSendEngine engine : engines) { + assertTrue("driver must release every restored flock", + engine.isCloseCompleted()); + } + assertEquals("retries must not create another thread", + 1, threadsCreated.get()); + } finally { + if (!fdsRestored) { + restoreFds(slotLocks, realFds); + } + runRetryDriver.countDown(); + Thread retryThread = retryThreadRef.get(); + if (retryThread != null) { + retryThread.join(10_000L); + } + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + @Test(timeout = 30_000L) + public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("start-failure"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + try { + setFd(slotLock, 1_000_000_000); + engine.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // This also proves the failed driver cleared its queue: the + // setter rejects replacement while any engine remains queued. + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + engine.close(); + assertTrue("explicit close must recover after retry-thread start failure", + engine.isCloseCompleted()); + } finally { + if (!engine.isCloseCompleted()) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + }); + } + + private static int fd(SlotLock slotLock) throws Exception { + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + return fdField.getInt(slotLock); + } + + private static void removeDir(String sfDir) { + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + private static void restoreFds(List slotLocks, List realFds) throws Exception { + for (int i = 0; i < slotLocks.size(); i++) { + SlotLock slotLock = slotLocks.get(i); + synchronized (slotLock) { + setFd(slotLock, realFds.get(i)); + } + } + } + + private static void setFd(SlotLock slotLock, int fd) throws Exception { + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + fdField.setInt(slotLock, fd); + } + + private static SlotLock slotLock(CursorSendEngine engine) throws Exception { + Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); + slotLockField.setAccessible(true); + return (SlotLock) slotLockField.get(engine); + } + + private String newSfDir(String suffix) { + String sfDir = Paths.get( + System.getProperty("java.io.tmpdir"), + "qdb-flock-release-retry-" + suffix + "-" + System.nanoTime() + ).toString(); + sfDirs.add(sfDir); + return sfDir; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java new file mode 100644 index 00000000..0feffe8a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -0,0 +1,350 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SegmentManagerUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-segmgr-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test + public void testEnumerationFindNextFailureRefusesGenerationAllocation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/enumeration"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String lowerName = "sf-0000000000000000.sfa"; + String lowerPath = dir + "/" + lowerName; + String higherPath = dir + "/sf-0000000000000007.sfa"; + MmapSegment lower = MmapSegment.create(lowerPath, 0L, segmentSize); + lower.close(); + MmapSegment initial = MmapSegment.create(higherPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + byte[] originalHigher = java.nio.file.Files.readAllBytes(Paths.get(higherPath)); + FailingFilesFacade facade = new FailingFilesFacade(null, dir, lowerName); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 4, facade)) { + try { + manager.register(ring, dir); + Assert.fail("register accepted a partially enumerated SF directory"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("could not fully enumerate")); + } + Assert.assertTrue("fault did not occur after the lower generation was observed", + facade.partialLowerObserved); + Assert.assertTrue("failed enumeration cursor was not closed", facade.partialFindClosed); + Assert.assertEquals("enumeration failure must not allocate or truncate a path", + 0, facade.openCleanCalls); + Assert.assertTrue("higher generation disappeared", Files.exists(higherPath)); + Assert.assertArrayEquals("partial enumeration changed the unseen higher generation", + originalHigher, java.nio.file.Files.readAllBytes(Paths.get(higherPath))); + } finally { + ring.close(); + } + }); + } + + @Test(timeout = 15_000L) + public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/unlink"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String failedPath = dir + "/sf-0000000000000000.sfa"; + String activePath = dir + "/sf-0000000000000001.sfa"; + MmapSegment initial = MmapSegment.create(failedPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + AckWatermark watermark = AckWatermark.open(dir); + Assert.assertNotNull(watermark); + watermark.write(-1L); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + fill(payload, 32, (byte) 0x11); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 32)); + ring.installHotSpare(MmapSegment.create(activePath, 1L, segmentSize)); + Assert.assertEquals(1L, ring.appendOrFsn(payload, 32)); + ring.acknowledge(0L); + byte[] original = java.nio.file.Files.readAllBytes(Paths.get(failedPath)); + + FailingFilesFacade facade = new FailingFilesFacade(failedPath, null, null); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + manager.register(ring, dir, watermark); + manager.start(); + Assert.assertTrue("manager never attempted the injected unlink", + facade.removeAttempted.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("failed unlink must retain conservative registered bytes", + ring.totalSegmentBytes(), readTotalBytes(manager)); + } + + Assert.assertTrue("failed unlink path must remain observable", Files.exists(failedPath)); + Assert.assertTrue("failed unlink changed the acknowledged segment bytes", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + Assert.assertNotNull("failed unlink removed the segment from ring bookkeeping", + ring.firstSealed()); + Assert.assertEquals(failedPath, ring.firstSealed().path()); + Assert.assertEquals("watermark advanced although unlink did not commit", + -1L, watermark.read()); + + fill(payload, 32, (byte) 0x5A); + Assert.assertEquals("non-DEDUP successor must continue at the next FSN", + 2L, ring.appendOrFsn(payload, 32)); + String successorPath = ring.getActive().path(); + Assert.assertNotEquals("successor reused the acknowledged path", + failedPath, successorPath); + Assert.assertEquals(dir + "/sf-0000000000000002.sfa", successorPath); + Assert.assertTrue("successor segment was not created", Files.exists(successorPath)); + Assert.assertTrue("distinct non-DEDUP payload was not written to the successor", + containsRun(java.nio.file.Files.readAllBytes(Paths.get(successorPath)), + (byte) 0x5A, 32)); + Assert.assertTrue("successor write overwrote the failed-unlink segment", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + + try (SegmentManager retryManager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + retryManager.register(ring, dir, watermark); + retryManager.start(); + retryManager.wakeWorker(); + awaitRetryCommit(failedPath, watermark); + } + MmapSegment firstSealed = ring.firstSealed(); + Assert.assertTrue("successful retry retained the acknowledged segment", + firstSealed == null || !failedPath.equals(firstSealed.path())); + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + ring.close(); + watermark.close(); + } + }); + } + + private static void awaitRetryCommit(String failedPath, AckWatermark watermark) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (Files.exists(failedPath) || watermark.read() != 0L) { + if (System.nanoTime() > deadline) { + throw new AssertionError("unlink retry did not remove the file and advance the watermark"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static boolean containsRun(byte[] bytes, byte value, int length) { + int run = 0; + for (byte b : bytes) { + run = b == value ? run + 1 : 0; + if (run == length) { + return true; + } + } + return false; + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static long readTotalBytes(SegmentManager manager) throws Exception { + Field field = SegmentManager.class.getDeclaredField("totalBytes"); + field.setAccessible(true); + Field lockField = SegmentManager.class.getDeclaredField("lock"); + lockField.setAccessible(true); + Object lock = lockField.get(manager); + synchronized (lock) { + return field.getLong(manager); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static final class FailingFilesFacade implements FilesFacade { + private static final long PARTIAL_FIND_PTR = Long.MAX_VALUE; + private final String enumerationFailureDir; + private final String partialLowerName; + private final String unlinkFailurePath; + private final CountDownLatch removeAttempted = new CountDownLatch(1); + private int openCleanCalls; + private boolean partialFindClosed; + private long partialFindNamePtr; + private boolean partialLowerObserved; + private int unlinkFailuresRemaining = 1; + + private FailingFilesFacade( + String unlinkFailurePath, + String enumerationFailureDir, + String partialLowerName + ) { + this.unlinkFailurePath = unlinkFailurePath; + this.enumerationFailureDir = enumerationFailureDir; + this.partialLowerName = partialLowerName; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + INSTANCE.freeNativePath(partialFindNamePtr); + partialFindNamePtr = 0L; + partialFindClosed = true; + } else { + INSTANCE.findClose(findPtr); + } + } + @Override + public long findFirst(String dir) { + if (dir.equals(enumerationFailureDir)) { + partialFindNamePtr = INSTANCE.allocNativePath(partialLowerName); + return PARTIAL_FIND_PTR; + } + return INSTANCE.findFirst(dir); + } + @Override + public long findName(long findPtr) { + return findPtr == PARTIAL_FIND_PTR ? partialFindNamePtr : INSTANCE.findName(findPtr); + } + @Override + public int findNext(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + partialLowerObserved = true; + return -1; + } + return INSTANCE.findNext(findPtr); + } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int openCleanRW(String path) { + openCleanCalls++; + return INSTANCE.openCleanRW(path); + } + @Override + public int openCleanRW(long pathPtr) { + openCleanCalls++; + return INSTANCE.openCleanRW(pathPtr); + } + @Override + public int openRW(String path) { return INSTANCE.openRW(path); } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (path.equals(unlinkFailurePath) && unlinkFailuresRemaining > 0) { + unlinkFailuresRemaining--; + removeAttempted.countDown(); + return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..139a445f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -64,6 +64,8 @@ public class TestWebSocketServer implements Closeable { // client-side pool actually closed the connections it opened. private final AtomicInteger liveConnections = new AtomicInteger(); private final int port; + private final AtomicInteger roleRejectCount = new AtomicInteger(); + private final CountDownLatch roleRejectLatch = new CountDownLatch(1); private final AtomicBoolean running = new AtomicBoolean(false); private final ServerSocket serverSocket; private final CountDownLatch startLatch = new CountDownLatch(1); @@ -165,6 +167,10 @@ public TestWebSocketServer(WebSocketServerHandler handler, this.port = serverSocket.getLocalPort(); } + public boolean awaitRoleReject(long timeout, TimeUnit unit) throws InterruptedException { + return roleRejectLatch.await(timeout, unit); + } + public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedException { return startLatch.await(timeout, unit); } @@ -221,6 +227,13 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Number of HTTP 421 role-reject responses sent over the server's lifetime. + */ + public int roleRejectCount() { + return roleRejectCount.get(); + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -586,6 +599,8 @@ private boolean performHandshake() throws IOException { "\r\n"; out.write(sb.getBytes(StandardCharsets.US_ASCII)); out.flush(); + roleRejectCount.incrementAndGet(); + roleRejectLatch.countDown(); return false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 6db9e009..896dc69a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -777,6 +777,100 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { }); } + @Test(timeout = 30_000) + public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender lease = pool.borrow(); + Sender delegate = getDelegate(lease); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + Assert.assertTrue("precondition: live flock fd", realFd >= 0); + + CountDownLatch borrowerAcquired = new CountDownLatch(1); + CountDownLatch borrowerParked = new CountDownLatch(1); + CountDownLatch releaseBorrower = new CountDownLatch(1); + AtomicReference borrowerFailure = new AtomicReference<>(); + AtomicReference recovered = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + recovered.set(pool.borrow()); + borrowerAcquired.countDown(); + if (!releaseBorrower.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release borrower"); + } + } catch (Throwable t) { + borrowerFailure.compareAndSet(null, t); + } finally { + PooledSender sender = recovered.get(); + if (sender != null) { + sender.close(); + } + } + }, "sender-pool-deferred-release-waiter"); + + try { + synchronized (slotLock) { + fdField.setInt(slotLock, 1_000_000_000); + } + invokeDiscardBroken(pool, lease); + Assert.assertEquals("failed release must retire the only slot", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + pool.setBeforeBorrowWaitHook(borrowerParked::countDown); + borrower.start(); + Assert.assertTrue("borrower must reach the condition wait with a long timeout", + borrowerParked.await(5, TimeUnit.SECONDS)); + // The hook runs under the pool lock immediately before awaitNanos. + // Acquiring that lock here proves awaitNanos atomically enqueued the + // borrower and released the lock before restoration can start. This + // read-only operation neither changes pool state nor signals a waiter. + pool.availableSize(); + + // This restored fd is the only source of progress: the retry driver + // confirms the release. There is no housekeeper or pool mutation. + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + Assert.assertTrue("deferred flock release must wake the parked borrower", + borrowerAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertNull("borrower must not fail", borrowerFailure.get()); + Assert.assertEquals("release wakeup must recover retired capacity", + 0, pool.leakedSlotCount()); + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrower.countDown(); + if (!engine.isCloseCompleted()) { + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + } + borrower.join(TimeUnit.SECONDS.toMillis(1)); + if (borrower.isAlive()) { + borrower.interrupt(); + borrower.join(TimeUnit.SECONDS.toMillis(5)); + } + Assert.assertFalse("borrower thread must finish", borrower.isAlive()); + } + if (borrowerFailure.get() != null) { + throw new AssertionError("borrower failed", borrowerFailure.get()); + } + } + } + }); + } + @Test public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -2201,6 +2295,84 @@ public void testDefaultConfigRecoversOutOfRangeSlotsAfterShrink() throws Excepti }); } + @Test + public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exception { + // A deferred pool can already have its sole in-range sender borrowed when + // startup recovery reaches an out-of-range slot left by a larger pool. + // A transient recoverer build failure must leave that candidate pending: + // after the primary lease returns, the SAME pool must retry and drain it. + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-1;close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", 1L).atNow(); + seed.flush(); + } + } + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean primaryReturned = new AtomicBoolean(); + AtomicInteger recoveryAttempts = new AtomicInteger(); + IntFunction factory = idx -> { + if (idx == 1) { + recoveryAttempts.incrementAndGet(); + if (!primaryReturned.get()) { + throw new LineSenderException("transient out-of-range recovery failure"); + } + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + PooledSender primary = pool.borrow(); + Object primarySlot = slotOf(primary); + + Assert.assertFalse("first recovery attempt must stop at the transient failure", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("exactly one out-of-range recovery attempt", 1, + recoveryAttempts.get()); + Assert.assertTrue("failed recovery must preserve the candidate", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("out-of-range failure must not consume in-range capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range recoverer must not enter retired-slot bookkeeping", + ((List) getField(pool, "retiredSlots")).isEmpty()); + + primary.close(); + primaryReturned.set(true); + invokeRunStartupRecoveryOnce(pool); + + Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, + recoveryAttempts.get()); + Assert.assertFalse("retry must drain the preserved out-of-range data", + hasSegmentFile(slot("default-1"))); + Assert.assertTrue("retry must deliver the recovered frame", handler.frames.get() >= 1); + Assert.assertEquals("successful out-of-range retry must not consume capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range retry must leave retired slots untouched", + ((List) getField(pool, "retiredSlots")).isEmpty()); + + PooledSender next = pool.borrow(); + try { + Assert.assertSame("normal borrow must reuse the returned primary slot", + primarySlot, slotOf(next)); + } finally { + next.close(); + } + } + } + }); + } + @Test public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java new file mode 100644 index 00000000..47d168f9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -0,0 +1,463 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.network; + +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.KqueueFacade; +import io.questdb.client.network.KqueueFacadeImpl; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.NetworkFacadeImpl; +import io.questdb.client.network.PlainSocket; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.Os; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class SocketTrafficShutdownTest { + private static final NetworkFacade NF = NetworkFacadeImpl.INSTANCE; + + private static class CompatibilityNetworkFacade implements NetworkFacade { + private final AtomicInteger closeCount; + + private CompatibilityNetworkFacade(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public int close(int fd) { + closeCount.incrementAndGet(); + return 0; + } + + @Override + public void close(int fd, org.slf4j.Logger logger) { + close(fd); + } + + @Override + public void configureKeepAlive(int fd) { + } + + @Override + public int configureNonBlocking(int fd) { + return 0; + } + + @Override + public int connect(int fd, long pSockaddr) { + return 0; + } + + @Override + public int connectAddrInfo(int fd, long pAddrInfo) { + return 0; + } + + @Override + public int connectAddrInfoTimeout(int fd, long pAddrInfo, int timeoutMillis) { + return 0; + } + + @Override + public int errno() { + return 0; + } + + @Override + public void freeAddrInfo(long pAddrInfo) { + } + + @Override + public void freeSockAddr(long pSockaddr) { + } + + @Override + public long getAddrInfo(CharSequence host, int port) { + return 0; + } + + @Override + public int getSndBuf(int fd) { + return 0; + } + + @Override + public int recvRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendToRaw(int fd, long lo, int len, long socketAddress) { + return 0; + } + + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + return 0; + } + + @Override + public int setMulticastInterface(int fd, int ipv4Address) { + return 0; + } + + @Override + public int setMulticastTtl(int fd, int ttl) { + return 0; + } + + @Override + public boolean setSndBuf(int fd, int size) { + return false; + } + + @Override + public int setTcpNoDelay(int fd, boolean noDelay) { + return 0; + } + + @Override + public long sockaddr(int address, int port) { + return 0; + } + + @Override + public int socketTcp(boolean blocking) { + return 0; + } + + @Override + public int socketUdp() { + return 0; + } + + @Override + public boolean testConnection(int fd, long buffer, int bufferSize) { + return false; + } + } + + private static class CompatibilitySocket implements Socket { + private final AtomicInteger closeCount; + + private CompatibilitySocket(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + + @Override + public int getFd() { + return -1; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void of(int fd) { + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) throws TlsSessionInitFailedException { + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + } + + @Test + public void testCompatibilityDefaultsDoNotBypassCustomTransportOwnership() { + AtomicInteger facadeCloseCount = new AtomicInteger(); + NetworkFacade customFacade = new CompatibilityNetworkFacade(facadeCloseCount); + + PlainSocket plainSocket = new PlainSocket(customFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + plainSocket.of(42); + assertUnsupported(plainSocket::closeTraffic); + Assert.assertEquals("facade compatibility fallback must not release a synthetic descriptor", + 0, facadeCloseCount.get()); + Assert.assertEquals(42, plainSocket.getFd()); + plainSocket.close(); + Assert.assertEquals(1, facadeCloseCount.get()); + + AtomicInteger socketCloseCount = new AtomicInteger(); + Socket customSocket = new CompatibilitySocket(socketCloseCount); + assertUnsupported(customSocket::closeTraffic); + Assert.assertEquals("socket compatibility fallback must not run destructive close", + 0, socketCloseCount.get()); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); + } + + @Test + public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exception { + AtomicInteger closeCount = new AtomicInteger(); + AtomicInteger shutdownCount = new AtomicInteger(); + NetworkFacade facade = (NetworkFacade) Proxy.newProxyInstance( + NetworkFacade.class.getClassLoader(), + new Class[]{NetworkFacade.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCount.incrementAndGet(); + return method.getReturnType() == int.class ? 0 : null; + } + if ("shutdown".equals(method.getName())) { + shutdownCount.incrementAndGet(); + return 0; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == long.class) { + return 0L; + } + return null; + } + ); + Socket socket = JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + facade, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + ); + socket.of(42); + Field sslEngineField = socket.getClass().getDeclaredField("sslEngine"); + Field stateField = socket.getClass().getDeclaredField("state"); + sslEngineField.setAccessible(true); + stateField.setAccessible(true); + sslEngineField.set(socket, SSLContext.getDefault().createSSLEngine()); + stateField.setInt(socket, 2); // JavaTlsClientSocket.STATE_TLS + Object[] tlsState = snapshotTlsState(socket); + + try { + socket.closeTraffic(); + + Object[] stateAfterTrafficClose = snapshotTlsState(socket); + for (int i = 0; i < tlsState.length; i++) { + if (i == 2) { + Assert.assertEquals("traffic cancellation must preserve TLS state", tlsState[i], stateAfterTrafficClose[i]); + } else { + Assert.assertSame("traffic cancellation must preserve SSLEngine/buffer references", + tlsState[i], stateAfterTrafficClose[i]); + } + } + Assert.assertEquals(1, shutdownCount.get()); + Assert.assertEquals("traffic cancellation must not release the delegate fd", 0, closeCount.get()); + Assert.assertEquals(42, socket.getFd()); + Assert.assertFalse(socket.isClosed()); + } finally { + // Restore a valid plaintext state so full close does not attempt a + // synthetic TLS close_notify with uninitialised session buffers. + sslEngineField.set(socket, null); + stateField.setInt(socket, 1); // JavaTlsClientSocket.STATE_PLAINTEXT + socket.close(); + } + Assert.assertEquals(1, closeCount.get()); + Assert.assertTrue(socket.isClosed()); + } + + @Test(timeout = 30_000L) + public void testTlsSocketTrafficGateUsesDelegateShutdownAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + NF, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + )); + } + + private static void assertShutdownWakesMacOsKqueue(Socket socket) throws Exception { + Assume.assumeTrue("real kqueue cancellation coverage runs on macOS", Os.type == Os.DARWIN); + + AtomicBoolean pollDone = new AtomicBoolean(); + AtomicInteger pollResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference pollFailure = new AtomicReference<>(); + CountDownLatch pollEntered = new CountDownLatch(1); + KqueueFacade facade = new KqueueFacade() { + private final KqueueFacade delegate = KqueueFacadeImpl.INSTANCE; + + @Override + public NetworkFacade getNetworkFacade() { + return delegate.getNetworkFacade(); + } + + @Override + public int kevent(int kq, long changeList, int nChanges, long eventList, int nEvents, int timeout) { + if (eventList != 0 && nEvents > 0) { + pollEntered.countDown(); + } + return delegate.kevent(kq, changeList, nChanges, eventList, nEvents, timeout); + } + + @Override + public int kqueue() { + return delegate.kqueue(); + } + }; + + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfoTimeout(fd, addrInfo, 5_000)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept(); Kqueue kqueue = new Kqueue(facade, 1)) { + try { + Assert.assertEquals(0, NF.configureNonBlocking(fd)); + socket.of(fd); + int retainedFd = fd; + fd = -1; + + kqueue.setWriteOffset(0); + kqueue.readFD(retainedFd, 0); + Assert.assertEquals(0, kqueue.register(1)); + + waiter = new Thread(() -> { + try { + pollResult.set(kqueue.poll(10_000)); + } catch (Throwable t) { + pollFailure.set(t); + } finally { + pollDone.set(true); + } + }, "socket-traffic-kqueue-waiter"); + waiter.start(); + + Assert.assertTrue("waiter did not enter the native kqueue wait", + pollEntered.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the read wait ready", pollDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native kqueue wait", waiter.isAlive()); + Assert.assertNull("native kqueue wait failed", pollFailure.get()); + Assert.assertTrue("shutdown must produce a readiness event", pollResult.get() > 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + } finally { + socket.closeTraffic(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + socket.close(); + } + } + } finally { + if (fd != -1) { + NF.close(fd); + } + } + } + + private static void assertUnsupported(Runnable operation) { + try { + operation.run(); + Assert.fail("expected unsupported traffic shutdown"); + } catch (UnsupportedOperationException expected) { + // Expected compatibility behavior: no native or destructive fallback. + } + } + + private static Object[] snapshotTlsState(Socket socket) throws Exception { + String[] fieldNames = { + "callerOutputBuffer", + "sslEngine", + "state", + "unwrapInputBuffer", + "unwrapOutputBuffer", + "wrapInputBuffer", + "wrapOutputBuffer" + }; + Object[] state = new Object[fieldNames.length]; + for (int i = 0; i < fieldNames.length; i++) { + Field field = socket.getClass().getDeclaredField(fieldNames[i]); + field.setAccessible(true); + state[i] = field.get(socket); + } + return state; + } +} From cad1bb8d27565381bb8965630b46beb74f2a8570 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 12 Jul 2026 00:34:05 +0100 Subject: [PATCH 16/64] fix(qwp): make close-time SF cleanup crash-safe and bound flock-release retries Close-time segment cleanup (review finding: failed cleanup published as reusable slot): - finishClose now persists the final acked FSN through the still-mapped watermark before closing the ring and before any unlink, so any cleanup failure or crash leaves the residue covered by a current watermark - unlinkAllSegmentFiles enumerates fully first, aborts with no unlinks on a failed directory walk, removes segments in ascending generation order and stops at the first failure, so residue is always a contiguous top slice that recovery seeds as fully acked (no replay, no duplicates) - the ack watermark is removed only after every segment is confirmed gone Flock-release retry driver (review finding: unbounded retry threads): - the shared retry driver now backs off exponentially per fully-failed round (100ms base, 5s cap), resets on progress, and is unparked when a fresh engine enqueues - after an injected driver-start failure, pool probes re-arm the retry via ensureFlockReleaseRetryScheduled() from isSlotLockReleased(), so retained capacity recovers without a second explicit close() New regression tests: CursorSendEngineCloseUnlinkFailureTest (close-time unlink fault injection; successor must not see replayable frames) and three FlockReleaseRetryDriverTest cases (backoff schedule, reset-on-progress, pool-probe recovery after start failure). --- .../qwp/client/QwpWebSocketSender.java | 24 +- .../client/sf/cursor/CursorSendEngine.java | 192 +++++++++++-- .../io/questdb/client/impl/SenderPool.java | 7 +- ...ursorSendEngineCloseUnlinkFailureTest.java | 209 ++++++++++++++ .../cursor/FlockReleaseRetryDriverTest.java | 261 ++++++++++++++++++ 5 files changed, 661 insertions(+), 32 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 9f454c9c..403748bf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1357,19 +1357,29 @@ public void close() { * manager-worker quiescence or I/O-thread exit path, this re-probes the * retained engine and latches true the moment that cleanup completes — pools re-probe retired * slots through this getter to recover their capacity. Monotonic: - * false→true only, never back. Lock-free (volatile reads) so pools may - * call it under their capacity lock. + * false→true only, never back. Cheap (volatile reads on every common + * path) so pools may call it under their capacity lock; only the rare + * orphaned-retry state below does more. + *

    + * The probe is also the recovery surface for a retained engine whose + * flock-release retry fell off the shared driver because the driver + * thread could not start (e.g. OOM at thread creation): close() is + * one-shot, so without the re-arm below that slot's capacity would stay + * lost until process exit. */ public boolean isSlotLockReleased() { if (slotLockReleased) { return true; } CursorSendEngine engine = retainedEngine; - if (engine != null && engine.isCloseCompleted()) { - // Benign latch race: concurrent callers may both observe the - // completed cleanup and both write true. - slotLockReleased = true; - return true; + if (engine != null) { + if (engine.isCloseCompleted()) { + // Benign latch race: concurrent callers may both observe the + // completed cleanup and both write true. + slotLockReleased = true; + return true; + } + engine.ensureFlockReleaseRetryScheduled(); } return false; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 95addd39..01910fcb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -31,9 +31,11 @@ import org.jetbrains.annotations.TestOnly; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongConsumer; /** * Facade that bundles a {@link SegmentRing} with a {@link SegmentManager} and @@ -69,10 +71,13 @@ public final class CursorSendEngine implements QuietCloseable { org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class); private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY = runnable -> new Thread(runnable, "qdb-sf-flock-release-retry"); + private static final long FLOCK_RELEASE_RETRY_BASE_PARK_NANOS = 100_000_000L; // 100 ms private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object(); + private static final long FLOCK_RELEASE_RETRY_MAX_PARK_NANOS = 5_000_000_000L; // 5 s private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>(); private static volatile Runnable afterFlockReleaseRetryFailureHook; private static volatile Runnable beforeDeferredCloseCreationHook; + private static volatile LongConsumer flockReleaseRetryParkOverride; private static Thread flockReleaseRetryThread; private static volatile ThreadFactory flockReleaseRetryThreadFactory = DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY; @@ -742,6 +747,27 @@ public synchronized void close() { */ private void finishClose(boolean fullyDrained) { try { + // On a fully-drained close, persist the final acked FSN through + // the still-mapped watermark BEFORE closing the ring/watermark + // and BEFORE unlinking any segment file. The manager persists + // the watermark only on its own tick, so it may lag the final + // ack. If the unlink below then fails (or the process dies + // mid-unlink), residual acknowledged .sfa files without an + // up-to-date watermark would seed the successor's recovery at + // lowestBase - 1 and replay already-acknowledged rows -- + // duplicates on a non-DEDUP table. The write is a single mmap + // store, so it succeeds even when the unlink is about to fail + // (e.g. the slot dir turned read-only). Quiescence is already + // established here, so no manager tick can race this write. + if (fullyDrained && watermark != null) { + try { + long finalAckedFsn = ring.ackedFsn(); + if (finalAckedFsn >= 0) { + watermark.write(finalAckedFsn); + } + } catch (Throwable ignored) { + } + } try { ring.close(); } catch (Throwable ignored) { @@ -761,13 +787,26 @@ private void finishClose(boolean fullyDrained) { } } if (fullyDrained) { + boolean segmentsRemoved = false; try { - unlinkAllSegmentFiles(sfDir); + segmentsRemoved = unlinkAllSegmentFiles(sfDir); } catch (Throwable ignored) { } - try { - AckWatermark.removeOrphan(sfDir); - } catch (Throwable ignored) { + // Remove the watermark ONLY once every segment file is + // confirmed gone. The watermark is what keeps residual + // acknowledged segments inert to a successor's recovery; + // removing it while any .sfa file survives would republish + // those already-acknowledged rows. + if (segmentsRemoved) { + try { + AckWatermark.removeOrphan(sfDir); + } catch (Throwable ignored) { + } + } else { + LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack " + + "watermark so residual acknowledged segments stay covered -- the next " + + "engine on this slot recovers them as fully acked and retries the " + + "unlink on its own close", sfDir); } } } finally { @@ -813,6 +852,17 @@ public static void setBeforeDeferredCloseCreationHook(Runnable hook) { beforeDeferredCloseCreationHook = hook; } + /** + * Replaces the shared retry driver's inter-round park with a callback + * receiving the park duration the driver would have used. Test-only: + * makes the retry cadence inspectable and rounds coordinatable without + * wall-clock waits. + */ + @TestOnly + public static void setFlockReleaseRetryParkOverride(LongConsumer override) { + flockReleaseRetryParkOverride = override; + } + /** * Overrides creation of the single shared flock-release retry thread. * Test-only: makes thread creation/start failure and persistent retry @@ -903,6 +953,11 @@ private boolean retryFlockReleaseIfReady() { } private static void runFlockReleaseRetryDriver() { + // Capped exponential backoff: a persistent unlock failure must not + // burn a fixed 10 rounds of native release syscalls per second + // forever, but a transient failure must still recover promptly. The + // ramp doubles per fully-failed round from 100 ms up to 5 s. + long parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; while (true) { final int roundSize; synchronized (FLOCK_RELEASE_RETRY_LOCK) { @@ -913,6 +968,7 @@ private static void runFlockReleaseRetryDriver() { } } boolean hasFailures = false; + boolean hasSuccesses = false; for (int i = 0; i < roundSize; i++) { final CursorSendEngine engine; synchronized (FLOCK_RELEASE_RETRY_LOCK) { @@ -920,6 +976,7 @@ private static void runFlockReleaseRetryDriver() { } if (engine.retryFlockReleaseIfReady()) { engine.flockReleaseRetryStarted.set(false); + hasSuccesses = true; } else { synchronized (FLOCK_RELEASE_RETRY_LOCK) { FLOCK_RELEASE_RETRY_QUEUE.addLast(engine); @@ -931,11 +988,22 @@ private static void runFlockReleaseRetryDriver() { } } } + if (hasSuccesses) { + // Progress: the failure condition is clearing, so retry the + // remaining engines on the base cadence again. + parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; + } if (hasFailures) { // Interruption must not abandon a retained flock, but clear // the flag so subsequent parks still throttle retries. Thread.interrupted(); - LockSupport.parkNanos(100_000_000L); + LongConsumer parkOverride = flockReleaseRetryParkOverride; + if (parkOverride != null) { + parkOverride.accept(parkNanos); + } else { + LockSupport.parkNanos(parkNanos); + } + parkNanos = Math.min(parkNanos * 2, FLOCK_RELEASE_RETRY_MAX_PARK_NANOS); } } } @@ -947,7 +1015,12 @@ private void startFlockReleaseRetry() { Throwable startFailure = null; synchronized (FLOCK_RELEASE_RETRY_LOCK) { FLOCK_RELEASE_RETRY_QUEUE.addLast(this); - if (flockReleaseRetryThread == null) { + if (flockReleaseRetryThread != null) { + // The driver may be parked on a ramped backoff; wake it so a + // freshly failed release gets its first driver retry promptly + // instead of inheriting older engines' full backoff. + LockSupport.unpark(flockReleaseRetryThread); + } else { try { Thread retryThread = flockReleaseRetryThreadFactory.newThread( CursorSendEngine::runFlockReleaseRetryDriver); @@ -973,11 +1046,13 @@ private void startFlockReleaseRetry() { + "retired capacity recovers after the transient failure [slot={}]", sfDir == null ? "" : sfDir); } else { - // A later explicit close() can still retry without repeating the - // one-time ring/watermark cleanup. The failed queue is cleared so - // the driver does not retain engines it cannot service. - LOG.error("Could not start SF flock-release retry driver; close() may be " - + "invoked again to retry [slot={}, error={}]", + // A later explicit close() or a pool's retired-slot probe + // (ensureFlockReleaseRetryScheduled) can still retry without + // repeating the one-time ring/watermark cleanup. The failed queue + // is cleared so the driver does not retain engines it cannot + // service. + LOG.error("Could not start SF flock-release retry driver; a retried close() " + + "or pool re-probe re-arms the retry [slot={}, error={}]", sfDir == null ? "" : sfDir, String.valueOf(startFailure)); } } @@ -1015,6 +1090,25 @@ public void setSlotLockReleaseListener(Runnable listener) { } } + /** + * Re-arms the shared flock-release retry for an engine whose terminal + * cleanup finished but whose confirmed flock release is still pending + * and no longer scheduled — the retry driver thread failed to start when + * the release first failed (e.g. OOM at thread creation). + * {@code Sender.close()} is one-shot by contract, so pool probes + * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a + * retired slot's capacity recoverable instead of lost until process + * exit. Cheap unless the engine is in that orphan state (volatile reads, + * then one failed CAS when the retry is already scheduled), so probes + * may call it under their capacity lock. + */ + public void ensureFlockReleaseRetryScheduled() { + if (closeCompleted || !terminalResourcesCleaned) { + return; + } + startFlockReleaseRetry(); + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ @@ -1112,37 +1206,91 @@ public long recoveredOrphanTipFsn() { return recoveredOrphanTipFsn; } + /** + * Ascending removal rank of a segment file name for + * {@link #unlinkAllSegmentFiles(String)}. {@code sf-initial.sfa} is + * always the fresh-start segment at baseSeq 0, so it ranks first. + * Spare files carry a monotonic generation ({@code sf-.sfa}) + * assigned in creation == rotation == baseSeq order, so the parsed + * generation ranks them. Anything else with the extension is not a + * live segment and ranks last. + */ + private static long segmentCleanupRank(String name) { + if ("sf-initial.sfa".equals(name)) { + return Long.MIN_VALUE; + } + if (name.length() == 23 && name.startsWith("sf-")) { + try { + return Long.parseUnsignedLong(name.substring(3, 19), 16); + } catch (NumberFormatException ignored) { + // fall through to the unrecognized-name rank + } + } + return Long.MAX_VALUE; + } + /** * Unlinks every {@code .sfa} file under {@code dir}. Called only on * clean shutdown when the ring confirms every published FSN has been * acked — at that moment the slot has no recoverable work and the * files are pure noise that would mislead the next sender's recovery. - * Best-effort: logs and continues on failures, since we're already on - * the close path. + *

    + * Removal runs in ascending segment order and STOPS at the first + * failure, so whatever survives (a failure here, or a crash mid-loop) + * is always a contiguous top slice of the ring: recovery's + * FSN-contiguity check still passes, and the retained ack watermark + * (== the final acked FSN == the highest frame on disk) still covers + * every surviving frame, so a successor replays nothing. + * + * @return {@code true} only when enumeration succeeded and every + * {@code .sfa} file was confirmed removed — the caller keeps the ack + * watermark on {@code false} so residual acknowledged segments stay + * covered. */ - private static void unlinkAllSegmentFiles(String dir) { - if (!io.questdb.client.std.Files.exists(dir)) return; + private static boolean unlinkAllSegmentFiles(String dir) { + if (!io.questdb.client.std.Files.exists(dir)) return true; long find = io.questdb.client.std.Files.findFirst(dir); if (find < 0) { LOG.warn("close-time unlink could not enumerate {}; " + "any residual sf-*.sfa files will be picked up by the next recovery", dir); - return; + return false; } - if (find == 0) return; + if (find == 0) return true; + ArrayList names = new ArrayList<>(); + int rc = 1; try { - int rc = 1; while (rc > 0) { String name = io.questdb.client.std.Files.utf8ToString( io.questdb.client.std.Files.findName(find)); rc = io.questdb.client.std.Files.findNext(find); - if (name == null || !name.endsWith(".sfa")) continue; - String path = dir + "/" + name; - if (!io.questdb.client.std.Files.remove(path)) { - LOG.warn("Failed to unlink fully-acked segment {} on close", path); + if (name != null && name.endsWith(".sfa")) { + names.add(name); } } } finally { io.questdb.client.std.Files.findClose(find); } + if (rc < 0) { + // A partial listing must not drive any unlink: removing only the + // files we happened to see could delete the segment holding the + // highest frame while a lower one survives, leaving residual + // state the retained watermark can no longer vouch for. + LOG.warn("close-time unlink could not fully enumerate {}; " + + "leaving all segment files for the next recovery", dir); + return false; + } + names.sort((a, b) -> { + int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b)); + return byRank != 0 ? byRank : a.compareTo(b); + }); + for (int i = 0, n = names.size(); i < n; i++) { + String path = dir + "/" + names.get(i); + if (!io.questdb.client.std.Files.remove(path)) { + LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + + "residual files stay a contiguous, watermark-covered range", path); + return false; + } + } + return true; } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 940e6049..e401f864 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1428,9 +1428,10 @@ private void recoverReleasedSlots() { * run since the retire. Restores {@code leakedSlots} capacity and signals * waiters so a parked borrow can admit a creation immediately. *

    - * Caller must hold {@code lock}. The probe is lock-free on the delegate - * side ({@code isSlotLockReleased()} reads volatiles only), so holding - * the pool lock across it cannot stall behind delegate teardown. + * Caller must hold {@code lock}. The probe is cheap on the delegate side + * ({@code isSlotLockReleased()} reads volatiles, and only re-arms the + * shared flock-release retry in the rare orphaned-retry state), so + * holding the pool lock across it cannot stall behind delegate teardown. * * @return {@code true} if at least one slot's capacity was recovered */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java new file mode 100644 index 00000000..a9df2b67 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java @@ -0,0 +1,209 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Regression for the close-time segment cleanup on a fully-drained slot. + *

    + * {@code CursorSendEngine.finishClose} unlinks the acknowledged {@code .sfa} + * files and then removes the ack watermark. When the unlink fails (transient + * I/O error, permission problem), the residual segment files hold rows the + * server already acknowledged. If the watermark does not cover them, a + * successor engine on the same slot seeds recovery from {@code lowestBase - 1} + * and replays every acknowledged row — duplicates on a non-DEDUP table. + *

    + * The test injects the unlink failure by dropping write permission on the + * slot directory (POSIX: unlink requires a writable parent directory), so it + * is skipped on Windows and when permissions are not enforced (root). + * The shared {@link SegmentManager} is deliberately never started: no worker + * thread exists, so no manager tick can persist the watermark behind the + * test's back, and the close-path quiescence barrier is trivially satisfied — + * fully deterministic, no timing coordination needed. + */ +public class CursorSendEngineCloseUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test(timeout = 20_000L) + public void testFailedCloseTimeUnlinkMustNotExposeAckedFramesToSuccessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String slot = tmpDir + "/slot"; + Path slotPath = Paths.get(slot); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60)); + CursorSendEngine pred = null; + CursorSendEngine succ = null; + boolean slotDirReadOnly = false; + try { + fill(payload, 32, (byte) 0x33); + pred = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals(0L, pred.appendBlocking(payload, 32)); + Assert.assertEquals(0L, pred.publishedFsn()); + // The server durably acknowledged FSN 0 in this session. + Assert.assertTrue(pred.acknowledge(0L)); + Assert.assertEquals(0L, pred.ackedFsn()); + + // Inject a close-time unlink failure: drop write permission on + // the slot dir. Prove the injection works with a probe file -- + // root (and some filesystems) ignore directory permissions. + String probePath = slot + "/probe"; + Assert.assertTrue(java.nio.file.Files.exists( + java.nio.file.Files.createFile(Paths.get(probePath)))); + try { + setPermissions(slotPath, "r-xr-xr-x"); + } catch (UnsupportedOperationException e) { + Assume.assumeNoException("POSIX permissions unavailable on this platform", e); + } + slotDirReadOnly = true; + boolean probeRemoved = Files.remove(probePath); + if (probeRemoved) { + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + } + Assume.assumeFalse("directory permissions not enforced (running as root?)", + probeRemoved); + + // Fully-drained close: tries to unlink the acknowledged + // segment files and fails. + pred.close(); + Assert.assertTrue("flock release needs no dir write; close must complete", + pred.isCloseCompleted()); + pred = null; + Assert.assertTrue("injected unlink failure must leave the acknowledged segment", + Files.exists(slot + "/sf-initial.sfa")); + + // The transient failure clears before the successor arrives. + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + Files.remove(probePath); + + succ = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(succ.wasRecoveredFromDisk()); + Assert.assertEquals(0L, succ.publishedFsn()); + // THE regression: FSN 0 was acknowledged by the server during + // the predecessor's session. The successor must not see it as + // replayable, or a non-DEDUP table receives duplicate rows. + Assert.assertTrue("successor exposes already-acknowledged frames for replay " + + "[ackedFsn=" + succ.ackedFsn() + + ", publishedFsn=" + succ.publishedFsn() + "]", + succ.ackedFsn() >= succ.publishedFsn()); + + // The successor's own fully-drained close retries the cleanup + // now that the failure has cleared: segments and watermark gone. + succ.close(); + Assert.assertTrue(succ.isCloseCompleted()); + succ = null; + Assert.assertFalse("successor close did not retry the segment unlink", + Files.exists(slot + "/sf-initial.sfa")); + Assert.assertFalse("watermark must be removed once no segment file remains", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } finally { + if (slotDirReadOnly) { + try { + setPermissions(slotPath, "rwxr-xr-x"); + } catch (Throwable ignored) { + } + } + if (pred != null) { + pred.close(); + } + if (succ != null) { + succ.close(); + } + manager.close(); + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static void setPermissions(Path path, String posix) throws Exception { + Set perms = PosixFilePermissions.fromString(posix); + java.nio.file.Files.setPosixFilePermissions(path, perms); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java index e6c0e143..0037437f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; @@ -36,6 +37,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -52,12 +54,98 @@ public class FlockReleaseRetryDriverTest { @After public void tearDown() { CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + CursorSendEngine.setFlockReleaseRetryParkOverride(null); CursorSendEngine.setFlockReleaseRetryThreadFactory(null); for (String sfDir : sfDirs) { removeDir(sfDir); } } + /** + * Driver-start failure must not strand retired capacity until process + * exit. {@code Sender.close()} is a one-shot no-op by contract, so the + * only recovery surface a pool has is its retired-slot probe + * ({@code isSlotLockReleased()}, called from the housekeeper tick and + * capacity-starved borrows) — that probe must re-arm the shared retry + * driver once thread creation works again. + */ + @Test(timeout = 30_000L) + public void testDriverStartFailureRecoversViaPoolProbe() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("probe-rearm"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(engine, true); + AtomicReference rearmedDriver = new AtomicReference<>(); + boolean recovered = false; + try { + setFd(slotLock, 1_000_000_000); + sender.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // Sender.close() is idempotent: repeat calls never reach the + // engine again, so they cannot restart the failed driver. + sender.close(); + assertEquals("no-op close must not retry the driver start", 1, starts.get()); + + // While the fault persists, each pool probe re-attempts the + // re-arm (and fails again) without publishing a release. + assertFalse("failed unlock must keep the slot reported as held", + sender.isSlotLockReleased()); + assertTrue("pool probe must re-attempt the driver start", + starts.get() >= 2); + + // The transient condition clears: thread creation works again + // (the failed start drained the queue, so the factory swap is + // legal) and the flock fd is back. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-rearmed-flock-release-retry"); + rearmedDriver.set(thread); + return thread; + }); + setFd(slotLock, realFd); + CountDownLatch released = new CountDownLatch(1); + engine.setSlotLockReleaseListener(released::countDown); + + // Production recovery surface: the pool re-probes the retired + // slot through isSlotLockReleased(). + sender.isSlotLockReleased(); + assertTrue("pool probe must re-arm the flock-release retry after " + + "a driver start failure", + released.await(10, TimeUnit.SECONDS)); + assertTrue(engine.isCloseCompleted()); + assertTrue("probe must expose the recovered release", + sender.isSlotLockReleased()); + recovered = true; + } finally { + Thread driver = rearmedDriver.get(); + if (driver != null) { + driver.join(10_000L); + } + if (!recovered) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + @Test(timeout = 30_000L) public void testPersistentFailuresShareOneRetryThread() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -149,6 +237,179 @@ public void testPersistentFailuresShareOneRetryThread() throws Exception { }); } + /** + * Schedule inspection for the shared driver's retry cadence: the + * inter-round park must grow exponentially from 100 ms and cap at 5 s, + * so a persistent unlock failure does not burn 10 syscalls per second + * per engine forever. The park override replaces the real park, so the + * test coordinates rounds without wall-clock waits. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffDoublesToCap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-backoff-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("backoff-cap"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + boolean fdRestored = false; + try { + setFd(slotLock, 1_000_000_000); + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + + // Eight failed rounds: enough to observe the full ramp and + // two capped parks. + for (int round = 1; round <= 8; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 8) { + proceed.release(); + } + } + + setFd(slotLock, realFd); + fdRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after the release succeeded", + driver.isAlive()); + assertTrue("restored flock must be released", engine.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(800_000_000L); + expected.add(1_600_000_000L); + expected.add(3_200_000_000L); + expected.add(5_000_000_000L); + expected.add(5_000_000_000L); + assertEquals("retry parks must double from 100ms and cap at 5s", + expected, parks); + } finally { + if (!fdRestored) { + setFd(slotLock, realFd); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + /** + * A successful release in a round is progress: the driver must reset its + * backoff to the base so the remaining engines are retried promptly + * while the failure condition is clearing. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffResetsOnProgress() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-reset-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engineA = new CursorSendEngine( + newSfDir("backoff-reset-a"), 4L * 1024 * 1024); + CursorSendEngine engineB = new CursorSendEngine( + newSfDir("backoff-reset-b"), 4L * 1024 * 1024); + SlotLock slotLockA = slotLock(engineA); + SlotLock slotLockB = slotLock(engineB); + int realFdA = fd(slotLockA); + int realFdB = fd(slotLockB); + boolean fdARestored = false; + boolean fdBRestored = false; + try { + setFd(slotLockA, 1_000_000_000); + setFd(slotLockB, 1_000_000_001); + engineA.close(); + engineB.close(); + + // Three failed rounds ramp the backoff to 400ms. + for (int round = 1; round <= 3; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 3) { + proceed.release(); + } + } + + // Engine A recovers; round 4 has one success and one failure, + // so its park must be back at the 100ms base. + setFd(slotLockA, realFdA); + fdARestored = true; + proceed.release(); + assertTrue("driver did not park after the mixed round", + parked.tryAcquire(10, TimeUnit.SECONDS)); + assertTrue("recovered engine must publish completion", + engineA.isCloseCompleted()); + + setFd(slotLockB, realFdB); + fdBRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after both releases succeeded", + driver.isAlive()); + assertTrue(engineB.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(100_000_000L); + assertEquals("a successful release must reset the backoff to base", + expected, parks); + } finally { + if (!fdARestored) { + setFd(slotLockA, realFdA); + } + if (!fdBRestored) { + setFd(slotLockB, realFdB); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + @Test(timeout = 30_000L) public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception { TestUtils.assertMemoryLeak(() -> { From a531164472c722c2ca5a771161eb0ae92fc8c039 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 12 Jul 2026 20:32:05 +0100 Subject: [PATCH 17/64] fix(qwp): harden SF crash recovery and cleanup Make acknowledged segment deletion crash-consistent, bound retry and cleanup paths, improve pool recovery complexity, and add deterministic lifecycle/native regression coverage. --- core/src/main/c/share/files.c | 15 + core/src/main/c/windows/files.c | 19 + .../qwp/client/QwpWebSocketSender.java | 263 ++++----- .../qwp/client/sf/cursor/AckWatermark.java | 70 ++- .../client/sf/cursor/CursorSendEngine.java | 59 +- .../sf/cursor/CursorWebSocketSendLoop.java | 63 +-- .../qwp/client/sf/cursor/MmapSegment.java | 19 + .../qwp/client/sf/cursor/SegmentManager.java | 297 ++++++++-- .../qwp/client/sf/cursor/SegmentRing.java | 157 ++++-- .../sf/cursor/SenderConnectionDispatcher.java | 6 + .../sf/cursor/SenderErrorDispatcher.java | 6 + .../sf/cursor/SenderProgressDispatcher.java | 6 + .../io/questdb/client/impl/SenderPool.java | 166 +++++- .../io/questdb/client/impl/SenderSlot.java | 13 + .../client/std/DefaultFilesFacade.java | 10 + .../java/io/questdb/client/std/Files.java | 16 + .../io/questdb/client/std/FilesFacade.java | 8 + .../java/io/questdb/client/std/ObjList.java | 2 +- .../client/SlotLockReleasedContractTest.java | 129 ++++- .../CursorSendEngineCrashConsistencyTest.java | 310 ++++++++++ ...CursorSendEngineSlotReacquisitionTest.java | 32 +- .../SegmentManagerCrashConsistencyTest.java | 529 ++++++++++++++++++ .../client/sf/cursor/SegmentManagerTest.java | 76 ++- .../SegmentManagerTrimDeregisterRaceTest.java | 8 +- .../SegmentManagerUnlinkFailureTest.java | 4 +- .../qwp/client/sf/cursor/SegmentRingTest.java | 140 ++++- .../test/impl/SenderPoolErrorSafetyTest.java | 107 ++++ .../client/test/impl/SenderPoolSfTest.java | 191 +++++++ .../network/SocketTrafficShutdownTest.java | 86 +++ .../questdb/client/test/std/ObjListTest.java | 17 + 30 files changed, 2464 insertions(+), 360 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 209c4490..fefd24fb 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -124,6 +124,21 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_RDONLY), fd); + if (fd < 0) { + return -1; + } + int res; + RESTARTABLE(fsync(fd), res); + int saved_errno = errno; + close(fd); + errno = saved_errno; + return res; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { int res; diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index d1d3b413..620bd81e 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -215,6 +215,25 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + jint fd = open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + if (fd < 0) { + return -1; + } + if (!FlushFileBuffers(FD_TO_HANDLE(fd))) { + SaveLastError(); + CloseHandle(FD_TO_HANDLE(fd)); + return -1; + } + CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { FILE_END_OF_FILE_INFO eof; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 403748bf..8300cedc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -195,6 +195,8 @@ public class QwpWebSocketSender implements Sender { // close() drain timeout in millis. Default applied at construction. // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. + private volatile boolean closeCleanupComplete; + private boolean closeCleanupStarted; private long closeFlushTimeoutMillis = 5_000L; private volatile boolean closed; // Test-only lifecycle witness. close() invokes and clears it strictly after @@ -1206,136 +1208,26 @@ public void close() { } if (!ioThreadStopped) { - // The I/O thread may still be using the socket and microbatch - // buffers. Freeing them would risk SIGSEGV. - LOG.error("I/O thread is still running, leaking WebSocket client and microbatch buffers"); - // The engine, however, need not leak: delegate its close to - // the I/O thread's exit path, which runs it strictly after - // the thread's last engine access — the mapping and slot - // lock release as soon as the stuck wire call resolves - // (bounded by OS timeouts). slotLockReleased intentionally - // stays false: the lock is released only when the delegated - // close actually runs, so the pool must not reuse the slot - // meanwhile. A false return means the thread exited between - // the failed close() and now — then closing here is safe. - if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null) { - if (cursorSendLoop.delegateEngineClose()) { - // The I/O thread adopted the engine close and runs it on - // its exit path. Keep the engine visible for re-probing: - // isSlotLockReleased() flips true once that close - // completes, letting a pool recover the retired slot. - retainedEngine = cursorEngine; - } else { - CursorSendEngine engine = cursorEngine; - try { - engine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - if (engine.isCloseCompleted()) { - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; - } else { - // Engine cleanup is pending on its manager worker's - // exit path (or leaked, for a shared manager). Report - // the retained flock now; the getter re-probes the - // retained engine so a late release is not lost. - slotLockReleased = false; - retainedEngine = engine; - } - } + // The worker may still touch every resource below. Hand the + // complete sender-owned tail to its exit path rather than + // permanently leaking everything except the engine. The + // callback is idempotence-gated by closeRemainingResources(). + if (ownsCursorEngine && cursorEngine != null) { + retainedEngine = cursorEngine; } - rethrowTerminal(terminalError); - return; - } - - if (buffer0 != null) { - try { - buffer0.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer0: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (buffer1 != null) { - try { - buffer1.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer1: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - - if (client != null) { - try { - client.close(); - } catch (Throwable t) { - LOG.error("Error closing WebSocket client: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - client = null; - } - - if (ownsCursorEngine && cursorEngine != null) { - CursorSendEngine engine = cursorEngine; - try { - engine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - if (engine.isCloseCompleted()) { - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; - } else { - // The manager worker did not quiesce. Preserve ownership - // and report the retained flock so pools retire this slot. - // Repeated Sender.close() calls remain no-ops by contract. - // Engine cleanup was handed to a safe manager-worker path: - // owned-manager exit or shared-manager ring-pass completion. - // The getter re-probes the retained engine so the pool can - // reclaim the slot once cleanup actually completes. - slotLockReleased = false; - retainedEngine = engine; + Runnable closeCallback = () -> closeRemainingResources(null); + if (cursorSendLoop != null && cursorSendLoop.delegateClose(closeCallback)) { + rethrowTerminal(terminalError); + return; } + // The worker exited between close() failing and delegation. + // Cleanup is safe here and its failures remain suppressed on + // the original close error. + terminalError = closeRemainingResources(terminalError); } else { - // This sender owns no cursor engine holding an SF flock. - slotLockReleased = true; + terminalError = closeRemainingResources(terminalError); } - // Shutdown order: dispatcher last, after the I/O loop has stopped - // producing into it. close() drains pending entries with a short - // deadline so any final errors land in the user's handler. - if (errorDispatcher != null) { - try { - errorDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing error dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (progressDispatcher != null) { - try { - progressDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing progress dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (connectionDispatcher != null) { - try { - connectionDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing connection dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - - LOG.info("QwpWebSocketSender closed"); - // If close() ended up holding the same instance the user already // caught earlier, suppress the rethrow. The user's catch block // wraps close() (try-with-resources), and Throwable refuses @@ -1347,6 +1239,11 @@ public void close() { } } + @TestOnly + public boolean isCloseCleanupComplete() { + return closeCleanupComplete; + } + /** * True once the store-and-forward slot flock has been released. False * means an I/O or manager worker did not stop and close() retained the @@ -1808,6 +1705,21 @@ public long getDroppedConnectionNotifications() { return d == null ? 0L : d.getDroppedNotifications(); } + @TestOnly + public SenderConnectionDispatcher getConnectionDispatcherForTesting() { + return connectionDispatcher; + } + + @TestOnly + public SenderErrorDispatcher getErrorDispatcherForTesting() { + return errorDispatcher; + } + + @TestOnly + public SenderProgressDispatcher getProgressDispatcherForTesting() { + return progressDispatcher; + } + /** * Number of {@link SenderError} notifications dropped because the * bounded inbox was full. Non-zero means the user-supplied @@ -2234,6 +2146,11 @@ public void setClientFactoryOverride(java.util.function.Supplier keys) { int tableCount = 0; for (int i = 0, n = keys.size(); i < n; i++) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 469c91ea..6e7ebc8c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; @@ -49,11 +50,11 @@ * offset 8: i64 fsn * *

    - * Zero-alloc, zero-syscall writes: {@link #open(String)} opens the + * Zero-alloc, store-only ACK writes: {@link #open(String)} opens the * file and maps the 16 bytes once. {@link #write(long)} is a single - * 8-byte aligned {@code Unsafe.putLong} into the mapped region. No - * malloc/free, no read/write syscalls, no rename, on the manager's hot - * tick path. An 8-byte aligned store is hardware-atomic on x86_64 and + * 8-byte aligned {@code Unsafe.putLong} into the mapped region. Ordinary + * ACK-only manager updates require no malloc/free, read/write syscalls, or + * rename. An 8-byte aligned store is hardware-atomic on x86_64 and * arm64, and disk-atomic within one sector (the file is 16 bytes — * trivially within one sector), so a torn FSN across a crash boundary * is not a concern. @@ -67,11 +68,13 @@ * complexity (multi-store with fence + read-side validate) for * marginal additional safety. *

    - * fsync cadence: intentionally NOT performed. Host crash falls - * back to recovery's {@code lowestBase - 1} seed (same as before this - * feature, no regression), at the cost of a bounded re-replay window - * for whatever durable-acks landed since the last successful page - * cache flush. + * fsync cadence: ordinary ACK-only manager updates call + * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim + * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs + * the slot directory before unlinking, and fsyncs it again after the batch. A + * fully drained close uses the same + * covering order so the durable watermark guards any acknowledged segment that + * a host crash restores. *

    * Lifecycle: single-writer (the {@link SegmentManager} worker * thread) after construction. Read once at engine startup (any thread, @@ -98,6 +101,7 @@ public final class AckWatermark implements QuietCloseable { private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class); private static final int MAGIC_OFFSET = 0; private final int fd; + private final FilesFacade filesFacade; private final long mmapAddress; private boolean closed; // Stamped once per process either at open() (if the file already @@ -108,8 +112,9 @@ public final class AckWatermark implements QuietCloseable { // construction; no synchronisation needed. private boolean magicWritten; - private AckWatermark(int fd, long mmapAddress, boolean magicAlreadyWritten) { + private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, boolean magicAlreadyWritten) { this.fd = fd; + this.filesFacade = filesFacade; this.mmapAddress = mmapAddress; this.magicWritten = magicAlreadyWritten; } @@ -122,7 +127,7 @@ public void close() { Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); } } @@ -136,6 +141,10 @@ public void close() { * stamps the magic and the new FSN atomically. */ public static AckWatermark open(String slotDir) { + return open(FilesFacade.INSTANCE, slotDir); + } + + static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; // Decide by size: existing-and-correct -> openRW preserves the // previous session's watermark (defeating which is the whole @@ -143,17 +152,17 @@ public static AckWatermark open(String slotDir) { // wrong-sized -> openCleanRW + allocate creates a fresh // FILE_SIZE-byte file (zero magic, read() reports INVALID until // the first write). - long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; if (existing == FILE_SIZE) { - fd = Files.openRW(filePath); + fd = filesFacade.openRW(filePath); } else { - fd = Files.openCleanRW(filePath); - if (fd >= 0 && !Files.allocate(fd, FILE_SIZE)) { + fd = filesFacade.openCleanRW(filePath); + if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) { // FilesFacade.allocate contract on a false return: // close the fd AND unlink the partial file. - Files.close(fd); - Files.remove(filePath); + filesFacade.close(fd); + filesFacade.remove(filePath); fd = -1; } } @@ -165,7 +174,7 @@ public static AckWatermark open(String slotDir) { long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath); - Files.close(fd); + filesFacade.close(fd); return null; } // Inspect the existing magic once at open time. If it's already @@ -173,7 +182,7 @@ public static AckWatermark open(String slotDir) { // shutdown), the first write() can skip the magic store // entirely and degenerate to a single 8-byte FSN put. int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET); - return new AckWatermark(fd, addr, magic == FILE_MAGIC); + return new AckWatermark(filesFacade, fd, addr, magic == FILE_MAGIC); } /** @@ -183,7 +192,11 @@ public static AckWatermark open(String slotDir) { * would only confuse the next session's seed. */ public static void removeOrphan(String slotDir) { - Files.remove(slotDir + "/" + FILE_NAME); + removeOrphan(FilesFacade.INSTANCE, slotDir); + } + + static boolean removeOrphan(FilesFacade filesFacade, String slotDir) { + return filesFacade.remove(slotDir + "/" + FILE_NAME); } /** @@ -203,6 +216,23 @@ public long read() { return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET); } + /** + * Flushes the mapped bytes and then the backing fd. The caller must sync + * the slot directory after this succeeds so a newly-created watermark's + * directory entry is durable before segment deletion begins. + */ + public void sync() { + if (closed) { + throw new IllegalStateException("ack watermark is closed"); + } + if (filesFacade.msync(mmapAddress, FILE_SIZE, false) != 0) { + throw new IllegalStateException("could not msync ack watermark"); + } + if (filesFacade.fsync(fd) != 0) { + throw new IllegalStateException("could not fsync ack watermark"); + } + } + /** * Atomically updates the persisted FSN. Single 8-byte aligned * store to the mapped region. The first write also stamps the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 01910fcb..1d46c489 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -26,6 +26,7 @@ import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; @@ -92,6 +93,7 @@ public final class CursorSendEngine implements QuietCloseable { // callback allocation failure cannot orphan manager resources. A timed-out // close can then hand it to either manager path without allocating. private final Runnable deferredClose; + private final FilesFacade filesFacade; private final SegmentManager manager; // We own the manager iff the user constructed us with no manager — in that // case close() also stops the manager. When the manager is shared across @@ -269,6 +271,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.sfDir = sfDir; this.segmentSizeBytes = segmentSizeBytes; this.manager = manager; + this.filesFacade = manager.filesFacade(); this.ownsManager = ownsManager; this.appendDeadlineNanos = appendDeadlineNanos; @@ -335,7 +338,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // open() returns null on any setup failure so a missing // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. - watermarkInProgress = AckWatermark.open(sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -386,8 +389,8 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // so the new session's first read() correctly reports // INVALID (magic=0 on a freshly zero-filled file). if (!memoryMode) { - AckWatermark.removeOrphan(sfDir); - watermarkInProgress = AckWatermark.open(sfDir); + AckWatermark.removeOrphan(filesFacade, sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); } MmapSegment initial; String initialPath = null; @@ -747,6 +750,7 @@ public synchronized void close() { */ private void finishClose(boolean fullyDrained) { try { + RuntimeException durabilityFailure = null; // On a fully-drained close, persist the final acked FSN through // the still-mapped watermark BEFORE closing the ring/watermark // and BEFORE unlinking any segment file. The manager persists @@ -764,8 +768,14 @@ private void finishClose(boolean fullyDrained) { long finalAckedFsn = ring.ackedFsn(); if (finalAckedFsn >= 0) { watermark.write(finalAckedFsn); + watermark.sync(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new IllegalStateException( + "could not fsync SF slot directory before segment cleanup"); + } } - } catch (Throwable ignored) { + } catch (RuntimeException e) { + durabilityFailure = e; } } try { @@ -786,21 +796,22 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } - if (fullyDrained) { + if (fullyDrained && watermark != null && durabilityFailure == null) { boolean segmentsRemoved = false; try { - segmentsRemoved = unlinkAllSegmentFiles(sfDir); + segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir); } catch (Throwable ignored) { } - // Remove the watermark ONLY once every segment file is - // confirmed gone. The watermark is what keeps residual - // acknowledged segments inert to a successor's recovery; - // removing it while any .sfa file survives would republish - // those already-acknowledged rows. + // Remove the watermark ONLY once every segment unlink is + // durable. Until the directory fsync succeeds, stable storage + // may still contain acknowledged segments and therefore still + // needs the durable watermark. if (segmentsRemoved) { - try { - AckWatermark.removeOrphan(sfDir); - } catch (Throwable ignored) { + if (filesFacade.fsyncDir(sfDir) != 0) { + durabilityFailure = new IllegalStateException( + "could not fsync SF slot directory after segment cleanup"); + } else { + AckWatermark.removeOrphan(filesFacade, sfDir); } } else { LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack " @@ -808,6 +819,12 @@ private void finishClose(boolean fullyDrained) { + "engine on this slot recovers them as fully acked and retries the " + "unlink on its own close", sfDir); } + } else if (fullyDrained && watermark == null && sfDir != null) { + LOG.warn("close-time segment cleanup skipped on slot {} because no ack watermark " + + "is available to cover a host crash during unlink", sfDir); + } + if (durabilityFailure != null) { + throw durabilityFailure; } } finally { // Reaching finishClose at all requires established quiescence, so @@ -1247,9 +1264,9 @@ private static long segmentCleanupRank(String name) { * watermark on {@code false} so residual acknowledged segments stay * covered. */ - private static boolean unlinkAllSegmentFiles(String dir) { - if (!io.questdb.client.std.Files.exists(dir)) return true; - long find = io.questdb.client.std.Files.findFirst(dir); + private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir) { + if (!filesFacade.exists(dir)) return true; + long find = filesFacade.findFirst(dir); if (find < 0) { LOG.warn("close-time unlink could not enumerate {}; " + "any residual sf-*.sfa files will be picked up by the next recovery", dir); @@ -1261,14 +1278,14 @@ private static boolean unlinkAllSegmentFiles(String dir) { try { while (rc > 0) { String name = io.questdb.client.std.Files.utf8ToString( - io.questdb.client.std.Files.findName(find)); - rc = io.questdb.client.std.Files.findNext(find); + filesFacade.findName(find)); + rc = filesFacade.findNext(find); if (name != null && name.endsWith(".sfa")) { names.add(name); } } } finally { - io.questdb.client.std.Files.findClose(find); + filesFacade.findClose(find); } if (rc < 0) { // A partial listing must not drive any unlink: removing only the @@ -1285,7 +1302,7 @@ private static boolean unlinkAllSegmentFiles(String dir) { }); for (int i = 0, n = names.size(); i < n; i++) { String path = dir + "/" + names.get(i); - if (!io.questdb.client.std.Files.remove(path)) { + if (!filesFacade.remove(path)) { LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + "residual files stay a contiguous, watermark-covered range", path); return false; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 32ec4b30..b8ae47d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -244,11 +244,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; - // Failed-stop hand-off flag: set by delegateEngineClose() when an owner's - // close() could not stop the I/O thread and the engine close is therefore - // performed by the I/O thread's exit path. Write-once, owner thread only; - // read by the I/O thread strictly after its shutdown-latch countdown (see - // the handshake contract on delegateEngineClose). + // Failed-stop hand-off callback: set when an owner could not stop the I/O + // thread and must defer worker-reachable cleanup until the thread exits. + // Read strictly after shutdownLatch.countDown(); see delegateClose(). + private volatile Runnable delegatedClose; + // Engine-only hand-off retained for BackgroundDrainer, whose remaining + // resources are owned by its run method rather than by a sender. private volatile boolean engineCloseDelegated; // The latched terminal failure — THE exception every checkError() call // rethrows. Write-once for the loop's lifetime: the only writer is @@ -834,27 +835,20 @@ public synchronized void close() { } /** - * Failed-stop hand-off for the engine. Called by an owner whose - * {@link #close()} threw because the I/O thread would not stop: the owner - * must not free the engine (munmap/Unsafe.free of segment memory) while - * the thread may still touch it with raw {@code Unsafe} reads. Setting - * the delegation flag makes the I/O thread run {@code engine.close()} on - * its exit path, strictly after its last engine access and after the - * shutdown-latch countdown — releasing the slot lock as soon as the - * stuck wire call resolves (bounded by OS timeouts) instead of leaking - * the mapping and lock forever. - *

    - * Returns {@code true} when the I/O thread is still live and has adopted - * the engine close; {@code false} when the thread has already exited — - * the caller must close the engine itself. - *

    - * Memory model — the classic store/load handshake: this method writes the - * volatile flag, then reads the latch count; the exit path counts the - * latch down, then reads the flag. Under the sequential consistency of - * volatile (and AQS latch state) accesses, if this method observes the - * latch still up, the exit path is guaranteed to observe the flag — no - * missed close. If both sides act, {@link CursorSendEngine#close()} is - * synchronized and idempotent, so the double close is benign. + * Hands complete owner cleanup to the I/O thread when it could not be + * stopped. The callback runs after the thread's last client, buffer, and + * engine access. Returns false if the thread already exited, in which case + * the caller must run the callback. The callback itself must be idempotent: + * the latch/callback handshake guarantees execution but permits both sides + * to race after the countdown. + */ + public boolean delegateClose(Runnable closeCallback) { + delegatedClose = closeCallback; + return shutdownLatch.getCount() != 0L; + } + + /** + * Engine-only failed-stop hand-off used by BackgroundDrainer. */ public boolean delegateEngineClose() { engineCloseDelegated = true; @@ -1692,15 +1686,14 @@ private void ioLoop() { } } shutdownLatch.countDown(); - // Failed-stop hand-off (see delegateEngineClose): the owner could - // not free the engine safely while this thread was alive, so the - // engine close — and with it the slot-lock release — happens - // here, strictly after this thread's last engine access. The flag - // is read only after the countDown: the store/load pairing with - // delegateEngineClose's flag-write-then-latch-read guarantees - // either this branch or the owner's fallback runs (or both — - // engine.close() is idempotent). - if (engineCloseDelegated) { + Runnable closeCallback = delegatedClose; + if (closeCallback != null) { + try { + closeCallback.run(); + } catch (Throwable ignored) { + // The owner callback logs individual cleanup failures. + } + } else if (engineCloseDelegated) { try { engine.close(); } catch (Throwable ignored) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..226e047f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -95,6 +95,10 @@ public final class MmapSegment implements QuietCloseable { // because the consumer must see writes in publication order — once the // producer bumps publishedCursor, every byte before it is fully written. private volatile long publishedCursor; + // Monotonic in-memory link to the segment that immediately follows this + // one. SegmentRing publishes it before promoting the successor to active; + // close deliberately retains it so a cursor can advance after head trim. + private volatile MmapSegment successor; // Bytes between the last valid frame and the file end that look like an // attempted-but-invalid frame write (non-zero bytes at the bail-out // position). Zero for fresh segments and for cleanly partially-filled @@ -438,6 +442,21 @@ public long sizeBytes() { return sizeBytes; } + void linkSuccessor(MmapSegment next) { + if (next == null) { + throw new IllegalArgumentException("successor must not be null"); + } + MmapSegment existing = successor; + if (existing != null && existing != next) { + throw new IllegalStateException("segment successor already linked"); + } + successor = next; + } + + MmapSegment successor() { + return successor; + } + /** * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]} * starting at the current append cursor, then advances both cursors diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 578c4f07..6bb617ad 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; /** * Background worker that keeps every registered {@link SegmentRing} supplied @@ -51,17 +52,6 @@ * in a JVM). Polls each ring on a configurable tick (default 1 ms) — short * enough that a producer rarely sees {@link SegmentRing#BACKPRESSURE_NO_SPARE} * in the steady state, long enough that an idle JVM doesn't burn CPU. - *

    - * baseSeq race window: the spare is created with - * {@code baseSeq = ring.nextSeqHint()} as observed by the manager. If the - * producer thread appends more frames before the rotation actually fires, - * the spare's baseSeq will be stale and {@link SegmentRing#appendOrFsn} will - * throw on the mismatch check. In practice this is benign — by the time - * {@link SegmentRing#needsHotSpare()} returns true the producer has very - * little room left in the active segment, and the manager polls fast enough - * to install before the producer fills the rest. Hardening to make the race - * impossible (lazy header write at rotation time) is a separate refinement - * deferred to PR2. */ public final class SegmentManager implements QuietCloseable { @@ -77,6 +67,13 @@ public final class SegmentManager implements QuietCloseable { private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state"); private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class); + private static final int MAX_TRIMS_PER_RING_PASS = 64; + private static final long TRIM_RETRY_INITIAL_NANOS = 4_000_000L; + private static final long TRIM_RETRY_MAX_NANOS = 1_024_000_000L; + private static final int TRIM_RETRY_NONE = 0; + private static final int TRIM_RETRY_POST_BARRIER = 3; + private static final int TRIM_RETRY_PRE_BARRIER = 1; + private static final int TRIM_RETRY_UNLINK = 2; private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; private final AtomicLong fileGeneration = new AtomicLong(); @@ -97,6 +94,11 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + private final LongSupplier ticks; + // Reused by the worker for one bounded trim quantum. Keeping the unlinked + // prefix here until the post-unlink directory barrier succeeds avoids both + // per-pass allocation and publishing logical removal before it is durable. + private final MmapSegment[] trimBatch = new MmapSegment[MAX_TRIMS_PER_RING_PASS]; // Test seam: runs after a deferred ring-pass cleanup returns. Null in // production; public sender/pool lifecycle tests use it to observe exact // callback completion without sleeps or polling. @@ -126,6 +128,10 @@ public final class SegmentManager implements QuietCloseable { // entry-state check inside the trim block closes for watermark writes and // totalBytes accounting. private volatile Runnable beforeTrimSyncHook; + // Test seam invoked exactly when a retry transition/recovery is logged. + // Null in production; persistent-failure tests use it to prove log bounds + // without binding to a particular SLF4J backend. + private volatile Runnable retryLogHook; // Entry the worker is claiming or servicing, or null between passes. // Volatile publication lets teardown find the entry after deregister() // removes it from rings. RingEntry.state is the authoritative barrier: @@ -174,11 +180,11 @@ public final class SegmentManager implements QuietCloseable { private volatile Thread workerThread; public SegmentManager(long segmentSizeBytes) { - this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); + this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } public SegmentManager(long segmentSizeBytes, long pollNanos) { - this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); + this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } /** @@ -204,11 +210,22 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) { * hold an initial active plus one hot spare. */ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) { - this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE); + this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE, System::nanoTime); } @TestOnly public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, filesFacade, System::nanoTime); + } + + @TestOnly + public SegmentManager( + long segmentSizeBytes, + long pollNanos, + long maxTotalBytes, + FilesFacade filesFacade, + LongSupplier ticks + ) { // The pathScratch field initializer has already allocated its native // buffer by the time this body runs, so a validation throw must free // it or every failed construction leaks 256 bytes of native memory @@ -227,6 +244,11 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, this.segmentSizeBytes = segmentSizeBytes; this.pollNanos = pollNanos; this.maxTotalBytes = maxTotalBytes; + this.ticks = ticks; + } + + FilesFacade filesFacade() { + return filesFacade; } @Override @@ -586,6 +608,11 @@ public void setBeforeTrimSyncHook(Runnable hook) { this.beforeTrimSyncHook = hook; } + @TestOnly + public void setRetryLogHook(Runnable hook) { + this.retryLogHook = hook; + } + @TestOnly public void setWorkerJoinTimeoutMillis(long millis) { this.workerJoinTimeoutMillis = millis; @@ -691,17 +718,18 @@ private String nextSparePath(String dir) { return displayPath; } - private void serviceRing(RingEntry e) { + private boolean serviceRing(RingEntry e) { // Publish before the CAS so deregister + await can always find the // entry. The state CAS is the ownership decision: if deregister won, // this stale snapshot pass skips the ring without touching it. inService = e; if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) { inService = null; - return; + return false; } + boolean hasMoreTrimmable; try { - serviceRing0(e); + hasMoreTrimmable = serviceRing0(e); } finally { e.finishService(); Runnable cleanup = e.quiescenceCleanup == null @@ -731,9 +759,10 @@ private void serviceRing(RingEntry e) { } } } + return hasMoreTrimmable; } - private void serviceRing0(RingEntry e) { + private boolean serviceRing0(RingEntry e) { // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) @@ -829,62 +858,207 @@ private void serviceRing0(RingEntry e) { } } - // 2. Trim any segments that the ring says are fully acked. For - // memory-mode rings, "trim" is just close() (Unsafe.free) — no - // file to unlink. - // - // The watermark write and totalBytes commit are registration-gated - // under `lock` so stale worker snapshots cannot touch the - // engine-owned watermark or mutate accounting after deregister() - // returns. Trimming still runs for stale snapshots: a successful - // close + unlink transfers the fully-acked segment out of the ring, - // preserving cleanup without touching deregistered accounting. - // - // munmap + unlink stay outside the lock — they can be slow - // and shouldn't block register/deregister or sibling rings. + // 2. Trim any segments that the ring says are fully acked. Memory + // mode only frees native memory. Disk mode first makes the current + // cumulative ACK durable, then durably establishes the watermark's + // directory entry, unlinks one bounded batch, and finally commits + // those directory removals before ring/accounting state changes. + // No syscall runs under the manager lock. Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } + MmapSegment first = e.ring.firstTrimmable(); + if (first == null) { + // Preserve the cheap mmap-only watermark cadence for ACKs that do + // not yet cover a complete sealed segment. + synchronized (lock) { + if (e.isRegistered() && e.watermark != null) { + long currentAck = e.ring.ackedFsn(); + if (currentAck > e.lastPersistedAck) { + e.watermark.write(currentAck); + e.lastPersistedAck = currentAck; + } + } + } + return false; + } + + if (memoryMode) { + int trimmed = 0; + while (trimmed < MAX_TRIMS_PER_RING_PASS) { + MmapSegment segment = e.ring.firstTrimmable(); + if (segment == null) { + break; + } + try { + segment.close(); + synchronized (lock) { + if (e.ring.removeTrimmable(segment)) { + trimmed++; + if (e.isRegistered()) { + totalBytes -= segment.sizeBytes(); + } + } + } + } catch (Throwable t) { + LOG.warn("Failed to trim memory segment", t); + return false; + } + } + return trimmed == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + } + + // A deferred disk retry does no sync, unlink, or logging work. Signed + // subtraction is the standard wrap-safe deadline comparison because + // the bounded delay is many orders of magnitude below half the long + // range, even when the monotonic clock wraps. + long now = ticks.getAsLong(); + if (e.trimRetryDelayNanos != 0 && now - e.trimRetryAtNanos < 0) { + return false; + } + + // A disk segment cannot be safely unlinked without durable ACK cover. + // The registration check and mmap store are atomic with deregister. + // Once the store wins, deregistration may proceed, but its owner must + // await this in-service pass before closing the watermark or ring. + if (e.watermark == null) { + if (!e.missingWatermarkLogged) { + e.missingWatermarkLogged = true; + LOG.warn("Cannot durably trim acknowledged segments in {} without an ack watermark", e.dir); + } + return false; + } + long durableAck; + synchronized (lock) { + if (!e.isRegistered()) { + return false; + } + durableAck = e.ring.ackedFsn(); + if (durableAck > e.lastPersistedAck) { + e.watermark.write(durableAck); + e.lastPersistedAck = durableAck; + } + } + try { + e.watermark.sync(); + if (filesFacade.fsyncDir(e.dir) != 0) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, t); + return false; + } + boolean trimFailed = false; - while (true) { - MmapSegment segment = e.ring.firstTrimmable(); - if (segment == null) { + Throwable trimFailure = null; + int unlinked = 0; + MmapSegment segment = first; + long coveredAck = durableAck; + while (segment != null && unlinked < MAX_TRIMS_PER_RING_PASS) { + long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; + if (lastSeq > coveredAck) { break; } + MmapSegment next = e.ring.nextSealedAfter(segment); String path = segment.path(); try { segment.close(); - if (path != null && !filesFacade.remove(path)) { + // A retry after a post-unlink directory-sync failure sees an + // already absent path. Treat absence as the prior successful + // unlink and retry the batch barrier rather than wedging. + if (!filesFacade.remove(path) && filesFacade.exists(path)) { trimFailed = true; - LOG.warn("Failed to unlink trimmed segment {}", path); break; } - synchronized (lock) { - if (e.ring.removeTrimmable(segment) && e.isRegistered()) { - totalBytes -= segment.sizeBytes(); - } - } + trimBatch[unlinked++] = segment; + segment = next; } catch (Throwable t) { trimFailed = true; - LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); + trimFailure = t; break; } } - // An unlink failure leaves the acknowledged segment in the ring and - // on disk. Do not advance the persisted watermark past it: recovery - // must continue to see the failed path as live bookkeeping rather - // than infer that trim committed. A later successful retry persists - // the current cumulative ACK normally. - synchronized (lock) { - if (!trimFailed && e.isRegistered() && e.watermark != null) { - long currentAck = e.ring.ackedFsn(); - if (currentAck > e.lastPersistedAck) { - e.watermark.write(currentAck); - e.lastPersistedAck = currentAck; + + if (unlinked > 0) { + try { + if (filesFacade.fsyncDir(e.dir) != 0) { + for (int i = 0; i < unlinked; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + for (int i = 0; i < unlinked; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); + return false; + } + synchronized (lock) { + for (int i = 0; i < unlinked; i++) { + MmapSegment removed = trimBatch[i]; + trimBatch[i] = null; + if (e.ring.removeTrimmable(removed) && e.isRegistered()) { + totalBytes -= removed.sizeBytes(); + } } } } + if (trimFailed) { + recordTrimFailure(e, TRIM_RETRY_UNLINK, now, trimFailure); + return false; + } + recordTrimSuccess(e); + return unlinked == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + } + + private void recordTrimFailure(RingEntry e, int failureKind, long now, Throwable failure) { + long delay = e.trimRetryDelayNanos == 0 + ? TRIM_RETRY_INITIAL_NANOS + : Math.min(e.trimRetryDelayNanos << 1, TRIM_RETRY_MAX_NANOS); + e.trimRetryAtNanos = now + delay; + e.trimRetryDelayNanos = delay; + if (e.trimRetryFailureKind != failureKind) { + e.trimRetryFailureKind = failureKind; + if (failure == null) { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind)); + } else { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind), failure); + } + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private void recordTrimSuccess(RingEntry e) { + if (e.trimRetryDelayNanos != 0) { + e.trimRetryAtNanos = 0; + e.trimRetryDelayNanos = 0; + e.trimRetryFailureKind = TRIM_RETRY_NONE; + LOG.info("Durable segment trim recovered in {}", e.dir); + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private static String trimFailureName(int failureKind) { + switch (failureKind) { + case TRIM_RETRY_PRE_BARRIER: + return "covering barrier"; + case TRIM_RETRY_UNLINK: + return "segment unlink"; + default: + return "directory commit barrier"; + } } private void workerLoop() { @@ -899,16 +1073,21 @@ private void workerLoop() { ringSnapshot.add(rings.getQuick(i)); } } + boolean hasMoreTrimmable = false; for (int i = 0, n = ringSnapshot.size(); i < n; i++) { if (!running) break; - serviceRing(ringSnapshot.getQuick(i)); + if (serviceRing(ringSnapshot.getQuick(i))) { + hasMoreTrimmable = true; + } } // Drop strong refs so a deregistered ring becomes collectable // before the next tick (otherwise the snapshot pins it for up // to pollNanos after deregister). ringSnapshot.clear(); if (!running) break; - LockSupport.parkNanos(pollNanos); + if (!hasMoreTrimmable) { + LockSupport.parkNanos(pollNanos); + } } } finally { // If a timed-out close() abandoned the reap, it handed @@ -969,6 +1148,14 @@ private static final class RingEntry { // Survives across multiple serviceRing ticks and avoids a // write-storm when ackedFsn is steady. long lastPersistedAck = -1L; + // Prevents a legacy disk registration without a watermark from + // flooding the log on every manager tick. + boolean missingWatermarkLogged; + // Zero-allocation manager-thread-only retry state. The deadline uses + // the manager's monotonic clock and the delay doubles to a fixed cap. + long trimRetryAtNanos; + long trimRetryDelayNanos; + int trimRetryFailureKind; // Updated lock-free by deferUntilRingQuiescent and pass completion. // The field updater avoids one AtomicReference allocation per ring. volatile Runnable quiescenceCleanup; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8d929ae1..8f5fddbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -63,6 +63,9 @@ public final class SegmentRing implements QuietCloseable { /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */ public static final long PAYLOAD_TOO_LARGE = -2L; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); + // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only + // operation count for deterministic traversal-complexity assertions. + private static long nextSealedComparisons; // Tally of baseSeq comparisons performed by sortByBaseSeq across every // openExisting() recovery on this JVM. Used by SegmentRingTest to // assert the sort stays O(N log N) without relying on wall-clock time @@ -70,6 +73,9 @@ public final class SegmentRing implements QuietCloseable { // in production: one volatile-free add per partition pass, dwarfed by // the mmap I/O the recovery does on every segment. private static long sortComparisons; + // References copied while compacting the logical sealed-segment head. + // Test-only operation count for deterministic trim-complexity assertions. + private static long trimMovedReferences; private final long maxBytesPerSegment; // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time @@ -78,6 +84,9 @@ public final class SegmentRing implements QuietCloseable { // looks at sealedSegments after observing a higher ackedFsn, by which // point the producer thread's add to sealedSegments has retired. private final ObjList sealedSegments = new ObjList<>(); + // Logical head of sealedSegments. Head removal nulls one entry and advances + // this index; occasional compaction bounds unused prefix slots. + private int sealedHead; // High-water byte offset within the active segment at which we proactively // ask the segment manager to provision a spare (if one isn't already // installed). Computed once as 3/4 of segment capacity -- leaves the manager @@ -286,6 +295,11 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) { + " check disk health"); } } + // Publish the immutable in-memory successor chain before exposing + // the recovered ring. The final sealed segment links to the active. + for (int i = 1, n = opened.size(); i < n; i++) { + opened.get(i - 1).linkSuccessor(opened.get(i)); + } // The newest segment becomes the active. Even if it's full, that's OK: // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager // installs a hot spare, the producer rotates. Same fast path as a @@ -379,15 +393,18 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { // full, so its frameCount is stable, and (b) the spare hasn't been // appended to yet (rebaseSeq enforces that). The segment manager's // earlier guess at baseSeq is irrelevant. - long actualBase = active.baseSeq() + active.frameCount(); + MmapSegment previous = active; + long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); - // Mutate sealedSegments under the same monitor used by - // snapshotSealedSegments -- the I/O thread reads through that - // path and must not see a half-resized ObjList. + // Publish the successor before the volatile active promotion. The + // same monitor protects the sealed list and nextSealedAfter's trim + // fallback, while the volatile link also remains readable from a + // current segment after the manager removes and closes it. synchronized (this) { - sealedSegments.add(active); + previous.linkSuccessor(spare); + sealedSegments.add(previous); + active = spare; } - active = spare; hotSpare = null; // Fresh active just consumed the spare → ask the manager to start // making the next one immediately, before this segment fills. @@ -439,13 +456,11 @@ public synchronized void close() { hotSpare.close(); hotSpare = null; } - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) { - s.close(); - } + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + sealedSegments.get(i).close(); } sealedSegments.clear(); + sealedHead = 0; } /** @@ -461,10 +476,8 @@ public synchronized ObjList drainTrimmable() { ObjList out = null; // Sealed segments are in baseSeq order, oldest first; once we hit one // that isn't fully acked, none of the later ones can be either. - // Synchronized so the I/O thread's snapshotSealedSegments() can't - // race against the remove(0) shuffling slots underneath it. - while (sealedSegments.size() > 0) { - MmapSegment s = sealedSegments.get(0); + while (sealedHead < sealedSegments.size()) { + MmapSegment s = sealedSegments.get(sealedHead); long lastSeq = s.baseSeq() + s.frameCount() - 1; if (lastSeq > acked) { break; @@ -473,7 +486,7 @@ public synchronized ObjList drainTrimmable() { out = new ObjList<>(); } out.add(s); - sealedSegments.remove(0); + removeSealedHead(); } return out; } @@ -490,7 +503,7 @@ public synchronized ObjList drainTrimmable() { * scan cost doesn't matter. */ public synchronized MmapSegment findSegmentContaining(long fsn) { - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { MmapSegment s = sealedSegments.get(i); long base = s.baseSeq(); if (fsn >= base && fsn < base + s.frameCount()) { @@ -513,7 +526,7 @@ public synchronized MmapSegment findSegmentContaining(long fsn) { * fallback -- see {@link #nextSealedAfter(MmapSegment)}. */ public synchronized MmapSegment firstSealed() { - return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; + return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; } /** @@ -523,10 +536,10 @@ public synchronized MmapSegment firstSealed() { * bookkeeping or allow its identifier to be reused. */ public synchronized MmapSegment firstTrimmable() { - if (sealedSegments.size() == 0) { + if (sealedHead == sealedSegments.size()) { return null; } - MmapSegment segment = sealedSegments.get(0); + MmapSegment segment = sealedSegments.get(sealedHead); long lastSeq = segment.baseSeq() + segment.frameCount() - 1; return lastSeq <= ackedFsn ? segment : null; } @@ -551,7 +564,7 @@ public synchronized MmapSegment firstTrimmable() { */ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) { long best = -1L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen); if (fsn > best) { best = fsn; @@ -571,7 +584,8 @@ public MmapSegment getActive() { * concurrent rotation. Cross-thread readers (typically the I/O loop) * should use {@link #snapshotSealedSegments(MmapSegment[])} instead. */ - public ObjList getSealedSegments() { + public synchronized ObjList getSealedSegments() { + compactSealedSegments(); return sealedSegments; } @@ -615,21 +629,28 @@ public boolean needsHotSpare() { * outpaces the I/O thread and sealed segments accumulate well beyond * any reasonable snapshot-array size. *

    - * Identity match is intentionally avoided: we compare {@code baseSeq} - * so the loop is robust against the case where {@code current} was - * trimmed out from under us (already ACK'd before the I/O thread - * advanced) -- we still return the next segment in baseSeq order rather - * than failing. Synchronized against rotation. + * Each segment publishes its successor once, before rotation exposes that + * successor as active. A constant-time head check detects when trimming + * removed the immediate successor and falls forward to the oldest live + * sealed segment. Synchronized against rotation and head removal. */ public synchronized MmapSegment nextSealedAfter(MmapSegment current) { - long currentBase = current.baseSeq(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s.baseSeq() > currentBase) { - return s; - } + nextSealedComparisons++; + MmapSegment successor = current.successor(); + if (successor == null) { + return null; } - return null; + if (successor == active) { + return null; + } + MmapSegment first = sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; + if (first != null && successor.baseSeq() >= first.baseSeq()) { + return successor; + } + // Head trimming may have removed the immediate successor while the + // I/O cursor still held an older segment. Trims only remove a prefix, + // so the current head is the first live segment after that prefix. + return first; } /** @@ -654,14 +675,14 @@ public long publishedFsn() { * Returns false if concurrent lifecycle activity changed the head. */ public synchronized boolean removeTrimmable(MmapSegment segment) { - if (sealedSegments.size() == 0 || sealedSegments.get(0) != segment) { + if (sealedHead == sealedSegments.size() || sealedSegments.get(sealedHead) != segment) { return false; } long lastSeq = segment.baseSeq() + segment.frameCount() - 1; if (lastSeq > ackedFsn) { return false; } - sealedSegments.remove(0); + removeSealedHead(); return true; } @@ -694,17 +715,12 @@ public void setManagerWakeup(Runnable wakeup) { * the I/O loop is about to do. */ public synchronized int snapshotSealedSegments(MmapSegment[] target) { - int n = sealedSegments.size(); - if (n > target.length) { - for (int i = 0; i < target.length; i++) { - target[i] = sealedSegments.get(i); - } - return -1; + int n = sealedSegments.size() - sealedHead; + int copyCount = Math.min(n, target.length); + for (int i = 0; i < copyCount; i++) { + target[i] = sealedSegments.get(sealedHead + i); } - for (int i = 0; i < n; i++) { - target[i] = sealedSegments.get(i); - } - return n; + return n > target.length ? -1 : n; } /** @@ -720,13 +736,38 @@ public synchronized long totalSegmentBytes() { if (a != null) total += a.sizeBytes(); MmapSegment hs = hotSpare; if (hs != null) total += hs.sizeBytes(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) total += s.sizeBytes(); + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + total += sealedSegments.get(i).sizeBytes(); } return total; } + private void compactSealedSegments() { + if (sealedHead > 0) { + int liveCount = sealedSegments.size() - sealedHead; + trimMovedReferences += liveCount; + sealedSegments.remove(0, sealedHead - 1); + sealedHead = 0; + } + } + + private void removeSealedHead() { + sealedSegments.setQuick(sealedHead++, null); + int size = sealedSegments.size(); + if (sealedHead == size) { + sealedSegments.clear(); + sealedHead = 0; + } else if (sealedHead >= 64 && sealedHead >= size - sealedHead) { + compactSealedSegments(); + } + } + + /** Returns the sealed-list operation count used by traversal tests. */ + @TestOnly + public static long getNextSealedComparisons() { + return nextSealedComparisons; + } + /** * Returns the cumulative count of baseSeq comparisons performed by * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} @@ -742,12 +783,30 @@ public static long getSortComparisons() { return sortComparisons; } + /** Returns the references moved by sealed-list compaction. */ + @TestOnly + public static long getTrimMovedReferences() { + return trimMovedReferences; + } + + /** Zeroes the counter exposed via {@link #getNextSealedComparisons()}. */ + @TestOnly + public static void resetNextSealedComparisons() { + nextSealedComparisons = 0; + } + /** Zeroes the counter exposed via {@link #getSortComparisons()}. */ @TestOnly public static void resetSortComparisons() { sortComparisons = 0; } + /** Zeroes the counter exposed via {@link #getTrimMovedReferences()}. */ + @TestOnly + public static void resetTrimMovedReferences() { + trimMovedReferences = 0; + } + /** * In-place quicksort over {@code list[lo, hi)} keyed by ascending * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java index 2dec7668..541f7dcf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,6 +147,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Non-blocking enqueue. Always admits the new event unless the dispatcher * is closed or {@code event} is null. Returns {@code true} when the new diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java index aaa0e4f9..fb796712 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderError; import io.questdb.client.SenderErrorHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -209,6 +210,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * True if at least one error has been delivered to a user-installed * (non-default) handler since this dispatcher started. Used by diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java index 2c0e90a1..9e01a630 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java @@ -26,6 +26,7 @@ import io.questdb.client.SenderProgressHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,6 +173,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Replace the user-supplied handler. Effective immediately for any * subsequent delivery. Pass {@code null} to install the no-op default. diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index e401f864..1bd60f3f 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -119,6 +119,10 @@ public final class SenderPool implements AutoCloseable { // RuntimeException Throwable (e.g. an -ea AssertionError) mid-prewarm, // exercising the Error-safe delegate cleanup loop. private final IntFunction senderFactory; + // Test seam: runs immediately after a delegate factory returns, before + // listener registration and SenderSlot construction. Null in production; + // error-safety tests inject a preallocated throwable at this ownership gap. + private final Runnable postFactoryHook; // Factory for startup-recovery delegates. Distinct from senderFactory so a // recoverer can force a non-blocking initial connect (initial_connect_mode= // OFF) regardless of user config: a recovery build runs on the @@ -189,6 +193,9 @@ public final class SenderPool implements AutoCloseable { // retiredSlots and returns any index whose flock has since dropped. // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; + // Deterministic white-box complexity counter. Counts delegate release + // probes performed by direct callbacks and fallback scans. Guarded by lock. + private long retiredSlotProbeCount; // The retired slots behind the leakedSlots count: runtime reclaim paths // (discardBroken/reapIdle via reclaimSlot) and the in-range startup- // recovery pass (recoverOneSlotStep, which retains the recoverer slot for @@ -284,6 +291,25 @@ public SenderPool( deferStartupRecovery, null, null, null); } + // Test-only constructor adding a deterministic fault hook for the ownership + // gap after a delegate factory returns. + @TestOnly + public SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + Runnable postFactoryHook + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, null, null, null, postFactoryHook); + } + // Full constructor adding the user-supplied ingest callbacks (error // handler, connection listener and background-drainer listener), applied // to every Sender the pool builds (see buildManagedSlotSender). The public @@ -301,6 +327,26 @@ public SenderPool( SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, errorHandler, connectionListener, + drainerListener, null); + } + + private SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + SenderErrorHandler errorHandler, + SenderConnectionListener connectionListener, + BackgroundDrainerListener drainerListener, + Runnable postFactoryHook ) { if (minSize < 0 || maxSize < 1 || minSize > maxSize) { throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize); @@ -319,6 +365,7 @@ public SenderPool( this.acquireTimeoutMillis = acquireTimeoutMillis; this.idleTimeoutMillis = idleTimeoutMillis; this.maxLifetimeMillis = maxLifetimeMillis; + this.postFactoryHook = postFactoryHook; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); this.retiredSlots = new ArrayList<>(maxSize); @@ -564,7 +611,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { // the retirement would be permanent even after the release // (fatal at maxSize=1: every later borrow would time out). leakedSlots++; - retiredSlots.add(retained[0]); + addRetiredSlot(retained[0]); LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with " + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered " @@ -699,8 +746,9 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, stopScan = true; } } catch (Throwable drainErr) { - LOG.warn("startup SF recovery: drain failed for slot {} ({})", + LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it", slotPath, drainErr.toString()); + stopScan = true; } finally { try { recoverer.delegate().close(); @@ -710,8 +758,9 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, } } } catch (Throwable scanErr) { - LOG.warn("startup SF recovery: scan failed for slot {} ({})", + LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it", slotPath, scanErr.toString()); + stopScan = true; } if (recoverer != null && !flockReleased(recoverer)) { retainedOut[0] = recoverer; @@ -1212,12 +1261,39 @@ public int leakedSlotCount() { } } - private SenderSlot createUnlocked(int slotIndex) { - Sender delegate = senderFactory.apply(slotIndex); - if (delegate instanceof QwpWebSocketSender) { - ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + private SenderSlot createSlot(IntFunction factory, int slotIndex) { + Sender delegate = factory.apply(slotIndex); + try { + if (postFactoryHook != null) { + postFactoryHook.run(); + } + SenderSlot slot = new SenderSlot(delegate, this, slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener( + () -> recoverReleasedSlot(slot)); + } + return slot; + } catch (Throwable failure) { + if (delegate instanceof QwpWebSocketSender) { + try { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(null); + } catch (Throwable deregistrationFailure) { + addSuppressed(failure, deregistrationFailure); + } + } + if (delegate != null) { + try { + delegate.close(); + } catch (Throwable closeFailure) { + addSuppressed(failure, closeFailure); + } + } + throw failure; } - return new SenderSlot(delegate, this, slotIndex); + } + + private SenderSlot createUnlocked(int slotIndex) { + return createSlot(senderFactory, slotIndex); } /** @@ -1228,11 +1304,18 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { - Sender delegate = recoverySenderFactory.apply(slotIndex); - if (delegate instanceof QwpWebSocketSender) { - ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + return createSlot(recoverySenderFactory, slotIndex); + } + + private static void addSuppressed(Throwable failure, Throwable cleanupFailure) { + if (failure != cleanupFailure) { + try { + failure.addSuppressed(cleanupFailure); + } catch (Throwable ignored) { + // Preserve the original construction failure even if recording + // the secondary cleanup failure cannot allocate. + } } - return new SenderSlot(delegate, this, slotIndex); } private Sender defaultSender(int slotIndex) { @@ -1404,7 +1487,7 @@ private boolean reclaimSlot(SenderSlot s, String context) { return true; } leakedSlots++; - retiredSlots.add(s); + addRetiredSlot(s); LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " + "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later", @@ -1412,10 +1495,25 @@ private boolean reclaimSlot(SenderSlot s, String context) { return false; } - private void recoverReleasedSlots() { + private boolean recoverReleasedSlot(SenderSlot s) { lock.lock(); try { - reprobeRetiredSlots(); + int retiredIndex = s.retiredIndex(); + if (retiredIndex < 0 + || retiredIndex >= retiredSlots.size() + || retiredSlots.get(retiredIndex) != s) { + // The callback raced retirement, or is stale/duplicate. The + // retirement path probes before insertion, and periodic scans + // remain as a fallback, so no capacity can be lost here. + return false; + } + retiredSlotProbeCount++; + if (!flockReleased(s)) { + return false; + } + recoverRetiredSlotAt(retiredIndex); + slotReleased.signalAll(); + return true; } finally { lock.unlock(); } @@ -1439,22 +1537,10 @@ private boolean reprobeRetiredSlots() { boolean recovered = false; for (int i = retiredSlots.size() - 1; i >= 0; i--) { SenderSlot s = retiredSlots.get(i); + retiredSlotProbeCount++; if (flockReleased(s)) { - // Order is irrelevant. Swap with the tail before removing so - // each recovery does O(1) list work instead of shifting the - // remaining retired entries. The tail has already been probed - // by this reverse scan and, if still present, is unreleased. - int last = retiredSlots.size() - 1; - if (i < last) { - retiredSlots.set(i, retiredSlots.get(last)); - } - retiredSlots.remove(last); - leakedSlots--; - freeSlotIndex(s.slotIndex()); + recoverRetiredSlotAt(i); recovered = true; - LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + - "pool capacity restored, now {} of {} usable [leakedSlots={}]", - s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); } } if (recovered) { @@ -1462,4 +1548,26 @@ private boolean reprobeRetiredSlots() { } return recovered; } + + private void addRetiredSlot(SenderSlot s) { + s.retiredIndex(retiredSlots.size()); + retiredSlots.add(s); + } + + private void recoverRetiredSlotAt(int retiredIndex) { + SenderSlot s = retiredSlots.get(retiredIndex); + int last = retiredSlots.size() - 1; + if (retiredIndex < last) { + SenderSlot moved = retiredSlots.get(last); + retiredSlots.set(retiredIndex, moved); + moved.retiredIndex(retiredIndex); + } + retiredSlots.remove(last); + s.retiredIndex(-1); + leakedSlots--; + freeSlotIndex(s.slotIndex()); + LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + + "pool capacity restored, now {} of {} usable [leakedSlots={}]", + s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index 19c93671..5d9ed6ff 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -48,6 +48,11 @@ final class SenderSlot { private final Sender delegate; private final SenderPool pool; private final int slotIndex; + // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. + // Guarded by the pool lock. The pool maintains this index across swap + // removals so the delegate's release callback can recover this exact slot + // without scanning every other retired slot. + private int retiredIndex = -1; // Monotonic lease id. Mutated only under the SenderPool lock (bumped in // borrow() when the slot is handed out and in giveBack()/discardBroken() // when it is returned). A PooledSender wrapper captures it live for its @@ -112,6 +117,14 @@ SenderPool pool() { return pool; } + int retiredIndex() { + return retiredIndex; + } + + void retiredIndex(int retiredIndex) { + this.retiredIndex = retiredIndex; + } + int slotIndex() { return slotIndex; } diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java index dca93e84..248ad61e 100644 --- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java @@ -86,6 +86,11 @@ public int fsync(int fd) { return Files.fsync(fd); } + @Override + public int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + @Override public long length(int fd) { return Files.length(fd); @@ -106,6 +111,11 @@ public int mkdir(String path, int mode) { return Files.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { return Files.openCleanRW(path); diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 02756c37..7a4038ed 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -408,6 +408,20 @@ public static String utf8ToString(long nameZ) { */ public static native int fsync(int fd); + /** + * Forces directory-entry updates under {@code dir} to durable storage. + * Returns 0 on success and non-zero on failure. Callers use this after + * unlinking files whose absence must survive a host crash. + */ + public static int fsyncDir(String dir) { + long ptr = pathPtr(dir); + try { + return fsyncDir0(ptr); + } finally { + freePathPtr(ptr); + } + } + /** * Truncates the file to exactly {@code size} bytes via {@code ftruncate}. * Returns {@code true} on success. Does NOT reserve disk space — the @@ -555,6 +569,8 @@ public static void munmap(long address, long len, int memoryTag) { static native int close0(int fd); + static native int fsyncDir0(long lpszName); + static native int openRO0(long lpszName); static native int openRW0(long lpszName); diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 1b408cf4..60000a3b 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -85,6 +85,10 @@ public interface FilesFacade { int fsync(int fd); + default int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + long length(int fd); /** @@ -107,6 +111,10 @@ public interface FilesFacade { int mkdir(String path, int mode); + default int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + int openCleanRW(String path); /** diff --git a/core/src/main/java/io/questdb/client/std/ObjList.java b/core/src/main/java/io/questdb/client/std/ObjList.java index 9e0e7b29..34d989ae 100644 --- a/core/src/main/java/io/questdb/client/std/ObjList.java +++ b/core/src/main/java/io/questdb/client/std/ObjList.java @@ -149,7 +149,7 @@ public void remove(int from, int to) { System.arraycopy(buffer, to + 1, buffer, from, move); } pos = Math.max(0, pos - (to - from + 1)); - Arrays.fill(buffer, pos, buffer.length - 1, null); + Arrays.fill(buffer, pos, buffer.length, null); } public void remove(int index) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index f2c560ea..63553ebc 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -26,14 +26,19 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.SenderError; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -457,6 +462,117 @@ public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Excep }); } + @Test(timeout = 30_000L) + public void testFailedIoStopReclaimsSenderResourcesAfterWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-full-cleanup-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + CountDownLatch enteredConnect = new CountDownLatch(1); + CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicReference ioThreadRef = new AtomicReference<>(); + StubWebSocketClient loopClient = new StubWebSocketClient(); + StubWebSocketClient senderClient = new StubWebSocketClient(); + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + try { + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return loopClient; + }; + loop = new CursorWebSocketSendLoop( + null, engine, 0L, 1_000L, stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setConnectionListener(event -> { + }); + sender.setErrorHandler(error -> { + }); + sender.setProgressHandler(ackedFsn -> { + }); + sender.setClientForTesting(senderClient); + sender.setCursorEngine(engine, true); + sender.setCursorSendLoopForTesting(loop); + + SenderConnectionDispatcher connectionDispatcher = sender.getConnectionDispatcherForTesting(); + SenderErrorDispatcher errorDispatcher = sender.getErrorDispatcherForTesting(); + SenderProgressDispatcher progressDispatcher = sender.getProgressDispatcherForTesting(); + Assert.assertTrue(connectionDispatcher.offer(new SenderConnectionEvent( + SenderConnectionEvent.Kind.DISCONNECTED, + null, SenderConnectionEvent.NO_PORT, + null, SenderConnectionEvent.NO_PORT, + SenderConnectionEvent.NO_ATTEMPT_NUMBER, + SenderConnectionEvent.NO_ROUND_NUMBER, + null, 0L))); + Assert.assertTrue(errorDispatcher.offer(new SenderError( + SenderError.Category.UNKNOWN, SenderError.Policy.RETRIABLE, + SenderError.NO_STATUS_BYTE, null, SenderError.NO_MESSAGE_SEQUENCE, + -1L, -1L, null, 0L))); + Assert.assertTrue(progressDispatcher.offer(0L)); + Thread connectionDispatcherThread = connectionDispatcher.getWorkerThreadForTesting(); + Thread errorDispatcherThread = errorDispatcher.getWorkerThreadForTesting(); + Thread progressDispatcherThread = progressDispatcher.getWorkerThreadForTesting(); + Assert.assertNotNull(connectionDispatcherThread); + Assert.assertNotNull(errorDispatcherThread); + Assert.assertNotNull(progressDispatcherThread); + + AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "full-cleanup-closer"); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after release", ioThread.isAlive()); + Assert.assertTrue("complete sender cleanup callback did not finish", + sender.isCloseCleanupComplete()); + Assert.assertTrue("loop-owned WebSocket client was not closed", loopClient.isClosed()); + Assert.assertTrue("sender-owned WebSocket client was not closed", senderClient.isClosed()); + Assert.assertFalse("connection dispatcher worker was not reclaimed", + connectionDispatcherThread.isAlive()); + Assert.assertFalse("error dispatcher worker was not reclaimed", + errorDispatcherThread.isAlive()); + Assert.assertFalse("progress dispatcher worker was not reclaimed", + progressDispatcherThread.isAlive()); + Assert.assertTrue("engine cleanup did not complete", engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull(probe); + } + } finally { + releaseConnect.countDown(); + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -528,16 +644,27 @@ private static void setField(Object target, String name, Object value) throws Ex * it (close is idempotent via the superclass). */ private static final class StubWebSocketClient extends WebSocketClient { + private final AtomicBoolean isClosed = new AtomicBoolean(); StubWebSocketClient() { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); } + @Override + public void close() { + isClosed.set(true); + super.close(); + } + @Override protected void ioWait(int timeout, int op) { throw new UnsupportedOperationException("stub: no socket"); } + boolean isClosed() { + return isClosed.get(); + } + @Override protected void setupIoWait() { // no-op diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java new file mode 100644 index 00000000..d6c3f665 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java @@ -0,0 +1,310 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class CursorSendEngineCrashConsistencyTest { + + @Test + public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{-1, 0, 1, 2, 4}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine engine = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, failAt); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + engine = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, engine.appendBlocking(payload, 32)); + Assert.assertTrue(engine.acknowledge(0L)); + ff.beginClose(); + try { + engine.close(); + if (failAt >= 0 && failAt != 3) { + Assert.fail("sync failure was swallowed at boundary " + failAt); + } + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, + failAt >= 0 && failAt != 3); + } + engine = null; + + if (failAt >= 0 && failAt <= 2) { + Assert.assertFalse("segment deletion started after watermark barrier failure", + ff.events.contains("segment-remove")); + Assert.assertTrue("watermark was removed after its durability barrier failed", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } else { + Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, + ff.durableSegments && !ff.durableWatermark); + } + if (failAt == -1) { + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", + "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"), + ff.events); + } + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, engine); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) { + if (failure == null) { + return cleanupFailure; + } + if (failure != cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + return failure; + } + + private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) { + if (engine != null) { + try { + engine.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable closeManager(Throwable failure, SegmentManager manager) { + if (manager != null) { + try { + manager.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable freePayload(Throwable failure, long payload) { + if (payload != 0) { + try { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static Throwable removeRoot(Throwable failure, String root) { + try { + removeRecursive(root); + Assert.assertFalse("test directory was not removed: " + root, Files.exists(root)); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + return failure; + } + + private static void rethrow(Throwable failure) { + if (failure != null) { + CursorSendEngineCrashConsistencyTest.throwUnchecked(failure); + } + } + + @SuppressWarnings("unchecked") + private static void throwUnchecked(Throwable failure) throws T { + throw (T) failure; + } + + private static final class CrashImageFilesFacade implements FilesFacade { + private final List events = new ArrayList<>(); + private final int failAt; + private final String slot; + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private int watermarkFd = -1; + + private CrashImageFilesFacade(String slot, int failAt) { + this.slot = slot; + this.failAt = failAt; + } + + private void beginClose() { + active = true; + events.clear(); + eventIndex = 0; + } + + private boolean fail(String event) { + events.add(event); + return eventIndex++ == failAt; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override + public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override + public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override + public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { + return active && fd == watermarkFd && fail("watermark-fsync") ? -1 : INSTANCE.fsync(fd); + } + @Override + public int fsyncDir(String dir) { + if (active && slot.equals(dir)) { + if (fail("dir-fsync")) return -1; + if (events.contains("segment-remove")) { + durableSegments = false; + } else { + durableWatermark = true; + } + } + return INSTANCE.fsyncDir(dir); + } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + return active && fail("watermark-msync") ? -1 : INSTANCE.msync(addr, len, async); + } + @Override + public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override + public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + if (fail("segment-remove")) return false; + } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) { + if (fail("watermark-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 6c08aeff..2c5f76c3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -85,14 +85,15 @@ public void tearDown() { /** * The structural guarantee: while the manager worker is provably still - * inside a service pass for the engine's ring, {@code close()} must NOT - * hand the slot to anyone else. With the quiescence barrier reverted, - * close() releases the slot lock immediately and the mid-test - * {@code SlotLock.acquire} probe succeeds. Once the pass finishes, its - * deferred cleanup must release the slot without a direct close retry. + * inside a service pass for the engine's ring, repeated direct + * {@code close()} calls must NOT hand the slot to anyone else. With the + * duplicate-cleanup-owner branch reverted, the second close runs cleanup + * inline and the mid-test {@code SlotLock.acquire} probe succeeds. Once + * the pass finishes, its deferred cleanup must run exactly once and + * release the slot without another close retry. */ @Test(timeout = 30_000L) - public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + public void testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { TestUtils.assertMemoryLeak(() -> { long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); String slot = tmpDir + "/slot"; @@ -103,11 +104,15 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger cleanupCount = new AtomicInteger(); AtomicReference hookErr = new AtomicReference<>(); boolean managerClosed = false; CursorSendEngine engine = null; try { - manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setAfterRingCleanupHook(() -> { + cleanupCount.incrementAndGet(); + cleanupFinished.countDown(); + }); manager.setBeforeInstallSyncHook(() -> { if (!fired.compareAndSet(false, true)) return; workerBlocked.countDown(); @@ -136,6 +141,15 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { Assert.assertFalse("incomplete close must remain observable to the owner", engine.isCloseCompleted()); + // Exercise CursorSendEngine.close() directly again while the + // same pass already owns deferredClose. Sender.close() cannot + // reach this branch because its second call is a no-op. + engine.close(); + Assert.assertFalse("repeated close must not steal deferred cleanup ownership", + engine.isCloseCompleted()); + Assert.assertEquals("cleanup ran while the worker pass was blocked", + 0, cleanupCount.get()); + // The slot must still be locked: a replacement engine (or raw // SlotLock) acquiring it now would race the stale worker. try { @@ -154,6 +168,8 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { releaseWorker.countDown(); Assert.assertTrue("ring pass did not finish deferred cleanup", cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("ring-pass cleanup must run exactly once", + 1, cleanupCount.get()); Assert.assertTrue("ring-pass cleanup must report complete cleanup", engine.isCloseCompleted()); engine = null; @@ -181,7 +197,7 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { } /** - * Owned-manager twin of {@link #testCloseRetainsSlotWhileWorkerIsMidServicePass}: + * Owned-manager twin of {@link #testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass}: * the ONLY construction shape production uses (Sender.build, BackgroundDrainer, * QwpWebSocketSender.connect all own their manager). The owned close path does * not run the per-ring barrier at all — it relies on {@code manager.close()}'s diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java new file mode 100644 index 00000000..44a348f5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java @@ -0,0 +1,529 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + *******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class SegmentManagerCrashConsistencyTest { + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim acknowledged segments"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static void awaitValue(AtomicInteger value, int expected, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (value.get() < expected) { + if (System.nanoTime() > deadline) { + throw new AssertionError(message + " [expected=" + expected + ", actual=" + value.get() + ']'); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static SegmentRing createRing(String root, long segmentSize, long payload, int sealedCount) { + SegmentRing ring = null; + boolean success = false; + try { + ring = new SegmentRing(MmapSegment.create(root + "/sf-0.sfa", 0, segmentSize), segmentSize); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 1)); + for (int i = 1; i <= sealedCount; i++) { + ring.installHotSpare(MmapSegment.create(root + "/sf-" + i + ".sfa", i, segmentSize)); + Assert.assertEquals(i, ring.appendOrFsn(payload, 1)); + } + ring.installHotSpare(MmapSegment.create(root + "/sf-" + (sealedCount + 1) + ".sfa", + sealedCount + 1L, segmentSize)); + Assert.assertTrue(ring.acknowledge(sealedCount - 1L)); + success = true; + return ring; + } finally { + if (!success && ring != null) ring.close(); + } + } + + private static AckWatermark openWatermark(FilesFacade ff, String root) throws Exception { + Method method = AckWatermark.class.getDeclaredMethod("open", FilesFacade.class, String.class); + method.setAccessible(true); + return (AckWatermark) method.invoke(null, ff, root); + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(dir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + @Test(timeout = 15_000L) + public void testBackgroundTrimDurabilityOrder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-order-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync", + "segment-remove", "segment-remove", "dir-fsync"), ff.events); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testBarrierFailuresPreserveCrashSafetyAndRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{0, 1, 2, 5}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-fault-" + failAt + "-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root, failAt); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + ff.awaitFailure(); + if (failAt == 5) { + Assert.assertNotNull("post-unlink barrier failure committed ring removal", + ring.firstSealed()); + ff.advance(TimeUnit.SECONDS.toNanos(2)); + manager.wakeWorker(); + awaitTrimmed(ring); + } + manager.close(); + } + if (failAt <= 2) { + Assert.assertEquals("unlink started before covering barrier failed", 0, ff.removeCalls.get()); + Assert.assertNotNull("barrier failure removed ring bookkeeping", ring.firstSealed()); + } else { + Assert.assertNull("post-unlink barrier retry did not commit ring removal", ring.firstSealed()); + } + Assert.assertFalse("segment deletion began without a durable covering watermark", + ff.removeCalls.get() > 0 && !ff.durableWatermark); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + @Test(timeout = 15_000L) + public void testDiskTrimWithoutWatermarkIsPreserved() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-no-watermark-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 1); + CountDownLatch servicePass = new CountDownLatch(1); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.setBeforeTrimSyncHook(servicePass::countDown); + manager.register(ring, root, null); + manager.start(); + Assert.assertTrue("manager did not reach the trim service pass", + servicePass.await(5, TimeUnit.SECONDS)); + } + Assert.assertNotNull(ring.firstSealed()); + Assert.assertTrue(Files.exists(root + "/sf-0.sfa")); + } finally { + if (ring != null) ring.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testMoreThanOneQuantumBatchesBarriers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-batch-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 65); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 70L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(65, ff.removeCalls.get()); + Assert.assertEquals("watermark must sync once per quantum", 2, ff.msyncCalls.get()); + Assert.assertEquals("directory barriers must be twice per quantum", 4, ff.dirSyncCalls.get()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testPersistentFailureDoesNotStarveSibling() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String base = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-sibling-" + System.nanoTime()).toString(); + String badRoot = base + "/bad"; + String goodRoot = base + "/good"; + Assert.assertEquals(0, Files.mkdir(base, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(badRoot, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(goodRoot, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(badRoot, "dir-fsync", 0); + AckWatermark badWatermark = null; + AckWatermark goodWatermark = null; + SegmentRing badRing = null; + SegmentRing goodRing = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + badRing = createRing(badRoot, segmentSize, payload, 1); + goodRing = createRing(goodRoot, segmentSize, payload, 1); + badWatermark = openWatermark(ff, badRoot); + goodWatermark = openWatermark(ff, goodRoot); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 16L, ff, ff::ticks)) { + manager.register(badRing, badRoot, badWatermark); + manager.register(goodRing, goodRoot, goodWatermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "bad sibling failure was not attempted"); + awaitTrimmed(goodRing); + Assert.assertNotNull("failed sibling unexpectedly trimmed", badRing.firstSealed()); + } + } finally { + if (badRing != null) badRing.close(); + if (goodRing != null) goodRing.close(); + if (badWatermark != null) badWatermark.close(); + if (goodWatermark != null) goodWatermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(badRoot); + removeRecursive(goodRoot); + removeRecursive(base); + } + }); + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresBackOffAndRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String[] failures = {"watermark-msync", "segment-remove", "post-dir-fsync"}; + for (String failure : failures) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-" + failure + '-' + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade( + root, failure, "segment-remove".equals(failure) ? 2 : 0); + if ("watermark-msync".equals(failure)) { + ff.ticks.set(Long.MAX_VALUE - 2_000_000L); + } + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + AtomicInteger logs = new AtomicInteger(); + AtomicInteger passes = new AtomicInteger(); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.setBeforeTrimSyncHook(passes::incrementAndGet); + manager.setRetryLogHook(logs::incrementAndGet); + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "initial persistent failure was not attempted"); + awaitValue(logs, 1, "initial failure transition was not logged"); + if ("segment-remove".equals(failure)) { + Assert.assertEquals("successful unlink prefix was not committed", 1L, + ring.firstSealed().baseSeq()); + Assert.assertFalse(Files.exists(root + "/sf-0.sfa")); + Assert.assertTrue(Files.exists(root + "/sf-1.sfa")); + } + + int operations = ff.operationCalls(); + for (int i = 0; i < 4; i++) { + int nextPass = passes.get() + 1; + manager.wakeWorker(); + awaitValue(passes, nextPass, "deferred retry pass did not run"); + } + Assert.assertEquals("deferred passes performed filesystem work", + operations, ff.operationCalls()); + Assert.assertEquals("deferred passes emitted logs", 1, logs.get()); + + long delay = 4_000_000L; + for (int attempt = 2; attempt <= 11; attempt++) { + int nextPass = passes.get() + 1; + operations = ff.operationCalls(); + ff.advance(delay - 1); + manager.wakeWorker(); + awaitValue(passes, nextPass, "pre-deadline pass did not run"); + Assert.assertEquals("pre-deadline pass performed filesystem work", + operations, ff.operationCalls()); + nextPass = passes.get() + 1; + ff.advance(1); + manager.wakeWorker(); + awaitValue(ff.failureCalls, attempt, "retry deadline did not enable attempt"); + awaitValue(passes, nextPass, "retry pass did not run"); + Assert.assertEquals("persistent failure log was not throttled", 1, logs.get()); + delay = Math.min(delay * 2, 1_024_000_000L); + } + + ff.failureEnabled = false; + ff.advance(delay); + manager.wakeWorker(); + awaitTrimmed(ring); + awaitValue(logs, 2, "recovery transition was not logged"); + Assert.assertEquals("recovery transition was not logged once", 2, logs.get()); + } + Assert.assertNull(ring.firstSealed()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + private static final class OrderingFilesFacade implements FilesFacade { + private final AtomicInteger dirSyncCalls = new AtomicInteger(); + private final List events = new ArrayList<>(); + private final int failAt; + private final AtomicInteger failureCalls = new AtomicInteger(); + private final AtomicInteger msyncCalls = new AtomicInteger(); + private final AtomicInteger removeCalls = new AtomicInteger(); + private final String persistentEvent; + private final int persistentRemoveOrdinal; + private final String root; + private final AtomicLong ticks = new AtomicLong(); + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private boolean expectPreDirSync = true; + private volatile boolean failureEnabled; + private volatile boolean failureObserved; + private int watermarkFd = -1; + + private OrderingFilesFacade(String root) { + this(root, -1); + } + + private OrderingFilesFacade(String root, int failAt) { + this.root = root; + this.failAt = failAt; + this.persistentEvent = null; + this.persistentRemoveOrdinal = 0; + } + + private OrderingFilesFacade(String root, String persistentEvent, int persistentRemoveOrdinal) { + this.root = root; + this.failAt = -1; + this.persistentEvent = persistentEvent; + this.persistentRemoveOrdinal = persistentRemoveOrdinal; + this.failureEnabled = true; + } + + private void advance(long nanos) { + ticks.addAndGet(nanos); + } + + private void awaitFailure() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!failureObserved) { + if (System.nanoTime() > deadline) { + throw new AssertionError("injected barrier failure was not reached"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private boolean fail(String event) { + events.add("post-dir-fsync".equals(event) ? "dir-fsync" : event); + boolean persistentMatch = failureEnabled && event.equals(persistentEvent) + && (!"segment-remove".equals(event) || persistentRemoveOrdinal <= removeCalls.get()); + boolean failed = eventIndex++ == failAt || persistentMatch; + if (failed) { + failureCalls.incrementAndGet(); + failureObserved = true; + } + return failed; + } + + private int operationCalls() { + return msyncCalls.get() + dirSyncCalls.get() + removeCalls.get(); + } + + private long ticks() { + return ticks.get(); + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { + if (active && fd == watermarkFd && fail("watermark-fsync")) return -1; + return INSTANCE.fsync(fd); + } + @Override public int fsyncDir(String dir) { + if (active && root.equals(dir)) { + dirSyncCalls.incrementAndGet(); + String event = expectPreDirSync ? "dir-fsync" : "post-dir-fsync"; + boolean failed = fail(event); + if (expectPreDirSync) { + if (!failed) { + durableWatermark = true; + expectPreDirSync = false; + } + } else { + expectPreDirSync = true; + if (!failed) durableSegments = false; + } + if (failed) return -1; + } + return INSTANCE.fsyncDir(dir); + } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int msync(long addr, long len, boolean async) { + if (active) { + msyncCalls.incrementAndGet(); + if (fail("watermark-msync")) return -1; + } + return INSTANCE.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + removeCalls.incrementAndGet(); + if (fail("segment-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java index 0c102fce..a00faea4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; @@ -36,6 +37,10 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -73,6 +78,65 @@ public void tearDown() { Files.remove(tmpDir); } + @Test + public void testAckBatchDoesNotDelaySiblingSpareProvisioning() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int sealedCount = 128; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + CountDownLatch busyDrained = new CountDownLatch(1); + CountDownLatch siblingInstall = new CountDownLatch(1); + AtomicInteger trimPass = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + try (SegmentRing busyRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentRing siblingRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60))) { + for (int i = 0; i < sealedCount; i++) { + assertEquals(i, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + } + assertEquals(sealedCount, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + busyRing.acknowledge(sealedCount - 1L); + + manager.register(busyRing, null); + manager.register(siblingRing, null); + manager.setBeforeInstallSyncHook(() -> { + try { + assertEquals("the busy ring must stop at the 64-segment quantum before sibling service", + 64, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + siblingInstall.countDown(); + } + }); + manager.setBeforeTrimSyncHook(() -> { + if (trimPass.incrementAndGet() == 4) { + try { + assertEquals("the next fair pass must fully drain the busy ring", + 0, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + busyDrained.countDown(); + } + } + }); + manager.start(); + assertTrue("manager did not reach sibling spare provisioning", + siblingInstall.await(10, TimeUnit.SECONDS)); + assertTrue("manager parked instead of immediately rescheduling the remaining ACK batch", + busyDrained.await(10, TimeUnit.SECONDS)); + if (failure.get() != null) { + throw new AssertionError("ACK batch fairness witness failed", failure.get()); + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testManagerProvisionsSpareWithinPollingTick() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -136,10 +200,12 @@ public void testManagerTrimsAckedSegmentFiles() throws Exception { String seg0Path = tmpDir + "/0000000000000000.sfa"; MmapSegment seg0 = MmapSegment.create(seg0Path, 0, segSize); long buf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L)) { + assertTrue("watermark must open", watermark != null); mgr.start(); - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Fill seg0 (2 frames) and force rotation by appending a third. for (int i = 0; i < 2; i++) ring.appendOrFsn(buf, 32); @@ -170,11 +236,13 @@ public void testMaxTotalBytesCapBlocksProvisioningUntilTrimFrees() throws Except long cap = 3 * segSize; MmapSegment seg0 = MmapSegment.create(tmpDir + "/0000000000000000.sfa", 0, segSize); long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L, cap)) { + assertTrue("watermark must open", watermark != null); mgr.start(); // register seeds totalBytes = 1*segSize (initial active). - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Manager provisions spare 1 → totalBytes = 2*segSize. assertTrue(waitFor(() -> !ring.needsHotSpare(), 2000)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java index 8cc05407..e622ac1b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java @@ -43,7 +43,6 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -186,11 +185,8 @@ public void testTrimPathDoesNotDoubleSubtractAfterDeregister() throws Exception + "`totalBytes -= sz` on a stillRegistered re-check " + "under the same lock that covers deregister.", 0L, observed); - assertFalse("stale SegmentManager snapshot skipped drainTrimmable() " - + "after deregister and left a fully-acked sealed " - + "segment on disk. The registration guard should " - + "protect watermark/accounting only; trim ownership " - + "transfer must still close and unlink " + activePath, + assertTrue("deregister before the durable barrier must preserve the segment " + + "for owner-side quiescent cleanup " + activePath, Files.exists(activePath)); } finally { mgr.setBeforeTrimSyncHook(null); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java index 0feffe8a..67e0be65 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -138,8 +138,8 @@ public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exce Assert.assertNotNull("failed unlink removed the segment from ring bookkeeping", ring.firstSealed()); Assert.assertEquals(failedPath, ring.firstSealed().path()); - Assert.assertEquals("watermark advanced although unlink did not commit", - -1L, watermark.read()); + Assert.assertEquals("failed unlink must remain covered by the durable cumulative watermark", + 0L, watermark.read()); fill(payload, 32, (byte) 0x5A); Assert.assertEquals("non-DEDUP successor must continue at the next FSN", diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index f1a0fcde..0d31320b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -38,6 +38,8 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -428,9 +430,10 @@ public void testAcknowledgeIsMonotonic() throws Exception { } @Test - public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws Exception { + public void testNextSealedAfterWalksThousandsOfSegmentsInLinearOperations() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Regression for "sealed snapshot grew unexpectedly large". + // Regression for both "sealed snapshot grew unexpectedly large" + // and a later quadratic list traversal at every segment boundary. // The cursor I/O loop used to copy the entire sealed list into a // fixed-size array (initial 16, grown once to 32) on every advance. // Under load — producer outpacing the WS sender, no maxTotalBytes @@ -462,8 +465,18 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws // After the loop we have `sealedCount` sealed segments and one // active (containing nothing yet — its base = sealedCount). // Now walk: oldest sealed, then nextSealedAfter() repeatedly. + SegmentRing.resetNextSealedComparisons(); MmapSegment cursor = ring.firstSealed(); assertNotNull(cursor); + for (int i = 1; i < sealedCount / 2; i++) { + cursor = ring.nextSealedAfter(cursor); + assertNotNull(cursor); + } + long halfWalkOperations = SegmentRing.getNextSealedComparisons(); + + SegmentRing.resetNextSealedComparisons(); + cursor = ring.firstSealed(); + assertNotNull(cursor); assertEquals(0, cursor.baseSeq()); int visited = 1; long prevBase = cursor.baseSeq(); @@ -478,8 +491,14 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws visited++; } assertEquals("must visit every sealed segment", sealedCount, visited); - // Walking past the last sealed → null (caller falls through to active). - assertNull(ring.nextSealedAfter(cursor)); + // The loop terminated when walking past the last sealed returned null. + long comparisons = SegmentRing.getNextSealedComparisons(); + assertTrue("walk inspected " + comparisons + " entries for " + sealedCount + + " segments; successor traversal must remain O(N)", + comparisons <= 2L * sealedCount); + assertTrue("doubling the walk grew operations from " + halfWalkOperations + + " to " + comparisons + "; expected linear scaling", + comparisons <= 2L * halfWalkOperations + 2L); } } finally { Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); @@ -528,6 +547,73 @@ public void testNextSealedAfterStillReturnsCorrectlyWhenCursorWasTrimmed() throw }); } + @Test(timeout = 30_000L) + public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 16); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment seg0 = MmapSegment.create(tmpDir + "/race-0.sfa", 0, segSize); + try (SegmentRing ring = new SegmentRing(seg0, segSize)) { + fillPattern(buf, 16, 0); + for (int i = 0; i < 3; i++) { + assertEquals(i, ring.appendOrFsn(buf, 16)); + ring.installHotSpare(MmapSegment.create( + tmpDir + "/race-" + (i + 1) + ".sfa", + ring.nextSeqHint(), segSize)); + } + MmapSegment cursor = ring.firstSealed(); + assertNotNull(cursor); + ring.acknowledge(1); + + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread rotate = new Thread(() -> { + ready.countDown(); + try { + start.await(); + assertEquals(3L, ring.appendOrFsn(buf, 16)); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-rotate"); + Thread trim = new Thread(() -> { + ready.countDown(); + try { + start.await(); + ObjList drained = ring.drainTrimmable(); + assertNotNull(drained); + assertEquals(2, drained.size()); + for (int i = 0; i < drained.size(); i++) { + drained.get(i).close(); + } + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-trim"); + + synchronized (ring) { + rotate.start(); + trim.start(); + ready.await(); + start.countDown(); + } + rotate.join(); + trim.join(); + assertNull("concurrent mutation failed: " + failure.get(), failure.get()); + + MmapSegment next = ring.nextSealedAfter(cursor); + assertNotNull(next); + assertEquals("trim fallback must skip both removed successors", 2L, next.baseSeq()); + assertEquals("rotation must leave the former active sealed", 1, ring.getSealedSegments().size()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + /** * Open-time sort regression: at the documented {@code sf_max_total_bytes * / sf_max_bytes} ceiling (~16K segments) an O(N²) sort over the @@ -600,6 +686,52 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { }); } + @Test + public void testRemovingAcknowledgedPrefixMovesLinearReferences() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + long previousMoves = -1; + for (int sealedCount = 64; sealedCount <= 256; sealedCount *= 2) { + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + for (int i = 0; i < 2 * sealedCount; i++) { + assertEquals(i, ring.appendOrFsn(buf, 1)); + ring.installHotSpare(MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + assertEquals(2L * sealedCount, ring.appendOrFsn(buf, 1)); + ring.acknowledge(sealedCount - 1L); + SegmentRing.resetTrimMovedReferences(); + MmapSegment segment; + int removed = 0; + while ((segment = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(segment)); + segment.close(); + removed++; + } + assertEquals(sealedCount, removed); + assertEquals("unacknowledged suffix must remain live", sealedCount, + ring.getSealedSegments().size()); + assertEquals(sealedCount, ring.firstSealed().baseSeq()); + long moves = SegmentRing.getTrimMovedReferences(); + assertTrue("removing " + sealedCount + " entries moved " + moves + + " references; expected amortized linear work", + moves <= 2L * sealedCount); + if (previousMoves >= 0) { + assertTrue("doubling the acknowledged prefix grew moved references from " + + previousMoves + " to " + moves, + moves <= 2L * previousMoves + sealedCount); + } + previousMoves = moves; + } + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testMaxBytesPerSegmentReturnsConfiguredValue() throws Exception { // Direct constructor path: the value passed in must round-trip through diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index 81055bb6..38561c46 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -32,6 +32,8 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.file.Paths; import java.util.concurrent.atomic.AtomicBoolean; @@ -83,6 +85,75 @@ public void preWarmClosesBuiltDelegatesWhenBuildThrowsError() throws Exception { }); } + @Test(timeout = 30_000) + public void elasticBorrowClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + })) { + try { + pool.borrow(); + Assert.fail("borrow must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("borrow must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed elastic creation must close its delegate", 1, closeCalls.get()); + } + }); + } + + @Test(timeout = 30_000) + public void preWarmClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try { + new SenderPool( + CFG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + }); + Assert.fail("prewarm must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("prewarm must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed prewarm creation must close its delegate", 1, closeCalls.get()); + }); + } + + @Test(timeout = 30_000) + public void recoveryClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AssertionError closeFailure = new AssertionError("injected close failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, closeFailure), false, + () -> { + throw failure; + })) { + Method createRecoverer = SenderPool.class.getDeclaredMethod("createRecoverer", int.class); + createRecoverer.setAccessible(true); + try { + createRecoverer.invoke(pool, -1); + Assert.fail("recovery creation must propagate the post-factory failure"); + } catch (InvocationTargetException e) { + Assert.assertSame("recovery must preserve throwable identity", failure, e.getCause()); + Assert.assertArrayEquals("close failure must remain secondary", + new Throwable[]{closeFailure}, failure.getSuppressed()); + } + Assert.assertEquals("failed recovery creation must close its delegate", 1, closeCalls.get()); + } + }); + } + // Companion to the catch (RuntimeException) -> track-normal-completion fix in // PooledSender.close(). flush() can exit with an Error (AssertionError under // -ea, OutOfMemoryError, ...) as well as a RuntimeException; the wrapper is @@ -232,6 +303,42 @@ public void borrowReleasesSfSlotIndexWhenCreationFails() throws Exception { }); } + private static Sender closeCountingSender(AtomicInteger closeCalls, Throwable closeFailure) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCalls.incrementAndGet(); + if (closeFailure != null) { + throw closeFailure; + } + return null; + } + if ("toString".equals(method.getName())) { + return "CloseCountingSender"; + } + if ("hashCode".equals(method.getName())) { + return System.identityHashCode(proxy); + } + if ("equals".equals(method.getName())) { + return proxy == args[0]; + } + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + }); + } + private static Sender fakeSender(AtomicBoolean closedFlag) { return (Sender) Proxy.newProxyInstance( Sender.class.getClassLoader(), diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 896dc69a..ea7e901a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; @@ -871,6 +872,105 @@ public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exce }); } + @Test + public void testDirectRetiredSlotCallbacksHaveLinearProbeCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + final int slotCount = 32; + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, slotCount, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[slotCount]; + Sender[] delegates = new Sender[slotCount]; + Runnable[] callbacks = new Runnable[slotCount]; + for (int i = 0; i < slotCount; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + Assert.assertNotNull(callbacks[i]); + } + for (int i = 0; i < slotCount; i++) { + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(slotCount, pool.leakedSlotCount()); + setLongField(pool, "retiredSlotProbeCount", 0); + + int[] geometricCheckpoints = {4, 8, 16, 32}; + int checkpoint = 0; + for (int i = 0; i < slotCount; i++) { + setBooleanField(delegates[i], "slotLockReleased", true); + callbacks[i].run(); + if (i + 1 == geometricCheckpoints[checkpoint]) { + Assert.assertEquals("direct release probes must grow linearly", + i + 1, getLongField(pool, "retiredSlotProbeCount")); + checkpoint++; + } + } + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + } + } + }); + } + + @Test + public void testDirectRetiredSlotCallbackFallbackAndIdempotence() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 0, 3, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[3]; + Sender[] delegates = new Sender[3]; + Runnable[] callbacks = new Runnable[3]; + for (int i = 0; i < 3; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(3, pool.leakedSlotCount()); + setLongField(pool, "retiredSlotProbeCount", 0); + + // Simulate callback registration becoming unavailable. The + // periodic housekeeper scan must remain a complete fallback. + ((QwpWebSocketSender) delegates[0]).setSlotLockReleaseListener(null); + setBooleanField(delegates[0], "slotLockReleased", true); + pool.reapIdle(); + Assert.assertEquals(2, pool.leakedSlotCount()); + + // A premature callback must not remove an unreleased slot. + callbacks[1].run(); + Assert.assertEquals(2, pool.leakedSlotCount()); + setBooleanField(delegates[1], "slotLockReleased", true); + callbacks[1].run(); + Assert.assertEquals(1, pool.leakedSlotCount()); + + // Duplicate and stale callbacks are idempotent and do not + // probe or mutate the slot after its direct removal. + setBooleanField(delegates[2], "slotLockReleased", true); + callbacks[2].run(); + long probesAfterRecovery = getLongField(pool, "retiredSlotProbeCount"); + callbacks[2].run(); + callbacks[1].run(); + Assert.assertEquals(probesAfterRecovery, + getLongField(pool, "retiredSlotProbeCount")); + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + } + } + }); + } + @Test public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -2373,6 +2473,85 @@ public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exce }); } + @Test + public void testDrainFailureRetriesInRangeAndOutOfRangeCandidates() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + if (attempts[idx].incrementAndGet() == 1) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("drain".equals(method.getName())) { + throw new LineSenderException("transient drain failure for slot " + idx); + } + if ("close".equals(method.getName())) { + return null; + } + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + }); + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertFalse("failed in-range drain must defer the same candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals(1, attempts[0].get()); + Assert.assertEquals(0, attempts[1].get()); + Assert.assertTrue("failed in-range drain must preserve durable data", + hasSegmentFile(slot("default-0"))); + + Assert.assertTrue("successful in-range retry may continue scanning", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("same live pool must retry the in-range candidate", + 2, attempts[0].get()); + Assert.assertFalse("in-range retry must drain the preserved data", + hasSegmentFile(slot("default-0"))); + + Assert.assertFalse("failed out-of-range drain must defer the same candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals(1, attempts[1].get()); + Assert.assertTrue("failed out-of-range drain must preserve durable data", + hasSegmentFile(slot("default-1"))); + + Assert.assertTrue("successful out-of-range retry may finish the candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("same live pool must retry the out-of-range candidate", + 2, attempts[1].get()); + Assert.assertFalse("out-of-range retry must drain the preserved data", + hasSegmentFile(slot("default-1"))); + Assert.assertFalse("final scan step must mark recovery complete", + invokeRunStartupRecoveryStep(pool)); + Assert.assertTrue("both recovered frames must be delivered", handler.frames.get() >= 2); + } + } + }); + } + @Test public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer @@ -3135,12 +3314,24 @@ private static void setBooleanField(Object target, String name, boolean value) t f.setBoolean(target, value); } + private static void setLongField(Object target, String name, long value) throws Exception { + Field f = target.getClass().getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + private static int getIntField(Object target, String name) throws Exception { Field f = target.getClass().getDeclaredField(name); f.setAccessible(true); return f.getInt(target); } + private static long getLongField(Object target, String name) throws Exception { + Field f = target.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.getLong(target); + } + private static Object getField(Object target, String name) throws Exception { Field f = target.getClass().getDeclaredField(name); f.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java index 47d168f9..6ae57621 100644 --- a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -34,7 +34,9 @@ import io.questdb.client.network.Socket; import io.questdb.client.network.SocketReadinessWaiter; import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Os; +import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -262,6 +264,90 @@ public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Excepti assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); } + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesWindowsRecvAndRetainsFd() throws Exception { + Assume.assumeTrue("real Winsock cancellation coverage runs on Windows", Os.type == Os.WINDOWS); + + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + AtomicBoolean recvDone = new AtomicBoolean(); + AtomicInteger recvResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference recvFailure = new AtomicReference<>(); + CountDownLatch recvStarted = new CountDownLatch(1); + + long buffer = 0; + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + long recvBuffer = buffer; + + waiter = new Thread(() -> { + recvStarted.countDown(); + try { + recvResult.set(socket.recv(recvBuffer, 1)); + } catch (Throwable t) { + recvFailure.set(t); + } finally { + recvDone.set(true); + } + }, "socket-traffic-windows-recv-waiter"); + waiter.setDaemon(true); + waiter.start(); + + Assert.assertTrue("waiter did not reach the receive call", + recvStarted.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the receive complete", recvDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native receive", waiter.isAlive()); + Assert.assertNull("native receive failed", recvFailure.get()); + Assert.assertTrue("shutdown must disconnect the native receive", recvResult.get() < 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the Winsock descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released Winsock descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + if (waiter != null && waiter.isAlive()) { + try { + socket.closeTraffic(); + } catch (Throwable ignored) { + // Full close below is the final wake-up fallback. + } + } + socket.close(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + if (buffer != 0 && (waiter == null || !waiter.isAlive())) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + @Test public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exception { AtomicInteger closeCount = new AtomicInteger(); diff --git a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java index b2efff7b..cc8c6e8b 100644 --- a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java +++ b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java @@ -28,6 +28,8 @@ import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; + /** * */ @@ -58,6 +60,21 @@ public void testRemoveFromTo() { Assert.assertEquals(list(), remove(list("a", "b", "c"), 4, 10)); } + @Test + public void testRemoveFromToClearsFinalBackingSlot() throws Exception { + ObjList values = new ObjList<>(16); + for (int i = 0; i < 16; i++) { + values.add(new Object()); + } + + values.remove(0, 0); + + Field bufferField = ObjList.class.getDeclaredField("buffer"); + bufferField.setAccessible(true); + Object[] buffer = (Object[]) bufferField.get(values); + Assert.assertNull(buffer[buffer.length - 1]); + } + private static ObjList remove(ObjList o, int from, int to) { o.remove(from, to); return o; From b8d965d53d8e6490cf1a772201b254fd17b3e049 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 12 Jul 2026 22:05:06 +0100 Subject: [PATCH 18/64] Make socket traffic shutdown idempotent Treat ENOTCONN and WSAENOTCONN as successful traffic shutdown because the peer may finish a graceful close before the owner cancels the I/O thread. Preserve all other failures and descriptor ownership. Add real loopback coverage for peer-first close and a synthetic failure case that verifies non-benign shutdown errors remain visible. --- core/src/main/c/share/net.c | 3 +- core/src/main/c/windows/net.c | 5 ++ .../network/SocketTrafficShutdownTest.java | 78 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 9b58fd65..f3bc1ddd 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -130,7 +130,8 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown (JNIEnv *e, jclass cl, jint fd) { - return shutdown((int) fd, SHUT_RDWR); + const int result = shutdown((int) fd, SHUT_RDWR); + return result == -1 && errno == ENOTCONN ? 0 : result; } JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index 9cc98000..d105b504 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -234,6 +234,11 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown (JNIEnv *e, jclass cl, jint fd) { const int result = shutdown((SOCKET) fd, SD_BOTH); if (result == SOCKET_ERROR) { + const int error = WSAGetLastError(); + if (error == WSAENOTCONN) { + return 0; + } + WSASetLastError(error); SaveLastError(); } return result; diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java index 6ae57621..3a737f50 100644 --- a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -259,6 +259,84 @@ public void testCompatibilityDefaultsDoNotBypassCustomTransportOwnership() { 0, socketCloseCount.get()); } + @Test(timeout = 30_000L) + public void testPlainSocketShutdownAfterPeerDisconnectRetainsFd() throws Exception { + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + + long buffer = 0; + int fd = -1; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + + peer.close(); + Assert.assertTrue("client must observe the peer disconnect", socket.recv(buffer, 1) < 0); + + socket.closeTraffic(); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + socket.close(); + if (buffer != 0) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + + @Test + public void testPlainSocketShutdownFailureStillThrows() { + AtomicInteger closeCount = new AtomicInteger(); + NetworkFacade failingFacade = new CompatibilityNetworkFacade(closeCount) { + @Override + public int errno() { + return 1234; + } + + @Override + public int shutdown(int fd) { + Assert.assertEquals(42, fd); + return -1; + } + }; + PlainSocket socket = new PlainSocket(failingFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + socket.of(42); + + try { + socket.closeTraffic(); + Assert.fail("expected genuine traffic shutdown failure"); + } catch (IllegalStateException expected) { + Assert.assertEquals("could not shut down socket traffic [fd=42, errno=1234]", expected.getMessage()); + } finally { + socket.close(); + } + + Assert.assertTrue(socket.isClosed()); + Assert.assertEquals("full close must release facade ownership exactly once", 1, closeCount.get()); + } + @Test(timeout = 30_000L) public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Exception { assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); From 79eb13c4b67397b6f968704372bd2bc304a3316a Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 13 Jul 2026 18:05:15 +0100 Subject: [PATCH 19/64] fix(client): serialize QuestDB.close() through shutdown completion QuestDBImpl.close() set the volatile `closed` flag before running the pool teardown chain, so a second concurrent close() caller could observe closed==true and return while the first caller was still draining and releasing pool resources -- a premature return that breaks the AutoCloseable expectation that shutdown has completed once close() returns. Make close() synchronized so the losing caller blocks on the monitor until the winner finishes teardown, then enters, sees `closed`, and no-ops. A bare CAS is insufficient: the losing caller would still return early. Adds a latch-controlled two-closer regression test (QuestDBImplCloseTest) that is red without the fix. --- .../io/questdb/client/impl/QuestDBImpl.java | 14 +- .../test/impl/QuestDBImplCloseTest.java | 191 ++++++++++++++++++ 2 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index e3da539b..3545d3fa 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -181,8 +181,20 @@ public Sender borrowSender() { return senderPool.borrow(); } + // synchronized so concurrent close() callers serialize THROUGH shutdown + // completion, not merely through the `closed` flip. `closed` is set before + // the teardown chain runs, so a plain volatile guard (or a bare CAS) would + // let a second caller observe closed==true and return while the first is + // still inside closeQuietly(senderPool) releasing the flock/mmap/I/O-thread + // resources -- a premature return that breaks the AutoCloseable contract + // that shutdown has completed once close() returns. The monitor makes the + // losing caller block until the winner finishes, then it enters, sees + // `closed` and returns a no-op. No deadlock: the teardown steps + // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close()) + // never call back into QuestDBImpl.close() on another thread, so nothing + // contends for this monitor from within the critical section. @Override - public void close() { + public synchronized void close() { if (closed) { return; } diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java new file mode 100644 index 00000000..c4a33e9c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java @@ -0,0 +1,191 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +/** + * P2 regression: {@link QuestDBImpl#close()} must not return before shutdown + * has completed, even when two threads call it concurrently. {@code closed} is + * volatile and set BEFORE the pool teardown chain runs, so a naive guard lets a + * second concurrent caller observe {@code closed == true} and RETURN while the + * first caller is still inside {@code closeQuietly(senderPool)} tearing down the + * flock/mmap/I/O-thread resources. After ANY {@code close()} returns -- the + * losing concurrent caller included -- callers must be able to assume shutdown + * has completed. + *

    + * The window is opened deterministically by injecting (via the {@code @TestOnly} + * senderFactory seam) a fake delegate whose {@code close()} parks on a latch. + * The last teardown step, {@code senderPool.close()}, closes the prewarmed idle + * delegate on the closing thread OUTSIDE the pool lock, so thread A parks there + * with {@code closed} already raised. Thread B then calls {@code close()}: + *

      + *
    • pre-fix -- B reads {@code closed == true} and returns immediately while A + * is still tearing down (premature return);
      • + *
    • fixed -- B blocks until A finishes the teardown, then returns.
      • + *
    + * Latch-coordinated (no {@code Thread.sleep} for correctness) with a JUnit + * timeout on the two-thread interleaving. + */ +public class QuestDBImplCloseTest { + + // Non-SF http config: the injected senderFactory replaces the native build, + // but the constructor's eager config probe must still parse it. + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + // RED (closed set before teardown, no serialization): while thread A is + // parked inside the delegate teardown of the final senderPool.close() step, + // thread B's close() sees closed==true and RETURNS -- closeReturnedEarly is + // true and the first assertion fails. GREEN (close() serialized through + // completion): B blocks on A until the teardown finishes, so its close() + // does not return until delegateCloses == 1. + @Test(timeout = 30_000) + public void concurrentCloseSecondCallerBlocksUntilShutdownCompletes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger delegateCloses = new AtomicInteger(); + CountDownLatch inDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + IntFunction senderFactory = slotIndex -> + parkingCloseSender(delegateCloses, inDelegateClose, releaseDelegateClose); + // queryMin = 0 -> QueryClientPool prewarms nothing, so the connect + // hook is never reached and its teardown is a no-op; the parking + // delegate is the only blocking teardown step. + Consumer connectHook = client -> { + }; + + QuestDBImpl questDB = newQuestDB(senderFactory, connectHook); + + // Thread A: enter close() and park inside the final teardown step + // (senderPool.close() -> idle delegate close()), with closed + // already raised. + Thread closerA = new Thread(questDB::close, "questdb-closer-A"); + closerA.start(); + Assert.assertTrue("closer A never reached the delegate teardown", + inDelegateClose.await(10, TimeUnit.SECONDS)); + + // Thread B: a concurrent close(). It must NOT return while A is + // still tearing down. + Thread closerB = new Thread(questDB::close, "questdb-closer-B"); + closerB.start(); + closerB.join(300); + boolean closeReturnedEarly = !closerB.isAlive(); + int closesWhenBReturned = delegateCloses.get(); + + // Always unpark the teardown so the test fails on the assertion, not + // its own timeout. + releaseDelegateClose.countDown(); + + Assert.assertFalse( + "concurrent close() returned while the first caller was still tearing down " + + "(delegateCloses=" + closesWhenBReturned + " when B returned): " + + "close() must serialize through shutdown completion", + closeReturnedEarly); + + // Once the teardown completes, B must return promptly and the + // delegate must have been torn down exactly once. + closerB.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("concurrent close() did not return after the teardown completed", + closerB.isAlive()); + closerA.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("first close() did not return after the teardown completed", + closerA.isAlive()); + Assert.assertEquals("the prewarmed delegate must be torn down exactly once", + 1, delegateCloses.get()); + }); + } + + private static QuestDBImpl newQuestDB( + IntFunction senderFactory, Consumer connectHook + ) { + return new QuestDBImpl( + SENDER_CFG, QUERY_CFG, + /*senderMin*/ 1, /*senderMax*/ 1, + /*queryMin*/ 0, /*queryMax*/ 1, + /*acquireTimeoutMillis*/ 250L, + /*idleTimeoutMillis*/ Long.MAX_VALUE, + /*maxLifetimeMillis*/ Long.MAX_VALUE, + /*housekeeperIntervalMillis*/ Long.MAX_VALUE, + senderFactory, connectHook); + } + + /** + * Proxy-backed fake Sender whose {@code close()} signals {@code inClose}, + * parks on {@code releaseClose}, then bumps {@code closes} -- a delegate + * teardown frozen mid-close so the test can probe what a concurrent + * close() does while it runs. + */ + private static Sender parkingCloseSender( + AtomicInteger closes, + CountDownLatch inClose, + CountDownLatch releaseClose + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + inClose.countDown(); + if (!releaseClose.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("test never released the parked close"); + } + closes.incrementAndGet(); + return null; + case "toString": + return "ParkingCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + } + }); + } +} From e8c2dcd8ef36a660175580787eb9e3f252e0e8f1 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 13 Jul 2026 21:32:35 +0100 Subject: [PATCH 20/64] fix(qwp): cancel in-flight connect during close, with a bounded backstop CursorWebSocketSendLoop.close() did an untimed shutdownLatch.await() while the I/O thread could be blocked in an in-flight foreground connect (connect_timeout=0 => OS SYN-retry ~60-130s). The connecting WebSocketClient is a walk-local in QwpWebSocketSender.connectWalk, invisible to close() (the `client` field is null on async-initial connect / stale on reconnect), so closeTraffic() could not reach it and close() hung -- risking Sender.close() exceeding the sidecar's 120s deadline. Publish a race-safe per-loop cancellation handle (ConnectCancellation) through the transport seam: connectWalk publishes the client it is about to block on before connect(); close() calls closeTraffic() on it to unwind a black-holed connect. The ReconnectFactory seam gains a Java-8 `default reconnect(ConnectCancellation)`, so every existing implementor stays source/binary compatible. Add a bounded backstop: close() awaits the shutdown latch for at most DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS (30s, ~4x under the sidecar deadline) and, on timeout, runs the same loud failed-stop protocol as the interrupt branch (delegates final teardown to the I/O thread's exit path, frees nothing under the live worker) -- so close() returns bounded even in the rare TOCTOU window where cancellation is a no-op. Also clear the in-flight handle on connect-failure paths so it never dangles at a disposed client. Adds CursorWebSocketSendLoopConnectPhaseCloseTest (async-initial + mid-flight cancellation, plus bounded-backstop-without-cancellation). --- .../qwp/client/QwpWebSocketSender.java | 58 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 195 +++++++- ...ebSocketSendLoopConnectPhaseCloseTest.java | 469 ++++++++++++++++++ 3 files changed, 715 insertions(+), 7 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8300cedc..3f272eb9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2768,22 +2768,37 @@ private WebSocketClient newWebSocketClient() { * reconnect and {@code close()} paths are therefore never queued * behind a drainer's endpoint walk. */ - private WebSocketClient buildAndConnect(ReconnectSupplier ctx) { + private WebSocketClient buildAndConnect(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { if (ctx.isBackground()) { // Lock-free: the walk below touches only internally-synchronized // hostTracker health state and walk-local/cursor-local state on // the background path. - return connectWalk(ctx); + return connectWalk(ctx, cancellation); } connectWalkLock.lock(); try { - return connectWalk(ctx); + return connectWalk(ctx, cancellation); } finally { connectWalkLock.unlock(); } } - private WebSocketClient connectWalk(ReconnectSupplier ctx) { + // Drop the in-flight connect handle once the walk has disposed a client on + // a connect/upgrade FAILURE, so inFlight never dangles at a disposed client + // across the inter-attempt backoff. Without this a concurrent + // close()->cancel() could closeTraffic() a client the walk no longer owns; + // proven a no-op on both production transports today (closed-fd closeTraffic + // no-ops), but a future custom transport that threw on a closed socket would + // spuriously loud-fail close(). Null-guarded: the no-arg reconnect() path + // and the Unsafe.allocateInstance bare-loop tests pass a null handle. Pairs + // with the success-path clear() after upgrade(). + private static void clearInFlight(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + if (cancellation != null) { + cancellation.clear(); + } + } + + private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { // Background (drainer) factories share this connect walk -- endpoint // list and hostTracker HEALTH state (never the shared round: a // background sweep walks its own RoundCursor and records with @@ -2871,9 +2886,34 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); + if (cancellation != null) { + // Publish the client we are about to block on so a + // concurrent CursorWebSocketSendLoop.close() can break its + // traffic and unwind a black-holed native connect + // (connect_timeout=0 => OS SYN-retry) instead of hanging on + // the untimed shutdown-latch await. The publish-then-check + // handshake pairs with ConnectCancellation.cancel(): if we + // observe cancellation here we skip the blocking connect + // entirely; otherwise cancel() observed this client and + // breaks it. The walk's per-attempt catch disposes the + // client and, since running has flipped false, the + // top-of-loop ctx.isAborted() gate ends the walk. + cancellation.publish(newClient); + if (cancellation.isCancelled()) { + throw new LineSenderException(ctx.abortMessage()); + } + } newClient.connect(ep.host, ep.port); int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); + if (cancellation != null) { + // connect()+upgrade() completed: this client is no longer + // blocking, so drop it from the in-flight handle before it + // becomes the loop's `client` field. close() must then break + // its traffic via the field path exactly once -- leaving it + // in the handle too would double-shut-down the socket. + cancellation.clear(); + } } catch (HttpClientException e) { // Close BEFORE classify: the sibling catch (Error) below does not // guard catch-arm bodies, so an Error thrown inside classify() @@ -2883,6 +2923,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // upgradeStatusCode) that are set during upgrade() and survive // close(). newClient.close(); + clearInFlight(cancellation); HttpClientException classified = QwpUpgradeFailures.classify(newClient, ep.host, ep.port, e); if (classified instanceof QwpIngressRoleRejectedException) { QwpIngressRoleRejectedException re = (QwpIngressRoleRejectedException) classified; @@ -2929,6 +2970,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { continue; } catch (Exception e) { newClient.close(); + clearInFlight(cancellation); hostTracker.recordTransportError(idx, !background); lastError = e; if (!background) { @@ -2951,6 +2993,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // the cursor reconnect loop, BackgroundDrainer) rethrows Error // rather than retrying, so this stays a loud one-shot failure. closeQuietlyOnError(newClient); + clearInFlight(cancellation); throw e; } // Guard the post-upgrade tail: from here until newClient is @@ -4008,7 +4051,12 @@ boolean isAborted() { @Override public WebSocketClient reconnect() { - return buildAndConnect(this); + return buildAndConnect(this, null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + return buildAndConnect(this, cancellation); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index b8ae47d9..97b2d7d0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -88,6 +89,29 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * sub-second confirmation latency once the upload completes * server-side. {@code 0} or negative disables the keepalive entirely. */ + /** + * Bounded-await backstop for {@link #close()}: the maximum time close() + * waits for the I/O thread to stop (count down {@code shutdownLatch}) + * before it loud-fails and delegates final teardown to the I/O thread's + * exit path. In the common case the round-2 in-flight connect cancellation + * (see {@link ConnectCancellation}) makes the I/O thread unwind and count + * the latch down within milliseconds, so this budget is never reached. It + * only fires in the pathological, astronomically-rare TOCTOU window where + * {@code cancel()}'s {@code closeTraffic()} lands between the pre-connect + * guard and native fd creation, so it is a no-op and the connect blocks the + * full OS SYN-retry (~60-130s per endpoint, possibly across several). + *

    + * {@code 30_000} ms is comfortably UNDER the sidecar's 120 s shutdown + * deadline (~4x headroom) yet ~1000x a healthy close's millisecond latch + * countdown, so it never prematurely abandons a legitimately draining + * close (the I/O thread only has to finish its current — now traffic-broken + * — native send/receive and run {@code ioLoop}'s finally). This bounds the + * WAIT inside close(), NOT the connect itself (rejected Option A): a + * legitimately slow SUCCESSFUL connect is unaffected — it is not concurrent + * with a close, and when close() does fire it cancels the in-flight connect + * rather than waiting it out. + */ + public static final long DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS = 30_000L; public static final long DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS = 200L; public static final long DEFAULT_PARK_NANOS = 50_000L; // 50us idle backoff /** @@ -163,6 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // the durable watermark can lag behind the OK watermark. private final ArrayDeque pendingDurable = new ArrayDeque<>(); private final ArrayDeque pendingDurablePool = new ArrayDeque<>(); + // Race-safe cancellation handle for an in-flight connect attempt. Passed + // into reconnectFactory.reconnect(...) on the I/O thread so close() can + // break a connect blocked mid-attempt (a black-holed native connect that + // neither unpark nor interrupt cancels). See ConnectCancellation. + private final ConnectCancellation connectCancellation = new ConnectCancellation(); // Optional reconnect plumbing. When non-null, a wire failure triggers a // reconnect attempt instead of a terminal fail(). The factory produces a // fresh, connected+upgraded WebSocketClient. @@ -225,6 +254,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // it is engine.ackedFsn() + 1, so the first replayed frame on the new // connection is wireSeq=0 and server-side cumulative ACKs still line up. private long fsnAtZero; + // Bounded-await backstop budget for close() (see + // DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via + // setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch + // deterministically without a multi-second real wait; production always + // uses the default. Read only on the owner thread inside close(). + private long shutdownAwaitTimeoutMillis = DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to @@ -783,8 +818,55 @@ public synchronized void close() { ); } } + // Cancel an in-flight connect attempt not yet installed as the + // `client` field: async initial connect leaves `client` null + // and a mid-flight reconnect leaves it pointing at the stale + // pre-drop client, so neither is reachable by the closeTraffic() + // above. The reconnect factory publishes the connecting client + // here before it blocks, so breaking its traffic unwinds a + // black-holed native connect (connect_timeout=0 => OS SYN-retry, + // ~60-130s) that would otherwise pin the untimed await below. + // Same loud-fail contract as the field client: a transport that + // cannot safely shut down traffic is not destructively closed; + // the worker's exit path retains ownership of final cleanup. try { - shutdownLatch.await(); + connectCancellation.cancel(); + } catch (Throwable e) { + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + try { + // Bounded backstop: never wait forever. The round-2 in-flight + // connect cancellation makes the I/O thread unwind and count + // the latch down in milliseconds, so this await returns true + // almost always. The timeout branch fires only in the + // pathological uninterruptible-connect TOCTOU window where + // cancel()'s closeTraffic() was a no-op and the connect blocks + // the full OS SYN-retry -- guaranteeing close() still returns + // (bounded, comfortably under the sidecar's 120s deadline) + // rather than hanging on an untimed await. + if (!shutdownLatch.await(shutdownAwaitTimeoutMillis, TimeUnit.MILLISECONDS)) { + // Latch still up after the budget: the I/O thread did not + // stop. Same failed-stop protocol as the interrupt branch + // below -- loud-fail without touching the client/engine + // (freeing native buffers under a still-live I/O thread + // risks a C5 SEGV; a quiet return would let the owner + // unmap the engine under it). The I/O thread's own exit + // path (ioLoop's finally) retains ownership of final + // cleanup; QwpWebSocketSender.close() keys its + // ioThreadStopped guard on this throw and BackgroundDrainer + // switches to delegateEngineClose(). ioThread stays set + // (not nulled below, since we throw) so a duplicate close() + // re-signals rather than silently succeeding. + throw new LineSenderException( + "cursor I/O thread did not stop: close() timed out after " + + shutdownAwaitTimeoutMillis + "ms awaiting shutdown; " + + "client/engine teardown is delegated to the I/O " + + "thread's exit path"); + } } catch (InterruptedException e) { // Re-assert the flag for the caller's stack, then decide. // If the I/O thread has genuinely not exited (latch still @@ -996,6 +1078,17 @@ public void setProgressDispatcher(SenderProgressDispatcher dispatcher) { this.progressDispatcher = dispatcher; } + /** + * Test seam: shrink the {@link #close()} bounded-await backstop + * (default {@link #DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS}) so a test can + * exercise the timeout branch deterministically without a multi-second + * real wait. Production never calls this. Set before {@link #close()}. + */ + @TestOnly + public void setShutdownAwaitTimeoutMillis(long millis) { + this.shutdownAwaitTimeoutMillis = millis; + } + public synchronized void start() { if (ioThread != null) { throw new IllegalStateException("already started"); @@ -1172,7 +1265,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { - WebSocketClient newClient = reconnectFactory.reconnect(); + WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { // close() ran while this connect attempt was in @@ -2049,6 +2142,104 @@ private boolean tryRetireOrphanTail() { @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; + + /** + * Cancellable variant of {@link #reconnect()}. The loop passes a + * per-attempt {@link ConnectCancellation} so a transport that blocks + * inside a native connect can publish the in-flight client to the + * handle BEFORE it blocks; {@link #close()} then breaks that client's + * traffic to unwind a black-holed connect promptly, rather than falling + * back to the bounded {@code shutdownLatch.await(...)} backstop (which + * only bounds the WAIT, not the connect). + *

    + * Default: ignore the handle and delegate to {@link #reconnect()} -- + * a transport that cannot publish its in-flight client is simply not + * cancellable mid-connect, and {@code close()} falls back to its + * existing field-client {@code closeTraffic()} / loud-fail path. The + * default keeps this a {@link FunctionalInterface} and preserves + * source compatibility for existing lambda / method-reference + * implementors (e.g. {@link #connectWithRetry}). + */ + default WebSocketClient reconnect(ConnectCancellation cancellation) throws Exception { + return reconnect(); + } + } + + /** + * Race-safe cancellation handle for a single in-flight connect attempt. + * Owned per-loop and passed into {@link ReconnectFactory#reconnect(ConnectCancellation)} + * on the I/O thread. The transport's connect walk {@link #publish}es the + * {@link WebSocketClient} it is about to block on before the blocking + * {@code connect()}, and {@link #clear}s it once the attempt is installed + * or disposed. {@link #close()} (owner thread) calls {@link #cancel()} to + * break the in-flight client's traffic path so a black-holed native + * connect unwinds and the I/O thread counts down the shutdown latch. + *

    + * Java 8: two volatiles, no locks. Visibility/ordering: + *

      + *
    • {@code inFlight} is written by the I/O thread (publish/clear) and + * read by the owner thread (cancel).
      • + *
    • {@code cancelled} is written by the owner thread (cancel) and read + * by the I/O thread (the pre-connect guard).
      • + *
    + * The publish (write inFlight, then read cancelled) / cancel (write + * cancelled, then read inFlight) handshake covers the close-vs-connect + * race in both directions: if the guard observes {@code cancelled==false} + * then, by the volatile total order, {@code cancel()} observed the + * published client and broke its traffic; if the guard observes + * {@code cancelled==true} it skips the blocking connect entirely. + */ + public static final class ConnectCancellation { + // The client the connect walk is currently about to block / blocked + // on. Written by the I/O thread only (publish/clear); read by the + // owner thread (cancel). volatile for cross-thread visibility. + private volatile WebSocketClient inFlight; + // Latched once close() requested cancellation. Written by the owner + // thread (cancel); read by the I/O thread's pre-connect guard. + private volatile boolean cancelled; + + public boolean isCancelled() { + return cancelled; + } + + /** + * I/O-thread hook: record the client the walk is about to block on, + * BEFORE the blocking {@code connect()}. Pairs with {@link #cancel()} + * so an attempt that publishes after cancellation is caught by the + * caller's {@link #isCancelled()} guard, and an attempt already + * blocked in {@code connect()} is broken by {@code cancel()}. + */ + public void publish(WebSocketClient client) { + inFlight = client; + } + + /** + * I/O-thread hook: the walk is done with the in-flight attempt + * (installed on success, disposed on failure). Drop the reference so a + * later {@link #cancel()} cannot touch a client the walk no longer + * owns. Single writer (I/O thread) for publish/clear, so no CAS is + * needed. + */ + public void clear() { + inFlight = null; + } + + /** + * Owner-thread hook from {@link #close()}: request cancellation, then + * break the traffic path of any in-flight connect so the I/O thread's + * connect walk unwinds. {@code cancelled} is written before reading + * {@code inFlight} to pair with {@link #publish(WebSocketClient)}. May + * throw when the transport cannot safely shut down traffic -- + * {@code close()} maps that to its existing loud-fail, exactly as it + * does for the field client's {@code closeTraffic()}. + */ + void cancel() { + cancelled = true; + WebSocketClient c = inFlight; + if (c != null) { + c.closeTraffic(); + } + } } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java new file mode 100644 index 00000000..0225d071 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java @@ -0,0 +1,469 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * TRUE-CANCELLATION proof (P1): {@code close()} must cancel a connect attempt + * that is blocked inside {@code reconnectFactory.reconnect(...)} (a black-holed + * native connect that neither unpark nor interrupt can cancel). The in-flight + * {@link WebSocketClient} is walk-local -- it is NOT the loop's {@code client} + * field (that field is {@code null} on the async-initial connect and points at + * the stale pre-drop client on a mid-flight reconnect) -- so before the fix + * {@code close()}'s field-client {@code closeTraffic()} could not reach it and + * {@code close()} blocked on the untimed {@code shutdownLatch.await()} for the + * whole OS SYN-retry window (~60-130s per endpoint). + *

    + * The fix publishes the in-flight client to a race-safe + * {@link CursorWebSocketSendLoop.ConnectCancellation} handle before the + * blocking connect, and {@code close()} breaks that client's traffic. These + * tests assert TRUE cancellation, not just return: the fake in-flight client's + * {@code closeTraffic()} is the ONLY thing that unblocks the parked + * {@code reconnect()}, and the tests witness that {@code closeTraffic()} was + * invoked (exactly once, by the closer thread) AND that {@code close()} + * returned. Deterministic latches only; {@code @Test(timeout=...)} fails a + * regression fast. + */ +public class CursorWebSocketSendLoopConnectPhaseCloseTest { + + /** + * Generous budget: with the fix, close() breaks the in-flight connect and + * returns well within this. Without the fix, close() blocks on the untimed + * shutdown latch for the entire (simulated) connect and this budget lapses. + */ + private static final long CLOSE_BUDGET_MILLIS = 5_000L; + + /** + * Small, injectable bounded-await backstop for the TOCTOU test. Shrunk from + * the production {@code DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS} (30 s) via + * {@link CursorWebSocketSendLoop#setShutdownAwaitTimeoutMillis(long)} so the + * timeout branch fires fast and deterministically -- no multi-second real + * wait -- while still leaving CLOSE_BUDGET_MILLIS of slack for the closer + * thread to return. + */ + private static final long BACKSTOP_MILLIS = 500L; + + /** + * Async-initial-connect path: the loop is built with a {@code null} client + * and a reconnect factory, so the I/O thread drives the very first connect + * through connectLoop while the {@code client} field stays {@code null}. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsAsyncInitialConnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(false); + } + + /** + * Mid-flight-reconnect path: an initial client is installed, its first + * receive fails, so connectLoop reconnects and blocks. The {@code client} + * field then points at the stale pre-drop client, never the in-flight one. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsMidFlightReconnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(true); + } + + /** + * BACKSTOP (bounded-await) proof: models the pathological, uninterruptible + * TOCTOU case where {@code cancel()}'s {@code closeTraffic()} CANNOT break + * the in-flight connect (as if cancellation landed after the pre-connect + * guard but before native fd creation, making closeTraffic() a no-op). The + * connect stays parked, so round-2 cancellation does NOT release the latch. + * Asserts {@code close()} STILL returns within the bounded backstop (does + * not hang) and surfaces the failed-stop contract: a loud + * {@link LineSenderException} whose message names the stalled I/O thread and + * the timeout, with client/engine teardown delegated to the I/O thread's + * own exit path (no destructive close under the still-live worker). + */ + @Test(timeout = 30_000L) + public void testCloseReturnsBoundedWhenCancellationCannotBreakConnect() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + // The ONLY thing that releases the parked connect. closeTraffic() + // deliberately does NOT touch it, modelling a cancel that cannot + // break the connect; the test itself counts it down, AFTER the + // backstop assertions, so the worker can unwind for the leak check. + final CountDownLatch release = new CountDownLatch(1); + + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + UninterruptibleInFlightClient c = new UninterruptibleInFlightClient(release); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Uninterruptible AND uncancellable: closeTraffic() does not + // release this park; only the test's release latch does. + c.awaitRelease(); + c.close(); + throw new LineSenderException("connect ended after release"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + // Shrink the bounded-await backstop so the timeout branch fires fast + // (no multi-second real wait); production uses the 30s default. + loop.setShutdownAwaitTimeoutMillis(BACKSTOP_MILLIS); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "backstop-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final UninterruptibleInFlightClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published", inFlight); + // cancel() DID attempt to break the connect -- but closeTraffic() + // is a no-op here, so the connect is still parked. + Assert.assertTrue("close() must have attempted to cancel the in-flight connect", + inFlight.trafficCloseCount.get() >= 1); + Assert.assertEquals("the parked connect must NOT have been released by closeTraffic()", + 1L, release.getCount()); + + // Decisive: close() returns within the bounded backstop even + // though cancellation could not break the connect. + Assert.assertTrue( + "close() must return within the bounded backstop (" + CLOSE_BUDGET_MILLIS + + "ms) even when cancellation cannot break the connect; instead it hung", + closedInTime); + + // Failed-stop contract: loud LineSenderException naming the + // stalled thread and the timeout. + final Throwable failure = closeFailure.get(); + Assert.assertNotNull("close() must loud-fail when the backstop times out", failure); + Assert.assertTrue("failed stop must be a LineSenderException, was " + failure, + failure instanceof LineSenderException); + Assert.assertTrue("message must name the stalled I/O thread: " + failure.getMessage(), + failure.getMessage().contains("cursor I/O thread did not stop")); + Assert.assertTrue("message must attribute the failed stop to the backstop timeout: " + + failure.getMessage(), + failure.getMessage().contains("timed out")); + // Teardown is delegated to the I/O thread's exit path, not done + // destructively under the live worker: no terminal manufactured. + Assert.assertNull("backstop timeout must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + // Now let the parked connect unwind so the worker exits and the + // leak check sees a clean teardown. Restore a generous backstop + // so the reconciling close() below awaits the worker's exit. + loop.setShutdownAwaitTimeoutMillis(CLOSE_BUDGET_MILLIS); + release.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + } + }); + } + + private void runConnectCancelledByCloseTraffic(boolean withInitialClient) throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Allocate the initial client INSIDE the leak-checked block so its + // native buffers are inside the baseline, not counted as over-free. + final WebSocketClient initialClient = withInitialClient ? new FailingInitialClient() : null; + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + + // A reconnect factory that models the real connect walk: it creates + // the client it is about to block on, PUBLISHES it to the loop's + // cancellation handle before the blocking connect, then parks in a + // way that ONLY closeTraffic() (i.e. cancellation) can release -- + // exactly a black-holed native connect that unpark/interrupt cannot + // cancel. + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + InFlightConnectClient c = new InFlightConnectClient(); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Black-holed connect: returns only once closeTraffic() breaks it. + c.awaitTrafficBreak(); + // Cancelled: dispose the half-built client (frees native + // buffers) and surface a transport error, exactly as the + // real connect walk's catch does on a broken connect. + c.close(); + throw new LineSenderException("connect cancelled by closeTraffic()"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "connect-phase-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final InFlightConnectClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published to the cancellation handle", + inFlight); + + // Let the worker finish unwinding regardless of outcome so the + // memory-leak check sees a clean teardown. + inFlight.trafficBroken.countDown(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + + Assert.assertNull("close() must not fail", closeFailure.get()); + Assert.assertTrue( + "close() must cancel the connect blocked inside reconnect() and return " + + "within " + CLOSE_BUDGET_MILLIS + "ms; instead it blocked on the " + + "untimed shutdown latch for the whole connect", + closedInTime); + // Decisive: closeTraffic() on the IN-FLIGHT client is what + // unblocked the parked connect -- and it was the closer thread, + // not the I/O worker, that broke it. + Assert.assertEquals( + "close() must break the in-flight connect's traffic exactly once", + 1, inFlight.trafficCloseCount.get()); + Assert.assertEquals( + "the closer thread (not the I/O worker) must cancel the in-flight connect", + closer, inFlight.trafficCloseThread.get()); + Assert.assertNull("ordinary connect-phase close must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + InFlightConnectClient inFlight = published.get(); + if (inFlight != null) { + inFlight.trafficBroken.countDown(); + } + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + if (initialClient != null) { + initialClient.close(); + } + } + }); + } + + /** + * The fake in-flight client. It never really connects: the factory parks in + * {@link #awaitTrafficBreak()} until {@link #closeTraffic()} releases it, + * which is the ONLY path out -- neither unpark nor interrupt can cancel it. + */ + private static final class InFlightConnectClient extends WebSocketClient { + final CountDownLatch trafficBroken = new CountDownLatch(1); + final AtomicInteger trafficCloseCount = new AtomicInteger(); + final AtomicReference trafficCloseThread = new AtomicReference<>(); + + private InFlightConnectClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void closeTraffic() { + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficBroken.countDown(); + } + + void awaitTrafficBreak() { + // Uninterruptible: only closeTraffic()'s countDown may release us, + // faithfully modelling a native connect that unpark/interrupt cannot + // cancel. + boolean interrupted = false; + while (trafficBroken.getCount() != 0L) { + try { + trafficBroken.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * The fake in-flight client for the BACKSTOP test. Its {@link #closeTraffic()} + * is a no-op w.r.t. releasing the park -- it only records that cancellation + * was attempted -- so a {@code close()}->{@code cancel()} CANNOT unblock the + * parked connect. This models the pathological uninterruptible/TOCTOU case; + * the ONLY release is the test-owned {@code release} latch, counted down + * AFTER the backstop assertions. + */ + private static final class UninterruptibleInFlightClient extends WebSocketClient { + final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final CountDownLatch release; + + private UninterruptibleInFlightClient(CountDownLatch release) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.release = release; + } + + @Override + public void closeTraffic() { + // Records the cancel attempt but does NOT release the parked + // connect: exactly a closeTraffic() that lands before the native fd + // exists (a no-op), so the connect blocks on regardless. + trafficCloseCount.incrementAndGet(); + } + + void awaitRelease() { + // Uninterruptible: neither unpark, interrupt, nor closeTraffic() + // releases us -- only the test's release latch. + boolean interrupted = false; + while (release.getCount() != 0L) { + try { + release.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * A pre-installed client whose first receive fails, driving the I/O loop + * into a reconnect so the in-flight client is published while the + * {@code client} field still points here (the mid-flight-reconnect path). + */ + private static final class FailingInitialClient extends WebSocketClient { + + private FailingInitialClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + throw new LineSenderException("initial wire dropped"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From d130f2c9002d0dcbde2d303cc86342b19e402f45 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 14 Jul 2026 00:14:19 +0100 Subject: [PATCH 21/64] fix(client): fail closed in SF recovery and add crash-safe boundary manifest SF crash recovery previously failed OPEN on operational errors: - findFirst() < 0 (permission/transient enumeration error) was treated as an empty directory, so the engine started fresh and openCleanRW(O_TRUNC) truncated the existing durable log (sf-initial.sfa) and deleted the ack watermark; - findNext() < 0 silently accepted an incomplete directory listing; - per-file openRW/mmap failures (EMFILE, ENOMEM, transient I/O) on valid segments were swallowed by the same catch as data corruption and skipped; - the FSN contiguity check only compares adjacent pairs, so a skipped LEADING segment silently dropped unacked rows (ackedFsn seeded past them) and a skipped TRAILING segment silently re-issued its FSNs to new payloads (overlapping FSNs on disk). Recovery now fails closed and proves slot integrity before mutating: - Enumeration errors (findFirst/findNext < 0) abort startup. - Operational open/mmap failures on recognized segments abort startup; only positively-identified corruption (new MmapSegmentCorruptionException: bad magic, sub-header size, negative baseSeq, unreadable header page) is skippable, and quarantine renames (.corrupt) are deferred until the surviving chain validates, so a failed recovery mutates nothing. Unsupported-version segments stay fatal (they belong to another client build; renaming them would strand that build's frames). - sf-manifest.bin: dual-slot, CRC-protected, generation-versioned boundary record (headBase/activeBase) written on rotation and ack-driven trim. Recovery validates the chain against the committed boundaries, which catches missing leading/trailing segments that contiguity alone cannot. Segments carry a manifest-required header flag so a deleted manifest next to migrated segments fails closed. Boundary updates are monotonic (clamped) and serialized with trim under the ring monitor; the trim path recomputes the successor under that monitor so a concurrent rotation can never make the head leapfrog a still-unacked sealed segment. - Segment files are created with O_EXCL (new Files.openRWExclusive natives) instead of O_TRUNC, so no code path can truncate an existing log. - Crash-window protocol, verified state by state: fresh start orders initial-segment durability -> manifest create -> flag stamp; rotation syncs the promoted spare's rebased header before the manifest references it and commits the manifest before any ring mutation; clean-drain close durably collapses boundaries to head==active before unlinking and removes the manifest last. Every window either recovers (equivalent empties from a fresh-start kill, an empty active at the chain end after a rotation crash, record-less manifest creation debris, drain-window survivors, stale below-head files) or fails closed with intact evidence. The one deliberate mutation on a failing path is quarantining a provably record-less manifest (it contains no boundaries by construction; flagged segments still fail closed afterwards). Regression tests (SegmentRecoveryIntegrityTest, 24 cases) cover the full required matrix: findFirst=-1, partial findNext=-1, openRW=-1, mmap failure -- each asserting byte-for-byte slot preservation; the engine-level O_TRUNC reproduction; leading/trailing/interior boundary loss; corrupt stray quarantine with sibling recovery; deferred-quarantine no-mutation on failure; flagged-segments-without-manifest; manifest creation-crash debris self-heal (zero-byte and record-less); drain-crash spare survivor; corrupt-active-with-empty-stand-in fail-closed; dual-slot generation selection and torn-record fallback. Independently reviewed for crash-consistency holes, lock ordering (manager lock -> ring monitor -> manifest monitor, acyclic), unacked-frame loss, and fd/mmap leaks; both review blockers (creation-debris brick, drain-window spare brick) are fixed and regression-tested. Windows native openRWExclusive0 uses CREATE_NEW; CI rebuilds platform binaries from the .c change. --- core/src/main/c/share/files.c | 7 + core/src/main/c/windows/files.c | 9 + .../client/sf/cursor/CursorSendEngine.java | 123 ++- .../qwp/client/sf/cursor/MmapSegment.java | 79 +- .../MmapSegmentCorruptionException.java | 50 ++ .../sf/cursor/MmapSegmentException.java | 8 +- .../qwp/client/sf/cursor/SegmentManager.java | 33 +- .../qwp/client/sf/cursor/SegmentRing.java | 665 ++++++++++---- .../qwp/client/sf/cursor/SfManifest.java | 282 ++++++ .../client/std/DefaultFilesFacade.java | 20 + .../java/io/questdb/client/std/Files.java | 20 + .../io/questdb/client/std/FilesFacade.java | 16 + .../qwp/client/sf/cursor/MmapSegmentTest.java | 43 +- .../cursor/SegmentRecoveryIntegrityTest.java | 827 ++++++++++++++++++ .../cursor/SegmentRingRecoveryUnlinkTest.java | 23 +- 15 files changed, 1975 insertions(+), 230 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index fefd24fb..01e18a6b 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -65,6 +65,13 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 return (jint) fd; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_CREAT | O_EXCL | O_RDWR, 0644), fd); + return (jint) fd; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { int fd; diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 620bd81e..7e57de94 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -103,6 +103,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 FILE_ATTRIBUTE_NORMAL); } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + return open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL); +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { jint fd = open_file((const char *) (uintptr_t) lpszName, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 1d46c489..9e606b94 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -286,9 +286,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs // already on disk and corrupting ACK translation, trim, and replay. - SegmentRing recovered = memoryMode ? null - : SegmentRing.openExisting(sfDir, segmentSizeBytes); - this.wasRecoveredFromDisk = recovered != null; + SegmentRing.Recovery recovery = memoryMode ? null + : SegmentRing.recover(filesFacade, sfDir, segmentSizeBytes); + SegmentRing recovered = recovery == null ? null : recovery.ring(); + this.wasRecoveredFromDisk = recovery != null + && recovery.status() == SegmentRing.RecoveryStatus.RECOVERED; if (recovered != null) { ringInProgress = recovered; // Seed ackedFsn to one below the lowest segment's baseSeq. @@ -394,18 +396,49 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } MmapSegment initial; String initialPath = null; + SfManifest initialManifest = null; if (memoryMode) { initial = MmapSegment.createInMemory(0L, segmentSizeBytes); } else { + // Created WITHOUT the manifest-required flag: the flag is + // a durable promise that sf-manifest.bin exists, so it + // must only be stamped after the manifest itself is on + // disk. Stamping it at create time would leave a crash + // window (initial durable, manifest not yet created) + // whose recovery hard-fails with "segment requires + // missing manifest" even though nothing was lost. initialPath = sfDir + "/sf-initial.sfa"; - initial = MmapSegment.create(initialPath, 0L, segmentSizeBytes); + initial = MmapSegment.create(filesFacade, initialPath, 0L, segmentSizeBytes); } try { - ringInProgress = new SegmentRing(initial, segmentSizeBytes); + if (!memoryMode) { + // Ordering is load-bearing for crash recovery: + // 1. initial segment durable (header + dirent), + // 2. manifest created (its own create fsyncs), + // 3. flag stamped on the segment. + // A crash after (1) recovers via the legacy + // (manifest-less) path; after (2) via the manifest + // path with an unflagged-but-valid empty active; + // after (3) it is the steady state. Every window + // self-heals without operator action. + initial.syncHeader(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new MmapSegmentException( + "could not sync fresh SF segment directory " + sfDir); + } + initialManifest = SfManifest.create(filesFacade, sfDir, 0L, 0L); + initial.markManifestRequired(); + } + ringInProgress = new SegmentRing(initial, segmentSizeBytes, initialManifest); + initialManifest = null; } catch (Throwable t) { initial.close(); + if (initialManifest != null) { + initialManifest.close(); + SfManifest.removeFile(filesFacade, sfDir); + } if (initialPath != null) { - Files.remove(initialPath); + filesFacade.remove(initialPath); } throw t; } @@ -1247,17 +1280,24 @@ private static long segmentCleanupRank(String name) { } /** - * Unlinks every {@code .sfa} file under {@code dir}. Called only on - * clean shutdown when the ring confirms every published FSN has been - * acked — at that moment the slot has no recoverable work and the - * files are pure noise that would mislead the next sender's recovery. + * Unlinks every {@code .sfa} file under {@code dir}, then the SF + * manifest. Called only on clean shutdown when the ring confirms every + * published FSN has been acked — at that moment the slot has no + * recoverable work and the files are pure noise that would mislead the + * next sender's recovery. *

    - * Removal runs in ascending segment order and STOPS at the first - * failure, so whatever survives (a failure here, or a crash mid-loop) - * is always a contiguous top slice of the ring: recovery's - * FSN-contiguity check still passes, and the retained ack watermark - * (== the final acked FSN == the highest frame on disk) still covers - * every surviving frame, so a successor replays nothing. + * Crash safety comes from a single durable manifest update committed + * BEFORE the first unlink: collapsing the boundaries to + * {@code head == active} declares every frame below the active base + * trimmed, so any subset of surviving files recovers cleanly (stale + * files are discarded, a surviving active recovers as the whole chain, + * zero files with a leftover manifest is the recognized drain window + * and recovers as EMPTY). Removal still runs in ascending segment order + * and STOPS at the first failure; the retained ack watermark (== the + * final acked FSN) covers every surviving frame, so a successor replays + * nothing. Legacy slots without a manifest keep the pre-manifest + * argument: ascending-order removal preserves a contiguous top slice + * that passes FSN-contiguity. * * @return {@code true} only when enumeration succeeded and every * {@code .sfa} file was confirmed removed — the caller keeps the ack @@ -1300,13 +1340,54 @@ private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b)); return byRank != 0 ? byRank : a.compareTo(b); }); - for (int i = 0, n = names.size(); i < n; i++) { - String path = dir + "/" + names.get(i); - if (!filesFacade.remove(path)) { - LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " - + "residual files stay a contiguous, watermark-covered range", path); + // Manifest-aware drain protocol. Before removing anything, durably + // collapse the committed boundaries to head == active. From that + // moment EVERY crash state is a valid recovery input: + // - surviving data below the active base is "stale below head" + // (acked by definition here) and is discarded by recovery; + // - a surviving active-base segment recovers as the whole chain; + // - empty spares are validated extras; + // - zero segment files with the manifest still present is the + // recognized drain window and recovers as EMPTY. + // Without this barrier, deleting the file the manifest calls "head" + // would make the next startup fail on a slot that lost nothing. + // The manifest itself is removed last, after every segment unlink. + SfManifest manifest = null; + try { + try { + manifest = SfManifest.open(filesFacade, dir); + } catch (Throwable t) { + LOG.warn("close-time unlink could not read the SF manifest in {}; leaving " + + "all files for the next recovery", dir, t); return false; } + if (manifest != null && !names.isEmpty()) { + try { + long activeBase = manifest.activeBase(); + manifest.update(activeBase, activeBase); + } catch (Throwable t) { + LOG.warn("close-time unlink could not commit drain boundaries in {}; " + + "leaving all files for the next recovery", dir, t); + return false; + } + } + for (int i = 0, n = names.size(); i < n; i++) { + String path = dir + "/" + names.get(i); + if (!filesFacade.remove(path)) { + LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + + "residual files stay a watermark-covered, manifest-consistent range", path); + return false; + } + } + } finally { + if (manifest != null) { + manifest.close(); + } + } + if (!SfManifest.removeFile(filesFacade, dir)) { + // Safe to proceed: a manifest with zero segment files is the + // recognized drain window and recovers as EMPTY. + LOG.warn("could not remove SF manifest in {} after full segment cleanup", dir); } return true; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 226e047f..ff7a00da 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -63,10 +63,12 @@ public final class MmapSegment implements QuietCloseable { public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian public static final int FRAME_HEADER_SIZE = 8; // u32 crc + u32 payloadLen public static final int HEADER_SIZE = 24; + public static final byte MANIFEST_REQUIRED_FLAG = 1; public static final byte VERSION = 1; private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); + private final FilesFacade filesFacade; private final String path; private final long sizeBytes; // memoryBacked: true when the segment buffer lives in malloc'd native @@ -106,9 +108,10 @@ public final class MmapSegment implements QuietCloseable { // recovery callers for diagnostics. Final after construction. private final long tornTailBytes; - private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes, + private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes, long baseSeq, long initialCursor, long frameCount, boolean memoryBacked, long tornTailBytes) { + this.filesFacade = filesFacade; this.path = path; this.fd = fd; this.mmapAddress = mmapAddress; @@ -149,9 +152,13 @@ public static MmapSegment create(String path, long baseSeq, long sizeBytes) { * that filesystem only. */ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes) { + return create(ff, path, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes, boolean manifestRequired) { long pathPtr = ff.allocNativePath(path); try { - return create(ff, pathPtr, path, baseSeq, sizeBytes); + return create(ff, pathPtr, path, baseSeq, sizeBytes, manifestRequired); } finally { ff.freeNativePath(pathPtr); } @@ -166,13 +173,17 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long * per-call {@code byte[]} + native-malloc the way the String overload does. */ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) { + return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes, boolean manifestRequired) { if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) { throw new IllegalArgumentException( "sizeBytes too small for header + one minimal frame: " + sizeBytes); } - int fd = ff.openCleanRW(pathPtr); + int fd = ff.openRWExclusive(pathPtr); if (fd < 0) { - throw new MmapSegmentException("openCleanRW failed for " + displayPath); + throw new MmapSegmentException("exclusive create failed for " + displayPath); } // Reserve real disk blocks and advance EOF to sizeBytes in one // call. ENOSPC surfaces here, before the producer thread starts @@ -190,21 +201,21 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + displayPath); } // Header goes straight into the mapping — no separate write syscall. Unsafe.getUnsafe().putInt(addr, FILE_MAGIC); Unsafe.getUnsafe().putByte(addr + 4, VERSION); - Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags + Unsafe.getUnsafe().putByte(addr + 5, manifestRequired ? MANIFEST_REQUIRED_FLAG : (byte) 0); // flags Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); + return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // mmap (or header writes) failed after a successful allocate — @@ -238,7 +249,7 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { Unsafe.getUnsafe().putShort(addr + 6, (short) 0); Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); + return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); } catch (Throwable t) { Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT); throw t; @@ -284,7 +295,11 @@ public static MmapSegment openExisting(String path) { public static MmapSegment openExisting(FilesFacade ff, String path) { long fileSize = ff.length(path); if (fileSize < HEADER_SIZE) { - throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize); + // Corruption, not an operational error: the bytes themselves prove + // this cannot be a whole segment (a create() is never durable at a + // sub-header size — allocate() reserves the full extent up front). + throw new MmapSegmentCorruptionException( + "file shorter than header: " + path + " size=" + fileSize); } int fd = ff.openRW(path); if (fd < 0) { @@ -292,17 +307,22 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + path); } int magic = Unsafe.getUnsafe().getInt(addr); if (magic != FILE_MAGIC) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); } byte version = Unsafe.getUnsafe().getByte(addr + 4); if (version != VERSION) { + // Deliberately NOT the corruption subtype: an unsupported + // version is a well-formed segment written by a different + // client build (e.g. after a downgrade). Quarantine-renaming + // it would strand its frames for the writer that CAN read it; + // failing recovery keeps the slot intact for that writer. throw new MmapSegmentException("unsupported version in " + path + ": " + version); } long baseSeq = Unsafe.getUnsafe().getLong(addr + 8); @@ -314,7 +334,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // checks (which would place the segment last in baseSeq order // and trip the FSN-gap throw, taking the whole recovery down). if (baseSeq < 0L) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "bad baseSeq in " + path + ": " + baseSeq); } long lastGood = scanFrames(addr, fileSize); @@ -328,10 +348,10 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { + "Investigate disk health or unexpected writer crash.", path, tornTail, lastGood, fileSize, count); } - return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); + return new MmapSegment(ff, path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // The header reads above (magic/version/baseSeq) run before @@ -347,7 +367,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // covers the header block and any future reader placed ahead of the // scan. if (isMmapAccessFault(t)) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "unreadable mapped header page in " + path + " (unbacked/sparse page 0): " + t.getMessage(), t); } @@ -379,16 +399,37 @@ public void close() { if (memoryBacked) { Unsafe.free(mmapAddress, sizeBytes, MemoryTag.NATIVE_DEFAULT); } else { - Files.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); + filesFacade.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); } mmapAddress = 0; } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); fd = -1; } } + boolean manifestRequired() { + return !memoryBacked && (Unsafe.getUnsafe().getByte(mmapAddress + 5) & MANIFEST_REQUIRED_FLAG) != 0; + } + + void markManifestRequired() { + if (memoryBacked || manifestRequired()) { + return; + } + Unsafe.getUnsafe().putByte(mmapAddress + 5, MANIFEST_REQUIRED_FLAG); + syncHeader(); + } + + void syncHeader() { + if (memoryBacked) { + return; + } + if (filesFacade.msync(mmapAddress, HEADER_SIZE, false) != 0 || filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment header " + path); + } + } + public boolean isFull() { return capacityRemaining() <= 0; } @@ -402,7 +443,7 @@ public void msync() { if (memoryBacked) return; // no on-disk pages to flush long pub = publishedCursor; if (pub > HEADER_SIZE) { - Files.msync(mmapAddress, pub, false); + filesFacade.msync(mmapAddress, pub, false); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java new file mode 100644 index 00000000..34a89f38 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Positively-identified segment corruption: the file's own bytes prove it is + * not (or no longer) a readable SF segment — truncated below the fixed header, + * wrong magic, a negative {@code baseSeq}, or an unreadable (unbacked/torn) + * header page. Distinct from its parent {@link MmapSegmentException}, which + * recovery treats as an operational failure (open/mmap error on a file + * whose contents may be perfectly intact) and must therefore be fatal. + *

    + * Recovery quarantines corruption (rename to {@code .corrupt}) and + * relies on manifest boundaries / FSN contiguity to decide whether the + * quarantined file was load-bearing; operational failures always abort + * startup so a transient {@code EMFILE}/{@code ENOMEM} can never silently + * drop durable frames. + */ +public final class MmapSegmentCorruptionException extends MmapSegmentException { + + public MmapSegmentCorruptionException(String message) { + super(message); + } + + public MmapSegmentCorruptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index eec0c0d9..e54eecb5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -29,8 +29,14 @@ * too short for header, etc. Indicates the segment is unusable, not that * the disk is full (the latter surfaces as backpressure on the producer * via {@link io.questdb.client.cutlass.qwp.client.LineSenderException}). + *

    + * Recovery distinguishes two flavors: this base type marks operational + * failures (open/mmap/enumeration errors — the file's contents may be fine, + * so recovery must fail closed), while the {@link MmapSegmentCorruptionException} + * subtype marks positively-identified corruption in the file's own + * bytes, which recovery may quarantine instead of aborting. */ -public final class MmapSegmentException extends RuntimeException { +public class MmapSegmentException extends RuntimeException { public MmapSegmentException(String message) { super(message); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 6bb617ad..5319ee65 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -808,12 +808,19 @@ private boolean serviceRing0(RingEntry e) { // rotation. spare = MmapSegment.create(filesFacade, pathScratch.ptr(), path, - e.ring.nextSeqHint(), segmentSizeBytes); + e.ring.nextSeqHint(), segmentSizeBytes, true); } Runnable installHook = beforeInstallSyncHook; if (installHook != null) { installHook.run(); } + if (!memoryMode) { + spare.syncHeader(); + if (filesFacade.fsyncDir(e.dir) != 0) { + throw new MmapSegmentException( + "could not sync hot-spare directory " + e.dir); + } + } // Install + commit atomically under the manager lock. // If `e.ring` was deregistered between the snapshot // above and now, abandoning the spare here is the only @@ -844,14 +851,15 @@ private boolean serviceRing0(RingEntry e) { } catch (Throwable ignored) { } } - // Remove the file even when spare is null (i.e. when - // MmapSegment.create itself threw): MmapSegment.create's - // catch already best-effort removes, but if anything - // before mmap (e.g. an exception thrown by the JVM - // between openCleanRW and the try block) leaves a file - // on disk, this is the second-line defense. Repeated - // unlink on an already-removed path is a harmless no-op. - if (path != null) { + // Only remove the file when the spare object exists, i.e. + // MmapSegment.create succeeded and ownership is ours but + // installation was rejected (ring deregistered/closed). + // When create() itself threw, its catch already removed + // anything it put on disk -- and with exclusive create + // (O_EXCL) a failure can also mean the path was ALREADY + // occupied by a file some other lifecycle owns, which a + // blanket unlink here would destroy. + if (path != null && spare != null) { filesFacade.remove(path); } } @@ -964,6 +972,13 @@ private boolean serviceRing0(RingEntry e) { MmapSegment next = e.ring.nextSealedAfter(segment); String path = segment.path(); try { + // Durably commit the head advance past this segment before + // unlinking it. The ring recomputes the successor under its + // own monitor -- computing it here from `next` would race + // with a concurrent rotation sealing the active, letting the + // head leapfrog a still-unacked sealed segment (whose file a + // later recovery would then discard as "stale below head"). + e.ring.advanceManifestHeadPast(segment); segment.close(); // A retry after a post-unlink directory-sync failure sees an // already absent path. Treat absence as the prior successful diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8f5fddbb..3ad376f3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; @@ -77,6 +78,7 @@ public final class SegmentRing implements QuietCloseable { // Test-only operation count for deterministic trim-complexity assertions. private static long trimMovedReferences; private final long maxBytesPerSegment; + private final SfManifest manifest; // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time // (the segment manager). For now, both sides synchronize via the single- @@ -129,10 +131,15 @@ public final class SegmentRing implements QuietCloseable { * frameCount == 0); typically supplied by the segment manager at startup. */ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { + this(initialActive, maxBytesPerSegment, null); + } + + SegmentRing(MmapSegment initialActive, long maxBytesPerSegment, SfManifest manifest) { if (initialActive == null) { throw new IllegalArgumentException("initialActive must not be null"); } this.active = initialActive; + this.manifest = manifest; this.maxBytesPerSegment = maxBytesPerSegment; // 3/4 of capacity gives the manager a full quarter-segment of producer // runway before backpressure kicks in. Long math, no float, no alloc. @@ -142,195 +149,488 @@ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { } /** - * Recovers a ring from segments already on disk in {@code sfDir}. Used at - * sender startup when the user's previous session left durable but - * not-yet-acked frames behind. Walks every {@code *.sfa} file in the - * directory, opens each via {@link MmapSegment#openExisting}, and - * arranges them by baseSeq: - *

      - *
    • Highest-baseSeq segment becomes the active (further appends land - * there until it fills, at which point normal rotation kicks in).
      • - *
    • All others become sealed segments awaiting ACK and trim.
      • - *
    - * Returns {@code null} if the directory is empty or contains no - * recognizable {@code .sfa} files -- the caller should then construct a - * fresh ring with {@link #SegmentRing(MmapSegment, long)} and a freshly - * created initial segment. - *

    - * Recovery is best-effort: a single bad-magic file is silently skipped - * (logged-then-ignored is the right call here; a stray unrelated file in - * the SF dir shouldn't take the whole sender down). A failure to open - * an otherwise-valid segment IS fatal -- the caller's data integrity - * depends on every segment being readable. + * Compatibility wrapper using the production facade. New startup code uses + * {@link #recover(FilesFacade, String, long)} so EMPTY is explicit. */ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) { - if (!Files.exists(sfDir)) { - return null; + return openExisting(FilesFacade.INSTANCE, sfDir, maxBytesPerSegment); + } + + /** + * Facade-aware variant of {@link #openExisting(String, long)}: every + * filesystem touch (enumeration, open, mmap, quarantine rename, manifest + * I/O) goes through {@code filesFacade} so recovery's fail-closed behavior + * can be fault-injected in tests. Returns {@code null} when the slot holds + * nothing recoverable; throws {@link MmapSegmentException} when the slot's + * state cannot be proven safe. + */ + public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + Recovery recovery = recover(filesFacade, sfDir, maxBytesPerSegment); + return recovery.status == RecoveryStatus.EMPTY ? null : recovery.ring; + } + + /** + * Exhaustively discovers and validates an SF slot without mutation. Only + * after enumeration, opens, CRC scans, contiguity and manifest boundaries + * all succeed does it migrate a legacy chain or discard validated spares. + */ + static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + if (!filesFacade.exists(sfDir)) { + return Recovery.empty(); } - ObjList opened = new ObjList<>(); - long find = Files.findFirst(sfDir); + ObjList names = new ObjList<>(); + long find = filesFacade.findFirst(sfDir); if (find < 0) { - LOG.warn("openExisting could not enumerate {} - treating as empty, " - + "but this may indicate a permission or transient error", sfDir); - return null; - } - if (find == 0) { - return null; + throw new MmapSegmentException("could not enumerate SF directory " + sfDir); } - // Outer try-catch: anything escaping the recovery body -- IOOBE from - // ObjList growth, OOM from native mmap during MmapSegment.openExisting, - // unforeseen RuntimeException from the contiguity check, etc. -- must - // not leave fds + mmaps owned by `opened` orphaned. Close every - // recovered segment and rethrow so the engine surfaces the failure. - try { + if (find > 0) { + int rc = 1; try { - int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); + String name = Files.utf8ToString(filesFacade.findName(find)); if (name != null && name.endsWith(".sfa")) { - String path = sfDir + "/" + name; - MmapSegment seg = null; - try { - seg = MmapSegment.openExisting(path); - // Filter out empty leftovers -- typically hot-spare - // segments the manager pre-allocated for a prior - // session that never got rotated into active. They - // carry the provisional baseSeq=0 and frameCount=0, - // which would otherwise collide with the real - // baseSeq=0 segment and trip the contiguity check - // below. No data to recover; close and unlink. - // Without the unlink the file persists across crash - // cycles and the disk leak compounds with every - // unclean shutdown. - // - // CAUTION: only unlink when the file is genuinely - // empty past the header. If frame[0] failed CRC - // (bit-rot, partial-page-write at crash, etc.) but - // valid frames followed, scanFrames returns - // lastGood=HEADER_SIZE and frameCount=0 -- yet - // tornTailBytes is non-zero. Treating that as - // "empty hot-spare" would silently destroy every - // surviving frame. Quarantine to .corrupt - // instead so a postmortem can recover what's left. - if (seg.frameCount() == 0) { - long torn = seg.tornTailBytes(); - seg.close(); - seg = null; - if (torn > 0) { - Files.rename(path, path + ".corrupt"); - } else { - Files.remove(path); - } - } else { - opened.add(seg); - seg = null; - } - } catch (MmapSegmentException t) { - // Per-file data error (bad magic, bad header, - // unsupported version, mmap rejection on this one - // file). Don't take down recovery for one corrupt - // .sfa -- log and skip so siblings still recover. - // Resource exhaustion (OOM) and programmer errors - // (IOOBE) deliberately propagate to the outer - // catch, which closes every already-recovered - // segment and rethrows: continuing the loop after - // an OOM would just fail again on the next file - // while silently leaking the segments we managed - // to recover before it. - LOG.warn("openExisting: skipping {} -- {}", path, t.toString()); - } finally { - // Close any seg whose ownership wasn't transferred - // (either to opened, or via the empty-branch close - // above). Fires on a propagating throw between - // open and transfer -- most importantly an OOM - // from ObjList.add growing its backing array - // after the mmap+fd were already acquired. - if (seg != null) { - try { - seg.close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing in-flight segment {}", - path, closeErr); - } - } - } + names.add(name); } - rc = Files.findNext(find); + rc = filesFacade.findNext(find); + } + if (rc < 0) { + throw new MmapSegmentException("could not fully enumerate SF directory " + sfDir); } } finally { - Files.findClose(find); + filesFacade.findClose(find); } - if (opened.size() == 0) { - return null; + } + + ObjList all = new ObjList<>(); + // Files whose own bytes prove corruption (bad magic, sub-header size, + // negative baseSeq, unreadable header page). They are excluded from + // the chain and quarantined to .corrupt — but only AFTER the + // surviving chain validates (or resolves to EMPTY), so a failed + // recovery never mutates the slot. Whether a quarantined file was + // load-bearing is decided by the manifest-boundary / contiguity + // checks below, not by the skip itself. Operational open errors + // (EMFILE, EACCES, mmap rejection, unsupported version) are NOT in + // this bucket: they throw the plain MmapSegmentException type and + // abort recovery, because the underlying file may be perfectly + // intact and silently dropping it could lose durable frames. + ObjList corruptPaths = null; + SfManifest manifest = null; + try { + for (int i = 0, n = names.size(); i < n; i++) { + String path = sfDir + "/" + names.get(i); + try { + all.add(MmapSegment.openExisting(filesFacade, path)); + } catch (MmapSegmentCorruptionException e) { + LOG.warn("recovery: {} is not a readable SF segment; excluding it and " + + "deferring quarantine until the surviving chain validates -- {}", + path, e.toString()); + if (corruptPaths == null) { + corruptPaths = new ObjList<>(); + } + corruptPaths.add(path); + } catch (MmapSegmentException e) { + throw new MmapSegmentException("recovery failed for recognized segment " + path, e); + } } - // Sort by baseSeq ascending. Worst-case segment count is - // sf_max_total_bytes / sf_max_bytes -- at the documented ceiling - // (1 TiB / 64 MiB) that is ~16K entries, where an O(N²) sort spends - // multiple seconds in compares + shifts before the I/O thread can - // start. In-place quicksort with median-of-three pivot keeps the - // no-allocation discipline of the surrounding code; median-of-three - // is required because readdir on many filesystems returns entries - // in lexicographic (== baseSeq-hex) order and a naive first-element - // pivot would degrade back to O(N²) on exactly that common case. - sortByBaseSeq(opened, 0, opened.size()); - // Sanity: the recovered segments must form a contiguous FSN range. - // Detect gaps so they don't silently produce duplicate or missing - // FSNs after recovery. A gap means a segment went missing (a - // manual deletion) or a sealed segment under-recovered -- its tail - // was cut short by a sparse/unbacked page or a mid-file media error - // (bad sector), the same class of fault scanFrames tolerates on the - // active segment but which corrupts the range on a sealed one. - for (int i = 1, n = opened.size(); i < n; i++) { - MmapSegment prev = opened.get(i - 1); - MmapSegment curr = opened.get(i); - long expected = prev.baseSeq() + prev.frameCount(); - if (curr.baseSeq() != expected) { - throw new MmapSegmentException( - "FSN gap in recovered segments: prev baseSeq=" + prev.baseSeq() - + " frameCount=" + prev.frameCount() - + " expected next baseSeq=" + expected - + " but got " + curr.baseSeq() - + " -- a segment was deleted, or a sealed segment's tail was" - + " truncated (sparse/unbacked page or disk media error);" - + " check disk health"); + manifest = SfManifest.open(filesFacade, sfDir); + if (all.size() == 0) { + if (corruptPaths != null) { + // Nothing valid survived. With a manifest this is still a + // hole we can prove (boundaries reference segments that are + // now unreadable) -- fail without mutating. Without one, + // legacy semantics apply: quarantine and start fresh. + if (manifest != null) { + throw new MmapSegmentException("every SF segment in " + sfDir + + " is corrupt but " + SfManifest.FILE_NAME + + " references durable data"); + } + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); } + if (manifest != null) { + // No .sfa files at all. Two legitimate protocols produce + // this: the close-time drain unlinks the last segment + // before it removes the manifest, and a fresh-start crash + // can leave a boundary-less (0,0) manifest behind. In + // both cases nothing recoverable exists, so accept EMPTY + // -- but shout, because a manual wipe of segment files + // looks identical and the operator should know. + LOG.warn("SF manifest exists in {} with no segment files " + + "(clean-drain or fresh-start crash window, or manual " + + "segment removal); discarding it and starting fresh", sfDir); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + } + return Recovery.empty(); } - // Publish the immutable in-memory successor chain before exposing - // the recovered ring. The final sealed segment links to the active. - for (int i = 1, n = opened.size(); i < n; i++) { - opened.get(i - 1).linkSuccessor(opened.get(i)); + + ObjList data = new ObjList<>(); + boolean requiresManifest = false; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + requiresManifest |= segment.manifestRequired(); + if (segment.frameCount() > 0) { + data.add(segment); + } } - // The newest segment becomes the active. Even if it's full, that's OK: - // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager - // installs a hot spare, the producer rotates. Same fast path as a - // mid-life ring. - int last = opened.size() - 1; - MmapSegment active = opened.get(last); - opened.remove(last); - SegmentRing ring = new SegmentRing(active, maxBytesPerSegment); - // Older segments become sealed in baseSeq order. - for (int i = 0, n = opened.size(); i < n; i++) { - ring.sealedSegments.add(opened.get(i)); + sortByBaseSeq(data, 0, data.size()); + if (manifest == null && requiresManifest) { + throw new MmapSegmentException("new-format SF segment exists but " + + SfManifest.FILE_NAME + " is missing"); } - return ring; + + MmapSegment active; + ObjList chain = new ObjList<>(); + long headBase; + long activeBase; + if (manifest == null) { + if (data.size() > 0) { + validateContiguous(data); + for (int i = 0, n = data.size(); i < n; i++) { + chain.add(data.get(i)); + } + active = chain.get(chain.size() - 1); + headBase = chain.get(0).baseSeq(); + activeBase = active.baseSeq(); + } else { + active = chooseEmptyInitial(all, sfDir); + if (active == null) { + // Legacy slot holding only empty leftovers, every one + // of them torn (a clean empty would have been chosen). + // Nothing recoverable: quarantine the torn evidence, + // drop the clean debris, start fresh -- exactly the + // pre-manifest behavior. + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove empty SF leftover {}", path); + } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); + } + chain.add(active); + headBase = active.baseSeq(); + activeBase = active.baseSeq(); + } + manifest = SfManifest.create(filesFacade, sfDir, headBase, activeBase); + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); + } + } else { + headBase = manifest.headBase(); + activeBase = manifest.activeBase(); + for (int i = 0, n = data.size(); i < n; i++) { + MmapSegment segment = data.get(i); + long end = segment.baseSeq() + segment.frameCount(); + if (segment.baseSeq() < headBase) { + if (end > headBase) { + throw new MmapSegmentException("segment overlaps committed SF head boundary"); + } + continue; // acknowledged stale file after manifest-before-unlink crash + } + if (segment.baseSeq() > activeBase) { + throw new MmapSegmentException("segment exists beyond committed SF active boundary"); + } + chain.add(segment); + } + if (chain.size() > 0) { + validateContiguous(chain); + if (chain.get(0).baseSeq() != headBase) { + throw new MmapSegmentException("missing expected SF head segment at base " + headBase); + } + } + active = findActive(all, activeBase); + if (active == null) { + if (chain.size() == 0 && headBase == activeBase && corruptPaths == null) { + // Clean-drain crash window: the close-time drain first + // durably collapses the boundaries to head == active + // (declaring every frame acked), then unlinks segments + // in ascending order -- so dying between the active's + // unlink and the spare's/manifest's leaves exactly + // this state: no data frame anywhere, no file at the + // committed active base, only empty spares and/or + // acked stale files. Nothing recoverable exists; + // accept EMPTY and clear the debris. Guarded on + // corruptPaths because an unreadable .sfa of unknown + // identity could be the real active -- in that case + // fail closed instead of guessing. + LOG.warn("SF manifest in {} has collapsed boundaries ({}) with no " + + "segment at the active base and no recovered frames; " + + "accepting the clean-drain crash window as empty", + sfDir, activeBase); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove drained SF leftover {}", path); + } + } + all.clear(); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + return Recovery.empty(); + } + throw new MmapSegmentException("missing expected SF active segment at base " + activeBase); + } + if (chain.size() == 0) { + if (headBase != activeBase || active.frameCount() != 0 || corruptPaths != null) { + // corruptPaths guard: with an unreadable .sfa in the + // slot, the innocent-looking empty at the active base + // could be a leftover spare coincidentally carrying + // the same provisional baseSeq as a corrupted real + // active -- accepting it would quarantine unacked + // frames and re-issue their FSNs. Fail closed. + throw new MmapSegmentException( + "missing SF chain between committed boundaries" + + (corruptPaths != null + ? " (a corrupt segment prevents proving the empty state)" : "")); + } + chain.add(active); + } else if (chain.get(chain.size() - 1) != active) { + MmapSegment tail = chain.get(chain.size() - 1); + long chainEnd = tail.baseSeq() + tail.frameCount(); + if (corruptPaths == null && active.frameCount() == 0 && active.baseSeq() == chainEnd) { + // Rotation committed (manifest fsync'd, promoted spare's + // header synced) but the process/OS died before a single + // frame of the new active reached disk: the sealed chain + // ends exactly where the empty active begins. Legal + // crash state -- resume appending into it. Refused when + // corrupt segments exist (same stand-in hazard as the + // empty-chain acceptance above). + chain.add(active); + } else { + throw new MmapSegmentException( + "missing expected SF active/tail segment at base " + activeBase); + } + } + for (int i = 0, n = chain.size() - 1; i < n; i++) { + if (chain.get(i).tornTailBytes() > 0) { + throw new MmapSegmentException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); + } + } + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); + } + } + + for (int i = 1, n = chain.size(); i < n; i++) { + chain.get(i - 1).linkSuccessor(chain.get(i)); + } + SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest); + manifest = null; + for (int i = 0, n = chain.size() - 1; i < n; i++) { + ring.sealedSegments.add(chain.get(i)); + } + // Ownership of the chain transferred. Clean up only validated + // extras; recovery is already successful, so cleanup failure + // must never turn startup into a partially-mutating error or + // orphan the constructed ring -- swallow and let the next + // startup re-examine the leftovers. Extras with a torn tail + // carry evidence of attempted writes -- keep the bytes under a + // .corrupt name instead of unlinking them. + try { + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (!containsIdentity(chain, segment)) { + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove validated stale/empty SF segment {}", path); + } + } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + } catch (Throwable cleanupError) { + LOG.warn("post-recovery cleanup failed; leftover files will be " + + "re-examined on the next startup", cleanupError); + } + return Recovery.recovered(ring); } catch (Throwable t) { - // Close every recovered MmapSegment that's still in `opened`. - // After the success path, `opened` no longer contains the active - // segment (removed above), but the sealed segments transferred to - // ring.sealedSegments are still owned by the ring once it's - // returned -- so this catch only fires before the return statement. - for (int i = 0, n = opened.size(); i < n; i++) { + for (int i = 0, n = all.size(); i < n; i++) { try { - opened.get(i).close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing recovered segment during cleanup", - closeErr); + all.get(i).close(); + } catch (Throwable closeError) { + LOG.warn("error closing SF segment after recovery failure", closeError); } } + if (manifest != null) { + manifest.close(); + } throw t; } } + /** + * Durably advances the manifest head past {@code trimming} (the sealed + * segment the manager is about to unlink). The successor and the current + * active are both read under the ring monitor, so a concurrent rotation + * (which also mutates the manifest under this monitor) can never make the + * head leapfrog a still-live sealed segment: if rotation sealed the old + * active after the caller's snapshot, {@code trimming.successor()} now + * points at that sealed segment, not at the new active. + */ + synchronized void advanceManifestHeadPast(MmapSegment trimming) { + if (manifest == null) { + return; + } + MmapSegment successor = trimming.successor(); + long newHeadBase = (successor == null || successor == active) + ? active.baseSeq() + : successor.baseSeq(); + manifest.update(newHeadBase, active.baseSeq()); + } + + /** + * Picks the clean (untorn) empty segment to reuse as a legacy slot's + * initial active, preferring {@code sf-initial.sfa}. Returns {@code null} + * when no clean empty exists; torn empties are never reused here because + * their bytes are quarantine evidence, not blank space. + */ + private static MmapSegment chooseEmptyInitial(ObjList all, String sfDir) { + String initialPath = sfDir + "/sf-initial.sfa"; + MmapSegment selected = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.frameCount() != 0 || segment.tornTailBytes() > 0) { + continue; + } + if (selected == null || initialPath.equals(segment.path())) { + selected = segment; + } + } + return selected; + } + + /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */ + private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) { + if (corruptPaths == null) { + return; + } + for (int i = 0, n = corruptPaths.size(); i < n; i++) { + quarantineFile(filesFacade, corruptPaths.get(i)); + } + } + + private static void quarantineFile(FilesFacade filesFacade, String path) { + if (filesFacade.rename(path, path + ".corrupt") != 0) { + LOG.warn("could not quarantine {} to {}.corrupt; it will be re-examined " + + "on the next recovery", path, path); + } + } + + private static boolean containsIdentity(ObjList list, MmapSegment value) { + for (int i = 0, n = list.size(); i < n; i++) { + if (list.get(i) == value) return true; + } + return false; + } + + /** + * Locates the segment the manifest's {@code activeBase} refers to. + * Preference order among same-base candidates: + *

      + *
    1. a segment with recovered frames (the durable chain tail);
      2. + *
    2. an empty segment with a torn tail (the promoted active whose + * first frame write was cut short — an attempted write marks it as + * the one rotation actually exposed);
      3. + *
    3. a clean empty segment.
      4. + *
    + * Multiple equivalent empties at the same base are NOT an error: a fresh + * start or a rotation crash routinely leaves both the initial/promoted + * segment and a provisioned hot spare carrying the same provisional + * baseSeq. They are interchangeable blanks — pick one deterministically + * and let the extras cleanup discard the rest. Bricking startup on this + * state would turn every "kill -9 shortly after start" into a manual + * repair. + */ + private static MmapSegment findActive(ObjList all, long activeBase) { + MmapSegment tornEmpty = null; + MmapSegment cleanEmpty = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.baseSeq() != activeBase) { + continue; + } + if (segment.frameCount() > 0) { + return segment; + } + if (segment.tornTailBytes() > 0) { + if (tornEmpty == null) { + tornEmpty = segment; + } + } else if (cleanEmpty == null) { + cleanEmpty = segment; + } + } + return tornEmpty != null ? tornEmpty : cleanEmpty; + } + + private static void validateContiguous(ObjList segments) { + for (int i = 1, n = segments.size(); i < n; i++) { + MmapSegment previous = segments.get(i - 1); + MmapSegment current = segments.get(i); + long expected = previous.baseSeq() + previous.frameCount(); + if (current.baseSeq() != expected) { + throw new MmapSegmentException("FSN gap in recovered segments: expected " + + expected + " but got " + current.baseSeq()); + } + } + } + + static final class Recovery { + private final SegmentRing ring; + private final RecoveryStatus status; + + private Recovery(RecoveryStatus status, SegmentRing ring) { + this.status = status; + this.ring = ring; + } + + static Recovery empty() { + return new Recovery(RecoveryStatus.EMPTY, null); + } + + SegmentRing ring() { + return ring; + } + + static Recovery recovered(SegmentRing ring) { + return new Recovery(RecoveryStatus.RECOVERED, ring); + } + + RecoveryStatus status() { + return status; + } + } + + enum RecoveryStatus { + EMPTY, + RECOVERED + } + /** * Highest FSN that the server has ACK'd. Read by the segment manager to * decide which sealed segments are safe to munmap + unlink. @@ -396,11 +696,45 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { MmapSegment previous = active; long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); + if (manifest != null) { + // Make the spare's rebased identity durable BEFORE the manifest + // references it. Without this barrier an OS crash could leave a + // durable manifest pointing at baseSeq=actualBase while the + // spare's on-disk header still carries the manager's + // provisional guess -- recovery would then find no segment at + // the committed active boundary and fail a startup that lost + // nothing. One msync per rotation, amortized over a whole + // segment of appends; runs outside the monitor because the + // spare is not yet visible to any other thread. + // + // Deliberately NOT msync'd here: the sealed predecessor's + // data pages. A power loss can therefore tear the sealed + // tail after the boundary is committed, and recovery will + // fail closed on chainEnd != activeBase. That is the + // intended semantics -- page-level durability of frame data + // follows the sender's opt-in msync cadence, and recovery + // must refuse to guess when the two disagree. + spare.syncHeader(); + } // Publish the successor before the volatile active promotion. The // same monitor protects the sealed list and nextSealedAfter's trim // fallback, while the volatile link also remains readable from a // current segment after the manager removes and closes it. synchronized (this) { + if (manifest != null) { + // Inside the monitor: serialized with the trim path's + // advanceManifestHeadPast so neither writer publishes a + // boundary computed from a state the other has already + // moved past (SfManifest additionally clamps monotonic). + // BEFORE any ring mutation: if the manifest fsync throws, + // the rotation never happened -- previous stays active, + // the spare stays installed, and the producer's retry + // re-runs this block from a consistent state. + long headBase = sealedHead < sealedSegments.size() + ? sealedSegments.get(sealedHead).baseSeq() + : previous.baseSeq(); + manifest.update(headBase, actualBase); + } previous.linkSuccessor(spare); sealedSegments.add(previous); active = spare; @@ -461,6 +795,9 @@ public synchronized void close() { } sealedSegments.clear(); sealedHead = 0; + if (manifest != null) { + manifest.close(); + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java new file mode 100644 index 00000000..f09ce94a --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -0,0 +1,282 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Crash-safe boundary record for an SF segment chain. Two fixed-size, + * independently CRC-protected records alternate on update. Recovery selects + * the valid record with the greatest generation, so a torn update cannot + * erase the previous committed head/active boundary. + */ +final class SfManifest implements QuietCloseable { + static final String FILE_NAME = "sf-manifest.bin"; + private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class); + private static final int CRC_OFFSET = 60; + private static final long FILE_SIZE = 128; + private static final int MAGIC = 0x314d4653; // SFM1 little-endian + private static final int RECORD_SIZE = 64; + private static final int VERSION = 1; + private final int fd; + private final FilesFacade filesFacade; + private final String path; + private long activeBase; + private boolean closed; + private long generation; + private long headBase; + + private SfManifest(FilesFacade filesFacade, String path, int fd, + long generation, long headBase, long activeBase) { + this.filesFacade = filesFacade; + this.path = path; + this.fd = fd; + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + + static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { + String path = dir + "/" + FILE_NAME; + int fd = filesFacade.openRWExclusive(path); + if (fd < 0) { + throw new MmapSegmentException("exclusive create failed for SF manifest " + path); + } + boolean success = false; + try { + if (!filesFacade.allocate(fd, FILE_SIZE)) { + throw new MmapSegmentException("could not allocate SF manifest " + path); + } + SfManifest manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1); + manifest.update(headBase, activeBase); + if (filesFacade.fsyncDir(dir) != 0) { + throw new MmapSegmentException("could not sync SF manifest directory " + dir); + } + success = true; + return manifest; + } finally { + if (!success) { + filesFacade.close(fd); + filesFacade.remove(path); + } + } + } + + static SfManifest open(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + if (!filesFacade.exists(path)) { + return null; + } + if (filesFacade.length(path) != FILE_SIZE) { + // A wrong-sized manifest is creation debris: create() reaches the + // full FILE_SIZE via allocate() before writing the first record, + // so a mis-sized file proves the crash happened before any + // boundary was ever committed — nothing can depend on it yet + // (segment flags are stamped only after create() returns). Treat + // as absent so startup self-heals; genuine post-creation loss is + // still caught by the manifest-required flag check. + quarantineDebris(filesFacade, path, "wrong size " + filesFacade.length(path)); + return null; + } + int fd = filesFacade.openRW(path); + if (fd < 0) { + throw new MmapSegmentException("could not open SF manifest " + path); + } + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Record first = readRecord(filesFacade, fd, buffer, 0); + Record second = readRecord(filesFacade, fd, buffer, RECORD_SIZE); + Record selected; + if (first == null) { + selected = second; + } else if (second == null || first.generation > second.generation) { + selected = first; + } else { + selected = second; + } + if (selected == null) { + // No valid record in either slot. create() makes the first + // record durable (write + fsync) before returning, and every + // later update() rewrites only ONE slot — so a torn update + // leaves the sibling record intact. Zero valid records + // therefore proves a creation crash, not boundary loss. + // Self-heal by treating the file as absent. If durable state + // DID depend on a manifest (flags stamped, i.e. double-slot + // bit rot), recovery still fails closed on the + // manifest-required flag check. + filesFacade.close(fd); + quarantineDebris(filesFacade, path, "no valid CRC-protected record"); + return null; + } + return new SfManifest(filesFacade, path, fd, selected.generation, + selected.headBase, selected.activeBase); + } catch (Throwable t) { + filesFacade.close(fd); + throw t; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + long activeBase() { + return activeBase; + } + + @Override + public void close() { + if (!closed) { + closed = true; + filesFacade.close(fd); + } + } + + long headBase() { + return headBase; + } + + /** + * Unlinks {@code dir}'s manifest file. Used when a slot is being reset to + * the "nothing durable" state (fresh-start cleanup, close-time drain, or + * recovery accepting a segment-less slot as empty). Returns {@code true} + * when the file is confirmed gone (removed, or never existed). + */ + static boolean removeFile(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + return filesFacade.remove(path) || !filesFacade.exists(path); + } + + synchronized void update(long newHeadBase, long newActiveBase) { + if (closed) { + throw new IllegalStateException("SF manifest is closed"); + } + // Committed boundaries only ever move forward: head advances on trim, + // active advances on rotation. Clamp instead of throwing because the + // two writers (producer rotation, manager trim) are serialized on the + // ring monitor but may each compute their argument from a snapshot + // the other has already moved past — e.g. rotation reads the sealed + // list while a trimmed-but-not-yet-removed head segment still sits in + // it. Regressing a durable boundary would let a later crash-recovery + // demand a segment file the trim path already unlinked (startup would + // fail on "missing head segment") or, worse, re-expose stale files + // below an already-committed head. + if (generation > 0) { + if (newHeadBase < headBase) { + newHeadBase = headBase; + } + if (newActiveBase < activeBase) { + newActiveBase = activeBase; + } + } + if (newHeadBase < 0 || newActiveBase < newHeadBase) { + throw new IllegalArgumentException("invalid SF manifest boundaries"); + } + if (generation > 0 && headBase == newHeadBase && activeBase == newActiveBase) { + return; + } + long nextGeneration = generation + 1; + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(buffer, MAGIC); + Unsafe.getUnsafe().putInt(buffer + 4, VERSION); + Unsafe.getUnsafe().putLong(buffer + 8, nextGeneration); + Unsafe.getUnsafe().putLong(buffer + 16, newHeadBase); + Unsafe.getUnsafe().putLong(buffer + 24, newActiveBase); + int crc = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + Unsafe.getUnsafe().putInt(buffer + CRC_OFFSET, crc); + long offset = (nextGeneration & 1L) * RECORD_SIZE; + if (filesFacade.write(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + throw new MmapSegmentException("short write updating SF manifest " + path); + } + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync SF manifest " + path); + } + generation = nextGeneration; + headBase = newHeadBase; + activeBase = newActiveBase; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + /** + * Moves creation-crash debris aside so a subsequent exclusive create can + * succeed. Prefers rename (keeps the bytes for postmortem); falls back to + * remove; throws when neither works — leaving the debris in place would + * wedge every subsequent {@link #create}. + */ + private static void quarantineDebris(FilesFacade filesFacade, String path, String reason) { + LOG.warn("SF manifest {} is creation-crash debris ({}); quarantining and starting " + + "from the segment files", path, reason); + if (filesFacade.rename(path, path + ".corrupt") == 0) { + return; + } + if (filesFacade.remove(path)) { + return; + } + throw new MmapSegmentException("could not quarantine invalid SF manifest " + path); + } + + private static Record readRecord(FilesFacade filesFacade, int fd, long buffer, long offset) { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + if (filesFacade.read(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + return null; + } + if (Unsafe.getUnsafe().getInt(buffer) != MAGIC + || Unsafe.getUnsafe().getInt(buffer + 4) != VERSION) { + return null; + } + int expected = Unsafe.getUnsafe().getInt(buffer + CRC_OFFSET); + int actual = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + if (expected != actual) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(buffer + 8); + long headBase = Unsafe.getUnsafe().getLong(buffer + 16); + long activeBase = Unsafe.getUnsafe().getLong(buffer + 24); + if (generation <= 0 || headBase < 0 || activeBase < headBase) { + return null; + } + return new Record(generation, headBase, activeBase); + } + + private static final class Record { + private final long activeBase; + private final long generation; + private final long headBase; + + private Record(long generation, long headBase, long activeBase) { + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + } +} diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java index 248ad61e..194d0972 100644 --- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java @@ -111,11 +111,21 @@ public int mkdir(String path, int mode) { return Files.mkdir(path, mode); } + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + @Override public int msync(long addr, long len, boolean async) { return Files.msync(addr, len, async); } + @Override + public void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { return Files.openCleanRW(path); @@ -136,6 +146,16 @@ public int openRW(long pathPtr) { return Files.openRW(pathPtr); } + @Override + public int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + @Override public long length(long pathPtr) { return Files.length(pathPtr); diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 7a4038ed..b0fdfb2e 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -154,6 +154,24 @@ public static int openRW(long pathPtr) { return openRW0(pathPtr); } + /** + * Atomically creates {@code path} for read-write access. Fails with -1 + * when the path already exists; existing bytes are never truncated. + */ + public static int openRWExclusive(String path) { + long ptr = pathPtr(path); + try { + return openRWExclusive0(ptr); + } finally { + freePathPtr(ptr); + } + } + + /** Native-path variant of {@link #openRWExclusive(String)}. */ + public static int openRWExclusive(long pathPtr) { + return openRWExclusive0(pathPtr); + } + /** * Opens {@code path} for append-only writes, creating it (mode 0644) if * absent. Every {@link #append(int, long, long)} writes at end-of-file @@ -575,6 +593,8 @@ public static void munmap(long address, long len, int memoryTag) { static native int openRW0(long lpszName); + static native int openRWExclusive0(long lpszName); + static native int openAppend0(long lpszName); static native int openCleanRW0(long lpszName); diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 60000a3b..082e680e 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -111,10 +111,18 @@ default int fsyncDir(String dir) { int mkdir(String path, int mode); + default long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + default int msync(long addr, long len, boolean async) { return Files.msync(addr, len, async); } + default void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + int openCleanRW(String path); /** @@ -130,6 +138,14 @@ default int msync(long addr, long len, boolean async) { /** Variant of {@link #openRW(String)} taking a pre-encoded native UTF-8 path pointer. */ int openRW(long pathPtr); + default int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + default int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + /** * Variant of {@code length(String)} taking a pre-encoded native UTF-8 path * pointer; same allocation-elision rationale as {@link #openRW(long)}. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 177ee5f6..cff8249b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -128,7 +128,7 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { assertTrue(expected.getMessage(), expected.getMessage().contains("pre-allocation failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); assertEquals("allocate must run exactly once", 1, ff.allocateCalls); assertEquals("fd must be closed on allocate failure", 1, ff.closeCalls); assertEquals("file must be removed on allocate failure", 1, ff.removeCalls); @@ -138,28 +138,31 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { } @Test - public void testCreateFailsCleanlyWhenOpenCleanRWReturnsMinusOne() throws Exception { + public void testCreateFailsCleanlyWhenExclusiveOpenReturnsMinusOne() throws Exception { TestUtils.assertMemoryLeak(() -> { String path = tmpDir + "/seg-noopen.sfa"; long sizeBytes = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 64; FaultyFilesFacade ff = new FaultyFilesFacade(); - ff.failOnOpenCleanRW = true; + ff.failOnOpenRWExclusive = true; try { MmapSegment.create(ff, path, 0L, sizeBytes).close(); - fail("expected MmapSegmentException from openCleanRW returning -1"); + fail("expected MmapSegmentException from openRWExclusive returning -1"); } catch (MmapSegmentException expected) { assertTrue(expected.getMessage(), - expected.getMessage().contains("openCleanRW failed")); + expected.getMessage().contains("exclusive create failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); - assertEquals("allocate must not run after openCleanRW failure", + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); + assertEquals("allocate must not run after exclusive-create failure", 0, ff.allocateCalls); assertEquals("close must not be called when no fd was opened", 0, ff.closeCalls); - assertEquals("remove must not be called when openCleanRW failed", + // With O_EXCL semantics a create failure can mean "path already + // exists and belongs to another lifecycle" -- create() must NOT + // unlink a file it never owned. + assertEquals("remove must not be called when exclusive create failed", 0, ff.removeCalls); - assertFalse("no file should exist when openCleanRW failed", + assertFalse("no file should exist when exclusive create failed", Files.exists(path)); }); } @@ -198,7 +201,7 @@ public void testCreateRepeatedAllocateFailuresDoNotAccumulateOrphans() throws Ex } assertEquals("no orphan files may survive repeated allocate failures", 0, survivors); - assertEquals(attempts, ff.openCleanRWCalls); + assertEquals(attempts, ff.openRWExclusiveCalls); assertEquals(attempts, ff.allocateCalls); assertEquals(attempts, ff.closeCalls); assertEquals(attempts, ff.removeCalls); @@ -495,9 +498,29 @@ private static final class FaultyFilesFacade implements FilesFacade { int closeCalls; boolean failOnAllocate; boolean failOnOpenCleanRW; + boolean failOnOpenRWExclusive; int openCleanRWCalls; + int openRWExclusiveCalls; int removeCalls; + @Override + public int openRWExclusive(String path) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(pathPtr); + } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java new file mode 100644 index 00000000..21116e68 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -0,0 +1,827 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +/** + * Regression tests for the SF recovery fail-open findings: directory + * enumeration errors, per-file open/mmap errors and boundary (leading / + * trailing) segment loss must fail startup without mutating the slot, while + * positively-identified corruption is quarantined only after the surviving + * chain validates. Also pins the crash-window states of the manifest + * protocol (fresh start, rotation, clean-drain) as recoverable. + */ +public class SegmentRecoveryIntegrityTest { + private static final String MANIFEST_NAME = "sf-manifest.bin"; + private static final long SEGMENT_SIZE = 64 * 1024; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-integrity-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + // ------------------------------------------------------------------ + // Operational failures must fail closed without touching a byte. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the SF directory cannot be enumerated"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testPartialFindNextFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int calls; + + @Override + public int findNext(long findPtr) { + // First advance succeeds, then the listing "fails" the way + // a readdir I/O error does. Recovery must treat the + // partial listing as fatal, not as end-of-directory. + return ++calls >= 2 ? -1 : super.findNext(findPtr); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the directory listing is incomplete"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not fully enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testOpenRWFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public int openRW(String path) { + // EMFILE/EACCES-style failure on a perfectly valid file. + return victim.equals(path) ? -1 : super.openRW(path); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be opened"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testMmapFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int victimFd = Integer.MIN_VALUE; + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (victim.equals(path)) { + victimFd = fd; + } + return fd; + } + + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + if (fd == victimFd) { + return Files.FAILED_MMAP_ADDRESS; + } + return super.mmap(fd, len, offset, flags, memoryTag); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be mapped"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testUnsupportedVersionFailsRecoveryWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String foreign = tmpDir + "/sf-0000000000000001.sfa"; + writeRawSegmentHeader(foreign, MmapSegment.FILE_MAGIC, (byte) 99, 2L); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("a segment written by a different client version must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + // NOT renamed .corrupt: the file belongs to the client build that + // can read it; recovery keeps the slot intact for that writer. + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Corruption is quarantined, but never before validation decides. + // ------------------------------------------------------------------ + + @Test + public void testCorruptStrayFileQuarantinedInLegacyDirAndSiblingsRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + String stray = tmpDir + "/zz-stray.sfa"; + writeRawSegmentHeader(stray, 0xDEADBEEF, (byte) 1, 0L); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("valid chain must recover around a corrupt stray file", ring); + try { + Assert.assertEquals("all five durable frames must be recovered", + 4, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertFalse("corrupt stray must be quarantined away from the .sfa scan", + Files.exists(stray)); + Assert.assertTrue("quarantine preserves the bytes as evidence", + Files.exists(stray + ".corrupt")); + Assert.assertTrue("legacy recovery must migrate the slot to the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptChainSegmentWithManifestFailsWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String seg = tmpDir + "/sf-initial.sfa"; + writeSegmentWithFrames(seg, 0, 3); + // Migrate once so the manifest exists and the segment is flagged. + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + // Bit-rot the magic in place. + overwriteInt(seg, 0, 0xBADC0DE); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("losing the only chain segment to corruption must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "corrupt"); + } + // Deferred quarantine: a FAILED recovery must not have renamed, + // deleted, or otherwise mutated anything. + assertDirUnchanged(before); + }); + } + + @Test + public void testFlaggedSegmentWithDeletedManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.remove(tmpDir + "/" + MANIFEST_NAME)); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a manifest-required segment without a manifest must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + // ------------------------------------------------------------------ + // Boundary evasion: missing leading/trailing segments must be caught. + // ------------------------------------------------------------------ + + @Test + public void testMissingActiveSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Chain [0..2) exists but the manifest says the active starts at 2. + // Pre-manifest recovery would silently promote the highest present + // segment and hand out overlapping FSNs. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing trailing/active segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + }); + } + + @Test + public void testMissingHeadSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The manifest promises a head at base 0, but only [2..5) survived. + // Pre-manifest recovery would pass contiguity on the remainder and + // silently lose the leading rows. + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing leading/head segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF head"); + } + }); + } + + @Test + public void testInteriorGapStillFailsRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000002.sfa", 5, 2); + writeManifest(1, 0, 5); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an interior FSN gap must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "FSN gap"); + } + }); + } + + // ------------------------------------------------------------------ + // Legal crash-window states must recover without operator action. + // ------------------------------------------------------------------ + + @Test + public void testFreshStartCrashWithTwoEmptyBaseZeroSegmentsRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Fresh engine start provisions sf-initial plus a hot spare, both + // empty at baseSeq 0. A process kill in that window must not brick + // the slot on "ambiguous" empties. + MmapSegment a = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + a.close(); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + MmapSegment b = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 0, SEGMENT_SIZE); + b.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("two equivalent empty segments must not brick recovery", ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + Assert.assertEquals(0, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + Assert.assertEquals("the redundant empty must have been cleaned up", + 1, countSfaFiles()); + }); + } + + @Test + public void testRotationCrashWindowEmptyActiveAtChainEndRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Crash after rotation committed (manifest fsync'd, spare header + // synced) but before any frame reached the new active: sealed + // chain [0..2) plus an empty active at base 2. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + spare.close(); + writeManifest(1, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("freshly-rotated empty active is a legal crash state", ring); + try { + Assert.assertEquals(1, ring.publishedFsn()); + Assert.assertEquals(2, ring.getActive().baseSeq()); + Assert.assertNotNull(ring.firstSealed()); + Assert.assertEquals(0, ring.firstSealed().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testDrainWindowManifestWithoutSegmentsRecoversEmptyAndRemovesManifest() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain close unlinks the last segment before the manifest; + // a crash between the two leaves this state. + writeManifest(3, 7, 9); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("segment-less slot must recover as EMPTY", ring); + Assert.assertFalse("the stale manifest must be discarded", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Engine crash between creating sf-initial.sfa (unflagged) and + // creating the manifest: recovers via legacy migration. + MmapSegment initial = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + initial.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("legacy migration must create the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + // ------------------------------------------------------------------ + // Engine level: enumeration failure must not truncate the durable log. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureCannotCreateInitialSegment() throws Exception { + TestUtils.assertMemoryLeak(() -> { + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + Assert.assertFalse("startup failure created sf-initial.sfa", + Files.exists(tmpDir + "/sf-initial.sfa")); + } + } finally { + manager.close(); + } + }); + } + + @Test + public void testFindFirstFailureDoesNotTruncateExistingLog() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The original failure mode: enumeration error -> treated as empty + // -> fresh start openCleanRW(O_TRUNC) destroys the durable log. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 4); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + // expected + } + } finally { + manager.close(); + } + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + /** Creates a real segment at {@code path} and appends {@code frames} 64-byte frames. */ + private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment segment = MmapSegment.create(path, baseSeq, SEGMENT_SIZE); + try { + for (int i = 0; i < frames; i++) { + for (int b = 0; b < 64; b++) { + Unsafe.getUnsafe().putByte(buf + b, (byte) (i * 31 + b)); + } + Assert.assertTrue("test frame must fit", segment.tryAppend(buf, 64) >= 0); + } + } finally { + segment.close(); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } + + /** Writes a raw 64-byte pseudo segment header (+zero padding) for corruption tests. */ + private static void writeRawSegmentHeader(String path, int magic, byte version, long baseSeq) { + int fd = Files.openCleanRW(path); + Assert.assertTrue("could not create " + path, fd >= 0); + try { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, magic); + Unsafe.getUnsafe().putByte(buf + 4, version); + Unsafe.getUnsafe().putLong(buf + 8, baseSeq); + Assert.assertEquals(64, Files.write(fd, buf, 64, 0)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** Overwrites a single int at {@code offset} in an existing file. */ + private static void overwriteInt(String path, long offset, int value) { + int fd = Files.openRW(path); + Assert.assertTrue("could not open " + path, fd >= 0); + try { + long buf = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(buf, value); + Assert.assertEquals(4, Files.write(fd, buf, 4, offset)); + } finally { + Unsafe.free(buf, 4, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Writes a valid {@code sf-manifest.bin} with one CRC-protected record, + * mirroring SfManifest's on-disk layout (two alternating 64-byte records + * in a 128-byte file; record slot = generation & 1). + */ + private void writeManifest(long generation, long headBase, long activeBase) { + String path = tmpDir + "/" + MANIFEST_NAME; + // openRW (not openCleanRW): callers may layer a second generation's + // record into the sibling slot of an existing manifest. + int fd = Files.openRW(path); + Assert.assertTrue("could not create manifest", fd >= 0); + try { + if (Files.length(path) < 128) { + Assert.assertTrue(Files.truncate(fd, 128)); + } + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, 0x314d4653); // SFM1 + Unsafe.getUnsafe().putInt(buf + 4, 1); // version + Unsafe.getUnsafe().putLong(buf + 8, generation); + Unsafe.getUnsafe().putLong(buf + 16, headBase); + Unsafe.getUnsafe().putLong(buf + 24, activeBase); + int crc = Crc32c.update(Crc32c.INIT, buf, 60); + Unsafe.getUnsafe().putInt(buf + 60, crc); + long offset = (generation & 1L) * 64; + Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Snapshot of the durable SF payload files (segments, quarantined + * segments, manifest): name -> content. Lifecycle noise such as the slot + * lock and ack watermark is deliberately excluded -- "recovery must not + * mutate the slot" is a statement about the durable log, not about lock + * bookkeeping. + */ + private Map snapshotDir() { + Map out = new HashMap<>(); + Path dir = Paths.get(tmpDir); + try (java.util.stream.Stream stream = java.nio.file.Files.list(dir)) { + stream.filter(java.nio.file.Files::isRegularFile) + .filter(p -> { + String name = p.getFileName().toString(); + return name.endsWith(".sfa") || name.endsWith(".corrupt") + || MANIFEST_NAME.equals(name); + }) + .forEach(p -> { + try { + out.put(p.getFileName().toString(), java.nio.file.Files.readAllBytes(p)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return out; + } + + /** Asserts the directory holds exactly the snapshotted files, byte for byte. */ + private void assertDirUnchanged(Map before) { + Map after = snapshotDir(); + Assert.assertEquals("file set must be unchanged", before.keySet(), after.keySet()); + for (Map.Entry e : before.entrySet()) { + Assert.assertArrayEquals("bytes of " + e.getKey() + " must be unchanged", + e.getValue(), after.get(e.getKey())); + } + } + + private int countSfaFiles() { + int count = 0; + for (String name : snapshotDir().keySet()) { + if (name.endsWith(".sfa")) { + count++; + } + } + return count; + } + + // ------------------------------------------------------------------ + // Manifest crash windows and record selection + // ------------------------------------------------------------------ + + @Test + public void testCreationCrashZeroByteManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // kill -9 between the manifest's O_EXCL create and its first + // durable record leaves a zero-byte file. No boundary was ever + // committed, so nothing can depend on it: startup must self-heal, + // not demand an operator delete the file. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("creation-crash manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("debris must be quarantined for postmortem", + Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue("a fresh valid manifest must replace the debris", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCreationCrashRecordlessManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Same window, later stage: allocate() completed (128 zero bytes) + // but the first record write/fsync never landed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 128)); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("record-less manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testRecordlessManifestWithFlaggedSegmentsFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Flagged segments prove a manifest create COMPLETED at some + // point, so a record-less manifest here is double-slot bit rot, + // not creation debris -- boundaries were lost. Fail closed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); // truncates + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 128)); + Files.close(fd); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("boundary loss next to flagged segments must fail closed"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + @Test + public void testDrainCrashSurvivingSpareRecoversEmpty() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain crash window: boundaries collapsed to head==active, + // the active-base file already unlinked, but an empty hot spare + // (provisional base > activeBase) survived. Everything was acked; + // this must recover as EMPTY, not brick on "missing active". + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000007.sfa", 5, SEGMENT_SIZE); + spare.close(); + writeManifest(4, 3, 3); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("drain crash window must recover as EMPTY", ring); + Assert.assertEquals("surviving spare must be cleaned up", 0, countSfaFiles()); + Assert.assertFalse("collapsed manifest must be removed", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptActiveWithSameBaseEmptyStandInFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The active (holding unacked frames) is corrupted while a + // leftover spare coincidentally carries the same base. Accepting + // the clean empty as the "rotation crash" active would quarantine + // the unacked frames and re-issue their FSNs -- recovery must + // fail closed instead. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String corruptActive = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(corruptActive, 2, 2); + overwriteInt(corruptActive, 0, 0xBADC0DE); // bit-rot the magic + MmapSegment standIn = MmapSegment.create(tmpDir + "/sf-0000000000000002.sfa", 2, SEGMENT_SIZE); + standIn.close(); + writeManifest(1, 0, 2); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an empty stand-in must not mask a corrupt active"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testManifestHigherGenerationRecordWins() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) says active=0; gen2 (slot 0) says active=2. If + // selection picked gen1, the empty at base 2 would sit beyond the + // committed active boundary; gen2 accepts it as the active. + writeManifest(1, 0, 0); + writeManifest(2, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals("the higher-generation record must win", + 2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testManifestTornNewerRecordFallsBackToOlder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) is valid and matches the segments; gen2 (slot 0) + // was torn mid-write (bad CRC). Selection must fall back to gen1 + // rather than reject the manifest or trust torn boundaries. + writeManifest(1, 0, 2); + writeManifest(2, 0, 4); + overwriteInt(tmpDir + "/" + MANIFEST_NAME, 60, 0xBADC0DE); // tear gen2's CRC (slot 0) + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("a torn newer record must fall back to the older slot", ring); + try { + Assert.assertEquals(2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + /** FilesFacade delegating everything to the production instance. */ + private static class DelegatingFacade implements FilesFacade { + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { return INSTANCE.openRW(path); } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java index 9f533fc7..a226eb0f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java @@ -76,11 +76,17 @@ public void tearDown() { } @Test - public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { + public void testRecoveryReusesSoleEmptyOrphanAsInitialActive() throws Exception { TestUtils.assertMemoryLeak(() -> { // Simulate a crashed prior session that left an unrotated hot spare // (valid SF01 header, frameCount=0). MmapSegment.create stamps the // header but writes no frames. + // + // Contract (manifest-era): a clean empty leftover is REUSED as the + // fresh ring's initial active instead of being unlinked and + // re-created. The anti-leak guarantee this test was written for + // still holds -- crash cycles converge to exactly one segment + // file, they don't accumulate. String orphanPath = tmpDir + "/sf-orphan.sfa"; MmapSegment empty = MmapSegment.create(orphanPath, 0L, SEGMENT_SIZE); empty.close(); @@ -89,12 +95,17 @@ public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { SegmentRing recovered = SegmentRing.openExisting(tmpDir, SEGMENT_SIZE); - Assert.assertNull( - "recovery returned a ring even though the only segment was empty", + Assert.assertNotNull( + "recovery must reuse the clean empty leftover as the initial active", recovered); - Assert.assertFalse( - "recovery left the empty orphan .sfa on disk — disk leak grows " - + "with every crash cycle", + try { + Assert.assertEquals("reused ring must be empty", -1, recovered.publishedFsn()); + Assert.assertEquals(0, recovered.getActive().baseSeq()); + Assert.assertEquals(orphanPath, recovered.getActive().path()); + } finally { + recovered.close(); + } + Assert.assertTrue("the reused segment file must still exist", Files.exists(orphanPath)); }); } From c448e34909bec79dd8fb0d666b3133f98319524b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 14 Jul 2026 00:58:27 +0100 Subject: [PATCH 22/64] fix(client): make SF recovery segment sort O(N log N) unconditionally sortByBaseSeq was a median-of-three Lomuto quicksort. Median-of-three covers the readdir orders a healthy slot produces (lexicographic enumeration yields already-sorted baseSeqs, hashed directory order is effectively random), but Lomuto partitioning is O(N^2) on organ-pipe, duplicate-heavy and median-of-three-killer orders. Exact simulation at the documented 16K-segment ceiling: 22.6M comparisons for organ-pipe, 50.7M for Musser's med3-killer permutation, 134M (full N^2/2) for mass-duplicate baseSeqs -- versus ~221K on the healthy paths. Such orders are only reachable through corrupted-yet-parseable or operator-copied headers, and recovery validation rejects those slots -- but the quadratic stall lands BEFORE validation gets to reject them. The sort is now an introsort: the median-of-three fast path is unchanged, and each root-to-leaf partition path carries a budget of 2*floor(log2(N)) passes. Loop-on-larger iterations count against the budget, so bad-split chains cannot hide in the loop. Ranges that exhaust it fall back to in-place heapsort (still Long.compareUnsigned, zero allocation), capping every adversarial pattern at ~3.8*N*log2(N) comparisons (organ-pipe 773K, duplicates 507K, med3-killer 861K at N=16384). Recursion depth stays under log2(N). The sortComparisons counter ticks +2 per sift-down level so it remains a strict upper bound in the fallback. New adversarial test drives the sort directly with in-memory segments at N=16384 across organ-pipe, all-duplicate, few-distinct, med3-killer and sign-bit-key patterns, asserting unsigned order plus comparisons < 8*N*log2(N) -- >2x headroom over the worst measured pattern, ~12x below the mildest quadratic blow-up. The existing sorted-input regression test is unchanged and still passes. --- .../qwp/client/sf/cursor/SegmentRing.java | 105 +++++++++++++++--- .../qwp/client/sf/cursor/SegmentRingTest.java | 95 ++++++++++++++++ 2 files changed, 186 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 3ad376f3..8b8e752c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -71,8 +71,9 @@ public final class SegmentRing implements QuietCloseable { // openExisting() recovery on this JVM. Used by SegmentRingTest to // assert the sort stays O(N log N) without relying on wall-clock time // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap - // in production: one volatile-free add per partition pass, dwarfed by - // the mmap I/O the recovery does on every segment. + // in production: one volatile-free add per partition pass (plus two per + // sift level in the rare heapsort fallback), dwarfed by the mmap I/O + // the recovery does on every segment. private static long sortComparisons; // References copied while compacting the logical sealed-segment head. // Test-only operation count for deterministic trim-complexity assertions. @@ -1110,9 +1111,10 @@ public static long getNextSealedComparisons() { * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} * (or process start). The count is incremented once per partition pass * for the median-of-three pivot pick plus once per element compared - * against the pivot, so a clean run on N segments adds roughly - * {@code 3 + (hi - lo - 1)} per recursive frame, summing to O(N log N). - * Exposed for {@code SegmentRingTest} to detect O(N²) regressions + * against the pivot ({@code 3 + (hi - lo - 1)} per pass), and by two per + * sift-down level when a range falls back to heapsort, so it strictly + * upper-bounds the true compare count and sums to O(N log N) on every + * input. Exposed for {@code SegmentRingTest} to detect O(N²) regressions * deterministically. */ @TestOnly @@ -1145,15 +1147,46 @@ public static void resetTrimMovedReferences() { } /** - * In-place quicksort over {@code list[lo, hi)} keyed by ascending - * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) - * on already-sorted input that lexicographic readdir produces (our - * filenames are zero-padded hex of {@code baseSeq}). Recursion depth is - * bounded by ~2 log₂(N) -- for the documented 16K-segment ceiling, well - * under the JVM default stack. + * Drives the recovery-time baseSeq sort over the whole list. Exposed so + * {@code SegmentRingTest} can feed adversarial orders (organ-pipe, mass + * duplicates, median-of-three killer, unsigned-boundary keys) straight + * into the sort and assert comparison bounds without staging thousands + * of segment files on disk. + */ + @TestOnly + public static void sortByBaseSeqForTest(ObjList list) { + sortByBaseSeq(list, 0, list.size()); + } + + /** + * In-place introsort over {@code list[lo, hi)} keyed by ascending + * unsigned {@code baseSeq}. Median-of-three quicksort handles the readdir + * orders a healthy slot produces (lexicographic enumeration of the + * generation-numbered filenames yields already-sorted baseSeqs; hashed + * directory order is effectively random), and a partition-pass budget of + * 2·⌊log₂(N)⌋ demotes any range that keeps splitting badly to in-place + * heapsort. Without that budget, Lomuto with a median-of-three pivot is + * O(N²) on organ-pipe, duplicate-heavy and median-of-three-killer orders + * -- reachable only through corrupted-yet-parseable or operator-copied + * headers, but at the documented 16K-segment ceiling that is 10⁷..10⁸ + * comparisons of startup stall before recovery validation gets to + * reject the slot, so the fallback makes O(N log N) unconditional. + * Recursion depth stays under log₂(N) (recurse on the smaller side, + * loop on the larger), well within the JVM default stack. */ private static void sortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + if (n > 1) { + sortByBaseSeq(list, lo, hi, 2 * (31 - Integer.numberOfLeadingZeros(n))); + } + } + + private static void sortByBaseSeq(ObjList list, int lo, int hi, int budget) { while (hi - lo > 1) { + if (budget-- == 0) { + heapSortByBaseSeq(list, lo, hi); + return; + } int mid = (lo + hi) >>> 1; long a = list.get(lo).baseSeq(); long b = list.get(mid).baseSeq(); @@ -1183,17 +1216,61 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) { } swap(list, store, hi - 1); // Recurse on the smaller partition; loop on the larger to keep - // recursion depth bounded by log₂(N). + // recursion depth bounded by log₂(N). Children inherit the + // remaining pass budget: it counts passes along a root-to-leaf + // path, so a chain of bad splits exhausts it after ~2 log₂(N) + // levels no matter how the work is divided. if (store - lo < hi - store - 1) { - sortByBaseSeq(list, lo, store); + sortByBaseSeq(list, lo, store, budget); lo = store + 1; } else { - sortByBaseSeq(list, store + 1, hi); + sortByBaseSeq(list, store + 1, hi, budget); hi = store; } } } + /** + * In-place heapsort over {@code list[lo, hi)} keyed by ascending unsigned + * {@code baseSeq}: the introsort fallback for ranges whose partition-pass + * budget ran out. Guaranteed O(N log N) for any key distribution and any + * initial order; no allocation. + */ + private static void heapSortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + for (int root = (n >>> 1) - 1; root >= 0; root--) { + siftDownByBaseSeq(list, lo, root, n); + } + for (int end = n - 1; end > 0; end--) { + swap(list, lo, lo + end); + siftDownByBaseSeq(list, lo, 0, end); + } + } + + private static void siftDownByBaseSeq(ObjList list, int lo, int root, int heapSize) { + while (true) { + int child = (root << 1) + 1; + if (child >= heapSize) { + return; + } + // At most two unsigned compares per level (sibling pick + parent + // test); bump the counter by the constant 2 up front -- same + // cheap-upper-bound convention as the partition pass. + sortComparisons += 2; + if (child + 1 < heapSize + && Long.compareUnsigned(list.get(lo + child).baseSeq(), + list.get(lo + child + 1).baseSeq()) < 0) { + child++; + } + if (Long.compareUnsigned(list.get(lo + root).baseSeq(), + list.get(lo + child).baseSeq()) >= 0) { + return; + } + swap(list, lo + root, lo + child); + root = child; + } + } + private static void swap(ObjList list, int i, int j) { if (i == j) return; MmapSegment tmp = list.get(i); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index 0d31320b..c6466de7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -686,6 +686,101 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { }); } + /** + * Adversarial-order companion to + * {@link #testLargeSegmentCountReopensInOrder}: that test covers the + * already-sorted readdir order median-of-three was chosen for; this one + * covers the orders median-of-three does NOT defend. On the + * pre-introsort quicksort, exact simulation at N=16384 measured ~22.6M + * comparisons for organ-pipe, ~134M (full N²/2) for mass-duplicate + * baseSeqs and ~50.7M for Musser's median-of-three-killer permutation. + * The heapsort fallback caps the worst of these at ~3.8·N·log₂(N) + * (~860K), so the 8·N·log₂(N) bound below keeps >2x headroom against + * harmless implementation drift while sitting ~26x under the mildest + * quadratic blow-up. Also pins unsigned key ordering: baseSeqs with the + * sign bit set must sort above {@code Long.MAX_VALUE}, not below zero. + *

    + * A healthy client cannot produce these orders (recovery validation + * rejects duplicate or non-contiguous baseSeqs moments after the sort), + * but corrupted-yet-parseable headers or operator file copies feed the + * sort BEFORE validation runs, so the sort itself must stay log-linear. + * Sorts in-memory segments directly: staging 16K adversarial header + * files on disk per pattern would dominate the test's runtime without + * adding coverage. + */ + @Test + public void testAdversarialSegmentOrdersSortLogLinear() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 16384; + final long bound = 8L * n * (long) (Math.log(n) / Math.log(2)); + + // Organ pipe: 0,1,...,n/2-1,n/2-1,...,1,0. + long[] organPipe = new long[n]; + for (int i = 0; i < n / 2; i++) { + organPipe[i] = i; + organPipe[n - 1 - i] = i; + } + assertAdversarialSortWithinBound("organ-pipe", organPipe, bound); + + long[] allDuplicates = new long[n]; + for (int i = 0; i < n; i++) { + allDuplicates[i] = 42; + } + assertAdversarialSortWithinBound("all-duplicates", allDuplicates, bound); + + long[] fewDistinct = new long[n]; + for (int i = 0; i < n; i++) { + fewDistinct[i] = i % 4; + } + assertAdversarialSortWithinBound("few-distinct", fewDistinct, bound); + + // Musser's median-of-three killer permutation of 1..n. + long[] med3Killer = new long[n]; + int half = n / 2; + for (int i = 1; i <= half; i++) { + if (i % 2 == 1) { + med3Killer[i - 1] = i; + med3Killer[i] = half + i; + } + med3Killer[half + i - 1] = 2L * i; + } + assertAdversarialSortWithinBound("median-of-three-killer", med3Killer, bound); + + // High-bit keys interleaved with small ones: exercises the + // unsigned comparison contract alongside the comparison bound. + long[] unsignedMix = new long[n]; + for (int i = 0; i < n; i++) { + unsignedMix[i] = (i % 2 == 0) ? (0x8000000000000000L | i) : i; + } + assertAdversarialSortWithinBound("unsigned-mix", unsignedMix, bound); + }); + } + + private static void assertAdversarialSortWithinBound(String label, long[] baseSeqs, long bound) { + final long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; + ObjList list = new ObjList<>(); + try { + for (long baseSeq : baseSeqs) { + list.add(MmapSegment.createInMemory(baseSeq, segSize)); + } + SegmentRing.resetSortComparisons(); + SegmentRing.sortByBaseSeqForTest(list); + long comparisons = SegmentRing.getSortComparisons(); + for (int i = 1, size = list.size(); i < size; i++) { + assertTrue(label + ": unsigned baseSeq order violated at index " + i, + Long.compareUnsigned(list.get(i - 1).baseSeq(), list.get(i).baseSeq()) <= 0); + } + assertTrue(label + " sort took " + comparisons + " comparisons (expected < " + bound + + " = 8 * N * log2(N) for N=" + baseSeqs.length + + "); regression suggests the introsort heapsort fallback stopped engaging", + comparisons < bound); + } finally { + for (int i = 0, size = list.size(); i < size; i++) { + list.get(i).close(); + } + } + } + @Test public void testRemovingAcknowledgedPrefixMovesLinearReferences() throws Exception { TestUtils.assertMemoryLeak(() -> { From 4b6ba13b06f015bf8b8379be1ad9cab453f8a7b8 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 14 Jul 2026 09:32:18 +0100 Subject: [PATCH 23/64] Fix client durability and creation shutdown --- .../cutlass/qwp/client/QwpQueryClient.java | 19 ++ .../qwp/client/sf/cursor/AckWatermark.java | 11 +- .../client/sf/cursor/CursorSendEngine.java | 133 ++++---- .../questdb/client/impl/QueryClientPool.java | 54 +++- .../io/questdb/client/impl/SenderPool.java | 68 ++-- .../CursorSendEngineCrashConsistencyTest.java | 180 ++++++++++- .../impl/QueryClientPoolErrorSafetyTest.java | 74 ++++- .../impl/QuestDBImplCloseLifecycleTest.java | 295 ++++++++++++++++++ .../impl/SenderPoolCloseLifecycleTest.java | 17 +- 9 files changed, 713 insertions(+), 138 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index b63b4fa9..f6587631 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -165,6 +165,10 @@ public class QwpQueryClient implements QuietCloseable { private final Random failoverRandom = new Random(); private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; + // Deterministic lifecycle barrier used by facade shutdown tests. Null in + // production; close() clears and invokes it after winning close ownership + // without allowing hook failures to prevent resource teardown. + private volatile Runnable beforeCloseHook; // Upper bound (ms) on each TCP connect attempt. 0 (default) falls back to // the OS connect timeout. private int connectTimeoutMs = 0; @@ -620,6 +624,16 @@ public void close() { // scratch, double-freeing it. return; } + Runnable hook = beforeCloseHook; + beforeCloseHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable ignored) { + // Omit diagnostics for this test-only hook: even rendering its + // failure must not prevent production resource cleanup. + } + } connected = false; lastCloseTimedOut = false; try { @@ -1004,6 +1018,11 @@ public void seedFailoverRandomForTest(long seed) { } } + @TestOnly + public void setBeforeCloseHookForTest(Runnable hook) { + beforeCloseHook = hook; + } + /** * Returns true if the most recent {@link #close()} call abandoned the I/O thread * because it failed to exit within the join timeout. The native buffer pool and diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 6e7ebc8c..625fbf22 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -134,9 +134,9 @@ public void close() { /** * Opens (creating if absent) the watermark file in {@code slotDir} * and maps it for the engine's lifetime. Returns {@code null} on - * any setup failure (open fail, mmap fail, unexpected size) — the - * caller falls back to the no-watermark behaviour, no exception - * escapes. Idempotent at the engine layer: a stale file from a + * any setup failure (open fail, mmap fail, unexpected size), leaving + * the caller to choose whether its durability contract permits operation + * without one. Idempotent at the engine layer: a stale file from a * prior session is reused as-is; the first {@link #write(long)} * stamps the magic and the new FSN atomically. */ @@ -167,13 +167,12 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { } } if (fd < 0) { - LOG.warn("ack watermark {} could not be opened (rc={}); proceeding without it", - filePath, fd); + LOG.warn("ack watermark {} could not be opened (rc={})", filePath, fd); return null; } long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { - LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath); + LOG.warn("ack watermark {} could not be mmapped", filePath); filesFacade.close(fd); return null; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 9e606b94..90c227fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -128,11 +128,12 @@ public final class CursorSendEngine implements QuietCloseable { // range with a cumulative self-acknowledge once everything below is // server-acked (CursorWebSocketSendLoop.tryRetireOrphanTail). private long recoveredOrphanTipFsn = -1L; - // Engine-owned mmap'd watermark file. {@code null} in memory mode and - // in disk mode if open() failed (we proceed without it; recovery just - // falls back to lowestBase - 1). Lifetime tied to the engine: opened - // in the constructor, closed by {@link #close()}. The segment manager - // writes through this on every tick where ackedFsn has advanced. + // Engine-owned mmap'd watermark file. {@code null} only in memory mode; + // disk mode fails construction unless the watermark is usable because + // segment-derived recovery cannot distinguish acknowledged residue from + // replayable frames. Lifetime tied to the engine: opened in the + // constructor, closed by {@link #close()}. The segment manager writes + // through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized @@ -179,6 +180,11 @@ public final class CursorSendEngine implements QuietCloseable { // With the CAS the worker's cleanup never blocks, so the join returns as // soon as the pass ends. private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean(); + // True only after a quiescent close reached the final watermark barrier + // and that barrier failed. It permits the shared retry driver to claim + // terminal cleanup; false while cleanup is merely waiting for manager + // quiescence, where an independent retry would race the live worker. + private volatile boolean terminalCleanupRetryReady; // Published only after ring/watermark/unlink cleanup is finished. A close // that loses terminalCleanupClaimed may retry the flock only after this // becomes true, otherwise it could expose the slot while cleanup is live. @@ -337,14 +343,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // - trim ran before persist: segments are gone (so // lowestBase is higher than watermark), watermark // is stale; max picks lowestBase - 1. - // open() returns null on any setup failure so a missing - // mmap doesn't take down the engine -- we just fall - // back to the bare lowestBase - 1 seed. + // Segment-derived state cannot tell whether frames in the + // lowest surviving segment were acknowledged. Starting + // without the watermark would therefore expose acknowledged + // residue for replay, so disk recovery must fail closed when + // the file cannot be opened or mapped. watermarkInProgress = AckWatermark.open(filesFacade, sfDir); + if (watermarkInProgress == null) { + throw new IllegalStateException( + "could not open required ack watermark for SF slot " + sfDir); + } long baseSeed = lowestBase - 1; - long watermarkFsn = watermarkInProgress != null - ? watermarkInProgress.read() - : AckWatermark.INVALID; + long watermarkFsn = watermarkInProgress.read(); // Reject watermarks past publishedFsn: a correctly // operating prior session cannot have produced one, so // a value above the on-disk frame ceiling is corruption @@ -393,6 +403,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(filesFacade, sfDir); watermarkInProgress = AckWatermark.open(filesFacade, sfDir); + if (watermarkInProgress == null) { + throw new IllegalStateException( + "could not open required ack watermark for SF slot " + sfDir); + } } MmapSegment initial; String initialPath = null; @@ -782,35 +796,36 @@ public synchronized void close() { * against a close() that holds the monitor while joining the worker. */ private void finishClose(boolean fullyDrained) { - try { - RuntimeException durabilityFailure = null; - // On a fully-drained close, persist the final acked FSN through - // the still-mapped watermark BEFORE closing the ring/watermark - // and BEFORE unlinking any segment file. The manager persists - // the watermark only on its own tick, so it may lag the final - // ack. If the unlink below then fails (or the process dies - // mid-unlink), residual acknowledged .sfa files without an - // up-to-date watermark would seed the successor's recovery at - // lowestBase - 1 and replay already-acknowledged rows -- - // duplicates on a non-DEDUP table. The write is a single mmap - // store, so it succeeds even when the unlink is about to fail - // (e.g. the slot dir turned read-only). Quiescence is already - // established here, so no manager tick can race this write. - if (fullyDrained && watermark != null) { - try { - long finalAckedFsn = ring.ackedFsn(); - if (finalAckedFsn >= 0) { - watermark.write(finalAckedFsn); - watermark.sync(); - if (filesFacade.fsyncDir(sfDir) != 0) { - throw new IllegalStateException( - "could not fsync SF slot directory before segment cleanup"); - } + // On a fully-drained close, persist the final acked FSN through the + // still-mapped watermark BEFORE closing the ring/watermark and BEFORE + // unlinking any segment file. The manager persists the watermark only + // on its own tick, so it may lag the final ack. If any part of this + // covering barrier fails, retain the ring, watermark and slot flock: + // publishing the slot would let a successor recover acknowledged + // residue from a stale durable watermark. The shared retry driver owns + // convergence after the one-shot public close reports the failure. + if (fullyDrained && watermark != null) { + try { + long finalAckedFsn = ring.ackedFsn(); + if (finalAckedFsn >= 0) { + watermark.write(finalAckedFsn); + watermark.sync(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new IllegalStateException( + "could not fsync SF slot directory before segment cleanup"); } - } catch (RuntimeException e) { - durabilityFailure = e; } + } catch (RuntimeException | Error e) { + terminalCleanupRetryReady = true; + terminalCleanupClaimed.set(false); + startFlockReleaseRetry(); + throw e; } + } + terminalCleanupRetryReady = false; + + try { + RuntimeException durabilityFailure = null; try { ring.close(); } catch (Throwable ignored) { @@ -829,7 +844,7 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } - if (fullyDrained && watermark != null && durabilityFailure == null) { + if (fullyDrained && watermark != null) { boolean segmentsRemoved = false; try { segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir); @@ -852,17 +867,15 @@ private void finishClose(boolean fullyDrained) { + "engine on this slot recovers them as fully acked and retries the " + "unlink on its own close", sfDir); } - } else if (fullyDrained && watermark == null && sfDir != null) { - LOG.warn("close-time segment cleanup skipped on slot {} because no ack watermark " - + "is available to cover a host crash during unlink", sfDir); } if (durabilityFailure != null) { throw durabilityFailure; } } finally { - // Reaching finishClose at all requires established quiescence, so - // releasing the flock is safe even if a step above threw. Leaking - // it would strand the slot until process exit for no reason. + // The final watermark covering barrier has succeeded, so terminal + // resources can be released even if later best-effort cleanup or + // its post-unlink directory sync failed. The durable watermark + // still covers any segment a host crash restores. // // ORDER MATTERS: explicitly release the flock, verify it, and // only then publish closeCompleted. Pools read isCloseCompleted() @@ -973,7 +986,19 @@ private Runnable createDeferredClose() { } private boolean retryFlockReleaseIfReady() { - if (closeCompleted || !terminalResourcesCleaned) { + if (closeCompleted) { + return true; + } + if (!terminalResourcesCleaned) { + if (!terminalCleanupRetryReady + || !terminalCleanupClaimed.compareAndSet(false, true)) { + return false; + } + try { + finishClose(fullyDrainedForDeferredClose); + } catch (Throwable ignored) { + return false; + } return closeCompleted; } boolean released; @@ -1091,8 +1116,8 @@ private void startFlockReleaseRetry() { } } if (startFailure == null) { - LOG.error("SF slot flock release failed during engine close; keeping " - + "closeCompleted=false and retrying on the shared driver so " + LOG.error("SF terminal cleanup or slot flock release failed during engine close; " + + "keeping closeCompleted=false and retrying on the shared driver so " + "retired capacity recovers after the transient failure [slot={}]", sfDir == null ? "" : sfDir); } else { @@ -1141,10 +1166,11 @@ public void setSlotLockReleaseListener(Runnable listener) { } /** - * Re-arms the shared flock-release retry for an engine whose terminal - * cleanup finished but whose confirmed flock release is still pending - * and no longer scheduled — the retry driver thread failed to start when - * the release first failed (e.g. OOM at thread creation). + * Re-arms the shared terminal retry for an engine whose final watermark + * barrier or confirmed flock release is still pending and no longer + * scheduled because the retry driver thread failed to start (e.g. OOM at + * thread creation). A close still waiting for worker quiescence cannot be + * retried here because terminal cleanup may not race that worker. * {@code Sender.close()} is one-shot by contract, so pool probes * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a * retired slot's capacity recoverable instead of lost until process @@ -1153,10 +1179,9 @@ public void setSlotLockReleaseListener(Runnable listener) { * may call it under their capacity lock. */ public void ensureFlockReleaseRetryScheduled() { - if (closeCompleted || !terminalResourcesCleaned) { - return; + if (!closeCompleted && (terminalResourcesCleaned || terminalCleanupRetryReady)) { + startFlockReleaseRetry(); } - startFlockReleaseRetry(); } /** diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java index 259ff10c..da9249c5 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java +++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java @@ -60,6 +60,10 @@ public final class QueryClientPool implements AutoCloseable { private final ArrayList all; private final ArrayDeque available; private final String configurationString; + // Signals completion of internally owned creation lifecycles. Kept + // separate from workerReleased so an acquire waiter cannot consume the only + // wakeup intended for close(). + private final Condition creationFinished; // Test seam. Production connects via QwpQueryClient.connect(); white-box // tests in io.questdb.client.test.impl reach the package-private constructor // by reflection to inject a hook that throws a non-RuntimeException @@ -146,6 +150,7 @@ public QueryClientPool( this.maxLifetimeMillis = maxLifetimeMillis; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.creationFinished = lock.newCondition(); this.workerReleased = lock.newCondition(); int built = 0; // Tracks a worker built by createUnlocked() but not yet added to `all`: @@ -231,16 +236,11 @@ public QueryWorker acquire() { // incremented, permanently shrinking pool capacity until // every acquire() times out. Restoring the reservation // for any throwable is safe. - lock.lock(); - inFlightCreations--; - workerReleased.signal(); - lock.unlock(); // createUnlocked() returns a fully connected client // (socket + native scratch + I/O thread), so if start() // threw afterwards we must close it here -- nothing else - // references it. createUnlocked() already self-cleans - // when connect() throws, leaving created == null, so - // this only fires on the start()-throws path. + // references it. Keep the creation reservation until + // that cleanup finishes so close() cannot return first. if (created != null) { try { created.shutdown(); @@ -249,33 +249,45 @@ public QueryWorker acquire() { // original creation failure rethrown below. } } + lock.lock(); + try { + inFlightCreations--; + creationFinished.signalAll(); + workerReleased.signal(); + } finally { + lock.unlock(); + } throw new QueryException((byte) 0, "failed to create query client: " + e.getMessage(), e); } lock.lock(); - inFlightCreations--; if (closed) { - // Pool was closed mid-creation -- tear the fresh worker - // down rather than leaking it, but OUTSIDE the lock: - // shutdown() joins the dispatch thread for up to - // SHUTDOWN_JOIN_MILLIS, and close()/release()/discard()/ - // cancelIfCurrent() all contend on this lock (whose - // contract is "held only briefly"). The accounting above - // already ran under the lock, and the worker never - // entered `all`, so close()'s snapshot loop cannot race - // this teardown. + // Keep inFlightCreations reserved while shutdown runs + // outside the lock. The worker never entered `all`, so + // this reservation is close()'s sole ownership record. lock.unlock(); try { created.shutdown(); } catch (Throwable ignored) { // Best-effort: an Error from teardown must not mask // the closed-pool signal. + } finally { + lock.lock(); + try { + inFlightCreations--; + creationFinished.signalAll(); + workerReleased.signalAll(); + } finally { + lock.unlock(); + } } throw new QueryException((byte) 0, "QuestDB handle is closed"); } all.add(created); // Stamp the first lease id for this freshly built worker. created.bumpGeneration(); + inFlightCreations--; + creationFinished.signalAll(); return created; } if (remainingNanos <= 0) { @@ -307,6 +319,14 @@ public void close() { } closed = true; workerReleased.signalAll(); + // A reservation owns every resource created outside the lock until + // the worker is either published in `all` or fully torn down. No + // user ever received these workers, so close() must not apply an + // abandoned-lease timeout. Preserve interrupts while waiting for + // the internal ownership count to reach zero. + while (inFlightCreations > 0) { + creationFinished.awaitUninterruptibly(); + } snapshot = new ArrayList<>(all); } finally { lock.unlock(); diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1bd60f3f..32540026 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -107,6 +107,10 @@ public final class SenderPool implements AutoCloseable { private final ArrayList all; private final ArrayDeque available; private final String configurationString; + // Signals completion of internally owned creation lifecycles. Kept + // separate from slotReleased so a capacity waiter cannot consume the only + // wakeup intended for close(). + private final Condition creationFinished; // User-supplied ingest callbacks, shared across every pooled Sender this // pool builds. Null -> each sender keeps its loud-not-silent default. private final SenderConnectionListener connectionListener; @@ -369,6 +373,7 @@ private SenderPool( this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); this.retiredSlots = new ArrayList<>(maxSize); + this.creationFinished = lock.newCondition(); this.slotReleased = lock.newCondition(); // Probe the config once, up front: this validates it eagerly (so a // bad config fails at construction even when minSize == 0) and tells @@ -810,33 +815,26 @@ public PooledSender borrow() { // idempotent, so undoing the reservation for any // throwable is safe. lock.lock(); - inFlightCreations--; - freeSlotIndex(slotIndex); - slotReleased.signal(); - lock.unlock(); + try { + inFlightCreations--; + freeSlotIndex(slotIndex); + creationFinished.signalAll(); + slotReleased.signal(); + } finally { + lock.unlock(); + } throw e; } lock.lock(); - inFlightCreations--; if (closed) { - // Pool was closed mid-creation -- destroy the new connection - // rather than leaking it. Other waiters have been signaled - // by close() already. The delegate is closed OUTSIDE the - // lock (mirroring retireLease): its close() can block for - // seconds (bounded ack drain, drainer-pool wind-down) or - // longer (unbounded I/O-thread latch await behind an - // OS-level connect), which held here would stall close(), - // giveBack/retireLease and reapIdle behind the pool lock. - // Accounting first, under the lock: for an SF slot the - // index reservation moves from inFlightCreations to - // closingSlots until the close below releases the flock, - // and pendingLeaseTeardowns keeps the out-of-lock close - // visible to close()'s outstanding-teardown wait. + // Pool was closed mid-creation. Keep inFlightCreations + // reserved while the delegate is closed outside the + // lock: close() waits on that counter, so the creation + // remains internally owned until its teardown completes. boolean reserved = created.slotIndex() >= 0; if (reserved) { closingSlots++; } - pendingLeaseTeardowns++; lock.unlock(); try { created.delegate().close(); @@ -844,23 +842,24 @@ public PooledSender borrow() { // Best-effort: an Error (e.g. -ea AssertionError) // from teardown must not mask the closed-pool signal. } finally { - // Re-lock to reclaim the SF slot index and signal a - // close() waiting on this teardown. MUST run even if - // the delegate close threw, otherwise the slot stays - // reserved forever and close() waits out its full - // budget on a teardown that already happened. lock.lock(); - pendingLeaseTeardowns--; - if (reserved) { - reclaimSlot(created, " after closed-mid-creation teardown"); + try { + if (reserved) { + reclaimSlot(created, " after closed-mid-creation teardown"); + } + inFlightCreations--; + creationFinished.signalAll(); + slotReleased.signalAll(); + } finally { + lock.unlock(); } - slotReleased.signalAll(); - lock.unlock(); } throw new LineSenderException("QuestDB handle is closed"); } all.add(created); created.bumpGeneration(); + inFlightCreations--; + creationFinished.signalAll(); return new PooledSender(created, created.generation()); } // Capacity-starved: re-probe retired slots BEFORE the terminal @@ -938,6 +937,8 @@ void markClosing() { * free their native memory under its feet -- a use-after-free / SEGV, not * an exception (C1). Instead: *

      + *
    1. waits for every internally owned creation to publish or complete its + * closed-mid-creation teardown; then
      2. *
    2. waits boundedly (up to {@code acquireTimeoutMillis}, hard-capped at * {@link #MAX_CLOSE_LEASE_WAIT_MILLIS}) for outstanding leases to come home -- {@link #giveBack} and {@link #discardBroken} * observe {@code closed} and tear each delegate down on the returning @@ -964,6 +965,15 @@ public void close() { closed = true; // Wake parked borrowers so they observe the shutdown and throw. slotReleased.signalAll(); + // A creation reservation represents pool-owned resources from the + // moment borrow() leaves the lock through either publication or + // completed cleanup. Unlike an abandoned user-visible lease, this + // ownership cannot be leaked after close() returns, so wait without + // a timeout. awaitUninterruptibly preserves the interrupt flag while + // maintaining the shutdown ownership invariant. + while (inFlightCreations > 0) { + creationFinished.awaitUninterruptibly(); + } // Bounded graceful wait for outstanding leases. A slot is borrowed // iff it is in `all` but not in `available`; retireLease's // delegate-close section (running outside the lock on a returning diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java index d6c3f665..a7017961 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java @@ -45,9 +45,9 @@ public class CursorSendEngineCrashConsistencyTest { @Test - public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception { + public void testCloseDurabilityOrderAndPostCleanupSyncFailurePropagation() throws Exception { TestUtils.assertMemoryLeak(() -> { - for (int failAt : new int[]{-1, 0, 1, 2, 4}) { + for (int failAt : new int[]{-1, 4}) { String root = Paths.get(System.getProperty("java.io.tmpdir"), "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString(); String slot = root + "/slot"; @@ -69,24 +69,16 @@ public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception ff.beginClose(); try { engine.close(); - if (failAt >= 0 && failAt != 3) { + if (failAt >= 0) { Assert.fail("sync failure was swallowed at boundary " + failAt); } } catch (IllegalStateException expected) { - Assert.assertTrue("unexpected close failure: " + expected, - failAt >= 0 && failAt != 3); + Assert.assertTrue("unexpected close failure: " + expected, failAt >= 0); } engine = null; - if (failAt >= 0 && failAt <= 2) { - Assert.assertFalse("segment deletion started after watermark barrier failure", - ff.events.contains("segment-remove")); - Assert.assertTrue("watermark was removed after its durability barrier failed", - Files.exists(slot + "/" + AckWatermark.FILE_NAME)); - } else { - Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, - ff.durableSegments && !ff.durableWatermark); - } + Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, + ff.durableSegments && !ff.durableWatermark); if (failAt == -1) { Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"), @@ -105,6 +97,138 @@ public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception }); } + @Test + public void testFinalWatermarkBarrierFailureRetainsSlotUntilRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (String barrier : Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync")) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-barrier-" + barrier + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine predecessor = null; + CursorSendEngine successor = null; + CrashImageFilesFacade ff = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + ff = new CrashImageFilesFacade(slot, -1); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + predecessor = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32)); + Assert.assertTrue(predecessor.acknowledge(0L)); + + ff.failPersistently(barrier); + try { + predecessor.close(); + Assert.fail("expected close to report failed final barrier " + barrier); + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, + expected.getMessage().contains("ack watermark") + || expected.getMessage().contains("slot directory")); + } + try { + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.fail("successor acquired a slot whose final ACK barrier failed: " + barrier); + } catch (IllegalStateException expected) { + Assert.assertTrue("successor failed for the wrong reason: " + expected, + expected.getMessage().contains("slot already in use")); + } + + ff.clearPersistentFailure(); + awaitCloseCompleted(predecessor); + predecessor = null; + + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals("successful retry must leave no ACKed frame to replay", + -1L, successor.publishedFsn()); + successor.close(); + Assert.assertTrue(successor.isCloseCompleted()); + successor = null; + } catch (Throwable t) { + failure = t; + } finally { + if (failure != null && ff != null) { + // Let a retained predecessor converge before cleanup. + // The exact red assertion remains attached to failure. + ff.clearPersistentFailure(); + } + failure = closeEngine(failure, successor); + failure = closeEngine(failure, predecessor); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + @Test + public void testRecoveredSlotRejectsMissingWatermark() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-missing-watermark-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine predecessor = null; + CursorSendEngine successor = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, -1); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + predecessor = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32)); + Assert.assertTrue(predecessor.acknowledge(0L)); + ff.beginClose(); + ff.blockSegmentRemove = true; + predecessor.close(); + Assert.assertTrue(predecessor.isCloseCompleted()); + predecessor = null; + Assert.assertTrue("precondition: ACKed segment residue must survive", + Files.exists(slot + "/sf-initial.sfa")); + + ff.failWatermarkOpen = true; + try { + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.fail("recovered disk slot started without a usable ACK watermark"); + } catch (IllegalStateException expected) { + Assert.assertTrue("successor failed for the wrong reason: " + expected, + expected.getMessage().contains("ack watermark")); + } + + ff.failWatermarkOpen = false; + ff.blockSegmentRemove = false; + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(successor.wasRecoveredFromDisk()); + Assert.assertTrue("successor exposes predecessor's ACKed frame", + successor.ackedFsn() >= successor.publishedFsn()); + successor.close(); + Assert.assertTrue(successor.isCloseCompleted()); + successor = null; + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, successor); + failure = closeEngine(failure, predecessor); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + }); + } + private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) { if (failure == null) { return cleanupFailure; @@ -115,6 +239,14 @@ private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupF return failure; } + private static void awaitCloseCompleted(CursorSendEngine engine) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!engine.isCloseCompleted() && System.nanoTime() < deadline) { + Thread.yield(); + } + Assert.assertTrue("close retry did not complete", engine.isCloseCompleted()); + } + private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) { if (engine != null) { try { @@ -196,9 +328,12 @@ private static final class CrashImageFilesFacade implements FilesFacade { private final int failAt; private final String slot; private boolean active; + private boolean blockSegmentRemove; private boolean durableSegments = true; private boolean durableWatermark; private int eventIndex; + private boolean failWatermarkOpen; + private String persistentFailure; private int watermarkFd = -1; private CrashImageFilesFacade(String slot, int failAt) { @@ -212,9 +347,20 @@ private void beginClose() { eventIndex = 0; } + private void clearPersistentFailure() { + persistentFailure = null; + } + private boolean fail(String event) { events.add(event); - return eventIndex++ == failAt; + return event.equals(persistentFailure) || eventIndex++ == failAt; + } + + private void failPersistently(String event) { + active = true; + events.clear(); + eventIndex = 0; + persistentFailure = event; } @Override @@ -269,6 +415,7 @@ public int msync(long addr, long len, boolean async) { } @Override public int openCleanRW(String path) { + if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1; int fd = INSTANCE.openCleanRW(path); if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; return fd; @@ -277,6 +424,7 @@ public int openCleanRW(String path) { public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } @Override public int openRW(String path) { + if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1; int fd = INSTANCE.openRW(path); if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; return fd; @@ -290,7 +438,7 @@ public long read(int fd, long addr, long len, long offset) { @Override public boolean remove(String path) { if (active && path.endsWith(".sfa")) { - if (fail("segment-remove")) return false; + if (blockSegmentRemove || fail("segment-remove")) return false; } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) { if (fail("watermark-remove")) return false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java index bc36d1ae..ca5e33aa 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java @@ -34,10 +34,13 @@ import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; // Error-safety of the three QueryClientPool creation paths the teardown-hardening @@ -230,16 +233,31 @@ public void preWarmDoesNotLeakNativeScratchOnErrorFromStart() throws Exception { // the dispatch thread for up to SHUTDOWN_JOIN_MILLIS; cancelIfCurrent's // contract is that this lock is "held only briefly") -- and still reclaim // everything: the client's NATIVE_DEFAULT scratch and the creation slot. - // RED (teardown dropped in the restructure): the scratch leaks and the - // baseline assertion fails. - // GREEN: shutdown() runs after the lock is released -> no leak, accounting - // restored, acquire() surfaces the closed pool. + // RED (rendering a throwing before-close hook's failure also throws after + // close ownership is won): QueryWorker suppresses that second failure, + // the pool releases its creation reservation, and the client's native + // scratch remains live. + // GREEN: close() suppresses the test-witness failure without rendering it, + // completes production teardown, and only then lets the pool release + // the reservation. @Test(timeout = 30_000) public void closedMidCreationTearsDownFreshWorkerWithoutLeak() throws Exception { TestUtils.assertMemoryLeak(() -> { CountDownLatch inConnect = new CountDownLatch(1); CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicInteger closeHookCalls = new AtomicInteger(); + AtomicReference createdClient = new AtomicReference<>(); Consumer connectHook = client -> { + createdClient.set(client); + client.setBeforeCloseHookForTest(() -> { + closeHookCalls.incrementAndGet(); + throw new AssertionError("injected query close hook failure") { + @Override + public String toString() { + throw new AssertionError("injected throwable rendering failure"); + } + }; + }); inConnect.countDown(); try { if (!releaseConnect.await(10, TimeUnit.SECONDS)) { @@ -268,25 +286,35 @@ public void closedMidCreationTearsDownFreshWorkerWithoutLeak() throws Exception Assert.assertTrue("acquirer never reached connect()", inConnect.await(10, TimeUnit.SECONDS)); - // Close the pool while the worker build is in flight (the fresh - // worker never entered `all`, so close()'s snapshot skips it), - // then let the build finish: acquire() must observe `closed`, - // tear the worker down on its own thread and throw. - pool.close(); + // Close the pool while the worker build is in flight. close() + // now waits for the internal creation reservation, so run it + // concurrently and wait until it parks on that reservation + // before releasing connect(). + Thread closer = new Thread(pool::close, "mid-creation-pool-closer"); + closer.start(); + awaitCreationWaiter(pool); + + // Let the build finish: acquire() must observe `closed`, tear + // the worker down on its own thread and throw. close() returns + // only after that teardown releases the reservation. releaseConnect.countDown(); acquirer.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("acquirer did not finish", acquirer.isAlive()); + Assert.assertFalse("pool close did not finish", closer.isAlive()); Assert.assertTrue( "acquire() must surface the closed pool, got: " + acquireOutcome.get(), acquireOutcome.get() instanceof QueryException && String.valueOf(acquireOutcome.get().getMessage()).contains("closed")); + Assert.assertEquals("pool released the creation reservation after teardown", + 0, inFlightCreations(pool)); long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); Assert.assertEquals( - "closed-mid-creation acquire() leaked the fresh worker's NATIVE_DEFAULT scratch", + "throwing close hook prevented fresh worker teardown before reservation release", baseline, after); - Assert.assertEquals("in-flight creation accounting must be restored", - 0, inFlightCreations(pool)); + createdClient.get().close(); + Assert.assertEquals("close hook must be one-shot", 1, closeHookCalls.get()); } finally { pool.close(); } @@ -299,6 +327,28 @@ private static Consumer alwaysThrow() { }; } + private static void awaitCreationWaiter(QueryClientPool pool) throws Exception { + Field lockField = QueryClientPool.class.getDeclaredField("lock"); + Field conditionField = QueryClientPool.class.getDeclaredField("creationFinished"); + lockField.setAccessible(true); + conditionField.setAccessible(true); + ReentrantLock lock = (ReentrantLock) lockField.get(pool); + Condition creationFinished = (Condition) conditionField.get(pool); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + lock.lock(); + try { + if (lock.hasWaiters(creationFinished)) { + return; + } + } finally { + lock.unlock(); + } + Thread.yield(); + } + Assert.fail("pool close did not wait on the creation reservation"); + } + // connectHook that connects nothing: fromConfig() has already committed the // NATIVE_DEFAULT scratch, so the client is half-built (scratch, no socket) // -- exactly the state createUnlocked() returns before start() runs. diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java new file mode 100644 index 00000000..dd697049 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java @@ -0,0 +1,295 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.QueryException; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QueryClientPool; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.impl.SenderPool; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +public class QuestDBImplCloseLifecycleTest { + + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + @Test(timeout = 30_000) + public void facadeCloseWaitsForQueryCreationAndTeardown() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + CountDownLatch inTeardown = new CountDownLatch(1); + CountDownLatch releaseTeardown = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(() -> { + inTeardown.countDown(); + awaitOrFail(releaseTeardown, "test never released query teardown"); + teardownCount.incrementAndGet(); + }); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB(0, 0, slotIndex -> fakeSender(null, null, null), connectHook); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "facade-query-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "facade-query-closer"); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + + closer.start(); + QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + awaitBooleanField(pool, "closed"); + awaitCreationWaiter(pool, + "facade close did not wait while query construction was internally owned"); + closer.interrupt(); + awaitCreationWaiter(pool, + "interrupt allowed facade close to abandon query creation ownership"); + + releaseCreation.countDown(); + Assert.assertTrue("query borrow never reached closed-mid-creation teardown", + inTeardown.await(10, TimeUnit.SECONDS)); + awaitCreationWaiter(pool, + "facade close abandoned query ownership during internal teardown"); + + releaseTeardown.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertFalse("facade close did not finish", closer.isAlive()); + Assert.assertEquals("internally owned query client must be torn down exactly once", + 1, teardownCount.get()); + Assert.assertTrue("facade close must preserve interruption after internal teardown", + closeReturnedInterrupted.get()); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + releaseTeardown.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseWaitsForSenderCreationAndTeardown() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + CountDownLatch inTeardown = new CountDownLatch(1); + CountDownLatch releaseTeardown = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, inTeardown, releaseTeardown); + }; + QuestDBImpl db = newQuestDB(0, 0, senderFactory, client -> { + }); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "facade-sender-borrower"); + Thread closer = new Thread(db::close, "facade-sender-closer"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + + closer.start(); + SenderPool pool = (SenderPool) getField(db, "senderPool"); + awaitBooleanField(pool, "closeStarted"); + awaitCreationWaiter(pool, + "facade close did not wait while sender construction was internally owned"); + + releaseCreation.countDown(); + Assert.assertTrue("sender borrow never reached closed-mid-creation teardown", + inTeardown.await(10, TimeUnit.SECONDS)); + awaitCreationWaiter(pool, + "facade close abandoned sender ownership during internal teardown"); + + releaseTeardown.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertFalse("facade close did not finish", closer.isAlive()); + Assert.assertEquals("internally owned sender must be torn down exactly once", + 1, teardownCount.get()); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + releaseTeardown.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + private static void awaitBooleanField(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (field.getBoolean(target)) { + return; + } + Thread.yield(); + } + Assert.fail("field did not become true: " + fieldName); + } + + private static void awaitCreationWaiter(Object pool, String message) throws Exception { + ReentrantLock lock = (ReentrantLock) getField(pool, "lock"); + Condition creationFinished = (Condition) getField(pool, "creationFinished"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + lock.lock(); + try { + if (lock.hasWaiters(creationFinished)) { + return; + } + } finally { + lock.unlock(); + } + Thread.yield(); + } + Assert.fail(message); + } + + private static void awaitOrFail(CountDownLatch latch, String message) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException(message); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(message, e); + } + } + + private static Sender fakeSender( + AtomicInteger teardownCount, + CountDownLatch inTeardown, + CountDownLatch releaseTeardown + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + if (inTeardown != null) { + inTeardown.countDown(); + awaitOrFail(releaseTeardown, "test never released sender teardown"); + } + if (teardownCount != null) { + teardownCount.incrementAndGet(); + } + return null; + case "toString": + return "FacadeCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class returnType = method.getReturnType(); + if (returnType == boolean.class) return false; + if (returnType == byte.class) return (byte) 0; + if (returnType == short.class) return (short) 0; + if (returnType == int.class) return 0; + if (returnType == long.class) return 0L; + if (returnType == float.class) return 0f; + if (returnType == double.class) return 0d; + if (returnType == char.class) return (char) 0; + if (returnType == void.class) return null; + if (returnType.isInstance(proxy)) return proxy; + return null; + } + }); + } + + private static Object getField(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } + + private static QuestDBImpl newQuestDB( + int senderMin, + int queryMin, + IntFunction senderFactory, + Consumer connectHook + ) { + return new QuestDBImpl( + SENDER_CFG, QUERY_CFG, + senderMin, 1, + queryMin, 1, + 10_000L, + Long.MAX_VALUE, + Long.MAX_VALUE, + Long.MAX_VALUE, + senderFactory, connectHook); + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java index b9b0136f..f116a4a6 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java @@ -286,10 +286,17 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { Assert.assertTrue("borrower never reached the factory", inCreate.await(10, TimeUnit.SECONDS)); - // Close the pool while the creation is in flight: nothing is - // outstanding (the new slot never entered `all`), so this - // returns promptly with `closed` raised. - pool.close(); + // Raise the same early shutdown signal used by QuestDBImpl so + // the creation deterministically takes the teardown branch. + Method markClosing = SenderPool.class.getDeclaredMethod("markClosing"); + markClosing.setAccessible(true); + markClosing.invoke(pool); + + // close() now owns the in-flight creation reservation until + // its closed-mid-creation teardown completes, so run it on a + // separate thread while the creation remains parked. + Thread poolCloser = new Thread(pool::close, "pool-closer"); + poolCloser.start(); // Let the creation finish: the borrower re-locks, observes the // closed pool and starts the delegate teardown, which parks. @@ -313,6 +320,8 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { lockFree); borrower.join(TimeUnit.SECONDS.toMillis(10)); + poolCloser.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("pool close did not finish", poolCloser.isAlive()); Assert.assertFalse("borrower did not finish", borrower.isAlive()); Assert.assertTrue( "borrow() must surface the closed pool, got: " + borrowOutcome.get(), From 21345b0639edcfbee1464d611935fad4e4402e1a Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Tue, 14 Jul 2026 18:32:41 +0100 Subject: [PATCH 24/64] Harden QWP client recovery and shutdown Protect the durable ACK watermark with redundant CRC records so torn writes cannot skip unacknowledged frames after restart. Bound sender and query pool shutdown waits while retaining late cleanup ownership, make segment recovery cleanup linear and ownership-safe, and count only genuine reconnect sends as replay telemetry. Add deterministic Java 8 regression coverage for crash recovery, pool shutdown, segment-count complexity, cleanup failures, and reconnect metrics. --- .../main/java/io/questdb/client/QuestDB.java | 15 +- .../qwp/client/sf/cursor/AckWatermark.java | 219 +++++++------ .../sf/cursor/CursorWebSocketSendLoop.java | 35 +- .../qwp/client/sf/cursor/SegmentRing.java | 206 +++++++++--- .../questdb/client/impl/QueryClientPool.java | 42 ++- .../io/questdb/client/impl/QuestDBImpl.java | 18 +- .../io/questdb/client/impl/SenderPool.java | 46 ++- .../qwp/client/InitialConnectAsyncTest.java | 163 ++++++++++ .../client/sf/cursor/AckWatermarkTest.java | 41 +++ .../sf/cursor/SegmentRingMembershipTest.java | 209 ++++++++++++ .../qwp/client/sf/cursor/SegmentRingTest.java | 19 +- .../impl/QuestDBImplCloseLifecycleTest.java | 305 +++++++++++++++++- 12 files changed, 1129 insertions(+), 189 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 1a9aacae..a4c7cb03 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -130,10 +130,17 @@ static QuestDB connect(CharSequence configurationString) { Sender borrowSender(); /** - * Shuts down the pools, closing every underlying {@link Sender} and - * query client. Idempotent. Threads currently blocked in - * {@link #borrowSender()} or {@link Query#submit()} are released with an - * error. + * Shuts down the pools and their published clients. Idempotent. Threads + * currently blocked in {@link #borrowSender()} or {@link Query#submit()} + * are released with an error. + *

      + * In-progress client creation: close() waits up to the builder's + * {@link QuestDBBuilder#acquireTimeoutMillis(long) acquire timeout}, + * hard-capped at 5 seconds, for a creation blocked in DNS, TCP, TLS, or a + * WebSocket handshake. If that budget expires, the creator retains cleanup + * ownership and closes its unpublished client (and releases any SF slot) + * when construction returns. This keeps close() bounded even when + * {@code connect_timeout} is unset without abandoning late resources. *

      * Outstanding leases: a borrowed {@link Sender} is never torn down * underneath the thread using it. Instead, close() waits up to the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 625fbf22..c1e6c96a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.std.Crc32c; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; @@ -43,80 +44,72 @@ * "everything <= N is durable"), so a single monotonic watermark suffices; * no per-frame bitmap is needed. *

      - * Layout (16 bytes, little-endian, mmap'd for the engine's lifetime): + * Layout (128 bytes, little-endian, mmap'd for the engine's lifetime): + * two independently CRC-protected 64-byte records. Each record contains: * 

      - *   offset 0:  u32 magic = 'AKW1' (set once on first write)
- *   offset 4:  u32 reserved (zero)
- *   offset 8:  i64 fsn
+ *   offset 0:   u32 magic = 'AKW1'
+ *   offset 4:   u32 version = 1
+ *   offset 8:   i64 generation
+ *   offset 16:  i64 fsn
+ *   offset 24:  reserved (zero-filled through offset 59)
+ *   offset 60:  u32 CRC32C of bytes [0, 60)
  *
      + * {@link #write(long)} rewrites the record not selected by the current + * generation and stores its CRC last. Recovery selects the valid record with + * the greatest generation. A torn update therefore falls back to the older + * valid record; if neither record validates, recovery conservatively uses the + * segment-derived seed. *

      - * Zero-alloc, store-only ACK writes: {@link #open(String)} opens the - * file and maps the 16 bytes once. {@link #write(long)} is a single - * 8-byte aligned {@code Unsafe.putLong} into the mapped region. Ordinary - * ACK-only manager updates require no malloc/free, read/write syscalls, or - * rename. An 8-byte aligned store is hardware-atomic on x86_64 and - * arm64, and disk-atomic within one sector (the file is 16 bytes — - * trivially within one sector), so a torn FSN across a crash boundary - * is not a concern. - *

      - * Why no CRC: the watermark is an optimization. If the on-disk - * value is somehow corrupted (filesystem bug, hardware fault), the - * recovery path's {@code max(lowestBase - 1, watermark)} clamp absorbs - * the inconsistency: a stale-low watermark just means more re-replay, a - * stale-high watermark is impossible because no write produces an FSN - * higher than the segments on disk could account for. CRC adds - * complexity (multi-store with fence + read-side validate) for - * marginal additional safety. + * Zero-alloc, store-only ACK writes: {@link #open(String)} maps both + * records once. Ordinary ACK-only manager updates mutate the inactive record + * in the mapping and require no malloc/free, read/write syscalls, or rename. *

      * fsync cadence: ordinary ACK-only manager updates call * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs * the slot directory before unlinking, and fsyncs it again after the batch. A - * fully drained close uses the same - * covering order so the durable watermark guards any acknowledged segment that - * a host crash restores. + * fully drained close uses the same covering order so the durable watermark + * guards any acknowledged segment that a host crash restores. *

      - * Lifecycle: single-writer (the {@link SegmentManager} worker - * thread) after construction. Read once at engine startup (any thread, - * before the manager observes the entry). Close releases the mapping - * and fd. Not thread-safe for concurrent writers. + * Lifecycle: single-writer (the {@link SegmentManager} worker thread) + * after construction. Read once at engine startup (any thread, before the + * manager observes the entry). Close releases the mapping and fd. Not + * thread-safe for concurrent writers. */ public final class AckWatermark implements QuietCloseable { /** - * Filename of the watermark within the slot directory. - * Dot-prefixed so directory enumerators that filter by extension - * (segment recovery, OrphanScanner) skip it automatically. + * Filename of the watermark within the slot directory. Dot-prefixed so + * directory enumerators that filter by extension skip it automatically. */ public static final String FILE_NAME = ".ack-watermark"; - public static final int FILE_SIZE = 16; + public static final int FILE_SIZE = 128; /** - * Sentinel returned by {@link #read()} when the watermark file is - * present but has not yet been written to (the magic field is zero - * because the OS zero-filled the freshly created file). + * Sentinel returned by {@link #read()} when neither watermark record is + * valid. */ public static final long INVALID = Long.MIN_VALUE; static final int FILE_MAGIC = 0x31574B41; // 'AKW1' little-endian - private static final int FSN_OFFSET = 8; + private static final int CRC_OFFSET = 60; + private static final int FSN_OFFSET = 16; private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class); private static final int MAGIC_OFFSET = 0; + private static final int RECORD_SIZE = 64; + private static final int VERSION = 1; private final int fd; private final FilesFacade filesFacade; private final long mmapAddress; private boolean closed; - // Stamped once per process either at open() (if the file already - // has the magic from a prior session) or on the first write() that - // observes it unset. After the flag flips, write() degenerates to - // a single 8-byte putLong against the mapped FSN slot -- no memory - // load of magic on the hot path. Manager-thread-only after - // construction; no synchronisation needed. - private boolean magicWritten; + private long fsn; + private long generation; - private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, boolean magicAlreadyWritten) { + private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, + long generation, long fsn) { this.fd = fd; this.filesFacade = filesFacade; + this.fsn = fsn; + this.generation = generation; this.mmapAddress = mmapAddress; - this.magicWritten = magicAlreadyWritten; } @Override @@ -132,13 +125,15 @@ public void close() { } /** - * Opens (creating if absent) the watermark file in {@code slotDir} - * and maps it for the engine's lifetime. Returns {@code null} on - * any setup failure (open fail, mmap fail, unexpected size), leaving - * the caller to choose whether its durability contract permits operation - * without one. Idempotent at the engine layer: a stale file from a - * prior session is reused as-is; the first {@link #write(long)} - * stamps the magic and the new FSN atomically. + * Opens (creating if absent) the watermark file in {@code slotDir} and + * maps it for the engine's lifetime. Returns {@code null} on any setup + * failure, leaving the caller to choose whether its durability contract + * permits operation without one. + *

      + * Wrong-sized files, including the legacy 16-byte non-CRC format, are + * reset. Trusting a legacy FSN would retain the torn-write ambiguity this + * format removes; resetting it causes conservative replay from the + * segment-derived seed instead. */ public static AckWatermark open(String slotDir) { return open(FilesFacade.INSTANCE, slotDir); @@ -146,12 +141,6 @@ public static AckWatermark open(String slotDir) { static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - // Decide by size: existing-and-correct -> openRW preserves the - // previous session's watermark (defeating which is the whole - // point of NOT calling openCleanRW unconditionally); missing or - // wrong-sized -> openCleanRW + allocate creates a fresh - // FILE_SIZE-byte file (zero magic, read() reports INVALID until - // the first write). long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; if (existing == FILE_SIZE) { @@ -159,8 +148,8 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { } else { fd = filesFacade.openCleanRW(filePath); if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) { - // FilesFacade.allocate contract on a false return: - // close the fd AND unlink the partial file. + // FilesFacade.allocate contract on a false return: close the + // fd and unlink the partial file. filesFacade.close(fd); filesFacade.remove(filePath); fd = -1; @@ -176,19 +165,15 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { filesFacade.close(fd); return null; } - // Inspect the existing magic once at open time. If it's already - // set (cross-session reopen, e.g. recovery after a clean - // shutdown), the first write() can skip the magic store - // entirely and degenerate to a single 8-byte FSN put. - int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET); - return new AckWatermark(filesFacade, fd, addr, magic == FILE_MAGIC); + Record selected = selectRecord(addr); + return selected == null + ? new AckWatermark(filesFacade, fd, addr, 0L, INVALID) + : new AckWatermark(filesFacade, fd, addr, selected.generation, selected.fsn); } /** - * Best-effort removal of a stale watermark file. Used by the - * engine startup path when no segments are recovered — a stale - * watermark file with no segments behind it is meaningless and - * would only confuse the next session's seed. + * Best-effort removal of a stale watermark file. Used by the engine + * startup path when no segments are recovered. */ public static void removeOrphan(String slotDir) { removeOrphan(FilesFacade.INSTANCE, slotDir); @@ -199,20 +184,20 @@ static boolean removeOrphan(FilesFacade filesFacade, String slotDir) { } /** - * Single-load read of the current FSN. Returns {@link #INVALID} if - * the file has never been written (magic field is zero, i.e. the - * file was freshly created by {@link #open(String)} and no - * {@link #write(long)} has run yet against this slot). + * Returns the FSN from the greatest-generation valid record, or + * {@link #INVALID} when neither record validates. */ public long read() { if (closed) return INVALID; - int magic = Unsafe.getUnsafe().getInt(mmapAddress + MAGIC_OFFSET); - if (magic != FILE_MAGIC) { - // Either freshly created (all zeros) or some kind of corruption. - // Either way, fall back to the segment-derived seed. - return INVALID; + Record selected = selectRecord(mmapAddress); + if (selected == null) { + fsn = INVALID; + generation = 0L; + } else { + fsn = selected.fsn; + generation = selected.generation; } - return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET); + return fsn; } /** @@ -233,29 +218,65 @@ public void sync() { } /** - * Atomically updates the persisted FSN. Single 8-byte aligned - * store to the mapped region. The first write also stamps the - * magic so the next session's {@link #read()} can distinguish - * "valid watermark" from "freshly created file". + * Updates the inactive record and selects it in memory only after its CRC + * has been stored. The next {@link #sync()} makes the complete update + * durable before any covered segment is deleted. *

      - * Caller responsibility: monotonic ordering. The manager's tick - * loop should only call this when {@code fsn} has advanced past - * the last write. + * Caller responsibility: monotonic ordering. The manager's tick loop only + * calls this when {@code fsn} has advanced past the last write. */ public void write(long fsn) { if (closed) return; - // Steady-state hot path: a single 8-byte aligned putLong, no - // memory load of the magic. Order matters only on the very - // first write of a fresh file: FSN first so the next reader - // that observes the magic (set immediately below) also - // observes a valid FSN -- no fence needed because the same - // thread reads both in program order, and crash recovery - // resumes a fresh process that sees whatever disk-level state - // the kernel flushed. - Unsafe.getUnsafe().putLong(mmapAddress + FSN_OFFSET, fsn); - if (!magicWritten) { - Unsafe.getUnsafe().putInt(mmapAddress + MAGIC_OFFSET, FILE_MAGIC); - magicWritten = true; + long nextGeneration = generation + 1L; + long recordAddress = mmapAddress + (nextGeneration & 1L) * RECORD_SIZE; + Unsafe.getUnsafe().setMemory(recordAddress, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(recordAddress + MAGIC_OFFSET, FILE_MAGIC); + Unsafe.getUnsafe().putInt(recordAddress + 4, VERSION); + Unsafe.getUnsafe().putLong(recordAddress + 8, nextGeneration); + Unsafe.getUnsafe().putLong(recordAddress + FSN_OFFSET, fsn); + int crc = Crc32c.update(Crc32c.INIT, recordAddress, CRC_OFFSET); + Unsafe.getUnsafe().putInt(recordAddress + CRC_OFFSET, crc); + generation = nextGeneration; + this.fsn = fsn; + } + + private static Record readRecord(long address) { + if (Unsafe.getUnsafe().getInt(address + MAGIC_OFFSET) != FILE_MAGIC + || Unsafe.getUnsafe().getInt(address + 4) != VERSION) { + return null; + } + int expectedCrc = Unsafe.getUnsafe().getInt(address + CRC_OFFSET); + int actualCrc = Crc32c.update(Crc32c.INIT, address, CRC_OFFSET); + if (expectedCrc != actualCrc) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(address + 8); + long fsn = Unsafe.getUnsafe().getLong(address + FSN_OFFSET); + if (generation <= 0 || fsn < -1L) { + return null; + } + return new Record(generation, fsn); + } + + private static Record selectRecord(long address) { + Record first = readRecord(address); + Record second = readRecord(address + RECORD_SIZE); + if (first == null) { + return second; + } + if (second == null || first.generation > second.generation) { + return first; + } + return second; + } + + private static final class Record { + private final long fsn; + private final long generation; + + private Record(long generation, long fsn) { + this.fsn = fsn; + this.generation = generation; } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 97b2d7d0..e83d907e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -304,11 +304,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long nextWireSeq; private volatile SenderProgressDispatcher progressDispatcher; // Frames sent during the post-reconnect catch-up window — i.e. frames - // whose FSN was already published before the wire dropped. A non-zero + // whose FSN completed a send on an earlier connection. A non-zero // value confirms replay is working; a sustained nonzero rate means // the connection is flapping and replay is doing real work each cycle. - // Set at swapClient time to publishedFsn at that moment; cleared back - // to -1 once trySendOne has caught up past it. Used to count replay + // Snapshotted when an established connection enters reconnect; cleared + // back to -1 once trySendOne has caught up past it. Used to count replay // frames without a per-frame branch on the steady-state path. private long replayTargetFsn = -1L; // Recovered orphaned deferred tail: frames [orphanSkipStartFsn .. @@ -1227,6 +1227,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + snapshotReplayTarget(); LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); @@ -1931,6 +1932,23 @@ private void sendDurableAckKeepaliveIfDue() { } } + /** + * Freezes the highest FSN that completed a send on the connection which + * just failed. Frames published before the first connection, or while a + * reconnect is in progress, have not been sent and therefore do not belong + * to the replay metric. Keep an older, higher target when another failure + * interrupts catch-up so the remaining frames still count as re-sends on + * the following connection. + */ + private void snapshotReplayTarget() { + if (hasEverConnected && nextWireSeq > 0L) { + long lastSentFsn = fsnAtZero + (nextWireSeq - 1L); + if (lastSentFsn > replayTargetFsn) { + replayTargetFsn = lastSentFsn; + } + } + } + /** * Reset wire state for a fresh connection: install the new client, * realign {@code fsnAtZero} to the next unacked FSN, restart wire @@ -1959,11 +1977,12 @@ private void swapClient(WebSocketClient newClient) { long replayStart = engine.ackedFsn() + 1L; this.fsnAtZero = replayStart; this.nextWireSeq = 0L; - // Snapshot publishedFsn at swap time — frames at FSN ≤ this value - // were already on the wire before the drop and will be replayed. - // trySendOne resets replayTargetFsn to -1 once we cross the boundary. - long pubAtSwap = engine.publishedFsn(); - this.replayTargetFsn = pubAtSwap >= replayStart ? pubAtSwap : -1L; + // snapshotReplayTarget froze the completed-send boundary when the + // outage began. ACK progress can move replayStart past that boundary; + // in that case no frame needs re-sending on this connection. + if (replayTargetFsn < replayStart) { + replayTargetFsn = -1L; + } // Drop any durable-ack tracking from the previous connection. The // new connection will re-OK every replayed batch and the server // re-emits cumulative durable-ack watermarks from scratch, so diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8b8e752c..4dd24b4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -32,6 +32,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.IdentityHashMap; + /** * Chain of {@link MmapSegment}s presented to the user thread as one logical * append-only log keyed by frame sequence number (FSN). Owns segment @@ -63,6 +65,8 @@ public final class SegmentRing implements QuietCloseable { public static final long BACKPRESSURE_NO_SPARE = -1L; /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */ public static final long PAYLOAD_TOO_LARGE = -2L; + private static final RetainedSegmentMembershipMode DEFAULT_MEMBERSHIP_MODE = + RetainedSegmentMembershipMode.IDENTITY; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only // operation count for deterministic traversal-complexity assertions. @@ -176,6 +180,15 @@ public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, lo * all succeed does it migrate a legacy chain or discard validated spares. */ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + return recover(filesFacade, sfDir, maxBytesPerSegment, null); + } + + static Recovery recover( + FilesFacade filesFacade, + String sfDir, + long maxBytesPerSegment, + MembershipObserver membershipObserver + ) { if (!filesFacade.exists(sfDir)) { return Recovery.empty(); } @@ -433,53 +446,73 @@ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerS } } + // Build membership and clean validated extras while every mmap and + // the manifest still have one local owner. In particular, an OOM + // while allocating the identity map falls through to the outer + // failure cleanup instead of stranding extras after transfer. + RetainedSegmentMembership retained = newDefaultMembership(chain, membershipObserver); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (!retained.contains(segment)) { + String path = null; + long torn = 0; + Throwable inspectionError = null; + try { + path = segment.path(); + torn = segment.tornTailBytes(); + } catch (Throwable error) { + inspectionError = error; + } + // A close failure is not best-effort: keep local ownership + // and fail through the outer cleanup rather than transfer a + // ring while an extra may still own an mmap or fd. + segment.close(); + all.setQuick(i, null); + if (inspectionError != null) { + warnPostRecoveryCleanupFailure(inspectionError); + continue; + } + try { + cleanupClosedExtra(filesFacade, path, torn); + } catch (Throwable cleanupError) { + warnPostRecoveryCleanupFailure(cleanupError); + } + } + } + quarantineCorrupt(filesFacade, corruptPaths); + for (int i = 1, n = chain.size(); i < n; i++) { chain.get(i - 1).linkSuccessor(chain.get(i)); } SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest); - manifest = null; for (int i = 0, n = chain.size() - 1; i < n; i++) { ring.sealedSegments.add(chain.get(i)); } - // Ownership of the chain transferred. Clean up only validated - // extras; recovery is already successful, so cleanup failure - // must never turn startup into a partially-mutating error or - // orphan the constructed ring -- swallow and let the next - // startup re-examine the leftovers. Extras with a torn tail - // carry evidence of attempted writes -- keep the bytes under a - // .corrupt name instead of unlinking them. - try { - for (int i = 0, n = all.size(); i < n; i++) { - MmapSegment segment = all.get(i); - if (!containsIdentity(chain, segment)) { - String path = segment.path(); - long torn = segment.tornTailBytes(); + // Allocate the return wrapper before transfer. Until this succeeds, + // the outer catch remains the sole owner and can close all segments + // and the manifest if construction or sealed-list growth fails. + Recovery recovery = Recovery.recovered(ring); + manifest = null; + all.clear(); + return recovery; + } catch (Throwable t) { + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment != null) { + try { segment.close(); - if (torn > 0) { - quarantineFile(filesFacade, path); - } else if (!filesFacade.remove(path)) { - LOG.warn("could not remove validated stale/empty SF segment {}", path); - } + } catch (Throwable closeError) { + warnRecoveryCloseFailure(closeError); } } - all.clear(); - quarantineCorrupt(filesFacade, corruptPaths); - } catch (Throwable cleanupError) { - LOG.warn("post-recovery cleanup failed; leftover files will be " - + "re-examined on the next startup", cleanupError); } - return Recovery.recovered(ring); - } catch (Throwable t) { - for (int i = 0, n = all.size(); i < n; i++) { + if (manifest != null) { try { - all.get(i).close(); + manifest.close(); } catch (Throwable closeError) { - LOG.warn("error closing SF segment after recovery failure", closeError); + warnRecoveryCloseFailure(closeError); } } - if (manifest != null) { - manifest.close(); - } throw t; } } @@ -525,13 +558,81 @@ private static MmapSegment chooseEmptyInitial(ObjList all, String s return selected; } + private static void cleanupClosedExtra(FilesFacade filesFacade, String path, long torn) { + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove validated stale/empty SF segment {}", path); + } + } + + private static RetainedSegmentMembership newDefaultMembership( + final ObjList chain, + final MembershipObserver observer + ) { + if (observer != null) { + observer.beforeMembershipAllocation(); + } + if (DEFAULT_MEMBERSHIP_MODE == RetainedSegmentMembershipMode.LINEAR) { + if (observer == null) { + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + for (int i = 0, n = chain.size(); i < n; i++) { + if (chain.get(i) == segment) { + return true; + } + } + return false; + } + }; + } + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + for (int i = 0, n = chain.size(); i < n; i++) { + observer.onMembershipOperation(); + if (chain.get(i) == segment) { + return true; + } + } + return false; + } + }; + } + + final IdentityHashMap retained = new IdentityHashMap<>(chain.size()); + for (int i = 0, n = chain.size(); i < n; i++) { + retained.put(chain.get(i), Boolean.TRUE); + } + if (observer == null) { + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + return retained.containsKey(segment); + } + }; + } + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + observer.onMembershipOperation(); + return retained.containsKey(segment); + } + }; + } + /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */ private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) { if (corruptPaths == null) { return; } for (int i = 0, n = corruptPaths.size(); i < n; i++) { - quarantineFile(filesFacade, corruptPaths.get(i)); + try { + quarantineFile(filesFacade, corruptPaths.get(i)); + } catch (Throwable cleanupError) { + warnPostRecoveryCleanupFailure(cleanupError); + } } } @@ -542,13 +643,6 @@ private static void quarantineFile(FilesFacade filesFacade, String path) { } } - private static boolean containsIdentity(ObjList list, MmapSegment value) { - for (int i = 0, n = list.size(); i < n; i++) { - if (list.get(i) == value) return true; - } - return false; - } - /** * Locates the segment the manifest's {@code activeBase} refers to. * Preference order among same-base candidates: @@ -601,6 +695,29 @@ private static void validateContiguous(ObjList segments) { } } + private static void warnPostRecoveryCleanupFailure(Throwable cleanupError) { + try { + LOG.warn("post-recovery cleanup failed; leftover files will be " + + "re-examined on the next startup", cleanupError); + } catch (Throwable ignored) { + // Cleanup diagnostics must not invalidate a recovered ring. + } + } + + private static void warnRecoveryCloseFailure(Throwable closeError) { + try { + LOG.warn("error closing SF resource after recovery failure", closeError); + } catch (Throwable ignored) { + // Preserve the original recovery failure and continue closing. + } + } + + interface MembershipObserver { + void beforeMembershipAllocation(); + + void onMembershipOperation(); + } + static final class Recovery { private final SegmentRing ring; private final RecoveryStatus status; @@ -632,6 +749,15 @@ enum RecoveryStatus { RECOVERED } + interface RetainedSegmentMembership { + boolean contains(MmapSegment segment); + } + + private enum RetainedSegmentMembershipMode { + IDENTITY, + LINEAR + } + /** * Highest FSN that the server has ACK'd. Read by the segment manager to * decide which sealed segments are safe to munmap + unlink. diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java index da9249c5..1fc86798 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java +++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java @@ -27,6 +27,8 @@ import io.questdb.client.QueryException; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; import org.jetbrains.annotations.TestOnly; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayDeque; import java.util.ArrayList; @@ -56,6 +58,11 @@ public final class QueryClientPool implements AutoCloseable { // close() can never block the caller unbounded. Tunable per pool via // closeQueryTimeoutMillis(long). static final long DEFAULT_CLOSE_QUERY_TIMEOUT_MILLIS = 5_000; + // Hard cap on close()'s wait for a creation blocked in DNS, TCP, TLS, or + // WebSocket setup. A late creator retains the client and native-scratch + // cleanup obligation until it returns and observes the closed pool. + static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000; + private static final Logger LOG = LoggerFactory.getLogger(QueryClientPool.class); private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; @@ -319,13 +326,34 @@ public void close() { } closed = true; workerReleased.signalAll(); - // A reservation owns every resource created outside the lock until - // the worker is either published in `all` or fully torn down. No - // user ever received these workers, so close() must not apply an - // abandoned-lease timeout. Preserve interrupts while waiting for - // the internal ownership count to reach zero. - while (inFlightCreations > 0) { - creationFinished.awaitUninterruptibly(); + // A creator can block in DNS, TCP, TLS, WebSocket upgrade, or the + // SERVER_INFO handshake. Wait boundedly rather than turning an + // unset connect_timeout into an unbounded QuestDB.close(). Timing + // out does not abandon the client or its native scratch: the creator + // keeps the reservation and, once construction returns, observes + // closed and shuts the worker down before releasing ownership. + // Preserve interruption while applying the same finite budget. + final long creationWaitMillis = Math.max(0, + Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS)); + final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis); + final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos; + long creationRemainingNanos = creationWaitNanos; + boolean creationWaitInterrupted = false; + while (inFlightCreations > 0 && creationRemainingNanos > 0) { + try { + creationFinished.awaitNanos(creationRemainingNanos); + } catch (InterruptedException e) { + creationWaitInterrupted = true; + } + creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime(); + } + if (creationWaitInterrupted) { + Thread.currentThread().interrupt(); + } + if (inFlightCreations > 0) { + LOG.warn("QueryClientPool.close(): {} query client creation(s) still in flight after {}ms; " + + "each creator retains cleanup ownership until construction returns", + inFlightCreations, creationWaitMillis); } snapshot = new ArrayList<>(all); } finally { diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index 3545d3fa..3d7be5d9 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -181,15 +181,15 @@ public Sender borrowSender() { return senderPool.borrow(); } - // synchronized so concurrent close() callers serialize THROUGH shutdown - // completion, not merely through the `closed` flip. `closed` is set before - // the teardown chain runs, so a plain volatile guard (or a bare CAS) would - // let a second caller observe closed==true and return while the first is - // still inside closeQuietly(senderPool) releasing the flock/mmap/I/O-thread - // resources -- a premature return that breaks the AutoCloseable contract - // that shutdown has completed once close() returns. The monitor makes the - // losing caller block until the winner finishes, then it enters, sees - // `closed` and returns a no-op. No deadlock: the teardown steps + // synchronized so concurrent close() callers serialize THROUGH the bounded + // shutdown sequence, not merely through the `closed` flip. `closed` is set + // before the teardown chain runs, so a plain volatile guard (or a bare CAS) + // would let a second caller observe closed==true and return while the first + // is still closing published resources. The monitor makes the losing caller + // block until the winner finishes, then it enters, sees `closed` and returns + // a no-op. A pool creator that outlives its finite shutdown wait retains and + // eventually cleans its unpublished resources on the creator thread; this + // monitor does not wait past that budget. No deadlock: the teardown steps // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close()) // never call back into QuestDBImpl.close() on another thread, so nothing // contends for this monitor from within the critical section. diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 32540026..66f09d90 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -96,6 +96,11 @@ public final class SenderPool implements AutoCloseable { // -- the residual window documented on recoverOneSlotStep -- because the // transport has no application-level connect timeout to clamp it. private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000; + // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP, + // TLS, or WebSocket setup. The creator retains ownership after this budget: + // once construction returns, borrow() observes closed and tears down the + // delegate before releasing its reservation (including an SF slot). + static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000; // Hard cap on close()'s outstanding-lease wait. The acquire timeout is a // BORROW policy -- Long.MAX_VALUE legitimately means "block until a slot // frees" -- and must never unbound SHUTDOWN: without this cap a forgotten @@ -937,8 +942,9 @@ void markClosing() { * free their native memory under its feet -- a use-after-free / SEGV, not * an exception (C1). Instead: *

        - *
      1. waits for every internally owned creation to publish or complete its - * closed-mid-creation teardown; then
        2. + *
      2. waits boundedly for internally owned creations; a late creator keeps + * ownership and performs its closed-mid-creation teardown asynchronously; + * then
        3. *
      3. waits boundedly (up to {@code acquireTimeoutMillis}, hard-capped at * {@link #MAX_CLOSE_LEASE_WAIT_MILLIS}) for outstanding leases to come home -- {@link #giveBack} and {@link #discardBroken} * observe {@code closed} and tear each delegate down on the returning @@ -965,14 +971,34 @@ public void close() { closed = true; // Wake parked borrowers so they observe the shutdown and throw. slotReleased.signalAll(); - // A creation reservation represents pool-owned resources from the - // moment borrow() leaves the lock through either publication or - // completed cleanup. Unlike an abandoned user-visible lease, this - // ownership cannot be leaked after close() returns, so wait without - // a timeout. awaitUninterruptibly preserves the interrupt flag while - // maintaining the shutdown ownership invariant. - while (inFlightCreations > 0) { - creationFinished.awaitUninterruptibly(); + // A creator can block in DNS, TCP, TLS, or WebSocket setup. Wait + // boundedly rather than turning an unset connect_timeout into an + // unbounded QuestDB.close(). Timing out does not abandon ownership: + // the reservation and any SF slot stay assigned to the creator, + // which observes closed and tears down a late result before releasing + // them. Preserve the caller's interrupt status while still applying + // the same finite wait budget. + final long creationWaitMillis = Math.max(0, + Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS)); + final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis); + final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos; + long creationRemainingNanos = creationWaitNanos; + boolean creationWaitInterrupted = false; + while (inFlightCreations > 0 && creationRemainingNanos > 0) { + try { + creationFinished.awaitNanos(creationRemainingNanos); + } catch (InterruptedException e) { + creationWaitInterrupted = true; + } + creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime(); + } + if (creationWaitInterrupted) { + Thread.currentThread().interrupt(); + } + if (inFlightCreations > 0) { + LOG.warn("SenderPool.close(): {} sender creation(s) still in flight after {}ms; " + + "each creator retains cleanup ownership and releases its SF slot when construction returns", + inFlightCreations, creationWaitMillis); } // Bounded graceful wait for outstanding leases. A slot is borrowed // iff it is in `all` but not in `available`; retireLease's diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java index 173374ec..5f04ca68 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java @@ -27,6 +27,7 @@ import io.questdb.client.Sender; import io.questdb.client.SenderError; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.jetbrains.annotations.NotNull; import org.junit.Assert; @@ -43,8 +44,10 @@ import java.nio.charset.StandardCharsets; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; /** * Behavior of {@code initial_connect_retry=async}: the producer-thread @@ -205,6 +208,78 @@ public void testAsyncDeliversBufferedRowsWhenServerArrivesLate() throws Exceptio } } + @Test + public void testAsyncMetricsDistinguishInitialSendFromReconnectReplay() throws Exception { + ReplayMetricsHandler handler = new ReplayMetricsHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + sfDirOpt() + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=200" + + ";close_flush_timeout_millis=2000;"; + Sender sender = Sender.fromConfig(cfg); + try { + QwpWebSocketSender wss = (QwpWebSocketSender) sender; + handler.bind(wss, server); + + // Publish before the server starts accepting. The frame has + // never been on any wire, so its eventual first delivery must + // increase sent, but not replayed. + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + awaitAtLeastOneConnectAttempt(wss); + Assert.assertEquals("the server must not handshake before start", 0, server.handshakeCount()); + + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("the initial frame must be ACKed", + handler.awaitInitialAck(5, TimeUnit.SECONDS)); + awaitMetricAtLeast("initial ACK", wss::getTotalAcks, 1L); + Assert.assertEquals("exactly one frame must be sent initially", + 1L, wss.getTotalFramesSent()); + long replayedAfterInitialDelivery = wss.getTotalFramesReplayed(); + + // The second frame reaches the established connection, but the + // fixture closes that connection without ACKing it. It must then + // arrive again on a genuinely new connection and count once as + // replayed. + sender.table("foo").longColumn("v", 2L).atNow(); + sender.flush(); + Assert.assertTrue("the fixture must close after the unacked frame", + handler.awaitDisconnect(5, TimeUnit.SECONDS)); + Assert.assertTrue("the reconnect must reach the gated server", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + + // Publish another frame while reconnect is blocked at a role + // reject, then reopen the gate. This frame has never been sent + // and must not inflate the replay counter after reconnect. + sender.table("foo").longColumn("v", 3L).atNow(); + sender.flush(); + server.setRejectWithRole(null); + + Assert.assertTrue("the unacked frame must be replayed", + handler.awaitReplay(5, TimeUnit.SECONDS)); + Assert.assertTrue("the outage-queued frame must be delivered", + handler.awaitOutageQueuedDelivery(5, TimeUnit.SECONDS)); + awaitMetricAtLeast("replayed frame", wss::getTotalFramesReplayed, 1L); + awaitMetricAtLeast("post-reconnect ACKs", wss::getTotalAcks, 3L); + + Assert.assertTrue("replay must arrive on a new WebSocket connection", + handler.wasReplayOnNewConnection()); + Assert.assertTrue("the server must observe a reconnect", + server.handshakeCount() >= 2); + Assert.assertEquals("initial delivery is not a replay", + 0L, replayedAfterInitialDelivery); + Assert.assertEquals("three first sends plus one genuine resend", + 4L, wss.getTotalFramesSent()); + Assert.assertEquals("only the genuine resend is replayed", + 1L, wss.getTotalFramesReplayed()); + } finally { + closeQuietly(sender); + } + } + } + @Test public void testAsyncReturnsImmediatelyWithNoServer() { // No server. With async mode, fromConfig must return fast — the @@ -345,6 +420,17 @@ private static void awaitAtLeastOneConnectAttempt(QwpWebSocketSender wss) { * before the budget expired would satisfy it even if the loop then * exited. */ + private static void awaitMetricAtLeast(String label, LongSupplier metric, long expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (metric.getAsLong() < expected) { + if (System.nanoTime() > deadlineNanos) { + throw new AssertionError(label + " did not reach " + expected + + " within 5s; current=" + metric.getAsLong()); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + private static void awaitReconnectAttemptsAdvance(QwpWebSocketSender wss) { long snapshot = wss.getTotalReconnectAttempts(); long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); @@ -476,6 +562,83 @@ public void onError(@NotNull SenderError err) { } } + private static class ReplayMetricsHandler implements TestWebSocketServer.WebSocketServerHandler { + private final CountDownLatch disconnect = new CountDownLatch(1); + private final CountDownLatch initialAck = new CountDownLatch(1); + private final CountDownLatch outageQueuedDelivery = new CountDownLatch(1); + private final AtomicLong received = new AtomicLong(); + private final CountDownLatch replay = new CountDownLatch(1); + private final AtomicBoolean replayOnNewConnection = new AtomicBoolean(); + private TestWebSocketServer.ClientHandler initialClient; + private volatile TestWebSocketServer server; + private volatile QwpWebSocketSender sender; + + boolean awaitDisconnect(long timeout, TimeUnit unit) throws InterruptedException { + return disconnect.await(timeout, unit); + } + + boolean awaitInitialAck(long timeout, TimeUnit unit) throws InterruptedException { + return initialAck.await(timeout, unit); + } + + boolean awaitOutageQueuedDelivery(long timeout, TimeUnit unit) throws InterruptedException { + return outageQueuedDelivery.await(timeout, unit); + } + + boolean awaitReplay(long timeout, TimeUnit unit) throws InterruptedException { + return replay.await(timeout, unit); + } + + void bind(QwpWebSocketSender sender, TestWebSocketServer server) { + this.sender = sender; + this.server = server; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long ordinal = received.incrementAndGet(); + try { + if (ordinal == 1L) { + initialClient = client; + client.sendBinary(AckHandler.buildAck(0L)); + initialAck.countDown(); + } else if (ordinal == 2L) { + // Receipt alone can race the sender's post-send metric + // increments. Wait until sendBinary returned before closing, + // proving this frame completed a send and must be replayed. + awaitSentFrames(2L); + server.setRejectWithRole("REPLICA"); + client.sendClose(WebSocketCloseCode.GOING_AWAY, "test reconnect"); + disconnect.countDown(); + } else if (ordinal == 3L) { + replayOnNewConnection.set(client != initialClient); + client.sendBinary(AckHandler.buildAck(0L)); + replay.countDown(); + } else if (ordinal == 4L) { + client.sendBinary(AckHandler.buildAck(1L)); + outageQueuedDelivery.countDown(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + boolean wasReplayOnNewConnection() { + return replayOnNewConnection.get(); + } + + private void awaitSentFrames(long expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (sender.getTotalFramesSent() < expected) { + if (System.nanoTime() > deadlineNanos) { + throw new AssertionError("sent-frame metric did not reach " + expected + + " before disconnect; current=" + sender.getTotalFramesSent()); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + } + /** * Raw-socket fixture: every accepted connection responds with HTTP * 401 Unauthorized and closes. Used to drive the async-init diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java index 001f62ce..27bef939 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Before; @@ -76,6 +78,45 @@ public void testCrossSessionPersistence() throws Exception { }); } + @Test + public void testFallsBackFromTornPlausibleHighValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + watermark.write(255L); + watermark.write(256L); + watermark.sync(); + } + + // Model a torn little-endian 255 -> 256 transition: the high byte + // reached storage while the low byte retained 0xff, yielding the + // false FSN 511. A recovered ring with publishedFsn=599 cannot + // reject that value using its segment ceiling. + long publishedFsn = 599L; + long corruptFsn = 511L; + assertTrue(corruptFsn <= publishedFsn); + String path = slotDir + "/" + AckWatermark.FILE_NAME; + int fd = Files.openRW(path); + assertTrue(fd >= 0); + long corruptByte = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(corruptByte, (byte) 0xff); + long fsnOffset = AckWatermark.FILE_SIZE == 16 ? 8L : 16L; + assertEquals(1L, Files.write(fd, corruptByte, 1, fsnOffset)); + assertEquals(0, Files.fsync(fd)); + } finally { + Files.close(fd); + Unsafe.free(corruptByte, 1, MemoryTag.NATIVE_DEFAULT); + } + + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + assertEquals("torn latest record must fall back to the older watermark", + 255L, watermark.read()); + } + }); + } + @Test public void testFreshFileReadsAsInvalid() throws Exception { // open() creates the file zero-filled, so magic is 0 and read() diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java new file mode 100644 index 00000000..d8813b7f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SegmentRingMembershipTest { + + private static final String CURSOR_PACKAGE = "io.questdb.client.cutlass.qwp.client.sf.cursor."; + private static final long SEGMENT_SIZE = MmapSegment.HEADER_SIZE + + MmapSegment.FRAME_HEADER_SIZE + 16; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-ring-membership-" + System.nanoTime()).toString(); + assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) { + return; + } + long find = Files.findFirst(tmpDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(tmpDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(tmpDir); + } + + @Test + public void testLargeRecoveryCountsMembershipPrimitives() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 2048; + createChain(n); + + CountingOperations operations = new CountingOperations(); + try (SegmentRing ring = recover(newMembershipObserver(operations, false))) { + assertRecoveredChain(ring, n); + } + long linearBound = 2L * n; + assertTrue("production default took " + operations.count + + " membership operations; expected fewer than " + linearBound, + operations.count < linearBound); + assertEquals("production default identity map must do one lookup per discovered segment", + n, operations.count); + }); + } + + @Test + public void testMembershipAllocationFailureRetainsLocalOwnership() throws Exception { + TestUtils.assertMemoryLeak(() -> { + createChain(8); + String extraPath = tmpDir + "/sf-extra.sfa"; + MmapSegment.create(extraPath, 8, SEGMENT_SIZE).close(); + try { + recover(newMembershipObserver(null, true)); + fail("expected injected membership allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("injected membership allocation failure", expected.getMessage()); + } + + try (SegmentRing ring = SegmentRing.openExisting( + FilesFacade.INSTANCE, + tmpDir, + SEGMENT_SIZE + )) { + assertRecoveredChain(ring, 8); + } + assertFalse("retry must close and remove the unselected empty extra", Files.exists(extraPath)); + }); + } + + private static void assertRecoveredChain(SegmentRing ring, int n) { + assertNotNull(ring); + assertEquals(n, ring.nextSeqHint()); + assertEquals(n - 1, ring.publishedFsn()); + assertEquals(n - 1, ring.getSealedSegments().size()); + } + + private void createChain(int n) { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) i); + } + for (int i = 0; i < n; i++) { + String name = String.format("sf-%05d.sfa", i); + try (MmapSegment segment = MmapSegment.create(tmpDir + "/" + name, i, SEGMENT_SIZE)) { + assertTrue("setup append " + i, segment.tryAppend(buf, 16) >= 0); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + private SegmentRing recover(Object membershipObserver) throws Exception { + Class observerClass = Class.forName(CURSOR_PACKAGE + "SegmentRing$MembershipObserver"); + Method recover = SegmentRing.class.getDeclaredMethod( + "recover", + FilesFacade.class, + String.class, + long.class, + observerClass + ); + recover.setAccessible(true); + Object recovery; + try { + recovery = recover.invoke(null, FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE, membershipObserver); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Error) { + throw (Error) cause; + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new AssertionError(cause); + } + Method ring = recovery.getClass().getDeclaredMethod("ring"); + ring.setAccessible(true); + return (SegmentRing) ring.invoke(recovery); + } + + private static Object newMembershipObserver( + final CountingOperations operations, + final boolean failAllocation + ) throws Exception { + Class observerClass = Class.forName(CURSOR_PACKAGE + "SegmentRing$MembershipObserver"); + return Proxy.newProxyInstance( + SegmentRing.class.getClassLoader(), + new Class[]{observerClass}, + (proxy, method, args) -> { + if ("beforeMembershipAllocation".equals(method.getName())) { + if (failAllocation) { + throw new OutOfMemoryError("injected membership allocation failure"); + } + } else if ("onMembershipOperation".equals(method.getName())) { + operations.onOperation(); + } + return null; + } + ); + } + + private static final class CountingOperations { + private long count; + + private void onOperation() { + count++; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index c6466de7..9bc08a6a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -616,22 +616,19 @@ public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Ex /** * Open-time sort regression: at the documented {@code sf_max_total_bytes - * / sf_max_bytes} ceiling (~16K segments) an O(N²) sort over the - * recovered segments burns multi-second wall time before the I/O thread - * can start. The previous selection-sort implementation regressed an - * earlier perf fix on the legacy {@code SegmentLog} path; this test - * guards the cursor path against the same regression. + * / sf_max_bytes} ceiling (~16K segments), an O(N²) sort over the + * recovered segments delays the I/O thread. This test guards the sort + * with a deterministic comparison count. *

        * Constructs N=2048 valid one-frame segments with names assigned in * lexicographic order — the exact pattern {@code readdir} produces on * many filesystems (and the worst case for a naive first-element pivot). * Recovers, asserts contiguous baseSeq ordering and total frame count, - * and bounds wall time at 5 s. With the median-of-three quicksort the - * test completes in well under a second; an O(N²) regression at this - * scale climbs back into multi-second territory. + * and bounds sort comparisons independently of CI timing or filesystem + * throughput. */ @Test - public void testLargeSegmentCountReopensInOrder() throws Exception { + public void testLargeSegmentCountRecoveryIsLogLinear() throws Exception { TestUtils.assertMemoryLeak(() -> { final int n = 2048; long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); @@ -663,6 +660,8 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { n, ring.nextSeqHint()); // publishedFsn = n - 1 (last frame visible). assertEquals(n - 1, ring.publishedFsn()); + assertEquals("full valid chain must retain every segment", + n - 1, ring.getSealedSegments().size()); // O(N log N) quicksort with good pivots does ~2-3 * N * log2(N) // comparisons; the partition-pass + median-of-three counter we // increment per recursive frame upper-bounds this at roughly @@ -688,7 +687,7 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { /** * Adversarial-order companion to - * {@link #testLargeSegmentCountReopensInOrder}: that test covers the + * {@link #testLargeSegmentCountRecoveryIsLogLinear}: that test covers the * already-sorted readdir order median-of-three was chosen for; this one * covers the orders median-of-three does NOT defend. On the * pre-introsort quicksort, exact simulation at N=16384 measured ~22.6M diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java index dd697049..65e57f85 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java @@ -31,6 +31,8 @@ import io.questdb.client.impl.QueryClientPool; import io.questdb.client.impl.QuestDBImpl; import io.questdb.client.impl.SenderPool; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -52,6 +54,294 @@ public class QuestDBImplCloseLifecycleTest { private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + @Test(timeout = 30_000) + public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringQueryCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(teardownCount::incrementAndGet); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB( + SENDER_CFG, 0, 0, 0, slotIndex -> fakeSender(null, null, null), connectHook); + QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "bounded-query-borrower"); + Thread closer = new Thread(db::close, "bounded-query-closer"); + long nativeBaseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.inFlightCreations()); + + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "facade close exceeded its zero creation-wait budget", + closer.isAlive()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.inFlightCreations()); + Assert.assertEquals("the still-constructing query client must remain live", 0, teardownCount.get()); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late query creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late query creation reservation must be released", 0, pool.inFlightCreations()); + Assert.assertEquals("late query cleanup must release native scratch", + nativeBaseline, Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT)); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringSenderCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, null, null); + }; + String senderConfig = "ws::addr=localhost:1;sf_dir=" + + System.getProperty("java.io.tmpdir") + "/qdb-bounded-pool-" + System.nanoTime() + ";"; + QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 0, senderFactory, client -> { + }); + SenderPool pool = (SenderPool) getField(db, "senderPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "bounded-sender-borrower"); + Thread closer = new Thread(db::close, "bounded-sender-closer"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("SF slot must stay reserved during creation", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "facade close exceeded its zero creation-wait budget", + closer.isAlive()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("close must not abandon the reserved SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late sender creation reservation must be released", + 0, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertFalse("late sender cleanup must release the SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringQueryCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(teardownCount::incrementAndGet); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB( + SENDER_CFG, 0, 0, 100, slotIndex -> fakeSender(null, null, null), connectHook); + QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + AtomicBoolean keepInterrupting = new AtomicBoolean(true); + AtomicInteger interruptCount = new AtomicInteger(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "interrupted-query-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "interrupted-query-closer"); + Thread interrupter = new Thread(() -> { + while (keepInterrupting.get()) { + interruptCount.incrementAndGet(); + closer.interrupt(); + Thread.yield(); + } + }, "query-close-interrupter"); + long nativeBaseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.inFlightCreations()); + + closer.start(); + awaitCreationWaiter(pool, + "facade close did not wait while query construction was internally owned"); + interrupter.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "repeated interrupts restarted the query creation-wait deadline", + closer.isAlive()); + Assert.assertTrue("test did not repeatedly interrupt query close", interruptCount.get() > 1); + Assert.assertTrue("facade close must restore query closer interruption", + closeReturnedInterrupted.get()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.inFlightCreations()); + Assert.assertEquals("the still-constructing query client must remain live", 0, teardownCount.get()); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late query creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late query creation reservation must be released", 0, pool.inFlightCreations()); + Assert.assertEquals("late query cleanup must release native scratch", + nativeBaseline, Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT)); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + keepInterrupting.set(false); + releaseCreation.countDown(); + interrupter.join(TimeUnit.SECONDS.toMillis(10)); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringSenderCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, null, null); + }; + String senderConfig = "ws::addr=localhost:1;sf_dir=" + + System.getProperty("java.io.tmpdir") + "/qdb-interrupted-pool-" + System.nanoTime() + ";"; + QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 100, senderFactory, client -> { + }); + SenderPool pool = (SenderPool) getField(db, "senderPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + AtomicBoolean keepInterrupting = new AtomicBoolean(true); + AtomicInteger interruptCount = new AtomicInteger(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "interrupted-sender-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "interrupted-sender-closer"); + Thread interrupter = new Thread(() -> { + while (keepInterrupting.get()) { + interruptCount.incrementAndGet(); + closer.interrupt(); + Thread.yield(); + } + }, "sender-close-interrupter"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("SF slot must stay reserved during creation", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + closer.start(); + awaitCreationWaiter(pool, + "facade close did not wait while sender construction was internally owned"); + interrupter.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "repeated interrupts restarted the sender creation-wait deadline", + closer.isAlive()); + Assert.assertTrue("test did not repeatedly interrupt sender close", interruptCount.get() > 1); + Assert.assertTrue("facade close must restore sender closer interruption", + closeReturnedInterrupted.get()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("close must not abandon the reserved SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late sender creation reservation must be released", + 0, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertFalse("late sender cleanup must release the SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + keepInterrupting.set(false); + releaseCreation.countDown(); + interrupter.join(TimeUnit.SECONDS.toMillis(10)); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + @Test(timeout = 30_000) public void facadeCloseWaitsForQueryCreationAndTeardown() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -281,12 +571,23 @@ private static QuestDBImpl newQuestDB( int queryMin, IntFunction senderFactory, Consumer connectHook + ) { + return newQuestDB(SENDER_CFG, senderMin, queryMin, 10_000L, senderFactory, connectHook); + } + + private static QuestDBImpl newQuestDB( + String senderConfig, + int senderMin, + int queryMin, + long acquireTimeoutMillis, + IntFunction senderFactory, + Consumer connectHook ) { return new QuestDBImpl( - SENDER_CFG, QUERY_CFG, + senderConfig, QUERY_CFG, senderMin, 1, queryMin, 1, - 10_000L, + acquireTimeoutMillis, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, From 94a0ae6b72e31f30ace12798a631ca2efd3dabbc Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Tue, 14 Jul 2026 19:26:48 +0100 Subject: [PATCH 25/64] Isolate flock retry ordering test EngineClosePublishAfterFlockReleaseTest started the shared retry driver after injecting an invalid slot-lock descriptor. It then returned before the driver's initial park expired. A following test could observe the global driver as active and fail, as Windows CI did. Inject a synchronous retry-driver start failure for this explicit-close retry test. Restore the factory in a nested finally block. Dedicated retry-driver tests continue to cover the asynchronous lifecycle. --- ...gineClosePublishAfterFlockReleaseTest.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index c142dd6a..0c28ea16 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -200,6 +200,15 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws E fdField.setAccessible(true); int realFd = fdField.getInt(slotLock); assertTrue("precondition: live flock fd", realFd >= 0); + + // This test owns the explicit close() retry. Prevent the shared + // asynchronous retry driver from racing it and from surviving into + // another test class; FlockReleaseRetryDriverTest covers the real + // driver lifecycle separately. A start failure synchronously clears + // the shared retry queue and resets the engine's scheduling flag. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + throw new IllegalStateException("retry driver disabled for explicit close retry test"); + }); try { // A non-negative fd no process has open: close(2) fails EBADF, // so finishClose's release() confirmation fails. @@ -230,11 +239,15 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws E // good — eventual completion implies reusable capacity. } } finally { - // If an assertion failed before the successful retry, restore - // and release the real fd so the test never leaks a flock. - if (!engine.isCloseCompleted()) { - fdField.setInt(slotLock, realFd); - assertTrue("restored fd must release cleanly", slotLock.release()); + try { + // If an assertion failed before the successful retry, restore + // and release the real fd so the test never leaks a flock. + if (!engine.isCloseCompleted()) { + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } + } finally { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); } } }); From d5673b3ffa5c66e3bb0d92bcf3191a4471ada9b1 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Tue, 14 Jul 2026 20:50:15 +0100 Subject: [PATCH 26/64] Continue direct pool startup recovery Direct SenderPool instances now continue store-and-forward recovery after a transient failure or startup budget exhaustion. The recovery driver uses elapsed-time budget accounting, throttles retries, and shuts down before delegate teardown. Guard driver startup so constructor failures quiesce the driver and close prewarmed delegates. Add deterministic recovery, lifecycle, cleanup, and mutation-sensitive tests. --- .../io/questdb/client/impl/SenderPool.java | 321 ++++++++--- .../client/test/impl/SenderPoolSfTest.java | 514 +++++++++++++++++- 2 files changed, 752 insertions(+), 83 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 66f09d90..49cac21d 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -40,6 +40,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Iterator; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -85,8 +86,8 @@ public final class SenderPool implements AutoCloseable { // Per-slot wall-clock cap on a single startup-recovery drain. Kept BELOW the // PoolHousekeeper stop/join budget (PoolHousekeeper.STOP_TIMEOUT_MILLIS) so a // recovery drain still in flight when close() arrives cannot outlive the - // housekeeper join -- the residual-budget bound that, together with the - // early markClosing() signal, keeps close() prompt (C1 fix). + // driver join -- the residual-budget bound that, together with the early + // markClosing() signal, keeps close() prompt (C1 fix). // // This caps only the DRAIN. The recovery build that precedes it is bounded // separately: recovery delegates force initial_connect_mode=OFF (see @@ -96,6 +97,10 @@ public final class SenderPool implements AutoCloseable { // -- the residual window documented on recoverOneSlotStep -- because the // transport has no application-level connect timeout to clamp it. private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000; + // A direct SenderPool has no PoolHousekeeper to retry a transient startup + // recovery failure. Its private driver waits this long between failed + // attempts so an unavailable server does not cause a hot retry loop. + private static final long RECOVERY_RETRY_INTERVAL_MILLIS = 1_000; // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP, // TLS, or WebSocket setup. The creator retains ownership after this budget: // once construction returns, borrow() observes closed and tears down the @@ -111,6 +116,11 @@ public final class SenderPool implements AutoCloseable { private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; + // Test-only constructor seam. Runs immediately before a possibly-started + // recovery driver is joined after constructor failure. Null in production; + // lifecycle tests use it to release a deliberately held driver and prove + // delegate cleanup happens only after the join. + private final Runnable beforeFailedStartupRecoveryJoinHook; private final String configurationString; // Signals completion of internally owned creation lifecycles. Kept // separate from slotReleased so a capacity waiter cannot consume the only @@ -134,13 +144,13 @@ public final class SenderPool implements AutoCloseable { private final Runnable postFactoryHook; // Factory for startup-recovery delegates. Distinct from senderFactory so a // recoverer can force a non-blocking initial connect (initial_connect_mode= - // OFF) regardless of user config: a recovery build runs on the - // PoolHousekeeper thread and must NOT inherit SYNC (auto-enabled by any - // reconnect_* knob), which would retry the connect for the whole reconnect - // budget inside build() -- far past PoolHousekeeper.STOP_TIMEOUT_MILLIS, so - // a close() landing during that build would make housekeeper.stop()'s join - // time out and leave the recoverer holding the slot flock after close() - // returned (M1). Mirrors senderFactory's test seam: an injected factory + // OFF) regardless of user config: a recovery build runs on a private direct + // driver or the PoolHousekeeper thread and must NOT inherit SYNC + // (auto-enabled by any reconnect_* knob), which would retry the connect for + // the whole reconnect budget inside build() -- far past + // PoolHousekeeper.STOP_TIMEOUT_MILLIS, so a close() landing during that + // build could make its driver join time out and leave the recoverer holding + // the slot flock after close() returned (M1). Mirrors senderFactory's test seam: an injected factory // (non-null) drives BOTH paths so white-box recovery tests keep control. private final IntFunction recoverySenderFactory; private final ReentrantLock lock = new ReentrantLock(); @@ -161,11 +171,28 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Direct pools run one inline recovery attempt for backwards-compatible + // startup behavior, then this daemon continues any backlog left by budget + // exhaustion or a transient failure. Deferred pools use PoolHousekeeper and + // leave this null. + private final Object startupRecoverySignal = new Object(); + private final Thread startupRecoveryThread; + // Test-only constructor seam for the failed-retry operation. Production + // uses waitForStartupRecoveryRetry(), which performs the one-second signal + // wait; lifecycle tests inject an event barrier without wall-clock checks. + private final Runnable startupRecoveryWaiter; + // Test seam: runs after the direct recovery driver's bounded join returns. + // Null in production; lifecycle tests prove the joined thread is quiescent. + private volatile Runnable afterStartupRecoveryJoinHook; // Test seam: runs immediately before a capacity-starved borrow enters its // condition wait, while it still holds the pool lock. Null in production; // concurrency tests use a latch here to prove that several borrowers have // all reached the wait path before recovering retired capacity. private volatile Runnable beforeBorrowWaitHook; + // Test seam: runs immediately before the direct recovery driver's bounded + // join. Null in production; lifecycle tests coordinate close() with a + // deliberately held driver. + private volatile Runnable beforeStartupRecoveryJoinHook; // Test seam: runs after a capacity-starved borrow's condition wait has // exhausted its positive timeout, before the loop's terminal pass. Null in // production; regression tests release a retired slot here to prove that @@ -223,14 +250,15 @@ public final class SenderPool implements AutoCloseable { // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a // concurrent borrow can neither over-allocate past maxSize nor target a - // dir being recovered. Only ever non-zero on the deferred (housekeeper- - // driven) recovery path, where recovery overlaps borrow()/return; on the - // inline construction path the pool is still single-threaded. Guarded by - // lock; only ever ticks for SF slots. + // dir being recovered. The inline constructor drive is single-threaded, + // but the continuing direct-pool driver and deferred housekeeper driver can + // overlap borrow()/return after publication. Guarded by lock; only ever + // ticks for SF slots. private int recoveringSlots; // Resumable startup-recovery scan cursor. Advanced only by the single - // recovery driver -- the inline constructor loop (single-threaded, - // unpublished) or the PoolHousekeeper thread (the sole deferred driver) -- + // recovery driver -- the inline constructor loop followed by its private + // direct-pool thread, or the PoolHousekeeper thread (the sole deferred + // driver) -- // so the cursor itself needs no lock; the per-slot reservation it performs // (slotInUse/recoveringSlots) is still taken under `lock` because borrow() // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) @@ -253,7 +281,7 @@ public SenderPool( long maxLifetimeMillis ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, - idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null); + idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null, null, null, null, null); } // Test-only constructor exposing the senderFactory seam: production builds @@ -282,8 +310,8 @@ public SenderPool( // reachable-but-not-acking server; the owner (QuestDBImpl) then drives // recovery one slot per tick on the PoolHousekeeper thread via // runStartupRecoveryStep(). White-box SF tests call this directly; the - // in-range recovery pass is concurrency-safe against borrow()/return on the - // deferred path -- see recoverOneSlotStep(). + // in-range recovery pass is concurrency-safe against borrow()/return after + // pool publication -- see recoverOneSlotStep(). @TestOnly public SenderPool( String configurationString, @@ -297,7 +325,7 @@ public SenderPool( ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, - deferStartupRecovery, null, null, null); + deferStartupRecovery, null, null, null, null, null, null, null); } // Test-only constructor adding a deterministic fault hook for the ownership @@ -316,7 +344,7 @@ public SenderPool( ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, - deferStartupRecovery, null, null, null, postFactoryHook); + deferStartupRecovery, null, null, null, postFactoryHook, null, null, null); } // Full constructor adding the user-supplied ingest callbacks (error @@ -340,7 +368,7 @@ public SenderPool( this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, deferStartupRecovery, errorHandler, connectionListener, - drainerListener, null); + drainerListener, null, null, null, null); } private SenderPool( @@ -355,7 +383,10 @@ private SenderPool( SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener, - Runnable postFactoryHook + Runnable postFactoryHook, + ThreadFactory recoveryThreadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook ) { if (minSize < 0 || maxSize < 1 || minSize > maxSize) { throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize); @@ -375,6 +406,10 @@ private SenderPool( this.idleTimeoutMillis = idleTimeoutMillis; this.maxLifetimeMillis = maxLifetimeMillis; this.postFactoryHook = postFactoryHook; + this.beforeFailedStartupRecoveryJoinHook = beforeFailedRecoveryJoinHook; + this.startupRecoveryWaiter = recoveryWaiter != null + ? recoveryWaiter + : this::waitForStartupRecoveryRetry; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); this.retiredSlots = new ArrayList<>(maxSize); @@ -410,17 +445,7 @@ private SenderPool( // propagate without running the cleanup below, leaking every // already-built delegate's flock + mmap'd ring + I/O thread and // resurrecting "sf slot already in use" on the next attempt. - for (int i = 0; i < built; i++) { - try { - all.get(i).delegate().close(); - } catch (Throwable ignored) { - // Best-effort cleanup: a delegate close() can throw an - // Error (e.g. an -ea AssertionError) as well as a - // RuntimeException. Swallow either so we still close the - // remaining pre-warmed slots and rethrow the original - // construction failure below. - } - } + closePrewarmedDelegates(built); throw e; } // Prewarm succeeded. Recover any unacked data a previous run left in @@ -429,18 +454,43 @@ private SenderPool( // (deferStartupRecovery=true) so QuestDB.build() never blocks on a slow // or reachable-but-not-acking server; direct constructions run it inline // here, while still single-threaded and unpublished. - if (!deferStartupRecovery) { - runStartupRecoveryToCompletion(); + Thread recoveryThread = null; + try { + if (!deferStartupRecovery) { + runStartupRecoveryToCompletion(); + if (storeAndForward && !recoveryComplete) { + ThreadFactory threadFactory = recoveryThreadFactory != null + ? recoveryThreadFactory + : SenderPool::createStartupRecoveryThread; + recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop); + recoveryThread.setDaemon(true); + } + } + this.startupRecoveryThread = recoveryThread; + if (recoveryThread != null) { + recoveryThread.start(); + } + } catch (Throwable e) { + // Thread allocation, configuration, or start can throw after + // prewarm transferred delegate ownership to this constructor. The + // pool cannot be returned, so stop a possibly-started driver before + // tearing down delegates it could otherwise still recover against. + // Preserve the construction throwable; cleanup is best-effort. + markClosing(); + stopFailedStartupRecoveryDriver(recoveryThread); + closePrewarmedDelegates(built); + throw e; } } /** - * Drives startup SF recovery to completion in a single call, bounded by one - * shared {@code acquireTimeoutMillis} wall-clock budget (and each individual - * drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the inline - * construction path -- single-threaded and unpublished -- and by manual / - * test drives. No-op when SF is off, the pool is shutting down, or recovery - * has already finished. Idempotent. + * Drives startup SF recovery toward completion in a single call, bounded by + * one shared {@code acquireTimeoutMillis} wall-clock budget (and each + * individual drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the + * inline construction path -- single-threaded and unpublished -- and by + * manual / test drives. Budget exhaustion or a transient slot failure leaves + * the cursor pending for the pool's continuing driver. No-op when SF is off, + * the pool is shutting down, or recovery has already finished. Idempotent. */ void runStartupRecoveryToCompletion() { if (!storeAndForward) { @@ -452,33 +502,64 @@ void runStartupRecoveryToCompletion() { // accepted for a single borrow, so we reuse it as the total budget; once // spent, the remaining slots wait for a later attempt (data stays // durable on disk). - final long deadlineNanos = - System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis); + final long startNanos = System.nanoTime(); while (!closed && !recoveryComplete) { - long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - if (remainingMillis <= 0) { + // Subtract nanoTime samples before comparing them with the duration. + // This avoids an overflowed absolute deadline when + // acquireTimeoutMillis is Long.MAX_VALUE and keeps the intended + // duration comparison explicit. + long elapsedNanos = System.nanoTime() - startNanos; + if (elapsedNanos >= budgetNanos) { LOG.warn("startup SF recovery: {}ms budget exhausted; " - + "skipping remaining slots", acquireTimeoutMillis); - recoveryComplete = true; + + "deferring remaining slots", acquireTimeoutMillis); return; } + long remainingNanos = budgetNanos - elapsedNanos; + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(remainingNanos); + if (remainingMillis == 0) { + // drain() accepts milliseconds; preserve a positive sub-ms + // remainder instead of treating truncation as exhaustion. + remainingMillis = 1; + } if (!recoverOneSlotStep(Math.min(remainingMillis, RECOVERY_DRAIN_BUDGET_MILLIS))) { return; } } } + private void runStartupRecoveryLoop() { + while (!closed && !recoveryComplete) { + boolean hasImmediateWork; + try { + hasImmediateWork = runStartupRecoveryStep(); + } catch (Throwable ignored) { + // Match PoolHousekeeper: startup recovery is best-effort and a + // delegate Error must not permanently kill its only driver. + hasImmediateWork = false; + } + if (closed || recoveryComplete) { + return; + } + if (!hasImmediateWork) { + startupRecoveryWaiter.run(); + if (Thread.currentThread().isInterrupted()) { + return; + } + } + } + } + /** * Recovers at most ONE stranded managed slot and reports whether more remain. - * Driven by the {@link PoolHousekeeper} one slot per step, back-to-back, on - * the reap-loop thread: each step performs at most one drain bounded by - * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the housekeeper - * stop/join budget -- on a delegate whose initial connect is forced OFF + * The private direct driver or {@link PoolHousekeeper} drives steps + * back-to-back: each step performs at most one drain bounded by + * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the driver stop/join + * budget -- on a delegate whose initial connect is forced OFF * ({@link #defaultRecoverySender}) so the build makes at most one connect - * attempt. The housekeeper re-checks {@code stop} between steps while - * {@link QuestDBImpl#close()} raises the {@code closed} shutdown signal (via - * {@link #markClosing()}) BEFORE stopping it, so a {@code close()} landing - * mid-recovery normally only has to wait out a single bounded drain. The one + * attempt. Each owner raises the {@code closed} shutdown signal before + * joining its driver, so a {@code close()} landing mid-recovery normally + * only has to wait out a single bounded drain. The one * exception is an in-flight connect to a black-holed host, which blocks on * the OS connect timeout -- see the residual-window note on * {@link #recoverOneSlotStep}. @@ -519,8 +600,8 @@ boolean runStartupRecoveryStep() { *

        * The in-range pass reserves each slot index under {@code lock} for the * duration of its recovery AND counts it in the borrow() capacity check (via - * {@code recoveringSlots}), so a concurrent borrow on the deferred path can - * neither target the dir being recovered nor over-allocate past + * {@code recoveringSlots}), so a concurrent borrow after pool publication + * can neither target the dir being recovered nor over-allocate past * {@code maxSize}. Prewarmed/borrowed slots (already live, holding their * flock) are skipped, as are empty slots (a cheap directory probe); only a * slot that actually holds stranded data spends the step's single drain. The @@ -530,13 +611,14 @@ boolean runStartupRecoveryStep() { * Best-effort throughout: a build/close Error or a slow drain is logged and * never propagates, since the data stays durable on disk for a later attempt. * A build failure or drain timeout stops the current drive but leaves its - * candidate pending so a later housekeeper tick can retry after a transient + * candidate pending so a later driver attempt can retry after a transient * condition clears; it does not poison recovery for the life of the pool. *

        - * Boundedness / residual window. Recovery is driven on the - * PoolHousekeeper thread, and {@code close()} relies on a step finishing - * within {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + - * drain + close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS} + * Boundedness / residual window. A continuing recovery step runs on + * either the direct pool's private driver or the PoolHousekeeper thread, and + * {@code close()} relies on a step finishing within + * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + drain + + * close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS} * and the build forces {@code initial_connect_mode=OFF} (see * {@link #defaultRecoverySender}), so it makes at most ONE connect attempt * instead of a SYNC reconnect-budget retry loop. That removes the @@ -544,7 +626,7 @@ boolean runStartupRecoveryStep() { * One residual window remains and is NOT closed here: a single in-flight * connect to a black-holed/firewalled host blocks on the OS connect timeout * (the transport exposes no application-level connect timeout to clamp it). - * If {@code close()} lands during that one connect the housekeeper join can + * If {@code close()} lands during that one connect, its driver join can * still time out and the detached build releases the slot flock shortly * after {@code close()} returns. No data is lost (the slot stays durable on * disk); the exposure is a brief "sf slot already in use" window on an @@ -567,8 +649,8 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { return false; } int i = recoveryInRangeNext; - // Reserve this index unless prewarm (or a concurrent borrow, on the - // deferred path) already holds it live. Count the reservation in + // Reserve this index unless prewarm (or a concurrent borrow after + // publication) already holds it live. Count the reservation in // recoveringSlots so the borrow() cap check cannot over-allocate // while this slot is held for recovery. boolean reserved; @@ -629,9 +711,9 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { i, maxSize - leakedSlots, maxSize, leakedSlots); } else { slotInUse[i] = false; - // On the deferred path a borrow may be waiting on capacity - // this recovery held; the freed index can now admit a - // creation. + // On a post-publication drive, a borrow may be waiting on + // capacity this recovery held; the freed index can now admit + // a creation. slotReleased.signal(); } } finally { @@ -699,6 +781,22 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { return false; } + private void closePrewarmedDelegates(int built) { + for (int i = 0; i < built; i++) { + try { + all.get(i).delegate().close(); + } catch (Throwable ignored) { + // Best-effort cleanup: one delegate close failure must not + // strand later prewarmed delegates or replace the original + // construction throwable. + } + } + } + + private static Thread createStartupRecoveryThread(Runnable runnable) { + return new Thread(runnable, "questdb-sender-pool-recovery"); + } + /** * Drains one candidate orphan slot dir within {@code remainingMillis}, * best-effort and never throwing. Builds a recoverer on {@code slotIndex} @@ -733,8 +831,8 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // Recovery delegate: forced OFF-mode initial connect (see // createRecoverer / defaultRecoverySender), so this build does // at most ONE connect attempt -- it never inherits the SYNC - // reconnect-budget retry loop that would block this - // (PoolHousekeeper) thread for minutes (M1). + // reconnect-budget retry loop that would block this recovery + // driver for minutes (M1). recoverer = createRecoverer(slotIndex); } catch (Throwable buildErr) { // A build/connect failure (e.g. server unreachable) will very @@ -921,10 +1019,10 @@ public void setBorrowWaitExpiredHook(Runnable hook) { /** * Raises the shutdown signal early -- without tearing down live delegates -- - * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper} - * thread stops promptly between slots. {@link QuestDBImpl#close()} calls this - * BEFORE stopping the housekeeper, so the housekeeper join cannot time out - * waiting on a fresh slot's recovery drain. Idle-delegate teardown and the + * so an in-flight startup-recovery step stops promptly between slots. Direct + * {@link #close()} calls this before joining its private driver; + * {@link QuestDBImpl#close()} calls it before stopping the housekeeper. + * Idle-delegate teardown and the * outstanding-lease wait still happen in {@link #close()} (guarded by * {@code closeStarted}, so this early signal never short-circuits them); * borrowed delegates returned after this signal are torn down on the @@ -933,6 +1031,9 @@ public void setBorrowWaitExpiredHook(Runnable hook) { */ void markClosing() { closed = true; + synchronized (startupRecoverySignal) { + startupRecoverySignal.notifyAll(); + } } /** @@ -959,6 +1060,11 @@ void markClosing() { */ @Override public void close() { + // Direct pools own their continuing recovery driver. Stop it before + // snapshotting/closing delegates; deferred pools have a null thread and + // QuestDBImpl stops their external PoolHousekeeper before calling here. + markClosing(); + stopStartupRecoveryDriver(); SenderSlot[] idleSnapshot; lock.lock(); try { @@ -1099,6 +1205,62 @@ public void giveBack(PooledSender ps) { retireLease(ps, " during pool shutdown"); } + @TestOnly + private void setStartupRecoveryJoinHooks(Runnable beforeJoinHook, Runnable afterJoinHook) { + this.beforeStartupRecoveryJoinHook = beforeJoinHook; + this.afterStartupRecoveryJoinHook = afterJoinHook; + } + + private void stopFailedStartupRecoveryDriver(Thread recoveryThread) { + if (recoveryThread == null || recoveryThread == Thread.currentThread()) { + return; + } + if (beforeFailedStartupRecoveryJoinHook != null) { + beforeFailedStartupRecoveryJoinHook.run(); + } + boolean interrupted = false; + while (recoveryThread.isAlive()) { + try { + recoveryThread.join(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void stopStartupRecoveryDriver() { + if (startupRecoveryThread == null || startupRecoveryThread == Thread.currentThread()) { + return; + } + if (beforeStartupRecoveryJoinHook != null) { + beforeStartupRecoveryJoinHook.run(); + } + try { + startupRecoveryThread.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (afterStartupRecoveryJoinHook != null) { + afterStartupRecoveryJoinHook.run(); + } + } + + private void waitForStartupRecoveryRetry() { + synchronized (startupRecoverySignal) { + if (closed || recoveryComplete) { + return; + } + try { + startupRecoverySignal.wait(RECOVERY_RETRY_INTERVAL_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + /** * Retires one lease on the calling (borrower's) thread: validates the * lease generation under the lock, removes the slot from {@code all}, @@ -1361,9 +1523,10 @@ private Sender defaultSender(int slotIndex) { /** * Same managed-slot delegate as {@link #defaultSender}, but with the * initial connect forced to {@link Sender.InitialConnectMode#OFF}. Used - * only for startup recovery, which runs on the PoolHousekeeper thread: OFF - * makes the build do at most ONE connect attempt instead of retrying for - * the whole reconnect budget (SYNC, auto-enabled by any reconnect_* knob), + * only for startup recovery, which runs on a private direct driver or the + * PoolHousekeeper thread: OFF makes the build do at most ONE connect attempt + * instead of retrying for the whole reconnect budget (SYNC, auto-enabled by + * any reconnect_* knob), * keeping a recovery step bounded below * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. See M1 / the residual-window * note on {@link #recoverOneSlotStep}. @@ -1432,8 +1595,8 @@ private Sender buildManagedSlotSender(int slotIndex, boolean forRecovery) { // its OWN slot (the one recoverOneSlotStep is processing); it must // never start a BackgroundDrainerPool for sibling/foreign orphans. // If it did, the delegate's close() -- called from - // drainCandidateSlotForRecovery() on the PoolHousekeeper thread, - // BEFORE its cursorEngine.close() releases the slot flock -- would + // drainCandidateSlotForRecovery() on the recovery driver, BEFORE + // its cursorEngine.close() releases the slot flock -- would // block in BackgroundDrainerPool.close() for up to // GRACEFUL_DRAIN_MILLIS + STOP_GRACE_MILLIS (3s) against a // reachable-but-not-acking server. That overruns a recovery step's diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index ea7e901a..78eb44a5 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -47,7 +47,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; @@ -58,6 +60,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -2633,6 +2636,355 @@ public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws E }); } + @Test + public void testDirectRecoveryContinuesAfterFiniteBudgetExhaustion() throws Exception { + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + CountDownLatch drained = new CountDownLatch(2); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 0, factory)) { + Assert.assertTrue("direct pool must continue recovery after its inline budget expires", + drained.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("in-range candidate must be recovered", 1, attempts[0].get()); + Assert.assertEquals("out-of-range candidate must be recovered", 1, attempts[1].get()); + } + } + + @Test + public void testDirectRecoveryCloseJoinsDriverAndStopsRemainingCandidates() throws Exception { + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + CountDownLatch afterJoin = new CountDownLatch(1); + CountDownLatch beforeJoin = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + CountDownLatch delegateCloseStarted = new CountDownLatch(1); + CountDownLatch drainStarted = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDrain = new CountDownLatch(1); + AtomicBoolean driverAliveAfterJoin = new AtomicBoolean(); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + AtomicReference closeFailure = new AtomicReference<>(); + IntFunction senderFactory = idx -> { + attempts[idx].incrementAndGet(); + return blockingFakeSender( + idx, drainStarted, releaseDrain, delegateCloseStarted, releaseDelegateClose); + }; + + SenderPool pool = newPoolWithFactory( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 0, 2, 0, senderFactory); + Thread recoveryThread = (Thread) getField(pool, "startupRecoveryThread"); + invokeSetStartupRecoveryJoinHooks( + pool, beforeJoin::countDown, + () -> { + driverAliveAfterJoin.set(recoveryThread.isAlive()); + afterJoin.countDown(); + }); + Thread closeThread = new Thread(() -> { + try { + pool.close(); + } catch (Throwable t) { + closeFailure.set(t); + } finally { + closeReturned.countDown(); + } + }, "test-direct-pool-close"); + try { + Assert.assertTrue("direct recovery driver must enter the first drain", + drainStarted.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("direct recovery driver must be running", recoveryThread.isAlive()); + + closeThread.start(); + Assert.assertTrue("close must enter its direct-driver join operation", + beforeJoin.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close itself must raise the shutdown signal before joining", + (Boolean) getField(pool, "closed")); + + releaseDrain.countDown(); + Assert.assertTrue("driver must remain deliberately held before termination", + delegateCloseStarted.await(10, TimeUnit.SECONDS)); + Assert.assertEquals("close must not return while its driver is held", + 1L, closeReturned.getCount()); + + releaseDelegateClose.countDown(); + Assert.assertTrue("close must return after the driver is released", + closeReturned.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close must complete its join operation", + afterJoin.await(10, TimeUnit.SECONDS)); + if (closeFailure.get() != null) { + throw new AssertionError("close failed", closeFailure.get()); + } + Assert.assertFalse("the recovery driver must be dead when close's join returns", + driverAliveAfterJoin.get()); + Assert.assertFalse("close must leave the direct recovery driver quiescent", + recoveryThread.isAlive()); + Assert.assertEquals("the in-flight candidate must have been built", 1, attempts[0].get()); + Assert.assertEquals("shutdown must prevent a later candidate build", 0, attempts[1].get()); + } finally { + releaseDrain.countDown(); + releaseDelegateClose.countDown(); + closeThread.join(TimeUnit.SECONDS.toMillis(10)); + pool.close(); + } + } + + @Test + public void testDirectRecoveryFailedAttemptEntersRetryWait() throws Throwable { + createCandidateSlot("default-0"); + CountDownLatch beforeJoin = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + CountDownLatch recoveryOperationObserved = new CountDownLatch(1); + CountDownLatch releaseWait = new CountDownLatch(1); + CountDownLatch waitEntered = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + AtomicReference closeFailure = new AtomicReference<>(); + IntFunction senderFactory = idx -> { + if (attempts.incrementAndGet() > 1) { + recoveryOperationObserved.countDown(); + } + throw new LineSenderException("injected recovery failure"); + }; + Runnable recoveryWaiter = () -> { + waitEntered.countDown(); + recoveryOperationObserved.countDown(); + try { + releaseWait.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + + SenderPool pool = newPoolWithRecoveryControls( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 0, 1, 0, senderFactory, null, recoveryWaiter, null); + invokeSetStartupRecoveryJoinHooks(pool, beforeJoin::countDown, null); + Thread closeThread = new Thread(() -> { + try { + pool.close(); + } catch (Throwable t) { + closeFailure.set(t); + } finally { + closeReturned.countDown(); + } + }, "test-retry-wait-close"); + try { + Assert.assertTrue("recovery must either wait or retry after the failed attempt", + recoveryOperationObserved.await(10, TimeUnit.SECONDS)); + Assert.assertEquals("a failed recovery attempt must enter the retry-wait operation", + 0L, waitEntered.getCount()); + Assert.assertEquals("the driver must wait immediately after its first failed attempt", + 1, attempts.get()); + + closeThread.start(); + Assert.assertTrue("close must reach the driver join while the waiter is held", + beforeJoin.await(10, TimeUnit.SECONDS)); + releaseWait.countDown(); + Assert.assertTrue("close must finish after the retry waiter is released", + closeReturned.await(10, TimeUnit.SECONDS)); + if (closeFailure.get() != null) { + throw new AssertionError("close failed", closeFailure.get()); + } + Assert.assertEquals("shutdown must prevent another recovery attempt", 1, attempts.get()); + } finally { + releaseWait.countDown(); + closeThread.join(TimeUnit.SECONDS.toMillis(10)); + pool.close(); + } + } + + @Test + public void testDirectRecoveryThreadCreationFailureClosesPrewarmedDelegates() throws Throwable { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread creation failure"); + AtomicInteger[] closeCalls = {new AtomicInteger(), new AtomicInteger()}; + IntFunction senderFactory = idx -> closeCountingSender(idx, closeCalls, idx == 0); + ThreadFactory threadFactory = runnable -> { + throw failure; + }; + + try { + newPoolWithRecoveryThreadFactory( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 2, 2, 0, senderFactory, threadFactory); + Assert.fail("construction must propagate the recovery thread creation failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } + Assert.assertEquals("cleanup must attempt the first prewarmed delegate", 1, closeCalls[0].get()); + Assert.assertEquals("one close failure must not strand the next delegate", 1, closeCalls[1].get()); + } + + @Test + public void testDirectRecoveryThreadCreationFailureReleasesPrewarmedSfFlocks() throws Exception { + TestUtils.assertMemoryLeak(() -> { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread creation failure"); + ThreadFactory threadFactory = runnable -> { + throw failure; + }; + + try (TestWebSocketServer ack = new TestWebSocketServer(new CountingAckHandler())) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + try { + newPoolWithRecoveryThreadFactory(config, 2, 2, 0, null, threadFactory); + Assert.fail("construction must propagate the recovery thread creation failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } catch (Throwable unexpected) { + throw new AssertionError("unexpected construction failure", unexpected); + } + + try (SlotLock ignored0 = SlotLock.acquire(slot("default-0")); + SlotLock ignored1 = SlotLock.acquire(slot("default-1"))) { + // Reacquisition proves failed construction closed both real + // prewarmed delegates and released their SF flocks. + } + } + }); + } + + @Test + public void testDirectRecoveryThreadStartFailureClosesPrewarmedDelegates() throws Throwable { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread start failure"); + AtomicBoolean cleanupBeforeDriverQuiescence = new AtomicBoolean(); + AtomicInteger failedJoinCalls = new AtomicInteger(); + AtomicInteger[] closeCalls = {new AtomicInteger(), new AtomicInteger()}; + AtomicReference recoveryThread = new AtomicReference<>(); + CountDownLatch releaseDriver = new CountDownLatch(1); + CountDownLatch running = new CountDownLatch(1); + IntFunction senderFactory = idx -> closeCountingSender( + idx, closeCalls, false, recoveryThread, cleanupBeforeDriverQuiescence, releaseDriver); + ThreadFactory threadFactory = runnable -> { + Thread thread = new Thread(() -> { + running.countDown(); + try { + releaseDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + runnable.run(); + }, "test-started-recovery-driver") { + @Override + public synchronized void start() { + super.start(); + try { + if (!running.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("recovery driver did not start"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted waiting for recovery driver", e); + } + throw failure; + } + }; + recoveryThread.set(thread); + return thread; + }; + Runnable beforeFailedJoin = () -> { + failedJoinCalls.incrementAndGet(); + releaseDriver.countDown(); + }; + + try { + newPoolWithRecoveryControls( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 2, 2, 0, senderFactory, threadFactory, null, beforeFailedJoin); + Assert.fail("construction must propagate the recovery thread start failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } finally { + releaseDriver.countDown(); + } + Assert.assertEquals("constructor cleanup must enter the failed-driver join", 1, failedJoinCalls.get()); + Assert.assertFalse("delegate cleanup must not begin while the failed driver is alive", + cleanupBeforeDriverQuiescence.get()); + Assert.assertFalse("a possibly-started failed driver must terminate before delegate cleanup", + recoveryThread.get().isAlive()); + Assert.assertEquals("cleanup must close the first prewarmed delegate", 1, closeCalls[0].get()); + Assert.assertEquals("cleanup must close the second prewarmed delegate", 1, closeCalls[1].get()); + } + + @Test + public void testDirectRecoveryRetriesTransientFailureAndRemainingSlots() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(2); + IntFunction factory = idx -> { + int attempt = attempts[idx].incrementAndGet(); + if (idx == 0 && attempt == 1) { + throw new LineSenderException("transient direct recovery failure"); + } + Sender delegate = Sender.builder(config).senderId("default-" + idx).build(); + return notifyingCloseSender(delegate, drained); + }; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertTrue("both recovery delegates must drain and close", + drained.await(15, TimeUnit.SECONDS)); + Assert.assertFalse("failed in-range candidate must be retried and delivered", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse("remaining out-of-range candidate must also be delivered", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("failed in-range candidate must be retried", 2, attempts[0].get()); + Assert.assertEquals("remaining out-of-range candidate must also be recovered", + 1, attempts[1].get()); + Assert.assertTrue("both recovered frames must reach the server", + handler.frames.get() >= 2); + } + } + }); + } + + @Test + public void testLongMaxStartupRecoveryBudgetDoesNotOverflow() throws Exception { + createCandidateSlot("default-0"); + AtomicInteger attempts = new AtomicInteger(); + IntFunction factory = idx -> { + attempts.incrementAndGet(); + return successfulRecoverySender(idx, new CountDownLatch(0)); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, Long.MAX_VALUE, factory)) { + Assert.assertEquals("Long.MAX_VALUE must leave a positive inline recovery budget", + 1, attempts.get()); + Assert.assertTrue("the inline scan must complete", (Boolean) getField(pool, "recoveryComplete")); + } + } + @Test public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception { // Regression for the startup-recovery budget (M1). @@ -3048,10 +3400,97 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Helpers. // ---------------------------------------------------------------------- + private static Sender closeCountingSender( + int idx, AtomicInteger[] closeCalls, boolean throwOnClose + ) { + return closeCountingSender(idx, closeCalls, throwOnClose, null, null, null); + } + + private static Sender closeCountingSender( + int idx, + AtomicInteger[] closeCalls, + boolean throwOnClose, + AtomicReference recoveryThread, + AtomicBoolean cleanupBeforeDriverQuiescence, + CountDownLatch releaseDriver + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + if (recoveryThread != null && recoveryThread.get().isAlive()) { + cleanupBeforeDriverQuiescence.set(true); + releaseDriver.countDown(); + } + closeCalls[idx].incrementAndGet(); + if (throwOnClose) { + throw new AssertionError("injected delegate close failure"); + } + return null; + case "toString": + return "CloseCountingSender-" + idx; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + throw new AssertionError("unexpected prewarmed sender call: " + method.getName()); + } + }); + } + + private void createCandidateSlot(String name) throws IOException { + java.nio.file.Path dir = Paths.get(slot(name)); + java.nio.file.Files.createDirectories(dir); + java.nio.file.Files.write(dir.resolve("0.sfa"), new byte[]{1}); + } + + private static Sender notifyingCloseSender(Sender delegate, CountDownLatch closed) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + try { + Object result = method.invoke(delegate, args); + if ("close".equals(method.getName())) { + closed.countDown(); + } + return result; + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + private String slot(String name) { return sfDir + "/" + name; } + private static Sender successfulRecoverySender(int idx, CountDownLatch drained) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "drain": + drained.countDown(); + return true; + case "close": + return null; + case "toString": + return "SuccessfulRecoverySender-" + idx; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + } + }); + } + private void assertPreallocatedExitHandoffCleansStartupRecoverer( int strandedIndex, int maxSize) throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -3362,6 +3801,46 @@ private static SenderPool newPoolWithFactory( return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, senderFactory); } + private static SenderPool newPoolWithRecoveryControls( + String cfg, + int min, + int max, + long acquireMs, + IntFunction senderFactory, + ThreadFactory threadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook + ) throws Throwable { + Constructor constructor = SenderPool.class.getDeclaredConstructor( + String.class, int.class, int.class, long.class, long.class, long.class, + IntFunction.class, boolean.class, + io.questdb.client.SenderErrorHandler.class, + io.questdb.client.SenderConnectionListener.class, + io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener.class, + Runnable.class, ThreadFactory.class, Runnable.class, Runnable.class); + constructor.setAccessible(true); + try { + return constructor.newInstance( + cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, + senderFactory, false, null, null, null, null, threadFactory, + recoveryWaiter, beforeFailedRecoveryJoinHook); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + private static SenderPool newPoolWithRecoveryThreadFactory( + String cfg, + int min, + int max, + long acquireMs, + IntFunction senderFactory, + ThreadFactory threadFactory + ) throws Throwable { + return newPoolWithRecoveryControls( + cfg, min, max, acquireMs, senderFactory, threadFactory, null, null); + } + // Uses the @TestOnly 8-arg constructor (deferStartupRecovery=true) so a test // can build a pool whose SF startup recovery is NOT run inline -- mirroring // the pooled QuestDB handle, which defers it to the housekeeper. @@ -3394,6 +3873,15 @@ private static void invokeMarkClosing(SenderPool pool) throws Exception { m.invoke(pool); } + private static void invokeSetStartupRecoveryJoinHooks( + SenderPool pool, Runnable beforeJoinHook, Runnable afterJoinHook + ) throws Exception { + Method m = SenderPool.class.getDeclaredMethod( + "setStartupRecoveryJoinHooks", Runnable.class, Runnable.class); + m.setAccessible(true); + m.invoke(pool, beforeJoinHook, afterJoinHook); + } + // Deferred pool (deferStartupRecovery=true) WITH an injected factory, so a // test can drive the housekeeper recovery path against fully controlled // (fake) recoverers. @@ -3402,9 +3890,23 @@ private static SenderPool newDeferredPoolWithFactory( return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, factory, true); } - // Fake Sender whose drain() (for slot 0 only) parks until released, opening a - // deterministic shutdown-during-recovery window. Holds no native resources. - private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, CountDownLatch release) { + private static Sender blockingFakeSender( + int idx, CountDownLatch drainStarted, CountDownLatch releaseDrain + ) { + return blockingFakeSender( + idx, drainStarted, releaseDrain, new CountDownLatch(0), new CountDownLatch(0)); + } + + // Fake Sender whose drain() and close() (for slot 0 only) park until + // released, opening deterministic shutdown and pre-termination windows. + // Holds no native resources. + private static Sender blockingFakeSender( + int idx, + CountDownLatch drainStarted, + CountDownLatch releaseDrain, + CountDownLatch closeStarted, + CountDownLatch releaseClose + ) { return (Sender) java.lang.reflect.Proxy.newProxyInstance( Sender.class.getClassLoader(), new Class[]{Sender.class}, @@ -3413,10 +3915,14 @@ private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, C case "drain": if (idx == 0) { drainStarted.countDown(); - release.await(); + releaseDrain.await(); } return true; case "close": + if (idx == 0) { + closeStarted.countDown(); + releaseClose.await(); + } return null; case "toString": return "BlockingFakeSender-" + idx; From 429153a467da6cd9759a4cafa233b7fca36e5443 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Wed, 15 Jul 2026 17:23:05 +0100 Subject: [PATCH 27/64] fix(client): batch manifest updates during segment trim --- .../qwp/client/sf/cursor/SegmentManager.java | 68 +++++-- .../qwp/client/sf/cursor/SegmentRing.java | 18 +- .../qwp/client/sf/cursor/SfManifest.java | 61 +++--- .../SegmentManagerManifestFsyncTest.java | 192 ++++++++++++++++++ 4 files changed, 286 insertions(+), 53 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 5319ee65..ba74e3e2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -962,37 +962,63 @@ private boolean serviceRing0(RingEntry e) { boolean trimFailed = false; Throwable trimFailure = null; int unlinked = 0; + int batchSize = 0; MmapSegment segment = first; long coveredAck = durableAck; - while (segment != null && unlinked < MAX_TRIMS_PER_RING_PASS) { + while (segment != null && batchSize < MAX_TRIMS_PER_RING_PASS) { long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; if (lastSeq > coveredAck) { break; } - MmapSegment next = e.ring.nextSealedAfter(segment); - String path = segment.path(); + trimBatch[batchSize++] = segment; + segment = e.ring.nextSealedAfter(segment); + } + if (batchSize > 0) { try { - // Durably commit the head advance past this segment before - // unlinking it. The ring recomputes the successor under its - // own monitor -- computing it here from `next` would race - // with a concurrent rotation sealing the active, letting the - // head leapfrog a still-unacked sealed segment (whose file a - // later recovery would then discard as "stale below head"). - e.ring.advanceManifestHeadPast(segment); - segment.close(); - // A retry after a post-unlink directory-sync failure sees an - // already absent path. Treat absence as the prior successful - // unlink and retry the batch barrier rather than wedging. - if (!filesFacade.remove(path) && filesFacade.exists(path)) { + // Durably commit the head advance past the LAST batch member + // before unlinking any of them. One commit covers the whole + // batch: head values are segment boundaries and the batch is + // a contiguous prefix of the sealed chain, so recovery + // discards every batch member as "stale below head" whether + // the crash lands mid-unlink or after -- byte-identical to a + // per-segment commit, minus up to 63 device flushes. The ring + // recomputes the successor under its own monitor -- + // computing it here from the walk above would race with a + // concurrent rotation sealing the active, letting the head + // leapfrog a still-unacked sealed segment (whose file a later + // recovery would then discard as "stale below head"). + e.ring.advanceManifestHeadPast(trimBatch[batchSize - 1]); + } catch (Throwable t) { + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_UNLINK, now, t); + return false; + } + while (unlinked < batchSize) { + MmapSegment trimming = trimBatch[unlinked]; + String path = trimming.path(); + try { + trimming.close(); + // A retry after a post-unlink directory-sync failure sees + // an already absent path. Treat absence as the prior + // successful unlink and retry the batch barrier rather + // than wedging. A retry after a mid-batch unlink failure + // re-collects from firstTrimmable(); its head advance + // clamps to the already-committed boundary (no fsync). + if (!filesFacade.remove(path) && filesFacade.exists(path)) { + trimFailed = true; + break; + } + unlinked++; + } catch (Throwable t) { trimFailed = true; + trimFailure = t; break; } - trimBatch[unlinked++] = segment; - segment = next; - } catch (Throwable t) { - trimFailed = true; - trimFailure = t; - break; + } + for (int i = unlinked; i < batchSize; i++) { + trimBatch[i] = null; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 4dd24b4c..8eda2212 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -518,13 +518,17 @@ static Recovery recover( } /** - * Durably advances the manifest head past {@code trimming} (the sealed - * segment the manager is about to unlink). The successor and the current - * active are both read under the ring monitor, so a concurrent rotation - * (which also mutates the manifest under this monitor) can never make the - * head leapfrog a still-live sealed segment: if rotation sealed the old - * active after the caller's snapshot, {@code trimming.successor()} now - * points at that sealed segment, not at the new active. + * Durably advances the manifest head past {@code trimming} (the LAST + * sealed segment of the bounded batch the manager is about to unlink). + * One durable commit covers every earlier batch member: head values are + * segment boundaries and the batch is a contiguous prefix of the sealed + * chain, so recovery discards each member as "stale below head" + * regardless of how far the unlink loop got. The successor and the + * current active are both read under the ring monitor, so a concurrent + * rotation (which also mutates the manifest under this monitor) can never + * make the head leapfrog a still-live sealed segment: if rotation sealed + * the old active after the caller's snapshot, {@code trimming.successor()} + * now points at that sealed segment, not at the new active. */ synchronized void advanceManifestHeadPast(MmapSegment trimming) { if (manifest == null) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java index f09ce94a..368c834c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -49,6 +49,10 @@ final class SfManifest implements QuietCloseable { private final int fd; private final FilesFacade filesFacade; private final String path; + // Preallocated record scratch for update(): the trim path calls update() + // once per batch and must not malloc/free per call. Guarded by this + // object's monitor (update() and close() are both synchronized). + private final long writeScratch; private long activeBase; private boolean closed; private long generation; @@ -62,6 +66,7 @@ private SfManifest(FilesFacade filesFacade, String path, int fd, this.generation = generation; this.headBase = headBase; this.activeBase = activeBase; + this.writeScratch = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); } static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { @@ -71,11 +76,12 @@ static SfManifest create(FilesFacade filesFacade, String dir, long headBase, lon throw new MmapSegmentException("exclusive create failed for SF manifest " + path); } boolean success = false; + SfManifest manifest = null; try { if (!filesFacade.allocate(fd, FILE_SIZE)) { throw new MmapSegmentException("could not allocate SF manifest " + path); } - SfManifest manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1); + manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1); manifest.update(headBase, activeBase); if (filesFacade.fsyncDir(dir) != 0) { throw new MmapSegmentException("could not sync SF manifest directory " + dir); @@ -84,7 +90,13 @@ static SfManifest create(FilesFacade filesFacade, String dir, long headBase, lon return manifest; } finally { if (!success) { - filesFacade.close(fd); + if (manifest != null) { + // close() frees the constructor-owned scratch buffer as + // well as the fd; closing only the raw fd would leak it. + manifest.close(); + } else { + filesFacade.close(fd); + } filesFacade.remove(path); } } @@ -151,9 +163,13 @@ long activeBase() { } @Override - public void close() { + public synchronized void close() { + // Synchronized against update() so the scratch buffer can never be + // freed under a concurrent writer; update() checks `closed` inside + // the same monitor. if (!closed) { closed = true; + Unsafe.free(writeScratch, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); filesFacade.close(fd); } } @@ -202,29 +218,24 @@ synchronized void update(long newHeadBase, long newActiveBase) { return; } long nextGeneration = generation + 1; - long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); - try { - Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); - Unsafe.getUnsafe().putInt(buffer, MAGIC); - Unsafe.getUnsafe().putInt(buffer + 4, VERSION); - Unsafe.getUnsafe().putLong(buffer + 8, nextGeneration); - Unsafe.getUnsafe().putLong(buffer + 16, newHeadBase); - Unsafe.getUnsafe().putLong(buffer + 24, newActiveBase); - int crc = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); - Unsafe.getUnsafe().putInt(buffer + CRC_OFFSET, crc); - long offset = (nextGeneration & 1L) * RECORD_SIZE; - if (filesFacade.write(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { - throw new MmapSegmentException("short write updating SF manifest " + path); - } - if (filesFacade.fsync(fd) != 0) { - throw new MmapSegmentException("could not sync SF manifest " + path); - } - generation = nextGeneration; - headBase = newHeadBase; - activeBase = newActiveBase; - } finally { - Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().setMemory(writeScratch, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(writeScratch, MAGIC); + Unsafe.getUnsafe().putInt(writeScratch + 4, VERSION); + Unsafe.getUnsafe().putLong(writeScratch + 8, nextGeneration); + Unsafe.getUnsafe().putLong(writeScratch + 16, newHeadBase); + Unsafe.getUnsafe().putLong(writeScratch + 24, newActiveBase); + int crc = Crc32c.update(Crc32c.INIT, writeScratch, CRC_OFFSET); + Unsafe.getUnsafe().putInt(writeScratch + CRC_OFFSET, crc); + long offset = (nextGeneration & 1L) * RECORD_SIZE; + if (filesFacade.write(fd, writeScratch, RECORD_SIZE, offset) != RECORD_SIZE) { + throw new MmapSegmentException("short write updating SF manifest " + path); + } + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync SF manifest " + path); } + generation = nextGeneration; + headBase = newHeadBase; + activeBase = newActiveBase; } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java new file mode 100644 index 00000000..d991c038 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java @@ -0,0 +1,192 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Regression test for manifest fsync amplification on the disk-trim path. + * A single trim pass covering N fully-acked sealed segments must commit the + * manifest head advance exactly once (one write + one fsync past the LAST + * batch member before any unlink), not once per trimmed segment. Recovery + * discards files "stale below head", so the single commit is byte-identical + * crash recovery — the per-segment commits are pure IO amplification. + */ +public class SegmentManagerManifestFsyncTest { + private static final String MANIFEST_NAME = "sf-manifest.bin"; + private static final long SEGMENT_SIZE = 64 * 1024; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-manifest-fsync-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test(timeout = 15_000L) + public void testTrimPassCommitsManifestHeadOncePerBatch() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // A contiguous chain of four 1-frame segments: three sealed + // (seqs 0..2) plus the active tail (seq 3). + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 1); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 1, 1); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000002.sfa", 2, 1); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000003.sfa", 3, 1); + ManifestFsyncCountingFacade ff = new ManifestFsyncCountingFacade(tmpDir); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + ring = SegmentRing.openExisting(ff, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("legacy chain must recover", ring); + Assert.assertNotNull("recovered ring must expose sealed segments", ring.firstSealed()); + Assert.assertTrue(ring.acknowledge(2)); + watermark = openWatermark(ff, tmpDir); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, TimeUnit.SECONDS.toNanos(60), SEGMENT_SIZE * 8L, ff)) { + manager.register(ring, tmpDir, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals( + "a trim pass must commit the manifest head once per batch, not once per segment", + 1, ff.manifestFsyncCalls.get()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + } + }); + } + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim acknowledged segments"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static AckWatermark openWatermark(FilesFacade ff, String root) throws Exception { + Method method = AckWatermark.class.getDeclaredMethod("open", FilesFacade.class, String.class); + method.setAccessible(true); + return (AckWatermark) method.invoke(null, ff, root); + } + + private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment segment = MmapSegment.create(path, baseSeq, SEGMENT_SIZE); + try { + for (int i = 0; i < frames; i++) { + for (int b = 0; b < 64; b++) { + Unsafe.getUnsafe().putByte(buf + b, (byte) (i * 31 + b)); + } + Assert.assertTrue("test frame must fit", segment.tryAppend(buf, 64) >= 0); + } + } finally { + segment.close(); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } + + private static final class ManifestFsyncCountingFacade implements FilesFacade { + private final AtomicInteger manifestFsyncCalls = new AtomicInteger(); + private final String manifestPath; + private volatile boolean active; + private volatile int manifestFd = -1; + + private ManifestFsyncCountingFacade(String root) { + this.manifestPath = root + "/" + MANIFEST_NAME; + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { + if (active && fd == manifestFd) { + manifestFsyncCalls.incrementAndGet(); + } + return INSTANCE.fsync(fd); + } + @Override public int fsyncDir(String dir) { return INSTANCE.fsyncDir(dir); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int msync(long addr, long len, boolean async) { return INSTANCE.msync(addr, len, async); } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (manifestPath.equals(path)) manifestFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public int openRWExclusive(String path) { + int fd = INSTANCE.openRWExclusive(path); + if (manifestPath.equals(path)) manifestFd = fd; + return fd; + } + @Override public int openRWExclusive(long pathPtr) { return INSTANCE.openRWExclusive(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} From b969bd9e5a106191b890ef1971dab70d475682f1 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Wed, 15 Jul 2026 19:55:01 +0100 Subject: [PATCH 28/64] fix(client): coordinate segment trim with I/O cursor --- .../client/sf/cursor/CursorSendEngine.java | 18 +- .../sf/cursor/CursorWebSocketSendLoop.java | 67 ++--- .../qwp/client/sf/cursor/SegmentManager.java | 174 ++++++------ .../qwp/client/sf/cursor/SegmentRing.java | 247 ++++++++++++++++-- .../SegmentManagerCrashConsistencyTest.java | 16 +- .../SegmentManagerManifestFsyncTest.java | 2 +- .../cursor/SegmentManagerPinnedTrimTest.java | 91 +++++++ .../SegmentManagerUnlinkFailureTest.java | 7 +- 8 files changed, 466 insertions(+), 156 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 90c227fa..30baa09b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -528,6 +528,10 @@ public MmapSegment activeSegment() { return ring.getActive(); } + MmapSegment pinActiveSegment() { + return ring.pinActiveSegment(); + } + /** * Append the payload, blocking up to {@link #appendDeadlineNanos} when * the cursor ring is at its memory/disk cap and waiting for ACK-driven @@ -613,7 +617,7 @@ public synchronized void close() { closed = true; // Capture drain state BEFORE closing the ring — once the ring is // closed, its accessors aren't safe to read. The active segment is - // never trimmed by drainTrimmable (only sealed segments are), so + // never trim-eligible (only sealed segments are), so // when everything published has been acked we have to unlink the // residual .sfa files here. Without this, the next sender (or a // drainer adopting this slot) would replay already-acked data @@ -1191,6 +1195,10 @@ public MmapSegment findSegmentContaining(long fsn) { return ring.findSegmentContaining(fsn); } + MmapSegment pinSegmentContaining(long fsn) { + return ring.pinSegmentContaining(fsn); + } + /** * Pass-through to {@link SegmentRing#firstSealed()}. */ @@ -1215,6 +1223,14 @@ public MmapSegment nextSealedAfter(MmapSegment current) { return ring.nextSealedAfter(current); } + MmapSegment advancePinnedSegment(MmapSegment current) { + return ring.advancePinnedSegment(current); + } + + void releasePinnedSegment(MmapSegment current) { + ring.releasePinnedSegment(current); + } + /** * I/O thread accessor: highest FSN whose frame is fully written. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index e83d907e..58c3d31e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -896,6 +896,9 @@ public synchronized void close() { } ioThread = null; } + // Covers close-before-start and start failures; normal I/O-thread + // exit already released the same pin in its finally block. + releaseSendingSegment(); // Close the current client. After a reconnect, swapClient has // replaced the original (and closed it); the owner only retains // the stale pre-reconnect reference. Without closing the live @@ -1102,17 +1105,24 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (Throwable t) { + running = false; + releaseSendingSegment(); + shutdownLatch.countDown(); + throw t; + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { t.start(); } catch (Throwable th) { // Thread.start() failed (e.g. native stack alloc OOM). ioLoop - // never ran, so its finally{shutdownLatch.countDown()} never - // fires. Release the latch and reset state so a subsequent - // close() doesn't block on a thread that doesn't exist. + // never ran, so its finally block cannot release the segment pin + // or count down the latch. running = false; + releaseSendingSegment(); shutdownLatch.countDown(); throw th; } @@ -1132,35 +1142,17 @@ private PendingDurableEntry acquirePendingEntry() { * else the active). Returns the same segment if it's still being written * (we're on the active and just need to wait for more publishedFsn). *

        - * Uses {@link CursorSendEngine#nextSealedAfter} so we never have to - * snapshot the full sealed list — important when the producer outpaces - * the I/O thread and the sealed list can grow to thousands of entries - * (cursor SF lets the producer fan out at memory speed; the wire path - * catches up at WebSocket speed). + * The ring switches the single I/O pin atomically with choosing the next + * segment, so trim can never unmap either side of the handoff. No sealed + * list snapshot is needed when the producer outpaces the wire path. */ private MmapSegment advanceSegment() { MmapSegment current = sendingSegment; - MmapSegment liveActive = engine.activeSegment(); - if (current == liveActive) { - // We're on the active — there's no "next", just wait for more - // bytes to be published into it. Caller's sendOne will see - // publishedOffset > sendOffset eventually and resume. - return current; + MmapSegment next = engine.advancePinnedSegment(current); + if (next != current) { + sendOffset = MmapSegment.HEADER_SIZE; } - sendOffset = MmapSegment.HEADER_SIZE; - MmapSegment next = engine.nextSealedAfter(current); - if (next != null) { - return next; - } - // current was the newest sealed (no later sealed exists). If it's - // still in the sealed list, the next segment must be the active; - // if it's been trimmed out from under us, fall back to the oldest - // remaining sealed before resorting to the active. - next = engine.firstSealed(); - if (next != null && next.baseSeq() > current.baseSeq()) { - return next; - } - return liveActive; + return next; } private void applyDurableAck() { @@ -1760,6 +1752,9 @@ private void ioLoop() { } } } finally { + // Release native-segment lifetime before publishing I/O-thread + // completion or running delegated engine cleanup. + releaseSendingSegment(); // Last act of the I/O thread: dispose of whatever client it // holds. This is the airtight half of the close()-vs-reconnect // race — when close()'s latch await is interrupted (drainer pool @@ -1809,7 +1804,7 @@ private void ioLoop() { * loop will then wait until the producer publishes more bytes. */ private void positionCursorAt(long targetFsn) { - MmapSegment seg = engine.findSegmentContaining(targetFsn); + MmapSegment seg = engine.pinSegmentContaining(targetFsn); if (seg == null) { // No segment currently advertises targetFsn. That normally means // targetFsn is just past publishedFsn and there is nothing to @@ -1818,14 +1813,14 @@ private void positionCursorAt(long targetFsn) { // The producer is concurrent with this I/O thread, though. It can // publish targetFsn after the first findSegmentContaining() returns // null but before or during the active-tip snapshot below. - sendingSegment = engine.activeSegment(); + sendingSegment = engine.pinActiveSegment(); sendOffset = sendingSegment.publishedOffset(); // The publishedOffset read is the producer's volatile publish // barrier. If it saw the new frame bytes, the frameCount write that // makes targetFsn discoverable is also visible, so a second lookup // must now find it. If the producer publishes later, sendOffset is // still at the old tip and trySendOne() will send the frame normally. - seg = engine.findSegmentContaining(targetFsn); + seg = engine.pinSegmentContaining(targetFsn); if (seg != null) { positionCursorInSegment(seg, targetFsn); } @@ -1905,6 +1900,14 @@ private void releasePendingEntry(PendingDurableEntry e) { pendingDurablePool.addFirst(e); } + private void releaseSendingSegment() { + MmapSegment segment = sendingSegment; + if (segment != null) { + engine.releasePinnedSegment(segment); + sendingSegment = null; + } + } + /** * Send a WebSocket PING to prod the server into flushing pending * STATUS_DURABLE_ACK frames, but only when the throttle interval has diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index ba74e3e2..07d47559 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -866,20 +866,21 @@ private boolean serviceRing0(RingEntry e) { } } - // 2. Trim any segments that the ring says are fully acked. Memory - // mode only frees native memory. Disk mode first makes the current - // cumulative ACK durable, then durably establishes the watermark's - // directory entry, unlinks one bounded batch, and finally commits - // those directory removals before ring/accounting state changes. - // No syscall runs under the manager lock. + // 2. Trim fully ACKed segments. The ring first transfers one bounded, + // unpinned prefix out of live traversal and into hidden pending + // ownership. Only then may this worker unmap it. Disk mode keeps the + // pending prefix until unlink + directory fsync are durable; memory + // mode commits each successfully freed prefix immediately. No + // syscall runs under the manager lock. Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } - MmapSegment first = e.ring.firstTrimmable(); - if (first == null) { + int pendingCount = e.ring.pendingTrimCount(); + MmapSegment first = pendingCount == 0 ? e.ring.firstTrimmable() : null; + if (pendingCount == 0 && first == null) { // Preserve the cheap mmap-only watermark cadence for ACKs that do - // not yet cover a complete sealed segment. + // not yet cover a complete, unpinned sealed segment. synchronized (lock) { if (e.isRegistered() && e.watermark != null) { long currentAck = e.ring.ackedFsn(); @@ -893,28 +894,37 @@ private boolean serviceRing0(RingEntry e) { } if (memoryMode) { - int trimmed = 0; - while (trimmed < MAX_TRIMS_PER_RING_PASS) { - MmapSegment segment = e.ring.firstTrimmable(); - if (segment == null) { + int batchSize = pendingCount > 0 + ? e.ring.copyPendingTrims(trimBatch) + : e.ring.stagePendingTrims( + trimBatch, MAX_TRIMS_PER_RING_PASS, e.ring.ackedFsn()); + int closed = 0; + Throwable closeFailure = null; + while (closed < batchSize) { + try { + trimBatch[closed].close(); + closed++; + } catch (Throwable t) { + closeFailure = t; break; } - try { - segment.close(); - synchronized (lock) { - if (e.ring.removeTrimmable(segment)) { - trimmed++; - if (e.isRegistered()) { - totalBytes -= segment.sizeBytes(); - } - } + } + if (closed > 0) { + synchronized (lock) { + long removedBytes = e.ring.commitPendingTrims(trimBatch, closed); + if (e.isRegistered()) { + totalBytes -= removedBytes; } - } catch (Throwable t) { - LOG.warn("Failed to trim memory segment", t); - return false; } } - return trimmed == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + if (closeFailure != null) { + LOG.warn("Failed to trim memory segment", closeFailure); + return false; + } + return e.ring.firstTrimmable() != null; } // A deferred disk retry does no sync, unlink, or logging work. Signed @@ -926,10 +936,9 @@ private boolean serviceRing0(RingEntry e) { return false; } - // A disk segment cannot be safely unlinked without durable ACK cover. - // The registration check and mmap store are atomic with deregister. - // Once the store wins, deregistration may proceed, but its owner must - // await this in-service pass before closing the watermark or ring. + // Every attempt repeats the cheap covering barrier. Besides keeping + // the latest cumulative ACK durable, this preserves the same strict + // pre-unlink/post-unlink directory ordering on pending retries. if (e.watermark == null) { if (!e.missingWatermarkLogged) { e.missingWatermarkLogged = true; @@ -959,101 +968,88 @@ private boolean serviceRing0(RingEntry e) { return false; } - boolean trimFailed = false; - Throwable trimFailure = null; - int unlinked = 0; - int batchSize = 0; - MmapSegment segment = first; - long coveredAck = durableAck; - while (segment != null && batchSize < MAX_TRIMS_PER_RING_PASS) { - long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; - if (lastSeq > coveredAck) { - break; - } - trimBatch[batchSize++] = segment; - segment = e.ring.nextSealedAfter(segment); - } - if (batchSize > 0) { + int batchSize; + if (pendingCount > 0) { + batchSize = e.ring.copyPendingTrims(trimBatch); + } else { try { - // Durably commit the head advance past the LAST batch member - // before unlinking any of them. One commit covers the whole - // batch: head values are segment boundaries and the batch is - // a contiguous prefix of the sealed chain, so recovery - // discards every batch member as "stale below head" whether - // the crash lands mid-unlink or after -- byte-identical to a - // per-segment commit, minus up to 63 device flushes. The ring - // recomputes the successor under its own monitor -- - // computing it here from the walk above would race with a - // concurrent rotation sealing the active, letting the head - // leapfrog a still-unacked sealed segment (whose file a later - // recovery would then discard as "stale below head"). - e.ring.advanceManifestHeadPast(trimBatch[batchSize - 1]); + // Under the ring monitor: advance the manifest past the last + // eligible member, then atomically hide the batch. No I/O pin + // can appear between the eligibility check and live removal. + batchSize = e.ring.stagePendingTrims( + trimBatch, MAX_TRIMS_PER_RING_PASS, durableAck); } catch (Throwable t) { - for (int i = 0; i < batchSize; i++) { + for (int i = 0; i < trimBatch.length; i++) { trimBatch[i] = null; } recordTrimFailure(e, TRIM_RETRY_UNLINK, now, t); return false; } - while (unlinked < batchSize) { - MmapSegment trimming = trimBatch[unlinked]; - String path = trimming.path(); - try { - trimming.close(); - // A retry after a post-unlink directory-sync failure sees - // an already absent path. Treat absence as the prior - // successful unlink and retry the batch barrier rather - // than wedging. A retry after a mid-batch unlink failure - // re-collects from firstTrimmable(); its head advance - // clamps to the already-committed boundary (no fsync). - if (!filesFacade.remove(path) && filesFacade.exists(path)) { - trimFailed = true; - break; - } - unlinked++; - } catch (Throwable t) { + if (batchSize == 0) { + return false; + } + } + + boolean trimFailed = false; + Throwable trimFailure = null; + int unlinked = 0; + while (unlinked < batchSize) { + MmapSegment trimming = trimBatch[unlinked]; + String path = trimming.path(); + try { + trimming.close(); + if (!filesFacade.remove(path) && filesFacade.exists(path)) { trimFailed = true; - trimFailure = t; break; } - } - for (int i = unlinked; i < batchSize; i++) { - trimBatch[i] = null; + unlinked++; + } catch (Throwable t) { + trimFailed = true; + trimFailure = t; + break; } } if (unlinked > 0) { try { if (filesFacade.fsyncDir(e.dir) != 0) { - for (int i = 0; i < unlinked; i++) { + for (int i = 0; i < batchSize; i++) { trimBatch[i] = null; } recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, null); return false; } } catch (Throwable t) { - for (int i = 0; i < unlinked; i++) { + for (int i = 0; i < batchSize; i++) { trimBatch[i] = null; } recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); return false; } - synchronized (lock) { - for (int i = 0; i < unlinked; i++) { - MmapSegment removed = trimBatch[i]; - trimBatch[i] = null; - if (e.ring.removeTrimmable(removed) && e.isRegistered()) { - totalBytes -= removed.sizeBytes(); + try { + synchronized (lock) { + long removedBytes = e.ring.commitPendingTrims(trimBatch, unlinked); + if (e.isRegistered()) { + totalBytes -= removedBytes; } } + } catch (Throwable t) { + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); + return false; } } + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } if (trimFailed) { recordTrimFailure(e, TRIM_RETRY_UNLINK, now, trimFailure); return false; } recordTrimSuccess(e); - return unlinked == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + return e.ring.firstTrimmable() != null; } private void recordTrimFailure(RingEntry e, int failureKind, long now, Throwable failure) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8eda2212..a6c1990d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -43,10 +43,10 @@ *

      4. Producer thread (single user thread): {@link #appendOrFsn}, * {@link #installHotSpare}, {@link #publishedFsn}.
        5. *
      5. I/O thread: {@link #publishedFsn} (read-only), {@link #acknowledge} - * (single writer), {@link #drainTrimmable} (single reader).
        6. + * (single writer), and one pinned segment cursor for native reads. *
      6. Segment manager: polls {@link #needsHotSpare}, hands new - * segments via {@link #installHotSpare}, drains trim-eligible segments - * via {@link #drainTrimmable} on its own cadence.
        7. + * segments via {@link #installHotSpare}, and stages trim-eligible + * segments into hidden cleanup ownership on its own cadence. * * No locks; the only cross-thread state is {@link #publishedFsn} (volatile, * single-writer) and {@link #ackedFsn} (volatile, single-writer). Hot-spare @@ -68,6 +68,7 @@ public final class SegmentRing implements QuietCloseable { private static final RetainedSegmentMembershipMode DEFAULT_MEMBERSHIP_MODE = RetainedSegmentMembershipMode.IDENTITY; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); + private static final int MAX_PENDING_TRIMS = 64; // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only // operation count for deterministic traversal-complexity assertions. private static long nextSealedComparisons; @@ -84,6 +85,11 @@ public final class SegmentRing implements QuietCloseable { private static long trimMovedReferences; private final long maxBytesPerSegment; private final SfManifest manifest; + // ACKed segments leave live traversal under the ring monitor before the + // manager unmaps them. They remain owned here until close + unlink + the + // directory barrier succeed, so failures stay retryable and accounted. + // At most one bounded manager batch is pending at a time. + private final ObjList pendingTrims = new ObjList<>(MAX_PENDING_TRIMS); // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time // (the segment manager). For now, both sides synchronize via the single- @@ -115,12 +121,15 @@ public final class SegmentRing implements QuietCloseable { // hotSpare: written by segment manager (installHotSpare), read+cleared by // producer thread on rotation. Volatile so the producer sees fresh installs. private volatile MmapSegment hotSpare; + // Segment whose native mapping the single I/O consumer may dereference. + // Guarded by this monitor: cursor lookup/switch and manager trim staging + // are atomic, so a pinned segment cannot be hidden or unmapped. + private MmapSegment ioPinnedSegment; // Optional callback the segment manager registers via setManagerWakeup // so the producer can wake the manager out of its poll-park the moment - // a spare is needed (rotation just consumed one, or active crossed the - // high-water mark while no spare is installed). Without this, the - // manager only notices on its next polling tick -- fine on average, - // but the worst-case wait is the full poll interval. Producer-thread-only. + // a spare is needed, and the I/O thread can wake it after releasing a + // segment that may now be trimmable. Without this, the manager only + // notices on its next polling tick. private Runnable managerWakeup; private long nextSeq; private volatile long publishedFsn; @@ -530,7 +539,7 @@ static Recovery recover( * the old active after the caller's snapshot, {@code trimming.successor()} * now points at that sealed segment, not at the new active. */ - synchronized void advanceManifestHeadPast(MmapSegment trimming) { + private synchronized void advanceManifestHeadPast(MmapSegment trimming) { if (manifest == null) { return; } @@ -913,6 +922,7 @@ public synchronized void close() { // / firstSealed / findSegmentContaining, so they don't iterate // half-freed state. closed = true; + ioPinnedSegment = null; if (active != null) { active.close(); active = null; @@ -926,6 +936,10 @@ public synchronized void close() { } sealedSegments.clear(); sealedHead = 0; + for (int i = 0, n = pendingTrims.size(); i < n; i++) { + pendingTrims.getQuick(i).close(); + } + pendingTrims.clear(); if (manifest != null) { manifest.close(); } @@ -946,6 +960,9 @@ public synchronized ObjList drainTrimmable() { // that isn't fully acked, none of the later ones can be either. while (sealedHead < sealedSegments.size()) { MmapSegment s = sealedSegments.get(sealedHead); + if (s == ioPinnedSegment) { + break; + } long lastSeq = s.baseSeq() + s.frameCount() - 1; if (lastSeq > acked) { break; @@ -971,21 +988,20 @@ public synchronized ObjList drainTrimmable() { * scan cost doesn't matter. */ public synchronized MmapSegment findSegmentContaining(long fsn) { - for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - long base = s.baseSeq(); - if (fsn >= base && fsn < base + s.frameCount()) { - return s; - } - } - MmapSegment a = active; - if (a != null) { - long base = a.baseSeq(); - if (fsn >= base && fsn < base + a.frameCount()) { - return a; - } + return findSegmentContaining0(fsn); + } + + /** + * Atomically finds and pins the segment containing {@code fsn}. The pin + * remains until the I/O cursor switches or releases it, preventing trim + * staging from hiding or unmapping the returned segment meanwhile. + */ + synchronized MmapSegment pinSegmentContaining(long fsn) { + MmapSegment segment = findSegmentContaining0(fsn); + if (segment != null) { + switchIoPin(segment); } - return null; + return segment; } /** @@ -998,16 +1014,19 @@ public synchronized MmapSegment firstSealed() { } /** - * Returns the oldest fully acknowledged sealed segment without removing - * it. The segment manager keeps it owned by the ring until close + unlink - * succeeds, so a failed unlink cannot make the path disappear from live - * bookkeeping or allow its identifier to be reused. + * Returns the oldest fully acknowledged, unpinned live segment without + * removing it. Staging later transfers it to hidden ring-owned cleanup + * state, where unlink/barrier failures remain retryable without exposing + * an unmapped segment to traversal. */ public synchronized MmapSegment firstTrimmable() { if (sealedHead == sealedSegments.size()) { return null; } MmapSegment segment = sealedSegments.get(sealedHead); + if (segment == ioPinnedSegment) { + return null; + } long lastSeq = segment.baseSeq() + segment.frameCount() - 1; return lastSeq <= ackedFsn ? segment : null; } @@ -1046,6 +1065,13 @@ public MmapSegment getActive() { return active; } + /** Atomically pins and returns the current active segment for I/O. */ + synchronized MmapSegment pinActiveSegment() { + MmapSegment segment = active; + switchIoPin(segment); + return segment; + } + /** * Direct view of sealed segments (oldest first). NOT thread-safe -- use * only from the producer thread, or alongside a lock that excludes @@ -1088,6 +1114,30 @@ public boolean needsHotSpare() { return hotSpare == null; } + /** + * Atomically advances the I/O pin from {@code current} to the next live + * sealed segment, or to the active segment when no sealed successor + * remains. An active cursor stays pinned in place until rotation seals it. + */ + synchronized MmapSegment advancePinnedSegment(MmapSegment current) { + assert ioPinnedSegment == current; + MmapSegment liveActive = active; + if (current == liveActive) { + return current; + } + MmapSegment next = nextSealedAfter(current); + if (next == null) { + MmapSegment first = sealedHead < sealedSegments.size() + ? sealedSegments.get(sealedHead) + : null; + next = first != null && first.baseSeq() > current.baseSeq() + ? first + : liveActive; + } + switchIoPin(next); + return next; + } + /** * Returns the sealed segment whose {@code baseSeq} immediately follows * {@code current.baseSeq()}, or {@code null} if no such segment exists @@ -1138,12 +1188,78 @@ public long publishedFsn() { return publishedFsn; } + /** + * Drops a directory-durable prefix from hidden trim ownership and returns + * its exact byte total for manager accounting. + */ + synchronized long commitPendingTrims(MmapSegment[] expected, int count) { + if (count < 0 || count > pendingTrims.size()) { + throw new IllegalArgumentException("invalid pending trim count: " + count); + } + long bytes = 0; + for (int i = 0; i < count; i++) { + MmapSegment segment = pendingTrims.getQuick(i); + if (segment != expected[i]) { + throw new IllegalStateException("pending trim prefix changed"); + } + bytes += segment.sizeBytes(); + } + if (count == pendingTrims.size()) { + pendingTrims.clear(); + } else if (count > 0) { + pendingTrims.remove(0, count - 1); + } + return bytes; + } + + /** Copies the hidden retry batch into manager-thread scratch storage. */ + synchronized int copyPendingTrims(MmapSegment[] target) { + int count = pendingTrims.size(); + if (target.length < count) { + throw new IllegalArgumentException("pending trim target is too small"); + } + for (int i = 0; i < count; i++) { + target[i] = pendingTrims.getQuick(i); + } + return count; + } + + /** Number of hidden trim entries retained for retry. */ + synchronized int pendingTrimCount() { + return pendingTrims.size(); + } + + @TestOnly + public synchronized int getPendingTrimCount() { + return pendingTrims.size(); + } + + @TestOnly + public synchronized MmapSegment pinSegmentContainingForTest(long fsn) { + return pinSegmentContaining(fsn); + } + + @TestOnly + public synchronized void releasePinnedSegmentForTest(MmapSegment expected) { + releasePinnedSegment(expected); + } + + /** Releases the I/O cursor pin and wakes trim if it still names {@code expected}. */ + synchronized void releasePinnedSegment(MmapSegment expected) { + if (ioPinnedSegment == expected) { + ioPinnedSegment = null; + wakeManager(); + } + } + /** * Commits removal of the segment returned by {@link #firstTrimmable()}. * Returns false if concurrent lifecycle activity changed the head. */ public synchronized boolean removeTrimmable(MmapSegment segment) { - if (sealedHead == sealedSegments.size() || sealedSegments.get(sealedHead) != segment) { + if (sealedHead == sealedSegments.size() + || sealedSegments.get(sealedHead) != segment + || segment == ioPinnedSegment) { return false; } long lastSeq = segment.baseSeq() + segment.frameCount() - 1; @@ -1154,6 +1270,47 @@ public synchronized boolean removeTrimmable(MmapSegment segment) { return true; } + /** + * Durably advances the manifest past, then hides, one bounded ACKed and + * unpinned sealed prefix. Once this returns, I/O lookup cannot discover + * any staged segment; the manager owns only the physical cleanup attempt. + */ + synchronized int stagePendingTrims(MmapSegment[] target, int maxCount, long coveredAck) { + if (pendingTrims.size() != 0) { + return 0; + } + if (maxCount < 0 || maxCount > MAX_PENDING_TRIMS || target.length < maxCount) { + throw new IllegalArgumentException("invalid trim batch size: " + maxCount); + } + int count = 0; + for (int i = sealedHead, n = sealedSegments.size(); i < n && count < maxCount; i++) { + MmapSegment segment = sealedSegments.get(i); + long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; + if (segment == ioPinnedSegment || lastSeq > coveredAck) { + break; + } + target[count++] = segment; + } + if (count == 0) { + return 0; + } + try { + advanceManifestHeadPast(target[count - 1]); + } catch (Throwable t) { + for (int i = 0; i < count; i++) { + target[i] = null; + } + throw t; + } + for (int i = 0; i < count; i++) { + MmapSegment segment = target[i]; + assert sealedSegments.get(sealedHead) == segment; + pendingTrims.add(segment); + removeSealedHead(); + } + return count; + } + /** * Registers a wakeup callback that the producer thread will invoke when * a hot spare is needed -- either right after a rotation has consumed the @@ -1207,9 +1364,30 @@ public synchronized long totalSegmentBytes() { for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { total += sealedSegments.get(i).sizeBytes(); } + for (int i = 0, n = pendingTrims.size(); i < n; i++) { + total += pendingTrims.getQuick(i).sizeBytes(); + } return total; } + private MmapSegment findSegmentContaining0(long fsn) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + MmapSegment segment = sealedSegments.get(i); + long base = segment.baseSeq(); + if (fsn >= base && fsn < base + segment.frameCount()) { + return segment; + } + } + MmapSegment liveActive = active; + if (liveActive != null) { + long base = liveActive.baseSeq(); + if (fsn >= base && fsn < base + liveActive.frameCount()) { + return liveActive; + } + } + return null; + } + private void compactSealedSegments() { if (sealedHead > 0) { int liveCount = sealedSegments.size() - sealedHead; @@ -1230,6 +1408,21 @@ private void removeSealedHead() { } } + private void switchIoPin(MmapSegment segment) { + MmapSegment previous = ioPinnedSegment; + ioPinnedSegment = segment; + if (previous != null && previous != segment) { + wakeManager(); + } + } + + private void wakeManager() { + Runnable wakeup = managerWakeup; + if (wakeup != null) { + wakeup.run(); + } + } + /** Returns the sealed-list operation count used by traversal tests. */ @TestOnly public static long getNextSealedComparisons() { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java index 44a348f5..f6b43b7d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java @@ -39,7 +39,7 @@ public class SegmentManagerCrashConsistencyTest { private static void awaitTrimmed(SegmentRing ring) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (ring.firstSealed() != null) { + while (ring.firstSealed() != null || ring.getPendingTrimCount() != 0) { if (System.nanoTime() > deadline) { throw new AssertionError("manager did not trim acknowledged segments"); } @@ -160,8 +160,12 @@ public void testBarrierFailuresPreserveCrashSafetyAndRetry() throws Exception { manager.start(); ff.awaitFailure(); if (failAt == 5) { - Assert.assertNotNull("post-unlink barrier failure committed ring removal", + Assert.assertNull("post-unlink barrier failure exposed a closed segment", + ring.findSegmentContaining(0L)); + Assert.assertNull("post-unlink barrier failure kept a closed segment live", ring.firstSealed()); + Assert.assertEquals("post-unlink barrier failure lost cleanup ownership", + 2, ring.getPendingTrimCount()); ff.advance(TimeUnit.SECONDS.toNanos(2)); manager.wakeWorker(); awaitTrimmed(ring); @@ -173,6 +177,8 @@ public void testBarrierFailuresPreserveCrashSafetyAndRetry() throws Exception { Assert.assertNotNull("barrier failure removed ring bookkeeping", ring.firstSealed()); } else { Assert.assertNull("post-unlink barrier retry did not commit ring removal", ring.firstSealed()); + Assert.assertEquals("post-unlink barrier retry retained pending ownership", + 0, ring.getPendingTrimCount()); } Assert.assertFalse("segment deletion began without a durable covering watermark", ff.removeCalls.get() > 0 && !ff.durableWatermark); @@ -332,8 +338,10 @@ public void testPersistentFailuresBackOffAndRecover() throws Exception { awaitValue(ff.failureCalls, 1, "initial persistent failure was not attempted"); awaitValue(logs, 1, "initial failure transition was not logged"); if ("segment-remove".equals(failure)) { - Assert.assertEquals("successful unlink prefix was not committed", 1L, - ring.firstSealed().baseSeq()); + Assert.assertNull("failed unlink left a closed segment live", + ring.firstSealed()); + Assert.assertEquals("successful unlink prefix was not committed", + 1, ring.getPendingTrimCount()); Assert.assertFalse(Files.exists(root + "/sf-0.sfa")); Assert.assertTrue(Files.exists(root + "/sf-1.sfa")); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java index d991c038..bb099824 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java @@ -102,7 +102,7 @@ public void testTrimPassCommitsManifestHeadOncePerBatch() throws Exception { private static void awaitTrimmed(SegmentRing ring) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (ring.firstSealed() != null) { + while (ring.firstSealed() != null || ring.getPendingTrimCount() != 0) { if (System.nanoTime() > deadline) { throw new AssertionError("manager did not trim acknowledged segments"); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java new file mode 100644 index 00000000..8e8a90ab --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java @@ -0,0 +1,91 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + *******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SegmentManagerPinnedTrimTest { + + @Test(timeout = 15_000L) + public void testPinnedActiveSurvivesRotationUntilIoRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + ring = new SegmentRing(MmapSegment.createInMemory(0L, segmentSize), segmentSize); + Unsafe.getUnsafe().putByte(payload, (byte) 1); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 1)); + MmapSegment pinned = ring.pinSegmentContainingForTest(0L); + Assert.assertNotNull(pinned); + Assert.assertTrue(ring.acknowledge(0L)); + + ring.installHotSpare(MmapSegment.createInMemory(1L, segmentSize)); + Assert.assertEquals(1L, ring.appendOrFsn(payload, 1)); + Assert.assertSame(pinned, ring.firstSealed()); + + CountDownLatch trimPass = new CountDownLatch(1); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.setBeforeTrimSyncHook(trimPass::countDown); + manager.register(ring, null); + manager.start(); + Assert.assertTrue("manager did not attempt trim", + trimPass.await(5, TimeUnit.SECONDS)); + } + + Assert.assertSame("manager trimmed the I/O-pinned segment", + pinned, ring.firstSealed()); + Assert.assertNotEquals("manager freed the I/O-pinned mapping", + 0L, pinned.address()); + Assert.assertEquals(0, ring.getPendingTrimCount()); + + ring.releasePinnedSegmentForTest(pinned); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.register(ring, null); + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals("released segment mapping was not freed", + 0L, pinned.address()); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null || ring.getPendingTrimCount() != 0) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim the released segment"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java index 67e0be65..6a076f2f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -135,9 +135,10 @@ public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exce Assert.assertTrue("failed unlink path must remain observable", Files.exists(failedPath)); Assert.assertTrue("failed unlink changed the acknowledged segment bytes", Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); - Assert.assertNotNull("failed unlink removed the segment from ring bookkeeping", + Assert.assertNull("failed unlink left a closed segment in live traversal", ring.firstSealed()); - Assert.assertEquals(failedPath, ring.firstSealed().path()); + Assert.assertEquals("failed unlink lost pending cleanup ownership", + 1, ring.getPendingTrimCount()); Assert.assertEquals("failed unlink must remain covered by the durable cumulative watermark", 0L, watermark.read()); @@ -165,6 +166,8 @@ public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exce MmapSegment firstSealed = ring.firstSealed(); Assert.assertTrue("successful retry retained the acknowledged segment", firstSealed == null || !failedPath.equals(firstSealed.path())); + Assert.assertEquals("successful retry retained pending cleanup ownership", + 0, ring.getPendingTrimCount()); } finally { Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); ring.close(); From a167e6503b9bfbb61c7cc88bd56ef4cadd69b618 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Wed, 15 Jul 2026 20:53:26 +0100 Subject: [PATCH 29/64] test(client): replace reflection with typed test seams --- .../qwp/client/QwpWebSocketSender.java | 25 ++ .../qwp/client/sf/cursor/AckWatermark.java | 33 +- .../client/sf/cursor/CursorSendEngine.java | 10 + .../qwp/client/sf/cursor/SegmentManager.java | 25 ++ .../qwp/client/sf/cursor/SegmentRing.java | 5 + .../qwp/client/sf/cursor/SlotLock.java | 47 +++ .../io/questdb/client/impl/PooledSender.java | 16 + .../questdb/client/impl/QueryClientPool.java | 15 + .../io/questdb/client/impl/QuestDBImpl.java | 10 + .../io/questdb/client/impl/SenderPool.java | 134 +++++- .../client/network/JavaTlsClientSocket.java | 52 +++ .../client/sf/cursor/AckWatermarkTest.java | 29 ++ ...gineClosePublishAfterFlockReleaseTest.java | 66 ++- .../cursor/FlockReleaseRetryDriverTest.java | 186 ++++----- .../cursor/SegmentManagerCloseRaceTest.java | 44 +- .../SegmentManagerTotalBytesRaceTest.java | 18 +- .../SegmentManagerTrimDeregisterRaceTest.java | 18 +- ...entManagerWatermarkDeregisterRaceTest.java | 31 +- .../qwp/client/sf/cursor/SlotLockTest.java | 18 +- .../impl/QuestDBImplCloseLifecycleTest.java | 92 +++-- .../client/test/impl/SenderPoolSfTest.java | 387 ++++++------------ .../network/SocketTrafficShutdownTest.java | 54 +-- 22 files changed, 716 insertions(+), 599 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3f272eb9..c3b4dfde 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1292,6 +1292,11 @@ public void setSlotLockReleaseListener(Runnable listener) { } } + @TestOnly + public void setSlotLockReleasedForTesting(boolean isReleased) { + slotLockReleased = isReleased; + } + @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { checkNotClosed(); @@ -1710,16 +1715,31 @@ public SenderConnectionDispatcher getConnectionDispatcherForTesting() { return connectionDispatcher; } + @TestOnly + public CursorSendEngine getCursorEngineForTesting() { + return cursorEngine; + } + @TestOnly public SenderErrorDispatcher getErrorDispatcherForTesting() { return errorDispatcher; } + @TestOnly + public Sender.InitialConnectMode getInitialConnectModeForTesting() { + return initialConnectMode; + } + @TestOnly public SenderProgressDispatcher getProgressDispatcherForTesting() { return progressDispatcher; } + @TestOnly + public Runnable getSlotLockReleaseListenerForTesting() { + return slotLockReleaseListener; + } + /** * Number of {@link SenderError} notifications dropped because the * bounded inbox was full. Non-zero means the user-supplied @@ -2151,6 +2171,11 @@ public void setClientForTesting(WebSocketClient client) { this.client = client; } + @TestOnly + public void setClosedForTesting(boolean isClosed) { + this.closed = isClosed; + } + /** * Installs a one-shot test witness that close-time drain invokes after it * observes a real unacknowledged target and before it starts waiting. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index c1e6c96a..eb8381c5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -30,6 +30,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,6 +103,7 @@ public final class AckWatermark implements QuietCloseable { private boolean closed; private long fsn; private long generation; + private boolean isStorageReleased; private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, long generation, long fsn) { @@ -116,12 +118,7 @@ private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, public void close() { if (closed) return; closed = true; - if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); - } - if (fd >= 0) { - filesFacade.close(fd); - } + releaseStorage(); } /** @@ -171,6 +168,16 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { : new AckWatermark(filesFacade, fd, addr, selected.generation, selected.fsn); } + /** + * Releases the native storage while deliberately leaving the logical + * closed flag clear. Test-only: recreates a stale racing writer without + * reflective access to descriptor and mapping internals. + */ + @TestOnly + public boolean releaseStorageButKeepWritableForTesting() { + return releaseStorage(); + } + /** * Best-effort removal of a stale watermark file. Used by the engine * startup path when no segments are recovered. @@ -240,6 +247,20 @@ public void write(long fsn) { this.fsn = fsn; } + private boolean releaseStorage() { + if (isStorageReleased) { + return false; + } + isStorageReleased = true; + if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { + Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); + } + if (fd >= 0) { + filesFacade.close(fd); + } + return true; + } + private static Record readRecord(long address) { if (Unsafe.getUnsafe().getInt(address + MAGIC_OFFSET) != FILE_MAGIC || Unsafe.getUnsafe().getInt(address + 4) != VERSION) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 30baa09b..6e48f036 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -900,6 +900,16 @@ private void finishClose(boolean fullyDrained) { } } + @TestOnly + public SegmentManager getManagerForTesting() { + return manager; + } + + @TestOnly + public SlotLock getSlotLockForTesting() { + return slotLock; + } + /** * Installs a hook invoked after each failed shared-driver flock release. * Test-only: makes persistent retry progress observable without sleeps. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 07d47559..88ef9cd9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -583,6 +583,31 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public SegmentRing getInServiceRingForTesting() { + RingEntry entry = inService; + return entry == null ? null : entry.ring; + } + + @TestOnly + public long getTotalBytesForTesting() { + synchronized (lock) { + return totalBytes; + } + } + + @TestOnly + public Thread getWorkerThreadForTesting() { + return workerThread; + } + + @TestOnly + public boolean isPathScratchAllocatedForTesting() { + synchronized (lock) { + return !scratchFreed; + } + } + @TestOnly public void setAfterRingCleanupHook(Runnable hook) { this.afterRingCleanupHook = hook; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index a6c1990d..f068b263 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -1229,6 +1229,11 @@ synchronized int pendingTrimCount() { return pendingTrims.size(); } + @TestOnly + public synchronized MmapSegment getHotSpareForTesting() { + return hotSpare; + } + @TestOnly public synchronized int getPendingTrimCount() { return pendingTrims.size(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 2bbcc109..3e066300 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -29,6 +29,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import java.nio.charset.StandardCharsets; @@ -56,6 +57,7 @@ */ public final class SlotLock implements QuietCloseable { + private static final int DEAD_FD_FOR_TESTING = 1_000_000_000; private static final String LOCK_FILE_NAME = ".lock"; private static final String LOCK_PID_FILE_NAME = ".lock.pid"; private final String slotDir; @@ -111,6 +113,26 @@ public static SlotLock acquire(String slotDir) { } } + /** + * Replaces the live descriptor with a known-dead value until the returned + * guard closes. Test-only: exercises release retry paths without exposing + * mutable descriptor state. + */ + @TestOnly + public synchronized ReleaseFailureForTesting injectReleaseFailureForTesting() { + if (fd < 0 || fd == DEAD_FD_FOR_TESTING) { + throw new IllegalStateException("slot lock is not held by a live descriptor"); + } + ReleaseFailureForTesting releaseFailure = new ReleaseFailureForTesting(fd); + fd = DEAD_FD_FOR_TESTING; + return releaseFailure; + } + + @TestOnly + public synchronized boolean isReleaseFailureInjectedForTesting() { + return fd == DEAD_FD_FOR_TESTING; + } + /** Slot dir this lock guards. */ public String slotDir() { return slotDir; @@ -183,6 +205,13 @@ private static String readHolder(String pidPath) { private static native int release0(int fd); + private synchronized void restoreFdForTesting(int savedFd) { + if (fd != DEAD_FD_FOR_TESTING) { + throw new IllegalStateException("slot lock release failure is not injected"); + } + fd = savedFd; + } + private static void writePid(String pidPath) { long pid; try { @@ -212,4 +241,22 @@ private static void writePid(String pidPath) { Files.close(wfd); } } + + @TestOnly + public final class ReleaseFailureForTesting implements QuietCloseable { + private final int savedFd; + private boolean isRestored; + + private ReleaseFailureForTesting(int savedFd) { + this.savedFd = savedFd; + } + + @Override + public synchronized void close() { + if (!isRestored) { + restoreFdForTesting(savedFd); + isRestored = true; + } + } + } } diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 1379c76c..7b4e5f80 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -32,6 +32,7 @@ import io.questdb.client.std.Decimal64; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.TestOnly; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -380,6 +381,21 @@ long generation() { return generation; } + @TestOnly + public Sender getDelegateForTesting() { + return slot.delegate(); + } + + @TestOnly + public int getSlotIndexForTesting() { + return slot.slotIndex(); + } + + @TestOnly + public boolean hasSameSlotForTesting(PooledSender that) { + return slot == that.slot; + } + SenderSlot slot() { return slot; } diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java index 1fc86798..75323a12 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java +++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java @@ -528,6 +528,16 @@ void release(QueryWorker w, long gen) { } } + @TestOnly + public boolean hasCreationWaiterForTesting() { + lock.lock(); + try { + return lock.hasWaiters(creationFinished); + } finally { + lock.unlock(); + } + } + // White-box accessor for tests: reports the current in-flight creation count // under the pool lock. A non-zero value after a failed acquire() means the // slot reservation was never released -- the capacity-shrink bug this guards @@ -542,6 +552,11 @@ public int inFlightCreations() { } } + @TestOnly + public boolean isClosedForTesting() { + return closed; + } + private QueryWorker createUnlocked() { QwpQueryClient client = QwpQueryClient.fromConfig(configurationString); try { diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index 3d7be5d9..75ee0a26 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -215,6 +215,16 @@ public synchronized void close() { closeQuietly(senderPool); } + @TestOnly + public QueryClientPool getQueryPoolForTesting() { + return queryPool; + } + + @TestOnly + public SenderPool getSenderPoolForTesting() { + return senderPool; + } + private static void closeQuietly(PoolHousekeeper housekeeper) { if (housekeeper == null) { return; diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 49cac21d..90658da2 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -347,6 +347,23 @@ public SenderPool( deferStartupRecovery, null, null, null, postFactoryHook, null, null, null); } + @TestOnly + public static SenderPool createWithRecoveryControlsForTesting( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + IntFunction senderFactory, + ThreadFactory recoveryThreadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook + ) { + return new SenderPool(configurationString, minSize, maxSize, acquireTimeoutMillis, + Long.MAX_VALUE, Long.MAX_VALUE, senderFactory, false, + null, null, null, null, recoveryThreadFactory, recoveryWaiter, + beforeFailedRecoveryJoinHook); + } + // Full constructor adding the user-supplied ingest callbacks (error // handler, connection listener and background-drainer listener), applied // to every Sender the pool builds (see buildManagedSlotSender). The public @@ -1007,6 +1024,111 @@ public PooledSender borrow() { } } + @TestOnly + public Sender buildRecoverySenderForTesting(int slotIndex) { + return defaultRecoverySender(slotIndex); + } + + @TestOnly + public Sender buildSenderForTesting(int slotIndex) { + return defaultSender(slotIndex); + } + + @TestOnly + public void discardBrokenForTesting(PooledSender sender) { + discardBroken(sender); + } + + @TestOnly + public int getInFlightCreationsForTesting() { + lock.lock(); + try { + return inFlightCreations; + } finally { + lock.unlock(); + } + } + + @TestOnly + public int getRetiredSlotCountForTesting() { + lock.lock(); + try { + return retiredSlots.size(); + } finally { + lock.unlock(); + } + } + + @TestOnly + public long getRetiredSlotProbeCountForTesting() { + lock.lock(); + try { + return retiredSlotProbeCount; + } finally { + lock.unlock(); + } + } + + @TestOnly + public Thread getStartupRecoveryThreadForTesting() { + return startupRecoveryThread; + } + + @TestOnly + public boolean hasCreationWaiterForTesting() { + lock.lock(); + try { + return lock.hasWaiters(creationFinished); + } finally { + lock.unlock(); + } + } + + @TestOnly + public boolean isCloseStartedForTesting() { + lock.lock(); + try { + return closeStarted; + } finally { + lock.unlock(); + } + } + + @TestOnly + public boolean isClosedForTesting() { + return closed; + } + + @TestOnly + public boolean isRecoveryCompleteForTesting() { + return recoveryComplete; + } + + @TestOnly + public boolean isSlotInUseForTesting(int slotIndex) { + lock.lock(); + try { + return slotInUse[slotIndex]; + } finally { + lock.unlock(); + } + } + + @TestOnly + public void markClosingForTesting() { + markClosing(); + } + + @TestOnly + public void runStartupRecoveryToCompletionForTesting() { + runStartupRecoveryToCompletion(); + } + + @TestOnly + public boolean runStartupRecoveryStepForTesting() { + return runStartupRecoveryStep(); + } + @TestOnly public void setBeforeBorrowWaitHook(Runnable hook) { this.beforeBorrowWaitHook = hook; @@ -1206,7 +1328,17 @@ public void giveBack(PooledSender ps) { } @TestOnly - private void setStartupRecoveryJoinHooks(Runnable beforeJoinHook, Runnable afterJoinHook) { + public void setRetiredSlotProbeCountForTesting(long count) { + lock.lock(); + try { + retiredSlotProbeCount = count; + } finally { + lock.unlock(); + } + } + + @TestOnly + public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runnable afterJoinHook) { this.beforeStartupRecoveryJoinHook = beforeJoinHook; this.afterStartupRecoveryJoinHook = afterJoinHook; } diff --git a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java index b5e43a35..f4dc94df 100644 --- a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java +++ b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java @@ -31,6 +31,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import javax.net.ssl.SSLContext; @@ -174,6 +175,26 @@ public void of(int fd) { state = STATE_PLAINTEXT; } + @TestOnly + public void setPlaintextStateForTesting() { + sslEngine = null; + state = STATE_PLAINTEXT; + } + + @TestOnly + public void setTlsStateForTesting(SSLEngine engine) { + if (state != STATE_PLAINTEXT) { + throw new IllegalStateException("socket must be in plaintext state"); + } + sslEngine = engine; + state = STATE_TLS; + } + + @TestOnly + public TlsStateForTesting snapshotTlsStateForTesting() { + return new TlsStateForTesting(this); + } + @Override public int recv(long bufferPtr, int bufferLen) { assert sslEngine != null; @@ -651,4 +672,35 @@ private int writeToSocket(int bytesToSend) { LIMIT_FIELD_OFFSET = Unsafe.getUnsafe().objectFieldOffset(limitField); CAPACITY_FIELD_OFFSET = Unsafe.getUnsafe().objectFieldOffset(capacityField); } + + @TestOnly + public static final class TlsStateForTesting { + private final ByteBuffer callerOutputBuffer; + private final SSLEngine sslEngine; + private final int state; + private final ByteBuffer unwrapInputBuffer; + private final ByteBuffer unwrapOutputBuffer; + private final ByteBuffer wrapInputBuffer; + private final ByteBuffer wrapOutputBuffer; + + private TlsStateForTesting(JavaTlsClientSocket socket) { + callerOutputBuffer = socket.callerOutputBuffer; + sslEngine = socket.sslEngine; + state = socket.state; + unwrapInputBuffer = socket.unwrapInputBuffer; + unwrapOutputBuffer = socket.unwrapOutputBuffer; + wrapInputBuffer = socket.wrapInputBuffer; + wrapOutputBuffer = socket.wrapOutputBuffer; + } + + public boolean hasSameStateForTesting(TlsStateForTesting that) { + return callerOutputBuffer == that.callerOutputBuffer + && sslEngine == that.sslEngine + && state == that.state + && unwrapInputBuffer == that.unwrapInputBuffer + && unwrapOutputBuffer == that.unwrapOutputBuffer + && wrapInputBuffer == that.wrapInputBuffer + && wrapOutputBuffer == that.wrapOutputBuffer; + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java index 27bef939..d9445af3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java @@ -143,6 +143,35 @@ public void testNegativeFsnRoundTrips() throws Exception { }); } + @Test + public void testPhysicalReleaseIsIdempotentAcrossTestingSeamAndClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AckWatermark watermark = AckWatermark.open(slotDir); + assertNotNull(watermark); + try { + assertTrue("first test release must relinquish physical storage", + watermark.releaseStorageButKeepWritableForTesting()); + assertFalse("repeated test release must not touch physical storage again", + watermark.releaseStorageButKeepWritableForTesting()); + watermark.close(); + assertFalse("close after test release must keep physical cleanup idempotent", + watermark.releaseStorageButKeepWritableForTesting()); + } finally { + watermark.close(); + } + + AckWatermark normallyClosed = AckWatermark.open(slotDir); + assertNotNull(normallyClosed); + try { + normallyClosed.close(); + assertFalse("ordinary close must record physical relinquishment", + normallyClosed.releaseStorageButKeepWritableForTesting()); + } finally { + normallyClosed.close(); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index 0c28ea16..81fd36ef 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -32,7 +32,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -192,14 +191,8 @@ public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws E public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws Exception { TestUtils.assertMemoryLeak(() -> { CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); - Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); - slotLockField.setAccessible(true); - SlotLock slotLock = (SlotLock) slotLockField.get(engine); + SlotLock slotLock = engine.getSlotLockForTesting(); assertNotNull("disk-mode engine must hold a slot lock", slotLock); - Field fdField = SlotLock.class.getDeclaredField("fd"); - fdField.setAccessible(true); - int realFd = fdField.getInt(slotLock); - assertTrue("precondition: live flock fd", realFd >= 0); // This test owns the explicit close() retry. Prevent the shared // asynchronous retry driver from racing it and from surviving into @@ -210,40 +203,39 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws E throw new IllegalStateException("retry driver disabled for explicit close retry test"); }); try { - // A non-negative fd no process has open: close(2) fails EBADF, - // so finishClose's release() confirmation fails. - fdField.setInt(slotLock, 1_000_000_000); - engine.close(); - assertFalse( - "closeCompleted was published despite an unconfirmed flock release; " - + "a pool observing this would free the slot index while the " - + "flock fd is still open", - engine.isCloseCompleted()); - // The REAL flock is still held — the incomplete report is true. - try { - SlotLock probe = SlotLock.acquire(sfDir); - probe.close(); - fail("slot must not be acquirable while the original flock fd is still open"); - } catch (IllegalStateException expected) { - // good — incomplete close really means "still locked". - } + try (SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting()) { + engine.close(); + assertFalse( + "closeCompleted was published despite an unconfirmed flock release; " + + "a pool observing this would free the slot index while the " + + "flock fd is still open", + engine.isCloseCompleted()); + // The REAL flock is still held — the incomplete report is true. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — incomplete close really means "still locked". + } - // Remove the injected fault. The retry must only re-attempt - // the retained fd release: ring/watermark cleanup stays - // exactly-once, while completion and slot reusability recover. - fdField.setInt(slotLock, realFd); - engine.close(); - assertTrue("retried close() must complete after the flock release succeeds", - engine.isCloseCompleted()); - try (SlotLock ignored = SlotLock.acquire(sfDir)) { - // good — eventual completion implies reusable capacity. + // Remove the injected fault. The retry must only re-attempt + // the retained fd release: ring/watermark cleanup stays + // exactly-once, while completion and slot reusability recover. + releaseFailure.close(); + engine.close(); + assertTrue("retried close() must complete after the flock release succeeds", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — eventual completion implies reusable capacity. + } } } finally { try { - // If an assertion failed before the successful retry, restore - // and release the real fd so the test never leaks a flock. + // The guard has restored the real fd on every exit. Release + // it if an assertion failed before the successful retry. if (!engine.isCloseCompleted()) { - fdField.setInt(slotLock, realFd); assertTrue("restored fd must release cleanly", slotLock.release()); } } finally { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java index 0037437f..967a4fab 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -32,7 +32,6 @@ import org.junit.After; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -83,14 +82,13 @@ public synchronized void start() { CursorSendEngine engine = new CursorSendEngine( newSfDir("probe-rearm"), 4L * 1024 * 1024); - SlotLock slotLock = slotLock(engine); - int realFd = fd(slotLock); + SlotLock slotLock = engine.getSlotLockForTesting(); QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); sender.setCursorEngine(engine, true); AtomicReference rearmedDriver = new AtomicReference<>(); boolean recovered = false; + SlotLock.ReleaseFailureForTesting releaseFailure = slotLock.injectReleaseFailureForTesting(); try { - setFd(slotLock, 1_000_000_000); sender.close(); assertEquals("retry driver start must be attempted once", 1, starts.get()); assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); @@ -115,7 +113,7 @@ public synchronized void start() { rearmedDriver.set(thread); return thread; }); - setFd(slotLock, realFd); + releaseFailure.close(); CountDownLatch released = new CountDownLatch(1); engine.setSlotLockReleaseListener(released::countDown); @@ -136,7 +134,7 @@ public synchronized void start() { } if (!recovered) { CursorSendEngine.setFlockReleaseRetryThreadFactory(null); - setFd(slotLock, realFd); + releaseFailure.close(); if (!slotLock.release()) { fail("restored flock fd did not release"); } @@ -175,21 +173,27 @@ public void testPersistentFailuresShareOneRetryThread() throws Exception { return thread; }); - List engines = new ArrayList<>(); - List slotLocks = new ArrayList<>(); - List realFds = new ArrayList<>(); - boolean fdsRestored = false; + List engines = new ArrayList<>(engineCount); + List releaseFailures = new ArrayList<>(engineCount); + boolean failuresRestored = false; try { for (int i = 0; i < engineCount; i++) { String sfDir = newSfDir("persistent-" + i); CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); - SlotLock slotLock = slotLock(engine); - int realFd = fd(slotLock); + SlotLock slotLock = engine.getSlotLockForTesting(); engines.add(engine); - slotLocks.add(slotLock); - realFds.add(realFd); - setFd(slotLock, 1_000_000_000); - engine.close(); + boolean isTracked = false; + SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting(); + try { + releaseFailures.add(releaseFailure); + isTracked = true; + engine.close(); + } finally { + if (!isTracked) { + releaseFailure.close(); + } + } assertFalse("injected unlock failure must keep close incomplete", engine.isCloseCompleted()); if (i == 0) { @@ -210,8 +214,8 @@ public void testPersistentFailuresShareOneRetryThread() throws Exception { engine.isCloseCompleted()); } - restoreFds(slotLocks, realFds); - fdsRestored = true; + restoreReleaseFailures(releaseFailures); + failuresRestored = true; Thread retryThread = retryThreadRef.get(); retryThread.join(10_000L); assertFalse("shared retry thread retained lifecycle resources after drain", @@ -223,8 +227,8 @@ public void testPersistentFailuresShareOneRetryThread() throws Exception { assertEquals("retries must not create another thread", 1, threadsCreated.get()); } finally { - if (!fdsRestored) { - restoreFds(slotLocks, realFds); + if (!failuresRestored) { + restoreReleaseFailures(releaseFailures); } runRetryDriver.countDown(); Thread retryThread = retryThreadRef.get(); @@ -264,11 +268,10 @@ public void testRetryBackoffDoublesToCap() throws Exception { CursorSendEngine engine = new CursorSendEngine( newSfDir("backoff-cap"), 4L * 1024 * 1024); - SlotLock slotLock = slotLock(engine); - int realFd = fd(slotLock); - boolean fdRestored = false; + SlotLock slotLock = engine.getSlotLockForTesting(); + boolean failureRestored = false; + SlotLock.ReleaseFailureForTesting releaseFailure = slotLock.injectReleaseFailureForTesting(); try { - setFd(slotLock, 1_000_000_000); engine.close(); assertFalse("injected unlock failure must keep close incomplete", engine.isCloseCompleted()); @@ -283,8 +286,8 @@ public void testRetryBackoffDoublesToCap() throws Exception { } } - setFd(slotLock, realFd); - fdRestored = true; + releaseFailure.close(); + failureRestored = true; proceed.release(); Thread driver = driverRef.get(); driver.join(10_000L); @@ -304,8 +307,8 @@ public void testRetryBackoffDoublesToCap() throws Exception { assertEquals("retry parks must double from 100ms and cap at 5s", expected, parks); } finally { - if (!fdRestored) { - setFd(slotLock, realFd); + if (!failureRestored) { + releaseFailure.close(); } proceed.release(1_000); Thread driver = driverRef.get(); @@ -345,60 +348,51 @@ public void testRetryBackoffResetsOnProgress() throws Exception { newSfDir("backoff-reset-a"), 4L * 1024 * 1024); CursorSendEngine engineB = new CursorSendEngine( newSfDir("backoff-reset-b"), 4L * 1024 * 1024); - SlotLock slotLockA = slotLock(engineA); - SlotLock slotLockB = slotLock(engineB); - int realFdA = fd(slotLockA); - int realFdB = fd(slotLockB); - boolean fdARestored = false; - boolean fdBRestored = false; + SlotLock slotLockA = engineA.getSlotLockForTesting(); + SlotLock slotLockB = engineB.getSlotLockForTesting(); try { - setFd(slotLockA, 1_000_000_000); - setFd(slotLockB, 1_000_000_001); - engineA.close(); - engineB.close(); - - // Three failed rounds ramp the backoff to 400ms. - for (int round = 1; round <= 3; round++) { - assertTrue("driver did not park after failed round " + round, - parked.tryAcquire(10, TimeUnit.SECONDS)); - if (round < 3) { - proceed.release(); + try (SlotLock.ReleaseFailureForTesting releaseFailureA = + slotLockA.injectReleaseFailureForTesting(); + SlotLock.ReleaseFailureForTesting releaseFailureB = + slotLockB.injectReleaseFailureForTesting()) { + engineA.close(); + engineB.close(); + + // Three failed rounds ramp the backoff to 400ms. + for (int round = 1; round <= 3; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 3) { + proceed.release(); + } } - } - - // Engine A recovers; round 4 has one success and one failure, - // so its park must be back at the 100ms base. - setFd(slotLockA, realFdA); - fdARestored = true; - proceed.release(); - assertTrue("driver did not park after the mixed round", - parked.tryAcquire(10, TimeUnit.SECONDS)); - assertTrue("recovered engine must publish completion", - engineA.isCloseCompleted()); - setFd(slotLockB, realFdB); - fdBRestored = true; - proceed.release(); - Thread driver = driverRef.get(); - driver.join(10_000L); - assertFalse("driver did not drain after both releases succeeded", - driver.isAlive()); - assertTrue(engineB.isCloseCompleted()); + // Engine A recovers; round 4 has one success and one failure, + // so its park must be back at the 100ms base. + releaseFailureA.close(); + proceed.release(); + assertTrue("driver did not park after the mixed round", + parked.tryAcquire(10, TimeUnit.SECONDS)); + assertTrue("recovered engine must publish completion", + engineA.isCloseCompleted()); - List expected = new ArrayList<>(); - expected.add(100_000_000L); - expected.add(200_000_000L); - expected.add(400_000_000L); - expected.add(100_000_000L); - assertEquals("a successful release must reset the backoff to base", - expected, parks); - } finally { - if (!fdARestored) { - setFd(slotLockA, realFdA); - } - if (!fdBRestored) { - setFd(slotLockB, realFdB); + releaseFailureB.close(); + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after both releases succeeded", + driver.isAlive()); + assertTrue(engineB.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(100_000_000L); + assertEquals("a successful release must reset the backoff to base", + expected, parks); } + } finally { proceed.release(1_000); Thread driver = driverRef.get(); if (driver != null) { @@ -424,10 +418,9 @@ public synchronized void start() { CursorSendEngine engine = new CursorSendEngine( newSfDir("start-failure"), 4L * 1024 * 1024); - SlotLock slotLock = slotLock(engine); - int realFd = fd(slotLock); + SlotLock slotLock = engine.getSlotLockForTesting(); + SlotLock.ReleaseFailureForTesting releaseFailure = slotLock.injectReleaseFailureForTesting(); try { - setFd(slotLock, 1_000_000_000); engine.close(); assertEquals("retry driver start must be attempted once", 1, starts.get()); assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); @@ -435,14 +428,14 @@ public synchronized void start() { // This also proves the failed driver cleared its queue: the // setter rejects replacement while any engine remains queued. CursorSendEngine.setFlockReleaseRetryThreadFactory(null); - setFd(slotLock, realFd); + releaseFailure.close(); engine.close(); assertTrue("explicit close must recover after retry-thread start failure", engine.isCloseCompleted()); } finally { if (!engine.isCloseCompleted()) { CursorSendEngine.setFlockReleaseRetryThreadFactory(null); - setFd(slotLock, realFd); + releaseFailure.close(); if (!slotLock.release()) { fail("restored flock fd did not release"); } @@ -451,12 +444,6 @@ public synchronized void start() { }); } - private static int fd(SlotLock slotLock) throws Exception { - Field fdField = SlotLock.class.getDeclaredField("fd"); - fdField.setAccessible(true); - return fdField.getInt(slotLock); - } - private static void removeDir(String sfDir) { long find = Files.findFirst(sfDir); if (find > 0) { @@ -476,27 +463,14 @@ private static void removeDir(String sfDir) { Files.remove(sfDir); } - private static void restoreFds(List slotLocks, List realFds) throws Exception { - for (int i = 0; i < slotLocks.size(); i++) { - SlotLock slotLock = slotLocks.get(i); - synchronized (slotLock) { - setFd(slotLock, realFds.get(i)); - } + private static void restoreReleaseFailures( + List releaseFailures + ) { + for (int i = 0; i < releaseFailures.size(); i++) { + releaseFailures.get(i).close(); } } - private static void setFd(SlotLock slotLock, int fd) throws Exception { - Field fdField = SlotLock.class.getDeclaredField("fd"); - fdField.setAccessible(true); - fdField.setInt(slotLock, fd); - } - - private static SlotLock slotLock(CursorSendEngine engine) throws Exception { - Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); - slotLockField.setAccessible(true); - return (SlotLock) slotLockField.get(engine); - } - private String newSfDir(String suffix) { String sfDir = Paths.get( System.getProperty("java.io.tmpdir"), diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index cee8e41d..f53e0a00 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -27,16 +27,13 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; -import io.questdb.client.std.bytes.DirectByteSink; import io.questdb.client.std.Files; -import io.questdb.client.std.str.DirectUtf8Sink; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -116,12 +113,9 @@ public void testManagerDoesNotInstallSpareIntoClosedRing() throws Exception { manager.close(); } - Field hotSpareField = SegmentRing.class.getDeclaredField("hotSpare"); - hotSpareField.setAccessible(true); - int leaked = 0; for (int i = 0; i < ITERATIONS; i++) { - Object hs = hotSpareField.get(rings[i]); + Object hs = rings[i].getHotSpareForTesting(); if (hs != null) { leaked++; // Don't leak in the test: close the survivor. @@ -691,39 +685,19 @@ private static void cleanupRecursively(String dir) { } } - private static Object readHotSpare(SegmentRing ring) throws Exception { - Field f = SegmentRing.class.getDeclaredField("hotSpare"); - f.setAccessible(true); - return f.get(ring); + private static Object readHotSpare(SegmentRing ring) { + return ring.getHotSpareForTesting(); } - private static Object readInServiceRing(SegmentManager manager) throws Exception { - Field inServiceF = SegmentManager.class.getDeclaredField("inService"); - inServiceF.setAccessible(true); - Object entry = inServiceF.get(manager); - if (entry == null) { - return null; - } - Field ringF = entry.getClass().getDeclaredField("ring"); - ringF.setAccessible(true); - return ringF.get(entry); + private static Object readInServiceRing(SegmentManager manager) { + return manager.getInServiceRingForTesting(); } - private static long readPathScratchImpl(SegmentManager manager) throws Exception { - Field pathScratchF = SegmentManager.class.getDeclaredField("pathScratch"); - pathScratchF.setAccessible(true); - DirectUtf8Sink pathScratch = (DirectUtf8Sink) pathScratchF.get(manager); - Field sinkF = DirectUtf8Sink.class.getDeclaredField("sink"); - sinkF.setAccessible(true); - DirectByteSink sink = (DirectByteSink) sinkF.get(pathScratch); - Field implF = DirectByteSink.class.getDeclaredField("impl"); - implF.setAccessible(true); - return implF.getLong(sink); + private static long readPathScratchImpl(SegmentManager manager) { + return manager.isPathScratchAllocatedForTesting() ? 1L : 0L; } - private static Thread readWorkerThread(SegmentManager manager) throws Exception { - Field workerThreadF = SegmentManager.class.getDeclaredField("workerThread"); - workerThreadF.setAccessible(true); - return (Thread) workerThreadF.get(manager); + private static Thread readWorkerThread(SegmentManager manager) { + return manager.getWorkerThreadForTesting(); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java index 644b2709..15b8e8a6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java @@ -33,7 +33,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -193,15 +192,8 @@ private static void awaitParked(Thread t) { } } - private static long readTotalBytes(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("totalBytes"); - f.setAccessible(true); - Field lockF = SegmentManager.class.getDeclaredField("lock"); - lockF.setAccessible(true); - Object lock = lockF.get(mgr); - synchronized (lock) { - return f.getLong(mgr); - } + private static long readTotalBytes(SegmentManager mgr) { + return mgr.getTotalBytesForTesting(); } private static void rmDirRecursive(String dir) { @@ -226,9 +218,7 @@ private static void rmDirRecursive(String dir) { Files.remove(dir); } - private static Thread workerThread(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("workerThread"); - f.setAccessible(true); - return (Thread) f.get(mgr); + private static Thread workerThread(SegmentManager mgr) { + return mgr.getWorkerThreadForTesting(); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java index e622ac1b..e99898af 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java @@ -35,7 +35,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -223,15 +222,8 @@ private static void awaitSpare(SegmentRing ring, String where) { } } - private static long readTotalBytes(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("totalBytes"); - f.setAccessible(true); - Field lockF = SegmentManager.class.getDeclaredField("lock"); - lockF.setAccessible(true); - Object lock = lockF.get(mgr); - synchronized (lock) { - return f.getLong(mgr); - } + private static long readTotalBytes(SegmentManager mgr) { + return mgr.getTotalBytesForTesting(); } private static void rmDirRecursive(String dir) { @@ -256,9 +248,7 @@ private static void rmDirRecursive(String dir) { Files.remove(dir); } - private static Thread workerThread(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("workerThread"); - f.setAccessible(true); - return (Thread) f.get(mgr); + private static Thread workerThread(SegmentManager mgr) { + return mgr.getWorkerThreadForTesting(); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java index 29794b68..605b8def 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java @@ -36,7 +36,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -133,8 +132,8 @@ public void testStaleWorkerDoesNotWriteThroughUnmappedWatermarkAfterDeregister() // closed=true, masking the original bug's plain-boolean guard. // Releasing the mmap/fd directly leaves the object in the stale // state that a racing worker is allowed to observe. - releaseWatermarkStorageButLeaveObjectWritable(watermark); - watermarkStorageReleased = true; + watermarkStorageReleased = + releaseWatermarkStorageButLeaveObjectWritable(watermark); resumeWorker.countDown(); if (hookErr.get() != null) { throw new AssertionError("install hook failed", hookErr.get()); @@ -154,32 +153,18 @@ public void testStaleWorkerDoesNotWriteThroughUnmappedWatermarkAfterDeregister() } catch (Throwable ignored) { // best-effort } - if (!watermarkStorageReleased) { - try { - watermark.close(); - } catch (Throwable ignored) { - // best-effort - } + try { + watermark.close(); + } catch (Throwable ignored) { + // best-effort } Unsafe.free(buf, 32, MemoryTag.NATIVE_DEFAULT); } }); } - private static void releaseWatermarkStorageButLeaveObjectWritable(AckWatermark watermark) throws Exception { - Field mmapAddressF = AckWatermark.class.getDeclaredField("mmapAddress"); - mmapAddressF.setAccessible(true); - long mmapAddress = mmapAddressF.getLong(watermark); - if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(mmapAddress, AckWatermark.FILE_SIZE, MemoryTag.MMAP_DEFAULT); - } - - Field fdF = AckWatermark.class.getDeclaredField("fd"); - fdF.setAccessible(true); - int fd = fdF.getInt(watermark); - if (fd >= 0) { - Files.close(fd); - } + private static boolean releaseWatermarkStorageButLeaveObjectWritable(AckWatermark watermark) { + return watermark.releaseStorageButKeepWritableForTesting(); } private static void rmDirRecursive(String dir) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 20930f50..644bf917 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -137,29 +137,19 @@ public void testFailedUnlockRetainsFdAndReportsFalse() throws Exception { TestUtils.assertMemoryLeak(() -> { String slot = parentDir + "/failed-release"; SlotLock lock = SlotLock.acquire(slot); - java.lang.reflect.Field fdField = SlotLock.class.getDeclaredField("fd"); - fdField.setAccessible(true); - int realFd = fdField.getInt(lock); - assertTrue("precondition: acquire must hold a live fd", realFd >= 0); - try { - // A non-negative descriptor no process has open makes the - // explicit flock/UnlockFileEx operation fail without consuming - // the real descriptor that continues to hold the slot lock. - fdField.setInt(lock, 1_000_000_000); + try (SlotLock.ReleaseFailureForTesting ignored = lock.injectReleaseFailureForTesting()) { assertFalse("release must report false when explicit unlock fails", lock.release()); - assertEquals("failed unlock must retain the fd for a safe retry", - 1_000_000_000, fdField.getInt(lock)); + assertTrue("failed unlock must retain the injected fd for a safe retry", + lock.isReleaseFailureInjectedForTesting()); assertFalse("repeat release must stay false while unlock keeps failing", lock.release()); // While the release is unconfirmed the real flock remains held. - try (SlotLock ignored = SlotLock.acquire(slot)) { + try (SlotLock ignoredLock = SlotLock.acquire(slot)) { fail("slot must not be acquirable while the original flock fd is still open"); } catch (IllegalStateException expected) { // good - unconfirmed release really means "still locked". } - } finally { - fdField.setInt(lock, realFd); } assertTrue("release must confirm once explicit unlock succeeds", lock.release()); assertTrue("confirmed release must stay confirmed", lock.release()); diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java index 65e57f85..1bbf7058 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java @@ -37,15 +37,12 @@ import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Field; import java.lang.reflect.Proxy; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.IntFunction; @@ -67,7 +64,7 @@ public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringQueryCreation() throws }; QuestDBImpl db = newQuestDB( SENDER_CFG, 0, 0, 0, slotIndex -> fakeSender(null, null, null), connectHook); - QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + QueryClientPool pool = db.getQueryPoolForTesting(); AtomicReference borrowOutcome = new AtomicReference<>(); Thread borrower = new Thread(() -> { try { @@ -128,7 +125,7 @@ public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringSenderCreation() throw + System.getProperty("java.io.tmpdir") + "/qdb-bounded-pool-" + System.nanoTime() + ";"; QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 0, senderFactory, client -> { }); - SenderPool pool = (SenderPool) getField(db, "senderPool"); + SenderPool pool = db.getSenderPoolForTesting(); AtomicReference borrowOutcome = new AtomicReference<>(); Thread borrower = new Thread(() -> { try { @@ -142,9 +139,9 @@ public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringSenderCreation() throw borrower.start(); Assert.assertTrue("sender borrow never reached construction", inCreation.await(10, TimeUnit.SECONDS)); - Assert.assertEquals(1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertEquals(1, pool.getInFlightCreationsForTesting()); Assert.assertTrue("SF slot must stay reserved during creation", - ((boolean[]) getField(pool, "slotInUse"))[0]); + pool.isSlotInUseForTesting(0)); closer.start(); closer.join(TimeUnit.SECONDS.toMillis(5)); @@ -153,18 +150,18 @@ public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringSenderCreation() throw closer.isAlive()); Assert.assertEquals( "close must retain late-completion cleanup ownership", - 1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + 1, pool.getInFlightCreationsForTesting()); Assert.assertTrue("close must not abandon the reserved SF slot", - ((boolean[]) getField(pool, "slotInUse"))[0]); + pool.isSlotInUseForTesting(0)); releaseCreation.countDown(); borrower.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); Assert.assertEquals("late sender creation reservation must be released", - 0, ((Integer) getField(pool, "inFlightCreations")).intValue()); + 0, pool.getInFlightCreationsForTesting()); Assert.assertFalse("late sender cleanup must release the SF slot", - ((boolean[]) getField(pool, "slotInUse"))[0]); + pool.isSlotInUseForTesting(0)); Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), borrowOutcome.get() instanceof LineSenderException && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); @@ -190,7 +187,7 @@ public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringQueryCreation() thr }; QuestDBImpl db = newQuestDB( SENDER_CFG, 0, 0, 100, slotIndex -> fakeSender(null, null, null), connectHook); - QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + QueryClientPool pool = db.getQueryPoolForTesting(); AtomicReference borrowOutcome = new AtomicReference<>(); AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); AtomicBoolean keepInterrupting = new AtomicBoolean(true); @@ -272,7 +269,7 @@ public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringSenderCreation() th + System.getProperty("java.io.tmpdir") + "/qdb-interrupted-pool-" + System.nanoTime() + ";"; QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 100, senderFactory, client -> { }); - SenderPool pool = (SenderPool) getField(db, "senderPool"); + SenderPool pool = db.getSenderPoolForTesting(); AtomicReference borrowOutcome = new AtomicReference<>(); AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); AtomicBoolean keepInterrupting = new AtomicBoolean(true); @@ -299,9 +296,9 @@ public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringSenderCreation() th borrower.start(); Assert.assertTrue("sender borrow never reached construction", inCreation.await(10, TimeUnit.SECONDS)); - Assert.assertEquals(1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertEquals(1, pool.getInFlightCreationsForTesting()); Assert.assertTrue("SF slot must stay reserved during creation", - ((boolean[]) getField(pool, "slotInUse"))[0]); + pool.isSlotInUseForTesting(0)); closer.start(); awaitCreationWaiter(pool, @@ -316,18 +313,18 @@ public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringSenderCreation() th closeReturnedInterrupted.get()); Assert.assertEquals( "close must retain late-completion cleanup ownership", - 1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + 1, pool.getInFlightCreationsForTesting()); Assert.assertTrue("close must not abandon the reserved SF slot", - ((boolean[]) getField(pool, "slotInUse"))[0]); + pool.isSlotInUseForTesting(0)); releaseCreation.countDown(); borrower.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); Assert.assertEquals("late sender creation reservation must be released", - 0, ((Integer) getField(pool, "inFlightCreations")).intValue()); + 0, pool.getInFlightCreationsForTesting()); Assert.assertFalse("late sender cleanup must release the SF slot", - ((boolean[]) getField(pool, "slotInUse"))[0]); + pool.isSlotInUseForTesting(0)); Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), borrowOutcome.get() instanceof LineSenderException && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); @@ -379,8 +376,8 @@ public void facadeCloseWaitsForQueryCreationAndTeardown() throws Exception { inCreation.await(10, TimeUnit.SECONDS)); closer.start(); - QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); - awaitBooleanField(pool, "closed"); + QueryClientPool pool = db.getQueryPoolForTesting(); + awaitClosed(pool); awaitCreationWaiter(pool, "facade close did not wait while query construction was internally owned"); closer.interrupt(); @@ -445,8 +442,8 @@ public void facadeCloseWaitsForSenderCreationAndTeardown() throws Exception { inCreation.await(10, TimeUnit.SECONDS)); closer.start(); - SenderPool pool = (SenderPool) getField(db, "senderPool"); - awaitBooleanField(pool, "closeStarted"); + SenderPool pool = db.getSenderPoolForTesting(); + awaitCloseStarted(pool); awaitCreationWaiter(pool, "facade close did not wait while sender construction was internally owned"); @@ -476,31 +473,44 @@ public void facadeCloseWaitsForSenderCreationAndTeardown() throws Exception { }); } - private static void awaitBooleanField(Object target, String fieldName) throws Exception { - Field field = target.getClass().getDeclaredField(fieldName); - field.setAccessible(true); + private static void awaitCloseStarted(SenderPool pool) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (System.nanoTime() < deadline) { - if (field.getBoolean(target)) { + if (pool.isCloseStartedForTesting()) { return; } Thread.yield(); } - Assert.fail("field did not become true: " + fieldName); + Assert.fail("sender pool close did not start"); } - private static void awaitCreationWaiter(Object pool, String message) throws Exception { - ReentrantLock lock = (ReentrantLock) getField(pool, "lock"); - Condition creationFinished = (Condition) getField(pool, "creationFinished"); + private static void awaitClosed(QueryClientPool pool) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (System.nanoTime() < deadline) { - lock.lock(); - try { - if (lock.hasWaiters(creationFinished)) { - return; - } - } finally { - lock.unlock(); + if (pool.isClosedForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail("query pool did not close"); + } + + private static void awaitCreationWaiter(QueryClientPool pool, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.hasCreationWaiterForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail(message); + } + + private static void awaitCreationWaiter(SenderPool pool, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.hasCreationWaiterForTesting()) { + return; } Thread.yield(); } @@ -560,12 +570,6 @@ private static Sender fakeSender( }); } - private static Object getField(Object target, String fieldName) throws Exception { - Field field = target.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - return field.get(target); - } - private static QuestDBImpl newQuestDB( int senderMin, int queryMin, diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 78eb44a5..533795ff 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -47,10 +47,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -122,8 +119,8 @@ public void testTwoConcurrentSfSendersGetDistinctSlots() throws Exception { // call, so comparing the wrappers is vacuously true. // Distinctness of the two borrowed senders lives in the // underlying slots (mirrors SenderPoolTest.slotOf usage). - Assert.assertNotSame("two borrows must hold distinct slots", - slotOf(a), slotOf(b)); + Assert.assertFalse("two borrows must hold distinct slots", + a.hasSameSlotForTesting(b)); Assert.assertTrue("slot default-0 must exist", Files.exists(slot("default-0"))); Assert.assertTrue("slot default-1 must exist", Files.exists(slot("default-1"))); Assert.assertEquals("exactly two slot dirs", 2, countSlotDirs()); @@ -223,8 +220,8 @@ public void testReturnedSenderReusesSameSlot() throws Exception { try { // borrow() now returns a fresh wrapper each time; the // recycled thing is the underlying slot. - Assert.assertSame("returned slot must be recycled", - getField(first, "slot"), getField(second, "slot")); + Assert.assertTrue("returned slot must be recycled", + first.hasSameSlotForTesting(second)); Assert.assertEquals("no new slot dir on recycle", 1, countSlotDirs()); Assert.assertTrue(Files.exists(slot("default-0"))); } finally { @@ -492,17 +489,17 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock() throws Exceptio // below is a no-op and leaves the forged flag in place. Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); // Route the wrapper through the pool's broken-eviction path. - invokeDiscardBroken(pool, a); + pool.discardBrokenForTesting(a); // The leaked index must NOT be returned to the free set, // and capacity must be accounted as permanently consumed. Assert.assertEquals("one slot must be retired as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("leaked slot index 0 must stay reserved", slotInUse[0]); + 1, pool.leakedSlotCount()); + Assert.assertTrue("leaked slot index 0 must stay reserved", + pool.isSlotInUseForTesting(0)); // The next borrow must take a fresh index -- never reuse the // still-locked default-0 dir. @@ -565,12 +562,12 @@ public void testLeakedSlotIsObservable() throws Exception { // discardBroken's re-close leaves the forged flag set. Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); - invokeDiscardBroken(pool, a); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(a); // Sanity: the slot really was retired as leaked. Assert.assertEquals("precondition: one slot must leak", - 1, getIntField(pool, "leakedSlots")); + 1, pool.leakedSlotCount()); // The leak must be observable via public API (metric). Assert.assertEquals("leaked slot must be observable via leakedSlotCount()", 1, pool.leakedSlotCount()); @@ -629,7 +626,7 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw pool.giveBack(a); Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); // Drive the sweep: the idle timeout has elapsed. Thread.sleep(10); @@ -637,9 +634,9 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw // The reap leaked branch must have fired. Assert.assertEquals("reapIdle must retire the still-locked slot as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("leaked slot index 0 must stay reserved", slotInUse[0]); + 1, pool.leakedSlotCount()); + Assert.assertTrue("leaked slot index 0 must stay reserved", + pool.isSlotInUseForTesting(0)); Assert.assertEquals("leaked slot must be observable via leakedSlotCount()", 1, pool.leakedSlotCount()); @@ -694,8 +691,8 @@ public void testRetiredSlotRecoveredByHousekeeperAfterLateFlockRelease() throws // slot as leaked. Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); - invokeDiscardBroken(pool, a); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(a); Assert.assertEquals("precondition: one slot must be retired", 1, pool.leakedSlotCount()); @@ -703,16 +700,15 @@ public void testRetiredSlotRecoveredByHousekeeperAfterLateFlockRelease() throws // the delegate now reports the flock dropped (in production // this flip comes from isSlotLockReleased() re-probing the // retained engine after the manager worker exited). - setBooleanField(delegate, "slotLockReleased", true); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(true); // The housekeeper tick is the recovery driver. pool.reapIdle(); Assert.assertEquals("recovered slot must leave the leaked count", 0, pool.leakedSlotCount()); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); Assert.assertFalse("recovered slot index 0 must return to the free set", - slotInUse[0]); + pool.isSlotInUseForTesting(0)); // Full capacity restored: with maxSize=2, two concurrent // borrows must succeed again (index 0 is reusable — its @@ -751,8 +747,8 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); - invokeDiscardBroken(pool, a); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(a); Assert.assertEquals("precondition: one slot must be retired", 1, pool.leakedSlotCount()); @@ -761,7 +757,7 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { Assert.assertTrue(Files.exists(slot("default-1"))); try { // Late release lands while the pool is capacity-starved. - setBooleanField(delegate, "slotLockReleased", true); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(true); // The next borrow hits the cap check, re-probes, frees // index 0, and must create on it instead of timing out. @@ -794,13 +790,8 @@ public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exce config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { PooledSender lease = pool.borrow(); Sender delegate = getDelegate(lease); - CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); - SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); - Field fdField = SlotLock.class.getDeclaredField("fd"); - fdField.setAccessible(true); - int realFd = fdField.getInt(slotLock); - Assert.assertTrue("precondition: live flock fd", realFd >= 0); - + CursorSendEngine engine = ((QwpWebSocketSender) delegate).getCursorEngineForTesting(); + SlotLock slotLock = engine.getSlotLockForTesting(); CountDownLatch borrowerAcquired = new CountDownLatch(1); CountDownLatch borrowerParked = new CountDownLatch(1); CountDownLatch releaseBorrower = new CountDownLatch(1); @@ -823,11 +814,10 @@ public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exce } }, "sender-pool-deferred-release-waiter"); + SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting(); try { - synchronized (slotLock) { - fdField.setInt(slotLock, 1_000_000_000); - } - invokeDiscardBroken(pool, lease); + pool.discardBrokenForTesting(lease); Assert.assertEquals("failed release must retire the only slot", 1, pool.leakedSlotCount()); Assert.assertFalse(engine.isCloseCompleted()); @@ -844,9 +834,7 @@ public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exce // This restored fd is the only source of progress: the retry driver // confirms the release. There is no housekeeper or pool mutation. - synchronized (slotLock) { - fdField.setInt(slotLock, realFd); - } + releaseFailure.close(); Assert.assertTrue("deferred flock release must wake the parked borrower", borrowerAcquired.await(5, TimeUnit.SECONDS)); Assert.assertNull("borrower must not fail", borrowerFailure.get()); @@ -856,9 +844,7 @@ public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exce pool.setBeforeBorrowWaitHook(null); releaseBorrower.countDown(); if (!engine.isCloseCompleted()) { - synchronized (slotLock) { - fdField.setInt(slotLock, realFd); - } + releaseFailure.close(); } borrower.join(TimeUnit.SECONDS.toMillis(1)); if (borrower.isAlive()) { @@ -892,30 +878,30 @@ public void testDirectRetiredSlotCallbacksHaveLinearProbeCount() throws Exceptio for (int i = 0; i < slotCount; i++) { leases[i] = pool.borrow(); delegates[i] = getDelegate(leases[i]); - callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + callbacks[i] = ((QwpWebSocketSender) delegates[i]).getSlotLockReleaseListenerForTesting(); Assert.assertNotNull(callbacks[i]); } for (int i = 0; i < slotCount; i++) { delegates[i].close(); - setBooleanField(delegates[i], "slotLockReleased", false); - invokeDiscardBroken(pool, leases[i]); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); } Assert.assertEquals(slotCount, pool.leakedSlotCount()); - setLongField(pool, "retiredSlotProbeCount", 0); + pool.setRetiredSlotProbeCountForTesting(0); int[] geometricCheckpoints = {4, 8, 16, 32}; int checkpoint = 0; for (int i = 0; i < slotCount; i++) { - setBooleanField(delegates[i], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(true); callbacks[i].run(); if (i + 1 == geometricCheckpoints[checkpoint]) { Assert.assertEquals("direct release probes must grow linearly", - i + 1, getLongField(pool, "retiredSlotProbeCount")); + i + 1, pool.getRetiredSlotProbeCountForTesting()); checkpoint++; } } Assert.assertEquals(0, pool.leakedSlotCount()); - Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + Assert.assertTrue(pool.getRetiredSlotCountForTesting() == 0); } } }); @@ -936,39 +922,39 @@ public void testDirectRetiredSlotCallbackFallbackAndIdempotence() throws Excepti for (int i = 0; i < 3; i++) { leases[i] = pool.borrow(); delegates[i] = getDelegate(leases[i]); - callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + callbacks[i] = ((QwpWebSocketSender) delegates[i]).getSlotLockReleaseListenerForTesting(); delegates[i].close(); - setBooleanField(delegates[i], "slotLockReleased", false); - invokeDiscardBroken(pool, leases[i]); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); } Assert.assertEquals(3, pool.leakedSlotCount()); - setLongField(pool, "retiredSlotProbeCount", 0); + pool.setRetiredSlotProbeCountForTesting(0); // Simulate callback registration becoming unavailable. The // periodic housekeeper scan must remain a complete fallback. ((QwpWebSocketSender) delegates[0]).setSlotLockReleaseListener(null); - setBooleanField(delegates[0], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[0]).setSlotLockReleasedForTesting(true); pool.reapIdle(); Assert.assertEquals(2, pool.leakedSlotCount()); // A premature callback must not remove an unreleased slot. callbacks[1].run(); Assert.assertEquals(2, pool.leakedSlotCount()); - setBooleanField(delegates[1], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[1]).setSlotLockReleasedForTesting(true); callbacks[1].run(); Assert.assertEquals(1, pool.leakedSlotCount()); // Duplicate and stale callbacks are idempotent and do not // probe or mutate the slot after its direct removal. - setBooleanField(delegates[2], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[2]).setSlotLockReleasedForTesting(true); callbacks[2].run(); - long probesAfterRecovery = getLongField(pool, "retiredSlotProbeCount"); + long probesAfterRecovery = pool.getRetiredSlotProbeCountForTesting(); callbacks[2].run(); callbacks[1].run(); Assert.assertEquals(probesAfterRecovery, - getLongField(pool, "retiredSlotProbeCount")); + pool.getRetiredSlotProbeCountForTesting()); Assert.assertEquals(0, pool.leakedSlotCount()); - Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + Assert.assertTrue(pool.getRetiredSlotCountForTesting() == 0); } } }); @@ -998,8 +984,8 @@ public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Excepti // slot. The idempotent second close in discardBroken() // leaves the forged state unchanged. delegates[i].close(); - setBooleanField(delegates[i], "slotLockReleased", false); - invokeDiscardBroken(pool, leases[i]); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); } Assert.assertEquals(8, pool.leakedSlotCount()); @@ -1009,17 +995,16 @@ public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Excepti // bitmap/count relationship. int[] released = {0, 2, 5, 7}; for (int i = 0; i < released.length; i++) { - setBooleanField(delegates[released[i]], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[released[i]]).setSlotLockReleasedForTesting(true); } pool.reapIdle(); Assert.assertEquals(4, pool.leakedSlotCount()); - Assert.assertEquals(4, ((List) getField(pool, "retiredSlots")).size()); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - for (int i = 0; i < slotInUse.length; i++) { + Assert.assertEquals(4, pool.getRetiredSlotCountForTesting()); + for (int i = 0; i < delegates.length; i++) { boolean mustRemainRetired = i == 1 || i == 3 || i == 4 || i == 6; Assert.assertEquals("slot reservation mismatch at index " + i, - mustRemainRetired, slotInUse[i]); + mustRemainRetired, pool.isSlotInUseForTesting(i)); } // Exactly the four restored reservations are reusable. @@ -1062,8 +1047,8 @@ public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() th } for (int i = 0; i < leases.length; i++) { delegates[i].close(); - setBooleanField(delegates[i], "slotLockReleased", false); - invokeDiscardBroken(pool, leases[i]); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); } Assert.assertEquals("all capacity must start retired", 6, pool.leakedSlotCount()); @@ -1127,7 +1112,7 @@ public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() th reparkedWaiters.countDown(); } }); - setBooleanField(delegates[2], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[2]).setSlotLockReleasedForTesting(true); pool.reapIdle(); // One restored index admits exactly one borrower. The @@ -1147,8 +1132,8 @@ public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() th // only one of the two proven waiters; signalAll() must // wake both and let them claim the two restored indices. pool.setBeforeBorrowWaitHook(null); - setBooleanField(delegates[0], "slotLockReleased", true); - setBooleanField(delegates[5], "slotLockReleased", true); + ((QwpWebSocketSender) delegates[0]).setSlotLockReleasedForTesting(true); + ((QwpWebSocketSender) delegates[5]).setSlotLockReleasedForTesting(true); pool.reapIdle(); Assert.assertTrue("all waiting borrowers must receive restored capacity", allAcquired.await(5, TimeUnit.SECONDS)); @@ -1157,7 +1142,7 @@ public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() th boolean[] seen = new boolean[6]; for (int i = 0; i < recovered.length; i++) { - int slotIndex = getIntField(slotOf(recovered[i]), "slotIndex"); + int slotIndex = recovered[i].getSlotIndexForTesting(); Assert.assertFalse("borrowers must receive distinct restored indices", seen[slotIndex]); seen[slotIndex] = true; @@ -1196,20 +1181,15 @@ public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throw Long.MAX_VALUE, Long.MAX_VALUE)) { PooledSender a = pool.borrow(); Sender delegate = getDelegate(a); - CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); - SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); - Field fdField = SlotLock.class.getDeclaredField("fd"); - fdField.setAccessible(true); - int realFd = fdField.getInt(slotLock); - Assert.assertTrue("precondition: live flock fd", realFd >= 0); + CursorSendEngine engine = ((QwpWebSocketSender) delegate).getCursorEngineForTesting(); + SlotLock slotLock = engine.getSlotLockForTesting(); + SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting(); try { // Inject one persistent explicit-unlock failure. // Delegate close must retire the only pool slot rather // than publish a release while the real flock remains held. - synchronized (slotLock) { - fdField.setInt(slotLock, 1_000_000_000); - } - invokeDiscardBroken(pool, a); + pool.discardBrokenForTesting(a); Assert.assertEquals("failed release must retire pool capacity", 1, pool.leakedSlotCount()); Assert.assertFalse(engine.isCloseCompleted()); @@ -1217,9 +1197,7 @@ public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throw // Remove the fault without calling close again: the // engine's error-path retry driver runs outside the // pool lock and must eventually publish completion. - synchronized (slotLock) { - fdField.setInt(slotLock, realFd); - } + releaseFailure.close(); long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (!engine.isCloseCompleted()) { if (System.nanoTime() > deadlineNs) { @@ -1241,11 +1219,9 @@ public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throw } } finally { if (!engine.isCloseCompleted()) { - synchronized (slotLock) { - fdField.setInt(slotLock, realFd); - Assert.assertTrue("restored fd must release cleanly", - slotLock.release()); - } + releaseFailure.close(); + Assert.assertTrue("restored fd must release cleanly", + slotLock.release()); } } } @@ -1278,9 +1254,9 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Assert.assertTrue(Files.exists(slot("default-0"))); Sender delegate = getDelegate(a); - CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + CursorSendEngine engine = ((QwpWebSocketSender) delegate).getCursorEngineForTesting(); Assert.assertNotNull("SF delegate must own a cursor engine", engine); - SegmentManager manager = (SegmentManager) getField(engine, "manager"); + SegmentManager manager = engine.getManagerForTesting(); CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); @@ -1317,7 +1293,7 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws throw new OutOfMemoryError("simulated callback allocation failure"); }); manager.setWorkerJoinTimeoutMillis(50L); - invokeDiscardBroken(pool, a); + pool.discardBrokenForTesting(a); Assert.assertEquals( "pool must retire the slot while the delegate's manager " + "worker holds the deferred cleanup", @@ -1339,9 +1315,8 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws pool.reapIdle(); Assert.assertEquals("recovered slot must leave the leaked count", 0, pool.leakedSlotCount()); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); Assert.assertFalse("recovered slot index 0 must return to the free set", - slotInUse[0]); + pool.isSlotInUseForTesting(0)); // The proof a forged flag cannot fake: both indices — // including the recovered one, whose flock was really @@ -1455,20 +1430,20 @@ public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { // min=0 (no prewarm connect), no stranded data (recovery no-op). try (SenderPool pool = new SenderPool(cfg, 0, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { // Normal managed-slot delegate: inherits the promoted SYNC. - Sender normal = invokeBuildSlotDelegate(pool, "defaultSender", 0); + Sender normal = pool.buildSenderForTesting(0); try { Assert.assertEquals( "ordinary pooled sender must honour the user's promoted SYNC mode", - Sender.InitialConnectMode.SYNC, readInitialConnectMode(normal)); + Sender.InitialConnectMode.SYNC, ((QwpWebSocketSender) normal).getInitialConnectModeForTesting()); } finally { normal.close(); } // Recovery delegate on a different slot: forced OFF. - Sender recoverer = invokeBuildSlotDelegate(pool, "defaultRecoverySender", 1); + Sender recoverer = pool.buildRecoverySenderForTesting(1); try { Assert.assertEquals( "recovery delegate must force OFF so build() makes at most one connect attempt", - Sender.InitialConnectMode.OFF, readInitialConnectMode(recoverer)); + Sender.InitialConnectMode.OFF, ((QwpWebSocketSender) recoverer).getInitialConnectModeForTesting()); } finally { recoverer.close(); } @@ -1614,7 +1589,7 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); if (idx == 0) { try { - setBooleanField(real, "closed", true); + ((QwpWebSocketSender) real).setClosedForTesting(true); } catch (Exception e) { throw new RuntimeException(e); } @@ -1632,10 +1607,11 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th Assert.assertNotNull("recovery must have built slot 0", forged.get()); // The retire branch must have fired during construction. Assert.assertEquals("recovery must retire the still-locked slot as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("retired slot 0 must stay reserved", slotInUse[0]); - Assert.assertFalse("slot 1 must remain free", slotInUse[1]); + 1, pool.leakedSlotCount()); + Assert.assertTrue("retired slot 0 must stay reserved", + pool.isSlotInUseForTesting(0)); + Assert.assertFalse("slot 1 must remain free", + pool.isSlotInUseForTesting(1)); // A later borrow must take the fresh slot 1, never re-pick // the still-locked default-0 (which would throw "sf slot @@ -1664,7 +1640,7 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th // close it for real, otherwise assertMemoryLeak trips. Sender leaked = forged.get(); if (leaked != null) { - setBooleanField(leaked, "closed", false); + ((QwpWebSocketSender) leaked).setClosedForTesting(false); leaked.close(); } } @@ -1725,7 +1701,7 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); if (idx == 0 && forged.compareAndSet(null, real)) { try { - setBooleanField(real, "closed", true); + ((QwpWebSocketSender) real).setClosedForTesting(true); } catch (Exception e) { throw new RuntimeException(e); } @@ -1753,7 +1729,7 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except // isSlotLockReleased() re-probing the retained engine after // the worker exited). Sender recoverer = forged.get(); - setBooleanField(recoverer, "closed", false); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); recoverer.close(); // The capacity-starved borrow must re-probe the startup- @@ -1763,8 +1739,8 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except try { Assert.assertEquals("borrow must recover the startup-retired slot's capacity", 0, pool.leakedSlotCount()); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("recovered index 0 must carry the new borrow", slotInUse[0]); + Assert.assertTrue("recovered index 0 must carry the new borrow", + pool.isSlotInUseForTesting(0)); Assert.assertEquals(1, countSlotDirs()); } finally { b.close(); @@ -1828,7 +1804,7 @@ public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Except Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); if (idx == 0 && forged.compareAndSet(null, real)) { try { - setBooleanField(real, "closed", true); + ((QwpWebSocketSender) real).setClosedForTesting(true); } catch (Exception e) { throw new RuntimeException(e); } @@ -1838,7 +1814,7 @@ public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Except try (SenderPool pool = newDeferredPoolWithFactory(cfg2, 0, 1, 0, factory)) { //noinspection StatementWithEmptyBody - while (invokeRunStartupRecoveryStep(pool)) { + while (pool.runStartupRecoveryStepForTesting()) { // drive the whole backlog, one housekeeper-tick unit at a time } Assert.assertNotNull("recovery must have built slot 0", forged.get()); @@ -1850,7 +1826,7 @@ public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Except // pool -- the release happens in the delegate, volatile // writes only. Sender recoverer = forged.get(); - setBooleanField(recoverer, "closed", false); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); recoverer.close(); // Try-once borrow: its single pass must probe, recover the @@ -1912,7 +1888,7 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); if (idx == 0 && forged.compareAndSet(null, real)) { try { - setBooleanField(real, "closed", true); + ((QwpWebSocketSender) real).setClosedForTesting(true); } catch (Exception e) { throw new RuntimeException(e); } @@ -1931,7 +1907,7 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception waitExpired.compareAndSet(false, true)); Sender recoverer = forged.get(); try { - setBooleanField(recoverer, "closed", false); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); } catch (Exception e) { throw new RuntimeException(e); } @@ -2437,10 +2413,9 @@ public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exce try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { PooledSender primary = pool.borrow(); - Object primarySlot = slotOf(primary); Assert.assertFalse("first recovery attempt must stop at the transient failure", - invokeRunStartupRecoveryStep(pool)); + pool.runStartupRecoveryStepForTesting()); Assert.assertEquals("exactly one out-of-range recovery attempt", 1, recoveryAttempts.get()); Assert.assertTrue("failed recovery must preserve the candidate", @@ -2448,11 +2423,11 @@ public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exce Assert.assertEquals("out-of-range failure must not consume in-range capacity", 0, pool.leakedSlotCount()); Assert.assertTrue("out-of-range recoverer must not enter retired-slot bookkeeping", - ((List) getField(pool, "retiredSlots")).isEmpty()); + pool.getRetiredSlotCountForTesting() == 0); primary.close(); primaryReturned.set(true); - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, recoveryAttempts.get()); @@ -2462,12 +2437,12 @@ public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exce Assert.assertEquals("successful out-of-range retry must not consume capacity", 0, pool.leakedSlotCount()); Assert.assertTrue("out-of-range retry must leave retired slots untouched", - ((List) getField(pool, "retiredSlots")).isEmpty()); + pool.getRetiredSlotCountForTesting() == 0); PooledSender next = pool.borrow(); try { - Assert.assertSame("normal borrow must reuse the returned primary slot", - primarySlot, slotOf(next)); + Assert.assertTrue("normal borrow must reuse the returned primary slot", + primary.hasSameSlotForTesting(next)); } finally { next.close(); } @@ -2522,33 +2497,33 @@ public void testDrainFailureRetriesInRangeAndOutOfRangeCandidates() throws Excep try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { Assert.assertFalse("failed in-range drain must defer the same candidate", - invokeRunStartupRecoveryStep(pool)); + pool.runStartupRecoveryStepForTesting()); Assert.assertEquals(1, attempts[0].get()); Assert.assertEquals(0, attempts[1].get()); Assert.assertTrue("failed in-range drain must preserve durable data", hasSegmentFile(slot("default-0"))); Assert.assertTrue("successful in-range retry may continue scanning", - invokeRunStartupRecoveryStep(pool)); + pool.runStartupRecoveryStepForTesting()); Assert.assertEquals("same live pool must retry the in-range candidate", 2, attempts[0].get()); Assert.assertFalse("in-range retry must drain the preserved data", hasSegmentFile(slot("default-0"))); Assert.assertFalse("failed out-of-range drain must defer the same candidate", - invokeRunStartupRecoveryStep(pool)); + pool.runStartupRecoveryStepForTesting()); Assert.assertEquals(1, attempts[1].get()); Assert.assertTrue("failed out-of-range drain must preserve durable data", hasSegmentFile(slot("default-1"))); Assert.assertTrue("successful out-of-range retry may finish the candidate", - invokeRunStartupRecoveryStep(pool)); + pool.runStartupRecoveryStepForTesting()); Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, attempts[1].get()); Assert.assertFalse("out-of-range retry must drain the preserved data", hasSegmentFile(slot("default-1"))); Assert.assertFalse("final scan step must mark recovery complete", - invokeRunStartupRecoveryStep(pool)); + pool.runStartupRecoveryStepForTesting()); Assert.assertTrue("both recovered frames must be delivered", handler.frames.get() >= 2); } } @@ -2679,9 +2654,9 @@ public void testDirectRecoveryCloseJoinsDriverAndStopsRemainingCandidates() thro SenderPool pool = newPoolWithFactory( "ws::addr=localhost:1;sf_dir=" + sfDir + ";", 0, 2, 0, senderFactory); - Thread recoveryThread = (Thread) getField(pool, "startupRecoveryThread"); - invokeSetStartupRecoveryJoinHooks( - pool, beforeJoin::countDown, + Thread recoveryThread = pool.getStartupRecoveryThreadForTesting(); + pool.setStartupRecoveryJoinHooksForTesting( + beforeJoin::countDown, () -> { driverAliveAfterJoin.set(recoveryThread.isAlive()); afterJoin.countDown(); @@ -2704,7 +2679,7 @@ public void testDirectRecoveryCloseJoinsDriverAndStopsRemainingCandidates() thro Assert.assertTrue("close must enter its direct-driver join operation", beforeJoin.await(10, TimeUnit.SECONDS)); Assert.assertTrue("close itself must raise the shutdown signal before joining", - (Boolean) getField(pool, "closed")); + pool.isClosedForTesting()); releaseDrain.countDown(); Assert.assertTrue("driver must remain deliberately held before termination", @@ -2763,7 +2738,7 @@ public void testDirectRecoveryFailedAttemptEntersRetryWait() throws Throwable { SenderPool pool = newPoolWithRecoveryControls( "ws::addr=localhost:1;sf_dir=" + sfDir + ";", 0, 1, 0, senderFactory, null, recoveryWaiter, null); - invokeSetStartupRecoveryJoinHooks(pool, beforeJoin::countDown, null); + pool.setStartupRecoveryJoinHooksForTesting(beforeJoin::countDown, null); Thread closeThread = new Thread(() -> { try { pool.close(); @@ -2981,7 +2956,7 @@ public void testLongMaxStartupRecoveryBudgetDoesNotOverflow() throws Exception { try (SenderPool pool = newPoolWithFactory(config, 0, 1, Long.MAX_VALUE, factory)) { Assert.assertEquals("Long.MAX_VALUE must leave a positive inline recovery budget", 1, attempts.get()); - Assert.assertTrue("the inline scan must complete", (Boolean) getField(pool, "recoveryComplete")); + Assert.assertTrue("the inline scan must complete", pool.isRecoveryCompleteForTesting()); } } @@ -3139,7 +3114,7 @@ public void testRecoveryStepStaysBoundedWithDrainOrphansAgainstNonAckingServer() SenderPool pool = newDeferredPool(cfg, 0, maxSize, acquireTimeoutMillis); try { long startNanos = System.nanoTime(); - invokeRunStartupRecoveryStep(pool); + pool.runStartupRecoveryStepForTesting(); long stepMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); // Headline guarantee: a recovery delegate must not stand up a @@ -3242,7 +3217,7 @@ public void testDeferredStartupRecoveryDoesNotBlockConstruction() throws Excepti // lost) for a later attempt -- exercising the deferred path's // concurrency-safe slot reservation too. long recoverStart = System.nanoTime(); - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); long recoverMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStart); Assert.assertTrue( "driven recovery must be bounded by the shared budget: took " + recoverMillis @@ -3297,7 +3272,7 @@ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { hasSegmentFile(slot("default-0"))); // Drive it (what the housekeeper does on its first tick). - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); Assert.assertTrue("driven recovery must empty default-0", awaitNoSegmentFile(slot("default-0"), 15_000)); Assert.assertTrue("recovered frames must reach the server", @@ -3306,7 +3281,7 @@ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { Files.exists(slot("default-0") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); // Idempotent: a second drive is a no-op and must not throw. - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); Assert.assertFalse("default-0 stays recovered", hasSegmentFile(slot("default-0"))); // Pool still usable for normal borrows. @@ -3367,7 +3342,7 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Mimic the housekeeper: drive steps back-to-back until done/closing. Thread recovery = new Thread(() -> { try { - while (invokeRunStartupRecoveryStep(pool)) { + while (pool.runStartupRecoveryStepForTesting()) { // keep stepping } } catch (Exception ignored) { @@ -3380,7 +3355,7 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Raise the shutdown signal mid-drain, exactly as QuestDBImpl.close() // does before stopping the housekeeper. - invokeMarkClosing(pool); + pool.markClosingForTesting(); releaseSlot0Drain.countDown(); recovery.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("recovery thread must finish", recovery.isAlive()); @@ -3527,8 +3502,8 @@ private void assertPreallocatedExitHandoffCleansStartupRecoverer( Sender sender = Sender.builder(config).senderId("default-" + idx).build(); if (idx == strandedIndex && instrumented.compareAndSet(false, true)) { try { - CursorSendEngine engine = (CursorSendEngine) getField(sender, "cursorEngine"); - SegmentManager manager = (SegmentManager) getField(engine, "manager"); + CursorSendEngine engine = ((QwpWebSocketSender) sender).getCursorEngineForTesting(); + SegmentManager manager = engine.getManagerForTesting(); CountDownLatch workerBlocked = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); manager.setBeforeTrimSyncHook(() -> { @@ -3719,78 +3694,8 @@ private static void rmDir(String dir) { Files.remove(dir); } - private static Sender getDelegate(PooledSender ps) throws Exception { - Field slotF = PooledSender.class.getDeclaredField("slot"); - slotF.setAccessible(true); - Object slot = slotF.get(ps); - Field f = slot.getClass().getDeclaredField("delegate"); - f.setAccessible(true); - return (Sender) f.get(slot); - } - - // Invokes one of the pool's private managed-slot delegate factories - // (defaultSender / defaultRecoverySender) so a test can inspect the raw - // delegate it would build for a given slot index. - private static Sender invokeBuildSlotDelegate(SenderPool pool, String methodName, int slotIndex) - throws Exception { - Method m = SenderPool.class.getDeclaredMethod(methodName, int.class); - m.setAccessible(true); - return (Sender) m.invoke(pool, slotIndex); - } - - // Reads the resolved initial-connect mode a built QwpWebSocketSender delegate - // is using (the value after the builder's SYNC auto-promotion / explicit - // override has been applied). - private static Sender.InitialConnectMode readInitialConnectMode(Sender delegate) throws Exception { - Field f = delegate.getClass().getDeclaredField("initialConnectMode"); - f.setAccessible(true); - return (Sender.InitialConnectMode) f.get(delegate); - } - - private static void setBooleanField(Object target, String name, boolean value) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - f.setBoolean(target, value); - } - - private static void setLongField(Object target, String name, long value) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - f.setLong(target, value); - } - - private static int getIntField(Object target, String name) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.getInt(target); - } - - private static long getLongField(Object target, String name) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.getLong(target); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - // Reads the package-private PooledSender.slot -- the identity that the pool - // actually recycles. Wrapper identity is useless for aliasing checks because - // borrow() allocates a fresh wrapper every call (mirrors SenderPoolTest and - // SenderPoolErrorSafetyTest). - private static Object slotOf(PooledSender pooledWrapper) throws Exception { - Field f = PooledSender.class.getDeclaredField("slot"); - f.setAccessible(true); - return f.get(pooledWrapper); - } - - private static void invokeDiscardBroken(SenderPool pool, PooledSender ps) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("discardBroken", PooledSender.class); - m.setAccessible(true); - m.invoke(pool, ps); + private static Sender getDelegate(PooledSender ps) { + return ps.getDelegateForTesting(); } // Uses the @TestOnly senderFactory seam so a test can inject a fake/forged @@ -3810,23 +3715,10 @@ private static SenderPool newPoolWithRecoveryControls( ThreadFactory threadFactory, Runnable recoveryWaiter, Runnable beforeFailedRecoveryJoinHook - ) throws Throwable { - Constructor constructor = SenderPool.class.getDeclaredConstructor( - String.class, int.class, int.class, long.class, long.class, long.class, - IntFunction.class, boolean.class, - io.questdb.client.SenderErrorHandler.class, - io.questdb.client.SenderConnectionListener.class, - io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener.class, - Runnable.class, ThreadFactory.class, Runnable.class, Runnable.class); - constructor.setAccessible(true); - try { - return constructor.newInstance( - cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, - senderFactory, false, null, null, null, null, threadFactory, - recoveryWaiter, beforeFailedRecoveryJoinHook); - } catch (InvocationTargetException e) { - throw e.getCause(); - } + ) { + return SenderPool.createWithRecoveryControlsForTesting( + cfg, min, max, acquireMs, senderFactory, threadFactory, + recoveryWaiter, beforeFailedRecoveryJoinHook); } private static SenderPool newPoolWithRecoveryThreadFactory( @@ -3849,39 +3741,6 @@ private static SenderPool newDeferredPool(String cfg, int min, int max, long acq return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, null, true); } - // Drives a deferred pool's startup recovery to completion (the housekeeper - // drives it one slot per tick; tests drive the whole backlog in one call). - private static void invokeRunStartupRecoveryOnce(SenderPool pool) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("runStartupRecoveryToCompletion"); - m.setAccessible(true); - m.invoke(pool); - } - - // Drives a SINGLE recovery step (the housekeeper's per-tick unit); returns - // whether more stranded slots remain. - private static boolean invokeRunStartupRecoveryStep(SenderPool pool) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("runStartupRecoveryStep"); - m.setAccessible(true); - return (Boolean) m.invoke(pool); - } - - // Raises the pool's shutdown signal early, exactly as QuestDBImpl.close() - // does before stopping the housekeeper. - private static void invokeMarkClosing(SenderPool pool) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("markClosing"); - m.setAccessible(true); - m.invoke(pool); - } - - private static void invokeSetStartupRecoveryJoinHooks( - SenderPool pool, Runnable beforeJoinHook, Runnable afterJoinHook - ) throws Exception { - Method m = SenderPool.class.getDeclaredMethod( - "setStartupRecoveryJoinHooks", Runnable.class, Runnable.class); - m.setAccessible(true); - m.invoke(pool, beforeJoinHook, afterJoinHook); - } - // Deferred pool (deferStartupRecovery=true) WITH an injected factory, so a // test can drive the housekeeper recovery path against fully controlled // (fake) recoverers. diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java index 3a737f50..67d66b95 100644 --- a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.network; +import io.questdb.client.network.JavaTlsClientSocket; import io.questdb.client.network.JavaTlsClientSocketFactory; import io.questdb.client.network.Kqueue; import io.questdb.client.network.KqueueFacade; @@ -43,7 +44,6 @@ import org.slf4j.LoggerFactory; import javax.net.ssl.SSLContext; -import java.lang.reflect.Field; import java.lang.reflect.Proxy; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -454,31 +454,22 @@ public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exc return null; } ); - Socket socket = JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( - facade, - LoggerFactory.getLogger(SocketTrafficShutdownTest.class) - ); + JavaTlsClientSocket socket = (JavaTlsClientSocket) JavaTlsClientSocketFactory + .INSECURE_NO_VALIDATION.newInstance( + facade, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + ); socket.of(42); - Field sslEngineField = socket.getClass().getDeclaredField("sslEngine"); - Field stateField = socket.getClass().getDeclaredField("state"); - sslEngineField.setAccessible(true); - stateField.setAccessible(true); - sslEngineField.set(socket, SSLContext.getDefault().createSSLEngine()); - stateField.setInt(socket, 2); // JavaTlsClientSocket.STATE_TLS - Object[] tlsState = snapshotTlsState(socket); + socket.setTlsStateForTesting(SSLContext.getDefault().createSSLEngine()); + JavaTlsClientSocket.TlsStateForTesting tlsState = socket.snapshotTlsStateForTesting(); try { socket.closeTraffic(); - Object[] stateAfterTrafficClose = snapshotTlsState(socket); - for (int i = 0; i < tlsState.length; i++) { - if (i == 2) { - Assert.assertEquals("traffic cancellation must preserve TLS state", tlsState[i], stateAfterTrafficClose[i]); - } else { - Assert.assertSame("traffic cancellation must preserve SSLEngine/buffer references", - tlsState[i], stateAfterTrafficClose[i]); - } - } + JavaTlsClientSocket.TlsStateForTesting stateAfterTrafficClose = + socket.snapshotTlsStateForTesting(); + Assert.assertTrue("traffic cancellation must preserve TLS state and buffer references", + tlsState.hasSameStateForTesting(stateAfterTrafficClose)); Assert.assertEquals(1, shutdownCount.get()); Assert.assertEquals("traffic cancellation must not release the delegate fd", 0, closeCount.get()); Assert.assertEquals(42, socket.getFd()); @@ -486,8 +477,7 @@ public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exc } finally { // Restore a valid plaintext state so full close does not attempt a // synthetic TLS close_notify with uninitialised session buffers. - sslEngineField.set(socket, null); - stateField.setInt(socket, 1); // JavaTlsClientSocket.STATE_PLAINTEXT + socket.setPlaintextStateForTesting(); socket.close(); } Assert.assertEquals(1, closeCount.get()); @@ -606,22 +596,4 @@ private static void assertUnsupported(Runnable operation) { } } - private static Object[] snapshotTlsState(Socket socket) throws Exception { - String[] fieldNames = { - "callerOutputBuffer", - "sslEngine", - "state", - "unwrapInputBuffer", - "unwrapOutputBuffer", - "wrapInputBuffer", - "wrapOutputBuffer" - }; - Object[] state = new Object[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { - Field field = socket.getClass().getDeclaredField(fieldNames[i]); - field.setAccessible(true); - state[i] = field.get(socket); - } - return state; - } } From fee9903ca6fc2b88d21ac63690d17debea7b6aa8 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Fri, 17 Jul 2026 16:31:13 +0100 Subject: [PATCH 30/64] test(client): fix closeStarted race in closed-mid-creation lock-probe test closedMidCreationTeardownRunsOutsideThePoolLock started the pool-closer thread and the idempotent-re-close lock probe with nothing ordering their entry into close(). On a slow scheduler (Windows CI, build 251972) the probe won the closeStarted race, became the primary closer and legitimately parked in the new bounded 5s in-flight-creation wait -- the parked teardown deliberately holds inFlightCreations at 1 -- so probe.join(5s) expired and the lock-free assertion failed even though the pool lock was free the whole time. Wait on the hasCreationWaiterForTesting() seam until the pool-closer has claimed closeStarted and parked in the creation wait (releasing the lock) before starting the probe, making the probe a true re-close. Also swap the reflective markClosing() invoke for the markClosingForTesting() seam. Reproduced by delaying the closer thread 400ms: the old test fails with the exact CI assertion in 5.1s; the fixed test passes in 0.5s under the same delay. --- .../impl/SenderPoolCloseLifecycleTest.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java index f116a4a6..188d9eb5 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java @@ -288,9 +288,7 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { // Raise the same early shutdown signal used by QuestDBImpl so // the creation deterministically takes the teardown branch. - Method markClosing = SenderPool.class.getDeclaredMethod("markClosing"); - markClosing.setAccessible(true); - markClosing.invoke(pool); + pool.markClosingForTesting(); // close() now owns the in-flight creation reservation until // its closed-mid-creation teardown completes, so run it on a @@ -298,6 +296,21 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { Thread poolCloser = new Thread(pool::close, "pool-closer"); poolCloser.start(); + // Make poolCloser the primary closer DETERMINISTICALLY before + // anything else moves: wait until it has claimed closeStarted + // and parked in the bounded in-flight-creation wait (which + // releases the lock). Without this the lock-probe close() + // below can win the closeStarted race, become the primary + // closer itself and legitimately block in that same 5s wait + // (the parked teardown holds inFlightCreations at 1) -- a + // false "lock held" failure under slow thread scheduling. + long closerParkedDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!pool.hasCreationWaiterForTesting()) { + Assert.assertTrue("pool-closer never parked in the in-flight-creation wait", + System.nanoTime() < closerParkedDeadline); + Thread.sleep(1); + } + // Let the creation finish: the borrower re-locks, observes the // closed pool and starts the delegate teardown, which parks. releaseCreate.countDown(); From f4e6271754d133e7e5e98537b932681c9bab1b09 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Fri, 17 Jul 2026 17:29:28 +0100 Subject: [PATCH 31/64] test(qwp): join the I/O worker before asserting it dead in blocked-send close test close() returns when the worker counts shutdownLatch down inside ioLoop's finally -- before the worker's exit tail and actual thread termination -- so Thread.isAlive() can be observably true for a scheduling beat after close() returns (linux-x64 failure in build 252736). Quiescence is the latch plus the cleanup-before-countdown ordering (already asserted), not thread death; every sibling test already uses join-then-assert. Reproduced by sleeping 200ms after the countdown: the old assert fails exactly like CI; the joined assert passes under the same window. --- .../CursorWebSocketSendLoopBlockedSendCloseTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java index 0a53e28e..20039fa5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java @@ -95,7 +95,14 @@ public void testCloseBreaksBlockedSendBeforeJoiningWorker() throws Exception { Thread ioThread = client.sendThread.get(); Assert.assertNotNull(ioThread); - Assert.assertFalse("I/O worker must be dead when close returns", ioThread.isAlive()); + // close() returns when the worker counts shutdownLatch down -- + // the worker's last action before its exit tail -- so the + // thread can be observably alive for a scheduling beat after + // close() returns. Quiescence is the latch plus the cleanup + // asserts below (cleanup runs before the countdown), not + // thread death: join briefly instead of asserting the race. + ioThread.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("I/O worker did not exit after close returned", ioThread.isAlive()); Assert.assertEquals("full client cleanup must run exactly once", 1, client.cleanupCount.get()); Assert.assertEquals("the I/O worker must own cleanup before publishing exit", ioThread, client.cleanupThread.get()); From 07034cdbb40487253db39d589779eb993d687f5c Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 19 Jul 2026 22:38:20 +0100 Subject: [PATCH 32/64] fix(qwp): recover SF segments with positioned reads Validate persisted segment bytes through bounded positioned reads before mmap. Fail closed on read and size instability, preserve corruption quarantine semantics, and cover sparse tails, buffer boundaries, and recovery I/O faults. --- .../qwp/client/sf/cursor/MmapSegment.java | 496 +++++++++--------- .../MmapSegmentCorruptionException.java | 6 +- .../sf/cursor/MmapSegmentException.java | 4 +- .../qwp/client/sf/cursor/SegmentRing.java | 20 +- .../io/questdb/client/std/FilesFacade.java | 28 +- .../cursor/MmapSegmentRecoveryFaultTest.java | 407 +++++++------- .../qwp/client/sf/cursor/MmapSegmentTest.java | 49 +- .../client/sf/cursor/PrReviewRedTests.java | 8 +- .../cursor/SegmentRecoveryIntegrityTest.java | 37 +- 9 files changed, 584 insertions(+), 471 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index ff7a00da..58afcd5b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -65,8 +65,8 @@ public final class MmapSegment implements QuietCloseable { public static final int HEADER_SIZE = 24; public static final byte MANIFEST_REQUIRED_FLAG = 1; public static final byte VERSION = 1; - private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); + private static final int RECOVERY_BUFFER_SIZE = 64 * 1024; private final FilesFacade filesFacade; private final String path; @@ -102,8 +102,8 @@ public final class MmapSegment implements QuietCloseable { // close deliberately retains it so a cursor can advance after head trim. private volatile MmapSegment successor; // Bytes between the last valid frame and the file end that look like an - // attempted-but-invalid frame write (non-zero bytes at the bail-out - // position). Zero for fresh segments and for cleanly partially-filled + // attempted-but-invalid frame write (the suffix contains non-zero bytes). + // Zero for fresh segments and for cleanly partially-filled // segments (uninitialised tail). Set only by openExisting; visible to // recovery callers for diagnostics. Final after construction. private final long tornTailBytes; @@ -257,16 +257,16 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { } /** - * Opens an existing segment file for recovery. mmaps it RW, validates the - * header magic / version, then scans frames forward verifying each CRC. + * Opens an existing segment file for recovery. Validates the header and + * scans frames through positioned reads, then mmaps the validated file RW. * The first bad CRC (or a frame whose declared length runs past the file * end) is treated as a torn tail; both cursors are positioned at the * start of that frame. Returns the segment ready for further appends. * Throws {@link MmapSegmentException} on header validation failure. *

        - * If recovery observes a torn tail (the bytes at the bail-out position - * are non-zero, indicating an attempted-but-failed frame write rather - * than clean unwritten space), a {@code WARN} is emitted with the byte + * If recovery observes a torn tail (the suffix after the last valid frame + * contains non-zero bytes, indicating an attempted-but-failed frame write + * rather than clean unwritten space), a {@code WARN} is emitted with the byte * count and the bytes are exposed via {@link #tornTailBytes()} so * operators can detect silent truncation from corruption or partial * writes. Clean partial fills (writer never attempted to write past the @@ -277,100 +277,90 @@ public static MmapSegment openExisting(String path) { } /** - * Facade-aware variant of {@link #openExisting(String)} that takes the file - * length (and open/close) through a {@link FilesFacade} instead of straight - * off {@link Files}. Production uses {@link FilesFacade#INSTANCE}; the seam - * exists so recovery's mmap-fault guard can be regression-tested on any - * filesystem. + * Facade-aware variant of {@link #openExisting(String)}. Recovery reads the + * file through {@link FilesFacade#read(int, long, long, long)} before it + * creates the mapping, so sparse or unbacked pages cannot raise SIGBUS in + * the JVM. The descriptor length is checked before and after the scan and + * again after mmap; a short read or size change is an operational failure + * and aborts recovery. *

        - * The mapping is sized to {@code ff.length(path)} and every recovery read - * runs straight out of it, so a facade that reports a length larger - * than the real file makes the mapping extend past end-of-file. A read of a - * page beyond real EOF raises SIGBUS on every filesystem — the same - * fault an unbacked/sparse page raises on ZFS, but reproduced - * deterministically on ext4/xfs, where a within-EOF hole would instead - * zero-fill and never exercise the guard. See - * {@link FilesFacade#length(String)}. + * The caller must prevent concurrent writers from modifying the file during + * recovery and for the lifetime of the returned segment. The length checks + * detect observed truncate/extend races, but no mmap-based implementation + * can remain safe against uncoordinated mutation after the final check. */ public static MmapSegment openExisting(FilesFacade ff, String path) { - long fileSize = ff.length(path); - if (fileSize < HEADER_SIZE) { - // Corruption, not an operational error: the bytes themselves prove - // this cannot be a whole segment (a create() is never durable at a - // sub-header size — allocate() reserves the full extent up front). - throw new MmapSegmentCorruptionException( - "file shorter than header: " + path + " size=" + fileSize); - } int fd = ff.openRW(path); if (fd < 0) { throw new MmapSegmentException("openRW failed for " + path); } long addr = Files.FAILED_MMAP_ADDRESS; + long fileSize = -1L; try { + fileSize = ff.length(fd); + if (fileSize < 0) { + throw new MmapSegmentException( + "could not stat open segment " + path + " [errno=" + Os.errno() + ']'); + } + if (fileSize < HEADER_SIZE) { + // Corruption, not an operational error: the bytes themselves prove + // this cannot be a whole segment (a create() is never durable at a + // sub-header size — allocate() reserves the full extent up front). + throw new MmapSegmentCorruptionException( + "file shorter than header: " + path + " size=" + fileSize); + } + + RecoveryScan scan = scanForRecovery(ff, fd, path, fileSize); + long finalSize = ff.length(fd); + if (finalSize < 0) { + throw new MmapSegmentException( + "could not re-stat open segment " + path + " [errno=" + Os.errno() + ']'); + } + if (finalSize != fileSize) { + throw new MmapSegmentException( + "segment size changed during recovery: " + path + + " [before=" + fileSize + ", after=" + finalSize + ']'); + } + addr = ff.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + path); } - int magic = Unsafe.getUnsafe().getInt(addr); - if (magic != FILE_MAGIC) { - throw new MmapSegmentCorruptionException( - "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); + long mappedSize = ff.length(fd); + if (mappedSize < 0) { + throw new MmapSegmentException( + "could not stat mapped segment " + path + " [errno=" + Os.errno() + ']'); } - byte version = Unsafe.getUnsafe().getByte(addr + 4); - if (version != VERSION) { - // Deliberately NOT the corruption subtype: an unsupported - // version is a well-formed segment written by a different - // client build (e.g. after a downgrade). Quarantine-renaming - // it would strand its frames for the writer that CAN read it; - // failing recovery keeps the slot intact for that writer. - throw new MmapSegmentException("unsupported version in " + path + ": " + version); - } - long baseSeq = Unsafe.getUnsafe().getLong(addr + 8); - // FSNs are non-negative by construction (see SegmentRing). - // A negative baseSeq on disk means bit-rot or a malicious file — - // refuse the segment so SegmentRing.openExisting's narrow catch - // skips it like any other unreadable .sfa rather than feeding - // the bad value into Long.compareUnsigned-based contiguity - // checks (which would place the segment last in baseSeq order - // and trip the FSN-gap throw, taking the whole recovery down). - if (baseSeq < 0L) { - throw new MmapSegmentCorruptionException( - "bad baseSeq in " + path + ": " + baseSeq); + if (mappedSize != fileSize) { + throw new MmapSegmentException( + "segment size changed while mapping: " + path + + " [before=" + fileSize + ", after=" + mappedSize + ']'); } - long lastGood = scanFrames(addr, fileSize); - long count = countFrames(addr, lastGood); - long tornTail = detectTornTail(addr, lastGood, fileSize); - if (tornTail > 0) { + if (scan.tornTailBytes > 0) { LOG.warn("SF segment {}: torn tail of {} bytes at offset {} " + "(file size {}, frames recovered {}). " + "Recovery will overwrite this region on next append; " + "frames past the tear (if any) are discarded. " + "Investigate disk health or unexpected writer crash.", - path, tornTail, lastGood, fileSize, count); + path, scan.tornTailBytes, scan.lastGood, fileSize, scan.frameCount); } - return new MmapSegment(ff, path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); + return new MmapSegment( + ff, + path, + fd, + addr, + fileSize, + scan.baseSeq, + scan.lastGood, + scan.frameCount, + false, + scan.tornTailBytes + ); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { ff.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); } ff.close(fd); - // The header reads above (magic/version/baseSeq) run before - // scanFrames and are otherwise unguarded. An unbacked page 0 -- - // an unflushed header on a truncate-based-allocate filesystem - // after a crash (create() does not msync), or a file truncated - // under the mapping -- faults at the Unsafe intrinsic site as a - // catchable InternalError. Convert it to a MmapSegmentException so - // SegmentRing's per-file catch skips just this .sfa, instead of - // letting the raw InternalError escape to SegmentRing's outer catch - // and abort recovery of every sibling segment. scanFrames and - // detectTornTail already handle their own in-mapping faults; this - // covers the header block and any future reader placed ahead of the - // scan. - if (isMmapAccessFault(t)) { - throw new MmapSegmentCorruptionException( - "unreadable mapped header page in " + path - + " (unbacked/sparse page 0): " + t.getMessage(), t); - } throw t; } } @@ -598,204 +588,220 @@ public long frameCount() { * recovery observes non-zero bytes past the bail-out point. {@code 0} for * fresh segments, memory-backed segments, and cleanly partially-filled * recovered segments. Operators / tests can read this to tell silent - * truncation (corruption) from a normal partial fill (no incident). - *

        - * One case this does NOT count: when the scan stops because the bail-out - * region is itself an unbacked mapped page (an in-mapping fault, not a - * backed torn header), that region cannot be probed, so this returns - * {@code 0} even though frames may have been discarded. That outcome is - * surfaced by the {@code WARN} in {@link #scanFrames} instead -- see it for - * the benign-tail-vs-media-error caveat. + * truncation (corruption) from a normal partial fill (no incident). Sparse + * holes read back as zeroes; an actual positioned-read failure aborts + * recovery instead of being classified as a clean tail. */ public long tornTailBytes() { return tornTailBytes; } /** - * True when {@code t} is the JVM's recoverable signal for a fault while - * accessing a memory-mapped region -- a SIGBUS/SIGSEGV that HotSpot - * translates into an {@code InternalError} at an {@code Unsafe} intrinsic - * site instead of aborting the process. It surfaces when a mapped page is - * not backed by real file blocks: a sparse {@code .sfa} tail on a - * filesystem whose pre-allocation leaves holes (e.g. ZFS, where a - * truncate-based {@code allocate} does not materialize blocks), or a file - * truncated under the mapping. Recovery treats this as an I/O boundary -- - * the same way MappedByteBuffer readers do -- not a fatal VM error. - *

        - * The message is matched on the fragment {@code "unsafe memory access - * operation"}, which is common to both HotSpot wordings and NOT - * version-stable as a whole: pre-21 JDKs (including the shipping/CI JDK 8) - * emit {@code "a fault occurred in a recent unsafe memory access operation - * in compiled Java code"}, while JDK 21+ shortened it to {@code "a fault - * occurred in an unsafe memory access operation"}. Matching the shared - * fragment keeps the guard effective on JDK 8/11/17 as well as 21+, while - * still being specific enough that a genuine VirtualMachineError (real OOM, - * StackOverflow) is never swallowed. + * Scans the header and frames through a bounded positioned-read buffer. + * Sparse file holes are returned by {@code pread}/{@code ReadFile} as zero + * bytes, while media errors and concurrent truncation surface as ordinary + * read failures. No recovery byte is dereferenced through mmap. */ - private static boolean isMmapAccessFault(Throwable t) { - if (!(t instanceof InternalError)) { - return false; - } - String msg = t.getMessage(); - return msg != null && msg.contains("unsafe memory access operation"); - } + private static RecoveryScan scanForRecovery( + FilesFacade ff, + int fd, + String path, + long fileSize + ) { + try (RecoveryReader reader = new RecoveryReader(ff, fd, path, fileSize)) { + int magic = reader.getInt(0L); + if (magic != FILE_MAGIC) { + throw new MmapSegmentCorruptionException( + "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); + } + byte version = reader.getByte(4L); + if (version != VERSION) { + // Deliberately NOT the corruption subtype: an unsupported + // version is a well-formed segment written by a different + // client build (e.g. after a downgrade). Quarantine-renaming + // it would strand its frames for the writer that CAN read it; + // failing recovery keeps the slot intact for that writer. + throw new MmapSegmentException("unsupported version in " + path + ": " + version); + } + long baseSeq = reader.getLong(8L); + // FSNs are non-negative by construction (see SegmentRing). A + // negative value on disk is positively identified corruption. + if (baseSeq < 0L) { + throw new MmapSegmentCorruptionException( + "bad baseSeq in " + path + ": " + baseSeq); + } - /** - * Forward scan that returns the offset just past the last frame whose - * CRC verifies. A torn-tail frame (declared length runs past EOF, or - * CRC mismatch) leaves both cursors at the start of that frame; the - * next {@link #tryAppend} will overwrite it. The scan only reads from - * the mapping — no syscalls. - */ - private static long scanFrames(long addr, long fileSize) { - long pos = HEADER_SIZE; - try { - while (pos + FRAME_HEADER_SIZE <= fileSize) { - int crcRead = Unsafe.getUnsafe().getInt(addr + pos); - int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4); - // Defensive: a corrupt length field could be enormous or negative, - // both of which would otherwise overrun the mapping. - if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) { - return pos; + long frameCount = 0L; + long pos = HEADER_SIZE; + while (fileSize - pos >= FRAME_HEADER_SIZE) { + int crcRead = reader.getInt(pos); + int payloadLen = reader.getInt(pos + 4L); + // Avoid addition overflow while rejecting a negative or + // beyond-EOF frame length as a torn tail. + if (payloadLen < 0 || payloadLen > fileSize - pos - FRAME_HEADER_SIZE) { + break; } - // CRC over the contiguous (payloadLen, payload) pair, folded - // via Unsafe reads rather than the native Crc32c.update. - // Recovery maps to the file's stat length, but a page inside - // that range can be unbacked: a sparse pre-allocation tail (a - // truncate-based allocate that never materialized blocks, as on - // ZFS), or -- via a torn write, since tryAppend writes the - // length field before copying the payload -- a real positive - // payloadLen whose payload spans into an unwritten hole. A raw - // read of an unbacked page raises SIGBUS; HotSpot translates - // that into a catchable InternalError ONLY at an Unsafe - // intrinsic site, NEVER inside JNI native code, so a native - // Crc32c.update over such a page aborts the whole JVM (and, - // empirically on pre-21 JDKs, a preceding Unsafe pre-touch does - // not reliably fault first once an earlier native CRC ran in - // the same scan). Folding over Unsafe keeps every fault - // catchable -- handled below as the boundary of recoverable - // data; a page that instead reads back as zeroes just fails the - // CRC check and ends the scan. Recovery is cold, so the slower - // table CRC here is immaterial. - int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen); + int crcCalc = reader.crc32c(pos + 4L, 4L + payloadLen); if (crcCalc != crcRead) { - return pos; + break; } pos += FRAME_HEADER_SIZE + payloadLen; + frameCount++; } - } catch (InternalError e) { - // The read at `pos` hit a mapped page that is not backed by real - // file blocks: the JVM translates the underlying SIGBUS into a - // recoverable InternalError instead of aborting the process. This - // happens when a prior session left a sparse segment tail (a - // truncate-based pre-allocation that does not materialize blocks, - // as on ZFS) or the file was truncated under the mapping. Every - // frame below `pos` already verified; treat the unreadable region - // exactly like unwritten space or a torn tail -- the boundary of - // recoverable data -- rather than letting the error abort recovery - // of the whole slot. Anything that is not the documented mmap - // access fault is a genuine VM error, so rethrow it. - if (!isMmapAccessFault(e)) { - throw e; - } - LOG.warn("SF segment recovery: unreadable mapped page at offset {} (file size {}); " - + "treating it as the end of recoverable data -- any frames beyond this " - + "offset are discarded. The usual cause is a benign sparse pre-allocation " - + "tail (e.g. a truncate-based allocate on ZFS) left by a prior session, " - + "but a mid-file media error (bad sector) is indistinguishable here; " - + "check disk health if this segment was expected to be fully written or " - + "if this recurs.", - pos, fileSize); + long tornTailBytes = detectTornTail(reader, pos, fileSize); + return new RecoveryScan(baseSeq, frameCount, pos, tornTailBytes); } - return pos; } /** - * CRC-32C (Castagnoli) of {@code [addr, addr + len)} read through - * {@link Unsafe}, seeded like {@code Crc32c.update(Crc32c.INIT, addr, len)} - * and bit-identical to it (verified) -- but every byte load is an Unsafe - * intrinsic, so a fault on an unbacked mapped page is a catchable - * {@link InternalError} instead of the uncatchable JNI SIGBUS the native - * {@link Crc32c} would raise. Byte-at-a-time via a precomputed table - * ({@code ~0.5 GiB/s}); used only on the cold recovery scan, never on the - * append hot path (which stays on the native, hardware-friendly path). + * Distinguishes attempted frame data from clean zero-filled space after the + * recovery boundary. The entire suffix is inspected: an all-zero corrupt + * frame header must not hide a non-zero payload or later frames and make a + * data-bearing segment look like a reusable empty spare. Positioned-read + * failures are intentionally not swallowed; they are operational failures, + * not proof of an unwritten tail. */ - private static int crc32cRecovery(long addr, long len) { - int crc = ~Crc32c.INIT; - for (long i = 0; i < len; i++) { - crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF]; + private static long detectTornTail(RecoveryReader reader, long lastGood, long fileSize) { + if (lastGood >= fileSize) { + return 0L; + } + return reader.hasNonZero(lastGood, fileSize - lastGood) + ? fileSize - lastGood + : 0L; + } + + private static final class RecoveryReader implements QuietCloseable { + private final long bufferAddress; + private final int fd; + private final FilesFacade filesFacade; + private final long fileSize; + private final String path; + private int bufferLength; + private long bufferOffset = -1L; + + private RecoveryReader(FilesFacade filesFacade, int fd, String path, long fileSize) { + this.filesFacade = filesFacade; + this.fd = fd; + this.path = path; + this.fileSize = fileSize; + this.bufferAddress = Unsafe.malloc(RECOVERY_BUFFER_SIZE, MemoryTag.NATIVE_DEFAULT); } - return ~crc; - } - /** - * Standard reflected CRC-32C byte table (polynomial {@code 0x82F63B78}), - * matching {@code crc32c_table[0]} in the native {@code crc32c.c}. Computed - * at class init to avoid 256 hand-transcribed literals; drives - * {@link #crc32cRecovery}. - */ - private static int[] buildCrc32cTable() { - int[] table = new int[256]; - for (int n = 0; n < 256; n++) { - int c = n; - for (int k = 0; k < 8; k++) { - c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1); + @Override + public void close() { + Unsafe.free(bufferAddress, RECOVERY_BUFFER_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + private int crc32c(long offset, long len) { + int crc = Crc32c.INIT; + while (len > 0L) { + ensure(offset, 1); + int bufferIndex = (int) (offset - bufferOffset); + long chunk = Math.min(len, bufferLength - bufferIndex); + crc = Crc32c.update(crc, bufferAddress + bufferIndex, chunk); + offset += chunk; + len -= chunk; } - table[n] = c; + return crc; } - return table; - } - /** - * Distinguishes "torn tail" (writer attempted a write past the last valid - * frame and failed — partial write, mid-stream corruption, bit rot) from - * clean unwritten space (manager-allocated segment with zero-filled tail). - * Returns the byte count from {@code lastGood} to {@code fileSize} when - * the bytes at the bail-out frame header are non-zero, else {@code 0}. - *

        - * Heuristic but robust for the common cases: {@link #create} truncates the - * file to size, leaving the tail zero-filled; the writer only writes - * non-zero bytes via {@link #tryAppend}, which writes the CRC and length - * fields together. So a non-zero byte at the failed-frame position - * implies an attempted write — exactly the case operators want flagged. - */ - private static long detectTornTail(long addr, long lastGood, long fileSize) { - if (lastGood >= fileSize) { - return 0L; + private void ensure(long offset, int requiredBytes) { + if (offset >= bufferOffset + && offset - bufferOffset <= bufferLength - requiredBytes) { + return; + } + if (offset < 0L || requiredBytes < 0 || offset > fileSize - requiredBytes) { + throw new MmapSegmentException( + "recovery read outside segment " + path + + " [offset=" + offset + ", required=" + requiredBytes + + ", fileSize=" + fileSize + ']'); + } + int len = (int) Math.min((long) RECOVERY_BUFFER_SIZE, fileSize - offset); + readFully(bufferAddress, len, offset); + bufferOffset = offset; + bufferLength = len; } - long probe = Math.min(FRAME_HEADER_SIZE, fileSize - lastGood); - try { - for (long i = 0; i < probe; i++) { - if (Unsafe.getUnsafe().getByte(addr + lastGood + i) != 0) { - return fileSize - lastGood; + + private byte getByte(long offset) { + ensure(offset, 1); + return Unsafe.getUnsafe().getByte(bufferAddress + offset - bufferOffset); + } + + private int getInt(long offset) { + ensure(offset, Integer.BYTES); + return Unsafe.getUnsafe().getInt(bufferAddress + offset - bufferOffset); + } + + private long getLong(long offset) { + ensure(offset, Long.BYTES); + return Unsafe.getUnsafe().getLong(bufferAddress + offset - bufferOffset); + } + + private boolean hasNonZero(long offset, long len) { + boolean nonZero = false; + while (len > 0L) { + ensure(offset, 1); + int bufferIndex = (int) (offset - bufferOffset); + int chunk = (int) Math.min(len, bufferLength - bufferIndex); + long address = bufferAddress + bufferIndex; + int i = 0; + while (i < chunk && ((address + i) & (Long.BYTES - 1L)) != 0L) { + nonZero |= Unsafe.getUnsafe().getByte(address + i++) != 0; + } + while (i <= chunk - Long.BYTES) { + nonZero |= Unsafe.getUnsafe().getLong(address + i) != 0L; + i += Long.BYTES; + } + while (i < chunk) { + nonZero |= Unsafe.getUnsafe().getByte(address + i++) != 0; } + offset += chunk; + len -= chunk; } - } catch (InternalError e) { - // The bail-out region is an unbacked (sparse) mapped page -- see - // scanFrames for the mechanism. An unbacked hole was never written, - // so it is clean unwritten space, not a torn write. Rethrow any - // error that is not the recoverable mmap access fault. - if (!isMmapAccessFault(e)) { - throw e; + return nonZero; + } + + private void readFully(long address, long len, long offset) { + long read = 0L; + while (read < len) { + long n = filesFacade.read(fd, address + read, len - read, offset + read); + if (n < 0L) { + throw new MmapSegmentException( + "could not read SF segment " + path + + " [offset=" + (offset + read) + + ", remaining=" + (len - read) + + ", errno=" + Os.errno() + ']'); + } + if (n == 0L) { + throw new MmapSegmentException( + "short read while recovering SF segment " + path + + " [offset=" + (offset + read) + + ", remaining=" + (len - read) + + ", fileSize=" + fileSize + ']'); + } + if (n > len - read) { + throw new MmapSegmentException( + "invalid read length while recovering SF segment " + path + + " [offset=" + (offset + read) + + ", requested=" + (len - read) + + ", actual=" + n + ']'); + } + read += n; } - return 0L; } - return 0L; } - /** - * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in - * lockstep with {@link #scanFrames} (which already validated CRCs); so - * this is just length-driven traversal, no CRC re-check. - */ - private static long countFrames(long addr, long lastGood) { - long pos = HEADER_SIZE; - long count = 0; - while (pos < lastGood) { - int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4); - pos += FRAME_HEADER_SIZE + payloadLen; - count++; - } - return count; + private static final class RecoveryScan { + private final long baseSeq; + private final long frameCount; + private final long lastGood; + private final long tornTailBytes; + + private RecoveryScan(long baseSeq, long frameCount, long lastGood, long tornTailBytes) { + this.baseSeq = baseSeq; + this.frameCount = frameCount; + this.lastGood = lastGood; + this.tornTailBytes = tornTailBytes; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java index 34a89f38..8a72a026 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java @@ -27,9 +27,9 @@ /** * Positively-identified segment corruption: the file's own bytes prove it is * not (or no longer) a readable SF segment — truncated below the fixed header, - * wrong magic, a negative {@code baseSeq}, or an unreadable (unbacked/torn) - * header page. Distinct from its parent {@link MmapSegmentException}, which - * recovery treats as an operational failure (open/mmap error on a file + * wrong magic, or a negative {@code baseSeq}. Distinct from its parent + * {@link MmapSegmentException}, which recovery treats as an operational + * failure (open/read/mmap error on a file * whose contents may be perfectly intact) and must therefore be fatal. *

        * Recovery quarantines corruption (rename to {@code .corrupt}) and diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index e54eecb5..dda38a8e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -28,10 +28,10 @@ * Hard failure of the MmapSegment layer — bad header, mmap rejection, file * too short for header, etc. Indicates the segment is unusable, not that * the disk is full (the latter surfaces as backpressure on the producer - * via {@link io.questdb.client.cutlass.qwp.client.LineSenderException}). + * via {@link io.questdb.client.cutlass.line.LineSenderException}). *

        * Recovery distinguishes two flavors: this base type marks operational - * failures (open/mmap/enumeration errors — the file's contents may be fine, + * failures (open/read/mmap/enumeration errors — the file's contents may be fine, * so recovery must fail closed), while the {@link MmapSegmentCorruptionException} * subtype marks positively-identified corruption in the file's own * bytes, which recovery may quarantine instead of aborting. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index f068b263..b839f779 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -226,16 +226,16 @@ static Recovery recover( ObjList all = new ObjList<>(); // Files whose own bytes prove corruption (bad magic, sub-header size, - // negative baseSeq, unreadable header page). They are excluded from - // the chain and quarantined to .corrupt — but only AFTER the - // surviving chain validates (or resolves to EMPTY), so a failed - // recovery never mutates the slot. Whether a quarantined file was - // load-bearing is decided by the manifest-boundary / contiguity - // checks below, not by the skip itself. Operational open errors - // (EMFILE, EACCES, mmap rejection, unsupported version) are NOT in - // this bucket: they throw the plain MmapSegmentException type and - // abort recovery, because the underlying file may be perfectly - // intact and silently dropping it could lose durable frames. + // negative baseSeq). They are excluded from the chain and quarantined + // to .corrupt — but only AFTER the surviving chain validates (or + // resolves to EMPTY), so a failed recovery never mutates the slot. + // Whether a quarantined file was load-bearing is decided by the + // manifest-boundary / contiguity checks below, not by the skip itself. + // Operational open/stat/read/mmap errors, observed size instability, + // and unsupported versions are NOT in this bucket: they throw the + // plain MmapSegmentException type and abort recovery, because the + // underlying file may be perfectly intact and silently dropping it + // could lose durable frames. ObjList corruptPaths = null; SfManifest manifest = null; try { diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 082e680e..7f436697 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -89,21 +89,18 @@ default int fsyncDir(String dir) { return Files.fsyncDir(dir); } + /** + * Returns the current byte length of the file referenced by open descriptor + * {@code fd}, or a negative value when the descriptor cannot be statted. + */ long length(int fd); /** - * Stat length of the file at {@code path}, in bytes. Default delegates to - * {@link Files#length(String)}. - * - *

        Test injection point: {@code MmapSegment.openExisting} maps the file to - * this length and scans straight out of the mapping, so a wrapping facade - * that returns a value larger than the real file makes the mapping - * extend past end-of-file. A read of a page beyond real EOF raises SIGBUS on - * every filesystem (which HotSpot translates to a catchable - * {@code InternalError} at an {@code Unsafe} intrinsic site) — the same fault - * a genuinely unbacked/sparse page raises on ZFS, but reproduced - * deterministically on ext4/xfs too. That is what lets recovery's mmap-fault - * guard be regression-tested on any CI runner rather than only on ZFS. + * Stat length of the file at {@code path}, in bytes. + * {@link DefaultFilesFacade} delegates to {@link Files#length(String)}. + * Code that already owns an open descriptor should prefer + * {@link #length(int)} so path replacement cannot make the stat refer to a + * different file. */ long length(String path); @@ -152,6 +149,13 @@ default int openRWExclusive(long pathPtr) { */ long length(long pathPtr); + /** + * Reads up to {@code len} bytes from the absolute file {@code offset} into + * native memory at {@code addr}, without changing the descriptor position. + * A positive result may be shorter than requested and callers that require + * the whole range must retry. For a non-zero request, {@code 0} means EOF + * or no progress and a negative result indicates an operating-system error. + */ long read(int fd, long addr, long len, long offset); boolean remove(String path); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java index 861f761c..fbe179e8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentCorruptionException; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; @@ -36,61 +37,17 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** - * Regression guard for the recovery-time SIGBUS hazard in {@link MmapSegment}. - *

        - * On recovery, {@link MmapSegment#openExisting} maps a persisted {@code .sfa} - * to its stat length and scans frames straight out of the mapping. When a prior - * session left a sparse segment tail -- a truncate-based pre-allocation that - * never materialized the tail blocks, as happens on ZFS -- a read of an - * unbacked page raises the JVM's recoverable - * {@code InternalError("...unsafe memory access operation...")} (a translated - * SIGBUS). Recovery must treat that page as the boundary of recoverable data, - * keep every frame below it, and hand back a usable segment -- not let the - * error abort recovery of the whole slot (the reported ZFS-CI flake). - *

        - * These tests drive the production entry point ({@code openExisting}), - * not the private scan methods via reflection. That matters for two reasons: - *

          - *
        • It exercises the real recovery path end to end.
          • - *
        • On pre-21 JDKs the mmap-fault {@code InternalError} is delivered - * imprecisely ("a fault occurred in a recent unsafe memory access - * operation in compiled Java code") and escapes a reflective - * {@code Method.invoke} frame instead of being caught inside the scan -- - * so a reflection-based test spuriously fails on the shipping JDK 8/11/17 - * even though the direct-call production path catches it fine.
          • - *
        - * The fault-delivery mechanism the fix rests on was verified directly on the - * shipping/CI Java floor -- JDK 8 (Temurin 1.8.0_492) -- not merely inferred - * from the adjacent pre-21 LTS releases: the whole class passes there in both - * interpreter ({@code -Xint}) and JIT modes, HotSpot emits the exact pre-21 - * message above, and a direct {@code try/catch} catches the fault in - * interpreter, C1, and C2 modes. {@code isMmapAccessFault}'s shared - * {@code "unsafe memory access operation"} fragment matches that message while - * the JDK 21+-only needle it replaced does not -- the guard is live on JDK 8. - * The unbacked tail is produced portably by truncating the file down (dropping - * the tail blocks) and back up to the mapping size (leaving a sparse hole). A - * hole-faulting filesystem (ZFS) then faults on the read exactly as in - * production -- the case the fix must survive rather than fold the CRC through - * the native, JNI-side {@code Crc32c} where a SIGBUS is uncatchable and aborts - * the JVM. A hole-zero-filling filesystem (ext4) instead reads the hole back as - * zeroes, which fails the frame CRC; either way recovery must stop at the same - * boundary and recover the same frames. - *

        - * Fail-on-revert on any filesystem. The sparse-hole tests above only - * fault on ZFS: on ext4/xfs the within-EOF hole zero-fills, so the scan stops - * via the CRC-mismatch / bad-magic branch and they stay green even with the - * mmap-fault guard reverted -- no regression protection on the ext4/xfs CI - * runners. The two {@code MapPastEof} tests below close that gap portably. - * They truncate the file down (freeing the tail blocks) and hand - * {@code openExisting} a {@link FilesFacade} that reports the original, larger - * length, so the mapping extends past real end-of-file. A read of a page beyond - * real EOF raises SIGBUS on every filesystem -- the same catchable - * {@code InternalError} an unbacked ZFS page raises -- so they exercise the - * real fault path (and fail on revert) on ext4/xfs too, not only on ZFS. + * Regression guards for recovery through positioned file reads. Recovery must + * consume every byte it validates before mmap creation so sparse or unbacked + * pages never raise SIGBUS during validation. Sparse holes are ordinary zero-filled + * reads; negative reads, premature EOF, and file-size changes are operational + * errors that fail recovery closed without returning a live mapping or mutating + * the segment. */ public class MmapSegmentRecoveryFaultTest { @@ -125,8 +82,8 @@ public void testRecoveryKeepsFramesBeforeUnbackedTail() throws Exception { long boundary = writeSegment(path, 7L, new int[]{payloadLen}); assertEquals("frame must fill exactly one page", page, boundary); - // Drop the tail blocks, then re-extend logically so [page, SEGMENT_BYTES) - // is an unbacked hole under the recovery mapping. + // Drop the tail blocks, then re-extend logically so + // [page, SEGMENT_BYTES) is an unbacked hole in the persisted file. punchSparseTail(path, page); try (MmapSegment seg = MmapSegment.openExisting(path)) { @@ -138,15 +95,9 @@ public void testRecoveryKeepsFramesBeforeUnbackedTail() throws Exception { } /** - * The harder case: a frame whose 8-byte header sits on a backed page but - * whose payload reaches into the unbacked hole (a torn write leaves a real - * positive {@code payloadLen} with the payload spanning the boundary). The - * CRC fold therefore reads across the backed-to-unbacked edge. Recovery - * must reject that frame and keep the one below it -- and, crucially, must - * do so via {@code Unsafe} reads: the native, JNI-side {@code Crc32c} over - * an unbacked page raises a SIGBUS that HotSpot cannot translate, aborting - * the whole JVM (verified: an {@code hs_err} in - * {@code Java_io_questdb_client_std_Crc32c_update}). + * The harder sparse case: a frame header is backed but its payload reaches + * into a hole. Positioned reads return zeroes for the hole, so CRC + * validation rejects that frame without dereferencing the mapping. */ @Test public void testRecoverySurvivesPayloadReachingUnbackedPage() throws Exception { @@ -159,10 +110,16 @@ public void testRecoverySurvivesPayloadReachingUnbackedPage() throws Exception { // the backed->unbacked edge. final long frame2Offset = boundary - 16; final int payloadLen2 = (int) page; - final int payloadLen1 = (int) (frame2Offset - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE); + final int payloadLen1 = (int) ( + frame2Offset - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE + ); long used = writeSegment(path, 11L, new int[]{payloadLen1, payloadLen2}); - assertEquals("frame 2's header must end 8 bytes below the page boundary", boundary - 8, frame2Offset + MmapSegment.FRAME_HEADER_SIZE); + assertEquals( + "frame 2's header must end 8 bytes below the page boundary", + boundary - 8, + frame2Offset + MmapSegment.FRAME_HEADER_SIZE + ); assertTrue("frame 2 payload must reach past the boundary", used > boundary); punchSparseTail(path, boundary); @@ -178,18 +135,8 @@ public void testRecoverySurvivesPayloadReachingUnbackedPage() throws Exception { } /** - * M1 regression: the header block (magic/version/baseSeq) is read before - * {@code scanFrames}, so an unbacked page 0 faults ahead of the guarded - * scan. {@link MmapSegment#openExisting} must surface that as a - * {@link MmapSegmentException} -- the per-file signal {@code SegmentRing} - * catches to skip just this {@code .sfa} -- and never let the raw - * {@code InternalError} escape and abort recovery of every sibling segment. - *

        - * Portable across filesystems: on a hole-faulting FS (ZFS) the fault is - * converted to a {@code MmapSegmentException} in {@code openExisting}'s - * catch; on a hole-zero-filling FS (ext4) page 0 reads back as zeroes, so - * the magic check fails and throws {@code MmapSegmentException} directly. - * Either way the file is skippable, not fatal. + * A sparse page zero header is positively identified as corrupt from the + * bytes returned by positioned read and is therefore skippable per-file. */ @Test public void testUnbackedHeaderPageIsSkippableNotFatal() throws Exception { @@ -200,101 +147,172 @@ public void testUnbackedHeaderPageIsSkippableNotFatal() throws Exception { punchSparseTail(path, 0L); try { MmapSegment.openExisting(path).close(); - fail("expected MmapSegmentException for an unbacked header page"); + fail("expected corruption for a sparse zero header"); + } catch (MmapSegmentCorruptionException expected) { + // ok -- SegmentRing's narrow corruption catch skips just this + // file instead of aborting recovery of the whole slot. + } catch (MmapSegmentException unexpected) { + fail("expected quarantinable corruption subtype, got " + unexpected); + } + }); + } + + @Test + public void testLargeFrameRecoveryCrossesReadBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-large-frame.sfa"; + // Put frame 2's length field across the first 64 KiB boundary, + // then make its payload span several recovery-buffer refills. + final int firstPayloadLen = 64 * 1024 + - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE - 5; + final int largePayloadLen = 3 * 64 * 1024 + 17; + assertEquals(64 * 1024 - 5, + MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + firstPayloadLen); + long expectedEnd = writeSegment(path, 13L, new int[]{firstPayloadLen, largePayloadLen, 31}); + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(3L, seg.frameCount()); + assertEquals(expectedEnd, seg.publishedOffset()); + assertEquals(0L, seg.tornTailBytes()); + } + }); + } + + @Test + public void testReadErrorFailsClosedBeforeMmap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-read-error.sfa"; + writeSegment(path, 17L, new int[]{256}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.failReadWithError = true; + ff.stopReadsAt = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected positioned-read failure"); } catch (MmapSegmentException expected) { - // ok -- SegmentRing's per-file catch skips just this file - // instead of aborting recovery of the whole slot. + assertFalse("operational read errors must not be quarantinable corruption", + expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("could not read")); } + assertEquals("mapping must not start after a failed scan", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("read failure must not mutate the segment", Files.exists(path)); }); } - /** - * Portable fail-on-revert guard for the recovery mmap-fault handling on the - * scan path. Unlike {@link #testRecoveryKeepsFramesBeforeUnbackedTail} - * (which only faults on ZFS), this maps the file past real EOF via the - * length-injecting facade, so the scan's read of the beyond-EOF page faults - * on ext4/xfs too. The fix must recognize that fault and keep - * recovery safe -- never a JVM abort, never a raw {@code InternalError} - * escaping into {@code SegmentRing}'s recovery loop. Revert the - * {@code scanFrames}/{@code openExisting} mmap-fault guard (or fold the CRC - * back through native {@code Crc32c}) and this errors or aborts the fork. - *

        - * Two handled outcomes are accepted, because which one occurs depends on - * whether the recovery methods are JIT-compiled at fault time: - *

          - *
        • Interpreter / C1: {@code scanFrames}'s own - * {@code catch (InternalError)} fires, so the frame below the tear is - * recovered and a usable segment is returned.
          • - *
        • C2: once {@code scanFrames} is inlined into - * {@code openExisting}, HotSpot delivers the async unsafe-access - * {@code InternalError} to {@code openExisting}'s outer - * {@code catch (Throwable)} instead of the inlined inner one, which - * converts the file to a skippable {@link MmapSegmentException}. - * Still fully handled -- {@code SegmentRing} skips just this - * {@code .sfa} rather than aborting the slot.
          • - *
        - * (The C2 delivery imprecision is a property of HotSpot's async - * unsafe-access fault handling, not of this seam; the seam only makes it - * reproducible off ZFS.) - */ @Test - public void testScanFaultOnMapPastEofIsHandledAnyFilesystem() throws Exception { + public void testReadErrorAfterDetectedTornBytesStillFailsClosed() throws Exception { TestUtils.assertMemoryLeak(() -> { - final String path = tmpDir + "/seg-mappasteof-scan.sfa"; - final long page = Files.PAGE_SIZE; - // One frame that ends exactly on the first page boundary. - final int payloadLen = (int) (page - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE); - long boundary = writeSegment(path, 5L, new int[]{payloadLen}); - assertEquals("frame must fill exactly one page", page, boundary); - // Free every block past the first page: the file is now exactly one - // (fully backed) page, with nothing beyond it on disk. - truncateTo(path, page); - // Report twice the real length so openExisting maps a second, - // beyond-EOF page; the scan faults reading it on any filesystem. - FilesFacade ff = new MapPastEofFacade(path, 2 * page); + final String path = tmpDir + "/seg-torn-then-read-error.sfa"; + long lastGood = writeSegment(path, 18L, new int[]{256}); + + // Make the failed-frame header non-zero in the first recovery + // buffer. Recovery must still read the rest of the suffix so an + // operational error in a later buffer cannot hide behind the + // already-established torn-tail signal. + int fd = Files.openRW(path); + assertTrue("openRW failed", fd >= 0); + long marker = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(marker, (byte) 1); + assertEquals(1L, Files.write(fd, marker, 1L, lastGood)); + } finally { + Unsafe.free(marker, 1, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.failReadWithError = true; + ff.stopReadsAt = 64L * 1024L; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected later positioned-read failure"); + } catch (MmapSegmentException expected) { + assertFalse(expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("could not read")); + } + assertTrue("fault must occur after recovery consumed the first buffer", ff.readCalls > 1); + assertEquals("mapping must not start after any suffix read fails", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("read failure must not mutate the segment", Files.exists(path)); + }); + } + + @Test + public void testShortReadFailsClosedBeforeMmap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-short-read.sfa"; + writeSegment(path, 19L, new int[]{256}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.stopReadsAt = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected premature EOF failure"); + } catch (MmapSegmentException expected) { + assertFalse("premature EOF must remain an operational failure", + expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("short read")); + } + assertEquals("mapping must not start after a failed scan", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("short read must not mutate the segment", Files.exists(path)); + }); + } + + @Test + public void testShortReadsAreRetried() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-partial-reads.sfa"; + long expectedEnd = writeSegment(path, 23L, new int[]{31, 127, 4097}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.maxReadSize = 1024; try (MmapSegment seg = MmapSegment.openExisting(ff, path)) { - // Interpreter / C1: graceful partial recovery. - assertEquals("the frame below the beyond-EOF page must be recovered", 1L, seg.frameCount()); - assertEquals("scan must stop at the beyond-EOF boundary", page, seg.publishedOffset()); - assertEquals("a beyond-EOF page is not a torn write", 0L, seg.tornTailBytes()); - } catch (MmapSegmentException skippedUnderC2) { - // C2: the inlined fault escaped to openExisting's outer catch and - // was converted to a per-file skip. Assert it is the recognized - // mmap fault (not some other data error) so a revert -- which - // lets a raw InternalError through instead -- still fails here. - assertTrue(skippedUnderC2.getMessage(), - skippedUnderC2.getMessage().contains("unsafe memory access operation")); + assertEquals(3L, seg.frameCount()); + assertEquals(expectedEnd, seg.publishedOffset()); } + assertTrue("recovery must loop over partial positioned reads", ff.readCalls > 1); + assertEquals(1, ff.mmapCalls); + assertEquals(1, ff.closeCalls); }); } - /** - * Portable fail-on-revert guard for the {@code openExisting} header-block - * guard. The file is truncated to empty and the facade reports a full page, - * so the very first header read (magic) lands on a beyond-EOF page and - * faults on any filesystem. {@code openExisting} must convert that to a - * {@link MmapSegmentException} -- the per-file signal {@code SegmentRing} - * skips on -- not let the raw {@code InternalError} escape and abort recovery - * of every sibling. Revert the header-block conversion and this throws - * {@code InternalError} instead of {@code MmapSegmentException}. - */ @Test - public void testHeaderFaultOnMapPastEofIsSkippableAnyFilesystem() throws Exception { + public void testSizeChangeFailsClosedBeforeMmap() throws Exception { TestUtils.assertMemoryLeak(() -> { - final String path = tmpDir + "/seg-mappasteof-header.sfa"; - final long page = Files.PAGE_SIZE; - writeSegment(path, 9L, new int[]{64}); - // Free every block: the file is now empty, so even page 0 (the - // header) is beyond EOF under the reported one-page mapping. - truncateTo(path, 0L); - FilesFacade ff = new MapPastEofFacade(path, page); + final String path = tmpDir + "/seg-size-change.sfa"; + writeSegment(path, 29L, new int[]{64}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.changeLengthAfterScan = true; try { MmapSegment.openExisting(ff, path).close(); - fail("expected MmapSegmentException for a beyond-EOF header page"); + fail("expected unstable-size failure"); } catch (MmapSegmentException expected) { - // ok -- SegmentRing's per-file catch skips just this file - // instead of aborting recovery of the whole slot. + assertFalse(expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("size changed")); } + assertEquals("mapping must not start for an unstable file", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("size-race detection must not mutate the segment", Files.exists(path)); + }); + } + + @Test + public void testSizeChangeWhileMappingFailsClosedAndUnmaps() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-size-change-while-mapping.sfa"; + writeSegment(path, 31L, new int[]{64}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.changeLengthOnMmap = true; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected size change during mmap to fail recovery"); + } catch (MmapSegmentException expected) { + assertFalse(expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("size changed while mapping")); + } + assertEquals("scan should map only after validation", 1, ff.mmapCalls); + assertEquals("rejected mapping must be released", 1, ff.munmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("injected size observation must not mutate the segment", Files.exists(path)); }); } @@ -311,9 +329,7 @@ private static long writeSegment(String path, long baseSeq, int[] payloadLens) { } long buf = Unsafe.malloc(maxLen, MemoryTag.NATIVE_DEFAULT); try { - for (int i = 0; i < maxLen; i++) { - Unsafe.getUnsafe().putByte(buf + i, (byte) (i | 1)); // all non-zero - } + Unsafe.getUnsafe().setMemory(buf, maxLen, (byte) 1); try (MmapSegment seg = MmapSegment.create(path, baseSeq, SEGMENT_BYTES)) { for (int len : payloadLens) { assertTrue("append must fit", seg.tryAppend(buf, len) >= 0); @@ -329,8 +345,8 @@ private static long writeSegment(String path, long baseSeq, int[] payloadLens) { * Turns {@code [keepBytes, SEGMENT_BYTES)} of the file into an unbacked * sparse hole: truncate down to {@code keepBytes} (frees the tail blocks), * then back up to {@code SEGMENT_BYTES} (re-extends the logical size without - * allocating blocks). Recovery maps the full stat length, so the hole is - * inside the mapping -- reads of it fault on ZFS and zero-fill on ext4. + * allocating blocks). Positioned recovery reads observe the hole as zeroes + * without dereferencing it through mmap. */ private static void punchSparseTail(String path, long keepBytes) { int fd = Files.openRW(path); @@ -344,36 +360,20 @@ private static void punchSparseTail(String path, long keepBytes) { } /** - * Shrinks the file to {@code keepBytes}, freeing every block past it, and - * leaves it there (no re-extend). Combined with a facade that reports a - * larger length, the freed region becomes a beyond-EOF part of the mapping - * that faults on read on any filesystem. + * Positioned-read fault seam. Calls not involved in recovery scanning + * delegate to the production {@link FilesFacade#INSTANCE}. */ - private static void truncateTo(String path, long keepBytes) { - int fd = Files.openRW(path); - assertTrue("openRW failed", fd >= 0); - try { - assertTrue("truncate failed", Files.truncate(fd, keepBytes)); - } finally { - Files.close(fd); - } - } - - /** - * A {@link FilesFacade} that reports an inflated stat length for one target - * path so {@code openExisting} maps that file past end-of-file (see - * {@link FilesFacade#length(String)}); every other call, including - * {@code length} for any other path, delegates to the production - * {@link FilesFacade#INSTANCE}. - */ - private static final class MapPastEofFacade implements FilesFacade { - private final long reportedLength; - private final String targetPath; - - MapPastEofFacade(String targetPath, long reportedLength) { - this.targetPath = targetPath; - this.reportedLength = reportedLength; - } + private static final class RecoveryReadFacade implements FilesFacade { + private boolean changeLengthAfterScan; + private boolean changeLengthOnMmap; + private int closeCalls; + private boolean failReadWithError; + private int lengthCalls; + private int maxReadSize = Integer.MAX_VALUE; + private int mmapCalls; + private int munmapCalls; + private int readCalls; + private long stopReadsAt = Long.MAX_VALUE; @Override public boolean allocate(int fd, long size) { @@ -387,6 +387,7 @@ public long allocNativePath(String path) { @Override public int close(int fd) { + closeCalls++; return INSTANCE.close(fd); } @@ -432,7 +433,13 @@ public int fsync(int fd) { @Override public long length(int fd) { - return INSTANCE.length(fd); + long length = INSTANCE.length(fd); + lengthCalls++; + if ((changeLengthAfterScan && lengthCalls > 1) + || (changeLengthOnMmap && mmapCalls > 0)) { + return length - 1L; + } + return length; } @Override @@ -442,7 +449,7 @@ public long length(long pathPtr) { @Override public long length(String path) { - return targetPath.equals(path) ? reportedLength : INSTANCE.length(path); + return INSTANCE.length(path); } @Override @@ -455,6 +462,18 @@ public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + mmapCalls++; + return INSTANCE.mmap(fd, len, offset, flags, memoryTag); + } + + @Override + public void munmap(long address, long len, int memoryTag) { + munmapCalls++; + INSTANCE.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); @@ -477,7 +496,15 @@ public int openRW(long pathPtr) { @Override public long read(int fd, long addr, long len, long offset) { - return INSTANCE.read(fd, addr, len, offset); + readCalls++; + if (offset >= stopReadsAt) { + return failReadWithError ? -1L : 0L; + } + long delegatedLen = Math.min(len, (long) maxReadSize); + if (delegatedLen > stopReadsAt - offset) { + delegatedLen = stopReadsAt - offset; + } + return INSTANCE.read(fd, addr, delegatedLen, offset); } @Override diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index cff8249b..3edf4f95 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -215,9 +215,9 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex // frames are followed by garbage. None cover frame[0] itself // being corrupt — yet a single bit-flip on the CRC of frame[0] // at rest (bit-rot, partial-page-write at crash) is the - // worst-case data-loss trigger: scanFrames bails at HEADER_SIZE - // and frameCount drops to 0, even though valid frames still - // sit on disk past the corrupt header. + // worst-case data-loss trigger: the recovery scan bails at + // HEADER_SIZE and frameCount drops to 0, even though valid frames + // still sit on disk past the corrupt header. // // Contract: tornTailBytes() must be non-zero (because non-zero // bytes exist past the last good frame), and openExisting @@ -258,7 +258,7 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex Files.exists(path)); try (MmapSegment seg = MmapSegment.openExisting(path)) { - assertEquals("scanFrames must bail at the corrupt frame[0]", + assertEquals("recovery scan must bail at the corrupt frame[0]", 0L, seg.frameCount()); assertEquals("publishedOffset must rewind to the header end", MmapSegment.HEADER_SIZE, seg.publishedOffset()); @@ -276,6 +276,47 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex }); } + @Test + public void testAllZeroCorruptFrameHeaderDoesNotHideNonZeroSuffix() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String path = tmpDir + "/seg-zero-frame-header.sfa"; + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(payload, 32, 0); + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + assertTrue(seg.tryAppend(payload, 32) >= 0L); + assertTrue(seg.tryAppend(payload, 32) >= 0L); + seg.msync(); + } + + // Zero both the CRC and length of frame[0]. Looking only at + // this 8-byte header would misclassify the segment as a clean + // empty spare even though its payload and frame[1] remain. + int fd = Files.openRW(path); + assertTrue("openRW must succeed", fd >= 0); + long zeroHeader = Unsafe.malloc(MmapSegment.FRAME_HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(zeroHeader, MmapSegment.FRAME_HEADER_SIZE, (byte) 0); + assertEquals(MmapSegment.FRAME_HEADER_SIZE, + Files.write(fd, zeroHeader, MmapSegment.FRAME_HEADER_SIZE, MmapSegment.HEADER_SIZE)); + Files.fsync(fd); + } finally { + Unsafe.free(zeroHeader, MmapSegment.FRAME_HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(0L, seg.frameCount()); + assertEquals(MmapSegment.HEADER_SIZE, seg.publishedOffset()); + assertEquals("non-zero bytes later in the suffix must preserve corruption evidence", + 4096L - MmapSegment.HEADER_SIZE, seg.tornTailBytes()); + } + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testFullSegmentRejectsFurtherAppends() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java index e551c6f8..ae987fee 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java @@ -79,8 +79,8 @@ public void tearDown() { /** * Finding C1 / C10 — first-frame CRC corruption silently deletes the segment. *

        - * If frame[0] of a recovered .sfa fails CRC validation, scanFrames returns - * lastGood=HEADER_SIZE, countFrames returns 0, and SegmentRing.openExisting + * If frame[0] of a recovered .sfa fails CRC validation, the recovery scan + * returns lastGood=HEADER_SIZE and frameCount=0, and SegmentRing.openExisting * unlinks the file as an "empty hot-spare leftover" — destroying every frame * that physically followed the corrupt header. The torn-tail WARN inside * MmapSegment.openExisting is dropped on the floor. @@ -128,8 +128,8 @@ public void testC1_recoveryMustNotUnlinkSegmentWithCorruptFirstFrame() throws Ex // Run recovery. SegmentRing recovered = SegmentRing.openExisting(tmpDir, 64 * 1024); try { - // The bug: openExisting sees frameCount=0 (because scanFrames - // bailed at the corrupt frame[0]) and treats the segment as + // The bug: openExisting sees frameCount=0 (because the recovery + // scan bailed at the corrupt frame[0]) and treats the segment as // an "empty hot-spare leftover" — closing AND UNLINKING the // file. The user's frames 1, 2, 3 are gone forever; the only // record was a WARN log line that's already been emitted. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java index 21116e68..a1186176 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -48,7 +48,7 @@ /** * Regression tests for the SF recovery fail-open findings: directory - * enumeration errors, per-file open/mmap errors and boundary (leading / + * enumeration errors, per-file open/read/mmap errors and boundary (leading / * trailing) segment loss must fail startup without mutating the slot, while * positively-identified corruption is quarantined only after the surviving * chain validates. Also pins the crash-window states of the manifest @@ -187,6 +187,41 @@ public long mmap(int fd, long len, long offset, int flags, int memoryTag) { }); } + @Test + public void testReadFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int victimFd = Integer.MIN_VALUE; + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (victim.equals(path)) { + victimFd = fd; + } + return fd; + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return fd == victimFd ? -1L : super.read(fd, addr, len, offset); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be read"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + @Test public void testUnsupportedVersionFailsRecoveryWithoutQuarantine() throws Exception { TestUtils.assertMemoryLeak(() -> { From 9c9e1ddbda9a3a0e21d2c5a677afe80b6b587fdc Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 20 Jul 2026 13:34:10 +0100 Subject: [PATCH 33/64] Add periodic store-and-forward syncing Add configurable background checkpoints for store-and-forward payloads.\nUse checked mmap and file-descriptor barriers and gate segment rotation\nuntil the predecessor is durable.\n\nPropagate the interval through foreground senders and orphan drainers,\npreserve failures for producer visibility, and sync replayable data on\nrecovery and close. Persist parent directory entries in periodic mode.\n\nDocument the bounded-RPO contract and cover configuration, scheduling,\nbarrier failures, rotation, recovery, and compatibility in tests. --- README.md | 14 +- .../main/java/io/questdb/client/Sender.java | 119 +++++--- .../qwp/client/QwpWebSocketSender.java | 34 ++- .../client/sf/cursor/BackgroundDrainer.java | 30 +- .../client/sf/cursor/CursorSendEngine.java | 93 ++++++- .../qwp/client/sf/cursor/MmapSegment.java | 66 ++++- .../qwp/client/sf/cursor/SegmentManager.java | 98 ++++++- .../qwp/client/sf/cursor/SegmentRing.java | 93 ++++++- .../qwp/client/sf/cursor/SlotLock.java | 12 + .../io/questdb/client/impl/ConfigSchema.java | 1 + .../java/io/questdb/client/std/Files.java | 10 + .../qwp/client/sf/SfFromConfigTest.java | 55 ++++ .../sf/cursor/CursorSendEngineTest.java | 68 +++++ .../qwp/client/sf/cursor/MmapSegmentTest.java | 88 +++++- .../SegmentManagerPeriodicSyncTest.java | 259 ++++++++++++++++++ .../test/impl/WsSenderConfigHonoredTest.java | 3 +- .../io/questdb/client/test/std/FilesTest.java | 10 + .../sender/WsStoreAndForwardExample.java | 9 +- 18 files changed, 978 insertions(+), 84 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java diff --git a/README.md b/README.md index cc1b25ae..01aa3370 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,9 @@ You can also let the client flush batches for you with the `auto_flush_rows` / ` `ws::addr=localhost:9000;auto_flush_rows=10000;auto_flush_interval=1000;`. **Confirm a batch is durably received.** Over QWP each flush returns a frame sequence number (FSN); `awaitAckedFsn` -blocks until the server has acknowledged it. Rows are safe in the store-and-forward log even if the ack has not landed -yet — they replay on reconnect. +blocks until the server has acknowledged it. With `sf_dir`, rows in the store-and-forward log replay after reconnect +or a producer-process restart. For periodic host-power-loss checkpoints, also configure +`sf_durability=periodic;sf_sync_interval_millis=5000;`. ```java try (Sender sender = db.borrowSender()) { @@ -259,7 +260,9 @@ try (QuestDB db = QuestDB.builder() List every cluster node in one `addr` server list; the single string configures both the ingest and query pools across all of them. On the query side, `target` selects the node role to route to (`any`, `primary`, or `replica`) and `failover=on` enables failover across the list. The ingest side reconnects across the same node list on its own — a -store-and-forward sender keeps buffering rows through a failover window and never drops them. +store-and-forward sender keeps buffering rows through a failover window and replays unacknowledged data. The default +`memory` durability mode protects process restarts, not host power loss; use periodic durability for background disk +checkpoints. ```java try (QuestDB db = QuestDB.connect( @@ -439,7 +442,10 @@ Applied by the query pool to select and fail over between the nodes in the `addr | `client_id` | | Opaque client identifier surfaced server-side for observability | The ingest side also accepts store-and-forward and reconnection tuning keys (`auto_flush_*`, `initial_connect_retry`, -`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). See the +`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). +`sf_durability=periodic` checkpoints mmap-published data in the background; `sf_sync_interval_millis` defaults to `5000` +in that mode. The interval is a target cadence: JVM scheduling and storage-sync latency add to the actual loss window. +Use `request_durable_ack=on` when end-to-end server durability is also required. See the [QuestDB documentation](https://questdb.com/docs/) for the full reference. ## Requirements diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..e2891201 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -819,17 +819,22 @@ enum InitialConnectMode { *

      7. {@link #MEMORY} — never fsync explicitly. Bytes live in the OS * page cache; survive a JVM crash but not an OS crash. Default * and the lowest-latency setting.
        8. - *
      8. {@link #FLUSH} — fsync the active segment at every - * {@code Sender.flush()} (and at the implicit close-flush). One - * fsync per user flush, regardless of frame count.
        9. - *
      9. {@link #APPEND} — fsync after every individual frame append. - * Strongest guarantee, slowest path; pay a disk fsync per row.
        10. + *
      10. {@link #PERIODIC} — checkpoint published frames in the background + * at {@code sf_sync_interval_millis}. The configured interval is a + * target cadence; scheduler and storage latency add to the actual + * power-loss recovery window.
        11. + *
      11. {@link #FLUSH} — reserved for a future synchronous + * {@code Sender.flush()} barrier; currently rejected by + * {@code build()}.
        12. + *
      12. {@link #APPEND} — reserved for a future barrier after every frame + * append; currently rejected by {@code build()}.
        13. * */ enum SfDurability { MEMORY, FLUSH, - APPEND + APPEND, + PERIODIC } /** @@ -972,6 +977,7 @@ final class LineSenderBuilder { // syscall cost so smaller segments give finer trim granularity and // make the cap arithmetic friendlier (cap / segment >> 2). private static final long DEFAULT_SEGMENT_BYTES = 4L * 1024 * 1024; + private static final long DEFAULT_SF_SYNC_INTERVAL_MILLIS = 5_000L; // Slot identity within sfDir. Each sender owns // and // takes an advisory exclusive lock there. Default "default" is fine for // single-sender deployments; multi-sender setups must set this explicitly @@ -1119,12 +1125,12 @@ public int getConnectTimeout() { // there is no separate on/off flag (presence of the directory is the switch). // null sfDir → memory-only async ingest (same lock-free architecture, no disk). private String sfDir; - // Durability contract for SF append/flush. Today only MEMORY is - // implemented; FLUSH and APPEND are deferred follow-ups (cursor needs - // to learn fsync first). + // Durability contract for SF append/flush. FLUSH and APPEND remain + // deferred follow-ups; PERIODIC uses the segment manager. private SfDurability sfDurability = SfDurability.MEMORY; private long sfMaxBytes = PARAMETER_NOT_SET_EXPLICITLY; private long sfMaxTotalBytes = PARAMETER_NOT_SET_EXPLICITLY; + private long sfSyncIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean shouldDestroyPrivKey; private boolean tlsEnabled; private TlsValidationMode tlsValidationMode; @@ -1443,9 +1449,9 @@ public Sender build() { // Setting sfDir enables store-and-forward (mmap'd, recoverable // across sender restarts); omitting it gives memory-only mode - // (same lock-free architecture, no disk involvement). The - // sf_durability != memory rejection lives in validateParameters - // so it is reached by build() and by no-connect validation alike. + // (same lock-free architecture, no disk involvement). + // Durability-combination validation lives in validateParameters + // so build() and no-connect validation apply the same rules. long actualSfMaxBytes = sfMaxBytes == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEGMENT_BYTES : sfMaxBytes; @@ -1534,15 +1540,25 @@ public Sender build() { "could not create sf_dir: " + sfDir + " rc=" + rc); } } + if (sfDurability == SfDurability.PERIODIC + && Files.fsyncParentDir(sfDir) != 0) { + throw new LineSenderException( + "could not sync parent directory for sf_dir: " + sfDir); + } slotPath = sfDir + "/" + senderId; } long actualSfAppendDeadlineNanos = sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; + long actualSfSyncIntervalNanos = sfDurability == SfDurability.PERIODIC + ? (sfSyncIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_SF_SYNC_INTERVAL_MILLIS : sfSyncIntervalMillis) * 1_000_000L + : 0L; CursorSendEngine cursorEngine = new CursorSendEngine( slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, + actualSfSyncIntervalNanos); int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; @@ -1623,7 +1639,8 @@ public Sender build() { orphans, maxBackgroundDrainers, actualSfMaxBytes, - actualSfMaxTotalBytes); + actualSfMaxTotalBytes, + actualSfSyncIntervalNanos); } } return connected; @@ -2701,20 +2718,18 @@ public LineSenderBuilder sfAppendDeadlineMillis(long millis) { * directory is the on-switch — there is no separate * enable/disable flag. SF is off iff {@code dir} was never set. *

        - * Every batch is persisted to disk before it leaves the wire and is - * reclaimed as soon as the server acknowledges it. On restart the - * sender replays only batches whose acknowledgement had not been - * received before the previous sender shut down — typically the last - * in-flight batches at close time. Acknowledged batches are not - * replayed: their disk space is freed during normal operation by an - * automatic per-frame trim that force-rotates the active segment - * once every frame in it has been acknowledged. + * The sender publishes each batch into a memory-mapped segment before + * transmission and reclaims acknowledged segments in the background. + * The default {@link SfDurability#MEMORY} mode relies on OS page-cache + * writeback: it survives a producer-process restart but not guaranteed + * host power loss. {@link SfDurability#PERIODIC} adds checked background + * storage barriers at the configured target cadence. *

        - * Note that {@link io.questdb.client.cutlass.qwp.client.QwpWebSocketSender#close()} - * under SF returns once data is on disk, not on server-ack, so a - * sender closed immediately after a flush may still have unacked - * batches in flight; those will be replayed by the next sender - * against the same directory. WebSocket transport only. + * On restart, the sender replays frames after its durable acknowledgement + * watermark. An acknowledgement that reached the server but not that + * watermark can replay, so applications that require row-level + * idempotence should configure server-side deduplication. + * WebSocket transport only. *

        * The sender takes ownership of the underlying SF storage and closes * it when the sender itself is closed. @@ -2734,7 +2749,10 @@ public LineSenderBuilder storeAndForwardDir(String dir) { /** * Selects the durability contract for SF appends and flushes. See - * {@link SfDurability} for the value semantics. + * {@link SfDurability} for the value semantics. The client currently + * supports {@link SfDurability#MEMORY} and + * {@link SfDurability#PERIODIC}; {@code build()} rejects the reserved + * {@code FLUSH} and {@code APPEND} values. *

        * Replaces the prior pair of independent {@code sf_fsync} and * {@code sf_fsync_on_flush} booleans — they were three states @@ -2788,6 +2806,22 @@ public LineSenderBuilder storeAndForwardMaxTotalBytes(long maxTotalBytes) { return this; } + /** + * Sets the target background checkpoint cadence for + * {@link SfDurability#PERIODIC}. Scheduler and storage latency add to + * the configured interval. Defaults to 5000 ms in periodic mode. + */ + public LineSenderBuilder storeAndForwardSyncIntervalMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("store_and_forward is only supported for WebSocket transport"); + } + if (millis <= 0L || millis > Long.MAX_VALUE / 1_000_000L) { + throw new LineSenderException("sf_sync_interval_millis is out of range: ").put(millis); + } + this.sfSyncIntervalMillis = millis; + return this; + } + private static boolean charsEqualsRange(CharSequence a, CharSequence b, int bStart, int bEnd) { int len = bEnd - bStart; if (a.length() != len) { @@ -2810,10 +2844,11 @@ private static int getValue(CharSequence configurationString, int pos, StringSin private static SfDurability parseDurabilityValue(@NotNull StringSink value) { if (Chars.equalsIgnoreCase("memory", value)) return SfDurability.MEMORY; + if (Chars.equalsIgnoreCase("periodic", value)) return SfDurability.PERIODIC; if (Chars.equalsIgnoreCase("flush", value)) return SfDurability.FLUSH; if (Chars.equalsIgnoreCase("append", value)) return SfDurability.APPEND; throw new LineSenderException("invalid sf_durability [value=").put(value) - .put(", allowed-values=[memory, flush, append]]"); + .put(", allowed-values=[memory, periodic, flush, append]]"); } private static int parseIntValue(@NotNull StringSink value, @NotNull String name) { @@ -3370,6 +3405,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "sf_durability"); storeAndForwardDurability(parseDurabilityValue(sink)); + } else if (Chars.equals("sf_sync_interval_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("sf_sync_interval_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "sf_sync_interval_millis"); + storeAndForwardSyncIntervalMillis(parseLongValue(sink, "sf_sync_interval_millis")); } else if (Chars.equals("close_flush_timeout_millis", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("close_flush_timeout_millis is only supported for WebSocket transport"); @@ -3695,6 +3736,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("sf_max_total_bytes")) { storeAndForwardMaxTotalBytes(wsSize(view, v, "sf_max_total_bytes")); } + if (view.has("sf_sync_interval_millis")) { + storeAndForwardSyncIntervalMillis(wsLong(view, v, "sf_sync_interval_millis")); + } s = view.getStr("sf_dir"); if (s != null) { @@ -3852,6 +3896,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("sf_max_total_bytes", sfMaxTotalBytes); m.put("sf_durability", sfDurability == null ? null : sfDurability.name()); m.put("sf_append_deadline_millis", sfAppendDeadlineMillis); + m.put("sf_sync_interval_millis", sfSyncIntervalMillis); m.put("close_flush_timeout_millis", closeFlushTimeoutMillis); m.put("durable_ack_keepalive_interval_millis", durableAckKeepaliveIntervalMillis); m.put("initial_connect_retry", initialConnectMode == null ? null : initialConnectMode.name()); @@ -4057,14 +4102,18 @@ private void validateParameters() { if (autoFlushIntervalMillis == Integer.MAX_VALUE) { throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol"); } - // The cursor send path does not fsync yet, so any sf_durability - // other than memory is rejected rather than silently downgraded. - // Validating it here (rather than at connect time) lets a - // no-connect config check reject it as a full build() does. - if (sfDurability != SfDurability.MEMORY) { + if ((sfDurability == SfDurability.FLUSH || sfDurability == SfDurability.APPEND)) { throw new LineSenderException( "sf_durability=" + sfDurability.name().toLowerCase() - + " is not yet supported (deferred follow-up; use sf_durability=memory)"); + + " is not yet supported (use sf_durability=memory or periodic)"); + } + if (sfDurability == SfDurability.PERIODIC && sfDir == null) { + throw new LineSenderException("sf_durability=periodic requires sf_dir"); + } + if (sfSyncIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY + && sfDurability != SfDurability.PERIODIC) { + throw new LineSenderException( + "sf_sync_interval_millis requires sf_durability=periodic"); } } else { throw new LineSenderException("unsupported protocol ") diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c3b4dfde..89bdf48f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -867,6 +867,7 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { if (cursorEngine == null) { return targetFsn < 0L; } + cursorEngine.checkDurability(); // Surface latched I/O errors before any early-return path, so a // caller polling with timeoutMillis <= 0 to drive their own loop // sees the terminal throw instead of an indefinite "not yet". @@ -882,6 +883,7 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { } long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L; while (cursorEngine.ackedFsn() < targetFsn) { + cursorEngine.checkDurability(); if (cursorSendLoop != null) { cursorSendLoop.checkError(); } @@ -1519,15 +1521,19 @@ public void flush() { @Override public long flushAndGetSequence() { checkNotClosed(); + if (cursorEngine != null) { + cursorEngine.checkDurability(); + } ensureNoInProgressRow(); ensureConnected(); long beforeFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L; - // Cursor SF: SF.append happens on the user thread inside - // sealAndSwapBuffer, so by the time we reach here every encoded - // batch is durable on its mmap'd segment. No processingCount to - // drain, no awaitPendingAcks. Just surface any I/O thread error. + // Cursor SF: append happens on the user thread inside + // sealAndSwapBuffer, so by the time we reach here every encoded batch + // is published in its mmap'd segment. PERIODIC stable-storage barriers + // run independently in the manager. No processingCount to drain and no + // awaitPendingAcks here; just surface any I/O thread error. flushPendingRows(deferCommit); if (!deferCommit && hasDeferredMessages) { sendCommitMessage(); @@ -2422,6 +2428,25 @@ public synchronized void startOrphanDrainers( int maxBackgroundDrainers, long segmentSizeBytes, long sfMaxTotalBytes + ) { + startOrphanDrainers( + orphanSlotPaths, + maxBackgroundDrainers, + segmentSizeBytes, + sfMaxTotalBytes, + 0L); + } + + /** + * Starts orphan drainers while preserving the foreground sender's periodic + * store-and-forward checkpoint interval. + */ + public synchronized void startOrphanDrainers( + io.questdb.client.std.ObjList orphanSlotPaths, + int maxBackgroundDrainers, + long segmentSizeBytes, + long sfMaxTotalBytes, + long syncIntervalNanos ) { if (orphanSlotPaths == null || orphanSlotPaths.size() == 0 || maxBackgroundDrainers <= 0) { @@ -2459,6 +2484,7 @@ public synchronized void startOrphanDrainers( io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer drainer = new io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer( slot, segmentSizeBytes, sfMaxTotalBytes, + syncIntervalNanos, factory, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..11128a9b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -106,6 +106,7 @@ public final class BackgroundDrainer implements Runnable { private final long segmentSizeBytes; private final long sfMaxTotalBytes; private final String slotPath; + private final long syncIntervalNanos; /** Latest known {@code engine.ackedFsn()}; published for visibility. */ private volatile long ackedFsn = -1L; private volatile String lastErrorMessage; @@ -170,10 +171,36 @@ public BackgroundDrainer( long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, long poisonMinEscalationWindowMillis + ) { + this(slotPath, segmentSizeBytes, sfMaxTotalBytes, 0L, clientFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, requestDurableAck, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis); + } + + /** + * Master constructor with the periodic SF checkpoint interval inherited + * from the sender that adopted the orphan slot. + */ + public BackgroundDrainer( + String slotPath, + long segmentSizeBytes, + long sfMaxTotalBytes, + long syncIntervalNanos, + CursorWebSocketSendLoop.ReconnectFactory clientFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean requestDurableAck, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; this.sfMaxTotalBytes = sfMaxTotalBytes; + this.syncIntervalNanos = syncIntervalNanos; this.clientFactory = clientFactory; this.reconnectMaxDurationMillis = reconnectMaxDurationMillis; this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; @@ -531,7 +558,8 @@ public void run() { // (no .failed sentinel — contention is expected, not an error). try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, - sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); + sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + syncIntervalNanos); } catch (IllegalStateException t) { String msg = t.getMessage(); if (msg != null && msg.contains("already in use")) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 6e48f036..2c4dd422 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -102,6 +102,7 @@ public final class CursorSendEngine implements QuietCloseable { private final SegmentRing ring; private final long segmentSizeBytes; private final String sfDir; + private final long syncIntervalNanos; // Held for the engine's lifetime in disk mode. {@code null} in memory // mode (no slot, no lock). Released by {@link #close()}; the kernel // also drops it on hard process exit. @@ -200,7 +201,7 @@ public final class CursorSendEngine implements QuietCloseable { */ public CursorSendEngine(String sfDir, long segmentSizeBytes) { this(sfDir, segmentSizeBytes, SegmentManager.UNLIMITED_TOTAL_BYTES, - DEFAULT_APPEND_DEADLINE_NANOS); + DEFAULT_APPEND_DEADLINE_NANOS, 0L); } /** @@ -212,7 +213,18 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { - this(sfDir, segmentSizeBytes, null, true, maxTotalBytes, appendDeadlineNanos); + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, 0L); + } + + /** + * Creates an engine with an optional periodic data-checkpoint interval. + * A positive interval requires disk-backed store-and-forward mode. + */ + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, + long syncIntervalNanos) { + this(sfDir, segmentSizeBytes, null, true, maxTotalBytes, + appendDeadlineNanos, syncIntervalNanos); } /** @@ -221,26 +233,35 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, 0L); + } + + public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + long syncIntervalNanos) { + this(sfDir, segmentSizeBytes, manager, false, + DEFAULT_APPEND_DEADLINE_NANOS, syncIntervalNanos); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, boolean ownsManager, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, manager, ownsManager, appendDeadlineNanos, 0L); + } + + private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + boolean ownsManager, long appendDeadlineNanos, + long syncIntervalNanos) { this(sfDir, segmentSizeBytes, manager, ownsManager, - SegmentManager.UNLIMITED_TOTAL_BYTES, appendDeadlineNanos); + SegmentManager.UNLIMITED_TOTAL_BYTES, appendDeadlineNanos, syncIntervalNanos); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, - boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos) { + boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos, + long syncIntervalNanos) { // Allocate the bound callback before constructing an owned manager. // Field initializers have completed, but no engine-owned native/disk // resource exists yet. If callback allocation throws, construction // stops without a manager whose native path scratch could be orphaned. this.deferredClose = createDeferredClose(); - if (ownsManager && manager == null) { - manager = new SegmentManager( - segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes); - } // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments @@ -248,6 +269,17 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // sfDir != null → store-and-forward mode. Segments are mmap'd files // under sfDir, recoverable across sender restarts. boolean memoryMode = sfDir == null; + // Validate before an owned manager allocates its native path scratch. + if (syncIntervalNanos < 0L) { + throw new IllegalArgumentException("syncIntervalNanos must not be negative"); + } + if (memoryMode && syncIntervalNanos > 0L) { + throw new IllegalArgumentException("periodic sync requires disk store-and-forward mode"); + } + if (ownsManager && manager == null) { + manager = new SegmentManager( + segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes); + } SlotLock acquiredLock = null; if (!memoryMode) { try { @@ -259,7 +291,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // recovery and create overlapping FSN ranges. SlotLock.acquire // also creates the slot dir if it doesn't exist yet — no // separate mkdir step needed here. - acquiredLock = SlotLock.acquire(sfDir); + acquiredLock = SlotLock.acquire(sfDir, syncIntervalNanos > 0L); } catch (Throwable t) { // Callback creation and owned-manager construction have already // completed. A slot-lock failure must close the owned manager's @@ -280,6 +312,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.filesFacade = manager.filesFacade(); this.ownsManager = ownsManager; this.appendDeadlineNanos = appendDeadlineNanos; + this.syncIntervalNanos = syncIntervalNanos; // Track the ring locally until every step succeeds — only commit it // to this.ring at the very end. If anything between ring allocation @@ -458,10 +491,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } } + if (syncIntervalNanos > 0L) { + ringInProgress.enablePeriodicSync(); + if (recovered != null) { + // Establish a durable baseline before exposing data that a + // previous MEMORY-mode process may have left in page cache. + ringInProgress.syncAllLiveSegments(); + } + } if (ownsManager) { manager.start(); } - manager.register(ringInProgress, sfDir, watermarkInProgress); + manager.register(ringInProgress, sfDir, watermarkInProgress, syncIntervalNanos); // All construction succeeded — commit the ring and // watermark references. this.ring = ringInProgress; @@ -564,11 +605,13 @@ public long appendBlocking(long payloadAddr, int payloadLen) { if (now >= deadlineNs) { throw new io.questdb.client.cutlass.line.LineSenderException( "cursor ring backpressured for ").put(appendDeadlineNanos / 1_000_000L) - .put(" ms — wire path is not draining (server slow / disconnected, or sf_max_total_bytes too small)"); + .put(" ms - wire path is not draining (server slow / disconnected, " + + "periodic disk sync is slow, or sf_max_total_bytes is too small)"); } if (now - lastBackpressureLogNs >= BACKPRESSURE_LOG_THROTTLE_NANOS) { lastBackpressureLogNs = now; - LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O drain — will throw after {} ms", + LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O or periodic disk sync; " + + "will throw after {} ms", backpressureStallCount.get(), appendDeadlineNanos / 1_000_000L); } LockSupport.parkNanos(50_000L); // 50 µs @@ -611,6 +654,10 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos return SegmentRing.BACKPRESSURE_NO_SPARE; } + public void checkDurability() { + ring.checkDurability(); + } + @Override public synchronized void close() { if (closed && closeCompleted) return; @@ -800,6 +847,16 @@ public synchronized void close() { * against a close() that holds the monitor while joining the worker. */ private void finishClose(boolean fullyDrained) { + if (!fullyDrained && syncIntervalNanos > 0L) { + try { + ring.syncAllLiveSegments(); + } catch (RuntimeException | Error e) { + terminalCleanupRetryReady = true; + terminalCleanupClaimed.set(false); + startFlockReleaseRetry(); + throw e; + } + } // On a fully-drained close, persist the final acked FSN through the // still-mapped watermark BEFORE closing the ring/watermark and BEFORE // unlinking any segment file. The manager persists the watermark only @@ -905,11 +962,21 @@ public SegmentManager getManagerForTesting() { return manager; } + @TestOnly + public SegmentRing getRingForTesting() { + return ring; + } + @TestOnly public SlotLock getSlotLockForTesting() { return slotLock; } + @TestOnly + public long getSyncIntervalNanosForTesting() { + return syncIntervalNanos; + } + /** * Installs a hook invoked after each failed shared-driver flock release. * Test-only: makes persistent retry progress observable without sleeps. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 58afcd5b..8758aabf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -54,9 +54,11 @@ * * The mapping is sized at construction and never grows. When * {@link #tryAppend} returns -1 the caller must rotate to a fresh segment. - * Closing the segment unmaps and closes the fd; data already written is - * durable under the page cache (and recoverable across JVM restarts) — call - * {@link #msync} for OS-crash durability. + * Closing the segment unmaps and closes the fd. Dirty mapped pages normally + * survive a producer-process restart in the OS page cache, but that is not a + * host-power-loss guarantee. {@link #msync} preserves the legacy mmap-only + * flush API; use the checked {@link #syncPublished()} mapping-plus-fd barrier + * for portable power-loss durability. */ public final class MmapSegment implements QuietCloseable { @@ -84,6 +86,10 @@ public final class MmapSegment implements QuietCloseable { // segment manager pre-creates spares before the producer knows the exact // baseSeq the new active will need. private long baseSeq; + // Highest published byte offset covered by a successful data barrier. + // The manager writes this after msync+fsync; the producer reads it before + // allowing rotation to seal the segment. + private volatile long durableCursor; private int fd; // frameCount: number of frames successfully appended. Single writer (the // producer thread in tryAppend); read cross-thread by the I/O thread via @@ -119,6 +125,7 @@ private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddre this.baseSeq = baseSeq; this.appendCursor = initialCursor; this.publishedCursor = initialCursor; + this.durableCursor = memoryBacked ? initialCursor : HEADER_SIZE; this.frameCount = frameCount; this.memoryBacked = memoryBacked; this.tornTailBytes = tornTailBytes; @@ -418,23 +425,62 @@ void syncHeader() { if (filesFacade.msync(mmapAddress, HEADER_SIZE, false) != 0 || filesFacade.fsync(fd) != 0) { throw new MmapSegmentException("could not sync segment header " + path); } + if (durableCursor < HEADER_SIZE) { + durableCursor = HEADER_SIZE; + } } public boolean isFull() { return capacityRemaining() <= 0; } + public boolean isPublishedDurable() { + return durableCursor >= publishedCursor; + } + /** - * Synchronously flushes dirty pages of {@code [HEADER_SIZE, publishedOffset())} - * to disk via {@code msync(MS_SYNC)}. Off the hot path — call only when - * the user has opted into OS-crash durability (e.g. {@code sf_msync_on_flush=on}). + * Preserves the original explicit mmap-flush behavior for callers that use + * this low-level API directly. Periodic durability uses + * {@link #syncPublished()}, which adds checked error handling and an fd + * barrier. */ public void msync() { - if (memoryBacked) return; // no on-disk pages to flush - long pub = publishedCursor; - if (pub > HEADER_SIZE) { - filesFacade.msync(mmapAddress, pub, false); + if (memoryBacked) { + return; + } + long published = publishedCursor; + if (published > HEADER_SIZE) { + filesFacade.msync(mmapAddress, published, false); + } + } + + /** + * Synchronously flushes every complete frame published when this method + * captures {@link #publishedCursor}. A concurrent producer may publish + * more bytes while the barrier runs; those bytes remain outside the + * returned durable boundary until a later call. + * + * @return the captured byte offset covered by the successful barrier + */ + public long syncPublished() { + long published = publishedCursor; + if (memoryBacked) { + durableCursor = published; + return published; + } + if (published <= durableCursor) { + return durableCursor; + } + if (filesFacade.msync(mmapAddress, published, false) != 0) { + throw new MmapSegmentException("could not sync segment data " + path); + } + // FlushViewOfFile alone is not power-loss durable on Windows. Keep the + // fd barrier on every platform so this method has one portable contract. + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment file " + path); } + durableCursor = published; + return published; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 88ef9cd9..d5fb005a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -94,6 +94,9 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + // Reused by the manager worker while it checkpoints one ring. The entry + // in-service state keeps these segment mappings alive until the pass ends. + private final ObjList syncScratch = new ObjList<>(); private final LongSupplier ticks; // Reused by the worker for one bounded trim quantum. Keeping the unlinked // prefix here until the post-unlink directory barrier succeeds avoids both @@ -164,6 +167,7 @@ public final class SegmentManager implements QuietCloseable { // returns. private boolean scratchFreed; private boolean scratchHandedToWorker; + private volatile long shortestSyncIntervalNanos = Long.MAX_VALUE; private boolean workerLoopExited; // Total bytes currently allocated across every segment owned by every // registered ring (active + sealed + hot-spare). Mutated by the manager @@ -533,7 +537,7 @@ public void deregister(SegmentRing ring) { * the high-water mark — no waiting on the next tick. */ public void register(SegmentRing ring, String dir) { - register(ring, dir, null); + register(ring, dir, null, 0L); } /** @@ -546,6 +550,20 @@ public void register(SegmentRing ring, String dir) { * {@code lowestSurvivingBaseSeq - 1}. */ public void register(SegmentRing ring, String dir, AckWatermark watermark) { + register(ring, dir, watermark, 0L); + } + + /** + * Registers a ring with an optional periodic data-checkpoint interval. + * A positive interval requires disk-backed store-and-forward mode. + */ + public void register(SegmentRing ring, String dir, AckWatermark watermark, long syncIntervalNanos) { + if (syncIntervalNanos < 0L) { + throw new IllegalArgumentException("syncIntervalNanos must not be negative"); + } + if (syncIntervalNanos > 0L && dir == null) { + throw new IllegalArgumentException("periodic sync requires a segment directory"); + } // Account for bytes the ring already owns when it joins. A recovered // ring (post-restart, orphan adoption) can come up at-or-above the cap; // without this seed, totalBytes stays at 0 and the per-tick cap check @@ -559,7 +577,7 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { // the in-flight mmap. Memory-mode rings have no dir; nothing to scan. long minNextGeneration = dir == null ? -1L : scanMaxGeneration(dir) + 1L; Runnable managerWakeup = this::wakeWorker; - RingEntry e = new RingEntry(ring, dir, watermark); + RingEntry e = new RingEntry(ring, dir, watermark, syncIntervalNanos, ticks.getAsLong()); // ObjList.add either throws before storing e or makes the entry visible. // Once visible, only non-throwing state commits may remain. synchronized (lock) { @@ -568,6 +586,12 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { } rings.add(e); totalBytes += ringBytes; + if (syncIntervalNanos > 0L) { + ring.enablePeriodicSync(); + if (syncIntervalNanos < shortestSyncIntervalNanos) { + shortestSyncIntervalNanos = syncIntervalNanos; + } + } } ring.setManagerWakeup(managerWakeup); // Nudge the worker so it picks up the new ring on its very next @@ -608,6 +632,27 @@ public boolean isPathScratchAllocatedForTesting() { } } + @TestOnly + public boolean serviceRingForTesting(SegmentRing ring) { + if (workerThread != null) { + throw new IllegalStateException("test service requires a stopped manager"); + } + RingEntry selected = null; + synchronized (lock) { + for (int i = 0, n = rings.size(); i < n; i++) { + RingEntry candidate = rings.getQuick(i); + if (candidate.ring == ring) { + selected = candidate; + break; + } + } + } + if (selected == null) { + throw new IllegalArgumentException("ring is not registered"); + } + return serviceRing(selected); + } + @TestOnly public void setAfterRingCleanupHook(Runnable hook) { this.afterRingCleanupHook = hook; @@ -788,13 +833,17 @@ private boolean serviceRing(RingEntry e) { } private boolean serviceRing0(RingEntry e) { + boolean memoryMode = e.dir == null; + if (!memoryMode) { + servicePeriodicSync(e, ticks.getAsLong()); + } + // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) // on this or a subsequent tick frees space. Logged at most once per // DISK_FULL_LOG_THROTTLE_NANOS so a sustained-disk-full state // doesn't drown the log. - boolean memoryMode = e.dir == null; if (e.ring.needsHotSpare()) { // Snapshot totalBytes under lock — register/deregister can mutate // it from caller threads. Heavy provisioning I/O happens outside @@ -1123,6 +1172,32 @@ private static String trimFailureName(int failureKind) { } } + private void servicePeriodicSync(RingEntry e, long now) { + if (e.syncIntervalNanos <= 0L + || (!e.ring.isSyncRequested() && now - e.nextDataSyncNanos < 0L)) { + return; + } + try { + e.ring.copyLiveSegmentsForSync(syncScratch); + for (int i = 0, n = syncScratch.size(); i < n; i++) { + syncScratch.getQuick(i).syncPublished(); + } + e.ring.clearSyncRequestIfActiveDurable(); + e.nextDataSyncNanos = now + e.syncIntervalNanos; + e.syncFailureLogged = false; + } catch (Throwable failure) { + e.ring.recordDurabilityFailure(failure); + if (!e.syncFailureLogged) { + e.syncFailureLogged = true; + LOG.error("Periodic SF data sync failed for {}", e.dir, failure); + } + long retry = Math.min(e.syncIntervalNanos, 1_000_000_000L); + e.nextDataSyncNanos = now + retry; + } finally { + syncScratch.clear(); + } + } + private void workerLoop() { try { while (running) { @@ -1148,7 +1223,7 @@ private void workerLoop() { ringSnapshot.clear(); if (!running) break; if (!hasMoreTrimmable) { - LockSupport.parkNanos(pollNanos); + LockSupport.parkNanos(Math.min(pollNanos, shortestSyncIntervalNanos)); } } } finally { @@ -1200,6 +1275,7 @@ private void workerLoop() { private static final class RingEntry { final String dir; final SegmentRing ring; + final long syncIntervalNanos; // Engine-owned ack watermark for this slot, or null in memory // mode and for callers that didn't supply one. Manager-thread // only after register; never closed here (owner closes). @@ -1213,6 +1289,8 @@ private static final class RingEntry { // Prevents a legacy disk registration without a watermark from // flooding the log on every manager tick. boolean missingWatermarkLogged; + long nextDataSyncNanos; + boolean syncFailureLogged; // Zero-allocation manager-thread-only retry state. The deadline uses // the manager's monotonic clock and the delay doubles to a fixed cap. long trimRetryAtNanos; @@ -1230,10 +1308,20 @@ private static final class RingEntry { // to DEREGISTERED. The field updater avoids per-entry allocation. volatile int state = ENTRY_REGISTERED; - RingEntry(SegmentRing ring, String dir, AckWatermark watermark) { + RingEntry( + SegmentRing ring, + String dir, + AckWatermark watermark, + long syncIntervalNanos, + long now + ) { this.ring = ring; this.dir = dir; this.watermark = watermark; + this.syncIntervalNanos = syncIntervalNanos; + // Run the first periodic pass immediately. This establishes a + // durable baseline for segments recovered from MEMORY mode. + this.nextDataSyncNanos = now; } void deregister() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index b839f779..15042098 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -118,6 +118,9 @@ public final class SegmentRing implements QuietCloseable { // call installHotSpare on a closed ring (whose hotSpare was just // zeroed by close()) -- the spare's mmap + fd would never be reclaimed. private boolean closed; + // First periodic data-barrier failure. The manager latches it and the + // producer observes it before its next append. + private volatile MmapSegmentException durabilityFailure; // hotSpare: written by segment manager (installHotSpare), read+cleared by // producer thread on rotation. Volatile so the producer sees fresh installs. private volatile MmapSegment hotSpare; @@ -132,7 +135,9 @@ public final class SegmentRing implements QuietCloseable { // notices on its next polling tick. private Runnable managerWakeup; private long nextSeq; + private boolean periodicSyncEnabled; private volatile long publishedFsn; + private volatile boolean syncRequested; // Plain (producer-thread-only) flag; set to true the first time we ask // the manager for a spare for the current active segment, cleared on // every rotation. Coalesces multiple high-water-mark crossings into a @@ -821,6 +826,7 @@ public boolean acknowledge(long seq) { * {@link #needsHotSpare}) to prepare the next spare. */ public long appendOrFsn(long payloadAddr, int payloadLen) { + checkDurability(); long offset = active.tryAppend(payloadAddr, payloadLen); if (offset == -1L) { // Active is full. Try to rotate. @@ -828,12 +834,20 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { if (spare == null) { return BACKPRESSURE_NO_SPARE; } + // Periodic mode must make the predecessor's complete published + // range durable before the manifest can name its successor. The + // manager performs the barrier; the producer uses the existing + // backpressure path while it waits. + MmapSegment previous = active; + if (requestSyncBeforeRotation(previous)) { + wakeManager(); + return BACKPRESSURE_NO_SPARE; + } // Pin the spare's baseSeq to whatever the active's nextSeq actually // is right now. This is the right moment because (a) the active is // full, so its frameCount is stable, and (b) the spare hasn't been // appended to yet (rebaseSeq enforces that). The segment manager's // earlier guess at baseSeq is irrelevant. - MmapSegment previous = active; long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); if (manifest != null) { @@ -847,13 +861,9 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { // segment of appends; runs outside the monitor because the // spare is not yet visible to any other thread. // - // Deliberately NOT msync'd here: the sealed predecessor's - // data pages. A power loss can therefore tear the sealed - // tail after the boundary is committed, and recovery will - // fail closed on chainEnd != activeBase. That is the - // intended semantics -- page-level durability of frame data - // follows the sender's opt-in msync cadence, and recovery - // must refuse to guess when the two disagree. + // MEMORY mode deliberately does not sync the predecessor's + // data pages. PERIODIC mode reached this point only after the + // manager covered the predecessor's complete published range. spare.syncHeader(); } // Publish the successor before the volatile active promotion. The @@ -913,6 +923,51 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { return fsn; } + public void checkDurability() { + MmapSegmentException failure = durabilityFailure; + if (failure != null) { + throw failure; + } + } + + synchronized void clearSyncRequestIfActiveDurable() { + if (active != null && active.isPublishedDurable()) { + syncRequested = false; + } + } + + synchronized void copyLiveSegmentsForSync(ObjList target) { + target.clear(); + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + target.add(sealedSegments.get(i)); + } + if (active != null) { + target.add(active); + } + } + + void enablePeriodicSync() { + periodicSyncEnabled = true; + syncRequested = true; + } + + boolean isSyncRequested() { + return syncRequested; + } + + void recordDurabilityFailure(Throwable failure) { + if (durabilityFailure == null) { + MmapSegmentException wrapped = failure instanceof MmapSegmentException + ? (MmapSegmentException) failure + : new MmapSegmentException("periodic SF data sync failed", failure); + synchronized (this) { + if (durabilityFailure == null) { + durabilityFailure = wrapped; + } + } + } + } + @Override public synchronized void close() { // Marking closed BEFORE freeing fields ensures any concurrent @@ -1244,11 +1299,24 @@ public synchronized MmapSegment pinSegmentContainingForTest(long fsn) { return pinSegmentContaining(fsn); } + @TestOnly + public void recordDurabilityFailureForTesting(Throwable failure) { + recordDurabilityFailure(failure); + } + @TestOnly public synchronized void releasePinnedSegmentForTest(MmapSegment expected) { releasePinnedSegment(expected); } + private synchronized boolean requestSyncBeforeRotation(MmapSegment previous) { + if (periodicSyncEnabled && !previous.isPublishedDurable()) { + syncRequested = true; + return true; + } + return false; + } + /** Releases the I/O cursor pin and wakes trim if it still names {@code expected}. */ synchronized void releasePinnedSegment(MmapSegment expected) { if (ioPinnedSegment == expected) { @@ -1353,6 +1421,15 @@ public synchronized int snapshotSealedSegments(MmapSegment[] target) { return n > target.length ? -1 : n; } + public void syncAllLiveSegments() { + ObjList segments = new ObjList<>(); + copyLiveSegmentsForSync(segments); + for (int i = 0, n = segments.size(); i < n; i++) { + segments.getQuick(i).syncPublished(); + } + clearSyncRequestIfActiveDurable(); + } + /** * Total mmap'd bytes the ring currently owns: active + hot spare (if * installed) + every sealed segment. Used by {@code SegmentManager} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 3e066300..765fc215 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -77,6 +77,14 @@ private SlotLock(String slotDir, int fd) { * or lock contention. */ public static SlotLock acquire(String slotDir) { + return acquire(slotDir, false); + } + + /** + * Acquires a slot and optionally makes the slot entry durable in its + * parent directory before any segment file is created. + */ + public static SlotLock acquire(String slotDir, boolean syncParentDirectory) { if (slotDir == null || slotDir.isEmpty()) { throw new IllegalArgumentException("slotDir must not be empty"); } @@ -87,6 +95,10 @@ public static SlotLock acquire(String slotDir) { "could not create slot dir: " + slotDir + " rc=" + rc); } } + if (syncParentDirectory && Files.fsyncParentDir(slotDir) != 0) { + throw new IllegalStateException( + "could not sync parent directory for SF slot: " + slotDir); + } String lockPath = slotDir + "/" + LOCK_FILE_NAME; String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME; int fd = Files.openRW(lockPath); diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index a255e638..9cd18134 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -88,6 +88,7 @@ public final class ConfigSchema { str("sf_durability", Side.INGRESS); str("sf_max_bytes", Side.INGRESS); str("sf_max_total_bytes", Side.INGRESS); + str("sf_sync_interval_millis", Side.INGRESS); str("transaction", Side.INGRESS); // EGRESS -- the QwpQueryClient applies. Typed where there is a range or diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index b0fdfb2e..17c74ca4 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -24,6 +24,7 @@ package io.questdb.client.std; +import java.io.File; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -440,6 +441,15 @@ public static int fsyncDir(String dir) { } } + /** + * Forces the directory entry naming {@code path} to durable storage by + * syncing its absolute parent directory. + */ + public static int fsyncParentDir(String path) { + File parent = new File(path).getAbsoluteFile().getParentFile(); + return parent == null ? 0 : fsyncDir(parent.getPath()); + } + /** * Truncates the file to exactly {@code size} bytes via {@code ftruncate}. * Returns {@code true} on success. Does NOT reserve disk space — the diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java index 0c27d698..1644d561 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java @@ -26,6 +26,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -165,6 +166,46 @@ public void testSfMaxTotalBytesAcceptsLargeValue() throws Exception { }); } + @Test + public void testPeriodicDurabilityDefaultAndExplicitInterval() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AckHandler handler = new AckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String base = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfDir + ";sf_durability=periodic;"; + try (Sender sender = Sender.fromConfig(base)) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + Assert.assertEquals(5_000_000_000L, + ws.getCursorEngineForTesting().getSyncIntervalNanosForTesting()); + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + } + + try (Sender sender = Sender.fromConfig(base + + "sender_id=explicit;sf_sync_interval_millis=123;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + Assert.assertEquals(123_000_000L, + ws.getCursorEngineForTesting().getSyncIntervalNanosForTesting()); + } + } + }); + } + + @Test + public void testPeriodicDurabilityRequiresDirectory() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender ignored = Sender.fromConfig( + "ws::addr=localhost:1;sf_durability=periodic;")) { + Assert.fail("expected periodic mode without sf_dir to fail"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains("requires sf_dir")); + } + }); + } + @Test public void testSfDurabilityAppendNotYetSupported() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -221,6 +262,20 @@ public void testSfDurabilityOnTcpRejected() throws Exception { }); } + @Test + public void testSfSyncIntervalRequiresPeriodicMode() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + + ";sf_sync_interval_millis=5000;"; + try (Sender ignored = Sender.fromConfig(config)) { + Assert.fail("expected interval without periodic mode to fail"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("requires sf_durability=periodic")); + } + }); + } + @Test public void testSfMaxBytesAcceptsSizeSuffixes() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index c11fe1fe..db570367 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; @@ -104,6 +105,26 @@ public void testAcknowledgePropagatesToRing() throws Exception { }); } + @Test + public void testAppendChecksLatchedDurabilityFailureBeforePublishing() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) { + MmapSegmentException failure = new MmapSegmentException("injected data-sync failure"); + engine.getRingForTesting().recordDurabilityFailureForTesting(failure); + try { + engine.appendBlocking(buf, 16); + fail("expected latched durability failure"); + } catch (MmapSegmentException expected) { + assertTrue(expected == failure); + } + assertEquals(-1L, engine.publishedFsn()); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testAppendBlockingNeverFailsUnderManagerSupply() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -400,6 +421,53 @@ public void testManagerPersistedWatermarkSurvivesRestart() throws Exception { }); } + @Test + public void testPeriodicRotationWaitsForDurablePredecessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + + 2L * (MmapSegment.FRAME_HEADER_SIZE + 64L); + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + CursorSendEngine engine = new CursorSendEngine( + tmpDir, segmentSize, segmentSize * 4L, + TimeUnit.SECONDS.toNanos(5), TimeUnit.HOURS.toNanos(1)); + try { + assertEquals(0L, engine.appendBlocking(buf, 64)); + assertEquals(1L, engine.appendBlocking(buf, 64)); + // The third append needs rotation. Its predecessor is dirty and + // the periodic deadline is an hour away, so only the explicit + // rotation request can make progress. + assertEquals(2L, engine.appendBlocking(buf, 64)); + ObjList sealed = engine.getRingForTesting().getSealedSegments(); + assertEquals(1, sealed.size()); + assertTrue(sealed.getQuick(0).isPublishedDurable()); + MmapSegment active = engine.getRingForTesting().getActive(); + assertFalse(active.isPublishedDurable()); + engine.close(); + assertTrue("close must sync the unacknowledged active", active.isPublishedDurable()); + } finally { + engine.close(); + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testPeriodicValidationPrecedesOwnedManagerAllocation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try { + new CursorSendEngine( + null, + 4096L, + 8192L, + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + 1L); + fail("expected periodic memory-mode validation failure"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("requires disk store-and-forward mode")); + } + }); + } + @Test public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 3edf4f95..3408a379 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -55,6 +55,60 @@ public void tearDown() { TestUtils.removeTmpDir(tmpDir); } + @Test + public void testBarrierChecksMsyncAndFsyncFailures() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + FaultyFilesFacade success = new FaultyFilesFacade(); + try (MmapSegment segment = MmapSegment.create( + success, tmpDir + "/barrier-ok.sfa", 0L, 4096L)) { + fillPattern(payload, 32, 1); + assertTrue(segment.tryAppend(payload, 32) >= 0L); + assertFalse(segment.isPublishedDurable()); + assertEquals(segment.publishedOffset(), segment.syncPublished()); + assertTrue(segment.isPublishedDurable()); + assertEquals(1, success.msyncCalls); + assertEquals(1, success.fsyncCalls); + } + + FaultyFilesFacade msyncFailure = new FaultyFilesFacade(); + msyncFailure.failOnMsync = true; + try (MmapSegment segment = MmapSegment.create( + msyncFailure, tmpDir + "/barrier-msync-fail.sfa", 0L, 4096L)) { + assertTrue(segment.tryAppend(payload, 32) >= 0L); + try { + segment.syncPublished(); + fail("expected msync failure"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment data")); + } + assertFalse(segment.isPublishedDurable()); + assertEquals(1, msyncFailure.msyncCalls); + assertEquals(0, msyncFailure.fsyncCalls); + } + + FaultyFilesFacade fsyncFailure = new FaultyFilesFacade(); + fsyncFailure.failOnFsync = true; + try (MmapSegment segment = MmapSegment.create( + fsyncFailure, tmpDir + "/barrier-fsync-fail.sfa", 0L, 4096L)) { + assertTrue(segment.tryAppend(payload, 32) >= 0L); + try { + segment.syncPublished(); + fail("expected fsync failure"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + assertFalse(segment.isPublishedDurable()); + assertEquals(1, fsyncFailure.msyncCalls); + assertEquals(1, fsyncFailure.fsyncCalls); + } + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testCapacityRemainingAccountsForFrameEnvelope() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -80,6 +134,27 @@ public void testCapacityRemainingAccountsForFrameEnvelope() throws Exception { }); } + @Test + public void testCompatibilityMsyncKeepsLegacyCallPattern() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + FaultyFilesFacade filesFacade = new FaultyFilesFacade(); + try (MmapSegment segment = MmapSegment.create( + filesFacade, tmpDir + "/compat-msync.sfa", 0L, 4096L)) { + assertTrue(segment.tryAppend(payload, 32) >= 0L); + segment.msync(); + segment.msync(); + assertEquals(2, filesFacade.msyncCalls); + assertEquals(0, filesFacade.fsyncCalls); + assertFalse(segment.isPublishedDurable()); + } + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testCreateAppendCloseReopenScansAllFrames() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -538,8 +613,12 @@ private static final class FaultyFilesFacade implements FilesFacade { int allocateCalls; int closeCalls; boolean failOnAllocate; + boolean failOnFsync; + boolean failOnMsync; boolean failOnOpenCleanRW; boolean failOnOpenRWExclusive; + int fsyncCalls; + int msyncCalls; int openCleanRWCalls; int openRWExclusiveCalls; int removeCalls; @@ -619,7 +698,8 @@ public void freeNativePath(long pathPtr) { @Override public int fsync(int fd) { - return INSTANCE.fsync(fd); + fsyncCalls++; + return failOnFsync ? -1 : INSTANCE.fsync(fd); } @Override @@ -642,6 +722,12 @@ public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + msyncCalls++; + return failOnMsync ? -1 : INSTANCE.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { openCleanRWCalls++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java new file mode 100644 index 00000000..2e395783 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java @@ -0,0 +1,259 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SegmentManagerPeriodicSyncTest { + + @Test + public void testDeadlineAndFailurePropagation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long intervalNanos = 100L; + final long segmentSize = 4096L; + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-manager-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 1L, segmentSize)); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + assertEquals(1, filesFacade.msyncCalls); + assertEquals(1, filesFacade.fsyncCalls); + + assertEquals(1L, ring.appendOrFsn(payload, 16)); + ticks.set(intervalNanos - 1L); + manager.serviceRingForTesting(ring); + assertEquals("checkpoint ran before its deadline", 1, filesFacade.msyncCalls); + assertEquals("checkpoint ran before its deadline", 1, filesFacade.fsyncCalls); + + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + assertEquals(2, filesFacade.msyncCalls); + assertEquals(2, filesFacade.fsyncCalls); + + assertEquals(2L, ring.appendOrFsn(payload, 16)); + filesFacade.isFsyncFailureEnabled = true; + ticks.set(intervalNanos * 2L); + manager.serviceRingForTesting(ring); + assertEquals(3, filesFacade.msyncCalls); + assertEquals(3, filesFacade.fsyncCalls); + try { + ring.appendOrFsn(payload, 16); + fail("expected manager data-sync failure to reach the producer"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + assertEquals("failed append must not enter the ring", 2L, ring.publishedFsn()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + private static final class CountingFilesFacade implements FilesFacade { + private boolean isFsyncFailureEnabled; + private int fsyncCalls; + private int msyncCalls; + + @Override + public boolean allocate(int fd, long size) { + return INSTANCE.allocate(fd, size); + } + + @Override + public long allocNativePath(String path) { + return INSTANCE.allocNativePath(path); + } + + @Override + public int close(int fd) { + return INSTANCE.close(fd); + } + + @Override + public boolean exists(String path) { + return INSTANCE.exists(path); + } + + @Override + public void findClose(long findPtr) { + INSTANCE.findClose(findPtr); + } + + @Override + public long findFirst(String dir) { + return INSTANCE.findFirst(dir); + } + + @Override + public long findName(long findPtr) { + return INSTANCE.findName(findPtr); + } + + @Override + public int findNext(long findPtr) { + return INSTANCE.findNext(findPtr); + } + + @Override + public int findType(long findPtr) { + return INSTANCE.findType(findPtr); + } + + @Override + public void freeNativePath(long pathPtr) { + INSTANCE.freeNativePath(pathPtr); + } + + @Override + public int fsync(int fd) { + fsyncCalls++; + return isFsyncFailureEnabled ? -1 : INSTANCE.fsync(fd); + } + + @Override + public long length(int fd) { + return INSTANCE.length(fd); + } + + @Override + public long length(String path) { + return INSTANCE.length(path); + } + + @Override + public long length(long pathPtr) { + return INSTANCE.length(pathPtr); + } + + @Override + public int lock(int fd) { + return INSTANCE.lock(fd); + } + + @Override + public int mkdir(String path, int mode) { + return INSTANCE.mkdir(path, mode); + } + + @Override + public int msync(long addr, long len, boolean async) { + msyncCalls++; + return INSTANCE.msync(addr, len, async); + } + + @Override + public int openCleanRW(String path) { + return INSTANCE.openCleanRW(path); + } + + @Override + public int openCleanRW(long pathPtr) { + return INSTANCE.openCleanRW(pathPtr); + } + + @Override + public int openRW(String path) { + return INSTANCE.openRW(path); + } + + @Override + public int openRW(long pathPtr) { + return INSTANCE.openRW(pathPtr); + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + + @Override + public boolean remove(String path) { + return INSTANCE.remove(path); + } + + @Override + public boolean remove(long pathPtr) { + return INSTANCE.remove(pathPtr); + } + + @Override + public int rename(String oldPath, String newPath) { + return INSTANCE.rename(oldPath, newPath); + } + + @Override + public boolean truncate(int fd, long size) { + return INSTANCE.truncate(fd, size); + } + + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index dba5765f..7fe602d3 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -63,8 +63,9 @@ public void testEveryIngressKeyIsHonored() { assertHonored("sf_dir=/var/probe", "sf_dir", "/var/probe"); assertHonored("sf_max_bytes=4096", "sf_max_bytes", 4096L); assertHonored("sf_max_total_bytes=8192", "sf_max_total_bytes", 8192L); - assertHonored("sf_durability=flush", "sf_durability", "FLUSH"); + assertHonored("sf_durability=periodic", "sf_durability", "PERIODIC"); assertHonored("sf_append_deadline_millis=1500", "sf_append_deadline_millis", 1500L); + assertHonored("sf_sync_interval_millis=5000", "sf_sync_interval_millis", 5000L); assertHonored("close_flush_timeout_millis=2500", "close_flush_timeout_millis", 2500L); assertHonored("durable_ack_keepalive_interval_millis=900", "durable_ack_keepalive_interval_millis", 900L); assertHonored("initial_connect_retry=async", "initial_connect_retry", "ASYNC"); diff --git a/core/src/test/java/io/questdb/client/test/std/FilesTest.java b/core/src/test/java/io/questdb/client/test/std/FilesTest.java index ebef927a..63252c45 100644 --- a/core/src/test/java/io/questdb/client/test/std/FilesTest.java +++ b/core/src/test/java/io/questdb/client/test/std/FilesTest.java @@ -73,6 +73,16 @@ public void tearDown() { Files.remove(tmpDir); } + @Test + public void testFsyncParentDir() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String child = tmpDir + "/child"; + assertEquals(0, Files.mkdir(child, Files.DIR_MODE_DEFAULT)); + assertEquals(0, Files.fsyncParentDir(child)); + assertTrue(Files.remove(child)); + }); + } + @Test public void testWriteReadRoundtrip() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java b/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java index 18cc7b52..082dc4c4 100644 --- a/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java +++ b/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java @@ -4,7 +4,7 @@ import io.questdb.client.Sender; /** - * Crash-durable ingest with store-and-forward ({@code sf_dir}). + * Process-restart and periodic power-loss protection with store-and-forward. *

        * Ingestion is asynchronous in every mode: {@code flush()} hands rows to a * background send engine that delivers them and tracks the server's @@ -17,7 +17,10 @@ *

      13. Store-and-forward ({@code sf_dir} set): the engine backs its * buffer with memory-mapped files, so unacked rows survive a producer * process restart -- on the next startup it recovers the tail from - * disk and replays it once the server is reachable.
        14. + * disk and replays it once the server is reachable. Add + * {@code sf_durability=periodic} for background power-loss checkpoints; + * the configured interval is a target cadence and storage latency adds + * to the actual recovery window. * * {@code request_durable_ack=on} (Enterprise + replication) makes the ack wait * for the durable upload to object storage rather than the ordinary WAL commit. @@ -28,6 +31,8 @@ public static void main(String[] args) { try (QuestDB db = QuestDB.connect( "ws::addr=localhost:9000;" + "sf_dir=/var/lib/questdb/sf;" + + "sf_durability=periodic;" + + "sf_sync_interval_millis=5000;" + "request_durable_ack=on;")) { try (Sender sender = db.borrowSender()) { From ca7a2dac40e957effe8a2eb1c163ee31af407388 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 20 Jul 2026 15:45:18 +0100 Subject: [PATCH 34/64] fix(qwp): fail closed on SF manifest with uncollapsed boundaries and no segment files Recovery treated every valid manifest with zero .sfa files as EMPTY and removed it, even when headBase < activeBase. No in-protocol crash can produce that state: the close-time drain durably collapses the boundaries to head == active before its first unlink, and a fresh start writes (0,0). Uncollapsed boundaries with no segment files therefore prove durable, never-declared-acked frames vanished outside the protocol (manual wipe, partial restore) -- yet recovery silently started fresh and deleted the manifest, destroying the only evidence of the loss. Accept the segment-less slot as EMPTY only when headBase == activeBase, matching the existing guard on the some-files clean-drain window; otherwise throw without mutating the slot. Fix the drain-window test to model the real crash state (9,9) and add a fail-closed (7,9) test asserting the directory is left byte-identical. --- .../qwp/client/sf/cursor/SegmentRing.java | 36 +++++++++++++------ .../cursor/SegmentRecoveryIntegrityTest.java | 36 +++++++++++++++++-- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 15042098..d0e3fbe8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -276,16 +276,32 @@ static Recovery recover( return Recovery.empty(); } if (manifest != null) { - // No .sfa files at all. Two legitimate protocols produce - // this: the close-time drain unlinks the last segment - // before it removes the manifest, and a fresh-start crash - // can leave a boundary-less (0,0) manifest behind. In - // both cases nothing recoverable exists, so accept EMPTY - // -- but shout, because a manual wipe of segment files - // looks identical and the operator should know. - LOG.warn("SF manifest exists in {} with no segment files " - + "(clean-drain or fresh-start crash window, or manual " - + "segment removal); discarding it and starting fresh", sfDir); + // No .sfa files at all. Only two protocols legitimately + // produce this, and both leave head == active: the + // close-time drain durably collapses the boundaries + // BEFORE its first unlink, and a fresh-start crash can + // leave a boundary-less (0,0) manifest behind. Uncollapsed + // boundaries therefore prove durable, never-declared-acked + // frames existed in [headBase, activeBase] whose files + // vanished outside the protocol (manual wipe, partial + // restore) -- fail closed and keep the manifest as + // evidence instead of silently starting fresh. + long manifestHeadBase = manifest.headBase(); + long manifestActiveBase = manifest.activeBase(); + if (manifestHeadBase != manifestActiveBase) { + throw new MmapSegmentException(SfManifest.FILE_NAME + " in " + sfDir + + " references durable data (headBase=" + manifestHeadBase + + ", activeBase=" + manifestActiveBase + + ") but no segment files exist"); + } + // Collapsed boundaries: nothing recoverable exists, so + // accept EMPTY -- but shout, because a manual wipe of a + // fully-acked slot looks identical and the operator + // should know. + LOG.warn("SF manifest exists in {} with collapsed boundaries ({}) and " + + "no segment files (clean-drain or fresh-start crash window, or " + + "manual segment removal); discarding it and starting fresh", + sfDir, manifestActiveBase); manifest.close(); manifest = null; if (!SfManifest.removeFile(filesFacade, sfDir)) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java index a1186176..0d3d560c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -423,9 +423,11 @@ public void testRotationCrashWindowEmptyActiveAtChainEndRecovers() throws Except @Test public void testDrainWindowManifestWithoutSegmentsRecoversEmptyAndRemovesManifest() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Clean-drain close unlinks the last segment before the manifest; - // a crash between the two leaves this state. - writeManifest(3, 7, 9); + // Clean-drain close durably collapses the boundaries to + // head == active before unlinking, then removes segments and + // finally the manifest; a crash between the last unlink and the + // manifest removal leaves collapsed boundaries with no files. + writeManifest(3, 9, 9); SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); Assert.assertNull("segment-less slot must recover as EMPTY", ring); @@ -434,6 +436,34 @@ public void testDrainWindowManifestWithoutSegmentsRecoversEmptyAndRemovesManifes }); } + @Test + public void testZeroSegmentFilesWithUncollapsedBoundariesFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // headBase(7) < activeBase(9): the manifest durably committed that + // segments [7..9] existed and were never declared acked. No + // in-protocol crash leaves this next to zero .sfa files -- the + // close-time drain collapses boundaries to head == active BEFORE + // its first unlink, and a fresh start writes (0,0). Uncollapsed + // boundaries with no segment files therefore prove durable data + // vanished outside the protocol: recovery must fail closed and + // keep the manifest as evidence, not silently start fresh. + writeManifest(3, 7, 9); + Map before = snapshotDir(); + + try { + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + if (ring != null) { + ring.close(); + } + Assert.fail("recovery must fail closed: manifest boundaries (7,9) reference " + + "durable frames but no segment file survives"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "references durable data"); + } + assertDirUnchanged(before); + }); + } + @Test public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() throws Exception { TestUtils.assertMemoryLeak(() -> { From 7e482c5fa2a1b70133fb865e7476ba8c20b498cf Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 20 Jul 2026 19:15:23 +0100 Subject: [PATCH 35/64] fix(client): wake a blocked Windows recv on closeTraffic On POSIX, shutdown(SHUT_RDWR) unblocks a recv() already in progress on another thread, which is how closeTraffic() wakes a worker parked in recv without releasing the fd. Winsock's shutdown(SD_BOTH) does not do this: an already-blocked recv() stays parked until data arrives, the peer resets, or the socket is closed. That left the QWP I/O worker stuck and failed SocketTrafficShutdownTest.testPlainSocketShutdownWakesWindowsRecvAndRetainsFd on the windows-msvc-2022-x64 job. Cancel outstanding I/O on the socket handle with CancelIoEx before the shutdown(). This wakes the blocked recv() while leaving the descriptor allocated, so the fd is retained (no reuse race) until close() releases it, matching the POSIX/macOS behaviour the sibling tests already rely on. Net.shutdown() is only reached via PlainSocket.closeTraffic(), so no other caller is affected. --- core/src/main/c/windows/net.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index d105b504..191295ab 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -232,6 +232,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_configureNonBlocking JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown (JNIEnv *e, jclass cl, jint fd) { + // Unlike POSIX, a Winsock shutdown() does not unblock a recv() that is + // already in progress on another thread. Cancel any outstanding I/O on the + // handle first so a worker parked in a blocking recv() wakes up. CancelIoEx + // does not close the socket, so the descriptor stays allocated and the fd + // cannot be reused underneath the worker before close() releases it. A + // FALSE return with ERROR_NOT_FOUND (nothing was pending) is expected here + // and deliberately ignored. + CancelIoEx((HANDLE) (SOCKET) fd, NULL); const int result = shutdown((SOCKET) fd, SD_BOTH); if (result == SOCKET_ERROR) { const int error = WSAGetLastError(); From 8d962e0526aa199ab77cd14292990cf1780e4371 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 20 Jul 2026 21:12:04 +0100 Subject: [PATCH 36/64] fix(qwp): wait for worker exit cleanups before reaping in SegmentManager.close() The bounded-join fallback in close() reaped the manager worker (workerThread=null => isWorkerReaped()) as soon as it observed workerLoopExited, which the worker publishes at the START of its exit block -- BEFORE it runs the deferred exit cleanups (the owning engine's finishClose(), which releases the slot flock and only then publishes closeCompleted). When the bounded join timed out mid-finishClose, close() reaped while the flock release was still in flight, so a caller reading isCloseCompleted() saw a stale false and a spurious flock-release retry could be scheduled. On the slow JDK 8 CI leg this raced SenderPoolSfTest.testPreallocatedExitHandoff* to failure ("worker exit must complete startup-recoverer cleanup without a sender or engine close retry"), and the leaked shared retry driver then cascaded into the FlockReleaseRetryDriverTest suite. Once workerLoopExited is true the worker has left its (possibly wedged) service loop and is running only finite exit cleanups, so give it a second bounded join to terminate before reaping. This reuses the same join-under-monitor pattern as the first join -- the worker uses a lock-free CAS for exactly-once cleanup and never blocks on the manager monitor -- and still reaps on timeout, so a pathologically slow cleanup cannot hang close(). The second join is budgeted with the fixed WORKER_JOIN_TIMEOUT_MILLIS, not the tunable workerJoinTimeoutMillis: the first join bounds a possibly-wedged service pass (tests shrink it to force the timed-out path), but the finite exit cleanups must be allowed to finish regardless. In production both values are equal. Validated under JDK 8: a deterministic repro (400ms finishClose delay) reproduces the exact CI assertion without this change and passes with it; the full previously-failing set (SenderPoolSfTest, FlockReleaseRetryDriverTest, EngineClosePublishAfterFlockReleaseTest, CursorWebSocketSendLoopRotationRaceTest) is green. --- .../qwp/client/sf/cursor/SegmentManager.java | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index d5fb005a..dab9d099 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -299,9 +299,50 @@ public synchronized void close() { return; } } - // The worker loop has already run its exit block; the thread - // is at most a few instructions from terminating and can no - // longer touch manager state. Fall through and reap it. + // The worker has left its service loop (workerLoopExited) and + // is now running only its finite exit cleanups -- the owning + // engine's finishClose(), which releases the slot flock and + // only THEN publishes closeCompleted. It can no longer wedge in + // a service pass, so give it a second bounded join to finish + // before reaping. Without this, a bounded-join timeout that + // lands mid-finishClose reaps the worker (workerThread=null => + // isWorkerReaped()) while the flock release is still in flight, + // so a caller reading isCloseCompleted() observes a stale false + // and a spurious flock-release retry gets scheduled. This reuses + // the same join-under-monitor pattern as the bounded join above + // -- the worker uses a lock-free CAS for exactly-once cleanup + // and never blocks on this monitor -- and still reaps on + // timeout, so a pathologically slow cleanup cannot hang close(). + // + // Budget with the fixed WORKER_JOIN_TIMEOUT_MILLIS, NOT the + // tunable workerJoinTimeoutMillis: the first join bounds a + // possibly-wedged SERVICE pass (tests shrink it to force the + // timed-out path), but the exit cleanups are finite and must be + // allowed to finish regardless of that tuning. In production + // both values are equal, so this only matters under a shrunk + // test override. + boolean cleanupInterrupted = Thread.interrupted(); + long cleanupDeadlineNanos = System.nanoTime() + WORKER_JOIN_TIMEOUT_MILLIS * 1_000_000L; + try { + while (t.isAlive()) { + long remainingMillis = (cleanupDeadlineNanos - System.nanoTime()) / 1_000_000L; + if (remainingMillis <= 0) { + break; + } + try { + t.join(remainingMillis); + } catch (InterruptedException ignored) { + cleanupInterrupted = true; + } + } + } finally { + if (cleanupInterrupted) { + Thread.currentThread().interrupt(); + } + } + // If the cleanups still have not finished (pathologically slow + // syscall), fall through and reap anyway -- identical to the + // best-effort behaviour before this second join existed. } workerThread = null; } From 178fbe9f3f70258bbef9ba5e91a7cb585dd604ba Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 01:08:30 +0100 Subject: [PATCH 37/64] fix(qwp): close SF manifest fd exactly once when quarantine fails in open() In the no-valid-record branch, SfManifest.open() closed the fd and then called quarantineDebris(), which throws when both rename and remove fail (e.g. a permission-degraded slot dir). The enclosing catch then closed the same fd a second time. With concurrent threads opening files, the OS can reuse the fd number in that window, so the second close could kill an unrelated live descriptor and silently corrupt whatever it backed. Mark ownership released (fd = -1) after the first close and guard the catch, so the fd is closed exactly once on every path. Adds a SegmentRecoveryIntegrityTest case: correctly-sized manifest with both records CRC-broken and a facade failing rename+remove asserts the manifest fd is closed exactly once (fails with 2 closes pre-fix). --- .../qwp/client/sf/cursor/SfManifest.java | 10 +- .../cursor/SegmentRecoveryIntegrityTest.java | 93 +++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java index 368c834c..39e2e208 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -145,13 +145,21 @@ static SfManifest open(FilesFacade filesFacade, String dir) { // bit rot), recovery still fails closed on the // manifest-required flag check. filesFacade.close(fd); + // Ownership released: quarantineDebris may throw (when both + // rename and remove fail) and the catch below must not close + // this fd again -- the OS may already have handed the number + // to another thread, and a double-close would silently kill + // an unrelated descriptor. + fd = -1; quarantineDebris(filesFacade, path, "no valid CRC-protected record"); return null; } return new SfManifest(filesFacade, path, fd, selected.generation, selected.headBase, selected.activeBase); } catch (Throwable t) { - filesFacade.close(fd); + if (fd != -1) { + filesFacade.close(fd); + } throw t; } finally { Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java index 0d3d560c..5935aa24 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -464,6 +464,66 @@ public void testZeroSegmentFilesWithUncollapsedBoundariesFailsClosed() throws Ex }); } + @Test + public void testUnquarantinableCorruptManifestClosesFdExactlyOnce() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Correctly-sized manifest with BOTH records CRC-broken: open() + // closes the fd and then tries to quarantine the debris. Make + // rename AND remove fail (permission-degraded slot dir) so + // quarantineDebris throws after the fd is already closed. The + // propagating failure must not close the fd a second time -- the + // OS may already have handed that number to another thread, and + // a double-close would silently kill an unrelated descriptor. + writeManifestBothRecordsCrcBroken(); + Map before = snapshotDir(); + + String manifestPath = tmpDir + "/" + MANIFEST_NAME; + int[] manifestFd = {-1}; + int[] manifestFdCloses = {0}; + FilesFacade facade = new DelegatingFacade() { + @Override + public int close(int fd) { + if (fd >= 0 && fd == manifestFd[0]) { + manifestFdCloses[0]++; + } + return super.close(fd); + } + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (manifestPath.equals(path)) { + manifestFd[0] = fd; + } + return fd; + } + + @Override + public boolean remove(String path) { + return !manifestPath.equals(path) && super.remove(path); + } + + @Override + public int rename(String oldPath, String newPath) { + return manifestPath.equals(oldPath) ? -1 : super.rename(oldPath, newPath); + } + }; + try { + SegmentRing ring = SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE); + if (ring != null) { + ring.close(); + } + Assert.fail("recovery must fail when corrupt-manifest quarantine cannot proceed"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not quarantine"); + } + Assert.assertTrue("manifest was never opened", manifestFd[0] >= 0); + Assert.assertEquals("manifest fd must be closed exactly once (a double-close can " + + "kill an unrelated descriptor)", 1, manifestFdCloses[0]); + assertDirUnchanged(before); + }); + } + @Test public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -643,6 +703,39 @@ private void writeManifest(long generation, long headBase, long activeBase) { } } + /** + * Writes a correctly-sized (128-byte) manifest whose A and B records are + * BOTH structurally plausible (magic, version, boundaries) but fail their + * CRC check -- the "no valid CRC-protected record" quarantine trigger. + */ + private void writeManifestBothRecordsCrcBroken() { + String path = tmpDir + "/" + MANIFEST_NAME; + int fd = Files.openRW(path); + Assert.assertTrue("could not create manifest", fd >= 0); + try { + Assert.assertTrue(Files.truncate(fd, 128)); + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + for (long generation = 1; generation <= 2; generation++) { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, 0x314d4653); // SFM1 + Unsafe.getUnsafe().putInt(buf + 4, 1); // version + Unsafe.getUnsafe().putLong(buf + 8, generation); + Unsafe.getUnsafe().putLong(buf + 16, 0); // headBase + Unsafe.getUnsafe().putLong(buf + 24, 2); // activeBase + int crc = Crc32c.update(Crc32c.INIT, buf, 60); + Unsafe.getUnsafe().putInt(buf + 60, crc + 1); // broken CRC + long offset = (generation & 1L) * 64; + Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + /** * Snapshot of the durable SF payload files (segments, quarantined * segments, manifest): name -> content. Lifecycle noise such as the slot From 0c81ad34130dab519ab0225ed2e9dec4eded0934 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 01:16:36 +0100 Subject: [PATCH 38/64] perf(qwp): fuse per-frame CRC into one native call in MmapSegment.tryAppend The append hot path computed the frame CRC-32C with two Crc32c.update JNI calls -- one over the u32 payloadLen field, one over the payload -- even though the layout [u32 crc][u32 payloadLen][payload] places the payload immediately after payloadLen (at lenAddr+4). The two CRC'd regions are physically contiguous, so a single update over 4 + payloadLen bytes yields the byte-identical CRC while crossing the JNI boundary once per frame instead of twice. This is the same form the recovery scanner already recomputes (reader.crc32c(pos + 4, 4 + payloadLen) in scanForRecovery), which validates every on-disk frame against the producer-written CRC, so the fused value is guaranteed identical and the round-trip is unchanged. No behavior change, no change to hashed or written bytes; saves one native-call dispatch per non-empty frame on the producer's zero-alloc hot path. Verified by MmapSegmentTest and MmapSegmentRecoveryFaultTest. --- .../cutlass/qwp/client/sf/cursor/MmapSegment.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 8758aabf..e7f86651 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -554,16 +554,17 @@ public long tryAppend(long payloadAddr, int payloadLen) { return -1L; } // CRC32C over the (payloadLen, payload) pair. Recovery scans validate - // each frame by recomputing this CRC over the on-disk bytes. + // each frame by recomputing this CRC over the on-disk bytes. The + // payloadLen field (4 bytes at lenAddr) and the payload (at lenAddr+4) + // are physically contiguous, so a single CRC pass covers both -- one + // native call per frame, byte-identical to the chained form the + // recovery scanner recomputes (see scanForRecovery). long lenAddr = mmapAddress + offset + 4; Unsafe.getUnsafe().putInt(lenAddr, payloadLen); if (payloadLen > 0) { Unsafe.getUnsafe().copyMemory(payloadAddr, mmapAddress + offset + FRAME_HEADER_SIZE, payloadLen); } - int crc = Crc32c.update(Crc32c.INIT, lenAddr, 4); - if (payloadLen > 0) { - crc = Crc32c.update(crc, mmapAddress + offset + FRAME_HEADER_SIZE, payloadLen); - } + int crc = Crc32c.update(Crc32c.INIT, lenAddr, 4L + payloadLen); Unsafe.getUnsafe().putInt(mmapAddress + offset, crc); appendCursor = offset + total; // Plain read + write of the volatile field. `frameCount++` would From 8fd842f2045c6cae9cdd502c4055be89dfeef61f Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 02:04:32 +0100 Subject: [PATCH 39/64] fix(client): make Windows fsyncDir0 best-effort for directory handles FlushFileBuffers is documented for file/volume handles; on NTFS it refuses a directory handle, commonly returning ERROR_ACCESS_DENIED (or ERROR_INVALID_FUNCTION on filesystems that do not implement it), and a GENERIC_WRITE open of a directory can itself be refused by ACLs. Because fsyncDir0 failures are fatal to their callers -- SfManifest.create, SlotLock.acquire (reached whenever sf_sync_interval_millis > 0), and the SegmentManager hot-spare/trim barriers -- SF disk mode with sf_durability=periodic could hard-fail to start on Windows, a Windows-only regression on the durability path. Treat the documented "directory cannot be flushed" signatures (ERROR_ACCESS_DENIED, ERROR_INVALID_FUNCTION), and an ACL-refused GENERIC_WRITE open, as best-effort success: create/rename/unlink of directory entries are made crash consistent by NTFS metadata journaling ($LogFile). This mirrors libgit2/PostgreSQL/SQLite, which do not rely on directory fsync on Windows. Genuine I/O errors, and a missing directory, still propagate as fatal. File-data fsync (Files_fsync) is unchanged. --- core/src/main/c/windows/files.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 7e57de94..9805740b 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -232,11 +232,36 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS); if (fd < 0) { + // A directory can be crash-consistent yet not openable for write: + // some ACL/filesystem configurations refuse a GENERIC_WRITE open of a + // directory handle with ERROR_ACCESS_DENIED. Directory-entry durability + // on NTFS is provided by metadata journaling ($LogFile), so degrade to + // best-effort success here rather than hard-failing the SF durability + // path. open_file() already recorded the error via SaveLastError(); a + // genuine failure such as a missing directory (ERROR_PATH_NOT_FOUND) + // still propagates as fatal. + if (GetLastError() == ERROR_ACCESS_DENIED) { + return 0; + } return -1; } if (!FlushFileBuffers(FD_TO_HANDLE(fd))) { - SaveLastError(); + DWORD err = GetLastError(); CloseHandle(FD_TO_HANDLE(fd)); + // FlushFileBuffers is documented for file/volume handles; NTFS refuses + // it on a directory handle (typically ERROR_ACCESS_DENIED, and + // ERROR_INVALID_FUNCTION on filesystems that do not implement it). + // create/rename/unlink of directory entries are made crash consistent + // by NTFS metadata journaling, so treat those "not supported" + // signatures as best-effort success. This mirrors libgit2/PostgreSQL/ + // SQLite, which do not rely on directory fsync on Windows, and keeps + // the SF manifest-create, slot-lock, and trim/unlink barriers from + // hard-failing. Any other error is a real I/O failure and stays fatal. + if (err == ERROR_ACCESS_DENIED || err == ERROR_INVALID_FUNCTION) { + return 0; + } + SetLastError(err); + SaveLastError(); return -1; } CloseHandle(FD_TO_HANDLE(fd)); From bd7c3cb714f4ce3e8fd7d74285474fe54cf6f648 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 10:03:34 +0100 Subject: [PATCH 40/64] perf(qwp): binary-search sealed segments in SegmentRing.findSegmentContaining0 findSegmentContaining0 walked sealedSegments[sealedHead..size) front to back to find the segment covering an fsn: O(live-sealed) per call, and the live window is unbounded when the producer outpaces the drain. The callers are cursor (re)positioning on reconnect (findSegmentContaining / pinSegmentContaining via CursorSendEngine), so the cost was per reconnect, not per row -- but it scaled with exactly the backlog a reconnect storm tends to coincide with. The sealed list is strictly ascending and contiguous in baseSeq on every mutation path: rotation rebases the promoted spare to previous.baseSeq() + previous.frameCount() before sealing, recovery sorts by unsigned baseSeq and rejects any gap via validateContiguous, and all removals are prefix-only. The only candidate that can contain fsn is therefore the rightmost sealed segment with baseSeq <= fsn. Binary-search for it (unsigned compare, matching sortByBaseSeq's key order) and re-check that single candidate with the original containment predicate; the active-segment fallback is unchanged. O(log live-sealed), zero allocation, same nulls/misses on every input including negative fsn, empty window, and post-trim sealedHead > 0. Adds two SegmentRingTest guards pinning the full boundary matrix (segment bases, interiors, last fsns, below/above range, single-frame segments, post-trim windows); both were run green against the pre-fix linear implementation first, so the rewrite is provably semantics-preserving. --- .../qwp/client/sf/cursor/SegmentRing.java | 29 +++- .../qwp/client/sf/cursor/SegmentRingTest.java | 124 ++++++++++++++++++ 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index d0e3fbe8..9099cd10 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -1054,9 +1054,10 @@ public synchronized ObjList drainTrimmable() { * to position the I/O thread's cursor at the first unacked frame for * replay. *

        - * Walks sealed first (oldest → newest) then the active. The sealed list - * is small enough -- and reconnects are rare enough -- that the linear - * scan cost doesn't matter. + * Binary-searches the sealed list, then falls back to the active + * segment: O(log live-sealed) per call, so repositioning stays cheap + * even when a producer-outpaces-drain backlog has grown the sealed + * list without bound. */ public synchronized MmapSegment findSegmentContaining(long fsn) { return findSegmentContaining0(fsn); @@ -1469,8 +1470,26 @@ public synchronized long totalSegmentBytes() { } private MmapSegment findSegmentContaining0(long fsn) { - for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { - MmapSegment segment = sealedSegments.get(i); + // The sealed list is strictly ascending and contiguous in baseSeq: + // rotation seals the predecessor exactly where the promoted spare's + // rebased baseSeq starts, and recovery sorts the chain then rejects + // any gap via validateContiguous. The only sealed segment that can + // cover fsn is therefore the rightmost one whose baseSeq is at or + // below fsn -- binary-search for it (unsigned, matching + // sortByBaseSeq's key order) instead of walking a list that grows + // without bound while the producer outpaces the drain. + int lo = sealedHead; + int hi = sealedSegments.size() - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + if (Long.compareUnsigned(sealedSegments.get(mid).baseSeq(), fsn) <= 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + if (hi >= sealedHead) { + MmapSegment segment = sealedSegments.get(hi); long base = segment.baseSeq(); if (fsn >= base && fsn < base + segment.frameCount()) { return segment; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index 9bc08a6a..0f412a51 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; public class SegmentRingTest { @@ -869,6 +870,129 @@ public void testMaxBytesPerSegmentSurvivesOpenExisting() throws Exception { }); } + /** + * Pins findSegmentContaining across the full boundary matrix so the + * O(log N) lookup rewrite is provably semantics-preserving: empty ring, + * miss below the head segment, exact segment base, mid-segment, last FSN + * in a segment, tail sealed segment, active-segment hits, miss above the + * published range, and misses below a trimmed live head (sealedHead > 0). + * Three frames per segment give every sealed segment a distinct base, + * mid and last FSN. + */ + @Test + public void testFindSegmentContainingBoundaryMatrix() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + + 3 * (MmapSegment.FRAME_HEADER_SIZE + 8); + long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 8, 0); + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + // Empty ring: no frame published anywhere, every lookup misses. + assertNull(ring.findSegmentContaining(-1)); + assertNull(ring.findSegmentContaining(0)); + + for (int i = 0; i < 12; i++) { + if (ring.needsHotSpare()) { + ring.installHotSpare( + MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + assertEquals(i, ring.appendOrFsn(buf, 8)); + } + // Sealed: [0-2], [3-5], [6-8]; active: [9-11]. + for (long fsn = 0; fsn < 12; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn / 3 * 3, seg.baseSeq()); + } + assertSame(ring.getActive(), ring.findSegmentContaining(9)); + assertSame(ring.getActive(), ring.findSegmentContaining(11)); + assertNull(ring.findSegmentContaining(-1)); + assertNull(ring.findSegmentContaining(12)); + assertNull(ring.findSegmentContaining(Long.MIN_VALUE)); + assertNull(ring.findSegmentContaining(Long.MAX_VALUE)); + + // Trim the two fully-ACK'd oldest sealed segments: their + // FSNs must now miss while the surviving window (with a + // non-zero sealedHead) still resolves every live FSN. + ring.acknowledge(5); + MmapSegment trimmable; + int removed = 0; + while ((trimmable = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(trimmable)); + trimmable.close(); + removed++; + } + assertEquals(2, removed); + for (long fsn = 0; fsn < 6; fsn++) { + assertNull("trimmed fsn " + fsn, ring.findSegmentContaining(fsn)); + } + for (long fsn = 6; fsn < 12; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn / 3 * 3, seg.baseSeq()); + } + } + } finally { + Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + /** + * Same lookup pinned over one-frame segments, where every FSN is both a + * segment base and a segment's last FSN: 33 sealed segments (odd live + * count), then a 17-segment trim leaving an even 16-segment window with + * sealedHead well inside the backing list. Complements the three-frame + * matrix by exercising both live-window parities and a larger list. + */ + @Test + public void testFindSegmentContainingSingleFrameSegments() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + for (int i = 0; i < 34; i++) { + assertEquals(i, ring.appendOrFsn(buf, 1)); + ring.installHotSpare( + MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + // Sealed: FSNs 0..32, one frame each; active: [33]. + for (long fsn = 0; fsn <= 33; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn, seg.baseSeq()); + } + assertNull(ring.findSegmentContaining(-1)); + assertNull(ring.findSegmentContaining(34)); + + ring.acknowledge(16); + MmapSegment trimmable; + int removed = 0; + while ((trimmable = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(trimmable)); + trimmable.close(); + removed++; + } + assertEquals(17, removed); + for (long fsn = 0; fsn < 17; fsn++) { + assertNull("trimmed fsn " + fsn, ring.findSegmentContaining(fsn)); + } + for (long fsn = 17; fsn <= 33; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn, seg.baseSeq()); + } + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + private static void fillPattern(long addr, int len, int seed) { for (int i = 0; i < len; i++) { Unsafe.getUnsafe().putByte(addr + i, (byte) (seed * 31 + i + 17)); From 1374448018a5924fed57b07143c83d5fb2e3a80e Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 10:03:49 +0100 Subject: [PATCH 41/64] fix(qwp): route AckWatermark mmap/munmap through the injected FilesFacade AckWatermark.open(FilesFacade, ...) routed exists/length/openRW/ allocate/close/msync/fsync through the facade but mapped the watermark with static Files.mmap and released it with static Files.munmap. Harmless in production -- the facade defaults delegate verbatim -- but inconsistent with MmapSegment, which routes mmap/munmap through the facade, and it made watermark-mmap fault injection impossible from a test facade: the injected override was silently bypassed on the one mapping that lives for the watermark's whole lifetime. Route both calls through the facade. The instance already stores the mapping facade in a final field used by sync() and releaseStorage(), so map and unmap always go through the same facade on every path, and the error-cleanup ordering (mmap failure closes the fd via the facade; release unmaps then closes exactly once) is unchanged. Widen open(FilesFacade, String) to public, matching the established facade-factory precedent (MmapSegment.create/openExisting, SegmentRing.openExisting) in the same exported package. This lets SegmentManagerCrashConsistencyTest and SegmentManagerManifestFsyncTest drop their reflective Method lookups for compile-time-checked calls. Adds AckWatermarkTest coverage with a delegating facade: mapping and release must be observed by the injected facade, and a facade that rejects mmap must fail open() with the fd closed and no mapping left. Both tests fail pre-fix (the static call bypassed the facade), pass post-fix. --- .../qwp/client/sf/cursor/AckWatermark.java | 11 +- .../client/sf/cursor/AckWatermarkTest.java | 100 ++++++++++++++++++ .../SegmentManagerCrashConsistencyTest.java | 7 +- .../SegmentManagerManifestFsyncTest.java | 7 +- 4 files changed, 112 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index eb8381c5..b57ad546 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -136,7 +136,12 @@ public static AckWatermark open(String slotDir) { return open(FilesFacade.INSTANCE, slotDir); } - static AckWatermark open(FilesFacade filesFacade, String slotDir) { + /** + * Facade-aware variant of {@link #open(String)}. Every filesystem call, + * including the lifetime mapping, goes through {@code filesFacade} so + * tests can observe or fault-inject the watermark mmap. + */ + public static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; @@ -156,7 +161,7 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { LOG.warn("ack watermark {} could not be opened (rc={})", filePath, fd); return null; } - long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + long addr = filesFacade.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { LOG.warn("ack watermark {} could not be mmapped", filePath); filesFacade.close(fd); @@ -253,7 +258,7 @@ private boolean releaseStorage() { } isStorageReleased = true; if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); + filesFacade.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); } if (fd >= 0) { filesFacade.close(fd); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java index d9445af3..4bd8ef76 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; @@ -38,6 +39,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class AckWatermarkTest { @@ -78,6 +80,22 @@ public void testCrossSessionPersistence() throws Exception { }); } + @Test + public void testFacadeMmapFaultFailsOpenAndClosesFd() throws Exception { + // The watermark mapping must be reachable from an injected facade so + // tests can fault-inject mmap. On a rejected mapping, open() must + // fail cleanly and release the fd through the same facade. + TestUtils.assertMemoryLeak(() -> { + MappingFilesFacade ff = new MappingFilesFacade(true); + assertNull("open must fail when the injected facade rejects mmap", + AckWatermark.open(ff, slotDir)); + assertEquals("facade must receive the watermark mmap call", 1, ff.mmapCalls); + assertEquals("failed open must close the watermark fd via the facade", + 1, ff.watermarkFdCloseCalls); + assertEquals("a rejected mapping must not be munmapped", 0, ff.munmapCalls); + }); + } + @Test public void testFallsBackFromTornPlausibleHighValue() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -143,6 +161,24 @@ public void testNegativeFsnRoundTrips() throws Exception { }); } + @Test + public void testOpenAndCloseRouteMappingThroughFacade() throws Exception { + // The lifetime mapping and its release must go through the injected + // FilesFacade, matching MmapSegment, so test facades can observe them. + TestUtils.assertMemoryLeak(() -> { + MappingFilesFacade ff = new MappingFilesFacade(false); + try (AckWatermark w = AckWatermark.open(ff, slotDir)) { + assertNotNull(w); + assertEquals("open must mmap through the injected facade", 1, ff.mmapCalls); + w.write(7L); + assertEquals(7L, w.read()); + } + assertEquals("close must munmap through the injected facade", 1, ff.munmapCalls); + assertEquals("close must release the watermark fd via the facade", + 1, ff.watermarkFdCloseCalls); + }); + } + @Test public void testPhysicalReleaseIsIdempotentAcrossTestingSeamAndClose() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -241,4 +277,68 @@ public void testWriteReadInSameSession() throws Exception { } }); } + + /** + * Delegating facade that counts mmap/munmap traffic and watermark-fd + * closes, optionally rejecting the mapping to exercise the open() + * failure path. + */ + private static final class MappingFilesFacade implements FilesFacade { + private final boolean failMmap; + private int mmapCalls; + private int munmapCalls; + private int watermarkFd = -1; + private int watermarkFdCloseCalls; + + private MappingFilesFacade(boolean failMmap) { + this.failMmap = failMmap; + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { + if (fd >= 0 && fd == watermarkFd) watermarkFdCloseCalls++; + return INSTANCE.close(fd); + } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + mmapCalls++; + if (failMmap) return Files.FAILED_MMAP_ADDRESS; + return INSTANCE.mmap(fd, len, offset, flags, memoryTag); + } + @Override public void munmap(long address, long len, int memoryTag) { + munmapCalls++; + INSTANCE.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.endsWith(AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.endsWith(AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java index f6b43b7d..126c79c8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java @@ -25,7 +25,6 @@ import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Method; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -77,10 +76,8 @@ private static SegmentRing createRing(String root, long segmentSize, long payloa } } - private static AckWatermark openWatermark(FilesFacade ff, String root) throws Exception { - Method method = AckWatermark.class.getDeclaredMethod("open", FilesFacade.class, String.class); - method.setAccessible(true); - return (AckWatermark) method.invoke(null, ff, root); + private static AckWatermark openWatermark(FilesFacade ff, String root) { + return AckWatermark.open(ff, root); } private static void removeRecursive(String dir) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java index bb099824..8b45567b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java @@ -37,7 +37,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Method; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -110,10 +109,8 @@ private static void awaitTrimmed(SegmentRing ring) { } } - private static AckWatermark openWatermark(FilesFacade ff, String root) throws Exception { - Method method = AckWatermark.class.getDeclaredMethod("open", FilesFacade.class, String.class); - method.setAccessible(true); - return (AckWatermark) method.invoke(null, ff, root); + private static AckWatermark openWatermark(FilesFacade ff, String root) { + return AckWatermark.open(ff, root); } private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { From 88d6b792541d5f8bdb514a8dca482e1efaaa5b5b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 12:17:59 +0100 Subject: [PATCH 42/64] fix(qwp): zero torn-tail residue during SF segment recovery Recovery resumed appending at lastGood but left the stale bytes in [lastGood, fileSize) on disk. Two ordinary crashes could then brick the client permanently: crash #1 tears the active near its end; recovery resumes and the session fills the segment, but the last frame stops short of the old residue, so rotation reseals the recovered file with a non-zero suffix; recovery #2 correctly refuses non-zero sealed suffixes ("corrupt torn tail in sealed SF segment") -- on every startup, since the plain exception is not quarantined. Primary slot: Sender.build() fails forever; orphan slot: the drainer drops a permanent .failed sentinel and the unacked data is silently stranded. Sibling hazard: the frame envelope binds neither position nor FSN, so a byte-aligned stale frame with a valid CRC past the tear could be resurrected at a recycled FSN by a later scan (stale replay). openExisting now zeroes the residue right after the scan validates and makes the zeroes durable with an msync+fsync barrier before the segment is returned, restoring the invariant fresh segments already have: all bytes past the append cursor are zero. The barrier is load-bearing in MEMORY durability mode, where rotation does not sync the sealed predecessor's data pages. A failed barrier aborts recovery fail-closed. tornTailBytes still reports the pre-sanitization observation and the sealed-suffix check in SegmentRing stays fail-closed on first sight. Regression tests: residue zeroed on disk; two-crash reseal at segment level; stale-frame non-resurrection; barrier-failure abort; end-to-end two-crash ring recovery (torn -> recover -> fill+rotate -> recover). --- .../qwp/client/sf/cursor/MmapSegment.java | 54 ++++- .../qwp/client/sf/cursor/SegmentRing.java | 27 ++- .../qwp/client/sf/cursor/MmapSegmentTest.java | 206 ++++++++++++++++++ .../qwp/client/sf/cursor/SegmentRingTest.java | 60 +++++ 4 files changed, 330 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index e7f86651..2fab2540 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -111,7 +111,10 @@ public final class MmapSegment implements QuietCloseable { // attempted-but-invalid frame write (the suffix contains non-zero bytes). // Zero for fresh segments and for cleanly partially-filled // segments (uninitialised tail). Set only by openExisting; visible to - // recovery callers for diagnostics. Final after construction. + // recovery callers for diagnostics. Records the observation BEFORE + // sanitization -- openExisting zeroes the on-disk residue, so the file + // behind a freshly returned segment never carries it. Final after + // construction. private final long tornTailBytes; private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes, @@ -273,11 +276,18 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { *

        * If recovery observes a torn tail (the suffix after the last valid frame * contains non-zero bytes, indicating an attempted-but-failed frame write - * rather than clean unwritten space), a {@code WARN} is emitted with the byte - * count and the bytes are exposed via {@link #tornTailBytes()} so - * operators can detect silent truncation from corruption or partial - * writes. Clean partial fills (writer never attempted to write past the - * last valid frame) do not log and report {@code 0}. + * rather than clean unwritten space), a {@code WARN} is emitted with the + * byte count, the observed size is exposed via {@link #tornTailBytes()}, + * and the residue is zeroed on disk (with an msync+fsync barrier) before + * the segment is returned. Sanitizing restores the invariant every fresh + * segment already has -- all bytes past the append cursor are zero. + * Without it, residue that resumed appends do not fully overwrite would + * survive into a sealed segment, whose recovery treats a non-zero suffix + * as fatal corruption (a permanent startup failure), and a byte-aligned + * stale frame with a valid CRC could be silently resurrected at a + * recycled FSN by a later scan. Clean partial fills (writer never + * attempted to write past the last valid frame) do not log, report + * {@code 0}, and skip the barrier. */ public static MmapSegment openExisting(String path) { return openExisting(FilesFacade.INSTANCE, path); @@ -344,10 +354,33 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { + " [before=" + fileSize + ", after=" + mappedSize + ']'); } if (scan.tornTailBytes > 0) { + // Zero the residue [lastGood, fileSize) and make the zeroes + // durable before anyone can append. Resumed appends restart at + // lastGood but stop wherever the last payload fits, so stale + // residue past that point would otherwise survive a + // seal-via-rotation -- and sealed-segment recovery treats a + // non-zero suffix as fatal corruption, permanently failing + // every subsequent startup. Zeroing also stops a byte-aligned + // stale frame with a valid CRC from being resurrected at a + // recycled FSN by a later recovery scan (the frame envelope + // binds neither position nor FSN). The fsync is load-bearing: + // in MEMORY durability mode rotation does not sync the sealed + // predecessor's data pages, so an unflushed zero-write plus a + // crash would regenerate the exact poisoned state this + // sanitization removes. One-time recovery cost, torn tails + // only; a failed barrier aborts recovery (fail closed) -- + // the zeroes may still reach disk via the page cache, which + // is safe: a retry either sees them (clean tail) or re-zeroes. + Unsafe.getUnsafe().setMemory(addr + scan.lastGood, fileSize - scan.lastGood, (byte) 0); + if (ff.msync(addr, fileSize, false) != 0 || ff.fsync(fd) != 0) { + throw new MmapSegmentException( + "could not sync zeroed torn tail in " + path + + " [errno=" + Os.errno() + ']'); + } LOG.warn("SF segment {}: torn tail of {} bytes at offset {} " + "(file size {}, frames recovered {}). " - + "Recovery will overwrite this region on next append; " - + "frames past the tear (if any) are discarded. " + + "The residue has been zeroed; frames past the tear " + + "(if any) are discarded. " + "Investigate disk health or unexpected writer crash.", path, scan.tornTailBytes, scan.lastGood, fileSize, scan.frameCount); } @@ -635,7 +668,10 @@ public long frameCount() { * recovery observes non-zero bytes past the bail-out point. {@code 0} for * fresh segments, memory-backed segments, and cleanly partially-filled * recovered segments. Operators / tests can read this to tell silent - * truncation (corruption) from a normal partial fill (no incident). Sparse + * truncation (corruption) from a normal partial fill (no incident). This + * is the pre-sanitization observation: {@link #openExisting} zeroes the + * residue on disk before returning, so re-opening the same file reports + * {@code 0} and a reseal of this segment presents a clean suffix. Sparse * holes read back as zeroes; an actual positioned-read failure aborts * recovery instead of being classified as a clean tail. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 9099cd10..58b37069 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -189,9 +189,14 @@ public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, lo } /** - * Exhaustively discovers and validates an SF slot without mutation. Only - * after enumeration, opens, CRC scans, contiguity and manifest boundaries - * all succeed does it migrate a legacy chain or discard validated spares. + * Exhaustively discovers and validates an SF slot without mutating frame + * data. Only after enumeration, opens, CRC scans, contiguity and manifest + * boundaries all succeed does it migrate a legacy chain or discard + * validated spares. One deliberate exception to the no-mutation rule: + * {@link MmapSegment#openExisting} zeroes torn-tail residue -- suffix + * bytes that already failed frame validation and can never be replayed -- + * as each segment opens, so even a recovery that later fails may have + * sanitized unusable residue. Valid frames are never touched. */ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { return recover(filesFacade, sfDir, maxBytesPerSegment, null); @@ -466,6 +471,16 @@ static Recovery recover( "missing expected SF active/tail segment at base " + activeBase); } } + // Sealed members must present an all-zero suffix. Fresh + // segments are zero-allocated and openExisting zeroes any + // torn-tail residue before a recovered active can take + // appends, so a reseal of a recovered segment cannot carry + // residue here. A non-zero suffix therefore means the file + // was mutated outside the writer protocol (or is legacy + // pre-sanitization poison; this run's openExisting has + // already zeroed it, so the restart after this fail-closed + // throw re-proves the chain clean). validateContiguous above + // independently catches sealed segments that LOST frames. for (int i = 0, n = chain.size() - 1; i < n; i++) { if (chain.get(i).tornTailBytes() > 0) { throw new MmapSegmentException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); @@ -1327,11 +1342,7 @@ public synchronized void releasePinnedSegmentForTest(MmapSegment expected) { } private synchronized boolean requestSyncBeforeRotation(MmapSegment previous) { - if (periodicSyncEnabled && !previous.isPublishedDurable()) { - syncRequested = true; - return true; - } - return false; + return false; // MUTANT: gate neutralized } /** Releases the I/O cursor pin and wakes trim if it still names {@code expected}. */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 3408a379..386b6eef 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -491,6 +491,212 @@ public void testRecoveryDoesNotFlagFreshUnusedSegment() throws Exception { }); } + @Test + public void testRecoveryZeroesTornTailResidueOnDisk() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Recovery must not only REPORT the torn tail, it must sanitize + // it: the first open still reports the observed residue (operator + // signal), but a second open of the untouched file must find a + // clean zero suffix. Pre-fix the residue survived forever. + String path = tmpDir + "/seg-zeroed.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + long lastGood; + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + for (int i = 0; i < 3; i++) { + fillPattern(buf, 16, i); + seg.tryAppend(buf, 16); + } + lastGood = seg.publishedOffset(); + long addr = seg.address(); + for (long off = lastGood; off + 4 <= 4096; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + seg.msync(); + } + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("first recovery must report the observed residue", + 4096L - lastGood, seg.tornTailBytes()); + } + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("first recovery must have zeroed the residue on disk", + 0L, seg.tornTailBytes()); + assertEquals(lastGood, seg.publishedOffset()); + assertEquals(3L, seg.frameCount()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testResealGapResidueCannotSurviveRecoveryAppends() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Two-crash reseal regression at segment level. Crash #1 leaves + // residue up to the file end; recovery resumes at lastGood; the + // resumed writer fills the segment but its last frame stops short + // of the file end (the remaining gap cannot fit another frame). + // Pre-fix the stale residue survived in that gap, so the segment + // -- sealed as-is by rotation -- failed the sealed-suffix-must- + // be-zero check on the NEXT recovery, permanently failing startup. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // reseal gap: a 5th 24-byte frame can never fit + String path = tmpDir + "/seg-reseal.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + // Session 1: two frames, then a crash mid-write near the end + // -- garbage from the torn frame all the way to the file end. + try (MmapSegment seg = MmapSegment.create(path, 0L, segSize)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); + fillPattern(buf, 16, 1); + seg.tryAppend(buf, 16); + long addr = seg.address(); + for (long off = seg.publishedOffset(); off + 4 <= segSize; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + seg.msync(); + } + // Recovery #1 + session 2: fill the segment to its rotation + // point. The 4th frame ends 12 bytes short of the file end -- + // a region session 2 never overwrites. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(2L, seg.frameCount()); + fillPattern(buf, 16, 2); + assertTrue(seg.tryAppend(buf, 16) >= 0); + fillPattern(buf, 16, 3); + assertTrue(seg.tryAppend(buf, 16) >= 0); + assertEquals("next append must not fit (rotation would seal here)", + -1L, seg.tryAppend(buf, 16)); + seg.msync(); + } + // Recovery #2: the state a sealed segment presents at the next + // startup. Its suffix must be clean or ring recovery bricks. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("all four frames must recover", 4L, seg.frameCount()); + assertEquals("no residue may survive in the reseal gap: a sealed " + + "segment with a non-zero suffix permanently fails " + + "ring recovery", + 0L, seg.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testDiscardedStaleFrameNotResurrectedByLaterRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The frame envelope [crc(len|payload)][len][payload] binds + // neither position nor FSN, so a stale frame with a valid CRC + // sitting past the tear -- byte-aligned with the resumed writer's + // frames, natural with fixed-size records -- would be silently + // re-adopted by the next recovery scan at a recycled FSN. + // Recovery #1 discarded it; recovery #2 must not bring it back. + String path = tmpDir + "/seg-stale.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + long frameB; + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); // frame A, FSN 0 + fillPattern(buf, 16, 1); + frameB = seg.tryAppend(buf, 16); // frame B, FSN 1 + fillPattern(buf, 16, 2); + seg.tryAppend(buf, 16); // frame C, FSN 2 + // The crash tears frame B only: flip its CRC. C keeps a + // valid CRC at a frame-aligned offset past the tear. + long addr = seg.address(); + int crc = Unsafe.getUnsafe().getInt(addr + frameB); + Unsafe.getUnsafe().putInt(addr + frameB, crc ^ 0x5A5A5A5A); + seg.msync(); + } + // Recovery #1: B fails CRC, so the scan stops at B; B and C + // are discarded (and, with sanitization, zeroed). + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("scan must stop at the torn frame", 1L, seg.frameCount()); + // The resumed writer re-issues FSN 1 with a fresh payload + // of the old B's exact size -- the byte-aligned case. + fillPattern(buf, 16, 7); + assertEquals(frameB, seg.tryAppend(buf, 16)); + seg.msync(); + } + // Recovery #2: pre-fix the scan walked A, B' and then adopted + // the STALE C (valid CRC) as a live frame -- data recovery #1 + // had already discarded, resurrected behind the engine's back. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("stale frame C must stay discarded, not be " + + "resurrected at a recycled FSN", 2L, seg.frameCount()); + assertEquals(0L, seg.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testTornTailZeroingSyncFailureAbortsRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Sanitization is load-bearing: if the zeroes cannot be made + // durable, recovery must fail closed rather than hand back a + // segment whose reseal could permanently fail the next startup. + // A failed attempt may still leave zeroes in the page cache; + // that is safe (a retry either sees a clean tail or re-zeroes), + // so the follow-up open with a healthy facade must succeed. + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + String msyncPath = tmpDir + "/seg-zero-msync-fail.sfa"; + String fsyncPath = tmpDir + "/seg-zero-fsync-fail.sfa"; + for (String path : new String[]{msyncPath, fsyncPath}) { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); + long addr = seg.address(); + Unsafe.getUnsafe().putInt(addr + seg.publishedOffset(), 0xCAFEBABE); + Unsafe.getUnsafe().putInt(addr + seg.publishedOffset() + 4, 16); + seg.msync(); + } + } + + FaultyFilesFacade msyncFailure = new FaultyFilesFacade(); + msyncFailure.failOnMsync = true; + try { + MmapSegment.openExisting(msyncFailure, msyncPath).close(); + fail("expected recovery to abort when the zeroed tail cannot be msync'd"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("zeroed torn tail")); + } + assertEquals(1, msyncFailure.msyncCalls); + assertEquals(0, msyncFailure.fsyncCalls); + try (MmapSegment seg = MmapSegment.openExisting(msyncPath)) { + assertEquals(1L, seg.frameCount()); + } + + FaultyFilesFacade fsyncFailure = new FaultyFilesFacade(); + fsyncFailure.failOnFsync = true; + try { + MmapSegment.openExisting(fsyncFailure, fsyncPath).close(); + fail("expected recovery to abort when the zeroed tail cannot be fsync'd"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("zeroed torn tail")); + } + assertEquals(1, fsyncFailure.msyncCalls); + assertEquals(1, fsyncFailure.fsyncCalls); + try (MmapSegment seg = MmapSegment.openExisting(fsyncPath)) { + assertEquals(1L, seg.frameCount()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testRecoverySignalsTornTailWithByteCount() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index 0f412a51..943b45c3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -331,6 +331,66 @@ public void testOpenExistingRecoversActivePlusSealed() throws Exception { }); } + @Test + public void testResealedTornRecoveredSegmentDoesNotBrickNextRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // End-to-end two-crash regression: + // crash #1 -> torn active with residue up to the file end; + // recovery #1 -> resume in it, fill it, rotate (reseals the + // recovered file with a tail gap the resumed appends never + // overwrote); + // crash #2 -> recovery #2 used to throw "corrupt torn tail in + // sealed SF segment" on EVERY startup, because nothing ever + // sanitized crash #1's residue and the sealed-suffix check + // correctly refuses non-zero sealed tails. openExisting now + // zeroes the residue at recovery #1, so recovery #2 must + // succeed and see the full chain. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // reseal gap: a 5th 24-byte frame can never fit + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + // Session 1 "crashes": two good frames, then garbage from the + // torn write all the way to the file end. + try (MmapSegment s0 = MmapSegment.create(tmpDir + "/r0.sfa", 0, segSize)) { + s0.tryAppend(buf, 16); + s0.tryAppend(buf, 16); + long addr = s0.address(); + for (long off = s0.publishedOffset(); off + 4 <= segSize; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + s0.msync(); + } + // Recovery #1 + session 2: resume in the torn segment, fill + // it, rotate into a spare. The recovered file is resealed + // with its 12-byte gap intact. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull(ring); + assertEquals(2, ring.nextSeqHint()); + assertEquals(2, ring.appendOrFsn(buf, 16)); + assertEquals(3, ring.appendOrFsn(buf, 16)); + ring.installHotSpare(MmapSegment.create(tmpDir + "/r1.sfa", 4, segSize)); + assertEquals("append must rotate into the spare", + 4, ring.appendOrFsn(buf, 16)); + assertEquals(1, ring.getSealedSegments().size()); + } + // Recovery #2 ("crash" #2): must not brick on the resealed + // recovered segment. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull("second recovery must survive the resealed segment", ring); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(0, ring.getSealedSegments().get(0).baseSeq()); + assertEquals(4, ring.getSealedSegments().get(0).frameCount()); + assertEquals(4, ring.getActive().baseSeq()); + assertEquals(1, ring.getActive().frameCount()); + assertEquals(5, ring.nextSeqHint()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testOpenExistingDetectsFsnGap() throws Exception { TestUtils.assertMemoryLeak(() -> { From 71cfbe2e01e06d6dff0ae3cae925d80ea1a9dfe5 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 21 Jul 2026 12:38:05 +0100 Subject: [PATCH 43/64] fix(qwp): restore requestSyncBeforeRotation gate clobbered by stray edit Commit 88d6b792 accidentally captured a concurrent tooling edit that neutralized requestSyncBeforeRotation to 'return false' (marked 'MUTANT: gate neutralized'): a mutation-testing session shared this worktree and injected the edit between test validation and git add. The neutralized gate allowed rotation in PERIODIC durability mode to proceed without requesting a sync for a predecessor whose published range was not yet durable. This restores the original logic byte-for-byte; the worktree now matches exactly the change set the 369-test sf suite validated. SegmentManagerPeriodicSyncTest, SegmentManagerCrashConsistencyTest, SegmentRingTest and MmapSegmentTest re-run green on the restoration. --- .../client/cutlass/qwp/client/sf/cursor/SegmentRing.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 58b37069..c9e01303 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -1342,7 +1342,11 @@ public synchronized void releasePinnedSegmentForTest(MmapSegment expected) { } private synchronized boolean requestSyncBeforeRotation(MmapSegment previous) { - return false; // MUTANT: gate neutralized + if (periodicSyncEnabled && !previous.isPublishedDurable()) { + syncRequested = true; + return true; + } + return false; } /** Releases the I/O cursor pin and wakes trim if it still names {@code expected}. */ From 80ad4ca6bb953a320b6881e6c19c5d7734c9f468 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 01:17:17 +0100 Subject: [PATCH 44/64] fix(qwp): harden orphan recovery and metadata crash safety --- .../qwp/client/QwpWebSocketSender.java | 13 +- .../qwp/client/sf/cursor/AckWatermark.java | 13 +- .../client/sf/cursor/BackgroundDrainer.java | 75 ++++- .../client/sf/cursor/CursorSendEngine.java | 4 +- .../sf/cursor/MmapSegmentException.java | 10 +- .../qwp/client/sf/cursor/SegmentManager.java | 4 +- .../qwp/client/sf/cursor/SegmentRing.java | 22 +- .../qwp/client/sf/cursor/SfManifest.java | 14 +- .../sf/cursor/SfOperationalException.java | 38 +++ .../client/sf/cursor/SfRecoveryException.java | 38 +++ .../qwp/client/sf/cursor/SlotLock.java | 70 +++- .../cursor/SlotLockContentionException.java | 37 +++ .../client/sf/cursor/AckWatermarkTest.java | 32 ++ .../BackgroundDrainerDurableAckRetryTest.java | 4 +- .../BackgroundDrainerSetupFailureTest.java | 313 ++++++++++++++++++ .../SegmentManagerUnlinkFailureTest.java | 3 +- .../cursor/SegmentRecoveryIntegrityTest.java | 90 ++++- .../qwp/client/sf/cursor/SlotLockTest.java | 103 ++++++ 18 files changed, 808 insertions(+), 75 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 89bdf48f..699a903b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -39,6 +39,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerPool; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; @@ -2330,7 +2331,7 @@ public synchronized void setDrainerListener(BackgroundDrainerListener listener) pool.setListener(listener); // ...and direct re-assignment for the ones already running (the // pool listener is only applied at submit time, never after). - ObjList live = + ObjList live = pool.snapshot(); for (int i = 0, n = live.size(); i < n; i++) { live.getQuick(i).setListener(listener); @@ -2473,16 +2474,16 @@ public synchronized void startOrphanDrainers( // (the factory needs the drainer, the drainer's constructor // needs the factory); the ref write happens-before the drainer // runs because submit() publishes the task afterwards. - final io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer[] ref = - new io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer[1]; + final BackgroundDrainer[] ref = + new BackgroundDrainer[1]; ReconnectSupplier factory = new ReconnectSupplier( () -> { - io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer d = ref[0]; + BackgroundDrainer d = ref[0]; return d != null && d.isStopRequested(); }, "drainer stop requested during connect"); - io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer drainer = - new io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer( + BackgroundDrainer drainer = + new BackgroundDrainer( slot, segmentSizeBytes, sfMaxTotalBytes, syncIntervalNanos, factory, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index b57ad546..b11ae561 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -45,8 +45,10 @@ * "everything <= N is durable"), so a single monotonic watermark suffices; * no per-frame bitmap is needed. *

        - * Layout (128 bytes, little-endian, mmap'd for the engine's lifetime): - * two independently CRC-protected 64-byte records. Each record contains: + * Layout (8192 bytes, little-endian, mmap'd for the engine's lifetime): + * two independently CRC-protected 64-byte records at offsets 0 and 4096. + * Placing each record at the start of a separate 4 KiB slot prevents one + * aligned 512-byte or 4 KiB sector tear from damaging both. Each record contains: * 

          *   offset 0:   u32 magic = 'AKW1'
  *   offset 4:   u32 version = 1
@@ -84,7 +86,7 @@ public final class AckWatermark implements QuietCloseable {
      * directory enumerators that filter by extension skip it automatically.
      */
     public static final String FILE_NAME = ".ack-watermark";
-    public static final int FILE_SIZE = 128;
+    public static final int FILE_SIZE = 8 * 1024;
     /**
      * Sentinel returned by {@link #read()} when neither watermark record is
      * valid.
@@ -96,6 +98,7 @@ public final class AckWatermark implements QuietCloseable {
     private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class);
     private static final int MAGIC_OFFSET = 0;
     private static final int RECORD_SIZE = 64;
+    private static final int RECORD_SLOT_SIZE = 4 * 1024;
     private static final int VERSION = 1;
     private final int fd;
     private final FilesFacade filesFacade;
@@ -240,7 +243,7 @@ public void sync() {
     public void write(long fsn) {
         if (closed) return;
         long nextGeneration = generation + 1L;
-        long recordAddress = mmapAddress + (nextGeneration & 1L) * RECORD_SIZE;
+        long recordAddress = mmapAddress + (nextGeneration & 1L) * RECORD_SLOT_SIZE;
         Unsafe.getUnsafe().setMemory(recordAddress, RECORD_SIZE, (byte) 0);
         Unsafe.getUnsafe().putInt(recordAddress + MAGIC_OFFSET, FILE_MAGIC);
         Unsafe.getUnsafe().putInt(recordAddress + 4, VERSION);
@@ -286,7 +289,7 @@ private static Record readRecord(long address) {
 
     private static Record selectRecord(long address) {
         Record first = readRecord(address);
-        Record second = readRecord(address + RECORD_SIZE);
+        Record second = readRecord(address + RECORD_SLOT_SIZE);
         if (first == null) {
             return second;
         }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index 11128a9b..997eb9b9 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -56,12 +56,15 @@
  *
      *

      * On terminal failure (auth-rejection on reconnect, a cluster-wide durable-ack - * capability gap that exhausts its settle budget, recovery error), the drainer - * drops a {@link OrphanScanner#FAILED_SENTINEL_NAME} sentinel into the slot - * before exiting. Future scans skip the slot until an operator clears the - * sentinel — bounded automatic retry, then human-in-the-loop. A transient - * all-replica failover window is NOT terminal: it is retried indefinitely - * (Invariant B), never quarantined on a wall-clock budget or attempt cap. + * capability gap that exhausts its settle budget, corrupt or incomplete durable + * recovery state), the drainer drops a + * {@link OrphanScanner#FAILED_SENTINEL_NAME} sentinel into the slot before + * exiting. Future scans skip the slot until an operator clears the sentinel — + * bounded automatic retry, then human-in-the-loop. Operational setup failures + * leave no sentinel so a later orphan scan can retry. JVM/programming Errors + * also leave no sentinel and propagate after teardown. A transient all-replica + * failover window is NOT terminal: it is retried indefinitely (Invariant B), + * never quarantined on a wall-clock budget or attempt cap. */ public final class BackgroundDrainer implements Runnable { @@ -406,9 +409,9 @@ public WebSocketClient connectWithDurableAckRetry() { // retrying cannot clear it, and spinning here would pin // the slot .lock forever with no .failed sentinel and only // a throttled, possibly-null-message WARN as a trace. - // Rethrow: run()'s outer catch quarantines the slot - // (markFailed + FAILED) and its finally releases the lock - // -- quarantine-and-exit, exactly as genuine terminals do. + // Rethrow: run() records the failure without attempting + // allocation-heavy logging or a .failed write, its finally + // releases the lock, and then the Error remains visible. throw (Error) t; } // INVARIANT B: a transport failure -- the whole cluster is @@ -560,15 +563,31 @@ public void run() { engine = new CursorSendEngine(slotPath, segmentSizeBytes, sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, syncIntervalNanos); - } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { - LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); - outcome = DrainOutcome.LOCKED_BY_OTHER; - return; - } + } catch (SlotLockContentionException t) { + LOG.info("orphan slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } catch (SfRecoveryException | MmapSegmentCorruptionException t) { + // The durable chain itself is proven corrupt or incomplete. + // Repeated scans cannot repair it, so preserve the terminal + // quarantine path in the outer catch below. throw t; + } catch (Exception t) { + // Every other pre-publication construction exception is + // retryable: setup I/O, resource pressure, unexpected setup + // faults, and future operational failures do not prove durable + // data corruption. The constructor has closed partial + // resources; SlotLock retains and retries any unconfirmed + // unlock. Leave no .failed sentinel for the next orphan scan. + // Error deliberately escapes to the outer Error path: it is + // observable after teardown but cannot quarantine intact data. + String msg = t.getMessage(); + LOG.warn("drainer setup temporarily unavailable for slot {}: {}", + slotPath, msg); + lastErrorMessage = msg; + outcome = DrainOutcome.FAILED; + return; } long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { @@ -618,6 +637,16 @@ public void run() { try { loop.checkError(); } catch (Throwable t) { + // The I/O loop latches a JVM/programming Error inside a + // LineSenderException so checkError() can cross the + // thread boundary. Preserve the original Error contract: + // no wire quarantine, tear down, then propagate it. + if (t instanceof Error) { + throw (Error) t; + } + if (t.getCause() instanceof Error) { + throw (Error) t.getCause(); + } if (loop.capabilityGapTerminal() != null) { // Capability gap mid-drain: recycle the wire, NOT // the slot. connectWithDurableAckRetry() owns the @@ -673,6 +702,18 @@ public void run() { // outer condition, which is false for the same reason. } outcome = DrainOutcome.STOPPED; + } catch (Error t) { + // Resource pressure and JVM/programming failures prove neither + // durable corruption nor a terminal server response. Log the Error + // best-effort, but never let a secondary logging failure (especially + // under OOME) mask the original Error or prevent teardown. + try { + LOG.error("drainer failed with Error for slot {}", slotPath, t); + } catch (Throwable ignored) { + } + lastErrorMessage = t.getMessage(); + outcome = DrainOutcome.FAILED; + throw t; } catch (Throwable t) { String msg = t.getMessage(); if (slotPath != null) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 2c4dd422..80f54486 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -383,7 +383,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // the file cannot be opened or mapped. watermarkInProgress = AckWatermark.open(filesFacade, sfDir); if (watermarkInProgress == null) { - throw new IllegalStateException( + throw new SfOperationalException( "could not open required ack watermark for SF slot " + sfDir); } long baseSeed = lowestBase - 1; @@ -437,7 +437,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man AckWatermark.removeOrphan(filesFacade, sfDir); watermarkInProgress = AckWatermark.open(filesFacade, sfDir); if (watermarkInProgress == null) { - throw new IllegalStateException( + throw new SfOperationalException( "could not open required ack watermark for SF slot " + sfDir); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index dda38a8e..03751c80 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -30,11 +30,13 @@ * the disk is full (the latter surfaces as backpressure on the producer * via {@link io.questdb.client.cutlass.line.LineSenderException}). *

      - * Recovery distinguishes two flavors: this base type marks operational + * Recovery distinguishes three flavors: this base type marks operational * failures (open/read/mmap/enumeration errors — the file's contents may be fine, - * so recovery must fail closed), while the {@link MmapSegmentCorruptionException} - * subtype marks positively-identified corruption in the file's own - * bytes, which recovery may quarantine instead of aborting. + * so recovery must fail closed); {@link MmapSegmentCorruptionException} marks + * positively-identified corruption in one file's own bytes, which + * recovery may quarantine when the surviving chain proves safe; and + * {@link SfRecoveryException} marks a terminal chain failure that needs + * operator intervention. */ public class MmapSegmentException extends RuntimeException { public MmapSegmentException(String message) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index dab9d099..f6705f7e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -763,7 +763,7 @@ private long scanMaxGeneration(String dir) { if (!filesFacade.exists(dir)) return max; long find = filesFacade.findFirst(dir); if (find < 0) { - throw new IllegalStateException("could not enumerate SF segment directory " + dir); + throw new SfOperationalException("could not enumerate SF segment directory " + dir); } if (find == 0) return max; try { @@ -784,7 +784,7 @@ private long scanMaxGeneration(String dir) { } } if (rc < 0) { - throw new IllegalStateException("could not fully enumerate SF segment directory " + dir); + throw new SfOperationalException("could not fully enumerate SF segment directory " + dir); } } finally { filesFacade.findClose(find); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c9e01303..9e1a4c4f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -273,7 +273,7 @@ static Recovery recover( // now unreadable) -- fail without mutating. Without one, // legacy semantics apply: quarantine and start fresh. if (manifest != null) { - throw new MmapSegmentException("every SF segment in " + sfDir + throw new SfRecoveryException("every SF segment in " + sfDir + " is corrupt but " + SfManifest.FILE_NAME + " references durable data"); } @@ -294,7 +294,7 @@ static Recovery recover( long manifestHeadBase = manifest.headBase(); long manifestActiveBase = manifest.activeBase(); if (manifestHeadBase != manifestActiveBase) { - throw new MmapSegmentException(SfManifest.FILE_NAME + " in " + sfDir + throw new SfRecoveryException(SfManifest.FILE_NAME + " in " + sfDir + " references durable data (headBase=" + manifestHeadBase + ", activeBase=" + manifestActiveBase + ") but no segment files exist"); @@ -328,7 +328,7 @@ static Recovery recover( } sortByBaseSeq(data, 0, data.size()); if (manifest == null && requiresManifest) { - throw new MmapSegmentException("new-format SF segment exists but " + throw new SfRecoveryException("new-format SF segment exists but " + SfManifest.FILE_NAME + " is missing"); } @@ -384,19 +384,19 @@ static Recovery recover( long end = segment.baseSeq() + segment.frameCount(); if (segment.baseSeq() < headBase) { if (end > headBase) { - throw new MmapSegmentException("segment overlaps committed SF head boundary"); + throw new SfRecoveryException("segment overlaps committed SF head boundary"); } continue; // acknowledged stale file after manifest-before-unlink crash } if (segment.baseSeq() > activeBase) { - throw new MmapSegmentException("segment exists beyond committed SF active boundary"); + throw new SfRecoveryException("segment exists beyond committed SF active boundary"); } chain.add(segment); } if (chain.size() > 0) { validateContiguous(chain); if (chain.get(0).baseSeq() != headBase) { - throw new MmapSegmentException("missing expected SF head segment at base " + headBase); + throw new SfRecoveryException("missing expected SF head segment at base " + headBase); } } active = findActive(all, activeBase); @@ -438,7 +438,7 @@ static Recovery recover( } return Recovery.empty(); } - throw new MmapSegmentException("missing expected SF active segment at base " + activeBase); + throw new SfRecoveryException("missing expected SF active segment at base " + activeBase); } if (chain.size() == 0) { if (headBase != activeBase || active.frameCount() != 0 || corruptPaths != null) { @@ -448,7 +448,7 @@ static Recovery recover( // the same provisional baseSeq as a corrupted real // active -- accepting it would quarantine unacked // frames and re-issue their FSNs. Fail closed. - throw new MmapSegmentException( + throw new SfRecoveryException( "missing SF chain between committed boundaries" + (corruptPaths != null ? " (a corrupt segment prevents proving the empty state)" : "")); @@ -467,7 +467,7 @@ static Recovery recover( // empty-chain acceptance above). chain.add(active); } else { - throw new MmapSegmentException( + throw new SfRecoveryException( "missing expected SF active/tail segment at base " + activeBase); } } @@ -483,7 +483,7 @@ static Recovery recover( // independently catches sealed segments that LOST frames. for (int i = 0, n = chain.size() - 1; i < n; i++) { if (chain.get(i).tornTailBytes() > 0) { - throw new MmapSegmentException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); + throw new SfRecoveryException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); } } for (int i = 0, n = chain.size(); i < n; i++) { @@ -738,7 +738,7 @@ private static void validateContiguous(ObjList segments) { MmapSegment current = segments.get(i); long expected = previous.baseSeq() + previous.frameCount(); if (current.baseSeq() != expected) { - throw new MmapSegmentException("FSN gap in recovered segments: expected " + throw new SfRecoveryException("FSN gap in recovered segments: expected " + expected + " but got " + current.baseSeq()); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java index 39e2e208..b253075a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -34,17 +34,19 @@ /** * Crash-safe boundary record for an SF segment chain. Two fixed-size, - * independently CRC-protected records alternate on update. Recovery selects - * the valid record with the greatest generation, so a torn update cannot - * erase the previous committed head/active boundary. + * independently CRC-protected records at offsets 0 and 4096 alternate on + * update. Recovery selects the valid record with the greatest generation. + * The separate 4 KiB slots prevent one aligned 512-byte or 4 KiB sector tear + * from erasing both the update and the previous committed boundary. */ final class SfManifest implements QuietCloseable { static final String FILE_NAME = "sf-manifest.bin"; private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class); private static final int CRC_OFFSET = 60; - private static final long FILE_SIZE = 128; + private static final long FILE_SIZE = 8 * 1024; private static final int MAGIC = 0x314d4653; // SFM1 little-endian private static final int RECORD_SIZE = 64; + private static final int RECORD_SLOT_SIZE = 4 * 1024; private static final int VERSION = 1; private final int fd; private final FilesFacade filesFacade; @@ -125,7 +127,7 @@ static SfManifest open(FilesFacade filesFacade, String dir) { long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); try { Record first = readRecord(filesFacade, fd, buffer, 0); - Record second = readRecord(filesFacade, fd, buffer, RECORD_SIZE); + Record second = readRecord(filesFacade, fd, buffer, RECORD_SLOT_SIZE); Record selected; if (first == null) { selected = second; @@ -234,7 +236,7 @@ synchronized void update(long newHeadBase, long newActiveBase) { Unsafe.getUnsafe().putLong(writeScratch + 24, newActiveBase); int crc = Crc32c.update(Crc32c.INIT, writeScratch, CRC_OFFSET); Unsafe.getUnsafe().putInt(writeScratch + CRC_OFFSET, crc); - long offset = (nextGeneration & 1L) * RECORD_SIZE; + long offset = (nextGeneration & 1L) * RECORD_SLOT_SIZE; if (filesFacade.write(fd, writeScratch, RECORD_SIZE, offset) != RECORD_SIZE) { throw new MmapSegmentException("short write updating SF manifest " + path); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java new file mode 100644 index 00000000..308b99ee --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java @@ -0,0 +1,38 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Operational SF setup failure that may clear on a later retry. Extending + * {@link IllegalStateException} preserves the existing caller-visible + * construction-failure contract while giving internal recovery code a typed + * signal that must not permanently quarantine an orphan slot. + */ +public final class SfOperationalException extends IllegalStateException { + + public SfOperationalException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java new file mode 100644 index 00000000..21ce1cf4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java @@ -0,0 +1,38 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Terminal SF recovery failure. The on-disk state proves that the durable + * segment chain is corrupt or incomplete and requires operator intervention. + * Operational filesystem failures continue to use {@link MmapSegmentException} + * so callers can retry them without quarantining otherwise recoverable data. + */ +public final class SfRecoveryException extends MmapSegmentException { + + public SfRecoveryException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 765fc215..c7615e7c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -60,8 +60,12 @@ public final class SlotLock implements QuietCloseable { private static final int DEAD_FD_FOR_TESTING = 1_000_000_000; private static final String LOCK_FILE_NAME = ".lock"; private static final String LOCK_PID_FILE_NAME = ".lock.pid"; + private static final Object RELEASE_RETRY_LOCK = new Object(); + private static SlotLock releaseRetryHead; private final String slotDir; private int fd; + private boolean isReleaseRetryPending; + private SlotLock releaseRetryNext; private SlotLock(String slotDir, int fd) { this.slotDir = slotDir; @@ -73,8 +77,8 @@ private SlotLock(String slotDir, int fd) { * acquires an exclusive {@code flock} on it. On contention, reads the * existing PID payload and throws with a descriptive message. * - * @throws IllegalStateException on dir-create failure, file-open failure, - * or lock contention. + * @throws SfOperationalException on directory or lock-file setup failure + * @throws SlotLockContentionException on lock contention */ public static SlotLock acquire(String slotDir) { return acquire(slotDir, false); @@ -88,22 +92,28 @@ public static SlotLock acquire(String slotDir, boolean syncParentDirectory) { if (slotDir == null || slotDir.isEmpty()) { throw new IllegalArgumentException("slotDir must not be empty"); } + // Construction cleanup may have retained locks after explicit unlock + // failures. Drive every pending owner before opening a new descriptor. + // Path text cannot identify a physical file portably (symlinks and + // Windows case aliases are counterexamples), while the pending list is + // cold, error-only state and normally empty. + retryPendingReleases(); if (!Files.exists(slotDir)) { int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT); if (rc != 0) { - throw new IllegalStateException( + throw new SfOperationalException( "could not create slot dir: " + slotDir + " rc=" + rc); } } if (syncParentDirectory && Files.fsyncParentDir(slotDir) != 0) { - throw new IllegalStateException( + throw new SfOperationalException( "could not sync parent directory for SF slot: " + slotDir); } String lockPath = slotDir + "/" + LOCK_FILE_NAME; String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME; int fd = Files.openRW(lockPath); if (fd < 0) { - throw new IllegalStateException( + throw new SfOperationalException( "could not open slot lock file: " + lockPath); } boolean ok = false; @@ -111,7 +121,7 @@ public static SlotLock acquire(String slotDir, boolean syncParentDirectory) { int rc = Files.lock(fd); if (rc != 0) { String holder = readHolder(pidPath); - throw new IllegalStateException( + throw new SlotLockContentionException( "sf slot already in use by another process [slot=" + slotDir + ", holder=" + holder + "]"); } @@ -185,9 +195,16 @@ public synchronized boolean release() { @Override public void close() { - // QuietCloseable contract: best-effort, no signal. Callers that - // must confirm the release use release() and check the result. - release(); + // QuietCloseable cannot report a failure, so retain this object on an + // allocation-free retry list when unlock is unconfirmed. Serialize the + // release attempt and publication: an acquire that starts after close + // returns must not miss the retained owner. An acquire already racing + // an in-progress close may still observe ordinary lock contention. + synchronized (RELEASE_RETRY_LOCK) { + if (!release()) { + retainForReleaseRetryLocked(); + } + } } private static String readHolder(String pidPath) { @@ -217,6 +234,14 @@ private static String readHolder(String pidPath) { private static native int release0(int fd); + private void retainForReleaseRetryLocked() { + if (!isReleaseRetryPending) { + isReleaseRetryPending = true; + releaseRetryNext = releaseRetryHead; + releaseRetryHead = this; + } + } + private synchronized void restoreFdForTesting(int savedFd) { if (fd != DEAD_FD_FOR_TESTING) { throw new IllegalStateException("slot lock release failure is not injected"); @@ -224,6 +249,33 @@ private synchronized void restoreFdForTesting(int savedFd) { fd = savedFd; } + private static void retryPendingReleases() { + synchronized (RELEASE_RETRY_LOCK) { + SlotLock previous = null; + SlotLock lock = releaseRetryHead; + while (lock != null) { + SlotLock next = lock.releaseRetryNext; + // release() reports operational unlock failure as false. Do + // not catch Error or unexpected programming failures here: + // hiding them as apparent lock contention would misdiagnose + // the process and create a new retry contract for VM errors. + if (lock.release()) { + if (previous == null) { + releaseRetryHead = next; + } else { + previous.releaseRetryNext = next; + } + lock.isReleaseRetryPending = false; + lock.releaseRetryNext = null; + lock = next; + continue; + } + previous = lock; + lock = next; + } + } + } + private static void writePid(String pidPath) { long pid; try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java new file mode 100644 index 00000000..6dc98f67 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java @@ -0,0 +1,37 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Signals that another sender or drainer currently owns an SF slot lock. + * This expected contention condition is distinct from operational failures + * that prevent the lock file from being opened or created. + */ +public final class SlotLockContentionException extends IllegalStateException { + + public SlotLockContentionException(String message) { + super(message); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java index 4bd8ef76..848305b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java @@ -239,6 +239,38 @@ public void testRepeatedWriteUpdatesValue() throws Exception { }); } + @Test + public void testSingleSectorTearLeavesPriorRecordRecoverable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + watermark.write(100L); + watermark.write(200L); + watermark.sync(); + } + + String path = slotDir + "/" + AckWatermark.FILE_NAME; + int fd = Files.openRW(path); + assertTrue(fd >= 0); + int tornBytes = (int) Math.min(512L, Files.length(path)); + long tornSector = Unsafe.malloc(tornBytes, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(tornSector, tornBytes, (byte) 0xA5); + assertEquals(tornBytes, Files.write(fd, tornSector, tornBytes, 0)); + assertEquals(0, Files.fsync(fd)); + } finally { + Files.close(fd); + Unsafe.free(tornSector, tornBytes, MemoryTag.NATIVE_DEFAULT); + } + + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + assertEquals("one aligned 512-byte tear must leave the prior record valid", + 100L, watermark.read()); + } + }); + } + @Test public void testStaleFileWithWrongSizeIsResetOnOpen() throws Exception { // A leftover file with an unexpected size (corruption, partial diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java index 0a6b69f8..ad9f5930 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java @@ -511,8 +511,8 @@ public void testJvmErrorEscapesConnectRetryLoop() throws Exception { // .lock forever with no .failed sentinel and only a throttled WARN as // a trace. A JVM/programming failure is not a transport outage: // retrying cannot clear it, so it must escape the loop on the FIRST - // sweep. run()'s outer catch then quarantines the slot (markFailed + - // FAILED) and its finally releases the lock -- quarantine-and-exit. + // sweep. run() records FAILED without quarantining recoverable data, + // releases the lock in finally, and propagates the Error. CountingListener listener = new CountingListener(); ScriptedFactory factory = ScriptedFactory.alwaysFailing( () -> new LinkageError("simulated JVM failure")); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java new file mode 100644 index 00000000..0e9d20e1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java @@ -0,0 +1,313 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicInteger; + +public class BackgroundDrainerSetupFailureTest { + + private static final long SEGMENT_BYTES = 1L << 20; + private String slotPath; + + @Before + public void setUp() { + slotPath = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-drainer-setup-failure-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(slotPath, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + removeRecursive(slotPath); + } + + @Test + public void testConnectErrorPropagatesWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + LinkageError injected = new LinkageError("injected drainer connect error"); + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw injected; + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + LinkageError thrown = null; + try { + drainer.run(); + } catch (LinkageError e) { + thrown = e; + } + + Assert.assertSame("post-publication Error must remain visible to the caller", + injected, thrown); + Assert.assertEquals("connect error must occur on the first attempt", + 1, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "injected drainer connect error"); + Assert.assertFalse("a post-publication Error must not quarantine recoverable data", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + try (CursorSendEngine ignored = new CursorSendEngine(slotPath, SEGMENT_BYTES)) { + Assert.assertTrue("drainer teardown must release the slot lock", + OrphanScanner.isCandidateOrphan(slotPath)); + } + }); + } + + @Test + public void testConstructionErrorPropagatesWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + OutOfMemoryError injected = new OutOfMemoryError("injected drainer construction error"); + CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> { + throw injected; + }); + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("construction error must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + OutOfMemoryError thrown = null; + try { + drainer.run(); + } catch (OutOfMemoryError e) { + thrown = e; + } finally { + CursorSendEngine.setBeforeDeferredCloseCreationHook(null); + } + + Assert.assertSame("construction Error must remain visible to the caller", + injected, thrown); + Assert.assertEquals("construction error must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "injected drainer construction error"); + Assert.assertFalse("a VM Error must not quarantine recoverable data", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + try (CursorSendEngine ignored = new CursorSendEngine(slotPath, SEGMENT_BYTES)) { + Assert.assertTrue("constructor teardown must release the slot lock", + OrphanScanner.isCandidateOrphan(slotPath)); + } + }); + } + + @Test + public void testCorruptRecoveredChainIsQuarantined() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + + String segmentPath = slotPath + "/sf-initial.sfa"; + long corruptByte = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + int fd = Files.openRW(segmentPath); + Assert.assertTrue("could not open segment for corruption", fd >= 0); + try { + Unsafe.getUnsafe().putByte(corruptByte, (byte) 0); + Assert.assertEquals(1L, Files.write(fd, corruptByte, 1, 0L)); + } finally { + Files.close(fd); + Unsafe.free(corruptByte, 1, MemoryTag.NATIVE_DEFAULT); + } + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("recovery failure must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + drainer.run(); + + Assert.assertEquals("corrupt recovery must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + Assert.assertTrue("a corrupt durable chain requires operator quarantine", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("corrupt segment evidence must remain on disk", + Files.exists(segmentPath)); + }); + } + + @Test + public void testLockOpenFailureDoesNotQuarantineRecoverableData() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + + String lockPath = slotPath + "/.lock"; + Assert.assertTrue("seeded lock file must exist", Files.exists(lockPath)); + Assert.assertTrue("could not remove seeded lock file", Files.remove(lockPath)); + Assert.assertEquals("could not plant deterministic lock-open failure", + 0, Files.mkdir(lockPath, Files.DIR_MODE_DEFAULT)); + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("lock setup failure must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + drainer.run(); + + Assert.assertEquals("lock failure must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "could not open slot lock file"); + Assert.assertFalse("an operational lock-open failure must remain retryable", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + }); + } + + @Test + public void testWatermarkOpenFailureDoesNotQuarantineRecoverableData() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + + String watermarkPath = slotPath + "/" + AckWatermark.FILE_NAME; + Assert.assertTrue("seeded watermark must exist", Files.exists(watermarkPath)); + Assert.assertTrue("could not remove seeded watermark", Files.remove(watermarkPath)); + Assert.assertEquals("could not plant deterministic watermark-open failure", + 0, Files.mkdir(watermarkPath, Files.DIR_MODE_DEFAULT)); + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("setup failure must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + drainer.run(); + + Assert.assertEquals("watermark failure must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "could not open required ack watermark"); + Assert.assertFalse("an operational watermark-open failure must remain retryable", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain on disk", + OrphanScanner.isCandidateOrphan(slotPath)); + }); + } + + private static void removeRecursive(String path) { + if (path == null || !Files.exists(path)) { + return; + } + long find = Files.findFirst(path); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = path + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(path); + } + + private void seedUnackedFrame() { + long buffer = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine engine = new CursorSendEngine(slotPath, SEGMENT_BYTES)) { + Unsafe.getUnsafe().setMemory(buffer, 16, (byte) 1); + Assert.assertEquals(0L, engine.appendBlocking(buffer, 16)); + } finally { + Unsafe.free(buffer, 16, MemoryTag.NATIVE_DEFAULT); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java index 6a076f2f..b9ef46a5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfOperationalException; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; @@ -82,7 +83,7 @@ public void testEnumerationFindNextFailureRefusesGenerationAllocation() throws E try { manager.register(ring, dir); Assert.fail("register accepted a partially enumerated SF directory"); - } catch (IllegalStateException expected) { + } catch (SfOperationalException expected) { Assert.assertTrue(expected.getMessage().contains("could not fully enumerate")); } Assert.assertTrue("fault did not occur after the lower generation was observed", diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java index 5935aa24..1b6260b9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -544,6 +544,58 @@ public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() thr }); } + @Test + public void testSingleSectorTearLeavesPriorManifestRecordRecoverable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 64; + long payload = Unsafe.malloc(65, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment initial = MmapSegment.create( + tmpDir + "/sf-initial.sfa", 0, segmentSize); + try { + Assert.assertTrue(initial.tryAppend(payload, 64) >= 0); + } finally { + initial.close(); + } + + SegmentRing ring = SegmentRing.openExisting( + FilesFacade.INSTANCE, tmpDir, segmentSize); + Assert.assertNotNull(ring); + try { + MmapSegment spare = MmapSegment.create( + tmpDir + "/sf-0000000000000001.sfa", 1, segmentSize); + ring.installHotSpare(spare); + Assert.assertEquals("oversized append must rotate but leave the new active empty", + SegmentRing.PAYLOAD_TOO_LARGE, ring.appendOrFsn(payload, 65)); + Assert.assertEquals(0L, ring.publishedFsn()); + Assert.assertEquals(1L, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + + String manifestPath = tmpDir + "/" + MANIFEST_NAME; + int tornBytes = (int) Math.min(512L, Files.length(manifestPath)); + overwriteRange(manifestPath, 0, tornBytes, (byte) 0xA5); + + SegmentRing recovered = SegmentRing.openExisting( + FilesFacade.INSTANCE, tmpDir, segmentSize); + Assert.assertNotNull("one aligned 512-byte tear must leave a manifest record valid", + recovered); + try { + Assert.assertEquals("recovery must fall back to the prior committed boundary", + 0L, recovered.publishedFsn()); + Assert.assertEquals(0L, recovered.getActive().baseSeq()); + } finally { + recovered.close(); + } + Assert.assertFalse("a surviving record must prevent manifest quarantine", + Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + } finally { + Unsafe.free(payload, 65, MemoryTag.NATIVE_DEFAULT); + } + }); + } + // ------------------------------------------------------------------ // Engine level: enumeration failure must not truncate the durable log. // ------------------------------------------------------------------ @@ -651,6 +703,24 @@ private static void writeRawSegmentHeader(String path, int magic, byte version, } } + /** Overwrites a byte range in an existing file and makes the modeled tear durable. */ + private static void overwriteRange(String path, long offset, int length, byte value) { + int fd = Files.openRW(path); + Assert.assertTrue("could not open " + path, fd >= 0); + try { + long buf = Unsafe.malloc(length, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, length, value); + Assert.assertEquals(length, Files.write(fd, buf, length, offset)); + Assert.assertEquals(0, Files.fsync(fd)); + } finally { + Unsafe.free(buf, length, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + /** Overwrites a single int at {@code offset} in an existing file. */ private static void overwriteInt(String path, long offset, int value) { int fd = Files.openRW(path); @@ -671,7 +741,7 @@ private static void overwriteInt(String path, long offset, int value) { /** * Writes a valid {@code sf-manifest.bin} with one CRC-protected record, * mirroring SfManifest's on-disk layout (two alternating 64-byte records - * in a 128-byte file; record slot = generation & 1). + * at offsets 0 and 4096 in an 8192-byte file). */ private void writeManifest(long generation, long headBase, long activeBase) { String path = tmpDir + "/" + MANIFEST_NAME; @@ -680,8 +750,8 @@ private void writeManifest(long generation, long headBase, long activeBase) { int fd = Files.openRW(path); Assert.assertTrue("could not create manifest", fd >= 0); try { - if (Files.length(path) < 128) { - Assert.assertTrue(Files.truncate(fd, 128)); + if (Files.length(path) < 8192) { + Assert.assertTrue(Files.truncate(fd, 8192)); } long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); try { @@ -693,7 +763,7 @@ private void writeManifest(long generation, long headBase, long activeBase) { Unsafe.getUnsafe().putLong(buf + 24, activeBase); int crc = Crc32c.update(Crc32c.INIT, buf, 60); Unsafe.getUnsafe().putInt(buf + 60, crc); - long offset = (generation & 1L) * 64; + long offset = (generation & 1L) * 4096; Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); } finally { Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); @@ -704,7 +774,7 @@ private void writeManifest(long generation, long headBase, long activeBase) { } /** - * Writes a correctly-sized (128-byte) manifest whose A and B records are + * Writes a correctly-sized (8192-byte) manifest whose A and B records are * BOTH structurally plausible (magic, version, boundaries) but fail their * CRC check -- the "no valid CRC-protected record" quarantine trigger. */ @@ -713,7 +783,7 @@ private void writeManifestBothRecordsCrcBroken() { int fd = Files.openRW(path); Assert.assertTrue("could not create manifest", fd >= 0); try { - Assert.assertTrue(Files.truncate(fd, 128)); + Assert.assertTrue(Files.truncate(fd, 8192)); long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); try { for (long generation = 1; generation <= 2; generation++) { @@ -725,7 +795,7 @@ private void writeManifestBothRecordsCrcBroken() { Unsafe.getUnsafe().putLong(buf + 24, 2); // activeBase int crc = Crc32c.update(Crc32c.INIT, buf, 60); Unsafe.getUnsafe().putInt(buf + 60, crc + 1); // broken CRC - long offset = (generation & 1L) * 64; + long offset = (generation & 1L) * 4096; Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); } } finally { @@ -819,12 +889,12 @@ public void testCreationCrashZeroByteManifestSelfHeals() throws Exception { @Test public void testCreationCrashRecordlessManifestSelfHeals() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Same window, later stage: allocate() completed (128 zero bytes) + // Same window, later stage: allocate() completed (8192 zero bytes) // but the first record write/fsync never landed. writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); Assert.assertTrue(fd >= 0); - Assert.assertTrue(Files.truncate(fd, 128)); + Assert.assertTrue(Files.truncate(fd, 8192)); Files.close(fd); SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); @@ -849,7 +919,7 @@ public void testRecordlessManifestWithFlaggedSegmentsFailsClosed() throws Except SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); // truncates Assert.assertTrue(fd >= 0); - Assert.assertTrue(Files.truncate(fd, 128)); + Assert.assertTrue(Files.truncate(fd, 8192)); Files.close(fd); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644bf917..3529a5d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; import io.questdb.client.std.Files; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -76,6 +77,8 @@ public void testSecondAcquireFailsOnLockContention() throws Exception { try (SlotLock ignored = SlotLock.acquire(slot)) { fail("expected slot contention to throw"); } catch (IllegalStateException expected) { + assertTrue("contention must have a typed signal", + expected instanceof SlotLockContentionException); String msg = expected.getMessage(); assertTrue("error must mention contention: " + msg, msg.contains("already in use")); @@ -132,6 +135,106 @@ public void testReleaseConfirmsAndIsIdempotent() throws Exception { * confirms and stays confirmed. Swapping in a known-bad descriptor gives * the slot-specific native primitive a deterministic unlock failure. */ + @Test + public void testFailedCloseRetainsRetryOwnerUntilNextAcquire() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-close"; + SlotLock lock = SlotLock.acquire(slot); + SlotLock.ReleaseFailureForTesting releaseFailure = + lock.injectReleaseFailureForTesting(); + try { + // CursorSendEngine construction cleanup uses QuietCloseable.close(). + // A failed close must retain an owner that a later acquire can + // drive, rather than dropping the sole reference and flock fd. + lock.close(); + assertTrue("failed close must retain the injected descriptor", + lock.isReleaseFailureInjectedForTesting()); + try (SlotLock ignored = SlotLock.acquire(slot)) { + fail("slot must stay locked while the release failure persists"); + } catch (SlotLockContentionException expected) { + // The retained real flock still protects the slot. + } + + releaseFailure.close(); + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals("the next acquire must retry the retained release owner", + slot, again.slotDir()); + } catch (SlotLockContentionException stillHeld) { + fail("failed construction cleanup dropped its retry owner: " + + stillHeld.getMessage()); + } + } finally { + releaseFailure.close(); + lock.release(); + } + }); + } + + @Test + public void testFailedCloseRetainsRetryOwnerWithEquivalentPathAlias() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-close-alias"; + SlotLock lock = SlotLock.acquire(slot); + SlotLock.ReleaseFailureForTesting releaseFailure = + lock.injectReleaseFailureForTesting(); + try { + lock.close(); + assertTrue("failed close must retain the injected descriptor", + lock.isReleaseFailureInjectedForTesting()); + + // Restore the real descriptor so the retained owner can make + // progress. The trailing separator preserves the caller's + // spelling but names the same physical .lock file. + releaseFailure.close(); + String slotAlias = slot + "/"; + try (SlotLock again = SlotLock.acquire(slotAlias)) { + assertEquals("an equivalent path must drive the retained release owner", + slotAlias, again.slotDir()); + } catch (SlotLockContentionException stillHeld) { + fail("equivalent path spelling could not find its retry owner: " + + stillHeld.getMessage()); + } + } finally { + releaseFailure.close(); + lock.release(); + } + }); + } + + @Test + public void testPersistentFailedCloseDoesNotBlockDifferentSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String failedSlot = parentDir + "/persistent-failed-close"; + SlotLock failedLock = SlotLock.acquire(failedSlot); + SlotLock.ReleaseFailureForTesting releaseFailure = + failedLock.injectReleaseFailureForTesting(); + try { + failedLock.close(); + // A repeat close must not enqueue the same intrusive node twice. + failedLock.close(); + + String independentSlot = parentDir + "/independent"; + try (SlotLock independent = SlotLock.acquire(independentSlot)) { + assertEquals("a persistent failure on one slot must not block another", + independentSlot, independent.slotDir()); + } + + releaseFailure.close(); + String progressSlot = parentDir + "/progress"; + try (SlotLock ignored = SlotLock.acquire(progressSlot)) { + // Any cold acquisition drives every failed-close owner. + } + try (SlotLock reacquired = SlotLock.acquire(failedSlot)) { + assertEquals("successful retry must remove the pending list entry", + failedSlot, reacquired.slotDir()); + } + } finally { + releaseFailure.close(); + failedLock.release(); + } + }); + } + @Test public void testFailedUnlockRetainsFdAndReportsFalse() throws Exception { TestUtils.assertMemoryLeak(() -> { From 958a36d45320ae1a6675243db84a197d9ec89964 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 10:00:11 +0100 Subject: [PATCH 45/64] fix(qwp): clear transient sync latch, unpin startup recovery, capture win32 errno at failure point Review-finding fixes: - C1: SegmentManager.servicePeriodicSync now clears SegmentRing.durabilityFailure after a subsequent fully-successful sync pass, so a transient fsync/msync failure no longer permanently bricks the producer; recovery is logged like the trim path. Regression test: testTransientSyncFailureClearsOnNextSuccess. - C2: startup SF recovery parks slots whose build persistently fails (immediately on SlotLockContentionException, after 3 consecutive failures otherwise), keeps scanning the remaining slots, rewinds the cursors while parked slots remain, and dedups the per-slot WARN. A wedged slot no longer livelocks the driver, floods the log, or starves higher slots' orphan data; the server-wide transient retry contract is preserved. 3 new regression tests in SenderPoolSfTest. - M1: windows open_file saves GetLastError() at the CreateFileW failure point (before free() can clobber it) and fsyncDir0 classifies the ERROR_ACCESS_DENIED best-effort degrade from the saved value via the new GetSavedLastError() (defined in os.c beside the TLS index it reads), mirroring the FlushFileBuffers branch's capture pattern. - M2: replaced leftover test reflection with the existing @TestOnly seams in CursorSendEngineSlotReacquisitionTest, QueryClientPoolErrorSafetyTest and SlotLockReleasedContractTest. - Stale comments updated: the durability-latch throw is transient, and the recovery streak constant parks on the 3rd consecutive failure. --- core/src/main/c/windows/errno.h | 6 + core/src/main/c/windows/files.c | 17 +- core/src/main/c/windows/os.c | 4 + .../qwp/client/QwpWebSocketSender.java | 8 +- .../qwp/client/sf/cursor/SegmentManager.java | 9 +- .../qwp/client/sf/cursor/SegmentRing.java | 22 +- .../io/questdb/client/impl/SenderPool.java | 239 +++++++++++++++--- .../client/SlotLockReleasedContractTest.java | 2 +- ...CursorSendEngineSlotReacquisitionTest.java | 35 +-- .../SegmentManagerPeriodicSyncTest.java | 74 ++++++ .../impl/QueryClientPoolErrorSafetyTest.java | 20 +- .../client/test/impl/SenderPoolSfTest.java | 183 ++++++++++++++ 12 files changed, 536 insertions(+), 83 deletions(-) diff --git a/core/src/main/c/windows/errno.h b/core/src/main/c/windows/errno.h index 093d52c2..a2971056 100644 --- a/core/src/main/c/windows/errno.h +++ b/core/src/main/c/windows/errno.h @@ -6,4 +6,10 @@ static DWORD dwTlsIndexLastError = 0; void SaveLastError(); +/* Returns the per-thread error code most recently stored by SaveLastError(). + * Both functions are defined in os.c, the only translation unit whose static + * dwTlsIndexLastError copy is initialised by TlsAlloc() in DllMain; every + * other includer's copy of the variable is an unused zero. */ +DWORD GetSavedLastError(); + #endif //ZLIB_ERRNO_H diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 9805740b..d3fa2fbb 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -77,11 +77,16 @@ static jint open_file(const char *utf8Path, } HANDLE h = CreateFileW(wide, desiredAccess, shareMode, NULL, creationDisposition, flagsAndAttributes, NULL); - free(wide); if (h == INVALID_HANDLE_VALUE) { + // Save the CreateFileW failure code BEFORE free(): the thread's + // last-error value may be overwritten by any subsequent API/CRT call + // (HeapFree can SetLastError), and callers such as fsyncDir0 rely on + // the saved value being the CreateFileW error. SaveLastError(); + free(wide); return -1; } + free(wide); return HANDLE_TO_FD(h); } @@ -237,10 +242,12 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 // directory handle with ERROR_ACCESS_DENIED. Directory-entry durability // on NTFS is provided by metadata journaling ($LogFile), so degrade to // best-effort success here rather than hard-failing the SF durability - // path. open_file() already recorded the error via SaveLastError(); a - // genuine failure such as a missing directory (ERROR_PATH_NOT_FOUND) - // still propagates as fatal. - if (GetLastError() == ERROR_ACCESS_DENIED) { + // path. Consult the error open_file() saved at the CreateFileW failure + // point rather than the live GetLastError(): the intervening free() + // and return path inside open_file() are not guaranteed to preserve + // the thread's last-error value. A genuine failure such as a missing + // directory (ERROR_PATH_NOT_FOUND) still propagates as fatal. + if (GetSavedLastError() == ERROR_ACCESS_DENIED) { return 0; } return -1; diff --git a/core/src/main/c/windows/os.c b/core/src/main/c/windows/os.c index 73bb4307..e50ce84c 100644 --- a/core/src/main/c/windows/os.c +++ b/core/src/main/c/windows/os.c @@ -237,3 +237,7 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Os_currentTimeMicros void SaveLastError() { TlsSetValue(dwTlsIndexLastError, (LPVOID) (DWORD_PTR) GetLastError()); }; + +DWORD GetSavedLastError() { + return (DWORD) (DWORD_PTR) TlsGetValue(dwTlsIndexLastError); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 699a903b..5a813c83 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -869,9 +869,11 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { return targetFsn < 0L; } cursorEngine.checkDurability(); - // Surface latched I/O errors before any early-return path, so a - // caller polling with timeoutMillis <= 0 to drive their own loop - // sees the terminal throw instead of an indefinite "not yet". + // Surface latched errors before any early-return path, so a caller + // polling with timeoutMillis <= 0 to drive their own loop sees the + // throw instead of an indefinite "not yet". The durability latch + // above is transient: it throws while latched, and clears once a + // later periodic sync pass fully succeeds so producers can resume. if (cursorSendLoop != null) { cursorSendLoop.checkError(); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index f6705f7e..5745b15c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -1224,8 +1224,15 @@ private void servicePeriodicSync(RingEntry e, long now) { syncScratch.getQuick(i).syncPublished(); } e.ring.clearSyncRequestIfActiveDurable(); + // The pass above covered every live segment's published range, so + // any earlier barrier failure has been remedied — unlatch it so a + // transient disk fault doesn't permanently brick the producer. + e.ring.clearDurabilityFailure(); e.nextDataSyncNanos = now + e.syncIntervalNanos; - e.syncFailureLogged = false; + if (e.syncFailureLogged) { + e.syncFailureLogged = false; + LOG.info("Periodic SF data sync recovered for {}", e.dir); + } } catch (Throwable failure) { e.ring.recordDurabilityFailure(failure); if (!e.syncFailureLogged) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 9e1a4c4f..0c75bc74 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -118,8 +118,10 @@ public final class SegmentRing implements QuietCloseable { // call installHotSpare on a closed ring (whose hotSpare was just // zeroed by close()) -- the spare's mmap + fd would never be reclaimed. private boolean closed; - // First periodic data-barrier failure. The manager latches it and the - // producer observes it before its next append. + // Latest periodic data-barrier failure. The manager latches it and the + // producer observes it before its next append; the manager clears it + // again once a subsequent periodic sync pass succeeds, so a transient + // disk fault does not permanently brick the producer. private volatile MmapSegmentException durabilityFailure; // hotSpare: written by segment manager (installHotSpare), read+cleared by // producer thread on rotation. Volatile so the producer sees fresh installs. @@ -961,6 +963,22 @@ public void checkDurability() { } } + /** + * Clears a latched periodic data-barrier failure. Called by the segment + * manager after a subsequent periodic sync pass over every live segment + * succeeds: the published range the failed barrier was meant to cover is + * durable now, so the producer may resume. The monitor serializes the + * clear with {@link #recordDurabilityFailure(Throwable)}; the volatile + * write publishes it to producer threads calling {@link #checkDurability()}. + */ + void clearDurabilityFailure() { + if (durabilityFailure != null) { + synchronized (this) { + durabilityFailure = null; + } + } + } + synchronized void clearSyncRequestIfActiveDurable() { if (active != null && active.isPublishedDurable()) { syncRequested = false; diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 90658da2..31c688fc 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; import org.jetbrains.annotations.TestOnly; @@ -97,6 +98,18 @@ public final class SenderPool implements AutoCloseable { // -- the residual window documented on recoverOneSlotStep -- because the // transport has no application-level connect timeout to clamp it. private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000; + // The scan parks ONE candidate slot and moves on (C2) on the slot's Nth + // (3rd) consecutive recoverOneSlotStep failure: two retries in place, + // park on the third. The retry-in-place policy + // assumes a failure is server-wide and will repeat for every remaining + // slot, but a slot-specific persistent condition (e.g. a corrupt slot dir + // that fails the recovery build every time) would otherwise pin the cursor + // forever: the driver would livelock retrying that one slot each second + // while every higher-index slot's durable orphan data starves. Small + // enough to bound how long one slot can block the scan; large enough that + // the dominant server-wide transient failure keeps its retry-the-same- + // candidate behavior for the first attempts. + private static final int RECOVERY_MAX_SLOT_FAILURE_STREAK = 3; // A direct SenderPool has no PoolHousekeeper to retry a transient startup // recovery failure. Its private driver waits this long between failed // attempts so an unavailable server does not cause a hot retry loop. @@ -267,10 +280,24 @@ public final class SenderPool implements AutoCloseable { // cursor; recoveryComplete latches true only when the whole scan finishes. // A transient build failure or drain timeout leaves the current candidate // pending so a later tick or explicit drive can retry it on the same pool. + // A slot-SPECIFIC failure -- flock contention with another live owner, or + // RECOVERY_MAX_SLOT_FAILURE_STREAK consecutive failures on one slot -- is + // instead "parked": the cursor advances past it so it cannot starve the + // remaining slots, recoveryDeferredThisCycle records the park, and once + // both passes finish with parked slots outstanding the cursors rewind for + // another cycle on a later tick instead of latching recoveryComplete. + // recoveryFailStreak/-Slot track consecutive failures on one candidate; + // recoveryWarnedSlots dedups the per-slot WARNs so an indefinitely + // retried slot logs once per failure episode, not once per retry. All of + // this state is owned by the single recovery driver like the cursors. private int recoveryInRangeNext; private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; private boolean recoveryComplete; + private int recoveryDeferredThisCycle; + private int recoveryFailStreak; + private int recoveryFailStreakSlot = -1; + private final IntList recoveryWarnedSlots = new IntList(); public SenderPool( String configurationString, @@ -630,6 +657,14 @@ boolean runStartupRecoveryStep() { * A build failure or drain timeout stops the current drive but leaves its * candidate pending so a later driver attempt can retry after a transient * condition clears; it does not poison recovery for the life of the pool. + * A slot-SPECIFIC condition -- flock contention with another live owner, or + * {@link #RECOVERY_MAX_SLOT_FAILURE_STREAK} consecutive failures on the + * same candidate -- instead parks that slot: the cursor advances past it so + * one wedged slot cannot starve the remaining slots' recovery, and once + * both passes finish with parked slots outstanding the scan rewinds for + * another cycle on a later tick (nothing is abandoned while the pool + * lives). Per-slot failure WARNs are deduplicated across retries via + * {@link #warnSlotOnce}. *

      * Boundedness / residual window. A continuing recovery step runs on * either the direct pool's private driver or the PoolHousekeeper thread, and @@ -702,7 +737,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { } // A real candidate -> spend the step on it. Advance the cursor only // after success so a transient build/drain failure remains retryable. - boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); + RecoveryDrainOutcome outcome = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { // Release the recovery reservation accounting; from here either @@ -736,12 +771,33 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { } finally { lock.unlock(); } - if (stopScan) { - // Stop this drive without advancing the cursor. The same live - // pool retries this candidate after the transient condition is - // removed instead of requiring pool recreation. + if (outcome == RecoveryDrainOutcome.CONTENDED) { + // Slot-specific by construction: another LIVE owner holds this + // slot's flock and may do so indefinitely. Park the slot -- + // advance past it; the end-of-scan cycle rewind re-probes it + // later -- so it cannot starve the remaining slots, and keep + // scanning within this step (no drain was spent on it). + recoveryDeferredThisCycle++; + recoveryInRangeNext++; + continue; + } + if (outcome == RecoveryDrainOutcome.FAILED) { + if (parkAfterFailure(i)) { + // The same candidate failed RECOVERY_MAX_SLOT_FAILURE_STREAK + // consecutive times: stop presuming the failure is + // server-wide. Park the slot and move on; the end-of-scan + // cycle rewind retries it later, so nothing is abandoned. + recoveryDeferredThisCycle++; + recoveryInRangeNext++; + return true; + } + // Presumed transient/server-wide: stop this drive without + // advancing the cursor. The same live pool retries this + // candidate after the transient condition is removed instead + // of paying a likely-identical failure per remaining slot. return false; } + clearFailStreak(i); recoveryInRangeNext++; return true; } @@ -769,7 +825,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryOutOfRangeNext++; continue; } - boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); + RecoveryDrainOutcome outcome = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); if (retained[0] != null) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this @@ -784,16 +840,46 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); } - if (stopScan) { - // Keep the out-of-range cursor on this candidate. In particular, - // a transient flock/build collision must be retried after the - // primary sender returns; no capacity bookkeeping is involved. + if (outcome == RecoveryDrainOutcome.CONTENDED) { + // Same parking rule as the in-range pass: a live flock holder + // (e.g. a sibling drainer that adopted this out-of-range slot) + // is slot-specific and possibly long-lived; the end-of-scan + // cycle rewind re-probes it after the holder lets go. No + // capacity bookkeeping is involved out of range. + recoveryDeferredThisCycle++; + recoveryOutOfRangeNext++; + continue; + } + if (outcome == RecoveryDrainOutcome.FAILED) { + if (parkAfterFailure(idx)) { + recoveryDeferredThisCycle++; + recoveryOutOfRangeNext++; + return true; + } + // Keep the out-of-range cursor on this candidate so a later + // tick retries it after the presumed-transient condition + // clears; no capacity bookkeeping is involved. return false; } + clearFailStreak(idx); recoveryOutOfRangeNext++; return true; } + if (recoveryDeferredThisCycle > 0) { + // At least one slot was parked this cycle (contended or + // persistently failing). Its data stays durable on disk, so + // instead of latching recoveryComplete -- which would abandon it + // until a restart or a lucky borrow of that index -- rewind the + // scan and let the driver's retry cadence run another cycle. + // Recovered, live and empty slots are re-skipped cheaply; parked + // slots get a fresh (bounded) attempt. Per-slot WARNs stay + // deduplicated across cycles via recoveryWarnedSlots. + recoveryDeferredThisCycle = 0; + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + return false; + } recoveryComplete = true; return false; } @@ -828,21 +914,25 @@ private static Thread createStartupRecoveryThread(Runnable runnable) { * in-range caller keeps it in {@link #retiredSlots} so a * late flock release can be re-probed. {@code null} when * the flock was released (or no recoverer was built). - * @return {@code true} if a build/drain failure occurred that will very - * likely repeat for every remaining slot, so the caller should stop scanning + * @return {@link RecoveryDrainOutcome#CONTENDED} when the slot flock is + * held by another live owner (slot-specific: park it and keep scanning); + * {@link RecoveryDrainOutcome#FAILED} on any other build/drain failure + * (presumed server-wide, likely to repeat for every remaining slot); + * {@link RecoveryDrainOutcome#DRAINED} when the candidate was drained or + * was no longer a candidate */ - private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, - long remainingMillis, SenderSlot[] retainedOut) { + private RecoveryDrainOutcome drainCandidateSlotForRecovery(int slotIndex, String slotPath, + long remainingMillis, SenderSlot[] retainedOut) { retainedOut[0] = null; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and // delegate().close() can retain it when an I/O or manager worker does // not stop. SenderSlot recoverer = null; - boolean stopScan = false; + RecoveryDrainOutcome outcome = RecoveryDrainOutcome.DRAINED; try { if (!OrphanScanner.isCandidateOrphan(slotPath)) { - return false; + return RecoveryDrainOutcome.DRAINED; } try { // Recovery delegate: forced OFF-mode initial connect (see @@ -851,13 +941,28 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // reconnect-budget retry loop that would block this recovery // driver for minutes (M1). recoverer = createRecoverer(slotIndex); + } catch (SlotLockContentionException contention) { + // The slot flock is held by another LIVE owner. That condition + // is scoped to THIS slot dir by construction -- it says nothing + // about the server or the remaining slots -- and it can persist + // for the owner's whole lifetime, so retrying it in place would + // starve every remaining slot (C2). Report it distinctly so the + // scan parks this slot and continues. + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: slot {} is held by another live owner ({}); " + + "parking it and continuing with the remaining slots", + slotPath, contention.toString()); + } + return RecoveryDrainOutcome.CONTENDED; } catch (Throwable buildErr) { // A build/connect failure (e.g. server unreachable) will very // likely repeat for every remaining slot, so stop here rather // than pay a connect timeout per slot. - LOG.warn("startup SF recovery: could not open slot {} ({}); " - + "deferring this and remaining slots", slotPath, buildErr.toString()); - return true; + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: could not open slot {} ({}); " + + "deferring this and remaining slots", slotPath, buildErr.toString()); + } + return RecoveryDrainOutcome.FAILED; } try { // Cap the drain at the remaining shared budget and short-circuit @@ -865,15 +970,19 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // will very likely do the same for every remaining slot -- the // same reasoning as the build-failure case above. if (!recoverer.delegate().drain(remainingMillis)) { - LOG.warn("startup SF recovery: drain did not ack slot {} " - + "within {}ms; deferring this and remaining slots", - slotPath, remainingMillis); - stopScan = true; + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: drain did not ack slot {} " + + "within {}ms; deferring this and remaining slots", + slotPath, remainingMillis); + } + outcome = RecoveryDrainOutcome.FAILED; } } catch (Throwable drainErr) { - LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it", - slotPath, drainErr.toString()); - stopScan = true; + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it", + slotPath, drainErr.toString()); + } + outcome = RecoveryDrainOutcome.FAILED; } finally { try { recoverer.delegate().close(); @@ -883,14 +992,69 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, } } } catch (Throwable scanErr) { - LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it", - slotPath, scanErr.toString()); - stopScan = true; + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it", + slotPath, scanErr.toString()); + } + outcome = RecoveryDrainOutcome.FAILED; } if (recoverer != null && !flockReleased(recoverer)) { retainedOut[0] = recoverer; } - return stopScan; + if (outcome == RecoveryDrainOutcome.DRAINED && recoveryWarnedSlots.contains(slotIndex)) { + recoveryWarnedSlots.remove(slotIndex); + LOG.info("startup SF recovery: slot {} recovered after earlier deferrals", slotPath); + } + return outcome; + } + + /** + * Clears the consecutive-failure streak after a successful drain so the + * next candidate starts from a clean count. Driver-thread state (see the + * scan cursor comment); no lock needed. + */ + private void clearFailStreak(int slotIndex) { + if (recoveryFailStreakSlot == slotIndex) { + recoveryFailStreakSlot = -1; + recoveryFailStreak = 0; + } + } + + /** + * Records one FAILED drain attempt on a candidate slot and reports whether + * the scan should park it: {@code true} once the SAME slot has failed + * {@link #RECOVERY_MAX_SLOT_FAILURE_STREAK} consecutive times, meaning the + * failure is evidently slot-specific rather than server-wide and must not + * pin the cursor any longer (C2). Driver-thread state; no lock needed. + */ + private boolean parkAfterFailure(int slotIndex) { + if (recoveryFailStreakSlot != slotIndex) { + recoveryFailStreakSlot = slotIndex; + recoveryFailStreak = 0; + } + if (++recoveryFailStreak < RECOVERY_MAX_SLOT_FAILURE_STREAK) { + return false; + } + recoveryFailStreakSlot = -1; + recoveryFailStreak = 0; + return true; + } + + /** + * Dedups per-slot recovery WARNs: a contended or persistently failing slot + * is retried indefinitely (its data stays durable on disk), and warning on + * every retry produced an unbounded one-WARN-per-second stream for the life + * of the process (C2). Returns {@code true} only the first time a slot + * fails since its last successful drain, so each failure episode logs + * exactly once (a follow-up failure with a DIFFERENT cause is deliberately + * folded into the same episode). Driver-thread state; no lock needed. + */ + private boolean warnSlotOnce(int slotIndex) { + if (recoveryWarnedSlots.contains(slotIndex)) { + return false; + } + recoveryWarnedSlots.add(slotIndex); + return true; } public PooledSender borrow() { @@ -1901,4 +2065,19 @@ private void recoverRetiredSlotAt(int retiredIndex) { "pool capacity restored, now {} of {} usable [leakedSlots={}]", s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); } + + // Outcome of one drainCandidateSlotForRecovery attempt, letting the scan + // tell a slot-specific contention (park the slot and keep scanning) apart + // from a failure presumed server-wide (retry the same candidate, bounded + // by RECOVERY_MAX_SLOT_FAILURE_STREAK). + private enum RecoveryDrainOutcome { + // createRecoverer() lost the slot flock to another live owner -- a + // condition scoped to that slot dir by construction that can outlive + // any retry interval. + CONTENDED, + // The candidate was drained, or was no longer a candidate. + DRAINED, + // Any other build/drain failure, presumed transient and server-wide. + FAILED + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 63553ebc..cc501a8c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -366,7 +366,7 @@ public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Excep wss = QwpWebSocketSender.createForTesting("localhost", 1); wss.setCursorEngine(engine, true); - setField(wss, "cursorSendLoop", loop); + wss.setCursorSendLoopForTesting(loop); // Drive the real early-bail close() on a thread whose pending // interrupt lands in loop.close()'s shutdownLatch.await(). diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 2c5f76c3..4c59bcfe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -38,7 +38,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -227,12 +226,12 @@ public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws AtomicReference hookErr = new AtomicReference<>(); // Production shape: private, owned manager (ownsManager=true). CursorSendEngine engine = new CursorSendEngine(slot, segSize); - SegmentManager manager = readManager(engine); + SegmentManager manager = engine.getManagerForTesting(); long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { // Phase 1: let the worker finish the initial spare install so // the hook below can only fire on the rotation-triggered pass. - SegmentRing ring = readRing(engine); + SegmentRing ring = engine.getRingForTesting(); long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (ring.needsHotSpare()) { if (System.nanoTime() > deadlineNs) { @@ -333,13 +332,13 @@ public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { AtomicReference hookErr = new AtomicReference<>(); // Production shape: private, owned manager (ownsManager=true). CursorSendEngine engine = new CursorSendEngine(slot, segSize); - SegmentManager manager = readManager(engine); + SegmentManager manager = engine.getManagerForTesting(); long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { // Phase 1: wait out the initial spare install so the park hook // can only fire on the rotation-triggered pass (see // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). - SegmentRing ring = readRing(engine); + SegmentRing ring = engine.getRingForTesting(); long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (ring.needsHotSpare()) { if (System.nanoTime() > deadlineNs) { @@ -445,13 +444,13 @@ public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws AtomicReference hookErr = new AtomicReference<>(); // Production shape: private, owned manager (ownsManager=true). CursorSendEngine engine = new CursorSendEngine(slot, segSize); - SegmentManager manager = readManager(engine); + SegmentManager manager = engine.getManagerForTesting(); long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { // Phase 1: wait out the initial spare install so the park hook // can only fire on the rotation-triggered pass (see // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). - SegmentRing ring = readRing(engine); + SegmentRing ring = engine.getRingForTesting(); long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (ring.needsHotSpare()) { if (System.nanoTime() > deadlineNs) { @@ -562,12 +561,12 @@ public void testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff AtomicReference hookErr = new AtomicReference<>(); // Production shape: private, owned manager (ownsManager=true). CursorSendEngine engine = new CursorSendEngine(slot, segSize); - SegmentManager manager = readManager(engine); + SegmentManager manager = engine.getManagerForTesting(); long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { // Phase 1: wait out the initial spare install so the park hook // can only fire on the rotation-triggered pass. - SegmentRing ring = readRing(engine); + SegmentRing ring = engine.getRingForTesting(); long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (ring.needsHotSpare()) { if (System.nanoTime() > deadlineNs) { @@ -711,12 +710,12 @@ public void testMemoryModeOwnedCloseHandsCleanupToWorkerExit() throws Exception // Memory mode: null sfDir, private owned manager — the exact // shape non-SF async ingest uses. CursorSendEngine engine = new CursorSendEngine(null, segSize); - SegmentManager manager = readManager(engine); + SegmentManager manager = engine.getManagerForTesting(); long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { // Phase 1: wait out the initial spare install so the park hook // can only fire on the rotation-triggered pass. - SegmentRing ring = readRing(engine); + SegmentRing ring = engine.getRingForTesting(); long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (ring.needsHotSpare()) { if (System.nanoTime() > deadlineNs) { @@ -794,7 +793,7 @@ public void testOwnedManagerCloseSkipsPerRingQuiescenceWait() throws Exception { TestUtils.assertMemoryLeak(() -> { String slot = tmpDir + "/owned-slot"; CursorSendEngine engine = new CursorSendEngine(slot, 4L * 1024 * 1024); - SegmentManager manager = readManager(engine); + SegmentManager manager = engine.getManagerForTesting(); AtomicBoolean perRingAwaited = new AtomicBoolean(); try { manager.setBeforeRingQuiescenceAwaitHook(() -> perRingAwaited.set(true)); @@ -829,18 +828,6 @@ public void testSameSlotReacquirableAfterNormalClose() throws Exception { }); } - private static SegmentManager readManager(CursorSendEngine engine) throws Exception { - Field field = CursorSendEngine.class.getDeclaredField("manager"); - field.setAccessible(true); - return (SegmentManager) field.get(engine); - } - - private static SegmentRing readRing(CursorSendEngine engine) throws Exception { - Field field = CursorSendEngine.class.getDeclaredField("ring"); - field.setAccessible(true); - return (SegmentRing) field.get(engine); - } - private static void rmDirRecursive(String dir) { if (!Files.exists(dir)) return; long find = Files.findFirst(dir); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java index 2e395783..199f3ccf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java @@ -114,6 +114,80 @@ public void testDeadlineAndFailurePropagation() throws Exception { }); } + @Test + public void testTransientSyncFailureClearsOnNextSuccess() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long intervalNanos = 100L; + final long segmentSize = 4096L; + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-recovery-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 1L, segmentSize)); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // First tick: initial sync succeeds. + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + + // One transient fsync failure on the next periodic barrier. + assertEquals(1L, ring.appendOrFsn(payload, 16)); + filesFacade.isFsyncFailureEnabled = true; + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + try { + ring.appendOrFsn(payload, 16); + fail("expected the failed data sync to reach the producer"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + assertEquals("failed append must not enter the ring", 1L, ring.publishedFsn()); + + // Disk recovers; the manager's retry (scheduled at + // now + min(interval, 1s)) succeeds on the next tick. + filesFacade.isFsyncFailureEnabled = false; + ticks.set(intervalNanos * 2L); + int msyncBefore = filesFacade.msyncCalls; + int fsyncBefore = filesFacade.fsyncCalls; + manager.serviceRingForTesting(ring); + assertTrue("retry barrier must have run msync", filesFacade.msyncCalls > msyncBefore); + assertTrue("retry barrier must have run fsync", filesFacade.fsyncCalls > fsyncBefore); + + // A transient failure must not brick the producer: the retry + // covered the published range, so appends resume. + assertEquals(2L, ring.appendOrFsn(payload, 16)); + assertEquals(2L, ring.publishedFsn()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + private static final class CountingFilesFacade implements FilesFacade { private boolean isFsyncFailureEnabled; private int fsyncCalls; diff --git a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java index ca5e33aa..2978ff46 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java @@ -34,13 +34,10 @@ import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Field; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; // Error-safety of the three QueryClientPool creation paths the teardown-hardening @@ -327,22 +324,11 @@ private static Consumer alwaysThrow() { }; } - private static void awaitCreationWaiter(QueryClientPool pool) throws Exception { - Field lockField = QueryClientPool.class.getDeclaredField("lock"); - Field conditionField = QueryClientPool.class.getDeclaredField("creationFinished"); - lockField.setAccessible(true); - conditionField.setAccessible(true); - ReentrantLock lock = (ReentrantLock) lockField.get(pool); - Condition creationFinished = (Condition) conditionField.get(pool); + private static void awaitCreationWaiter(QueryClientPool pool) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); while (System.nanoTime() < deadline) { - lock.lock(); - try { - if (lock.hasWaiters(creationFinished)) { - return; - } - } finally { - lock.unlock(); + if (pool.hasCreationWaiterForTesting()) { + return; } Thread.yield(); } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 533795ff..0b7d6164 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -35,6 +35,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -2943,6 +2944,188 @@ public void testDirectRecoveryRetriesTransientFailureAndRemainingSlots() throws }); } + @Test + public void testDirectRecoveryContendedSlotDoesNotStarveRemainingSlots() throws Exception { + // C2 regression, end to end: a slot whose createRecoverer PERSISTENTLY + // throws SlotLockContentionException (its flock is held by another live + // owner, e.g. a sibling process sharing the slot dir) must not pin the + // startup-recovery cursor: pre-fix the driver retried that one slot + // every second forever and the higher-index slot's durable orphan data + // was never forwarded under idle load. The transient-failure twin + // (testDirectRecoveryRetriesTransientFailureAndRemainingSlots) covers a + // failure that CLEARS; this covers one that never does. + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(1); + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + if (idx == 0) { + // Persistent: never clears for the life of the pool. + throw new SlotLockContentionException( + "sf slot already in use by another process [slot=" + + slot("default-0") + ", holder=pid=test]"); + } + Sender delegate = Sender.builder(config).senderId("default-" + idx).build(); + return notifyingCloseSender(delegate, drained); + }; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertTrue( + "a persistently contended slot 0 must not starve slot 1's recovery", + drained.await(15, TimeUnit.SECONDS)); + Assert.assertFalse("higher-index slot's orphan data must be delivered", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("recovered slot must be drained exactly once", + 1, attempts[1].get()); + Assert.assertTrue("contended slot must have been probed", attempts[0].get() >= 1); + Assert.assertTrue("contended slot's durable data must be preserved on disk", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("recovered frame must reach the server", handler.frames.get() >= 1); + Assert.assertFalse("recovery must not report complete while the contended slot holds data", + pool.isRecoveryCompleteForTesting()); + } + } + }); + } + + @Test + public void testStartupRecoveryParksContendedSlotAndContinuesScan() throws Exception { + // C2, white-box and fully step-driven (no live driver, no wall clock): + // one recovery step must park a contended in-range slot 0, continue to + // the out-of-range slot 1 within the SAME step, keep the parked slot + // retryable across scan cycles (never abandoned, never complete), and + // WARN about the contended slot exactly once, not once per retry. + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(1); + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + if (idx == 0) { + throw new SlotLockContentionException( + "sf slot already in use by another process [slot=" + + slot("default-0") + ", holder=pid=test]"); + } + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + Logger poolLogger = (Logger) LoggerFactory.getLogger(SenderPool.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + Level savedLevel = poolLogger.getLevel(); + poolLogger.setLevel(Level.ALL); + poolLogger.addAppender(appender); + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 0, factory)) { + // Cycle 1: park the contended slot 0, then drain slot 1 in the + // same step (the park must not consume the step's single drain). + Assert.assertTrue("step must park the contended slot and drain the next candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(1, attempts[0].get()); + Assert.assertEquals("higher-index slot must be recovered despite the parked slot", + 1, attempts[1].get()); + Assert.assertEquals(0, drained.getCount()); + Assert.assertFalse("cycle with a parked slot must defer, not complete", + pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("recovery must not report complete while the contended slot holds data", + pool.isRecoveryCompleteForTesting()); + + // Cycle 2: the parked slot is re-probed (retryable, not abandoned). + Assert.assertTrue(pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("parked slot must be re-probed on the next cycle", + 2, attempts[0].get()); + Assert.assertFalse(pool.runStartupRecoveryStepForTesting()); + + // Bounded logging: the contended slot warned once, not per retry. + long contentionWarns = appender.list.stream().filter(e -> + e.getLevel().isGreaterOrEqual(Level.WARN) + && e.getFormattedMessage().contains("default-0")).count(); + Assert.assertEquals("contended slot must WARN once per episode, not per retry; captured=" + + appender.list, 1, contentionWarns); + } finally { + poolLogger.detachAppender(appender); + poolLogger.setLevel(savedLevel); + appender.stop(); + } + } + + @Test + public void testStartupRecoveryParksPersistentlyFailingSlotAfterBoundedRetries() throws Exception { + // C2, generic-failure flavor, step-driven: a slot whose recovery build + // persistently fails with a NON-contention error keeps the existing + // retry-in-place behavior for the first attempts (the server-wide + // transient heuristic) but must be parked after a bounded streak so it + // cannot starve the higher-index in-range slot, and must be re-probed + // on the next scan cycle rather than abandoned. + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(1); + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + if (idx == 0) { + throw new LineSenderException("persistent per-slot recovery failure"); + } + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 2, 0, factory)) { + // Attempts 1 and 2: presumed transient, same candidate retried. + Assert.assertFalse("first failure must defer the same candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("second failure must defer the same candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(2, attempts[0].get()); + Assert.assertEquals("failing slot must not have blocked past its streak yet", + 0, attempts[1].get()); + + // Attempt 3 exhausts the streak: the slot is parked and the scan + // may continue to the next candidate. + Assert.assertTrue("streak exhaustion must park the slot and continue the scan", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(3, attempts[0].get()); + Assert.assertTrue("higher-index slot must be recovered despite the parked slot", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("higher-index slot must be drained exactly once", + 1, attempts[1].get()); + Assert.assertEquals(0, drained.getCount()); + + // End of cycle: parked slot outstanding -> defer, not complete. + Assert.assertFalse("cycle with a parked slot must defer, not complete", + pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("recovery must not report complete while the parked slot holds data", + pool.isRecoveryCompleteForTesting()); + + // Next cycle: the parked slot is re-probed with a fresh streak. + Assert.assertFalse("re-probed slot restarts its bounded retry streak", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(4, attempts[0].get()); + } + } + @Test public void testLongMaxStartupRecoveryBudgetDoesNotOverflow() throws Exception { createCandidateSlot("default-0"); From db8938ecb95b07e3304939fec87ef64da3b39af3 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 11:23:33 +0100 Subject: [PATCH 46/64] fix(qwp): defer torn-tail zeroing until chain validation proves residue dead 88d6b792 zeroed torn-tail residue inside MmapSegment.openExisting, i.e. for every file recovery opens and before any chain-level check runs. The scan stops at the first bad CRC, so after a mid-file tear in a SEALED chain member the frames past the tear -- individually valid CRCs, real unacked payloads, the only surviving copy -- were classified as residue and durably destroyed, while the FSN-gap check then failed recovery closed anyway. Pre-88d6b792 the same fail-closed brick preserved the bytes for operator extraction; post-88d6b792 the first startup attempt erased them and every later startup failed with a misleading FSN gap. openExisting is now a pure observer: it scans, reports tornTailBytes and never mutates. The zeroing moved to MmapSegment.sanitizeTornTail (same setMemory + msync/fsync barrier, same fail-closed abort, idempotent, refused after appends resume) and SegmentRing.recover invokes it only where validation has proven the residue non-load-bearing: - the segment selected as the resumed active, at the single ring-construction funnel after every chain/manifest check has passed (the active is the only segment that takes appends and the only one a rotation can reseal, so this preserves both original hazard fixes: the two-crash reseal brick and stale-frame FSN resurrection); - sealed members whose frame accounting validated complete (contiguity plus boundary matching prove the suffix is dead bytes), which keeps the legacy-poison self-heal: sanitize, fail closed on first sight, restart proves the chain clean. A tear that cost frames now fails closed before any mutation with every byte left on disk. The three stale comments asserting the old behavior ("a failed recovery never mutates the slot", "bytes that already failed frame validation", "the restart re-proves the chain clean") are rewritten to match reality. Tests: openExisting observe-only (byte-identical file across repeated opens); sanitize zeroes durably, keeps the diagnostic, idempotent, refused after appends; barrier failure fails closed at sanitize level with no barrier at open; mid-file tear in a sealed member fails closed twice with byte-identical .sfa files (operator extraction preserved); proven-dead sealed residue sanitized then heals after one restart; ring recovery durably sanitizes the resumed active with zero appends; frame[0]-corruption evidence now provably survives re-opens. The five 88d6b792 regression scenarios remain pinned at ring/segment level. sf package: 385 tests green. --- .../qwp/client/sf/cursor/MmapSegment.java | 139 +++++++++---- .../qwp/client/sf/cursor/SegmentRing.java | 101 +++++++-- .../qwp/client/sf/cursor/MmapSegmentTest.java | 170 ++++++++++++--- .../qwp/client/sf/cursor/SegmentRingTest.java | 195 +++++++++++++++++- 4 files changed, 510 insertions(+), 95 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 2fab2540..35939395 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -111,11 +111,18 @@ public final class MmapSegment implements QuietCloseable { // attempted-but-invalid frame write (the suffix contains non-zero bytes). // Zero for fresh segments and for cleanly partially-filled // segments (uninitialised tail). Set only by openExisting; visible to - // recovery callers for diagnostics. Records the observation BEFORE - // sanitization -- openExisting zeroes the on-disk residue, so the file - // behind a freshly returned segment never carries it. Final after - // construction. + // recovery callers for diagnostics. openExisting only OBSERVES the + // residue -- it never mutates the file: after a mid-file tear the suffix + // can hold unreachable valid-CRC frames that are the only surviving copy + // of real payloads. Ring recovery decides after the chain fully validates + // whether to zero it (sanitizeTornTail on the resumed active or on a + // proven-dead sealed suffix) or to fail closed leaving the bytes on disk + // for operator extraction. Final after construction. private final long tornTailBytes; + // Set once sanitizeTornTail has durably zeroed the observed residue. + // Written and read only on the recovery path (single-threaded, before + // the segment is published to producer/consumer threads). + private boolean tornTailSanitized; private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes, long baseSeq, long initialCursor, long frameCount, @@ -277,17 +284,18 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { * If recovery observes a torn tail (the suffix after the last valid frame * contains non-zero bytes, indicating an attempted-but-failed frame write * rather than clean unwritten space), a {@code WARN} is emitted with the - * byte count, the observed size is exposed via {@link #tornTailBytes()}, - * and the residue is zeroed on disk (with an msync+fsync barrier) before - * the segment is returned. Sanitizing restores the invariant every fresh - * segment already has -- all bytes past the append cursor are zero. - * Without it, residue that resumed appends do not fully overwrite would - * survive into a sealed segment, whose recovery treats a non-zero suffix - * as fatal corruption (a permanent startup failure), and a byte-aligned - * stale frame with a valid CRC could be silently resurrected at a - * recycled FSN by a later scan. Clean partial fills (writer never - * attempted to write past the last valid frame) do not log, report - * {@code 0}, and skip the barrier. + * byte count and the observed size is exposed via {@link #tornTailBytes()}. + * The residue itself is NOT touched: after a mid-file tear the suffix can + * still hold unreachable frames with valid CRCs -- the only surviving copy + * of real payloads -- so whether it may be destroyed is a chain-level + * decision this method cannot make. {@link SegmentRing} recovery invokes + * {@link #sanitizeTornTail()} only once the chain has fully validated and + * only on residue that validation proves non-load-bearing (the resumed + * active's tail, which appends are about to reclaim anyway, and sealed + * suffixes whose frame accounting is proven complete); every fail-closed + * path leaves the bytes intact on disk for operator extraction. Clean + * partial fills (writer never attempted to write past the last valid + * frame) do not log and report {@code 0}. */ public static MmapSegment openExisting(String path) { return openExisting(FilesFacade.INSTANCE, path); @@ -354,33 +362,19 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { + " [before=" + fileSize + ", after=" + mappedSize + ']'); } if (scan.tornTailBytes > 0) { - // Zero the residue [lastGood, fileSize) and make the zeroes - // durable before anyone can append. Resumed appends restart at - // lastGood but stop wherever the last payload fits, so stale - // residue past that point would otherwise survive a - // seal-via-rotation -- and sealed-segment recovery treats a - // non-zero suffix as fatal corruption, permanently failing - // every subsequent startup. Zeroing also stops a byte-aligned - // stale frame with a valid CRC from being resurrected at a - // recycled FSN by a later recovery scan (the frame envelope - // binds neither position nor FSN). The fsync is load-bearing: - // in MEMORY durability mode rotation does not sync the sealed - // predecessor's data pages, so an unflushed zero-write plus a - // crash would regenerate the exact poisoned state this - // sanitization removes. One-time recovery cost, torn tails - // only; a failed barrier aborts recovery (fail closed) -- - // the zeroes may still reach disk via the page cache, which - // is safe: a retry either sees them (clean tail) or re-zeroes. - Unsafe.getUnsafe().setMemory(addr + scan.lastGood, fileSize - scan.lastGood, (byte) 0); - if (ff.msync(addr, fileSize, false) != 0 || ff.fsync(fd) != 0) { - throw new MmapSegmentException( - "could not sync zeroed torn tail in " + path - + " [errno=" + Os.errno() + ']'); - } + // Observe-only: report the residue, never touch it here. A + // torn tail may be an interrupted append (dead bytes) OR a + // mid-file tear with unreachable valid-CRC frames past it -- + // the only surviving copy of real payloads. Only chain-level + // validation can tell the two apart, so the destroy/preserve + // decision belongs to SegmentRing.recover: it invokes + // sanitizeTornTail on the segment it resumes as active (and + // on proven-dead sealed suffixes) AFTER the chain proves the + // residue is not load-bearing, and fails closed with every + // byte left on disk in all other cases. LOG.warn("SF segment {}: torn tail of {} bytes at offset {} " + "(file size {}, frames recovered {}). " - + "The residue has been zeroed; frames past the tear " - + "(if any) are discarded. " + + "The residue is preserved pending chain validation. " + "Investigate disk health or unexpected writer crash.", path, scan.tornTailBytes, scan.lastGood, fileSize, scan.frameCount); } @@ -669,9 +663,9 @@ public long frameCount() { * fresh segments, memory-backed segments, and cleanly partially-filled * recovered segments. Operators / tests can read this to tell silent * truncation (corruption) from a normal partial fill (no incident). This - * is the pre-sanitization observation: {@link #openExisting} zeroes the - * residue on disk before returning, so re-opening the same file reports - * {@code 0} and a reseal of this segment presents a clean suffix. Sparse + * is the open-time observation and is never updated: a later + * {@link #sanitizeTornTail()} zeroes the on-disk residue (after which a + * re-open reports {@code 0}) but leaves this diagnostic intact. Sparse * holes read back as zeroes; an actual positioned-read failure aborts * recovery instead of being classified as a clean tail. */ @@ -679,6 +673,65 @@ public long tornTailBytes() { return tornTailBytes; } + /** + * Durably zeroes the torn-tail residue this segment was opened with: + * {@code [sizeBytes - tornTailBytes, sizeBytes)} is set to zero and made + * durable with an msync+fsync barrier. Restores the invariant every fresh + * segment already has -- all bytes past the append cursor are zero -- so + * residue that resumed appends do not fully overwrite cannot survive into + * a reseal (where sealed-suffix recovery would treat it as fatal + * corruption on every subsequent startup) and a byte-aligned stale frame + * with a valid CRC cannot be resurrected at a recycled FSN by a later + * scan (the frame envelope binds neither position nor FSN). + *

      + * DESTRUCTIVE by design and therefore never called by + * {@link #openExisting}: after a mid-file tear the residue can hold + * unreachable valid-CRC frames that are the only surviving copy of + * unacked payloads. The caller ({@link SegmentRing} recovery) must first + * prove the residue is not load-bearing -- the resumed active's tail + * (reclaimed by appends anyway) or a sealed suffix whose frame accounting + * validated complete against the chain. Must run before any append + * resumes; appending first would put live frames inside the zeroed range, + * so that ordering is rejected. Idempotent; no-op when no residue was + * observed. The fsync is load-bearing: in MEMORY durability mode rotation + * does not sync the sealed predecessor's data pages, so an unflushed + * zero-write plus a crash would regenerate the exact poisoned state this + * sanitization removes. A failed barrier throws and the caller fails + * closed; the zeroes may still reach disk via the page cache, which is + * safe: a retry either observes a clean tail or re-zeroes. + */ + public void sanitizeTornTail() { + if (tornTailBytes == 0 || tornTailSanitized) { + return; + } + long residueStart = sizeBytes - tornTailBytes; + if (appendCursor != residueStart) { + throw new MmapSegmentException( + "torn tail must be sanitized before appends resume in " + path + + " [appendCursor=" + appendCursor + + ", residueStart=" + residueStart + ']'); + } + Unsafe.getUnsafe().setMemory(mmapAddress + residueStart, tornTailBytes, (byte) 0); + if (filesFacade.msync(mmapAddress, sizeBytes, false) != 0 || filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException( + "could not sync zeroed torn tail in " + path + + " [errno=" + Os.errno() + ']'); + } + tornTailSanitized = true; + LOG.warn("SF segment {}: zeroed {} bytes of torn-tail residue at offset {}; " + + "frames past the tear (if any) are discarded", + path, tornTailBytes, residueStart); + } + + /** + * True when recovery observed torn-tail residue that + * {@link #sanitizeTornTail()} has not yet durably zeroed -- i.e. the + * on-disk suffix still carries the observed bytes. + */ + public boolean hasUnsanitizedTornTail() { + return tornTailBytes > 0 && !tornTailSanitized; + } + /** * Scans the header and frames through a bounded positioned-read buffer. * Sparse file holes are returned by {@code pread}/{@code ReadFile} as zero diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 0c75bc74..1aee2ab3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -194,11 +194,14 @@ public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, lo * Exhaustively discovers and validates an SF slot without mutating frame * data. Only after enumeration, opens, CRC scans, contiguity and manifest * boundaries all succeed does it migrate a legacy chain or discard - * validated spares. One deliberate exception to the no-mutation rule: - * {@link MmapSegment#openExisting} zeroes torn-tail residue -- suffix - * bytes that already failed frame validation and can never be replayed -- - * as each segment opens, so even a recovery that later fails may have - * sanitized unusable residue. Valid frames are never touched. + * validated spares. Torn-tail residue is handled destroy-last: + * {@link MmapSegment#openExisting} only observes it, and recovery zeroes + * it exclusively where validation has proven it non-load-bearing -- the + * resumed active's tail (about to be reclaimed by appends) and sealed + * suffixes whose frame accounting validated complete against the chain. + * A tear that cost frames (e.g. a mid-file tear in a sealed member) + * fails closed BEFORE any of that, leaving every byte on disk for + * operator extraction. */ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { return recover(filesFacade, sfDir, maxBytesPerSegment, null); @@ -240,7 +243,10 @@ static Recovery recover( // Files whose own bytes prove corruption (bad magic, sub-header size, // negative baseSeq). They are excluded from the chain and quarantined // to .corrupt — but only AFTER the surviving chain validates (or - // resolves to EMPTY), so a failed recovery never mutates the slot. + // resolves to EMPTY), so a failed recovery never mutates the slot + // (single exception: proven-dead sealed residue is zeroed just before + // the fail-closed first-sight throw in sanitizeSealedResidue -- bytes + // the already-validated chain proves no replay can ever need). // Whether a quarantined file was load-bearing is decided by the // manifest-boundary / contiguity checks below, not by the skip itself. // Operational open/stat/read/mmap errors, observed size instability, @@ -347,6 +353,14 @@ static Recovery recover( active = chain.get(chain.size() - 1); headBase = chain.get(0).baseSeq(); activeBase = active.baseSeq(); + // Legacy sealed members carrying pre-manifest torn + // residue: contiguity just proved their frame accounting + // complete, so the residue is dead bytes. Zero it now + // (silently -- legacy slots predate the fail-closed + // sealed-suffix contract) so the migrated chain presents + // the all-zero sealed suffixes the manifest-era + // invariant requires. + sanitizeSealedResidue(chain, false); } else { active = chooseEmptyInitial(all, sfDir); if (active == null) { @@ -473,21 +487,21 @@ static Recovery recover( "missing expected SF active/tail segment at base " + activeBase); } } - // Sealed members must present an all-zero suffix. Fresh - // segments are zero-allocated and openExisting zeroes any - // torn-tail residue before a recovered active can take - // appends, so a reseal of a recovered segment cannot carry - // residue here. A non-zero suffix therefore means the file - // was mutated outside the writer protocol (or is legacy - // pre-sanitization poison; this run's openExisting has - // already zeroed it, so the restart after this fail-closed - // throw re-proves the chain clean). validateContiguous above - // independently catches sealed segments that LOST frames. - for (int i = 0, n = chain.size() - 1; i < n; i++) { - if (chain.get(i).tornTailBytes() > 0) { - throw new SfRecoveryException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); - } - } + // Sealed members must present an all-zero suffix: fresh + // segments are zero-allocated and this recovery sanitizes + // the resumed active before it takes appends, so a reseal + // can never carry residue forward. Reaching this point + // proves every sealed member's frame accounting is complete + // (validateContiguous plus the head/active boundary matching + // above), so any suffix residue here is dead bytes -- legacy + // pre-sanitization poison or an out-of-protocol scribble + // past the last frame, never lost frames. Zero it durably, + // then still fail closed on first sight so the incident is + // surfaced; the restart then proves the chain clean. A tear + // that DID cost frames never gets here: the contiguity or + // boundary checks above throw first, with every byte left + // on disk for operator extraction. + sanitizeSealedResidue(chain, true); for (int i = 0, n = chain.size(); i < n; i++) { chain.get(i).markManifestRequired(); } @@ -528,6 +542,21 @@ static Recovery recover( } quarantineCorrupt(filesFacade, corruptPaths); + // The resumed active is the only segment that takes appends and + // the only one a later rotation reseals, so recovery zeroes ITS + // residue exactly here -- every chain/manifest check has passed + // and the ring is about to be exposed. Resumed appends restart + // at lastGood but stop wherever the last payload fits, so + // unzeroed residue would survive a seal-via-rotation (bricking + // the next startup's sealed-suffix check) and a byte-aligned + // stale frame with a valid CRC could be resurrected at a + // recycled FSN by a later scan. The durable barrier inside + // sanitizeTornTail is load-bearing in MEMORY durability mode, + // where rotation does not sync the sealed predecessor's data + // pages. A failed barrier aborts recovery (fail closed); the + // retry re-observes the same residue because openExisting never + // mutates. + active.sanitizeTornTail(); for (int i = 1, n = chain.size(); i < n; i++) { chain.get(i - 1).linkSuccessor(chain.get(i)); } @@ -746,6 +775,36 @@ private static void validateContiguous(ObjList segments) { } } + /** + * Durably zeroes torn-tail residue on the chain's sealed members (every + * element but the last, which is the active). Callers must have already + * proven each sealed member's frame accounting complete -- contiguity + * plus head/active boundary matching -- which is exactly what makes the + * residue provably dead: a tear that cost frames breaks those checks and + * fails recovery before any mutation, preserving the bytes (potentially + * the only copy of unreachable valid-CRC frames) for operator + * extraction. With {@code failClosedOnSight} the incident is still + * surfaced as a first-sight startup failure after sanitizing (the + * restart then proves the chain clean); without it the chain proceeds + * immediately (legacy migration, which predates the sealed-suffix + * contract). + */ + private static void sanitizeSealedResidue(ObjList chain, boolean failClosedOnSight) { + String firstTornPath = null; + for (int i = 0, n = chain.size() - 1; i < n; i++) { + MmapSegment sealed = chain.get(i); + if (sealed.tornTailBytes() > 0) { + sealed.sanitizeTornTail(); + if (firstTornPath == null) { + firstTornPath = sealed.path(); + } + } + } + if (failClosedOnSight && firstTornPath != null) { + throw new SfRecoveryException("corrupt torn tail in sealed SF segment " + firstTornPath); + } + } + private static void warnPostRecoveryCleanupFailure(Throwable cleanupError) { try { LOG.warn("post-recovery cleanup failed; leftover files will be " diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 386b6eef..b63a0e97 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -35,6 +35,7 @@ import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -343,6 +344,16 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex + "hot-spare leftover; got " + seg.tornTailBytes(), seg.tornTailBytes() > 0L); } + // A second open must see the SAME evidence: openExisting is + // observe-only, so the valid-CRC frames past the corrupt + // frame[0] -- the only copy of that data -- stay on disk for + // operator extraction (chain-level recovery of such a member + // fails closed without mutating it). + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(0L, seg.frameCount()); + assertEquals("observe-only recovery must preserve the residue", + 4096L - MmapSegment.HEADER_SIZE, seg.tornTailBytes()); + } assertTrue("openExisting must not unlink the corrupt file", Files.exists(path)); } finally { @@ -492,12 +503,57 @@ public void testRecoveryDoesNotFlagFreshUnusedSegment() throws Exception { } @Test - public void testRecoveryZeroesTornTailResidueOnDisk() throws Exception { + public void testOpenExistingObservesTornTailWithoutMutation() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Recovery must not only REPORT the torn tail, it must sanitize - // it: the first open still reports the observed residue (operator - // signal), but a second open of the untouched file must find a - // clean zero suffix. Pre-fix the residue survived forever. + // openExisting must be a pure observer: a torn tail can hide + // unreachable valid-CRC frames past a mid-file tear -- the only + // copy of real payloads -- and whether they may be destroyed is + // a chain-level decision (SegmentRing sanitizes the segment it + // resumes as active, plus proven-dead sealed residue, and fails + // closed preserving the bytes otherwise). Repeated opens must + // keep reporting the same observation over identical bytes. + String path = tmpDir + "/seg-observe.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + long lastGood; + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + for (int i = 0; i < 3; i++) { + fillPattern(buf, 16, i); + seg.tryAppend(buf, 16); + } + lastGood = seg.publishedOffset(); + long addr = seg.address(); + for (long off = lastGood; off + 4 <= 4096; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + seg.msync(); + } + byte[] before = java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(path)); + for (int open = 0; open < 2; open++) { + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("open #" + open + " must report the residue", + 4096L - lastGood, seg.tornTailBytes()); + assertEquals(3L, seg.frameCount()); + assertTrue(seg.hasUnsanitizedTornTail()); + } + assertArrayEquals( + "observe-only recovery must not change a single byte (open #" + open + ')', + before, java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(path))); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testSanitizeTornTailZeroesResidueOnDisk() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Once ring recovery proves the residue is not load-bearing it + // calls sanitizeTornTail: the open keeps reporting the observed + // residue (operator signal), but after sanitization a later open + // of the untouched file must find a clean zero suffix and intact + // frames. String path = tmpDir + "/seg-zeroed.sfa"; long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); long lastGood; @@ -515,11 +571,18 @@ public void testRecoveryZeroesTornTailResidueOnDisk() throws Exception { seg.msync(); } try (MmapSegment seg = MmapSegment.openExisting(path)) { - assertEquals("first recovery must report the observed residue", + assertEquals("recovery must report the observed residue", + 4096L - lastGood, seg.tornTailBytes()); + assertTrue(seg.hasUnsanitizedTornTail()); + seg.sanitizeTornTail(); + assertEquals("the observation must survive sanitization for diagnostics", 4096L - lastGood, seg.tornTailBytes()); + assertFalse(seg.hasUnsanitizedTornTail()); + // Idempotent: a second call is a no-op, not a re-zero. + seg.sanitizeTornTail(); } try (MmapSegment seg = MmapSegment.openExisting(path)) { - assertEquals("first recovery must have zeroed the residue on disk", + assertEquals("sanitize must have zeroed the residue on disk", 0L, seg.tornTailBytes()); assertEquals(lastGood, seg.publishedOffset()); assertEquals(3L, seg.frameCount()); @@ -561,9 +624,12 @@ public void testResealGapResidueCannotSurviveRecoveryAppends() throws Exception } // Recovery #1 + session 2: fill the segment to its rotation // point. The 4th frame ends 12 bytes short of the file end -- - // a region session 2 never overwrites. + // a region session 2 never overwrites. Sanitize before the + // appends, exactly as SegmentRing.recover does for the + // segment it resumes as active. try (MmapSegment seg = MmapSegment.openExisting(path)) { assertEquals(2L, seg.frameCount()); + seg.sanitizeTornTail(); fillPattern(buf, 16, 2); assertTrue(seg.tryAppend(buf, 16) >= 0); fillPattern(buf, 16, 3); @@ -615,9 +681,11 @@ public void testDiscardedStaleFrameNotResurrectedByLaterRecovery() throws Except seg.msync(); } // Recovery #1: B fails CRC, so the scan stops at B; B and C - // are discarded (and, with sanitization, zeroed). + // are discarded, and the resume-as-active sanitization (the + // same call SegmentRing.recover makes) zeroes them on disk. try (MmapSegment seg = MmapSegment.openExisting(path)) { assertEquals("scan must stop at the torn frame", 1L, seg.frameCount()); + seg.sanitizeTornTail(); // The resumed writer re-issues FSN 1 with a fresh payload // of the old B's exact size -- the byte-aligned case. fillPattern(buf, 16, 7); @@ -639,14 +707,15 @@ public void testDiscardedStaleFrameNotResurrectedByLaterRecovery() throws Except } @Test - public void testTornTailZeroingSyncFailureAbortsRecovery() throws Exception { + public void testSanitizeTornTailSyncFailureFailsClosed() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Sanitization is load-bearing: if the zeroes cannot be made - // durable, recovery must fail closed rather than hand back a - // segment whose reseal could permanently fail the next startup. - // A failed attempt may still leave zeroes in the page cache; - // that is safe (a retry either sees a clean tail or re-zeroes), - // so the follow-up open with a healthy facade must succeed. + // The sanitize barrier is load-bearing: if the zeroes cannot be + // made durable, ring recovery must fail closed rather than expose + // a segment whose reseal could permanently fail the next startup. + // openExisting itself runs no barrier -- it never writes. A + // failed attempt may still leave zeroes in the page cache; that + // is safe (a retry either sees a clean tail or re-zeroes), so + // the follow-up open with a healthy facade must succeed. long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { String msyncPath = tmpDir + "/seg-zero-msync-fail.sfa"; @@ -664,12 +733,18 @@ public void testTornTailZeroingSyncFailureAbortsRecovery() throws Exception { FaultyFilesFacade msyncFailure = new FaultyFilesFacade(); msyncFailure.failOnMsync = true; - try { - MmapSegment.openExisting(msyncFailure, msyncPath).close(); - fail("expected recovery to abort when the zeroed tail cannot be msync'd"); - } catch (MmapSegmentException expected) { - assertTrue(expected.getMessage(), - expected.getMessage().contains("zeroed torn tail")); + try (MmapSegment seg = MmapSegment.openExisting(msyncFailure, msyncPath)) { + assertEquals("observe-only open must not run the zeroing barrier", + 0, msyncFailure.msyncCalls); + try { + seg.sanitizeTornTail(); + fail("expected sanitize to abort when the zeroed tail cannot be msync'd"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("zeroed torn tail")); + } + assertTrue("a failed barrier must leave the residue unsanitized", + seg.hasUnsanitizedTornTail()); } assertEquals(1, msyncFailure.msyncCalls); assertEquals(0, msyncFailure.fsyncCalls); @@ -679,12 +754,16 @@ public void testTornTailZeroingSyncFailureAbortsRecovery() throws Exception { FaultyFilesFacade fsyncFailure = new FaultyFilesFacade(); fsyncFailure.failOnFsync = true; - try { - MmapSegment.openExisting(fsyncFailure, fsyncPath).close(); - fail("expected recovery to abort when the zeroed tail cannot be fsync'd"); - } catch (MmapSegmentException expected) { - assertTrue(expected.getMessage(), - expected.getMessage().contains("zeroed torn tail")); + try (MmapSegment seg = MmapSegment.openExisting(fsyncFailure, fsyncPath)) { + try { + seg.sanitizeTornTail(); + fail("expected sanitize to abort when the zeroed tail cannot be fsync'd"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("zeroed torn tail")); + } + assertTrue("a failed barrier must leave the residue unsanitized", + seg.hasUnsanitizedTornTail()); } assertEquals(1, fsyncFailure.msyncCalls); assertEquals(1, fsyncFailure.fsyncCalls); @@ -697,6 +776,41 @@ public void testTornTailZeroingSyncFailureAbortsRecovery() throws Exception { }); } + @Test + public void testSanitizeTornTailRefusedAfterAppendsResume() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Zeroing [residueStart, EOF) after appends resumed would destroy + // freshly appended frames, so that ordering is rejected outright: + // sanitization is a recovery-time-only operation that must run + // before the segment takes traffic. + String path = tmpDir + "/seg-sanitize-late.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); + long addr = seg.address(); + Unsafe.getUnsafe().putInt(addr + seg.publishedOffset(), 0xCAFEBABE); + seg.msync(); + } + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertTrue(seg.hasUnsanitizedTornTail()); + fillPattern(buf, 16, 1); + assertTrue(seg.tryAppend(buf, 16) >= 0); + try { + seg.sanitizeTornTail(); + fail("sanitize after appends must be refused"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("before appends resume")); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testRecoverySignalsTornTailWithByteCount() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index 943b45c3..aeb77b00 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -342,9 +343,9 @@ public void testResealedTornRecoveredSegmentDoesNotBrickNextRecovery() throws Ex // crash #2 -> recovery #2 used to throw "corrupt torn tail in // sealed SF segment" on EVERY startup, because nothing ever // sanitized crash #1's residue and the sealed-suffix check - // correctly refuses non-zero sealed tails. openExisting now - // zeroes the residue at recovery #1, so recovery #2 must - // succeed and see the full chain. + // correctly refuses non-zero sealed tails. Recovery #1 now + // sanitizes the segment it resumes as active before any + // append, so recovery #2 must succeed and see the full chain. long segSize = MmapSegment.HEADER_SIZE + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + 12; // reseal gap: a 5th 24-byte frame can never fit @@ -391,6 +392,190 @@ public void testResealedTornRecoveredSegmentDoesNotBrickNextRecovery() throws Ex }); } + @Test + public void testMidFileTearInSealedMemberFailsClosedWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Bit-rot mid-file in a SEALED chain member: the scan stops at + // the first bad CRC, so frames past the tear -- individually + // valid CRCs, real unacked payloads, the ONLY surviving copy -- + // are classified as torn-tail residue. Recovery must fail closed + // on the FSN gap the lost frames create BEFORE any sanitization + // decision, leaving every byte on disk for operator extraction, + // and must keep doing so identically on retry. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16); + String m0Path = tmpDir + "/m0.sfa"; + String m1Path = tmpDir + "/m1.sfa"; + String m2Path = tmpDir + "/m2.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 3); + MmapSegment s0 = MmapSegment.create(m0Path, 0, segSize); + for (int i = 0; i < 4; i++) s0.tryAppend(buf, 16); + s0.close(); + MmapSegment s1 = MmapSegment.create(m1Path, 4, segSize); + for (int i = 0; i < 4; i++) s1.tryAppend(buf, 16); + s1.close(); + MmapSegment s2 = MmapSegment.create(m2Path, 8, segSize); + s2.tryAppend(buf, 16); + s2.close(); + // First recovery migrates the legacy chain, creates the + // manifest and validates clean: sealed m0+m1, active m2. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull(ring); + assertEquals(2, ring.getSealedSegments().size()); + } + // Bit-rot strikes sealed m0 mid-file: clobber the CRC of its + // SECOND frame. Frames 2..3 keep valid CRCs but are + // unreachable to the sequential scan. + long frame1Offset = MmapSegment.HEADER_SIZE + + MmapSegment.FRAME_HEADER_SIZE + 16; + int fd = Files.openRW(m0Path); + assertTrue("openRW must succeed", fd >= 0); + long bad = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(bad, 0xDEADBEEF); + assertEquals(4L, Files.write(fd, bad, 4, frame1Offset)); + Files.fsync(fd); + } finally { + Unsafe.free(bad, 4, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + byte[] m0Before = readFileBytes(m0Path); + byte[] m1Before = readFileBytes(m1Path); + byte[] m2Before = readFileBytes(m2Path); + for (int attempt = 0; attempt < 2; attempt++) { + try { + Misc.free(SegmentRing.openExisting(tmpDir, segSize)); + throw new AssertionError( + "mid-file tear in a sealed member must fail recovery"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("FSN gap")); + } + assertArrayEquals("attempt #" + attempt + + ": failed recovery must not mutate the torn sealed member", + m0Before, readFileBytes(m0Path)); + assertArrayEquals(m1Before, readFileBytes(m1Path)); + assertArrayEquals(m2Before, readFileBytes(m2Path)); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testProvenDeadSealedResidueSanitizedThenHealsAfterOneRestart() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Legacy pre-sanitization poison: a sealed member whose frame + // accounting is COMPLETE (contiguity + boundaries prove no frame + // lost) but whose suffix gap carries residue from a pre-fix + // client. The residue is provably dead bytes, so recovery zeroes + // it durably, still fails closed on first sight to surface the + // incident, and the restart proves the chain clean. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // sealed suffix gap that can never fit a frame + String m0Path = tmpDir + "/p0.sfa"; + String m1Path = tmpDir + "/p1.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 5); + MmapSegment s0 = MmapSegment.create(m0Path, 0, segSize); + for (int i = 0; i < 4; i++) s0.tryAppend(buf, 16); + long gapStart = s0.publishedOffset(); + s0.close(); + MmapSegment s1 = MmapSegment.create(m1Path, 4, segSize); + s1.tryAppend(buf, 16); + s1.close(); + // First recovery creates the manifest over the clean chain. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull(ring); + assertEquals(1, ring.getSealedSegments().size()); + } + // Poison the sealed suffix gap the way a pre-fix client's + // reseal-after-recovery did. + int fd = Files.openRW(m0Path); + assertTrue("openRW must succeed", fd >= 0); + long junk = Unsafe.malloc(12, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 3; i++) { + Unsafe.getUnsafe().putInt(junk + i * 4L, 0xCAFEBABE); + } + assertEquals(12L, Files.write(fd, junk, 12, gapStart)); + Files.fsync(fd); + } finally { + Unsafe.free(junk, 12, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + // Recovery #1: proven-dead residue is sanitized, incident + // still fails closed on first sight. + try { + Misc.free(SegmentRing.openExisting(tmpDir, segSize)); + throw new AssertionError("poisoned sealed suffix must fail closed on first sight"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("corrupt torn tail in sealed SF segment")); + } + // The heal: residue durably zeroed, all frames intact. + try (MmapSegment seg = MmapSegment.openExisting(m0Path)) { + assertEquals("frames must be untouched", 4L, seg.frameCount()); + assertEquals("proven-dead residue must be zeroed on disk", + 0L, seg.tornTailBytes()); + } + // Recovery #2: the restart proves the chain clean. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull("restart after the sanitizing fail-closed throw must succeed", ring); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(4, ring.getSealedSegments().get(0).frameCount()); + assertEquals(4, ring.getActive().baseSeq()); + assertEquals(1, ring.getActive().frameCount()); + assertEquals(5, ring.nextSeqHint()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testRingRecoverySanitizesResumedActiveTornTail() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The funnel guarantee: whatever segment recovery selects as the + // resumed active has its torn residue durably zeroed before the + // ring is exposed -- even if the session then crashes without a + // single append, the reseal/resurrection hazards are gone. + String path = tmpDir + "/a0.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 9); + try (MmapSegment s0 = MmapSegment.create(path, 0, 4096)) { + s0.tryAppend(buf, 16); + s0.tryAppend(buf, 16); + long addr = s0.address(); + for (long off = s0.publishedOffset(); off + 4 <= 4096; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + s0.msync(); + } + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, 4096)) { + assertNotNull(ring); + assertEquals("the observation must survive for diagnostics", + true, ring.getActive().tornTailBytes() > 0); + } + // No appends happened; the zeroing must already be durable. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(2L, seg.frameCount()); + assertEquals("ring recovery must have sanitized the resumed active", + 0L, seg.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testOpenExistingDetectsFsnGap() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -1053,6 +1238,10 @@ public void testFindSegmentContainingSingleFrameSegments() throws Exception { }); } + private static byte[] readFileBytes(String path) throws java.io.IOException { + return java.nio.file.Files.readAllBytes(Paths.get(path)); + } + private static void fillPattern(long addr, int len, int seed) { for (int i = 0; i < len; i++) { Unsafe.getUnsafe().putByte(addr + i, (byte) (seed * 31 + i + 17)); From e0ebdf00367b4753be8f6652097bfa8a0d4fb441 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 12:02:23 +0100 Subject: [PATCH 47/64] test(qwp): close the C-2 durability test gaps with mutant-killer coverage A live mutation accident proved the suite blind to durability-critical code: 88d6b792 shipped requestSyncBeforeRotation neutralized ('return false; // MUTANT: gate neutralized') and 369 tests stayed green; only human re-reading caught it (71cfbe2e). Re-running that mutant against today's suite still passed 385/385, as did deleting SfManifest.update's monotonic clamp. Every gap below is now covered, and both mutants were re-applied after writing the tests to verify they die. - SegmentManagerPeriodicSyncTest.testRotationGateDefersRotationUntil- PredecessorDurable: kills the gate mutant at three points (rotation must backpressure while the predecessor is non-durable, the gate's sync request must run the barrier before the interval deadline, the retried append must rotate). Verified: mutant now fails 'expected -1 but was 2'. - SegmentManagerPeriodicSyncTest.testSyncPassStopsAtFirstFailureThen- RetryCoversAllSegments: >= 2 non-durable live segments (recovered segments start non-durable) prove the pass aborts at the first barrier failure and the healed retry covers every segment before the producer unlatches. - SfManifestClampTest: direct pin on the update() monotonic clamp, including independent per-field clamping and durable persistence across reopen. Verified: clamp-deletion mutant now fails 'expected 10 but was 5'. SfManifest and the five members under test are widened to public within the already-exported internal package (JPMS forbids same-package test classes; matches MmapSegment/SegmentRing). - CursorSendEngineTest.testCheckDurabilitySurfacesLatchedFailureTo- SenderEntryPoints: pins the engine delegation seam behind flush()/awaitAckedFsn() (zero references before): quiet when clean, throws the latched instance repeatedly until cleared. - SegmentManagerCloseRaceTest.testSecondBoundedJoinReapsWorker- FinishingExitCleanups: first join times out against a worker parked in its exit cleanups (workerLoopExited set before cleanups run), the second fixed-budget join must reap and close() must free the scratch itself. Every existing close-race test only reached the first join. - CursorSendEngineClosePartialEnumerationTest: torn close-time directory enumeration (findNext < 0 mid-walk, injected via a FilesFacade proxy) must drive ZERO unlinks, keep the watermark, and a successor's fully-drained close retries the cleanup. The sibling unlink-failure test only covered failing unlinks via the root-skipped permission trick. - SourceHygieneTest: tripwire that fails the build on 'MUTANT' markers in main sources -- the exact artifact 88d6b792 shipped. sf package: 392 tests green (385 + 7 new). --- .../qwp/client/sf/cursor/SfManifest.java | 17 +- .../client/test/SourceHygieneTest.java | 88 ++++++++ ...SendEngineClosePartialEnumerationTest.java | 163 +++++++++++++++ .../sf/cursor/CursorSendEngineTest.java | 29 +++ .../cursor/SegmentManagerCloseRaceTest.java | 101 ++++++++++ .../SegmentManagerPeriodicSyncTest.java | 190 ++++++++++++++++++ .../client/sf/cursor/SfManifestClampTest.java | 109 ++++++++++ 7 files changed, 691 insertions(+), 6 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/SourceHygieneTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java index b253075a..b6afda99 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -38,8 +38,13 @@ * update. Recovery selects the valid record with the greatest generation. * The separate 4 KiB slots prevent one aligned 512-byte or 4 KiB sector tear * from erasing both the update and the previous committed boundary. + *

      + * Public within this exported-internal package (like {@link MmapSegment} and + * {@link SegmentRing}) so the boundary contract -- notably the monotonic + * clamp in {@link #update} -- can be pinned by direct unit tests; it is not + * part of the supported client API. */ -final class SfManifest implements QuietCloseable { +public final class SfManifest implements QuietCloseable { static final String FILE_NAME = "sf-manifest.bin"; private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class); private static final int CRC_OFFSET = 60; @@ -71,7 +76,7 @@ private SfManifest(FilesFacade filesFacade, String path, int fd, this.writeScratch = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); } - static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { + public static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { String path = dir + "/" + FILE_NAME; int fd = filesFacade.openRWExclusive(path); if (fd < 0) { @@ -104,7 +109,7 @@ static SfManifest create(FilesFacade filesFacade, String dir, long headBase, lon } } - static SfManifest open(FilesFacade filesFacade, String dir) { + public static SfManifest open(FilesFacade filesFacade, String dir) { String path = dir + "/" + FILE_NAME; if (!filesFacade.exists(path)) { return null; @@ -168,7 +173,7 @@ static SfManifest open(FilesFacade filesFacade, String dir) { } } - long activeBase() { + public long activeBase() { return activeBase; } @@ -184,7 +189,7 @@ public synchronized void close() { } } - long headBase() { + public long headBase() { return headBase; } @@ -199,7 +204,7 @@ static boolean removeFile(FilesFacade filesFacade, String dir) { return filesFacade.remove(path) || !filesFacade.exists(path); } - synchronized void update(long newHeadBase, long newActiveBase) { + public synchronized void update(long newHeadBase, long newActiveBase) { if (closed) { throw new IllegalStateException("SF manifest is closed"); } diff --git a/core/src/test/java/io/questdb/client/test/SourceHygieneTest.java b/core/src/test/java/io/questdb/client/test/SourceHygieneTest.java new file mode 100644 index 00000000..9351e95d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/SourceHygieneTest.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertTrue; + +/** + * Tripwire against mutation-tooling edits leaking into shipped sources. + *

      + * Commit {@code 88d6b792} accidentally captured a concurrent mutation-testing + * edit that neutralized {@code SegmentRing.requestSyncBeforeRotation} to + * {@code return false; // MUTANT: gate neutralized} -- injected into a shared + * worktree between test validation and {@code git add}. The full suite stayed + * green and only human re-reading caught it ({@code 71cfbe2e}). Mutation + * tools mark their edits precisely so they can be found; this test makes that + * marker a build failure instead of a code-review lottery ticket. + */ +public class SourceHygieneTest { + + private static final String MARKER = "MUTANT"; + + @Test + public void testNoMutationToolMarkersInMainSources() throws IOException { + // Surefire runs with the module directory (core/) as cwd. + Path root = Paths.get("src", "main", "java"); + if (!Files.isDirectory(root)) { + root = Paths.get("core", "src", "main", "java"); + } + assertTrue("main source root not found from " + Paths.get("").toAbsolutePath() + + " -- fix the path resolution rather than skipping the tripwire", + Files.isDirectory(root)); + + final List offenders = new ArrayList<>(); + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.toString().endsWith(".java")) { + List lines = Files.readAllLines(file, StandardCharsets.UTF_8); + for (int i = 0; i < lines.size(); i++) { + if (lines.get(i).contains(MARKER)) { + offenders.add(file + ":" + (i + 1) + " " + lines.get(i).trim()); + } + } + } + return FileVisitResult.CONTINUE; + } + }); + assertTrue("mutation-tool markers must never reach main sources " + + "(a neutralized durability gate shipped exactly this way in 88d6b792); " + + "offending lines:\n" + String.join("\n", offenders), + offenders.isEmpty()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java new file mode 100644 index 00000000..9aefc2c1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Close-time unlink under a TORN directory enumeration ({@code findNext} + * fails after the listing already produced entries). A partial listing must + * not drive any unlink: removing only the files the walk happened to see + * could delete the segment holding the highest frame while a lower one + * survives, leaving residual state the retained ack watermark can no longer + * vouch for. The contract is all-or-nothing: abort the cleanup, keep every + * {@code .sfa} file and the watermark, and let the next recovery (or a + * successor's fully-drained close) retry. + *

      + * The sibling {@link CursorSendEngineCloseUnlinkFailureTest} injects a + * failing UNLINK (permission trick, root-skipped); nothing exercised the + * enumeration-abort branch itself. Same determinism trick as the sibling: + * the shared manager is never started, so no worker touches the slot and no + * concurrent enumeration can trip the armed fault. + */ +public class CursorSendEngineClosePartialEnumerationTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-engine-close-enum-fault-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test(timeout = 20_000L) + public void testTornCloseTimeEnumerationUnlinksNothingAndSuccessorRetries() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final AtomicBoolean failFindNext = new AtomicBoolean(); + final AtomicInteger sfaRemoveAttempts = new AtomicInteger(); + FilesFacade faultFacade = (FilesFacade) Proxy.newProxyInstance( + FilesFacade.class.getClassLoader(), + new Class[]{FilesFacade.class}, + (proxy, method, args) -> { + if (failFindNext.get() && "findNext".equals(method.getName())) { + return -1; + } + if ("remove".equals(method.getName()) + && args != null && args.length == 1 + && args[0] instanceof String + && ((String) args[0]).endsWith(".sfa")) { + sfaRemoveAttempts.incrementAndGet(); + } + try { + return method.invoke(FilesFacade.INSTANCE, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + + long segSize = 4096L; + String slot = tmpDir + "/slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + SegmentManager manager = new SegmentManager( + segSize, + SegmentManager.DEFAULT_POLL_NANOS, + segSize * 4L, + faultFacade, + System::nanoTime); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + CursorSendEngine engine = new CursorSendEngine(slot, segSize, manager); + boolean engineClosed = false; + try { + Assert.assertEquals(0L, engine.appendBlocking(buf, 16)); + Assert.assertTrue(engine.acknowledge(0L)); + + // Fully drained close, but the directory listing tears + // mid-walk: the cleanup must abort BEFORE the first + // unlink. + failFindNext.set(true); + engine.close(); + engineClosed = true; + Assert.assertTrue("close must complete despite the aborted cleanup", + engine.isCloseCompleted()); + Assert.assertEquals( + "a torn enumeration must drive ZERO segment unlinks -- " + + "removing only the files the walk happened to see can " + + "strand residual state the watermark cannot vouch for", + 0, sfaRemoveAttempts.get()); + Assert.assertTrue("the segment file must survive the aborted cleanup", + Files.exists(slot + "/sf-initial.sfa")); + } finally { + if (!engineClosed) { + engine.close(); + } + } + + // Heal the directory walk; a successor adopts the slot, + // recovers the residual (fully acknowledged) state, and its + // own fully-drained close retries the cleanup successfully. + failFindNext.set(false); + CursorSendEngine successor = new CursorSendEngine(slot, segSize, manager); + boolean successorClosed = false; + try { + Assert.assertTrue("successor must recover the residual slot state", + successor.wasRecoveredFromDisk()); + successor.close(); + successorClosed = true; + Assert.assertTrue(successor.isCloseCompleted()); + Assert.assertFalse( + "successor's fully-drained close must retry and complete the unlink", + Files.exists(slot + "/sf-initial.sfa")); + } finally { + if (!successorClosed) { + successor.close(); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + manager.close(); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index db570367..0cf5ab69 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -105,6 +105,35 @@ public void testAcknowledgePropagatesToRing() throws Exception { }); } + @Test + public void testCheckDurabilitySurfacesLatchedFailureToSenderEntryPoints() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // QwpWebSocketSender.flushAndGetSequence()/awaitAckedFsn() call + // engine.checkDurability() as their FIRST durability act, so this + // delegation seam is what stands between a latched periodic + // data-barrier failure and a producer that keeps publishing into + // an unsyncable slot. It had zero test references before this + // pin. Contract: quiet when clean, throws the LATCHED instance + // (not a copy) on every call until the manager's healed pass + // clears it -- callers poll it, so it must be repeatable, not + // one-shot. + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) { + engine.checkDurability(); // clean: must not throw + MmapSegmentException failure = new MmapSegmentException("injected data-sync failure"); + engine.getRingForTesting().recordDurabilityFailureForTesting(failure); + for (int i = 0; i < 2; i++) { + try { + engine.checkDurability(); + fail("latched durability failure must surface on call #" + i); + } catch (MmapSegmentException expected) { + assertTrue("the latched instance itself must surface", + expected == failure); + } + } + } + }); + } + @Test public void testAppendChecksLatchedDurabilityFailureBeforePublishing() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index f53e0a00..b8c0a0c1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -693,6 +693,107 @@ private static Object readInServiceRing(SegmentManager manager) { return manager.getInServiceRingForTesting(); } + /** + * Pins the SECOND bounded join in {@link SegmentManager#close()}: when + * the first (tunable) join times out but the worker has already left its + * service loop ({@code workerLoopExited}) and is running only its finite + * exit cleanups, close() must NOT hand off and walk away -- it gives the + * cleanups a second, fixed-budget join and reaps the worker once they + * finish. Without it, a bounded-join timeout landing mid-cleanup reaps + * the worker while an engine's flock release is still in flight, so + * callers observe a stale not-yet-closed state and schedule spurious + * flock-release retries. Every existing close-race test parks the worker + * MID-PASS (first-join territory); none reached this branch. + */ + @Test(timeout = 20_000L) + public void testSecondBoundedJoinReapsWorkerFinishingExitCleanups() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/second-join-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupEntered = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + AtomicReference err = new AtomicReference<>(); + Thread releaser = null; + try { + manager.register(ring, slot); + manager.start(); + // Deferred exit cleanup that parks: once the worker runs it, + // workerLoopExited is already true (set before cleanups) and + // the thread is alive in its finite exit phase -- exactly the + // second-join state. + Assert.assertTrue("live worker must accept the exit-cleanup handoff", + manager.deferUntilWorkerExit(() -> { + cleanupEntered.countDown(); + try { + if (!releaseCleanup.await(10, TimeUnit.SECONDS)) { + err.compareAndSet(null, new AssertionError( + "timed out waiting for test to release the exit cleanup")); + } + } catch (Throwable t) { + err.compareAndSet(null, t); + } + })); + Thread worker = readWorkerThread(manager); + Assert.assertNotNull(worker); + + // Hold the cleanup past the first join (200ms) and release it + // well inside the second join's fixed 5s budget. + releaser = new Thread(() -> { + try { + if (!cleanupEntered.await(10, TimeUnit.SECONDS)) { + err.compareAndSet(null, new AssertionError( + "worker never reached its exit cleanups")); + return; + } + Thread.sleep(400L); + } catch (Throwable t) { + err.compareAndSet(null, t); + } finally { + releaseCleanup.countDown(); + } + }, "second-join-releaser"); + releaser.start(); + + // close(): running=false, worker exits its loop promptly (it + // is idle), sets workerLoopExited, parks in the cleanup. The + // first join (200ms) expires against the parked cleanup; the + // fall-through must take the second join, which reaps once + // the releaser lets the cleanup finish. + manager.setWorkerJoinTimeoutMillis(200L); + manager.close(); + + Assert.assertEquals("worker must have been parked in its exit cleanups", + 0, cleanupEntered.getCount()); + Assert.assertTrue( + "second bounded join must reap a worker that finishes its exit " + + "cleanups inside the fixed budget -- close() returned unreaped", + manager.isWorkerReaped()); + worker.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker must be dead after the second join reaped it", + worker.isAlive()); + Assert.assertEquals( + "close() must free the path scratch itself after the second join " + + "confirmed termination (no handoff on this branch)", + 0L, readPathScratchImpl(manager)); + if (err.get() != null) { + throw new AssertionError("async participant failed", err.get()); + } + } finally { + releaseCleanup.countDown(); + if (releaser != null) { + releaser.join(TimeUnit.SECONDS.toMillis(10)); + } + Thread.interrupted(); + manager.close(); + ring.close(); + } + }); + } + private static long readPathScratchImpl(SegmentManager manager) { return manager.isPathScratchAllocatedForTesting() ? 1L : 0L; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java index 199f3ccf..738bf14d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java @@ -114,6 +114,196 @@ public void testDeadlineAndFailurePropagation() throws Exception { }); } + @Test + public void testRotationGateDefersRotationUntilPredecessorDurable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The PERIODIC-mode rotation gate (requestSyncBeforeRotation): + // rotation must NOT seal a predecessor whose published range is + // not yet durable. This is the exact gate commit 88d6b792 + // accidentally shipped neutralized (`return false; // MUTANT`) + // with the whole suite staying green -- this test is the mutant + // killer. Three independent kill points: + // 1. the rotating append must return BACKPRESSURE_NO_SPARE + // while the predecessor is non-durable (a neutralized gate + // rotates immediately and returns the FSN); + // 2. the gate's sync request must run the barrier on the very + // next service pass BEFORE the interval deadline (a gate + // that fails to set syncRequested leaves the pass idle); + // 3. after the barrier, the retried append must rotate. + final long intervalNanos = 100L; + // Exactly two 16-byte frames fit: header 24 + 2 * (8 + 16) = 72. + final long segmentSize = MmapSegment.HEADER_SIZE + + 2 * (MmapSegment.FRAME_HEADER_SIZE + 16); + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-gate-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 2L, segmentSize)); + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + assertEquals(0L, ring.appendOrFsn(payload, 16)); + manager.serviceRingForTesting(ring); + assertTrue("first pass must leave the active durable", active.isPublishedDurable()); + assertEquals(1, filesFacade.msyncCalls); + assertEquals(1, filesFacade.fsyncCalls); + + // Fill the segment; the second frame is published but NOT + // yet durable. + assertEquals(1L, ring.appendOrFsn(payload, 16)); + assertTrue("segment must be full", active.isFull()); + assertTrue("published range must be ahead of the durable cursor", + !active.isPublishedDurable()); + + // Kill point 1: the gate must refuse to rotate and + // backpressure the producer instead. A spare IS installed, + // so this return can only come from the durability gate. + assertEquals("rotation must be deferred while the predecessor's " + + "published range is not durable", + SegmentRing.BACKPRESSURE_NO_SPARE, ring.appendOrFsn(payload, 16)); + + // Kill point 2: the deadline (tick 100) has NOT been + // reached, so this pass runs the barrier only because the + // gate requested it. + manager.serviceRingForTesting(ring); + assertEquals("gate-requested barrier must run before the deadline", + 2, filesFacade.msyncCalls); + assertEquals("gate-requested barrier must run before the deadline", + 2, filesFacade.fsyncCalls); + assertTrue(active.isPublishedDurable()); + + // Kill point 3: with the predecessor durable the retried + // append rotates into the spare. + assertEquals(2L, ring.appendOrFsn(payload, 16)); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(2L, ring.getActive().baseSeq()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testSyncPassStopsAtFirstFailureThenRetryCoversAllSegments() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // servicePeriodicSync barriers every LIVE segment (sealed first, + // active last) and aborts at the first failure. With >= 2 + // non-durable segments a mid-pass failure must skip the later + // segments, latch the producer, and the healed retry must cover + // EVERY segment before the latch clears. Recovered segments are + // the deterministic way to hold two non-durable live segments: + // recovery constructs them with durableCursor at the header, and + // syncPublished() skips already-durable segments. + final long intervalNanos = 100L; + final long segmentSize = MmapSegment.HEADER_SIZE + + 2 * (MmapSegment.FRAME_HEADER_SIZE + 16); + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-multiseg-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + // Sealed chain member (full: FSNs 0..1) + active (FSN 2). + try (MmapSegment s0 = MmapSegment.create( + filesFacade, dir + "/r0.sfa", 0L, segmentSize)) { + s0.tryAppend(payload, 16); + s0.tryAppend(payload, 16); + s0.msync(); + } + try (MmapSegment s1 = MmapSegment.create( + filesFacade, dir + "/r1.sfa", 2L, segmentSize)) { + s1.tryAppend(payload, 16); + s1.msync(); + } + ring = SegmentRing.openExisting(filesFacade, dir, segmentSize); + assertTrue(ring != null); + assertEquals(1, ring.getSealedSegments().size()); + // Pre-install the hot spare so the service pass does not + // provision one mid-test: its header write would pollute the + // exact barrier-call accounting below. + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 3L, segmentSize)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 8L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // First pass with the disk failing: the sealed member's + // barrier is attempted first (msync succeeds, fsync fails) + // and the pass must STOP there -- the active's barrier must + // not run after a failure. + int msyncBefore = filesFacade.msyncCalls; + int fsyncBefore = filesFacade.fsyncCalls; + filesFacade.isFsyncFailureEnabled = true; + manager.serviceRingForTesting(ring); + assertEquals("only the first (sealed) segment's barrier may be attempted", + msyncBefore + 1, filesFacade.msyncCalls); + assertEquals("the pass must abort at the first fsync failure", + fsyncBefore + 1, filesFacade.fsyncCalls); + try { + ring.appendOrFsn(payload, 16); + fail("mid-pass barrier failure must latch the producer"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + + // Heal. The retry (scheduled at now + min(interval, 1s)) + // must cover BOTH segments -- the one that failed and the + // one the aborted pass never reached -- before unlatching. + filesFacade.isFsyncFailureEnabled = false; + msyncBefore = filesFacade.msyncCalls; + fsyncBefore = filesFacade.fsyncCalls; + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + assertEquals("the healed retry must barrier every live segment", + msyncBefore + 2, filesFacade.msyncCalls); + assertEquals("the healed retry must barrier every live segment", + fsyncBefore + 2, filesFacade.fsyncCalls); + assertEquals("producer must resume once the pass covered all segments", + 3L, ring.appendOrFsn(payload, 16)); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + @Test public void testTransientSyncFailureClearsOnNextSuccess() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java new file mode 100644 index 00000000..426c6d1f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java @@ -0,0 +1,109 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfManifest; + +import io.questdb.client.std.FilesFacade; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Direct pin on {@link SfManifest#update}'s monotonic clamp. Committed + * boundaries only ever move forward (head advances on trim, active on + * rotation); the two writers are serialized on the ring monitor but may each + * compute arguments from a snapshot the other has already moved past. + * Regressing a durable boundary would let a later crash-recovery demand a + * segment file the trim path already unlinked (permanent "missing head + * segment" startup failure) or re-expose stale files below a committed head. + *

      + * Before this test existed, deleting the clamp passed the entire sf suite + * (verified by live mutation run) -- the same blindness class as the + * neutralized rotation gate that commit 88d6b792 accidentally shipped. This + * test lives in the production package deliberately: the manifest API is + * package-private and the clamp deserves a direct unit pin, not an + * integration-distance one. + */ +public class SfManifestClampTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-manifest-clamp-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test + public void testUpdateClampsRegressingBoundariesAndPersistsForward() throws Exception { + TestUtils.assertMemoryLeak(() -> { + FilesFacade ff = FilesFacade.INSTANCE; + SfManifest manifest = SfManifest.create(ff, tmpDir, 10L, 20L); + try { + assertEquals(10L, manifest.headBase()); + assertEquals(20L, manifest.activeBase()); + + // A fully regressing update must be clamped on both fields. + manifest.update(5L, 15L); + assertEquals("regressing headBase must be clamped", 10L, manifest.headBase()); + assertEquals("regressing activeBase must be clamped", 20L, manifest.activeBase()); + + // Fields clamp independently: head may advance while a stale + // active snapshot is clamped in the same call. + manifest.update(12L, 18L); + assertEquals(12L, manifest.headBase()); + assertEquals("stale activeBase snapshot must be clamped independently", + 20L, manifest.activeBase()); + + // Forward motion is untouched. + manifest.update(12L, 25L); + assertEquals(12L, manifest.headBase()); + assertEquals(25L, manifest.activeBase()); + } finally { + manifest.close(); + } + + // The clamp must hold in the durable record, not just in memory: + // reopen and verify the forward-only boundaries survived. + SfManifest reopened = SfManifest.open(ff, tmpDir); + assertNotNull("manifest must reopen", reopened); + try { + assertEquals(12L, reopened.headBase()); + assertEquals(25L, reopened.activeBase()); + } finally { + reopened.close(); + } + }); + } +} From 7a716efe6779d4fef8cced080a60211c6658e94c Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 12:08:04 +0100 Subject: [PATCH 48/64] fix(qwp): attach the throwable to the drainer's retryable-setup warning The catch (Exception) setup path in BackgroundDrainer logged only t.getMessage() and stored it as lastErrorMessage. For a JVM-raised NPE (getMessage() == null on JDK 8, the release target) that produced 'drainer setup temporarily unavailable for slot ...: null' with no exception class and no stack trace -- and since this path deliberately leaves no .failed sentinel, a deterministic setup bug is retried on every orphan scan while emitting that same contentless line forever. The outer catches in the same method already attach the throwable (LOG.error(..., slotPath, msg, t)); only this inner retryable catch dropped it. Log the throwable (SLF4J attaches the stack trace) and carry t.toString() -- class plus message -- into lastErrorMessage so the telemetry surface shows 'java.lang.NullPointerException' instead of 'null'. Diagnostics only; the retry-not-quarantine policy is unchanged. sf package: 391 tests green. --- .../qwp/client/sf/cursor/BackgroundDrainer.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 997eb9b9..6f6c8c42 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -582,9 +582,15 @@ public void run() { // unlock. Leave no .failed sentinel for the next orphan scan. // Error deliberately escapes to the outer Error path: it is // observable after teardown but cannot quarantine intact data. - String msg = t.getMessage(); + // This path retries on every orphan scan, so the log line is + // the ONLY diagnostic a deterministic bug (e.g. an unexpected + // NPE, whose getMessage() is null) ever produces: attach the + // throwable for the stack trace and carry class+message into + // the telemetry surface, mirroring the outer setup-failure + // catch below. + String msg = t.toString(); LOG.warn("drainer setup temporarily unavailable for slot {}: {}", - slotPath, msg); + slotPath, msg, t); lastErrorMessage = msg; outcome = DrainOutcome.FAILED; return; From cce30917c9b47ec9d1fcd04f4632dc9975c1c7b9 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 14:10:35 +0100 Subject: [PATCH 49/64] fix(qwp): make SF barrier retries genuine re-persists (C-1 unsound latch clear) After a failed writeback the kernel marks the dirtied pages clean and reports the error once per open file description (errseq_t; fsyncgate), so re-running msync+fsync over the same range returns a vacuous 0: durableCursor advanced, the PERIODIC rotation gate opened, and the round-3 unlatch (958a36d4) resumed the producer -- all without the data being durable. A clean non-durable page is also evictable, so reclaim could silently replace the only good copy with stale disk content. - MmapSegment.syncPublished now brackets the barrier: best-effort mlock pin over the page-aligned [durableCursor, published) range, then msync+fsync; the failure arm re-dirties the pinned range (one same-value store per page -- kernel dirty tracking is value-blind) before the pin is released. The pin closes the clean-page reclaim race by construction; the re-dirty guarantees the next barrier performs real writeback and reports real errors, keeping the manager's clearDurabilityFailure() honest. - mlock/munlock natives added (POSIX mlock, Windows VirtualLock) and exposed through Files with an UnsatisfiedLinkError guard plus FilesFacade defaults. Refusal (RLIMIT_MEMLOCK, missing capability) is a soft downgrade: one deduped WARN, barrier outcome and latch semantics unaffected. - 4 regression tests: bracket ordering on success, re-dirty-before-unpin on failure, an fsyncgate facade model (first fsync fails, later msync/fsync return vacuous 0 without delegating) proving the latch only clears over re-dirtied pages, and mlock-refusal degrade. The drop-the-redirty mutant was applied and killed (2 failures) before restoring. Native libraries are built from source, so the new symbols ship with every build; the UnsatisfiedLinkError guard remains as defense-in-depth for environments running a stale prebuilt library. --- core/src/main/c/share/files.c | 12 + core/src/main/c/windows/files.c | 13 + .../qwp/client/sf/cursor/MmapSegment.java | 80 ++++++- .../qwp/client/sf/cursor/SegmentManager.java | 9 +- .../java/io/questdb/client/std/Files.java | 47 ++++ .../io/questdb/client/std/FilesFacade.java | 18 ++ .../SegmentManagerPeriodicSyncTest.java | 225 ++++++++++++++++++ 7 files changed, 393 insertions(+), 11 deletions(-) diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 01e18a6b..b469e17b 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -388,3 +388,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_msync (JNIEnv *e, jclass cl, jlong addr, jlong len, jboolean async) { return msync((void *) (uintptr_t) addr, (size_t) len, async ? MS_ASYNC : MS_SYNC); } + +/* Best-effort page pin. Callers treat a refusal (RLIMIT_MEMLOCK, missing + * privilege) as a soft downgrade, so no errno capture is required here. */ +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return mlock((void *) (uintptr_t) addr, (size_t) len); +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_munlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return munlock((void *) (uintptr_t) addr, (size_t) len); +} diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index d3fa2fbb..e9770e9d 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -689,3 +689,16 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_msync (void) async; return 0; } + +/* Best-effort page pin. VirtualLock is quota-bound to the process working-set + * minimum; callers treat any refusal as a soft downgrade, so no + * SaveLastError() -- the refusal is not surfaced as an error. */ +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return VirtualLock((LPVOID) (uintptr_t) addr, (SIZE_T) len) ? 0 : -1; +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_munlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return VirtualUnlock((LPVOID) (uintptr_t) addr, (SIZE_T) len) ? 0 : -1; +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 35939395..b8002908 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -31,9 +31,12 @@ import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicBoolean; + /** * One mmap-backed SF segment file. The user thread (the single producer) * appends frames into the mapping; the I/O thread (the single consumer) reads @@ -68,6 +71,10 @@ public final class MmapSegment implements QuietCloseable { public static final byte MANIFEST_REQUIRED_FLAG = 1; public static final byte VERSION = 1; private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); + // Deduplicates the process-wide "mlock refused" warning: the refusal is a + // soft downgrade (see syncPublished) and must not spam the log once per + // barrier when RLIMIT_MEMLOCK or the platform says no. + private static final AtomicBoolean MLOCK_REFUSAL_WARNED = new AtomicBoolean(); private static final int RECOVERY_BUFFER_SIZE = 64 * 1024; private final FilesFacade filesFacade; @@ -103,6 +110,9 @@ public final class MmapSegment implements QuietCloseable { // because the consumer must see writes in publication order — once the // producer bumps publishedCursor, every byte before it is fully written. private volatile long publishedCursor; + // Number of failure-arm re-dirty passes performed by syncPublished. + // Cold-path only; volatile so tests observing from another thread see it. + private volatile long redirtyPasses; // Monotonic in-memory link to the segment that immediately follows this // one. SegmentRing publishes it before promoting the successor to active; // close deliberately retains it so a cursor can advance after head trim. @@ -486,6 +496,21 @@ public void msync() { * captures {@link #publishedCursor}. A concurrent producer may publish * more bytes while the barrier runs; those bytes remain outside the * returned durable boundary until a later call. + *

      + * The not-yet-durable range is pinned with a best-effort {@code mlock} + * for the duration of the barrier and, on failure, re-dirtied before the + * pin is released. Rationale (fsyncgate): after a failed writeback the + * kernel marks the affected pages clean and reports the error once per + * open file description, so a naive retry would msync+fsync clean pages + * and return a vacuous 0 without persisting anything -- and an unpinned + * clean page can be reclaimed and later re-faulted from stale disk + * content, silently replacing the only good copy of the frames. The pin + * guarantees the failure arm re-dirties the genuine in-memory bytes; the + * re-dirty guarantees the next barrier performs real writeback and + * reports real errors, so the manager's unlatch-on-success stays honest. + * A refused mlock (RLIMIT_MEMLOCK, missing capability, stale native + * library) never affects the barrier outcome; it only widens the + * microseconds-scale window between the failed syscall and the re-dirty. * * @return the captured byte offset covered by the successful barrier */ @@ -498,16 +523,55 @@ public long syncPublished() { if (published <= durableCursor) { return durableCursor; } - if (filesFacade.msync(mmapAddress, published, false) != 0) { - throw new MmapSegmentException("could not sync segment data " + path); + long lockOffset = durableCursor & -Files.PAGE_SIZE; + long lockAddr = mmapAddress + lockOffset; + long lockLen = published - lockOffset; + boolean locked = filesFacade.mlock(lockAddr, lockLen) == 0; + if (!locked && MLOCK_REFUSAL_WARNED.compareAndSet(false, true)) { + LOG.warn("mlock refused for SF barrier range in {} (degrading to re-dirty-only retry protection); " + + "raise RLIMIT_MEMLOCK or grant CAP_IPC_LOCK to close the post-failure reclaim window", path); } - // FlushViewOfFile alone is not power-loss durable on Windows. Keep the - // fd barrier on every platform so this method has one portable contract. - if (filesFacade.fsync(fd) != 0) { - throw new MmapSegmentException("could not sync segment file " + path); + boolean durable = false; + try { + if (filesFacade.msync(mmapAddress, published, false) != 0) { + throw new MmapSegmentException("could not sync segment data " + path); + } + // FlushViewOfFile alone is not power-loss durable on Windows. Keep the + // fd barrier on every platform so this method has one portable contract. + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment file " + path); + } + durable = true; + durableCursor = published; + return published; + } finally { + if (!durable) { + redirtyRange(lockAddr, mmapAddress + published); + } + if (locked) { + filesFacade.munlock(lockAddr, lockLen); + } + } + } + + /** + * Marks every page overlapping {@code [fromAddr, endAddr)} dirty again by + * re-storing one byte per page. Every touched offset lies inside the + * published prefix (or the immutable header magic after aligning down), + * so the same-value store races with nothing; kernel dirty tracking is + * value-blind, so the store is a real dirtying event that forces the next + * writeback to re-submit the whole page. + */ + private void redirtyRange(long fromAddr, long endAddr) { + for (long addr = fromAddr; addr < endAddr; addr += Files.PAGE_SIZE) { + Unsafe.getUnsafe().putByte(addr, Unsafe.getUnsafe().getByte(addr)); } - durableCursor = published; - return published; + redirtyPasses++; + } + + @TestOnly + public long redirtyPassesForTest() { + return redirtyPasses; } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 5745b15c..e4a23090 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -1224,9 +1224,12 @@ private void servicePeriodicSync(RingEntry e, long now) { syncScratch.getQuick(i).syncPublished(); } e.ring.clearSyncRequestIfActiveDurable(); - // The pass above covered every live segment's published range, so - // any earlier barrier failure has been remedied — unlatch it so a - // transient disk fault doesn't permanently brick the producer. + // The pass above covered every live segment's published range, and + // a failed barrier re-dirties its range under an mlock pin (see + // MmapSegment.syncPublished), so a success here is a genuine + // re-persist -- not a vacuous retry over pages a failed writeback + // marked clean (fsyncgate). Unlatch so a transient disk fault + // doesn't permanently brick the producer. e.ring.clearDurabilityFailure(); e.nextDataSyncNanos = now + e.syncIntervalNanos; if (e.syncFailureLogged) { diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 17c74ca4..522dddc3 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -573,6 +573,53 @@ public static void munmap(long address, long len, int memoryTag) { */ public static native int msync(long addr, long len, boolean async); + // Guards the optional mlock/munlock natives: a packaged native library + // built before these symbols existed throws UnsatisfiedLinkError on first + // use. Flip to unavailable and report refusal (-1) from then on, so + // callers degrade to their unpinned tier instead of crashing. + private static volatile boolean mlockLinked = true; + + /** + * Best-effort page pin over {@code [addr, addr+len)} of an mmap'd region. + * Whole pages containing any part of the range are locked; {@code addr} + * should be page-aligned for portability. Returns 0 on success, -1 when + * the platform refuses (RLIMIT_MEMLOCK, missing privilege) or the loaded + * native library predates the symbol. Callers must treat refusal as a + * soft downgrade, never an error. + */ + public static int mlock(long addr, long len) { + if (!mlockLinked) { + return -1; + } + try { + return mlock0(addr, len); + } catch (UnsatisfiedLinkError err) { + mlockLinked = false; + return -1; + } + } + + /** + * Releases a pin taken by {@link #mlock(long, long)}. Best-effort: a + * refusal is ignorable ({@code munmap} implicitly drops any remaining + * locks on the range). + */ + public static int munlock(long addr, long len) { + if (!mlockLinked) { + return -1; + } + try { + return munlock0(addr, len); + } catch (UnsatisfiedLinkError err) { + mlockLinked = false; + return -1; + } + } + + private static native int mlock0(long addr, long len); + + private static native int munlock0(long addr, long len); + /** * Returns a native pointer to the current entry's null-terminated name * (UTF-8). Pointer is valid only until the next {@link #findNext(long)} diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 7f436697..921684a0 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -108,6 +108,16 @@ default int fsyncDir(String dir) { int mkdir(String path, int mode); + /** + * Best-effort page pin over {@code [addr, addr+len)} of an mmap'd region. + * Returns 0 when the range is locked, non-zero when the platform refuses + * (RLIMIT_MEMLOCK, missing privilege, or a native library without the + * symbol). Callers must treat refusal as a soft downgrade, never an error. + */ + default int mlock(long addr, long len) { + return Files.mlock(addr, len); + } + default long mmap(int fd, long len, long offset, int flags, int memoryTag) { return Files.mmap(fd, len, offset, flags, memoryTag); } @@ -116,6 +126,14 @@ default int msync(long addr, long len, boolean async) { return Files.msync(addr, len, async); } + /** + * Releases a pin taken by {@link #mlock(long, long)}. Best-effort; + * refusals are ignorable ({@code munmap} implicitly unlocks). + */ + default int munlock(long addr, long len) { + return Files.munlock(addr, len); + } + default void munmap(long address, long len, int memoryTag) { Files.munmap(address, len, memoryTag); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java index 738bf14d..83604657 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -378,10 +379,206 @@ public void testTransientSyncFailureClearsOnNextSuccess() throws Exception { }); } + @Test + public void testBarrierPinsPublishedRangeAndUnpinsOnSuccess() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long segmentSize = 4096L; + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-pin-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + active.syncPublished(); + + assertEquals("barrier must pin the not-yet-durable range", 1, filesFacade.mlockCalls); + assertEquals("successful barrier must release the pin", 1, filesFacade.munlockCalls); + // durableCursor starts at HEADER_SIZE, which aligns down to + // page 0, so the pin covers [0, published). + assertEquals("pin must cover the whole not-yet-durable range", + active.publishedOffset(), filesFacade.lastMlockLen); + assertEquals("success path must not re-dirty", 0L, active.redirtyPassesForTest()); + assertTrue(active.isPublishedDurable()); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testFailedBarrierRedirtiesUnderPinBeforeUnlock() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long segmentSize = 4096L; + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-redirty-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + long[] redirtyAtUnlock = new long[1]; + filesFacade.onMunlock = () -> redirtyAtUnlock[0] = active.redirtyPassesForTest(); + filesFacade.isFsyncFailureEnabled = true; + try { + active.syncPublished(); + fail("expected the fsync failure to surface"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + + assertEquals("failed barrier must re-dirty the covered range", + 1L, active.redirtyPassesForTest()); + assertEquals("failed barrier must still release the pin", 1, filesFacade.munlockCalls); + assertEquals("re-dirty must happen BEFORE the pin is released", + 1L, redirtyAtUnlock[0]); + assertFalse("failed barrier must not advance durability", active.isPublishedDurable()); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testConsumedErrorRetryClearsLatchOnlyOverRedirtiedPages() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long intervalNanos = 100L; + final long segmentSize = 4096L; + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-fsyncgate-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 1L, segmentSize)); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // First tick: initial sync genuinely succeeds. + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + + // fsyncgate model: the next barrier's fsync fails once; from + // then on the facade behaves like the real kernel after EIO -- + // pages clean, error consumed -- returning 0 from msync/fsync + // WITHOUT persisting anything. + assertEquals(1L, ring.appendOrFsn(payload, 16)); + filesFacade.isFsyncGateModeEnabled = true; + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + try { + ring.appendOrFsn(payload, 16); + fail("expected the failed data sync to latch the producer"); + } catch (MmapSegmentException expected) { + } + assertEquals("the failed barrier must have re-dirtied its range before any vacuous retry", + 1L, active.redirtyPassesForTest()); + + // Retry pass: the facade's vacuous 0 is backed by genuinely + // re-dirtied pages, so unlatching is honest. Without the + // re-dirty (the C-1 mutant) this scenario is exactly the + // unsound clear: latch gone, durableCursor advanced, nothing + // persisted and no dirty page left for any future barrier. + ticks.set(intervalNanos * 2L); + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + assertEquals("producer must resume after the covered retry", + 2L, ring.appendOrFsn(payload, 16)); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testMlockRefusalDegradesWithoutAffectingBarrier() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long segmentSize = 4096L; + CountingFilesFacade filesFacade = new CountingFilesFacade(); + filesFacade.isMlockRefusalEnabled = true; + String dir = TestUtils.createTmpDir("qdb-periodic-mlock-refusal-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + int msyncBefore = filesFacade.msyncCalls; + int fsyncBefore = filesFacade.fsyncCalls; + active.syncPublished(); + + assertEquals("refused pin must not skip the mapping barrier", + msyncBefore + 1, filesFacade.msyncCalls); + assertEquals("refused pin must not skip the fd barrier", + fsyncBefore + 1, filesFacade.fsyncCalls); + assertEquals(1, filesFacade.mlockCalls); + assertEquals("a refused pin must not be unlocked", 0, filesFacade.munlockCalls); + assertTrue("refusal must not affect the barrier outcome", active.isPublishedDurable()); + assertEquals(0L, active.redirtyPassesForTest()); + assertEquals("producer must remain unaffected", 1L, ring.appendOrFsn(payload, 16)); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + private static final class CountingFilesFacade implements FilesFacade { private boolean isFsyncFailureEnabled; + // fsyncgate model: the first fsync fails; afterwards every msync and + // fsync returns 0 WITHOUT delegating to the real syscall -- the + // kernel-accurate shape of a vacuous retry after a consumed EIO + // (clean pages, seen errseq cursor). + private boolean isFsyncGateModeEnabled; + private boolean fsyncGateErrorConsumed; + private boolean isMlockRefusalEnabled; private int fsyncCalls; private int msyncCalls; + private int mlockCalls; + private int munlockCalls; + private long lastMlockLen; + private Runnable onMunlock; @Override public boolean allocate(int fd, long size) { @@ -436,6 +633,13 @@ public void freeNativePath(long pathPtr) { @Override public int fsync(int fd) { fsyncCalls++; + if (isFsyncGateModeEnabled) { + if (!fsyncGateErrorConsumed) { + fsyncGateErrorConsumed = true; + return -1; + } + return 0; + } return isFsyncFailureEnabled ? -1 : INSTANCE.fsync(fd); } @@ -464,12 +668,33 @@ public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int mlock(long addr, long len) { + mlockCalls++; + lastMlockLen = len; + return isMlockRefusalEnabled ? -1 : 0; + } + @Override public int msync(long addr, long len, boolean async) { msyncCalls++; + if (isFsyncGateModeEnabled && fsyncGateErrorConsumed) { + // consumed-error semantics: no dirty pages, seen errseq -> 0 + return 0; + } return INSTANCE.msync(addr, len, async); } + @Override + public int munlock(long addr, long len) { + munlockCalls++; + Runnable hook = onMunlock; + if (hook != null) { + hook.run(); + } + return 0; + } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); From 38cdf3e33005808b00e9688df8e8beeb027f4c7b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 14:36:37 +0100 Subject: [PATCH 50/64] test(qwp): close the three remaining C-2 test-gate rows with verified mutant-killers Round-3 review re-audit of e0ebdf00 found three rows not genuinely closed; each is now pinned by a test whose mutant was applied and verified to die before restoring: - C-2a: close-path unlink stop-on-first-failure was empirically unprotected -- a continue-past-first-failed-remove mutant in CursorSendEngine.unlinkAllSegmentFiles survived the whole suite (2,722 green). e0ebdf00 mapped the row to the periodic sync pass (different subsystem); the enumeration and all-unlinks-fail siblings cannot discriminate stop-vs-continue. New CursorSendEngineCloseUnlinkStopOnFirstFailureTest: legacy (manifest-less) slot, two FSN-contiguous sealed segments recovered from disk, FilesFacade refusing only the lowest .sfa removal once; asserts the higher generation is never attempted and survives, and a successor recovers the contiguous residue and completes the cleanup. Mutant now fails 'expected:<0> but was:<1>'. - C-2b: the sender-level checkDurability call sites (QwpWebSocketSender awaitAckedFsn pre-check, wait-loop, and flushAndGetSequence) had no user-visible-layer coverage -- deleting all three survived the whole suite; only the engine seam was pinned. New QwpWebSocketSenderCursorEngineAttachmentTest. testLatchedDurabilityFailureSurfacesThroughSenderFlushAndAwait: unconnected createForTesting sender + attached engine + latched failure; empty flush()/flushAndGetSequence() and awaitAckedFsn(fsn,0) must each throw the latched instance repeatedly (empty flush and <=0-timeout polls publish nothing, so the ring gate never runs -- these call sites are their only durability act, and they must fire before connection setup). Mutant now dies on the first flush(). - C-2c: testSecondBoundedJoinReapsWorkerFinishingExitCleanups passed under the exact pre-fix revert of 8d962e05: both variants null workerThread (isWorkerReaped() cannot discriminate) and the worker.join(5s) preceding the isAlive assert destroyed the liveness observable. Added the discriminating assertion at the moment close() returns: the second bounded join must hold close() until the parked cleanup finishes (+400ms), while the revert returns at ~200ms with the worker alive. Verified deterministic both ways: fails under the revert, passes with the fix. Full core suite: 2,724 green (2,722 + 2 new). --- ...ocketSenderCursorEngineAttachmentTest.java | 50 +++++ ...gineCloseUnlinkStopOnFirstFailureTest.java | 193 ++++++++++++++++++ .../cursor/SegmentManagerCloseRaceTest.java | 14 ++ 3 files changed, 257 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java index 5bfb0863..9184fd3c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -126,6 +127,55 @@ public void testSameSharedEngineCannotTransferOwnership() throws Exception { }); } + @Test + public void testLatchedDurabilityFailureSurfacesThroughSenderFlushAndAwait() throws Exception { + assertMemoryLeak(() -> { + // The ring gate (SegmentRing.appendOrFsn) guards every publish + // path, but an EMPTY flush()/flushAndGetSequence() and a + // timeoutMillis <= 0 awaitAckedFsn() poll publish nothing: the + // sender-level cursorEngine.checkDurability() call sites are + // their ONLY durability act. Without them an empty flush + // silently succeeds against an unsyncable slot and a poll loop + // spins on "not yet" forever instead of surfacing the latched + // barrier failure. Deleting any of the three sender call sites + // previously survived the whole suite -- this pins the + // user-visible layer (the engine seam has its own pin in + // CursorSendEngineTest). + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, false); + MmapSegmentException failure = new MmapSegmentException("injected data-sync failure"); + engine.getRingForTesting().recordDurabilityFailureForTesting(failure); + for (int i = 0; i < 2; i++) { // repeatable until cleared, not one-shot + try { + sender.flush(); + Assert.fail("empty flush() must surface the latched durability failure, call #" + i); + } catch (MmapSegmentException expected) { + Assert.assertSame("the latched instance itself must surface", failure, expected); + } + try { + sender.flushAndGetSequence(); + Assert.fail("empty flushAndGetSequence() must surface the latched durability " + + "failure, call #" + i); + } catch (MmapSegmentException expected) { + Assert.assertSame(failure, expected); + } + try { + sender.awaitAckedFsn(0L, 0L); + Assert.fail("awaitAckedFsn(fsn, 0) must throw instead of polling 'not yet', " + + "call #" + i); + } catch (MmapSegmentException expected) { + Assert.assertSame(failure, expected); + } + } + } finally { + sender.close(); + engine.close(); + } + }); + } + private static void assertSecondAttachmentRejected( QwpWebSocketSender sender, CursorSendEngine engine, diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java new file mode 100644 index 00000000..8fe040d7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java @@ -0,0 +1,193 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Close-time unlink STOP-ON-FIRST-FAILURE on a legacy (manifest-less) slot. + * Removal runs in ascending generation order and must stop at the first + * failed unlink so the residue is always a contiguous top slice that passes + * FSN-contiguity at the next recovery. Continuing past a failed + * low-generation remove deletes higher generations and can leave + * non-contiguous residue -- a startup brick on a slot that lost nothing. + *

      + * The siblings cover the OTHER close-cleanup contracts: + * {@link CursorSendEngineClosePartialEnumerationTest} proves a torn + * enumeration drives zero unlinks, and the unlink-failure sibling fails ALL + * removals (permission trick), which cannot discriminate stop-vs-continue + * (every file survives either way). A continue-past-first-failure mutant in + * {@code unlinkAllSegmentFiles} previously survived the whole suite; this + * test kills it via the higher-generation-survival assertions. Same + * determinism trick as the siblings: the shared manager is never started, so + * no worker touches the slot and no concurrent cleanup can trip the armed + * fault. + */ +public class CursorSendEngineCloseUnlinkStopOnFirstFailureTest { + + private static final long SEGMENT_SIZE = 4096L; + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-engine-close-unlink-stop-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test(timeout = 20_000L) + public void testCloseUnlinkStopsAtFirstFailedRemoveOnLegacySlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String slot = tmpDir + "/legacy-slot"; + final String lowPath = slot + "/sf-initial.sfa"; + final String highPath = slot + "/sf-0000000000000002.sfa"; + final AtomicBoolean armLowestRemoveFailure = new AtomicBoolean(); + final AtomicInteger lowRemoveAttempts = new AtomicInteger(); + final AtomicInteger highRemoveAttempts = new AtomicInteger(); + FilesFacade faultFacade = (FilesFacade) Proxy.newProxyInstance( + FilesFacade.class.getClassLoader(), + new Class[]{FilesFacade.class}, + (proxy, method, args) -> { + if ("remove".equals(method.getName()) && args[0] instanceof String) { + String path = (String) args[0]; + if (path.equals(lowPath)) { + lowRemoveAttempts.incrementAndGet(); + if (armLowestRemoveFailure.get()) { + // EBUSY-style transient refusal: no unlink happens. + return false; + } + } else if (path.equals(highPath)) { + highRemoveAttempts.incrementAndGet(); + } + } + try { + return method.invoke(FilesFacade.INSTANCE, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + // Legacy slot: two FSN-contiguous segments on disk, NO manifest. + long buf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + try { + MmapSegment low = MmapSegment.create(lowPath, 0L, SEGMENT_SIZE); + Assert.assertTrue("setup: append must land", low.tryAppend(buf, 32) >= 0); + Assert.assertTrue("setup: append must land", low.tryAppend(buf, 32) >= 0); + low.close(); + MmapSegment high = MmapSegment.create(highPath, 2L, SEGMENT_SIZE); + Assert.assertTrue("setup: append must land", high.tryAppend(buf, 32) >= 0); + high.close(); + + manager = new SegmentManager( + SEGMENT_SIZE, + SegmentManager.DEFAULT_POLL_NANOS, + SEGMENT_SIZE * 4L, + faultFacade, + System::nanoTime); + + CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE, manager); + boolean engineClosed = false; + try { + Assert.assertTrue("engine must recover the legacy two-segment chain", + engine.wasRecoveredFromDisk()); + Assert.assertEquals("recovered chain must publish FSNs 0..2", + 2L, engine.publishedFsn()); + Assert.assertTrue(engine.acknowledge(2L)); + + // Fully drained close with the LOWEST-generation removal + // refused once: ascending-order removal must STOP there. + armLowestRemoveFailure.set(true); + engine.close(); + engineClosed = true; + Assert.assertTrue("close must complete despite the aborted cleanup", + engine.isCloseCompleted()); + Assert.assertEquals("cleanup must attempt the lowest generation first", + 1, lowRemoveAttempts.get()); + Assert.assertEquals( + "removal must STOP at the first failed unlink -- continuing would " + + "delete higher generations and could leave non-contiguous " + + "residue that fails FSN-contiguity at the next recovery", + 0, highRemoveAttempts.get()); + Assert.assertTrue("the refused lowest segment must survive", + Files.exists(lowPath)); + Assert.assertTrue("higher-generation segment must survive the stopped cleanup", + Files.exists(highPath)); + } finally { + if (!engineClosed) { + engine.close(); + } + } + + // Heal the fault; a successor adopts the contiguous residue + // and its fully-drained close completes the cleanup. + armLowestRemoveFailure.set(false); + CursorSendEngine successor = new CursorSendEngine(slot, SEGMENT_SIZE, manager); + boolean successorClosed = false; + try { + Assert.assertTrue("successor must recover the contiguous residue", + successor.wasRecoveredFromDisk()); + successor.close(); + successorClosed = true; + Assert.assertTrue(successor.isCloseCompleted()); + Assert.assertFalse("successor's fully-drained close must complete the unlink", + Files.exists(lowPath)); + Assert.assertFalse("successor's fully-drained close must complete the unlink", + Files.exists(highPath)); + } finally { + if (!successorClosed) { + successor.close(); + } + } + } finally { + Unsafe.free(buf, 32, MemoryTag.NATIVE_DEFAULT); + if (manager != null) { + manager.close(); + } + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index b8c0a0c1..46ea1b0d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -766,6 +766,20 @@ public void testSecondBoundedJoinReapsWorkerFinishingExitCleanups() throws Excep manager.setWorkerJoinTimeoutMillis(200L); manager.close(); + // The discriminating observable: with the second bounded join, + // close() blocks until the parked cleanup finishes (released at + // +400ms) and reaps a DEAD worker; the pre-fix code reaped on + // observing workerLoopExited and returned at ~200ms with the + // worker still alive in its cleanup. Assert liveness at the + // moment close() returns -- before any join in this test can + // mask it. (isWorkerReaped() alone cannot discriminate: both + // variants null workerThread.) + Assert.assertFalse( + "close() must not return while the worker is still alive in its exit " + + "cleanups -- the second bounded join has to hold close() until " + + "they finish", + worker.isAlive()); + Assert.assertEquals("worker must have been parked in its exit cleanups", 0, cleanupEntered.getCount()); Assert.assertTrue( From fd424e3e213931af5e62d4e651242407283bec04 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 15:15:42 +0100 Subject: [PATCH 51/64] fix(qwp): drainer retries once over a chain the sealed-residue sanitize already healed The fail-closed first-sight throw in sanitizeSealedResidue fires AFTER the proven-dead sealed residue has been durably zeroed (msync+fsync), so the chain on disk is already healed when it propagates. The orphan drainer classified that throw as terminal SfRecoveryException and dropped a .failed sentinel -- stranding a fully replayable backlog behind a quarantine no scan revisits, for an incident recovery had just repaired. Introduce SfSanitizedResidueException (a refinement of SfRecoveryException, so producer startup keeps its fail-closed restart-proves-clean semantics) and have the drainer intercept it for a single in-place construction retry. Any second failure, including a repeat of the refinement from a non-sticking heal, falls through to the existing terminal classification. New drainer test proves the full arc: poisoned sealed suffix, heal durably on disk before the throw, one retry, connect reached, no sentinel, backlog still scanner-eligible. SegmentRingTest now pins the thrown type at the sanitize site. --- .../client/sf/cursor/BackgroundDrainer.java | 26 +++- .../sf/cursor/MmapSegmentException.java | 4 +- .../qwp/client/sf/cursor/SegmentRing.java | 13 +- .../client/sf/cursor/SfRecoveryException.java | 8 +- .../cursor/SfSanitizedResidueException.java | 50 ++++++++ .../BackgroundDrainerSetupFailureTest.java | 111 ++++++++++++++++++ .../qwp/client/sf/cursor/SegmentRingTest.java | 4 + 7 files changed, 206 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 6f6c8c42..c83f9f93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -560,9 +560,29 @@ public void run() { // holds it, the engine constructor throws and we exit silently // (no .failed sentinel — contention is expected, not an error). try { - engine = new CursorSendEngine(slotPath, segmentSizeBytes, - sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, - syncIntervalNanos); + try { + engine = new CursorSendEngine(slotPath, segmentSizeBytes, + sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + syncIntervalNanos); + } catch (SfSanitizedResidueException first) { + // First sight of proven-dead sealed residue: recovery + // durably zeroed it BEFORE failing closed, so the chain + // on disk is already healed and a .failed sentinel here + // would strand a replayable backlog no scan revisits + // (the sentinel gates isCandidateOrphan and nothing in + // production clears it). Retry once over the healed + // chain; the WARN keeps the incident surfaced. Any + // failure of the retry is genuine and takes the normal + // classification below -- including a repeat of this + // type, which the SfRecoveryException arm then treats + // as the terminal quarantine a non-sticking heal is. + LOG.warn("drainer slot {}: sealed SF residue sanitized during recovery ({}); " + + "retrying engine construction over the healed chain", + slotPath, first.getMessage()); + engine = new CursorSendEngine(slotPath, segmentSizeBytes, + sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + syncIntervalNanos); + } } catch (SlotLockContentionException t) { LOG.info("orphan slot already locked, skipping: {} ({})", slotPath, t.getMessage()); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index 03751c80..371d980c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -36,7 +36,9 @@ * positively-identified corruption in one file's own bytes, which * recovery may quarantine when the surviving chain proves safe; and * {@link SfRecoveryException} marks a terminal chain failure that needs - * operator intervention. + * operator intervention (with one self-healed refinement, + * {@link SfSanitizedResidueException}, that unattended callers retry once + * instead of quarantining). */ public class MmapSegmentException extends RuntimeException { public MmapSegmentException(String message) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 1aee2ab3..ef5855c5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -784,10 +784,13 @@ private static void validateContiguous(ObjList segments) { * fails recovery before any mutation, preserving the bytes (potentially * the only copy of unreachable valid-CRC frames) for operator * extraction. With {@code failClosedOnSight} the incident is still - * surfaced as a first-sight startup failure after sanitizing (the - * restart then proves the chain clean); without it the chain proceeds - * immediately (legacy migration, which predates the sealed-suffix - * contract). + * surfaced as a first-sight {@link SfSanitizedResidueException} after + * sanitizing: the residue is already durably zeroed when it propagates, + * so a retry proves the chain clean (attended callers get that via + * restart; unattended callers key off the distinct type to retry + * instead of quarantining a just-healed slot). Without the flag the + * chain proceeds immediately (legacy migration, which predates the + * sealed-suffix contract). */ private static void sanitizeSealedResidue(ObjList chain, boolean failClosedOnSight) { String firstTornPath = null; @@ -801,7 +804,7 @@ private static void sanitizeSealedResidue(ObjList chain, boolean fa } } if (failClosedOnSight && firstTornPath != null) { - throw new SfRecoveryException("corrupt torn tail in sealed SF segment " + firstTornPath); + throw new SfSanitizedResidueException("corrupt torn tail in sealed SF segment " + firstTornPath); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java index 21ce1cf4..d3f266c3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java @@ -29,8 +29,14 @@ * segment chain is corrupt or incomplete and requires operator intervention. * Operational filesystem failures continue to use {@link MmapSegmentException} * so callers can retry them without quarantining otherwise recoverable data. + *

      + * One refinement is deliberately non-terminal for unattended callers: + * {@link SfSanitizedResidueException} marks a first-sight failure thrown + * AFTER recovery durably healed the chain, so a single retry validates + * clean. Catch sites that quarantine on this type must intercept the + * refinement first. */ -public final class SfRecoveryException extends MmapSegmentException { +public class SfRecoveryException extends MmapSegmentException { public SfRecoveryException(String message) { super(message); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java new file mode 100644 index 00000000..8a58c5b2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java @@ -0,0 +1,50 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * First-sight failure over a chain that recovery has already healed. + * Thrown from the fail-closed branch of the sealed-residue sanitize: frame + * accounting was proven complete (contiguity plus boundary checks), the + * proven-dead suffix residue was durably zeroed ({@code msync} + + * {@code fsync} completed — a sync failure throws + * {@link MmapSegmentException} instead and never reaches this type), and + * the throw exists solely to surface the incident on the startup that + * observed it. An immediate re-open re-runs the same proofs over the + * zeroed suffix and succeeds. + *

      + * Attended callers (producer startup, where an operator or supervisor + * restarts the process) should keep the parent's fail-closed semantics: + * the restart proves the chain clean. Unattended callers (the orphan + * drainer) may retry construction once instead of quarantining — dropping + * a {@code .failed} sentinel over a just-healed slot would strand its + * replayable backlog until an operator clears the sentinel by hand. + */ +public final class SfSanitizedResidueException extends SfRecoveryException { + + public SfSanitizedResidueException(String message) { + super(message); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java index 0e9d20e1..a8bd96c9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; @@ -196,6 +197,116 @@ public void testCorruptRecoveredChainIsQuarantined() throws Exception { }); } + @Test + public void testSealedResidueFirstSightHealsAndDoesNotQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // A sealed member whose frame accounting is complete but whose + // suffix gap carries legacy pre-sanitization poison. Recovery + // durably zeroes the residue BEFORE failing closed on first + // sight (SegmentRing's sanitize-then-throw), so at the instant + // the drainer sees the throw the chain on disk is already + // healed. The drainer must retry construction over the healed + // chain and reach connect -- not strand the replayable backlog + // behind a .failed sentinel that no orphan scan revisits. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // sealed suffix gap that can never fit a frame + String p0Path = slotPath + "/p0.sfa"; + String p1Path = slotPath + "/p1.sfa"; + long gapStart; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 16, (byte) 5); + MmapSegment s0 = MmapSegment.create(p0Path, 0, segSize); + for (int i = 0; i < 4; i++) { + s0.tryAppend(buf, 16); + } + gapStart = s0.publishedOffset(); + s0.close(); + MmapSegment s1 = MmapSegment.create(p1Path, 4, segSize); + s1.tryAppend(buf, 16); + s1.close(); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + + // Adopt the chain once so the slot carries a manifest, lock and + // watermark like any real producer slot (without a manifest the + // legacy-migration path sanitizes silently and never fails + // closed); close releases the lock. + try (CursorSendEngine seed = new CursorSendEngine(slotPath, segSize)) { + Assert.assertEquals("highest published FSN must cover frames 0..4", + 4L, seed.publishedFsn()); + } + + // Poison the sealed suffix gap the way a pre-sanitization + // client's reseal-after-recovery did. + int fd = Files.openRW(p0Path); + Assert.assertTrue("openRW must succeed", fd >= 0); + long junk = Unsafe.malloc(12, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 3; i++) { + Unsafe.getUnsafe().putInt(junk + i * 4L, 0xCAFEBABE); + } + Assert.assertEquals(12L, Files.write(fd, junk, 12, gapStart)); + Files.fsync(fd); + } finally { + Unsafe.free(junk, 12, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + + LinkageError injected = new LinkageError("injected drainer connect error"); + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + segSize, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw injected; + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + LinkageError thrown = null; + try { + drainer.run(); + } catch (LinkageError e) { + thrown = e; + } + + // The heal precedes the first-sight throw: frames intact, + // residue durably zeroed. This holds with or without the + // drainer fix -- it is exactly what makes quarantine wrong. + try (MmapSegment sealed = MmapSegment.openExisting(p0Path)) { + Assert.assertEquals("frames must be untouched", 4L, sealed.frameCount()); + Assert.assertEquals("proven-dead residue must be zeroed on disk", + 0L, sealed.tornTailBytes()); + } + + Assert.assertSame("drainer must reach connect over the healed chain", + injected, thrown); + Assert.assertEquals("exactly one connect attempt after the healed retry", + 1, connectAttempts.get()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "injected drainer connect error"); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + Assert.assertFalse("a just-healed slot must not be quarantined", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("replayable backlog must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + try (CursorSendEngine engine = new CursorSendEngine(slotPath, segSize)) { + Assert.assertEquals("backlog must remain fully replayable", + 4L, engine.publishedFsn()); + Assert.assertTrue("drainer teardown must release the slot lock", + OrphanScanner.isCandidateOrphan(slotPath)); + } + }); + } + @Test public void testLockOpenFailureDoesNotQuarantineRecoverableData() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index aeb77b00..dffc1b7d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfSanitizedResidueException; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Misc; @@ -515,6 +516,9 @@ public void testProvenDeadSealedResidueSanitizedThenHealsAfterOneRestart() throw Misc.free(SegmentRing.openExisting(tmpDir, segSize)); throw new AssertionError("poisoned sealed suffix must fail closed on first sight"); } catch (MmapSegmentException expected) { + assertTrue("first-sight throw must be the healed-residue refinement so " + + "unattended callers can retry: " + expected.getClass().getName(), + expected instanceof SfSanitizedResidueException); assertTrue(expected.getMessage(), expected.getMessage().contains("corrupt torn tail in sealed SF segment")); } From 98b739777783f3d827502b83b7a2ace132fa3e57 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 16:00:16 +0100 Subject: [PATCH 52/64] fix(qwp): probe the slot flock O(1) before paying a recovery build on parked slots A managed slot whose flock is held by another live owner is parked as CONTENDED and re-probed on every startup-recovery retry cycle -- every second on the direct pool's private driver, for as long as the owner lives. Each re-probe paid a full recovery build just to reach SlotLock.acquire and throw: two isCandidateOrphan dir enumerations, a complete config re-parse plus builder graph, parent-dir fsync barriers in periodic durability mode, and an owned SegmentManager allocation torn straight back down. Add SlotLock.probeHolder: a side-effect-light non-blocking flock probe that never creates dirs or files, never throws, and routes its momentary hold through the standard close() so an unconfirmed unlock can never leak from a probe. The recovery scan asks the probe first and parks on a held flock for a few syscalls; an indeterminate probe falls through to the full build, which keeps sole ownership of error classification. Races are benign both ways: a free probe can still lose the acquire (parks exactly as before) and a stale held probe is re-observed next cycle. New SenderPoolSfTest proves the arc with a real externally held flock: three parked cycles cost zero builds, and the first cycle after release recovers the slot with exactly one build. --- .../qwp/client/sf/cursor/SlotLock.java | 51 ++++++++++++ .../io/questdb/client/impl/SenderPool.java | 24 ++++++ .../client/test/impl/SenderPoolSfTest.java | 82 +++++++++++++++++++ 3 files changed, 157 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index c7615e7c..17264c5a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -135,6 +135,57 @@ public static SlotLock acquire(String slotDir, boolean syncParentDirectory) { } } + /** + * Side-effect-light contention probe: reports the current holder of the + * slot flock without creating the slot dir or lock file and without + * paying a full engine build. Opens the existing {@code .lock} file + * (absent means nothing can hold a flock on it), try-locks it + * non-blocking, and releases immediately on success. + *

      + * Returns a non-null holder description (the {@code .lock.pid} payload, + * or {@code "unknown"}) when the flock is currently held by a live + * owner; {@code null} when the lock is free or the probe could not + * determine state (missing lock file, open failure). Callers must treat + * {@code null} as "proceed to a full acquire", which owns real error + * classification -- the probe never throws. + *

      + * Races are benign in both directions: a free probe can still lose the + * subsequent acquire to a concurrent owner (the caller handles that + * contention exactly as before), and a held probe that goes stale the + * moment the owner exits is simply re-observed on the caller's next + * cycle. The probe's momentary hold can make a concurrent acquirer see + * spurious contention -- the same class of race two real contenders + * already have. + */ + public static String probeHolder(String slotDir) { + if (slotDir == null || slotDir.isEmpty()) { + return null; + } + // Same pre-step as acquire(): a lock retained by THIS process after + // an unconfirmed unlock would otherwise read as a live holder for as + // long as the retry list carries it. + retryPendingReleases(); + String lockPath = slotDir + "/" + LOCK_FILE_NAME; + if (!Files.exists(lockPath)) { + return null; + } + int fd = Files.openRW(lockPath); + if (fd < 0) { + return null; + } + if (Files.lock(fd) != 0) { + String holder = readHolder(slotDir + "/" + LOCK_PID_FILE_NAME); + Files.close(fd); + return holder; + } + // The flock was free and is momentarily ours. Route the release + // through the standard close() so an unconfirmed unlock is retained + // on the retry list exactly like a normal owner's -- a probe must + // never leak a held flock. + new SlotLock(slotDir, fd).close(); + return null; + } + /** * Replaces the live descriptor with a known-dead value until the returned * guard closes. Test-only: exercises release retry paths without exposing diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 31c688fc..7bd2145c 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; @@ -934,6 +935,29 @@ private RecoveryDrainOutcome drainCandidateSlotForRecovery(int slotIndex, String if (!OrphanScanner.isCandidateOrphan(slotPath)) { return RecoveryDrainOutcome.DRAINED; } + // O(1) contention pre-probe (flock only). A slot parked as + // CONTENDED is re-probed on every retry cycle for as long as its + // live owner runs -- potentially that owner's whole lifetime -- + // and the full recovery build below (config re-parse, builder + // graph, parent-dir fsync barriers in periodic durability, owned + // SegmentManager allocation) would exist only to reach + // SlotLock.acquire and throw. Ask the flock directly first so + // the steady-state cost of a held slot is a few syscalls per + // cycle, not a build. Races are benign in both directions: a + // free probe can still lose the acquire inside the build (the + // contention catch below parks exactly as before), and a held + // probe that goes stale is re-observed on the next cycle. An + // indeterminate probe (null) falls through to the build, which + // owns real error classification. + String probedHolder = SlotLock.probeHolder(slotPath); + if (probedHolder != null) { + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: slot {} is held by another live owner (holder={}); " + + "parking it and continuing with the remaining slots", + slotPath, probedHolder); + } + return RecoveryDrainOutcome.CONTENDED; + } try { // Recovery delegate: forced OFF-mode initial connect (see // createRecoverer / defaultRecoverySender), so this build does diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 0b7d6164..59f03dd7 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -36,6 +36,8 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -3840,6 +3842,86 @@ private static boolean awaitAtLeast(AtomicInteger counter, int target, long time return counter.get() >= target; } + @Test + public void testContendedSlotReprobeUsesFlockProbeNotFullBuild() throws Exception { + // A slot whose flock is held by another LIVE owner is parked and + // re-probed on every retry cycle -- potentially for the owner's whole + // lifetime. The re-probe must ask the flock directly (O(1) probe, + // a few syscalls), not pay a full recovery build (config re-parse, + // builder graph, parent-dir fsync barriers in periodic durability, + // owned SegmentManager allocation) per cycle just to reach + // SlotLock.acquire and throw. + TestUtils.assertMemoryLeak(() -> { + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + String slot0 = slot("default-0"); + + // Seed one unacked frame so slot 0 is a candidate orphan. The + // group root normally comes from Sender.build(); this test seeds + // the slot directly, so create the parent first (mkdir in + // SlotLock.acquire is non-recursive). + Assert.assertEquals(0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT)); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine seed = new CursorSendEngine(slot0, 1 << 20)) { + Unsafe.getUnsafe().setMemory(buf, 16, (byte) 1); + Assert.assertEquals(0L, seed.appendBlocking(buf, 16)); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + Assert.assertTrue("seeded slot must be a candidate orphan", + OrphanScanner.isCandidateOrphan(slot0)); + + AtomicInteger builds = new AtomicInteger(); + IntFunction factory = idx -> { + builds.incrementAndGet(); + // Fidelity with the production build: the recovery build's + // fate is decided by the real slot flock, exactly like + // defaultRecoverySender's engine construction. + SlotLock lock = SlotLock.acquire(slot("default-" + idx)); + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("drain".equals(method.getName())) { + return Boolean.TRUE; + } + if ("close".equals(method.getName())) { + lock.close(); + return null; + } + throw new AssertionError( + "unexpected recovery sender call: " + method.getName()); + }); + }; + + SlotLock held = SlotLock.acquire(slot0); + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + // Steady-state re-probe of a parked contended slot: three + // full cycles while the flock is held. + for (int cycle = 0; cycle < 3; cycle++) { + Assert.assertFalse("a cycle with only a contended slot must defer", + pool.runStartupRecoveryStepForTesting()); + } + Assert.assertEquals("re-probing a contended slot must be a flock probe, " + + "not a full recovery build per cycle", 0, builds.get()); + Assert.assertTrue("parked slot must keep its durable data", + hasSegmentFile(slot0)); + + // The probe must not dampen recovery: once the owner lets + // go, the very next cycle pays exactly one real build and + // drains the slot. + held.close(); + Assert.assertTrue("released slot must be recovered on the next cycle", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("released slot must be recovered with exactly one build", + 1, builds.get()); + Assert.assertFalse("final scan step must mark recovery complete", + pool.runStartupRecoveryStepForTesting()); + } finally { + held.close(); // idempotent when already released + } + }); + } + private static boolean awaitNoSegmentFile(String slotPath, long timeoutMillis) throws InterruptedException { long deadline = System.currentTimeMillis() + timeoutMillis; From 8722bf1f32f00877eb3f4744ba747de3526b7578 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 16:55:08 +0100 Subject: [PATCH 53/64] docs(qwp): state the recovery no-mutation guarantee exactly as strong as it is Two in-diff comments claimed more than the code proves. The recovery-scan comment asserted 'a failed recovery never mutates the slot (single exception: ...sanitizeSealedResidue)'. There are more windows where a recovery that later fails has already durably mutated: the legacy-migration sanitize zeroes residue before SfManifest.create (or any later step) can fail, and validated-extra cleanup plus corrupt-file quarantine run before the active-sanitize barrier or ring construction can fail. Restate the invariant as what the code actually guarantees -- a failed recovery never mutates COMMITTED CHAIN BYTES -- and enumerate the windows, each confined to proven-dead zeroes or preserve-by-rename quarantines. The db8938ec-derived javadoc claimed sanitizeTornTail runs 'only on residue that validation proves non-load-bearing', gliding over the resumed active with '(reclaimed by appends anyway)'. No such proof exists for the active -- it has no successor to bound its accounting, and past a mid-file tear its residue can hold valid-CRC frames of real unacked payloads. Zeroing them is a deliberate POLICY (replay cannot cross the tear; unzeroed residue risks the reseal-brick and stale-frame resurrection hazards), not a validation-derived proof. Say so at every site that carried the stronger wording: openExisting's javadoc, the scan-time observe-only comment, sanitizeTornTail's contract, and the active-sanitize call site in SegmentRing. Comments only; no behavior change. --- .../qwp/client/sf/cursor/MmapSegment.java | 43 ++++++++++++------- .../qwp/client/sf/cursor/SegmentRing.java | 23 +++++++--- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index b8002908..edf70252 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -299,11 +299,16 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { * still hold unreachable frames with valid CRCs -- the only surviving copy * of real payloads -- so whether it may be destroyed is a chain-level * decision this method cannot make. {@link SegmentRing} recovery invokes - * {@link #sanitizeTornTail()} only once the chain has fully validated and - * only on residue that validation proves non-load-bearing (the resumed - * active's tail, which appends are about to reclaim anyway, and sealed - * suffixes whose frame accounting is proven complete); every fail-closed - * path leaves the bytes intact on disk for operator extraction. Clean + * {@link #sanitizeTornTail()} only once the chain has fully validated, + * and the justification differs by role: sealed suffixes are zeroed on + * PROOF (frame accounting validated complete, so the residue can hold no + * replayable frame -- a tear that cost frames fails closed instead, with + * every byte left on disk for operator extraction), while the resumed + * active's tail is zeroed by POLICY -- past a mid-file tear it can still + * hold valid-CRC frames of real unacked payloads, but they are + * unreachable by replay (the FSN sequence breaks at the tear) and + * leaving them risks the reseal-brick and stale-frame-resurrection + * hazards. Clean * partial fills (writer never attempted to write past the last valid * frame) do not log and report {@code 0}. */ @@ -375,13 +380,17 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // Observe-only: report the residue, never touch it here. A // torn tail may be an interrupted append (dead bytes) OR a // mid-file tear with unreachable valid-CRC frames past it -- - // the only surviving copy of real payloads. Only chain-level - // validation can tell the two apart, so the destroy/preserve - // decision belongs to SegmentRing.recover: it invokes - // sanitizeTornTail on the segment it resumes as active (and - // on proven-dead sealed suffixes) AFTER the chain proves the - // residue is not load-bearing, and fails closed with every - // byte left on disk in all other cases. + // the only surviving copy of real payloads. The + // destroy/preserve decision belongs to SegmentRing.recover. + // For sealed members chain validation PROVES the residue + // dead (complete frame accounting) before sanitizeTornTail + // runs, and a tear that cost frames fails closed with every + // byte left on disk. For the segment resumed as active no + // such proof exists (there is no successor to bound it): its + // residue is zeroed by policy once the rest of the chain + // validates, because replay cannot cross the tear and + // unzeroed residue risks reseal-brick and stale-frame + // resurrection. LOG.warn("SF segment {}: torn tail of {} bytes at offset {} " + "(file size {}, frames recovered {}). " + "The residue is preserved pending chain validation. " @@ -752,9 +761,13 @@ public long tornTailBytes() { * {@link #openExisting}: after a mid-file tear the residue can hold * unreachable valid-CRC frames that are the only surviving copy of * unacked payloads. The caller ({@link SegmentRing} recovery) must first - * prove the residue is not load-bearing -- the resumed active's tail - * (reclaimed by appends anyway) or a sealed suffix whose frame accounting - * validated complete against the chain. Must run before any append + * establish the residue is not load-bearing, and the strength of that + * claim differs by role: a sealed suffix is PROVEN dead (frame + * accounting validated complete against the chain), while the resumed + * active's tail past a mid-file tear is discarded by POLICY, not proof + * -- it can hold valid-CRC frames of real unacked payloads that are + * unreachable by replay (the FSN sequence breaks at the tear), and + * leaving them risks the two hazards above. Must run before any append * resumes; appending first would put live frames inside the zeroed range, * so that ordering is rejected. Idempotent; no-op when no residue was * observed. The fsync is load-bearing: in MEMORY durability mode rotation diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index ef5855c5..d366fbc5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -243,10 +243,18 @@ static Recovery recover( // Files whose own bytes prove corruption (bad magic, sub-header size, // negative baseSeq). They are excluded from the chain and quarantined // to .corrupt — but only AFTER the surviving chain validates (or - // resolves to EMPTY), so a failed recovery never mutates the slot - // (single exception: proven-dead sealed residue is zeroed just before - // the fail-closed first-sight throw in sanitizeSealedResidue -- bytes - // the already-validated chain proves no replay can ever need). + // resolves to EMPTY). The precise invariant: a failed recovery never + // mutates COMMITTED CHAIN BYTES. It is not "never mutates the slot" + // -- several windows durably mutate before a later step can still + // fail: proven-dead sealed residue is zeroed just before the + // fail-closed first-sight throw in sanitizeSealedResidue; the + // legacy-migration sanitize zeroes proven-dead residue before + // SfManifest.create (or any later step) can fail; and validated- + // extra cleanup plus corrupt-file quarantine run before the + // active-sanitize barrier or ring construction can fail. Every such + // window is confined to bytes the already-validated chain proves no + // replay can ever need, or to preserve-by-rename quarantines that + // keep the bytes on disk. // Whether a quarantined file was load-bearing is decided by the // manifest-boundary / contiguity checks below, not by the skip itself. // Operational open/stat/read/mmap errors, observed size instability, @@ -555,7 +563,12 @@ static Recovery recover( // where rotation does not sync the sealed predecessor's data // pages. A failed barrier aborts recovery (fail closed); the // retry re-observes the same residue because openExisting never - // mutates. + // mutates. Unlike sealed suffixes, the active's residue is NOT + // proven dead -- past a mid-file tear it can hold valid-CRC + // frames of real unacked payloads. Zeroing is policy: replay + // cannot cross the tear, and preserving the bytes would trade + // the two hazards above for data no recovery path can use (see + // sanitizeTornTail). active.sanitizeTornTail(); for (int i = 1, n = chain.size(); i < n; i++) { chain.get(i - 1).linkSuccessor(chain.get(i)); From ac240447312ab75847fdb7264da41a69c4f7dec4 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 18:16:27 +0100 Subject: [PATCH 54/64] fix(qwp): count a retired slot with stranded data as a recovery deferral The in-range recovery scan skipped any reserved index as "live" without a deferral tick. A RETIRED index (wedged close() kept the slot flock) aliases into that branch: the scan could finish a cycle with zero deferrals and latch recoveryComplete while the retired slot's dir still held unacked durable data. From there nothing in-process ever drained it -- reprobeRetiredSlots() only restores capacity -- so the data waited for a restart or a lucky borrow of that exact index, which steady low load may never produce. Treat a retired index whose dir is still a candidate orphan as a deferral, the same rule as a CONTENDED park: the end-of-scan rewind keeps the cycle alive, and once the deferred cleanup releases the flock the scan reserves the freed index and drains it itself -- no borrow required. Data-inert (durable on disk; a restart already rescanned); this closes the drain- liveness gap for the life of the pool. The regression test drives the scan by hand: forge the wedged retire, walk the scan (pre-fix it latched complete right here), heal the flock, and assert the scan itself delivers the stranded data and only then completes. --- .../io/questdb/client/impl/SenderPool.java | 57 ++++++++- .../client/test/impl/SenderPoolSfTest.java | 118 ++++++++++++++++++ 2 files changed, 171 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 7bd2145c..b67daac6 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -287,6 +287,9 @@ public final class SenderPool implements AutoCloseable { // remaining slots, recoveryDeferredThisCycle records the park, and once // both passes finish with parked slots outstanding the cursors rewind for // another cycle on a later tick instead of latching recoveryComplete. + // A RETIRED index whose dir still holds data is deferred the same way on + // every walk (see the reserved-skip branch in recoverOneSlotStep) so the + // latch can never strand a retired slot's data while the pool lives. // recoveryFailStreak/-Slot track consecutive failures on one candidate; // recoveryWarnedSlots dedups the per-slot WARNs so an indefinitely // retried slot logs once per failure episode, not once per retry. All of @@ -649,7 +652,12 @@ boolean runStartupRecoveryStep() { * can neither target the dir being recovered nor over-allocate past * {@code maxSize}. Prewarmed/borrowed slots (already live, holding their * flock) are skipped, as are empty slots (a cheap directory probe); only a - * slot that actually holds stranded data spends the step's single drain. The + * slot that actually holds stranded data spends the step's single drain. A + * RETIRED index (a wedged close() kept its flock; see {@link #reclaimSlot} + * and the retire branch below) is likewise never drained in place, but + * while its dir still holds data it counts as a deferral, so the scan keeps + * cycling instead of latching {@code recoveryComplete} past stranded data + * that only a restart or a lucky borrow of that index would ever deliver. The * out-of-range pass needs no reservation: those indices have no * {@code slotInUse} entry and are never allocated by borrow(). *

      @@ -702,26 +710,49 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { return false; } int i = recoveryInRangeNext; + String slotPath = sfDir + "/" + slotBaseId + "-" + i; // Reserve this index unless prewarm (or a concurrent borrow after // publication) already holds it live. Count the reservation in // recoveringSlots so the borrow() cap check cannot over-allocate // while this slot is held for recovery. boolean reserved; + boolean retired = false; lock.lock(); try { reserved = slotInUse[i]; if (!reserved) { slotInUse[i] = true; recoveringSlots++; + } else { + retired = isRetiredSlotIndex(i); } } finally { lock.unlock(); } if (reserved) { + // A reserved index is normally LIVE (prewarm or a concurrent + // borrow owns it) and is none of recovery's business. A RETIRED + // index is different: its flock is held by this pool's own + // wedged former delegate, and any unacked data in its dir is + // exactly as stranded as a CONTENDED slot's. Without counting + // it as a deferral the scan would latch recoveryComplete and + // abandon that data until a restart or a lucky borrow of this + // exact index -- which steady low load may never produce. Count + // it -- the same rule as a CONTENDED park -- so the end-of-scan + // rewind keeps the cycle alive; once the deferred engine + // cleanup releases the flock, reprobeRetiredSlots()/the release + // callback frees the index and a later cycle reserves and + // drains it right here. The isCandidateOrphan probe (a few + // syscalls, outside the lock) keeps an already-clean retired + // dir from cycling the scan forever, and racing a concurrent + // recover/borrow can only over-count -- costing one extra + // cheap rewound walk, never a missed candidate. + if (retired && OrphanScanner.isCandidateOrphan(slotPath)) { + recoveryDeferredThisCycle++; + } recoveryInRangeNext++; continue; } - String slotPath = sfDir + "/" + slotBaseId + "-" + i; if (!OrphanScanner.isCandidateOrphan(slotPath)) { // No stranded data: release the reservation and keep scanning; // an empty slot must not cost a whole step. @@ -868,8 +899,8 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { } if (recoveryDeferredThisCycle > 0) { - // At least one slot was parked this cycle (contended or - // persistently failing). Its data stays durable on disk, so + // At least one slot was parked this cycle (contended, persistently + // failing, or retired with data still on disk). Its data stays durable on disk, so // instead of latching recoveryComplete -- which would abandon it // until a restart or a lucky borrow of that index -- rewind the // scan and let the driver's retry cadence run another cycle. @@ -2073,6 +2104,24 @@ private void addRetiredSlot(SenderSlot s) { retiredSlots.add(s); } + /** + * Whether {@code idx} is currently held by a RETIRED slot (see + * {@link #reclaimSlot} and the in-range recovery retire branch) rather + * than a live one. Lets the recovery scan tell "reserved because + * borrowed/prewarmed" apart from "reserved because a wedged close() left + * the flock held" for its deferral accounting: only the latter's dir can + * hold stranded data no borrow is coming for. Caller must hold + * {@code lock}; retiredSlots is bounded by maxSize, so the walk is cheap. + */ + private boolean isRetiredSlotIndex(int idx) { + for (int i = 0, n = retiredSlots.size(); i < n; i++) { + if (retiredSlots.get(i).slotIndex() == idx) { + return true; + } + } + return false; + } + private void recoverRetiredSlotAt(int retiredIndex) { SenderSlot s = retiredSlots.get(retiredIndex); int last = retiredSlots.size() - 1; diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 59f03dd7..4a326208 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -1753,6 +1753,124 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except }); } + @Test + public void testRecoveryScanStaysAliveWhileRetiredSlotHoldsStrandedData() throws Exception { + // Drain-liveness regression: a slot RETIRED mid-scan (recoverer close() + // returned with the flock still held) keeps slotInUse[i] set, so the + // next walk over its index took the "reserved => live" skip WITHOUT + // counting a deferral and latched recoveryComplete with the slot's + // unacked data still on disk. From there the data was stranded until a + // restart or a lucky borrow of that exact index -- which steady low + // load may never produce (testStartupRetiredSlotRecoveredAfterLate- + // FlockRelease passes only because its borrow IS that lucky borrow). + // The scan must instead treat a retired index whose dir is still a + // candidate orphan as a deferral -- the same rule as a CONTENDED park: + // the cycle keeps rewinding, and once the late flock release restores + // the index the SCAN itself drains the data, no borrow required. This + // test is RED (latches complete) until that deferral is counted. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: ack-ing server; the FIRST recovery build of slot 0 is + // forged into the wedged shape (closed=true makes the recovery + // drain throw and close() a no-op, so the real flock stays held + // and slotLockReleased stays false) so the scan's own drain FAILs + // and the slot is retired. deferStartupRecovery=true keeps the + // scan entirely under this test's control -- no background driver. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = new SenderPool(cfg2, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Step 1: builds the forged recoverer, fails its drain, and + // retires the slot (leakedSlots=1, slotInUse[0] stays set). + // The FAILED outcome stops the drive without advancing. + long deadline = System.currentTimeMillis() + 10_000; + while (pool.leakedSlotCount() == 0 + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("scan must retire the wedged slot", + 1, pool.leakedSlotCount()); + + // Re-drive the scan: the retired index is skipped as + // reserved. Pre-fix the first full walk latched + // recoveryComplete; post-fix the still-candidate retired + // dir counts as a deferral and the cycle keeps rewinding. + for (int k = 0; k < 8; k++) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("retired dir must still hold the stranded data", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse( + "scan must NOT latch recoveryComplete while a retired slot still " + + "holds stranded durable data -- that abandons the data " + + "until a restart or a lucky borrow of that exact index", + pool.isRecoveryCompleteForTesting()); + + // The late flock release: the "wedged worker" finishes + // (un-forge + real close). The housekeeper tick re-probes + // retiredSlots and frees the index; the still-alive scan + // must then reserve it and drain the stranded data ITSELF + // -- no borrow anywhere in this phase. + Sender recoverer = forged.get(); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); + recoverer.close(); + pool.reapIdle(); + Assert.assertEquals("late release must restore the retired capacity", + 0, pool.leakedSlotCount()); + + deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("scan must complete once the stranded data is drained", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the scan itself must have drained the stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + } + } + }); + } + @Test public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Exception { // Boundary twin of testStartupRetiredSlotRecoveredAfterLateFlockRelease: From ccb92290b4c784a1f9a052092a08d95ab0101903 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 18:16:27 +0100 Subject: [PATCH 55/64] test(qwp): drain-liveness fuzz for the SF recovery scan and retire machinery Seeded, single-threaded (replayable) randomized schedules of recovery steps, housekeeper ticks, borrows and late flock releases against stranded slot dirs and wedge-forged recovery builds. The forged recoverers model a wedged worker faithfully: alive but streaming into a silent sink, with close-flush off, so neither the forge nor the heal can deliver data on the scan's behalf. Oracle: after every wedge heals and the pool quiesces, the scan must converge and an ordinary-lifecycle close must leave no slot dir with undelivered durable data -- no restart, no lucky borrow. Iteration 0 pins the wedged-retire shape deterministically (prologue drives the retire before any random traffic, heals only at the end) so the suite reds on the scan- abandonment bug class regardless of seed; the remaining iterations randomize freely. Failure messages carry the reproducer seeds. --- .../test/impl/SenderPoolSfFuzzTest.java | 471 ++++++++++++++++++ 1 file changed, 471 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java new file mode 100644 index 00000000..69ffafa6 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java @@ -0,0 +1,471 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.impl.PooledSender; +import io.questdb.client.impl.SenderPool; +import io.questdb.client.std.Files; +import io.questdb.client.std.Rnd; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntFunction; + +/** + * Randomised drain-liveness fuzz for the {@link SenderPool} SF recovery scan, + * the retire/re-probe machinery, and their interleaving with ordinary + * borrow/return traffic. + *

      + * Every iteration builds the same fault family the deterministic tests pin + * one shape of (see {@code SenderPoolSfTest} + * {@code testRecoveryScanStaysAliveWhileRetiredSlotHoldsStrandedData}): + *

        + *
      1. a previous run strands unacked durable data under a random subset of + * the in-range slot dirs (silent server, close without acks);
        2. + *
      2. a new pool over the same {@code sf_dir} faces an ack-ing server, but a + * random subset of its recovery builds are forged into the wedged-close + * shape: {@code close()} returns with the slot flock still held, so the + * scan retires the slot ({@code leakedSlots++}, index stays reserved) + * with its data still on disk;
        3. + *
      3. a random single-threaded schedule of recovery steps, housekeeper + * ticks, borrows (returned or discarded) and late flock releases runs + * the scan, the retire bookkeeping and the borrow cap math against each + * other in randomized order.
        4. + *
      + * After the schedule every wedge is healed (the "worker" exits, the flock + * genuinely drops) and the pool is driven quiescently -- housekeeper tick + + * recovery step, no new load. The oracle is the store-and-forward delivery + * contract at the heart of the drain-liveness bug class: once faults heal and + * the system goes quiet, EVERY durably-accepted row must reach the server IN + * THIS PROCESS through nothing but the pool's ordinary lifecycle -- no + * restart, no lucky borrow required. Concretely: {@code recoveryComplete} + * latches and {@code leakedSlots} returns to zero under quiescent driving, + * and after {@code pool.close()} (which drains live-owned delegates the + * ordinary way -- a slot ADOPTED by a borrow mid-schedule is its owner's to + * deliver, not the scan's) no slot dir may still hold a segment file. Pre-fix, + * any iteration that retires a stranded slot mid-scan latches + * {@code recoveryComplete} past the stranded data (the retired index was + * skipped as "reserved" without counting a deferral); nothing ever drains that + * dir -- close() never owned it -- and the post-close audit fails. + *

      + * The schedule is single-threaded on purpose: with one driver the whole + * iteration is a pure function of the seed, so any failure replays exactly + * with {@code TestUtils.generateRandom(null, s0, s1)} (seeds are printed by + * the harness and repeated in the failure message). Iteration 0 is pinned to + * the worst case -- every slot stranded, slot 0 wedged, and a deterministic + * two-step prologue that drives the scan into the wedged-retire shape BEFORE + * any random traffic (a random schedule could otherwise draw a borrow first, + * adopt slot 0 and never materialize the wedge) -- so the suite cannot go + * green on an unlucky seed while the bug is present; later iterations + * randomize freely. + */ +public class SenderPoolSfFuzzTest { + + private static final int ITERATIONS = 4; + private static final long CONVERGE_BUDGET_MILLIS = 20_000; + + @Test + public void testRetiredSlotDrainLivenessFuzz() throws Exception { + long s0 = System.nanoTime(); + long s1 = System.currentTimeMillis(); + Rnd rnd = TestUtils.generateRandom(null, s0, s1); + try { + for (int iter = 0; iter < ITERATIONS; iter++) { + runOneIteration(rnd, iter); + } + } catch (Throwable t) { + throw new AssertionError("fuzz failure with seeds=" + s0 + "L," + s1 + "L", t); + } + } + + private void runOneIteration(Rnd rnd, int iter) throws Exception { + String sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-sf-pool-fuzz-" + System.nanoTime() + "-" + iter).toString(); + try { + TestUtils.assertMemoryLeak(() -> { + int maxSize = 1 + rnd.nextInt(3); // 1..3 + // Iteration 0 pins the guaranteed-red shape: every slot + // stranded so the wedged recovery build below has data behind + // it. Later iterations may strand any subset (0 = a plain + // clean-scan iteration, still a valid latch/borrow interplay). + int stranded = iter == 0 ? maxSize : rnd.nextInt(maxSize + 1); + + // Phase 1: strand unacked data under default-0..(stranded-1). + if (stranded > 0) { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (SenderPool seed = new SenderPool(seedCfg, stranded, stranded, + 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] s = new PooledSender[stranded]; + for (int i = 0; i < stranded; i++) { + s[i] = seed.borrow(); + } + for (int i = 0; i < stranded; i++) { + int nRows = 1 + rnd.nextInt(3); + for (int r = 0; r < nRows; r++) { + s[i].table("fuzz").longColumn("v", r).atNow(); + s[i].flush(); + } + } + for (int i = stranded - 1; i >= 0; i--) { + s[i].close(); + } + } + } + for (int i = 0; i < stranded; i++) { + Assert.assertTrue("iter " + iter + ": default-" + i + " must hold unacked data", + hasSegmentFile(sfDir + "/default-" + i)); + } + } + + // Phase 2: ack-ing server; wedge a random subset of RECOVERY + // builds (iteration 0 always wedges slot 0). The forge is + // scoped by inRecoveryStep so a borrow can never receive a + // forged delegate -- exactly like a real wedged close(), which + // only the recovery/reclaim paths ever observe. + boolean[] wedge = new boolean[maxSize]; + for (int i = 0; i < maxSize; i++) { + wedge[i] = i < stranded && (iter == 0 ? i == 0 : rnd.nextBoolean()); + } + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + // close_flush_timeout bounds the ordinary-lifecycle drain + // pool.close() performs on live-owned delegates; against + // the ack-ing server it completes in milliseconds. + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=10000;"; + // The forged recoverers model a WEDGED worker faithfully: + // alive (setClosedForTesting only forges the flag; the I/O + // loop keeps pumping) but delivering nothing. They are + // therefore built against a SILENT sink -- an alive worker + // streaming into a black hole -- so the adopted chain is + // never acked and never trimmed; and with close-flush OFF, + // so the heal close() drops the flock WITHOUT draining the + // chain on the data's behalf. Either kindness (an ack-ing + // endpoint or a draining close) would hand the oracle a + // delivery the production wedge semantics do not provide, + // masking scan abandonment. + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] inRecoveryStep = new boolean[1]; + Sender[] forged = new Sender[maxSize]; + List unhealed = new ArrayList<>(); + IntFunction factory = idx -> { + boolean forgeNow = inRecoveryStep[0] && idx < maxSize + && wedge[idx] && forged[idx] == null; + Sender real = Sender.builder(forgeNow ? cfgWedge : cfg) + .senderId("default-" + idx).build(); + if (forgeNow) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + forged[idx] = real; + unhealed.add(real); + } + return real; + }; + + try (SenderPool pool = new SenderPool(cfg, 0, maxSize, 100, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + try { + if (iter == 0) { + // Deterministic prologue for the pinned + // iteration: step the scan until it builds the + // forged recoverer for slot 0 and RETIRES it + // with its data still on disk (step 1), then + // walk once more so the unfixed skip-without- + // deferral path is on the table (step 2). + // Without this a random schedule could borrow + // first, adopt slot 0 as a LIVE slot, and the + // bug shape would never materialize. + for (int k = 0; k < 2; k++) { + inRecoveryStep[0] = true; + try { + pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + } + Assert.assertEquals( + "iter 0 prologue must retire the wedged slot with " + + "its stranded data still on disk", + 1, pool.leakedSlotCount()); + Assert.assertTrue("iter 0 prologue must leave slot 0 stranded", + hasSegmentFile(sfDir + "/default-0")); + } + // The randomized schedule. Single-threaded, so the + // iteration replays exactly from the seed. + int ops = 8 + rnd.nextInt(12); + for (int op = 0; op < ops; op++) { + switch (rnd.nextInt(5)) { + case 0: + case 1: // bias toward driving the scan + inRecoveryStep[0] = true; + try { + pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + break; + case 2: // housekeeper tick: reap + re-probe retired + pool.reapIdle(); + break; + case 3: { // ordinary traffic, returned to the pool + // The lease ALWAYS comes home: a leaked + // lease would (correctly) block close()'s + // ordinary-lifecycle drain and turn a + // write hiccup into a bogus audit red. + PooledSender ps = null; + try { + ps = pool.borrow(); + ps.table("fuzz").longColumn("live", op).atNow(); + ps.flush(); + } catch (LineSenderException e) { + // Legal under wedged-retire capacity + // starvation: borrow timed out. + } finally { + if (ps != null) { + ps.close(); + } + } + break; + } + case 4: // late flock release for one wedged slot + // Pinned iteration 0 keeps its wedge held + // through the WHOLE schedule: a mid-schedule + // heal would let a later borrow adopt the + // freed index and deliver the data through + // its own lifecycle -- legal, but it would + // mask the scan-abandonment bug the pinned + // iteration exists to catch. With the wedge + // held, no borrow can ever own slot 0, so + // post-heal the SCAN is the only possible + // deliverer (fixed) versus nobody (bug). + if (iter != 0 && !unhealed.isEmpty()) { + Sender s = unhealed.remove(rnd.nextInt(unhealed.size())); + ((QwpWebSocketSender) s).setClosedForTesting(false); + s.close(); + } + break; + } + } + + // Heal every remaining wedge: the "workers" exit and + // the flocks genuinely drop. + while (!unhealed.isEmpty()) { + Sender s = unhealed.remove(unhealed.size() - 1); + ((QwpWebSocketSender) s).setClosedForTesting(false); + s.close(); + } + + // Quiescent convergence: housekeeper tick + recovery + // step only -- NO new borrows. The delivery contract + // must not depend on future load. + long deadline = System.currentTimeMillis() + CONVERGE_BUDGET_MILLIS; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.reapIdle(); + boolean more; + inRecoveryStep[0] = true; + try { + more = pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + if (!more && !pool.isRecoveryCompleteForTesting()) { + Thread.sleep(5); + } + } + + // In-pool audit: the scan itself must have + // converged -- a latched-early scan is exactly the + // drain-liveness bug shape. + Assert.assertTrue( + "iter " + iter + ": recovery scan must complete once every " + + "flock is healed -- a latched-early scan strands " + + "retired slots' data until restart", + pool.isRecoveryCompleteForTesting()); + Assert.assertEquals( + "iter " + iter + ": healed flocks must restore all retired capacity", + 0, pool.leakedSlotCount()); + } finally { + // A failed assertion must not leak forged natives: + // un-forge and close for real (idempotent). + for (Sender s : unhealed) { + try { + ((QwpWebSocketSender) s).setClosedForTesting(false); + s.close(); + } catch (Throwable ignore) { + // best-effort teardown + } + } + unhealed.clear(); + } + } + + // Post-close audit: the pool exited through its ordinary + // lifecycle (close() drains live-owned delegates; the scan + // drained everything nobody owned). NOTHING may remain + // durably-accepted-but-undelivered in this process -- no + // restart, no lucky borrow. Pre-fix, a slot retired + // mid-scan fails exactly here: the latched scan abandoned + // it and close() never owned it. + for (int i = 0; i < maxSize; i++) { + Assert.assertTrue( + "iter " + iter + ": default-" + i + " must not hold " + + "undelivered durable data after quiescent convergence " + + "and an ordinary-lifecycle close", + awaitNoSegmentFile(sfDir + "/default-" + i, 15_000)); + } + } + }); + } finally { + rmDir(sfDir); + } + } + + // ------------------------------------------------------------------ + // Local copies of the SenderPoolSfTest harness helpers (private there). + // ------------------------------------------------------------------ + + private static boolean hasSegmentFile(String slotPath) { + if (!Files.exists(slotPath)) { + return false; + } + long find = Files.findFirst(slotPath); + if (find <= 0) { + return false; + } + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + rc = Files.findNext(find); + if (name != null && name.endsWith(".sfa")) { + return true; + } + } + } finally { + Files.findClose(find); + } + return false; + } + + private static boolean awaitNoSegmentFile(String slotPath, long timeoutMillis) + throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (!hasSegmentFile(slotPath)) { + return true; + } + Thread.sleep(10); + } + return !hasSegmentFile(slotPath); + } + + private static void rmDir(String dir) { + if (dir == null || !Files.exists(dir)) { + return; + } + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDir(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static final class CountingAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final Map seqByClient = + new ConcurrentHashMap<>(); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + AtomicLong seq = seqByClient.computeIfAbsent(client, c -> new AtomicLong(0)); + try { + client.sendBinary(buildAck(seq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); // STATUS_OK + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + } + + private static final class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // No ack -- frames stay unacked on disk so recovery sees candidates. + } + } +} From 94ddc606ba100ef66f72cd6ab99beae7e734454d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 18:56:55 +0100 Subject: [PATCH 56/64] fix(qwp): re-arm the recovery scan when a released retired slot still holds data Two remaining drain-liveness windows around retired-slot flock releases: 1. Post-latch runtime retire: discardBroken/reapIdle can retire a wedged slot AFTER the startup scan legitimately latched recoveryComplete. The late release then restored capacity only -- nothing re-drained the dir (the scan was done, close() never owns an unowned dir), so the data waited for a restart or a lucky borrow of that exact index. 2. Mid-cycle release (found by the drain-liveness fuzz): a retire AND its release can both land while the scan is alive but after the cursor already passed that index -- it was LIVE at walk time, so no retired-candidate deferral was counted, and recoveryComplete was still false at release, so no post-latch path applied. A cycle ending with zero deferrals then latched past the freed dir's stranded data. recoverRetiredSlotAt now probes the freed dir: if it is still a candidate orphan it either un-latches and rewinds the scan (post-latch case; the volatile latch publishes the rewound cursors to the unlocked driver gates) or raises recoveryRearmRequested (alive case; cursors stay driver-owned). The end-of-cycle latch decision consumes the flag under the pool lock -- the producer runs under the same lock, so no release can slip between the check and the latch write. Deferred pools re-arm through the housekeeper's unconditional per-tick step; a direct pool whose private driver already exited gets the driver revived (daemon, same loop, joined by close()). Three deterministic pins, each red before this change: the post-latch re-arm, the revived direct driver draining with no external help, and the mid-cycle release that must not latch past data it never revisited. --- .../io/questdb/client/impl/SenderPool.java | 176 +++++++++- .../client/test/impl/SenderPoolSfTest.java | 308 ++++++++++++++++++ 2 files changed, 469 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index b67daac6..6ccc83d8 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -191,6 +191,18 @@ public final class SenderPool implements AutoCloseable { // leave this null. private final Object startupRecoverySignal = new Object(); private final Thread startupRecoveryThread; + // Revived direct-pool recovery driver: spawned by recoverRetiredSlotAt() + // when a retired slot's late flock release re-arms an already-latched + // scan on a pool whose original private driver has exited. Deferred + // pools re-arm through the PoolHousekeeper tick instead and never spawn + // one. Joined by stopStartupRecoveryDriver() on close. Volatile: written + // under `lock`, read by close() without it. + private volatile Thread revivedStartupRecoveryThread; + // Driver policy captured for the re-arm path: deferred pools are driven + // by PoolHousekeeper (never spawn private drivers); direct pools own + // their (revivable) private driver, built via recoveryThreadFactory. + private final boolean deferStartupRecovery; + private final ThreadFactory recoveryThreadFactory; // Test-only constructor seam for the failed-retry operation. Production // uses waitForStartupRecoveryRetry(), which performs the one-second signal // wait; lifecycle tests inject an event barrier without wall-clock checks. @@ -290,6 +302,13 @@ public final class SenderPool implements AutoCloseable { // A RETIRED index whose dir still holds data is deferred the same way on // every walk (see the reserved-skip branch in recoverOneSlotStep) so the // latch can never strand a retired slot's data while the pool lives. + // If the retire lands only AFTER the scan latched (a runtime + // discardBroken/reapIdle reclaim), the late flock release re-arms the + // scan instead: recoverRetiredSlotAt() rewinds the cursors and clears + // recoveryComplete -- volatile, so the un-latch (written under `lock` by + // release callbacks and reprobes) is visible to the unlocked driver + // gates -- and, for direct pools whose private driver already exited, + // revives the driver. Deferred pools re-arm through the housekeeper tick. // recoveryFailStreak/-Slot track consecutive failures on one candidate; // recoveryWarnedSlots dedups the per-slot WARNs so an indefinitely // retried slot logs once per failure episode, not once per retry. All of @@ -297,7 +316,18 @@ public final class SenderPool implements AutoCloseable { private int recoveryInRangeNext; private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; - private boolean recoveryComplete; + private volatile boolean recoveryComplete; + // Set by recoverRetiredSlotAt() whenever a released retired slot's dir is + // still a candidate orphan, and consumed (under `lock`) by the scan's + // end-of-cycle latch decision. Closes the mid-cycle window: a retire AND + // its release can both land while the scan is alive but after the cursor + // already passed that index (it was LIVE when walked -- no retired- + // candidate deferral -- and recoveryComplete was still false at release, + // so the post-latch re-arm does not apply). Without this flag such a + // cycle could end with zero deferrals and latch past the freed dir's + // stranded data. Producer and consumer both run under `lock`, so no + // set can slip between the driver's check and its latch write. + private volatile boolean recoveryRearmRequested; private int recoveryDeferredThisCycle; private int recoveryFailStreak; private int recoveryFailStreakSlot = -1; @@ -455,6 +485,8 @@ private SenderPool( this.maxLifetimeMillis = maxLifetimeMillis; this.postFactoryHook = postFactoryHook; this.beforeFailedStartupRecoveryJoinHook = beforeFailedRecoveryJoinHook; + this.deferStartupRecovery = deferStartupRecovery; + this.recoveryThreadFactory = recoveryThreadFactory; this.startupRecoveryWaiter = recoveryWaiter != null ? recoveryWaiter : this::waitForStartupRecoveryRetry; @@ -912,7 +944,26 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryOutOfRangeNext = 0; return false; } - recoveryComplete = true; + // Latch decision under `lock`: a mid-cycle flock release can free a + // still-candidate dir AFTER this cycle's cursor passed its index (it + // was live/retired at walk time -- no deferral) and BEFORE the latch + // (so the post-latch re-arm in recoverRetiredSlotAt does not apply + // either). recoverRetiredSlotAt raises recoveryRearmRequested under + // the same lock, so consuming it here mutually excludes the race: no + // release can slip between this check and the latch write. + lock.lock(); + try { + if (recoveryRearmRequested) { + recoveryRearmRequested = false; + recoveryDeferredThisCycle = 0; + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + return false; + } + recoveryComplete = true; + } finally { + lock.unlock(); + } return false; } @@ -1583,19 +1634,30 @@ private void stopFailedStartupRecoveryDriver(Thread recoveryThread) { } private void stopStartupRecoveryDriver() { - if (startupRecoveryThread == null || startupRecoveryThread == Thread.currentThread()) { - return; - } - if (beforeStartupRecoveryJoinHook != null) { - beforeStartupRecoveryJoinHook.run(); - } - try { - startupRecoveryThread.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + if (startupRecoveryThread != null && startupRecoveryThread != Thread.currentThread()) { + if (beforeStartupRecoveryJoinHook != null) { + beforeStartupRecoveryJoinHook.run(); + } + try { + startupRecoveryThread.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (afterStartupRecoveryJoinHook != null) { + afterStartupRecoveryJoinHook.run(); + } } - if (afterStartupRecoveryJoinHook != null) { - afterStartupRecoveryJoinHook.run(); + // A revived driver (late-release re-arm on a direct pool; see + // reviveDirectRecoveryDriverIfNeeded) is joined with the same bound: + // `closed` is already raised, so its loop exits on the next waiter + // wake-up, within RECOVERY_RETRY_INTERVAL_MILLIS < STOP_TIMEOUT. + Thread revived = revivedStartupRecoveryThread; + if (revived != null && revived != Thread.currentThread()) { + try { + revived.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } } @@ -2019,7 +2081,10 @@ private static boolean flockReleased(SenderSlot s) { * capacity and no borrow reuses the still-locked dir unless * {@link #reprobeRetiredSlots} later observes the deferred cleanup's * release and recovers the index. Either way {@code closingSlots} is - * decremented. + * decremented. A later release that frees a dir still holding unacked + * data also re-arms the startup recovery scan (see + * {@link #recoverRetiredSlotAt}) so the data is drained in-process + * instead of waiting for a restart or a lucky borrow of that index. *

      * Caller must hold {@code lock} and is responsible for signalling waiters * (only the free path admits a new creation). Shared by @@ -2137,6 +2202,87 @@ private void recoverRetiredSlotAt(int retiredIndex) { LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + "pool capacity restored, now {} of {} usable [leakedSlots={}]", s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + // The freed dir may still hold unacked durable data: a runtime retire + // (discardBroken/reapIdle) can land AFTER the startup scan latched + // recoveryComplete, and close() never drains an unowned dir, so + // without re-arming the scan that data would wait for a restart or a + // lucky borrow of exactly this index. When the latch is already set, + // rewind the cursors FIRST, then clear the volatile latch (the write + // order publishes the rewound cursors to the next driver's unlocked + // gate read), then revive the private driver when this is a direct + // pool whose original driver already exited; recoveryComplete==true + // here also guarantees no driver is mid-step, so touching the + // driver-owned cursors is safe. When the scan is still ALIVE, only + // raise recoveryRearmRequested: the cursor may already have passed + // this index while it was live/retired, and the flag makes the + // end-of-cycle latch decision rewind instead -- the cursors stay + // strictly driver-owned. The isCandidateOrphan probe is a bounded + // directory scan; holding the pool lock across it mirrors + // reprobeRetiredSlots()' delegate probes. The pass-2 work list is + // kept and only its cursor rewound, exactly like the end-of-scan + // cycle rewind. + if (sfDir != null + && OrphanScanner.isCandidateOrphan(sfDir + "/" + slotBaseId + "-" + s.slotIndex())) { + if (recoveryComplete) { + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + recoveryDeferredThisCycle = 0; + recoveryRearmRequested = false; + recoveryComplete = false; + LOG.info("SF slot {} released with stranded data still on disk; re-arming the " + + "startup recovery scan to drain it", s.slotIndex()); + reviveDirectRecoveryDriverIfNeeded(); + } else { + recoveryRearmRequested = true; + LOG.info("SF slot {} released with stranded data while the recovery scan is " + + "mid-cycle; flagging a rewind so this cycle cannot latch past it", + s.slotIndex()); + } + } + } + + /** + * Re-arms a driver for an un-latched scan (see {@link #recoverRetiredSlotAt}). + * Deferred pools need nothing: {@code PoolHousekeeper} calls + * {@link #runStartupRecoveryStep()} on every tick and its gate re-opens + * with the cleared latch. A direct pool's private driver exits once the + * scan completes, so if neither the original nor a previously revived + * driver is alive, spawn a fresh one running the same loop -- it drains + * the backlog, re-latches, and exits; {@code close()} joins it via + * {@link #stopStartupRecoveryDriver}. Caller must hold {@code lock}; + * {@code Thread.start()} is cheap enough to run under it. Best-effort: a + * spawn failure is logged and never propagates into the release callback + * that triggered the re-arm (the data stays durable for a restart). A + * driver observed alive here can, in principle, be exiting past its final + * latch check; that microscopic window degrades to the pre-fix behavior + * (the data waits for the next release event or a restart) and is + * accepted rather than risking two concurrent drivers on the + * driver-owned cursors. + */ + private void reviveDirectRecoveryDriverIfNeeded() { + if (deferStartupRecovery || closed) { + return; + } + if (startupRecoveryThread != null && startupRecoveryThread.isAlive()) { + return; + } + Thread revived = revivedStartupRecoveryThread; + if (revived != null && revived.isAlive()) { + return; + } + try { + ThreadFactory factory = recoveryThreadFactory != null + ? recoveryThreadFactory + : SenderPool::createStartupRecoveryThread; + Thread t = factory.newThread(this::runStartupRecoveryLoop); + t.setDaemon(true); + revivedStartupRecoveryThread = t; + t.start(); + } catch (Throwable e) { + revivedStartupRecoveryThread = null; + LOG.warn("could not revive the startup recovery driver ({}); the released slot's " + + "data stays durable on disk for a later attempt or restart", e.toString()); + } } // Outcome of one drainCandidateSlotForRecovery attempt, letting the scan diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 4a326208..e7b48aa3 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -1871,6 +1871,314 @@ public void testRecoveryScanStaysAliveWhileRetiredSlotHoldsStrandedData() throws }); } + @Test + public void testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease() throws Exception { + // Residual drain-liveness gap, runtime flavour: the startup scan + // finishes LEGITIMATELY (recoveryComplete latches), and only LATER + // does a runtime reclaim (discardBroken) retire a slot whose close() + // kept the flock -- with unacked durable data still in its dir. When + // the wedged worker finally releases the flock, recoverRetiredSlotAt + // restores the CAPACITY but nothing re-drains the dir: the scan is + // latched and close() never owns the dir, so the data waits for a + // restart or a lucky borrow of that exact index. Pin the fix: the + // flock RELEASE of a retired slot whose dir is still a candidate + // orphan must re-arm the scan (un-latch + rewind) so the ordinary + // driver cadence drains it in THIS process. RED at the un-latch + // assertion until recoverRetiredSlotAt re-arms. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + // The wedged sender: alive worker streaming into a silent sink + // (rows stay durably unacked) whose close() must never drain + // the chain on the data's behalf. + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] wedgeNextBuild = new boolean[1]; + IntFunction factory = idx -> + Sender.builder(wedgeNextBuild[0] ? cfgWedge : cfg) + .senderId("default-" + idx).build(); + + try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Phase 1: the scan completes legitimately (clean dirs). + pool.runStartupRecoveryToCompletionForTesting(); + Assert.assertTrue("precondition: scan must have latched complete", + pool.isRecoveryCompleteForTesting()); + + // Phase 2: runtime wedge AFTER the latch. The borrowed + // sender writes durably against the silent sink, then its + // close() is forged to keep the flock (closed=true makes + // discardBroken's close a no-op). + wedgeNextBuild[0] = true; + PooledSender ps = pool.borrow(); + wedgeNextBuild[0] = false; + for (int i = 0; i < 3; i++) { + ps.table("resid").longColumn("v", i).atNow(); + ps.flush(); + } + Sender delegate = ps.getDelegateForTesting(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + Assert.assertEquals("runtime reclaim must retire the wedged slot", + 1, pool.leakedSlotCount()); + Assert.assertTrue("retired dir must hold the stranded data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("retire alone must not disturb the latch", + pool.isRecoveryCompleteForTesting()); + + // Phase 3: the late release -- the wedged worker exits + // WITHOUT delivering (close-flush 0 against the silent + // sink). The release listener restores capacity; the fix + // must ALSO re-arm the scan because the freed dir is still + // a candidate orphan. + ((QwpWebSocketSender) delegate).setClosedForTesting(false); + delegate.close(); + Assert.assertEquals("release must restore the retired capacity", + 0, pool.leakedSlotCount()); + Assert.assertFalse( + "flock release of a retired slot with stranded data must re-arm " + + "the recovery scan -- otherwise the data waits for a restart " + + "or a lucky borrow of that exact index", + pool.isRecoveryCompleteForTesting()); + + // Phase 4: the ordinary driver cadence (housekeeper proxy) + // must now drain the dir and re-latch. + long deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("re-armed scan must complete once the data is drained", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the re-armed scan must deliver the stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + } + } + }); + } + + @Test + public void testRevivedDirectDriverDrainsRuntimeRetiredSlotAfterLateRelease() throws Exception { + // Direct-pool twin of + // testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease: a + // direct pool (deferStartupRecovery=false) finishes its inline scan at + // construction and lets its private recovery driver die. A runtime + // retire after that leaves NO driver to pick up the re-armed scan + // when the flock releases -- the un-latch alone is not enough. Pin + // the revival: the release must respawn the direct pool's private + // driver, which drains the freed dir with NO test-side stepping and + // no borrow. RED (segments never clear) until release revives the + // driver. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] wedgeNextBuild = new boolean[1]; + IntFunction factory = idx -> + Sender.builder(wedgeNextBuild[0] ? cfgWedge : cfg) + .senderId("default-" + idx).build(); + + // deferStartupRecovery=false: the inline scan over the clean + // dir completes at construction; no private driver survives. + try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, false)) { + Assert.assertTrue("precondition: inline scan must have latched complete", + pool.isRecoveryCompleteForTesting()); + + wedgeNextBuild[0] = true; + PooledSender ps = pool.borrow(); + wedgeNextBuild[0] = false; + for (int i = 0; i < 3; i++) { + ps.table("resid").longColumn("v", i).atNow(); + ps.flush(); + } + Sender delegate = ps.getDelegateForTesting(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + Assert.assertEquals("runtime reclaim must retire the wedged slot", + 1, pool.leakedSlotCount()); + Assert.assertTrue("retired dir must hold the stranded data", + hasSegmentFile(slot("default-0"))); + + // Late release without delivery. From here the pool must + // recover the data ENTIRELY on its own: no recovery-step + // calls, no reapIdle, no borrow -- the revived private + // driver is the only possible deliverer. + ((QwpWebSocketSender) delegate).setClosedForTesting(false); + delegate.close(); + + Assert.assertTrue( + "the revived direct driver must drain the freed dir on its own -- " + + "no steps, no reap, no borrow", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + long deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + Assert.assertTrue("the revived driver must re-latch completion", + pool.isRecoveryCompleteForTesting()); + Assert.assertEquals(0, pool.leakedSlotCount()); + } + } + }); + } + + @Test + public void testMidCycleFlockReleaseCannotLatchScanPastStrandedData() throws Exception { + // Third drain-liveness shape (found by SenderPoolSfFuzzTest): a + // retire AND its flock release both land while the scan is ALIVE but + // after the cursor already passed that index in its current cycle -- + // the index was LIVE (borrowed) when walked, so no retired-candidate + // deferral was counted, and the release fires with recoveryComplete + // still false, so the post-latch re-arm path does not apply either. + // If the rest of the cycle finds nothing to defer, the scan latches + // with the freed dir's data stranded. Pin the fix: a release that + // frees a still-candidate dir mid-cycle must flag a rewind that the + // latch decision consumes, so the cycle cannot latch past it. RED at + // the post-latch assertion until the rearm flag exists. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + // Choreography state: slot 0 is the wedge target (borrowed + // BEFORE the walk so the cursor skips it as live). Slot 1's + // FIRST recovery build points at the silent sink so its drain + // times out (FAILED, flock released, cursor parked ON slot 1 + // past slot 0); its second build is real and drains. + boolean[] wedgeNextBuild = new boolean[1]; + int[] slot1Builds = new int[1]; + IntFunction factory = idx -> { + String c = cfg; + if (wedgeNextBuild[0]) { + c = cfgWedge; + } else if (idx == 1 && slot1Builds[0]++ == 0) { + c = cfgWedge; + } + return Sender.builder(c).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = new SenderPool(cfg, 0, 2, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Strand slot 1 only: plant unacked rows through a + // borrowed silent-sink sender... simpler: write via a + // wedge-built borrow on slot 0 FIRST (it will stay live + // through the walk), and seed slot 1 by a second borrow + // released before recovery starts. + wedgeNextBuild[0] = true; + PooledSender live0 = pool.borrow(); // creates slot 0, silent sink + wedgeNextBuild[0] = false; + wedgeNextBuild[0] = true; + PooledSender seed1 = pool.borrow(); // creates slot 1, silent sink + wedgeNextBuild[0] = false; + for (int i = 0; i < 2; i++) { + live0.table("midcycle").longColumn("v", i).atNow(); + live0.flush(); + seed1.table("midcycle").longColumn("w", i).atNow(); + seed1.flush(); + } + // Return slot 1's seeder through the HEALTHY discard path + // (real close, flock released, data left unacked on disk). + Sender seed1Delegate = seed1.getDelegateForTesting(); + pool.discardBrokenForTesting(seed1); + Assert.assertEquals("seeding must not leak capacity", 0, pool.leakedSlotCount()); + Assert.assertTrue("slot 1 must hold unacked data", + hasSegmentFile(slot("default-1"))); + Assert.assertTrue("slot 0 must hold unacked data", + hasSegmentFile(slot("default-0"))); + + // Step A: the walk skips slot 0 (live borrow -- NO + // deferral, correctly), reaches slot 1, builds the + // silent-sink recoverer, and the drain times out: FAILED, + // cursor parked ON slot 1, scan alive, zero deferrals. + Assert.assertFalse(pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("scan must still be running", + pool.isRecoveryCompleteForTesting()); + + // Mid-cycle: retire slot 0 (wedged close) and heal it + // immediately -- the release lands with the scan alive and + // the cursor already PAST slot 0. + Sender live0Delegate = live0.getDelegateForTesting(); + ((QwpWebSocketSender) live0Delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(live0); + Assert.assertEquals(1, pool.leakedSlotCount()); + ((QwpWebSocketSender) live0Delegate).setClosedForTesting(false); + live0Delegate.close(); + Assert.assertEquals("release must restore capacity", 0, pool.leakedSlotCount()); + Assert.assertTrue("slot 0 data must still be stranded", + hasSegmentFile(slot("default-0"))); + + // Step B: cursor resumes AT slot 1 (never revisits slot 0 + // this cycle), slot 1's second build is real and drains, + // the cycle ends with zero deferrals. Pre-fix this latched + // recoveryComplete past slot 0's stranded data; the fix's + // rearm flag must force a rewind instead. + long deadline = System.currentTimeMillis() + 10_000; + while (hasSegmentFile(slot("default-1")) + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertFalse("slot 1 must have drained", hasSegmentFile(slot("default-1"))); + // One more step reaches the end-of-cycle latch decision + // (pass 2 is empty, zero deferrals were counted). + pool.runStartupRecoveryStepForTesting(); + Assert.assertFalse( + "a mid-cycle flock release that frees a still-candidate dir must " + + "flag a rewind -- the cycle must NOT latch recoveryComplete " + + "past the stranded data it never revisited", + pool.isRecoveryCompleteForTesting()); + + // The rewound cycle must now drain slot 0 and only then + // complete. + deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("rewound scan must complete after draining slot 0", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the rewound scan must deliver slot 0's stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + // Silence the unused warning; the seeder delegate was + // closed for real by discardBroken (healthy path). + Assert.assertNotNull(seed1Delegate); + } + } + }); + } + @Test public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Exception { // Boundary twin of testStartupRetiredSlotRecoveredAfterLateFlockRelease: From e86835480b38c2c9ea880e8ca364c970c1cb19a2 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 18:56:55 +0100 Subject: [PATCH 57/64] test(qwp): fuzz both retired-slot drain-liveness shapes Extends the drain-liveness fuzz with a runtime-wedge action -- a borrow built against the silent sink, written through, then wedged-closed and discarded, so reclaimSlot retires it with its rows durably unacked -- and a second pinned iteration for the post-latch residual: the scan completes legitimately over clean dirs first, then the runtime wedge lands, and only the end-of-schedule heal may deliver the data back through the re-armed scan. Random iterations draw the runtime wedge freely so runtime retires also interleave with a still-running scan (that interleaving is what surfaced the mid-cycle release gap this change's sibling fix closes). The fault injector switches off at the end-of-schedule heal (faultsHealed): without it, a wedge[idx] flag whose first recovery build only happened DURING the quiescent convergence would forge a brand-new wedge after "every fault healed" -- one the schedule can never heal -- and the scan would correctly refuse to complete, failing the audit for the wrong reason. Discrimination matrix: both fixes green; residual fix removed reds at the pinned iteration 1; both fixes removed reds at the pinned iteration 0. --- .../test/impl/SenderPoolSfFuzzTest.java | 142 +++++++++++++++--- 1 file changed, 124 insertions(+), 18 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java index 69ffafa6..3220d80e 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java @@ -88,17 +88,31 @@ * The schedule is single-threaded on purpose: with one driver the whole * iteration is a pure function of the seed, so any failure replays exactly * with {@code TestUtils.generateRandom(null, s0, s1)} (seeds are printed by - * the harness and repeated in the failure message). Iteration 0 is pinned to - * the worst case -- every slot stranded, slot 0 wedged, and a deterministic - * two-step prologue that drives the scan into the wedged-retire shape BEFORE - * any random traffic (a random schedule could otherwise draw a borrow first, - * adopt slot 0 and never materialize the wedge) -- so the suite cannot go - * green on an unlucky seed while the bug is present; later iterations - * randomize freely. + * the harness and repeated in the failure message). Two iterations are pinned + * so the suite cannot go green on an unlucky seed while either bug shape is + * present; the rest randomize freely: + *

        + *
      • Iteration 0 -- scan abandonment. Every slot stranded, slot 0 + * wedged, and a deterministic two-step prologue drives the scan into + * the wedged-retire shape BEFORE any random traffic (a random schedule + * could otherwise draw a borrow first, adopt slot 0 and never + * materialize the wedge). Pre-fix, the scan skipped the retired index + * without a deferral and latched {@code recoveryComplete} past the + * stranded data.
        • + *
      • Iteration 1 -- post-latch runtime retire (the residual). The + * scan completes LEGITIMATELY over clean dirs first; only then does a + * borrowed sender -- built against the silent sink so its rows stay + * durably unacked -- take the wedged-close discard path + * ({@code reclaimSlot} retires it). Pre-fix, the late flock release + * restored capacity but nothing re-armed the latched scan, so the + * data waited for a restart or a lucky borrow of that index.
        • + *
      + * Random iterations (2+) additionally draw the runtime-wedge op freely, so + * runtime retires also interleave with a still-running scan. */ public class SenderPoolSfFuzzTest { - private static final int ITERATIONS = 4; + private static final int ITERATIONS = 5; private static final long CONVERGE_BUDGET_MILLIS = 20_000; @Test @@ -121,11 +135,13 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { try { TestUtils.assertMemoryLeak(() -> { int maxSize = 1 + rnd.nextInt(3); // 1..3 - // Iteration 0 pins the guaranteed-red shape: every slot - // stranded so the wedged recovery build below has data behind - // it. Later iterations may strand any subset (0 = a plain - // clean-scan iteration, still a valid latch/borrow interplay). - int stranded = iter == 0 ? maxSize : rnd.nextInt(maxSize + 1); + // Iteration 0 pins the guaranteed-red scan-abandonment shape: + // every slot stranded so the wedged recovery build below has + // data behind it. Iteration 1 pins the residual and needs a + // CLEAN start so the scan latches legitimately first. Later + // iterations may strand any subset (0 = a plain clean-scan + // iteration, still a valid latch/borrow interplay). + int stranded = iter == 0 ? maxSize : iter == 1 ? 0 : rnd.nextInt(maxSize + 1); // Phase 1: strand unacked data under default-0..(stranded-1). if (stranded > 0) { @@ -168,6 +184,8 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { for (int i = 0; i < maxSize; i++) { wedge[i] = i < stranded && (iter == 0 ? i == 0 : rnd.nextBoolean()); } + // Iteration 1's fault is a RUNTIME wedge, not an in-scan one. + boolean forceRuntimeWedge = iter == 1; CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler); TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { @@ -196,12 +214,27 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { + ";close_flush_timeout_millis=0;"; boolean[] inRecoveryStep = new boolean[1]; + // Latched by the end-of-schedule heal: from that point the + // fault injector is OFF. Without this, a wedge[idx] flag + // whose first recovery build only happens DURING the + // quiescent convergence would forge a brand-new wedge + // after "every fault healed" -- a fault the schedule can + // never heal -- and the scan would (correctly!) refuse to + // complete, failing the audit for the wrong reason. + boolean[] faultsHealed = new boolean[1]; + boolean[] wedgeNextBorrow = new boolean[1]; + Sender[] lastWedgeBuild = new Sender[1]; Sender[] forged = new Sender[maxSize]; List unhealed = new ArrayList<>(); IntFunction factory = idx -> { - boolean forgeNow = inRecoveryStep[0] && idx < maxSize + boolean forgeNow = inRecoveryStep[0] && !faultsHealed[0] && idx < maxSize && wedge[idx] && forged[idx] == null; - Sender real = Sender.builder(forgeNow ? cfgWedge : cfg) + // A runtime-wedge borrow is also built against the + // silent sink (rows must stay durably unacked) but is + // NOT forged closed at build: the op writes through it + // first, then forges and discards. + boolean wedgeBorrow = !inRecoveryStep[0] && wedgeNextBorrow[0]; + Sender real = Sender.builder(forgeNow || wedgeBorrow ? cfgWedge : cfg) .senderId("default-" + idx).build(); if (forgeNow) { try { @@ -211,6 +244,8 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { } forged[idx] = real; unhealed.add(real); + } else if (wedgeBorrow) { + lastWedgeBuild[0] = real; } return real; }; @@ -243,11 +278,49 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { Assert.assertTrue("iter 0 prologue must leave slot 0 stranded", hasSegmentFile(sfDir + "/default-0")); } + if (forceRuntimeWedge) { + // Pinned residual prologue: latch the scan + // legitimately FIRST, then retire a live + // borrow whose rows are durably unacked. No + // heal until the end of the schedule, so the + // post-heal re-arm (or its absence) is the + // only thing that decides the audit. + long latchDeadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < latchDeadline) { + inRecoveryStep[0] = true; + try { + pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + } + Assert.assertTrue("iter 1 prologue: scan must latch over clean dirs", + pool.isRecoveryCompleteForTesting()); + wedgeNextBorrow[0] = true; + PooledSender ps = pool.borrow(); + wedgeNextBorrow[0] = false; + Assert.assertSame("iter 1 prologue: borrow must create the wedge target", + lastWedgeBuild[0], ps.getDelegateForTesting()); + lastWedgeBuild[0] = null; + for (int r = 0; r < 2; r++) { + ps.table("fuzz").longColumn("resid", r).atNow(); + ps.flush(); + } + Sender delegate = ps.getDelegateForTesting(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + unhealed.add(delegate); + Assert.assertEquals("iter 1 prologue must retire the runtime wedge", + 1, pool.leakedSlotCount()); + Assert.assertTrue("iter 1 prologue must leave its slot stranded", + hasSegmentFile(sfDir + "/default-0")); + } // The randomized schedule. Single-threaded, so the // iteration replays exactly from the seed. int ops = 8 + rnd.nextInt(12); for (int op = 0; op < ops; op++) { - switch (rnd.nextInt(5)) { + switch (rnd.nextInt(iter >= 2 ? 6 : 5)) { case 0: case 1: // bias toward driving the scan inRecoveryStep[0] = true; @@ -291,17 +364,50 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { // held, no borrow can ever own slot 0, so // post-heal the SCAN is the only possible // deliverer (fixed) versus nobody (bug). - if (iter != 0 && !unhealed.isEmpty()) { + if (iter >= 2 && !unhealed.isEmpty()) { Sender s = unhealed.remove(rnd.nextInt(unhealed.size())); ((QwpWebSocketSender) s).setClosedForTesting(false); s.close(); } break; + case 5: { // runtime wedge: retire a live borrow undelivered + wedgeNextBorrow[0] = true; + PooledSender ps = null; + try { + ps = pool.borrow(); + } catch (LineSenderException e) { + // Capacity-starved: legal, move on. + break; + } finally { + // Disarm regardless: a reused borrow + // must not leave the flag armed for + // an unrelated later creation. + wedgeNextBorrow[0] = false; + } + Sender delegate = ps.getDelegateForTesting(); + if (delegate == lastWedgeBuild[0]) { + lastWedgeBuild[0] = null; + ps.table("fuzz").longColumn("wedged", op).atNow(); + ps.flush(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + unhealed.add(delegate); + } else { + // Borrow reused an idle (ack-server) + // sender; nothing to wedge. + lastWedgeBuild[0] = null; + ps.close(); + } + break; + } } } // Heal every remaining wedge: the "workers" exit and - // the flocks genuinely drop. + // the flocks genuinely drop. Also switch the fault + // injector off -- no new wedge may be born after + // this point (see faultsHealed). + faultsHealed[0] = true; while (!unhealed.isEmpty()) { Sender s = unhealed.remove(unhealed.size() - 1); ((QwpWebSocketSender) s).setClosedForTesting(false); From ab2b3b69708a4b309353acf765c1667d661a68e7 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 19:11:17 +0100 Subject: [PATCH 58/64] fix(qwp): make the recovery-driver revival handoff transactional The revive path decided "will a driver observe my un-latch?" by probing Thread.isAlive() -- an out-of-band signal not ordered with the driver's own exit decision. A driver past its final latch check could still report alive, so a release landing in that window skipped the spawn and left the un-latched scan ownerless until the next release event or restart (a microscopic, documented-but-real lost-wakeup corner on direct pools). Ownership is now a lock-guarded token (recoveryDriverRunning). The loop drops it transactionally: releaseRecoveryDriverOwnership re-checks the latch under the pool lock before the drop, so an un-latch that raced the exit keeps this thread driving; the producer (recoverRetiredSlotAt) clears the latch and reads the token under the same lock, so it spawns a successor exactly when no owner remains. Mutual exclusion makes the two decisions serial: no interleaving leaves an un-latched scan ownerless, and cursor ownership hands off cleanly (predecessor's last cursor touch happens-before its token drop happens-before the successor's spawn, all via one lock). Loop semantics are otherwise preserved: waiter only after a workless step on a live un-latched scan, prompt exits on close/interrupt (which also drop the token so a still-live pool can revive later). --- .../io/questdb/client/impl/SenderPool.java | 103 +++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 6ccc83d8..d49a24e1 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -198,6 +198,16 @@ public final class SenderPool implements AutoCloseable { // one. Joined by stopStartupRecoveryDriver() on close. Volatile: written // under `lock`, read by close() without it. private volatile Thread revivedStartupRecoveryThread; + // Ownership token for the private (direct-pool) recovery driver duty. + // Guarded by `lock`. TRUE while some thread owns runStartupRecoveryLoop; + // the loop drops it transactionally on exit (re-checking the latch under + // the lock first -- see releaseRecoveryDriverOwnership), and + // reviveDirectRecoveryDriverIfNeeded() spawns a successor only when it + // is FALSE. Replaces a Thread.isAlive() heuristic whose read raced the + // driver's final latch check: a driver observed "alive" could already + // have committed to exit, leaving an un-latched scan ownerless until the + // next release event or a restart. + private boolean recoveryDriverRunning; // Driver policy captured for the re-arm path: deferred pools are driven // by PoolHousekeeper (never spawn private drivers); direct pools own // their (revivable) private driver, built via recoveryThreadFactory. @@ -544,6 +554,9 @@ private SenderPool( : SenderPool::createStartupRecoveryThread; recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop); recoveryThread.setDaemon(true); + // Ownership precedes start(): the loop's transactional + // exit is the only place the token is dropped. + recoveryDriverRunning = true; } } this.startupRecoveryThread = recoveryThread; @@ -609,7 +622,25 @@ void runStartupRecoveryToCompletion() { } private void runStartupRecoveryLoop() { - while (!closed && !recoveryComplete) { + while (true) { + if (closed || Thread.currentThread().isInterrupted()) { + // Unconditional exits: still drop ownership so a later re-arm + // on a live pool can spawn a successor (the revive gate + // ignores a closing pool anyway). + releaseRecoveryDriverOwnership(false); + return; + } + if (recoveryComplete) { + // Transactional exit: re-check the latch under `lock` so an + // un-latch (recoverRetiredSlotAt's re-arm) cannot slip + // between this thread's decision to exit and its ownership + // drop -- the lost-wakeup race a Thread.isAlive() probe had. + if (releaseRecoveryDriverOwnership(true)) { + return; + } + // An un-latch landed first: this thread stays the driver. + continue; + } boolean hasImmediateWork; try { hasImmediateWork = runStartupRecoveryStep(); @@ -618,18 +649,37 @@ private void runStartupRecoveryLoop() { // delegate Error must not permanently kill its only driver. hasImmediateWork = false; } - if (closed || recoveryComplete) { - return; - } - if (!hasImmediateWork) { + if (!hasImmediateWork && !closed && !recoveryComplete) { startupRecoveryWaiter.run(); - if (Thread.currentThread().isInterrupted()) { - return; - } } } } + /** + * Drops this thread's private-driver ownership token, transactionally + * against the re-arm path. With {@code recheckLatch}, an un-latch that + * raced this driver's exit decision is detected under {@code lock} and + * ownership is KEPT ({@code false} is returned; the caller keeps + * driving). The producer ({@link #recoverRetiredSlotAt}) clears the + * latch and reads {@code recoveryDriverRunning} under the same lock, so + * exactly one of the two parties always owns the follow-up: either this + * driver observes the cleared latch, or the producer observes the + * dropped token and spawns a successor. No interleaving leaves an + * un-latched scan ownerless. + */ + private boolean releaseRecoveryDriverOwnership(boolean recheckLatch) { + lock.lock(); + try { + if (recheckLatch && !closed && !recoveryComplete) { + return false; + } + recoveryDriverRunning = false; + return true; + } finally { + lock.unlock(); + } + } + /** * Recovers at most ONE stranded managed slot and reports whether more remain. * The private direct driver or {@link PoolHousekeeper} drives steps @@ -2246,28 +2296,28 @@ private void recoverRetiredSlotAt(int retiredIndex) { * Deferred pools need nothing: {@code PoolHousekeeper} calls * {@link #runStartupRecoveryStep()} on every tick and its gate re-opens * with the cleared latch. A direct pool's private driver exits once the - * scan completes, so if neither the original nor a previously revived - * driver is alive, spawn a fresh one running the same loop -- it drains - * the backlog, re-latches, and exits; {@code close()} joins it via - * {@link #stopStartupRecoveryDriver}. Caller must hold {@code lock}; - * {@code Thread.start()} is cheap enough to run under it. Best-effort: a - * spawn failure is logged and never propagates into the release callback - * that triggered the re-arm (the data stays durable for a restart). A - * driver observed alive here can, in principle, be exiting past its final - * latch check; that microscopic window degrades to the pre-fix behavior - * (the data waits for the next release event or a restart) and is - * accepted rather than risking two concurrent drivers on the - * driver-owned cursors. + * scan completes, so when no thread currently owns the driver duty + * ({@code recoveryDriverRunning} false), spawn a successor running the + * same loop -- it drains the backlog, re-latches, and exits; close() + * joins it via {@link #stopStartupRecoveryDriver}. Ownership is a + * lock-guarded token, not a {@code Thread.isAlive()} probe: the loop + * drops it transactionally ({@link #releaseRecoveryDriverOwnership} + * re-checks the latch under this same lock first), so a driver that is + * mid-exit either observes this caller's un-latch and keeps driving, or + * has already dropped the token and a successor is spawned here -- no + * interleaving leaves the un-latched scan ownerless. Caller must hold + * {@code lock}; {@code Thread.start()} is cheap enough to run under it. + * Best-effort: a spawn failure is logged and never propagates into the + * release callback that triggered the re-arm (the data stays durable + * for a restart). */ private void reviveDirectRecoveryDriverIfNeeded() { if (deferStartupRecovery || closed) { return; } - if (startupRecoveryThread != null && startupRecoveryThread.isAlive()) { - return; - } - Thread revived = revivedStartupRecoveryThread; - if (revived != null && revived.isAlive()) { + if (recoveryDriverRunning) { + // A live driver's exit re-checks the latch under this lock, so + // it is guaranteed to observe the un-latch that brought us here. return; } try { @@ -2276,10 +2326,11 @@ private void reviveDirectRecoveryDriverIfNeeded() { : SenderPool::createStartupRecoveryThread; Thread t = factory.newThread(this::runStartupRecoveryLoop); t.setDaemon(true); + recoveryDriverRunning = true; revivedStartupRecoveryThread = t; t.start(); } catch (Throwable e) { - revivedStartupRecoveryThread = null; + recoveryDriverRunning = false; LOG.warn("could not revive the startup recovery driver ({}); the released slot's " + "data stays durable on disk for a later attempt or restart", e.toString()); } From c0e7fc85becb2f10d6e03cc0631b573832c011ab Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 19:34:53 +0100 Subject: [PATCH 59/64] fix(qwp): save win32 errno at the failure point in the open_file siblings Completes the errno-capture pattern 958a36d4 established for open_file/ fsyncDir0: seven sites across five natives -- length0 (CreateFileW and GetFileSizeEx failures), mkdir0, remove0, rename0, and findFirst0 (both the utf8_to_wide and FindFirstFileW failures) -- called SaveLastError() only AFTER free()/CloseHandle(), either of which may overwrite the thread's last-error value (HeapFree can SetLastError). The saved errno for a failed operation could therefore be an unrelated code or 0. Inert to behavior: repo-wide, nothing branches on the saved value for these operations -- fsyncDir0 was the sole control-flow consumer and already reads its own capture via GetSavedLastError() -- so the exposure was wrong [errno=N] diagnostics in Windows failure messages. Save now happens at the failure point; cleanup follows on every path unchanged. Windows-only TU: verified by mechanical review (order-only transform, brace/paren balance identical, statement-level clobber audit clean); compile coverage via the native build pipeline. --- core/src/main/c/windows/files.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index e9770e9d..e2b97af7 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -360,18 +360,24 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Files_length0 OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - free(wide); if (h == INVALID_HANDLE_VALUE) { + /* Save the CreateFileW failure code BEFORE free(): any subsequent + * API/CRT call (HeapFree can SetLastError) may overwrite the + * thread's last-error value -- same pattern as open_file(). */ SaveLastError(); + free(wide); return -1; } + free(wide); LARGE_INTEGER sz; BOOL ok = GetFileSizeEx(h, &sz); - CloseHandle(h); if (!ok) { + /* Save before CloseHandle() can clobber the last-error value. */ SaveLastError(); + CloseHandle(h); return -1; } + CloseHandle(h); return (jlong) sz.QuadPart; } @@ -414,11 +420,13 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 return -1; } BOOL ok = CreateDirectoryW(wide, NULL); - free(wide); if (!ok) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(wide); return -1; } + free(wide); return 0; } @@ -447,11 +455,13 @@ JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_remove0 } else { ok = DeleteFileW(wide); } - free(wide); if (!ok) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(wide); return JNI_FALSE; } + free(wide); return JNI_TRUE; } @@ -469,12 +479,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_rename0 return -1; } BOOL ok = MoveFileExW(oldW, newW, MOVEFILE_REPLACE_EXISTING); - free(oldW); - free(newW); if (!ok) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(oldW); + free(newW); return -1; } + free(oldW); + free(newW); return 0; } @@ -512,11 +525,13 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Files_findFirst0 pattern[pathLen] = '\0'; wchar_t *wide = utf8_to_wide(pattern); - free(pattern); if (!wide) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(pattern); return 0; } + free(pattern); qdb_find_t *find = (qdb_find_t *) malloc(sizeof(qdb_find_t)); if (!find) { @@ -524,12 +539,14 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Files_findFirst0 return 0; } find->handle = FindFirstFileW(wide, &find->data); - free(wide); if (find->handle == INVALID_HANDLE_VALUE) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(wide); free(find); return 0; } + free(wide); find->hasEntry = 1; win_findname_to_utf8(find); return (jlong) (uintptr_t) find; From 36a61177c5e56a016b83acd8e521ed435bf98a28 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 22:13:55 +0100 Subject: [PATCH 60/64] test(qwp): pin the orphan drainer's inherited periodic sync interval end to end The build -> startOrphanDrainers -> BackgroundDrainer -> CursorSendEngine inheritance was verified-correct but mutation-invisible: passing 0 anywhere along the chain would drain a PERIODIC slot without checkpoints and still pass the suite (the only startOrphanDrainers test call sites used the 4-arg overload, which itself hardcodes 0L). The new integration test adopts a fabricated orphan under sf_durability=periodic against a never-acking server -- the drain can never finish, so the drainer and its engine stay deterministically observable in the pool snapshot -- and asserts the configured interval at the foreground engine, the drainer, and the drainer's engine, failing on value rather than timeout. Seams: @TestOnly drainer-pool accessor on the sender plus engine capture and inherited-interval getter on the drainer. --- .../qwp/client/QwpWebSocketSender.java | 9 ++ .../client/sf/cursor/BackgroundDrainer.java | 26 +++++ .../client/sf/OrphanScanIntegrationTest.java | 104 ++++++++++++++++++ 3 files changed, 139 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 5a813c83..92de0565 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1729,6 +1729,15 @@ public CursorSendEngine getCursorEngineForTesting() { return cursorEngine; } + /** + * Background orphan-drainer pool, or {@code null} when + * {@code drain_orphans} is off or no orphan slot was adopted. + */ + @TestOnly + public BackgroundDrainerPool getDrainerPoolForTesting() { + return drainerPool; + } + @TestOnly public SenderErrorDispatcher getErrorDispatcherForTesting() { return errorDispatcher; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index c83f9f93..6f6d7eff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -112,6 +112,12 @@ public final class BackgroundDrainer implements Runnable { private final long syncIntervalNanos; /** Latest known {@code engine.ackedFsn()}; published for visibility. */ private volatile long ackedFsn = -1L; + /** + * Engine constructed by {@link #run()}, captured for test observation + * only (e.g. asserting the inherited periodic sync interval). May + * reference an already-closed engine once the drain ends. + */ + private volatile CursorSendEngine engineForTesting; private volatile String lastErrorMessage; /** * Optional observer for durable-ack-unavailable transients and the @@ -517,6 +523,25 @@ public boolean isStopRequested() { return stopRequested; } + /** + * Engine this drainer constructed, or {@code null} until {@link #run()} + * gets past engine construction. The reference outlives the drain, so + * tests can read construction-time state (it may be closed by then). + */ + @TestOnly + public CursorSendEngine getEngineForTesting() { + return engineForTesting; + } + + /** + * Periodic SF checkpoint interval this drainer inherited from the + * adopting sender at construction time. + */ + @TestOnly + public long getSyncIntervalNanosForTesting() { + return syncIntervalNanos; + } + /** * Stop check for the runner thread's park loops that also folds a * pending thread interrupt into the stop protocol. The pool delivers @@ -615,6 +640,7 @@ public void run() { outcome = DrainOutcome.FAILED; return; } + engineForTesting = engine; long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/OrphanScanIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/OrphanScanIntegrationTest.java index eccc030b..d146ad8d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/OrphanScanIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/OrphanScanIntegrationTest.java @@ -25,6 +25,9 @@ package io.questdb.client.test.cutlass.qwp.client.sf; import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerPool; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.std.Files; import io.questdb.client.std.ObjList; @@ -148,6 +151,107 @@ public void testScanFindsOrphanFromPriorSenderUnderSameGroupRoot() throws Except }); } + /** + * The orphan drainer must inherit the adopting sender's PERIODIC + * store-and-forward checkpoint interval end to end: + * {@code Sender.build → startOrphanDrainers → BackgroundDrainer → + * CursorSendEngine}. A {@code 0} anywhere along that chain would make + * the drainer silently replay a PERIODIC slot without periodic + * checkpoints — no error, no log — which is exactly the mutation this + * test exists to catch. + */ + @Test + public void testOrphanDrainerInheritsForegroundPeriodicSyncInterval() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Phase 1: fabricate an orphan slot holding unacked backlog + // (same recipe as + // testScanFindsOrphanFromPriorSenderUnderSameGroupRoot). + SilentHandler ghostSilent = new SilentHandler(); + try (TestWebSocketServer ghostServer = new TestWebSocketServer(ghostSilent)) { + ghostServer.start(); + Assert.assertTrue(ghostServer.awaitStart(5, TimeUnit.SECONDS)); + + String ghostCfg = "ws::addr=localhost:" + ghostServer.getPort() + + ";sf_dir=" + sfDir + ";sender_id=ghost;close_flush_timeout_millis=0;"; + try (Sender ghost = Sender.fromConfig(ghostCfg)) { + ghost.table("foo").longColumn("v", 7L).atNow(); + ghost.flush(); + Assert.assertTrue("ghost frame must reach the wire before close", + ghostSilent.awaitFrame(5, TimeUnit.SECONDS)); + } + } + Assert.assertEquals("ghost slot must be a candidate orphan", + 1, OrphanScanner.scan(sfDir, "primary").size()); + + // Phase 2: adopt with a PERIODIC-durability sender. The server + // never acks, so the adopted slot can never finish draining — + // the drainer (and its engine) stay pinned in the pool snapshot + // while we assert the inheritance chain. The asserts below fail + // on the VALUE, not on a timeout, when inheritance breaks. + try (TestWebSocketServer primaryServer = new TestWebSocketServer(new SilentHandler())) { + primaryServer.start(); + Assert.assertTrue(primaryServer.awaitStart(5, TimeUnit.SECONDS)); + + String primaryCfg = "ws::addr=localhost:" + primaryServer.getPort() + + ";sf_dir=" + sfDir + + ";sender_id=primary" + + ";sf_durability=periodic" + + ";sf_sync_interval_millis=123" + + ";drain_orphans=true;"; + try (Sender primary = Sender.fromConfig(primaryCfg)) { + QwpWebSocketSender ws = (QwpWebSocketSender) primary; + // Anchor: the foreground engine carries the configured + // interval (already pinned by SfFromConfigTest; repeated + // here so both ends of the inheritance are asserted on + // the same sender instance). + Assert.assertEquals(123_000_000L, + ws.getCursorEngineForTesting().getSyncIntervalNanosForTesting()); + + BackgroundDrainerPool pool = ws.getDrainerPoolForTesting(); + Assert.assertNotNull("a candidate orphan under drain_orphans=true " + + "must have built the drainer pool", pool); + + // The drainer constructs its engine on its own thread — + // await it. Once seen it cannot leave the snapshot: with + // no acks the drain never completes. + BackgroundDrainer drainer = null; + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadlineNanos) { + ObjList snap = pool.snapshot(); + if (snap.size() > 0 && snap.get(0).getEngineForTesting() != null) { + drainer = snap.get(0); + break; + } + Thread.sleep(10); + } + Assert.assertNotNull( + "orphan drainer must construct its engine within the deadline", + drainer); + Assert.assertEquals("exactly one orphan slot was fabricated", + 1, pool.snapshot().size()); + + // The carried finding: sender → drainer leg. + Assert.assertEquals( + "drainer must inherit the adopting sender's " + + "sf_sync_interval_millis", + 123_000_000L, drainer.getSyncIntervalNanosForTesting()); + // Drainer → engine leg: the interval actually reached the + // engine that performs the periodic checkpoints. + Assert.assertEquals( + "drainer's engine must inherit the adopting sender's " + + "sf_sync_interval_millis", + 123_000_000L, + drainer.getEngineForTesting().getSyncIntervalNanosForTesting()); + + // Fast teardown: the drainer can never finish against the + // silent server — stop it now so close() spends the + // graceful-drain window on unwinding, not waiting. + drainer.requestStop(); + } + } + }); + } + @Test public void testFailedSentinelHidesOrphanFromScan() throws Exception { TestUtils.assertMemoryLeak(() -> { From 6f8c7012d84feeee5cd2fe578fe434d188ab61ce Mon Sep 17 00:00:00 2001 From: bluestreak Date: Wed, 22 Jul 2026 22:14:01 +0100 Subject: [PATCH 61/64] test(qwp): prove legacy migration silently sanitizes proven-dead sealed residue Deleting sanitizeSealedResidue(chain, false) in the no-manifest migration branch passed all 395 SF/recovery tests: every legacy-migration test migrated a clean chain (the sanitize ran as a no-op), and the existing residue test poisons the sealed suffix only after the first recovery has created the manifest, exercising exclusively the fail-closed (chain, true) pass. The new black-box twin plants the pre-fix-client junk in the sealed suffix gap BEFORE the first recovery ever runs and pins the one-pass silent heal: no first-sight throw, residue durably zeroed on disk, frames untouched, manifest created, and a clean restart -- no spurious SfSanitizedResidueException deferred to production's next start. --- .../qwp/client/sf/cursor/SegmentRingTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index dffc1b7d..d2aa903f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -543,6 +543,86 @@ public void testProvenDeadSealedResidueSanitizedThenHealsAfterOneRestart() throw }); } + /** + * The legacy-migration twin of + * {@link #testProvenDeadSealedResidueSanitizedThenHealsAfterOneRestart}: + * the SAME proven-dead sealed residue, but already on disk BEFORE the + * first recovery ever runs — a pre-manifest slot written by a pre-fix + * client. Legacy slots predate the fail-closed sealed-suffix contract, + * so migration must zero the residue SILENTLY (no first-sight throw) + * and hand the manifest era a chain that already satisfies the + * all-zero sealed-suffix invariant — the next restart comes up clean + * instead of tripping a spurious {@link SfSanitizedResidueException} + * over bytes migration should have healed. + */ + @Test + public void testLegacyMigrationSilentlySanitizesProvenDeadSealedResidue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // sealed suffix gap that can never fit a frame + String m0Path = tmpDir + "/q0.sfa"; + String m1Path = tmpDir + "/q1.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 6); + MmapSegment s0 = MmapSegment.create(m0Path, 0, segSize); + for (int i = 0; i < 4; i++) s0.tryAppend(buf, 16); + long gapStart = s0.publishedOffset(); + s0.close(); + MmapSegment s1 = MmapSegment.create(m1Path, 4, segSize); + s1.tryAppend(buf, 16); + s1.close(); + // Poison the sealed suffix gap BEFORE any recovery: no + // manifest exists yet, so this is a legacy slot carrying + // pre-fix-client residue into migration. + int fd = Files.openRW(m0Path); + assertTrue("openRW must succeed", fd >= 0); + long junk = Unsafe.malloc(12, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 3; i++) { + Unsafe.getUnsafe().putInt(junk + i * 4L, 0xCAFEBABE); + } + assertEquals(12L, Files.write(fd, junk, 12, gapStart)); + Files.fsync(fd); + } finally { + Unsafe.free(junk, 12, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + // Migration: contiguity proves the residue dead, so the + // legacy chain recovers in ONE pass — silently. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull("legacy migration must not fail closed over " + + "proven-dead sealed residue", ring); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(4, ring.getSealedSegments().get(0).frameCount()); + assertEquals(4, ring.getActive().baseSeq()); + } + // The silent heal: residue durably zeroed, frames intact, + // slot migrated to the manifest era. + try (MmapSegment seg = MmapSegment.openExisting(m0Path)) { + assertEquals("frames must be untouched", 4L, seg.frameCount()); + assertEquals("legacy migration must durably zero the dead residue", + 0L, seg.tornTailBytes()); + } + assertTrue("legacy migration must create the manifest", + Files.exists(tmpDir + "/sf-manifest.bin")); + // The manifest-era invariant holds: the restart's fail-closed + // sealed-suffix pass finds nothing to surface. A + // SfSanitizedResidueException here means migration skipped + // the sanitize and deferred the incident to production's + // next restart. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull("restart over the migrated chain must come up clean", ring); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(5, ring.nextSeqHint()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testRingRecoverySanitizesResumedActiveTornTail() throws Exception { TestUtils.assertMemoryLeak(() -> { From c5297f7a3e171e66b31cf95216349fa5131edf7d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Thu, 23 Jul 2026 02:21:48 +0100 Subject: [PATCH 62/64] refactor(qwp): drive direct-pool SF recovery from one long-lived background thread Direct SF pools no longer run inline, construction-time recovery, and no longer revive a per-attempt recovery driver. Each direct pool now owns a single long-lived background recovery thread: it scans retired/stranded slots off the construction path, parks indefinitely when the scan is complete, and re-parks with a bounded wait between failed attempts. It exits only on close or interrupt. Release callbacks no longer create or revive threads. They re-arm the scan cursor under the pool lock and then LockSupport.unpark the single driver; the retained-permit ordering closes the callback-before-park lost-wakeup window. This removes the double-driver race and retires the transactional revival token (recoveryDriverRunning, releaseRecoveryDriverOwnership, reviveDirectRecoveryDriverIfNeeded, revivedStartupRecoveryThread, startupRecoverySignal) in favor of one scan with one owner. Construction no longer blocks on replay. The clean-release re-arm fix is preserved: reclaimSlot() calls rearmRecoveryScanIfStranded() so a clean flock release that frees a dir still holding stranded data re-arms and unparks the scan. Deferred pools own no private thread and remain PoolHousekeeper-driven; unpark(null) there is a harmless no-op. Shutdown unparks and joins only the single driver and never blocks on delivery; durable leftovers wait for restart. Java 8 stays green via Compat.onSpinWait, and the test-only manual drive helpers now refuse direct pools to keep the one-owner invariant. --- .../io/questdb/client/impl/SenderPool.java | 414 +++++++----------- .../test/impl/SenderPoolSfFuzzTest.java | 28 +- .../client/test/impl/SenderPoolSfTest.java | 390 +++++++++++++---- 3 files changed, 493 insertions(+), 339 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index d49a24e1..2d01059e 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -45,6 +45,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.ReentrantLock; import java.util.function.IntFunction; @@ -81,6 +82,11 @@ * to the free set once its delegate has released the {@code flock}, tracked * via {@code closingSlots} so a concurrent borrow can never reclaim a slot * dir whose lock is still held. + *

      + * Recovery of managed SF slots is asynchronous: a direct pool owns one + * long-lived background recovery thread, while a pooled {@code QuestDB} + * handle delegates the same serial scan to its {@link PoolHousekeeper}. + * Construction never performs network recovery inline. */ public final class SenderPool implements AutoCloseable { @@ -185,32 +191,14 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; - // Direct pools run one inline recovery attempt for backwards-compatible - // startup behavior, then this daemon continues any backlog left by budget - // exhaustion or a transient failure. Deferred pools use PoolHousekeeper and - // leave this null. - private final Object startupRecoverySignal = new Object(); + // Direct SF pools own one long-lived private recovery driver. It starts + // after construction state is initialized, scans only in the background, + // and parks when recovery completes; a later release re-arms the same scan + // and unparks this same thread. Deferred pools use PoolHousekeeper and + // leave this null. Joined by stopStartupRecoveryDriver() on close. private final Thread startupRecoveryThread; - // Revived direct-pool recovery driver: spawned by recoverRetiredSlotAt() - // when a retired slot's late flock release re-arms an already-latched - // scan on a pool whose original private driver has exited. Deferred - // pools re-arm through the PoolHousekeeper tick instead and never spawn - // one. Joined by stopStartupRecoveryDriver() on close. Volatile: written - // under `lock`, read by close() without it. - private volatile Thread revivedStartupRecoveryThread; - // Ownership token for the private (direct-pool) recovery driver duty. - // Guarded by `lock`. TRUE while some thread owns runStartupRecoveryLoop; - // the loop drops it transactionally on exit (re-checking the latch under - // the lock first -- see releaseRecoveryDriverOwnership), and - // reviveDirectRecoveryDriverIfNeeded() spawns a successor only when it - // is FALSE. Replaces a Thread.isAlive() heuristic whose read raced the - // driver's final latch check: a driver observed "alive" could already - // have committed to exit, leaving an un-latched scan ownerless until the - // next release event or a restart. - private boolean recoveryDriverRunning; - // Driver policy captured for the re-arm path: deferred pools are driven - // by PoolHousekeeper (never spawn private drivers); direct pools own - // their (revivable) private driver, built via recoveryThreadFactory. + // Driver policy: deferred pools are driven by PoolHousekeeper (never spawn + // a private driver); direct SF pools own the single thread above. private final boolean deferStartupRecovery; private final ThreadFactory recoveryThreadFactory; // Test-only constructor seam for the failed-retry operation. Production @@ -286,16 +274,14 @@ public final class SenderPool implements AutoCloseable { // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a // concurrent borrow can neither over-allocate past maxSize nor target a - // dir being recovered. The inline constructor drive is single-threaded, - // but the continuing direct-pool driver and deferred housekeeper driver can - // overlap borrow()/return after publication. Guarded by lock; only ever - // ticks for SF slots. + // dir being recovered. Both the direct-pool background driver and deferred + // housekeeper driver can overlap borrow()/return after publication. + // Guarded by lock; only ever ticks for SF slots. private int recoveringSlots; - // Resumable startup-recovery scan cursor. Advanced only by the single - // recovery driver -- the inline constructor loop followed by its private - // direct-pool thread, or the PoolHousekeeper thread (the sole deferred - // driver) -- - // so the cursor itself needs no lock; the per-slot reservation it performs + // Resumable startup-recovery scan cursor. Advanced only by the pool's + // single recovery driver -- the direct pool's long-lived private thread or + // the PoolHousekeeper thread (the sole deferred driver) -- so the cursor + // itself needs no lock; the per-slot reservation it performs // (slotInUse/recoveringSlots) is still taken under `lock` because borrow() // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) // for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily @@ -314,11 +300,11 @@ public final class SenderPool implements AutoCloseable { // latch can never strand a retired slot's data while the pool lives. // If the retire lands only AFTER the scan latched (a runtime // discardBroken/reapIdle reclaim), the late flock release re-arms the - // scan instead: recoverRetiredSlotAt() rewinds the cursors and clears - // recoveryComplete -- volatile, so the un-latch (written under `lock` by - // release callbacks and reprobes) is visible to the unlocked driver - // gates -- and, for direct pools whose private driver already exited, - // revives the driver. Deferred pools re-arm through the housekeeper tick. + // scan instead: recoverRetiredSlotAt() rewinds the cursors, clears + // recoveryComplete and unparks the direct pool's existing driver. The + // volatile un-latch (written under `lock` by release callbacks and + // reprobes) is visible to its unlocked gate; deferred pools observe it on + // the next housekeeper tick. // recoveryFailStreak/-Slot track consecutive failures on one candidate; // recoveryWarnedSlots dedups the per-slot WARNs so an indefinitely // retried slot logs once per failure episode, not once per retry. All of @@ -327,16 +313,18 @@ public final class SenderPool implements AutoCloseable { private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; private volatile boolean recoveryComplete; - // Set by recoverRetiredSlotAt() whenever a released retired slot's dir is - // still a candidate orphan, and consumed (under `lock`) by the scan's - // end-of-cycle latch decision. Closes the mid-cycle window: a retire AND - // its release can both land while the scan is alive but after the cursor - // already passed that index (it was LIVE when walked -- no retired- - // candidate deferral -- and recoveryComplete was still false at release, - // so the post-latch re-arm does not apply). Without this flag such a - // cycle could end with zero deferrals and latch past the freed dir's - // stranded data. Producer and consumer both run under `lock`, so no - // set can slip between the driver's check and its latch write. + // Set by rearmRecoveryScanIfStranded() whenever a freed index leaves a + // dir that is still a candidate orphan while the scan is alive -- reached + // from a retired slot's late flock release (recoverRetiredSlotAt) or from + // reclaimSlot's clean-release arm -- and consumed (under `lock`) by the + // scan's end-of-cycle latch decision. Closes the mid-cycle window: the + // release can land while the scan is alive but after the cursor already + // passed that index (it was LIVE when walked -- no retired-candidate + // deferral -- and recoveryComplete was still false at release, so the + // post-latch re-arm does not apply). Without this flag such a cycle + // could end with zero deferrals and latch past the freed dir's stranded + // data. Producer and consumer both run under `lock`, so no set can slip + // between the driver's check and its latch write. private volatile boolean recoveryRearmRequested; private int recoveryDeferredThisCycle; private int recoveryFailStreak; @@ -358,9 +346,9 @@ public SenderPool( // Test-only constructor exposing the senderFactory seam: production builds // via the full constructor below (senderFactory null -> the real // defaultSender()). White-box tests inject a factory that throws a - // non-RuntimeException Throwable mid-prewarm. Recovery runs inline here - // (deferStartupRecovery=false); the pooled QuestDB handle uses the 8-arg - // overload to defer it to the housekeeper thread. + // non-RuntimeException Throwable mid-prewarm. A direct SF pool owns one + // background recovery thread; the pooled QuestDB handle uses the 8-arg + // overload to defer recovery to the housekeeper thread. @TestOnly public SenderPool( String configurationString, @@ -376,13 +364,12 @@ public SenderPool( } // Test-only constructor adding the deferStartupRecovery toggle. - // deferStartupRecovery=true skips the inline, construction-time SF recovery - // (recoverOneSlotStep) so QuestDB.build() never blocks on a slow or - // reachable-but-not-acking server; the owner (QuestDBImpl) then drives - // recovery one slot per tick on the PoolHousekeeper thread via - // runStartupRecoveryStep(). White-box SF tests call this directly; the - // in-range recovery pass is concurrency-safe against borrow()/return after - // pool publication -- see recoverOneSlotStep(). + // deferStartupRecovery=true leaves the pool without a private recovery + // thread; the owner (QuestDBImpl) drives recovery one slot per tick on the + // PoolHousekeeper thread via runStartupRecoveryStep(). White-box SF tests + // call this directly; the in-range recovery pass is concurrency-safe + // against borrow()/return after pool publication -- see + // recoverOneSlotStep(). @TestOnly public SenderPool( String configurationString, @@ -538,26 +525,18 @@ private SenderPool( closePrewarmedDelegates(built); throw e; } - // Prewarm succeeded. Recover any unacked data a previous run left in - // this pool's own managed slots that prewarm did not already re-adopt. - // The pooled QuestDB handle defers this to the housekeeper thread - // (deferStartupRecovery=true) so QuestDB.build() never blocks on a slow - // or reachable-but-not-acking server; direct constructions run it inline - // here, while still single-threaded and unpublished. + // Prewarm succeeded. Direct SF pools recover exclusively on one + // background thread; deferred pools are driven by PoolHousekeeper. + // Assign the final field before start() so the runnable and every + // callback observe the one published driver reference. Thread recoveryThread = null; try { - if (!deferStartupRecovery) { - runStartupRecoveryToCompletion(); - if (storeAndForward && !recoveryComplete) { - ThreadFactory threadFactory = recoveryThreadFactory != null - ? recoveryThreadFactory - : SenderPool::createStartupRecoveryThread; - recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop); - recoveryThread.setDaemon(true); - // Ownership precedes start(): the loop's transactional - // exit is the only place the token is dropped. - recoveryDriverRunning = true; - } + if (storeAndForward && !deferStartupRecovery) { + ThreadFactory threadFactory = recoveryThreadFactory != null + ? recoveryThreadFactory + : SenderPool::createStartupRecoveryThread; + recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop); + recoveryThread.setDaemon(true); } this.startupRecoveryThread = recoveryThread; if (recoveryThread != null) { @@ -579,13 +558,13 @@ private SenderPool( /** * Drives startup SF recovery toward completion in a single call, bounded by * one shared {@code acquireTimeoutMillis} wall-clock budget (and each - * individual drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the - * inline construction path -- single-threaded and unpublished -- and by - * manual / test drives. Budget exhaustion or a transient slot failure leaves - * the cursor pending for the pool's continuing driver. No-op when SF is off, - * the pool is shutting down, or recovery has already finished. Idempotent. + * individual drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used only by + * manual / test drives on pools without a private recovery thread. Budget + * exhaustion or a transient slot failure leaves the cursor pending for a + * later drive. No-op when SF is off, the pool is shutting down, or recovery + * has already finished. Idempotent. */ - void runStartupRecoveryToCompletion() { + private void runStartupRecoveryWithinBudget() { if (!storeAndForward) { return; } @@ -622,64 +601,29 @@ void runStartupRecoveryToCompletion() { } private void runStartupRecoveryLoop() { - while (true) { - if (closed || Thread.currentThread().isInterrupted()) { - // Unconditional exits: still drop ownership so a later re-arm - // on a live pool can spawn a successor (the revive gate - // ignores a closing pool anyway). - releaseRecoveryDriverOwnership(false); - return; - } - if (recoveryComplete) { - // Transactional exit: re-check the latch under `lock` so an - // un-latch (recoverRetiredSlotAt's re-arm) cannot slip - // between this thread's decision to exit and its ownership - // drop -- the lost-wakeup race a Thread.isAlive() probe had. - if (releaseRecoveryDriverOwnership(true)) { - return; + while (!closed && !Thread.currentThread().isInterrupted()) { + boolean hasImmediateWork = false; + if (!recoveryComplete) { + try { + hasImmediateWork = runStartupRecoveryStep(); + } catch (Throwable ignored) { + // Match PoolHousekeeper: startup recovery is best-effort + // and a delegate Error must not kill its only driver. } - // An un-latch landed first: this thread stays the driver. - continue; } - boolean hasImmediateWork; - try { - hasImmediateWork = runStartupRecoveryStep(); - } catch (Throwable ignored) { - // Match PoolHousekeeper: startup recovery is best-effort and a - // delegate Error must not permanently kill its only driver. - hasImmediateWork = false; + if (closed || Thread.currentThread().isInterrupted()) { + return; } - if (!hasImmediateWork && !closed && !recoveryComplete) { + if (!hasImmediateWork) { + // Remain the sole owner for the pool's lifetime. The default + // waiter parks indefinitely after completion and for a bounded + // retry interval after failure. A release callback supplies an + // unpark permit, which survives a callback-before-park race. startupRecoveryWaiter.run(); } } } - /** - * Drops this thread's private-driver ownership token, transactionally - * against the re-arm path. With {@code recheckLatch}, an un-latch that - * raced this driver's exit decision is detected under {@code lock} and - * ownership is KEPT ({@code false} is returned; the caller keeps - * driving). The producer ({@link #recoverRetiredSlotAt}) clears the - * latch and reads {@code recoveryDriverRunning} under the same lock, so - * exactly one of the two parties always owns the follow-up: either this - * driver observes the cleared latch, or the producer observes the - * dropped token and spawns a successor. No interleaving leaves an - * un-latched scan ownerless. - */ - private boolean releaseRecoveryDriverOwnership(boolean recheckLatch) { - lock.lock(); - try { - if (recheckLatch && !closed && !recoveryComplete) { - return false; - } - recoveryDriverRunning = false; - return true; - } finally { - lock.unlock(); - } - } - /** * Recovers at most ONE stranded managed slot and reports whether more remain. * The private direct driver or {@link PoolHousekeeper} drives steps @@ -994,13 +938,14 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryOutOfRangeNext = 0; return false; } - // Latch decision under `lock`: a mid-cycle flock release can free a - // still-candidate dir AFTER this cycle's cursor passed its index (it - // was live/retired at walk time -- no deferral) and BEFORE the latch - // (so the post-latch re-arm in recoverRetiredSlotAt does not apply - // either). recoverRetiredSlotAt raises recoveryRearmRequested under - // the same lock, so consuming it here mutually excludes the race: no - // release can slip between this check and the latch write. + // Latch decision under `lock`: a mid-cycle release (a retired slot's + // late flock release or a clean reclaim) can free a still-candidate + // dir AFTER this cycle's cursor passed its index (it was live/retired + // at walk time -- no deferral) and BEFORE the latch (so the + // post-latch re-arm in rearmRecoveryScanIfStranded does not apply + // either). rearmRecoveryScanIfStranded raises recoveryRearmRequested + // under the same lock, so consuming it here mutually excludes the + // race: no release can slip between this check and the latch write. lock.lock(); try { if (recoveryRearmRequested) { @@ -1440,12 +1385,18 @@ public void markClosingForTesting() { } @TestOnly - public void runStartupRecoveryToCompletionForTesting() { - runStartupRecoveryToCompletion(); + public void runStartupRecoveryWithinBudgetForTesting() { + if (startupRecoveryThread != null) { + throw new IllegalStateException("manual recovery drive requires a deferred pool"); + } + runStartupRecoveryWithinBudget(); } @TestOnly public boolean runStartupRecoveryStepForTesting() { + if (startupRecoveryThread != null) { + throw new IllegalStateException("manual recovery drive requires a deferred pool"); + } return runStartupRecoveryStep(); } @@ -1473,9 +1424,9 @@ public void setBorrowWaitExpiredHook(Runnable hook) { */ void markClosing() { closed = true; - synchronized (startupRecoverySignal) { - startupRecoverySignal.notifyAll(); - } + // LockSupport retains a permit when shutdown races the driver's park, + // so the one direct recovery thread cannot miss its close wake-up. + LockSupport.unpark(startupRecoveryThread); } /** @@ -1502,7 +1453,7 @@ void markClosing() { */ @Override public void close() { - // Direct pools own their continuing recovery driver. Stop it before + // Direct pools own one long-lived recovery driver. Stop it before // snapshotting/closing delegates; deferred pools have a null thread and // QuestDBImpl stops their external PoolHousekeeper before calling here. markClosing(); @@ -1697,30 +1648,18 @@ private void stopStartupRecoveryDriver() { afterStartupRecoveryJoinHook.run(); } } - // A revived driver (late-release re-arm on a direct pool; see - // reviveDirectRecoveryDriverIfNeeded) is joined with the same bound: - // `closed` is already raised, so its loop exits on the next waiter - // wake-up, within RECOVERY_RETRY_INTERVAL_MILLIS < STOP_TIMEOUT. - Thread revived = revivedStartupRecoveryThread; - if (revived != null && revived != Thread.currentThread()) { - try { - revived.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } } private void waitForStartupRecoveryRetry() { - synchronized (startupRecoverySignal) { - if (closed || recoveryComplete) { - return; - } - try { - startupRecoverySignal.wait(RECOVERY_RETRY_INTERVAL_MILLIS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + if (closed) { + return; + } + if (recoveryComplete) { + LockSupport.park(this); + } else { + LockSupport.parkNanos( + this, + TimeUnit.MILLISECONDS.toNanos(RECOVERY_RETRY_INTERVAL_MILLIS)); } } @@ -2131,10 +2070,12 @@ private static boolean flockReleased(SenderSlot s) { * capacity and no borrow reuses the still-locked dir unless * {@link #reprobeRetiredSlots} later observes the deferred cleanup's * release and recovers the index. Either way {@code closingSlots} is - * decremented. A later release that frees a dir still holding unacked - * data also re-arms the startup recovery scan (see - * {@link #recoverRetiredSlotAt}) so the data is drained in-process - * instead of waiting for a restart or a lucky borrow of that index. + * decremented. Any release that frees a dir still holding unacked data + * -- the immediate clean release here or a retired slot's later flock + * release -- also re-arms the startup recovery scan (see + * {@link #rearmRecoveryScanIfStranded}) so the data is drained + * in-process instead of waiting for a restart or a lucky borrow of that + * index. *

      * Caller must hold {@code lock} and is responsible for signalling waiters * (only the free path admits a new creation). Shared by @@ -2149,6 +2090,13 @@ private boolean reclaimSlot(SenderSlot s, String context) { closingSlots--; if (flockReleased(s)) { freeSlotIndex(s.slotIndex()); + // A clean release can free a dir that still holds unacked durable + // data: discardBroken of a sender whose server went away tears + // down its workers fine and drops the flock, but the rows stay on + // disk and no borrow may ever land on this index again. Same + // stranding hazard as a retired slot's late release -- probe and + // re-arm the scan on this path too. + rearmRecoveryScanIfStranded(s.slotIndex()); return true; } leakedSlots++; @@ -2254,86 +2202,62 @@ private void recoverRetiredSlotAt(int retiredIndex) { s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); // The freed dir may still hold unacked durable data: a runtime retire // (discardBroken/reapIdle) can land AFTER the startup scan latched - // recoveryComplete, and close() never drains an unowned dir, so - // without re-arming the scan that data would wait for a restart or a - // lucky borrow of exactly this index. When the latch is already set, - // rewind the cursors FIRST, then clear the volatile latch (the write - // order publishes the rewound cursors to the next driver's unlocked - // gate read), then revive the private driver when this is a direct - // pool whose original driver already exited; recoveryComplete==true - // here also guarantees no driver is mid-step, so touching the - // driver-owned cursors is safe. When the scan is still ALIVE, only - // raise recoveryRearmRequested: the cursor may already have passed - // this index while it was live/retired, and the flag makes the - // end-of-cycle latch decision rewind instead -- the cursors stay - // strictly driver-owned. The isCandidateOrphan probe is a bounded - // directory scan; holding the pool lock across it mirrors - // reprobeRetiredSlots()' delegate probes. The pass-2 work list is - // kept and only its cursor rewound, exactly like the end-of-scan - // cycle rewind. - if (sfDir != null - && OrphanScanner.isCandidateOrphan(sfDir + "/" + slotBaseId + "-" + s.slotIndex())) { - if (recoveryComplete) { - recoveryInRangeNext = 0; - recoveryOutOfRangeNext = 0; - recoveryDeferredThisCycle = 0; - recoveryRearmRequested = false; - recoveryComplete = false; - LOG.info("SF slot {} released with stranded data still on disk; re-arming the " - + "startup recovery scan to drain it", s.slotIndex()); - reviveDirectRecoveryDriverIfNeeded(); - } else { - recoveryRearmRequested = true; - LOG.info("SF slot {} released with stranded data while the recovery scan is " - + "mid-cycle; flagging a rewind so this cycle cannot latch past it", - s.slotIndex()); - } - } + // recoveryComplete, and close() never drains an unowned dir. Shared + // re-arm tail with reclaimSlot's clean-release path. + rearmRecoveryScanIfStranded(s.slotIndex()); } /** - * Re-arms a driver for an un-latched scan (see {@link #recoverRetiredSlotAt}). - * Deferred pools need nothing: {@code PoolHousekeeper} calls - * {@link #runStartupRecoveryStep()} on every tick and its gate re-opens - * with the cleared latch. A direct pool's private driver exits once the - * scan completes, so when no thread currently owns the driver duty - * ({@code recoveryDriverRunning} false), spawn a successor running the - * same loop -- it drains the backlog, re-latches, and exits; close() - * joins it via {@link #stopStartupRecoveryDriver}. Ownership is a - * lock-guarded token, not a {@code Thread.isAlive()} probe: the loop - * drops it transactionally ({@link #releaseRecoveryDriverOwnership} - * re-checks the latch under this same lock first), so a driver that is - * mid-exit either observes this caller's un-latch and keeps driving, or - * has already dropped the token and a successor is spawned here -- no - * interleaving leaves the un-latched scan ownerless. Caller must hold - * {@code lock}; {@code Thread.start()} is cheap enough to run under it. - * Best-effort: a spawn failure is logged and never propagates into the - * release callback that triggered the re-arm (the data stays durable - * for a restart). + * Re-arms the startup recovery scan when a just-freed slot index left a + * dir that is still a candidate orphan (unacked durable data, no failure + * sentinel). Shared tail of every path that returns an SF index to the + * free set: the retired slot's late flock release + * ({@link #recoverRetiredSlotAt}) and the immediate clean release + * ({@link #reclaimSlot}'s freed arm). Without it the freed dir is + * unowned -- the scan is latched, close() never drains an unowned dir -- + * so the data would wait for a restart or a lucky borrow of exactly this + * index, which steady low load may never produce. + *

      + * When the latch is already set, rewind the cursors FIRST, then clear + * the volatile latch (the write order publishes the rewound cursors to + * the direct driver's unlocked gate read). Since the sole direct driver + * remains parked for the pool's lifetime, the callback only has to unpark + * it; no thread creation or ownership handoff occurs. When the scan is + * still ALIVE, only raise {@code recoveryRearmRequested}: the cursor may + * already have passed this index while it was live/retired, and the flag + * makes the end-of-cycle latch decision rewind instead -- the cursors + * stay strictly driver-owned. The pass-2 work list is kept and only its + * cursor rewound, exactly like the end-of-scan cycle rewind. + *

      + * No-op on a closed pool: teardown must not flip the latch; data at rest + * on disk is the designed safe state and the next process's startup scan + * delivers it. The isCandidateOrphan probe is a bounded directory scan; + * holding the pool lock across it mirrors {@link #reprobeRetiredSlots}' + * delegate probes. Caller must hold {@code lock}. */ - private void reviveDirectRecoveryDriverIfNeeded() { - if (deferStartupRecovery || closed) { - return; - } - if (recoveryDriverRunning) { - // A live driver's exit re-checks the latch under this lock, so - // it is guaranteed to observe the un-latch that brought us here. + private void rearmRecoveryScanIfStranded(int slotIdx) { + if (closed + || sfDir == null + || !OrphanScanner.isCandidateOrphan(sfDir + "/" + slotBaseId + "-" + slotIdx)) { return; } - try { - ThreadFactory factory = recoveryThreadFactory != null - ? recoveryThreadFactory - : SenderPool::createStartupRecoveryThread; - Thread t = factory.newThread(this::runStartupRecoveryLoop); - t.setDaemon(true); - recoveryDriverRunning = true; - revivedStartupRecoveryThread = t; - t.start(); - } catch (Throwable e) { - recoveryDriverRunning = false; - LOG.warn("could not revive the startup recovery driver ({}); the released slot's " - + "data stays durable on disk for a later attempt or restart", e.toString()); - } + if (recoveryComplete) { + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + recoveryDeferredThisCycle = 0; + recoveryRearmRequested = false; + recoveryComplete = false; + LOG.info("SF slot {} released with stranded data still on disk; re-arming the " + + "startup recovery scan to drain it", slotIdx); + } else { + recoveryRearmRequested = true; + LOG.info("SF slot {} released with stranded data while the recovery scan is " + + "mid-cycle; flagging a rewind so this cycle cannot latch past it", + slotIdx); + } + // The permit is retained if the callback wins the race with park(). + // Deferred pools have no private thread, so unpark(null) is a no-op. + LockSupport.unpark(startupRecoveryThread); } // Outcome of one drainCandidateSlotForRecovery attempt, letting the scan diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java index 3220d80e..345fa2f8 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java @@ -107,8 +107,11 @@ * restored capacity but nothing re-armed the latched scan, so the * data waited for a restart or a lucky borrow of that index.

      3. * - * Random iterations (2+) additionally draw the runtime-wedge op freely, so - * runtime retires also interleave with a still-running scan. + * Random iterations (2+) additionally draw the runtime-fault op freely, and + * each draw discards its borrow either through the wedged close (flock kept, + * slot retired, healed at end of schedule) or through a CLEAN release (workers + * stop, the flock drops, {@code reclaimSlot}'s freed arm must re-arm the scan + * itself), so both discard shapes interleave with a still-running scan. */ public class SenderPoolSfFuzzTest { @@ -370,7 +373,7 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { s.close(); } break; - case 5: { // runtime wedge: retire a live borrow undelivered + case 5: { // runtime fault: discard a live borrow undelivered wedgeNextBorrow[0] = true; PooledSender ps = null; try { @@ -389,9 +392,22 @@ private void runOneIteration(Rnd rnd, int iter) throws Exception { lastWedgeBuild[0] = null; ps.table("fuzz").longColumn("wedged", op).atNow(); ps.flush(); - ((QwpWebSocketSender) delegate).setClosedForTesting(true); - pool.discardBrokenForTesting(ps); - unhealed.add(delegate); + if (rnd.nextBoolean()) { + // Wedged close: the flock is + // kept, the slot retires; + // healed at end of schedule. + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + unhealed.add(delegate); + } else { + // Clean release: close() tears + // down fine, the flock drops, + // rows stay durably unacked. + // reclaimSlot's freed arm must + // re-arm the scan or the + // post-close audit reds. + pool.discardBrokenForTesting(ps); + } } else { // Borrow reused an idle (ack-server) // sender; nothing to wedge. diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index e7b48aa3..76c0f555 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -63,9 +63,11 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; import java.util.function.IntFunction; /** @@ -1606,9 +1608,11 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th // get a fresh slot (default-1), proving capacity dropped by one. SenderPool pool = newPoolWithFactory(cfg2, 0, 2, 500, factory); try { - // The forge must actually have reached recovery's build. - Assert.assertNotNull("recovery must have built slot 0", forged.get()); - // The retire branch must have fired during construction. + Assert.assertTrue( + "background recovery must build and retire the still-locked slot", + awaitCondition( + () -> forged.get() != null && pool.leakedSlotCount() == 1, + 10_000)); Assert.assertEquals("recovery must retire the still-locked slot as leaked", 1, pool.leakedSlotCount()); Assert.assertTrue("retired slot 0 must stay reserved", @@ -1713,7 +1717,11 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except }; try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 500, factory)) { - Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertTrue( + "background recovery must build and retire slot 0", + awaitCondition( + () -> forged.get() != null && pool.leakedSlotCount() == 1, + 10_000)); Assert.assertEquals("precondition: startup recovery must retire the slot", 1, pool.leakedSlotCount()); @@ -1909,7 +1917,7 @@ public void testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease() thr try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { // Phase 1: the scan completes legitimately (clean dirs). - pool.runStartupRecoveryToCompletionForTesting(); + pool.runStartupRecoveryWithinBudgetForTesting(); Assert.assertTrue("precondition: scan must have latched complete", pool.isRecoveryCompleteForTesting()); @@ -1968,17 +1976,93 @@ public void testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease() thr } @Test - public void testRevivedDirectDriverDrainsRuntimeRetiredSlotAfterLateRelease() throws Exception { - // Direct-pool twin of - // testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease: a - // direct pool (deferStartupRecovery=false) finishes its inline scan at - // construction and lets its private recovery driver die. A runtime - // retire after that leaves NO driver to pick up the re-armed scan - // when the flock releases -- the un-latch alone is not enough. Pin - // the revival: the release must respawn the direct pool's private - // driver, which drains the freed dir with NO test-side stepping and - // no borrow. RED (segments never clear) until release revives the - // driver. + public void testDiscardBrokenCleanFlockReleaseRearmsRecoveryScan() throws Exception { + // The mirror of testRuntimeRetireAfterScanCompleteRearmsRecovery- + // OnFlockRelease on the MORE COMMON reclaim shape: discardBroken of + // a sender whose server went silent, but whose close() tears down + // cleanly -- workers stop, the flock drops, reclaimSlot frees the + // index directly (no retire, no release listener). The dir still + // holds durably-unacked rows, the scan is latched, close() never + // drains an unowned dir: pre-fix NOTHING re-drained it and the data + // waited for a restart or a lucky borrow of exactly this index. Pin + // the fix: the freed arm must probe the dir and re-arm the scan + // (un-latch + rewind) just like the retired-slot release does. RED + // at the un-latch assertion until reclaimSlot's freed arm re-arms. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer silentSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + silentSink.start(); + Assert.assertTrue(silentSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + // The broken sender: alive workers streaming into a silent + // sink (rows stay durably unacked); close-flush OFF so its + // clean close() drops the flock WITHOUT delivering. + String cfgBroken = "ws::addr=localhost:" + silentSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] brokenNextBuild = new boolean[1]; + IntFunction factory = idx -> + Sender.builder(brokenNextBuild[0] ? cfgBroken : cfg) + .senderId("default-" + idx).build(); + + try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Phase 1: the scan completes legitimately (clean dirs). + pool.runStartupRecoveryWithinBudgetForTesting(); + Assert.assertTrue("precondition: scan must have latched complete", + pool.isRecoveryCompleteForTesting()); + + // Phase 2: borrow, write durably against the silent sink, + // then discard broken. close() is NOT forged: it tears + // down cleanly and releases the flock, so reclaimSlot + // takes the freed arm -- no retire, no leaked capacity. + brokenNextBuild[0] = true; + PooledSender ps = pool.borrow(); + brokenNextBuild[0] = false; + for (int i = 0; i < 3; i++) { + ps.table("clean").longColumn("v", i).atNow(); + ps.flush(); + } + pool.discardBrokenForTesting(ps); + Assert.assertEquals( + "clean release must take the freed arm, not retire", + 0, pool.leakedSlotCount()); + Assert.assertTrue("freed dir must still hold the stranded data", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse( + "a clean flock release that frees a dir with stranded data must " + + "re-arm the recovery scan -- otherwise the data waits for a " + + "restart or a lucky borrow of that exact index", + pool.isRecoveryCompleteForTesting()); + + // Phase 3: the ordinary driver cadence must now drain the + // dir and re-latch -- no borrow anywhere in this phase. + long deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("re-armed scan must complete once the data is drained", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the re-armed scan must deliver the stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + } + } + }); + } + + @Test + public void testLongLivedDirectDriverDrainsRuntimeRetiredSlotAfterLateRelease() throws Exception { + // A direct SF pool owns exactly one recovery thread for its lifetime. + // It parks after a clean scan, then a late flock release re-arms the + // cursors and unparks that SAME thread -- no constructor-inline drive, + // no replacement thread, no test-side step, reap or borrow. TestUtils.assertMemoryLeak(() -> { CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler); @@ -1996,13 +2080,24 @@ public void testRevivedDirectDriverDrainsRuntimeRetiredSlotAfterLateRelease() th IntFunction factory = idx -> Sender.builder(wedgeNextBuild[0] ? cfgWedge : cfg) .senderId("default-" + idx).build(); + AtomicInteger driverCreations = new AtomicInteger(); + AtomicReference driverRef = new AtomicReference<>(); + ThreadFactory threadFactory = runnable -> { + driverCreations.incrementAndGet(); + Thread driver = new Thread(runnable, "test-long-lived-recovery-driver"); + driverRef.set(driver); + return driver; + }; - // deferStartupRecovery=false: the inline scan over the clean - // dir completes at construction; no private driver survives. - try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, - Long.MAX_VALUE, Long.MAX_VALUE, factory, false)) { - Assert.assertTrue("precondition: inline scan must have latched complete", - pool.isRecoveryCompleteForTesting()); + try (SenderPool pool = newPoolWithRecoveryControls( + cfg, 0, 1, 500, factory, threadFactory, null, null)) { + Thread driver = pool.getStartupRecoveryThreadForTesting(); + Assert.assertSame(driverRef.get(), driver); + Assert.assertTrue("initial background scan must latch complete", + awaitCondition(pool::isRecoveryCompleteForTesting, 10_000)); + Assert.assertTrue("direct recovery driver must park, not exit", driver.isAlive()); + Assert.assertEquals("construction must create exactly one recovery driver", + 1, driverCreations.get()); wedgeNextBuild[0] = true; PooledSender ps = pool.borrow(); @@ -2019,28 +2114,114 @@ public void testRevivedDirectDriverDrainsRuntimeRetiredSlotAfterLateRelease() th Assert.assertTrue("retired dir must hold the stranded data", hasSegmentFile(slot("default-0"))); - // Late release without delivery. From here the pool must - // recover the data ENTIRELY on its own: no recovery-step - // calls, no reapIdle, no borrow -- the revived private - // driver is the only possible deliverer. ((QwpWebSocketSender) delegate).setClosedForTesting(false); delegate.close(); Assert.assertTrue( - "the revived direct driver must drain the freed dir on its own -- " - + "no steps, no reap, no borrow", + "the parked direct driver must drain the freed dir on its own", awaitNoSegmentFile(slot("default-0"), 15_000)); Assert.assertTrue("replayed frames must reach the server", awaitAtLeast(handler.frames, 1, 15_000)); - long deadline = System.currentTimeMillis() + 10_000; - while (!pool.isRecoveryCompleteForTesting() - && System.currentTimeMillis() < deadline) { - Thread.sleep(10); - } - Assert.assertTrue("the revived driver must re-latch completion", - pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the same driver must re-latch completion", + awaitCondition(pool::isRecoveryCompleteForTesting, 10_000)); + Assert.assertSame("re-arm must keep the original driver", + driver, pool.getStartupRecoveryThreadForTesting()); + Assert.assertTrue("the original driver must remain parked and alive", + driver.isAlive()); + Assert.assertEquals("re-arm must not create another recovery driver", + 1, driverCreations.get()); Assert.assertEquals(0, pool.leakedSlotCount()); } + Assert.assertFalse("pool close must stop the one recovery driver", + driverRef.get().isAlive()); + } + }); + } + + @Test + public void testDirectRecoveryRearmBeforeParkKeepsWakePermit() throws Exception { + // Deterministically put the sole driver between its completion check + // and park(), then re-arm first. A notify-style wake could be lost in + // this window; LockSupport retains one permit, so the subsequent park + // returns immediately and the SAME driver drains the new candidate. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer silentSink = new TestWebSocketServer(new SilentHandler())) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + silentSink.start(); + Assert.assertTrue(silentSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + String cfgBroken = "ws::addr=localhost:" + silentSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + AtomicBoolean brokenNextBuild = new AtomicBoolean(); + IntFunction factory = idx -> + Sender.builder(brokenNextBuild.get() ? cfgBroken : cfg) + .senderId("default-" + idx).build(); + AtomicBoolean firstWait = new AtomicBoolean(true); + AtomicBoolean releasePark = new AtomicBoolean(); + AtomicInteger driverCreations = new AtomicInteger(); + CountDownLatch beforePark = new CountDownLatch(1); + Runnable recoveryWaiter = () -> { + if (firstWait.compareAndSet(true, false)) { + beforePark.countDown(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!releasePark.get() + && !Thread.currentThread().isInterrupted() + && System.nanoTime() < deadline) { + io.questdb.client.std.Compat.onSpinWait(); + } + if (!releasePark.get()) { + throw new AssertionError("timed out before completion park"); + } + } + LockSupport.park(); + }; + ThreadFactory threadFactory = runnable -> { + driverCreations.incrementAndGet(); + return new Thread(runnable, "test-rearm-before-park"); + }; + + SenderPool pool = null; + try { + pool = newPoolWithRecoveryControls( + cfg, 0, 1, 500, factory, threadFactory, recoveryWaiter, null); + SenderPool recoveryPool = pool; + Assert.assertTrue("driver must reach the completion-park boundary", + beforePark.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("initial scan must have latched complete", + recoveryPool.isRecoveryCompleteForTesting()); + + brokenNextBuild.set(true); + PooledSender ps = recoveryPool.borrow(); + brokenNextBuild.set(false); + ps.table("wake").longColumn("v", 1).atNow(); + ps.flush(); + recoveryPool.discardBrokenForTesting(ps); + Assert.assertTrue("clean release must leave a candidate on disk", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse("release must re-arm before the driver parks", + recoveryPool.isRecoveryCompleteForTesting()); + + // The callback already called unpark(). Let the driver + // reach park only now; it must consume the retained permit. + releasePark.set(true); + Assert.assertTrue("retained wake permit must drive recovery immediately", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frame must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + Assert.assertTrue("same driver must re-latch completion", + awaitCondition(recoveryPool::isRecoveryCompleteForTesting, 10_000)); + Assert.assertEquals("re-arm must never create a second driver", + 1, driverCreations.get()); + } finally { + releasePark.set(true); + if (pool != null) { + pool.close(); + } + } } }); } @@ -2189,9 +2370,9 @@ public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Except // already dropped and a probe would have restored capacity and admitted // a creation. Deterministic: no housekeeper runs in this test, so // borrow() is the only reprobe driver. Recovery is driven manually via - // the deferred-pool step helper because the inline path reuses - // acquireTimeoutMillis as its recovery budget -- 0 would skip recovery - // outright. This test is RED until the probe is hoisted above the + // a deferred pool so no private background thread can race the borrow + // boundary this test isolates. This test is RED until the probe is + // hoisted above the // timeout check. TestUtils.assertMemoryLeak(() -> { // Phase 1: strand unacked data under default-0 so startup recovery @@ -2326,7 +2507,11 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception }; try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 100, factory)) { - Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertTrue( + "background recovery must build and retire slot 0", + awaitCondition( + () -> forged.get() != null && pool.leakedSlotCount() == 1, + 10_000)); Assert.assertEquals("precondition: startup recovery must retire the slot", 1, pool.leakedSlotCount()); @@ -2856,7 +3041,7 @@ public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exce primary.close(); primaryReturned.set(true); - pool.runStartupRecoveryToCompletionForTesting(); + pool.runStartupRecoveryWithinBudgetForTesting(); Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, recoveryAttempts.get()); @@ -3041,7 +3226,7 @@ public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws E } @Test - public void testDirectRecoveryContinuesAfterFiniteBudgetExhaustion() throws Exception { + public void testDirectBackgroundRecoveryDrainsAllCandidates() throws Exception { createCandidateSlot("default-0"); createCandidateSlot("default-1"); CountDownLatch drained = new CountDownLatch(2); @@ -3053,7 +3238,7 @@ public void testDirectRecoveryContinuesAfterFiniteBudgetExhaustion() throws Exce String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; try (SenderPool pool = newPoolWithFactory(config, 0, 1, 0, factory)) { - Assert.assertTrue("direct pool must continue recovery after its inline budget expires", + Assert.assertTrue("direct background driver must drain every candidate", drained.await(5, TimeUnit.SECONDS)); Assert.assertEquals("in-range candidate must be recovered", 1, attempts[0].get()); Assert.assertEquals("out-of-range candidate must be recovered", 1, attempts[1].get()); @@ -3555,36 +3740,48 @@ public void testStartupRecoveryParksPersistentlyFailingSlotAfterBoundedRetries() } @Test - public void testLongMaxStartupRecoveryBudgetDoesNotOverflow() throws Exception { + public void testDirectRecoveryRunsOnlyInBackground() throws Exception { createCandidateSlot("default-0"); - AtomicInteger attempts = new AtomicInteger(); + CountDownLatch factoryEntered = new CountDownLatch(1); + CountDownLatch releaseFactory = new CountDownLatch(1); + AtomicReference factoryThread = new AtomicReference<>(); IntFunction factory = idx -> { - attempts.incrementAndGet(); + factoryThread.set(Thread.currentThread()); + factoryEntered.countDown(); + try { + if (!releaseFactory.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release recovery factory"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("recovery factory interrupted", e); + } return successfulRecoverySender(idx, new CountDownLatch(0)); }; String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + Thread constructorThread = Thread.currentThread(); try (SenderPool pool = newPoolWithFactory(config, 0, 1, Long.MAX_VALUE, factory)) { - Assert.assertEquals("Long.MAX_VALUE must leave a positive inline recovery budget", - 1, attempts.get()); - Assert.assertTrue("the inline scan must complete", pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("background recovery must enter its sender factory", + factoryEntered.await(10, TimeUnit.SECONDS)); + Assert.assertNotSame("constructor thread must never run startup recovery", + constructorThread, factoryThread.get()); + Assert.assertTrue("direct SF pool must own one background driver", + pool.getStartupRecoveryThreadForTesting().isAlive()); + releaseFactory.countDown(); + Assert.assertTrue("background scan must complete after the factory is released", + awaitCondition(pool::isRecoveryCompleteForTesting, 10_000)); + } finally { + releaseFactory.countDown(); } } @Test - public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception { - // Regression for the startup-recovery budget (M1). - // recoverOneSlotStep() runs synchronously in the SenderPool - // constructor. A previous run can strand unacked data in EVERY in-range - // slot, and if the server is reachable but does not ack, each slot's - // drain blocks for the full acquireTimeoutMillis. Without a shared, - // whole-scan budget (and a short-circuit on the first drain that fails - // to ack) construction blocks for (maxSize - minSize) * - // acquireTimeoutMillis -- here 4 * 1s = 4s -- so QuestDB.build() stalls - // proportionally to the recovery backlog. The fix caps the TOTAL - // recovery at ~one acquireTimeoutMillis and stops scanning the moment a - // drain fails to ack, so construction must return well inside that - // ceiling no matter how many slots are stranded. + public void testDirectStartupRecoveryDoesNotBlockConstruction() throws Exception { + // Direct SF recovery is exclusively background work. A previous run + // can strand unacked data in every managed slot and a reachable server + // can fail to acknowledge every drain, but construction must not pay + // any recovery drain budget: it starts one driver and returns. final long acquireTimeoutMillis = 1_000L; final int maxSize = 4; @@ -3618,11 +3815,9 @@ public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception { } // Phase 2: restart against a STILL-silent (reachable but - // never-acking) server. Construction triggers startup recovery over - // all four stranded slots. close_flush_timeout=0 makes each - // recoverer's close() a fast close, so the measured window isolates - // the drain budget: pre-fix it is maxSize * acquireTimeoutMillis; - // post-fix it is bounded by ~one acquireTimeoutMillis. + // never-acking) server. The private recovery thread may begin + // immediately, but construction must return without waiting for + // its connect/drain attempt. try (TestWebSocketServer silent2 = new TestWebSocketServer(new SilentHandler())) { int port = silent2.getPort(); silent2.start(); @@ -3634,16 +3829,11 @@ public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception { SenderPool pool = new SenderPool(cfg, 0, maxSize, acquireTimeoutMillis, Long.MAX_VALUE, Long.MAX_VALUE); long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); try { - // Headline guarantee: total recovery is bounded by the - // shared budget, NOT maxSize * acquireTimeoutMillis. The 3x - // ceiling leaves generous CI margin over the ~1x post-fix - // cost while still decisively failing the pre-fix 4x stall. Assert.assertTrue( - "startup recovery must be bounded by a shared budget, not per-slot: took " - + elapsedMillis + "ms with acquireTimeout=" + acquireTimeoutMillis - + "ms over " + maxSize + " stranded slots (pre-fix ~" - + (maxSize * acquireTimeoutMillis) + "ms)", - elapsedMillis < 3 * acquireTimeoutMillis); + "direct construction must not block on startup recovery: took " + + elapsedMillis + "ms with acquireTimeout=" + + acquireTimeoutMillis + "ms", + elapsedMillis < acquireTimeoutMillis); // Durability, not loss: the silent server never acked, so // the stranded data is deferred (still on disk for a later @@ -3814,12 +4004,15 @@ public void testDeferredStartupRecoveryDoesNotBlockConstruction() throws Excepti "deferred construction must not block on recovery: took " + elapsedMillis + "ms with acquireTimeout=" + acquireTimeoutMillis + "ms", elapsedMillis < acquireTimeoutMillis); + Assert.assertNull("deferred pool must not own a private recovery thread", + pool.getStartupRecoveryThreadForTesting()); - // Recovery has not been driven yet, so every stranded slot is - // still on disk -- proving construction skipped inline recovery. + // Recovery has not been driven yet, so every stranded slot + // is still on disk -- proving construction left recovery to + // the external housekeeper. for (int i = 0; i < maxSize; i++) { Assert.assertTrue( - "deferred construction must NOT recover inline: default-" + i, + "deferred construction must leave recovery undriven: default-" + i, hasSegmentFile(slot("default-" + i))); } @@ -3828,7 +4021,7 @@ public void testDeferredStartupRecoveryDoesNotBlockConstruction() throws Excepti // lost) for a later attempt -- exercising the deferred path's // concurrency-safe slot reservation too. long recoverStart = System.nanoTime(); - pool.runStartupRecoveryToCompletionForTesting(); + pool.runStartupRecoveryWithinBudgetForTesting(); long recoverMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStart); Assert.assertTrue( "driven recovery must be bounded by the shared budget: took " + recoverMillis @@ -3848,10 +4041,10 @@ public void testDeferredStartupRecoveryDoesNotBlockConstruction() throws Excepti @Test public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { - // Deferring recovery off the constructor must not lose it: once driven - // (by the housekeeper in production; explicitly here) against an acking - // server, a deferred pool recovers its stranded managed slots exactly as - // the inline path would. The drive is also idempotent. + // A deferred pool must not lose recovery: once driven (by the + // housekeeper in production; explicitly here) against an acking server, + // it recovers the same stranded managed slots as the direct background + // driver. The manual drive is also idempotent. TestUtils.assertMemoryLeak(() -> { // Phase 1: seed unacked data into default-0 (silent server). try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { @@ -3883,7 +4076,7 @@ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { hasSegmentFile(slot("default-0"))); // Drive it (what the housekeeper does on its first tick). - pool.runStartupRecoveryToCompletionForTesting(); + pool.runStartupRecoveryWithinBudgetForTesting(); Assert.assertTrue("driven recovery must empty default-0", awaitNoSegmentFile(slot("default-0"), 15_000)); Assert.assertTrue("recovered frames must reach the server", @@ -3892,7 +4085,7 @@ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { Files.exists(slot("default-0") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); // Idempotent: a second drive is a no-op and must not throw. - pool.runStartupRecoveryToCompletionForTesting(); + pool.runStartupRecoveryWithinBudgetForTesting(); Assert.assertFalse("default-0 stays recovered", hasSegmentFile(slot("default-0"))); // Pool still usable for normal borrows. @@ -4154,13 +4347,22 @@ private void assertPreallocatedExitHandoffCleansStartupRecoverer( SenderPool pool = null; try { pool = newPoolWithFactory(config, 0, maxSize, 2_000, factory); + SenderPool recoveryPool = pool; + Assert.assertTrue( + "background recovery must build the stranded slot", + awaitCondition(() -> engineRef.get() != null, 10_000)); CursorSendEngine engine = engineRef.get(); SegmentManager manager = managerRef.get(); - Assert.assertNotNull("startup recovery must build the stranded slot", engine); if (strandedIndex < maxSize) { + Assert.assertTrue( + "in-range recoverer must retire after its close times out", + awaitCondition(() -> recoveryPool.leakedSlotCount() == 1, 10_000)); Assert.assertEquals("in-range recoverer must remain retired while worker is live", 1, pool.leakedSlotCount()); } else { + Assert.assertTrue( + "out-of-range scan must finish after the recoverer close returns", + awaitCondition(recoveryPool::isRecoveryCompleteForTesting, 10_000)); Assert.assertEquals("out-of-range recovery must not consume pool capacity", 0, pool.leakedSlotCount()); } @@ -4268,6 +4470,18 @@ private static boolean awaitAtLeast(AtomicInteger counter, int target, long time return counter.get() >= target; } + private static boolean awaitCondition(BooleanSupplier condition, long timeoutMillis) + throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return true; + } + Thread.sleep(10); + } + return condition.getAsBoolean(); + } + @Test public void testContendedSlotReprobeUsesFlockProbeNotFullBuild() throws Exception { // A slot whose flock is held by another LIVE owner is parked and @@ -4425,8 +4639,8 @@ private static SenderPool newPoolWithRecoveryThreadFactory( } // Uses the @TestOnly 8-arg constructor (deferStartupRecovery=true) so a test - // can build a pool whose SF startup recovery is NOT run inline -- mirroring - // the pooled QuestDB handle, which defers it to the housekeeper. + // can build a pool with no private recovery thread -- mirroring the pooled + // QuestDB handle, which delegates recovery to the housekeeper. // senderFactory=null -> the real defaultSender(). private static SenderPool newDeferredPool(String cfg, int min, int max, long acquireMs) { return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, null, true); From 5bd76e05db3fb9925c695e466ce3db017ce97ef4 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Thu, 23 Jul 2026 11:03:06 +0100 Subject: [PATCH 63/64] test(qwp): pin the sf_sync_interval_millis config rejection paths Both new user-visible rejection branches for sf_sync_interval_millis shipped untested: the out-of-range guard (<=0, or past the Long.MAX_VALUE/1_000_000 nanosecond-overflow boundary) and the non-WebSocket transport guard in the config-string parser. Add three cases to SfFromConfigTest: - testSfSyncIntervalRejectsZero: =0 is a realistic operator typo, rejected with "sf_sync_interval_millis is out of range". - testSfSyncIntervalOverflowBoundary: two-sided pin of the millis->nanos overflow guard -- Long.MAX_VALUE/1_000_000 is honored with exact nanos, one millisecond more is rejected -- so a later change to the operator or the divisor cannot regress silently. - testSfSyncIntervalRejectsOnNonWebSocketTransport: an http config is rejected with "sf_sync_interval_millis is only supported for WebSocket transport". --- .../qwp/client/sf/SfFromConfigTest.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java index 1644d561..fc2f1833 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java @@ -276,6 +276,78 @@ public void testSfSyncIntervalRequiresPeriodicMode() throws Exception { }); } + @Test + public void testSfSyncIntervalRejectsZero() throws Exception { + // A realistic operator typo: sf_sync_interval_millis=0. The setter + // rejects it at parse time (millis <= 0), before build() cross-checks + // periodic mode, so the out-of-range message is what surfaces. + TestUtils.assertMemoryLeak(() -> { + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + + ";sf_durability=periodic;sf_sync_interval_millis=0;"; + try (Sender ignored = Sender.fromConfig(config)) { + Assert.fail("expected sf_sync_interval_millis=0 to be rejected"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("sf_sync_interval_millis is out of range")); + } + }); + } + + @Test + public void testSfSyncIntervalOverflowBoundary() throws Exception { + // Pin the exact millis->nanos overflow guard: millis * 1_000_000 must + // not overflow a long. The largest accepted value is + // Long.MAX_VALUE / 1_000_000; one millisecond more must be rejected. + // Both sides are asserted so a later change to the operator (> vs >=) + // or the 1_000_000 divisor cannot regress silently. + final long maxValidMillis = Long.MAX_VALUE / 1_000_000L; + TestUtils.assertMemoryLeak(() -> { + AckHandler handler = new AckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Accept side: the boundary value is honored and converts to + // nanos without overflow. + String accepted = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfDir + ";sf_durability=periodic" + + ";sf_sync_interval_millis=" + maxValidMillis + ";"; + try (Sender sender = Sender.fromConfig(accepted)) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + Assert.assertEquals(maxValidMillis * 1_000_000L, + ws.getCursorEngineForTesting().getSyncIntervalNanosForTesting()); + } + } + + // Reject side: one millisecond past the boundary is out of range. + String rejected = "ws::addr=localhost:1;sf_dir=" + sfDir + + ";sf_durability=periodic;sf_sync_interval_millis=" + + (maxValidMillis + 1) + ";"; + try (Sender ignored = Sender.fromConfig(rejected)) { + Assert.fail("expected the overflow-boundary value to be rejected"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("sf_sync_interval_millis is out of range")); + } + }); + } + + @Test + public void testSfSyncIntervalRejectsOnNonWebSocketTransport() throws Exception { + // The config-string parser rejects sf_sync_interval_millis on any + // non-WebSocket schema before the value is parsed. + TestUtils.assertMemoryLeak(() -> { + String config = "http::addr=localhost:1;sf_sync_interval_millis=123;"; + try (Sender ignored = Sender.fromConfig(config)) { + Assert.fail("expected sf_sync_interval_millis on http to be rejected"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains( + "sf_sync_interval_millis is only supported for WebSocket transport")); + } + }); + } + @Test public void testSfMaxBytesAcceptsSizeSuffixes() throws Exception { TestUtils.assertMemoryLeak(() -> { From ab5a13a0324db510a1a799abea999dfb59ba520b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Thu, 23 Jul 2026 11:57:15 +0100 Subject: [PATCH 64/64] perf(qwp): skip proven-durable sealed prefix in periodic SF sync servicePeriodicSync copied and scanned the whole live sealed list under the ring monitor every tick and called syncPublished() on each. But a segment is made durable before it is sealed (the requestSyncBeforeRotation gate), so every sealed barrier early-returns -- the copy was O(live-sealed) monitor-held work that grows without bound under a producer-outpaces-drain backlog. Track a monotonic durability frontier (firstNonDurableSealed) and copy only [frontier, size) + active via copyPendingSyncSegments. The frontier only advances past segments observed durable and durability never regresses, so it is a conservative lower bound that can never skip a segment still needing a barrier; recovery-resumed non-durable sealed segments are covered because it starts at 0. Maintained under the ring monitor and shifted with sealed-list compaction, so steady-state copy is O(1). Adds a frontier invariant assert plus tests for the durable-prefix skip and the compaction index-shift. Also folds in minor local cleanups (Arrays.fill, RetainedSegmentMembership lambdas, assertTrue). --- .../qwp/client/sf/cursor/SegmentManager.java | 22 +-- .../qwp/client/sf/cursor/SegmentRing.java | 125 ++++++++++++++---- .../SegmentManagerPeriodicSyncTest.java | 81 ++++++++++++ .../qwp/client/sf/cursor/SegmentRingTest.java | 66 ++++++++- 4 files changed, 250 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index e4a23090..91deddd3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -34,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; @@ -1094,9 +1095,7 @@ private boolean serviceRing0(RingEntry e) { batchSize = e.ring.stagePendingTrims( trimBatch, MAX_TRIMS_PER_RING_PASS, durableAck); } catch (Throwable t) { - for (int i = 0; i < trimBatch.length; i++) { - trimBatch[i] = null; - } + Arrays.fill(trimBatch, null); recordTrimFailure(e, TRIM_RETRY_UNLINK, now, t); return false; } @@ -1219,17 +1218,20 @@ private void servicePeriodicSync(RingEntry e, long now) { return; } try { - e.ring.copyLiveSegmentsForSync(syncScratch); + e.ring.copyPendingSyncSegments(syncScratch); for (int i = 0, n = syncScratch.size(); i < n; i++) { syncScratch.getQuick(i).syncPublished(); } e.ring.clearSyncRequestIfActiveDurable(); - // The pass above covered every live segment's published range, and - // a failed barrier re-dirties its range under an mlock pin (see - // MmapSegment.syncPublished), so a success here is a genuine - // re-persist -- not a vacuous retry over pages a failed writeback - // marked clean (fsyncgate). Unlatch so a transient disk fault - // doesn't permanently brick the producer. + // The pass above covered every not-yet-durable live range -- the + // proven-durable sealed prefix is skipped precisely because its + // syncPublished would early-return (see copyPendingSyncSegments), + // so any latched failure necessarily belongs to a range we just + // re-barriered. A failed barrier re-dirties its range under an + // mlock pin (see MmapSegment.syncPublished), so a success here is a + // genuine re-persist -- not a vacuous retry over pages a failed + // writeback marked clean (fsyncgate). Unlatch so a transient disk + // fault doesn't permanently brick the producer. e.ring.clearDurabilityFailure(); e.nextDataSyncNanos = now + e.syncIntervalNanos; if (e.syncFailureLogged) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index d366fbc5..d3f43e25 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -100,6 +100,19 @@ public final class SegmentRing implements QuietCloseable { // Logical head of sealedSegments. Head removal nulls one entry and advances // this index; occasional compaction bounds unused prefix slots. private int sealedHead; + // Frontier index into sealedSegments: every sealed segment in + // [sealedHead, firstNonDurableSealed) has been proven durable by an earlier + // periodic pass and never needs re-scanning -- publishedCursor is frozen at + // seal and durableCursor only advances, so durability never regresses. The + // periodic sync (copyPendingSyncSegments) skips this proven-durable prefix, + // keeping the steady-state copy-under-monitor O(1) instead of O(live-sealed) + // work that would otherwise grow with a producer-outpaces-drain backlog. + // Rotation seals only already-durable predecessors (the + // requestSyncBeforeRotation gate), so the sole source of non-durable sealed + // segments is a crash-recovery resume; those are covered because the + // frontier starts at 0. Maintained entirely under this monitor, in the same + // coordinate space as sealedHead (shifted by compaction, reset on clear). + private int firstNonDurableSealed; // High-water byte offset within the active segment at which we proactively // ask the segment manager to provision a spare (if one isn't already // installed). Computed once as 3/4 of segment capacity -- leaves the manager @@ -668,29 +681,23 @@ private static RetainedSegmentMembership newDefaultMembership( } if (DEFAULT_MEMBERSHIP_MODE == RetainedSegmentMembershipMode.LINEAR) { if (observer == null) { - return new RetainedSegmentMembership() { - @Override - public boolean contains(MmapSegment segment) { - for (int i = 0, n = chain.size(); i < n; i++) { - if (chain.get(i) == segment) { - return true; - } - } - return false; - } - }; - } - return new RetainedSegmentMembership() { - @Override - public boolean contains(MmapSegment segment) { + return segment -> { for (int i = 0, n = chain.size(); i < n; i++) { - observer.onMembershipOperation(); if (chain.get(i) == segment) { return true; } } return false; + }; + } + return segment -> { + for (int i = 0, n = chain.size(); i < n; i++) { + observer.onMembershipOperation(); + if (chain.get(i) == segment) { + return true; + } } + return false; }; } @@ -699,19 +706,11 @@ public boolean contains(MmapSegment segment) { retained.put(chain.get(i), Boolean.TRUE); } if (observer == null) { - return new RetainedSegmentMembership() { - @Override - public boolean contains(MmapSegment segment) { - return retained.containsKey(segment); - } - }; + return retained::containsKey; } - return new RetainedSegmentMembership() { - @Override - public boolean contains(MmapSegment segment) { - observer.onMembershipOperation(); - return retained.containsKey(segment); - } + return segment -> { + observer.onMembershipOperation(); + return retained.containsKey(segment); }; } @@ -1070,6 +1069,46 @@ synchronized void copyLiveSegmentsForSync(ObjList target) { } } + /** + * Copies the live segments that may still need a durability barrier: every + * sealed segment from the {@link #firstNonDurableSealed} frontier onward, + * plus the active segment. First advances the frontier past any sealed + * segments an earlier pass (or rotation's pre-seal barrier) has since made + * durable. Used by the periodic sync path in place of + * {@link #copyLiveSegmentsForSync}: the proven-durable prefix would + * otherwise be re-copied under this monitor and re-scanned every tick as + * no-op {@link MmapSegment#syncPublished()} early-returns -- O(live-sealed) + * work that grows without bound under a producer-outpaces-drain backlog. + * The frontier is a conservative lower bound (it only ever advances past + * segments observed durable, and durability never regresses), so this can + * never skip a segment that still needs a barrier. + */ + synchronized void copyPendingSyncSegments(ObjList target) { + target.clear(); + // Invariant maintained by every mutation site (rotation append, trim's + // removeSealedHead, compaction shift, close). A frontier that drifted + // above size would silently skip un-fsynced segments, so guard it in + // tests; the clamp below keeps production safe if it is ever violated. + assert firstNonDurableSealed >= sealedHead && firstNonDurableSealed <= sealedSegments.size() + : "durability frontier out of range: firstNonDurableSealed=" + firstNonDurableSealed + + " sealedHead=" + sealedHead + " size=" + sealedSegments.size(); + int i = firstNonDurableSealed; + if (i < sealedHead) { + i = sealedHead; + } + int n = sealedSegments.size(); + while (i < n && sealedSegments.get(i).isPublishedDurable()) { + i++; + } + firstNonDurableSealed = i; + for (; i < n; i++) { + target.add(sealedSegments.get(i)); + } + if (active != null) { + target.add(active); + } + } + void enablePeriodicSync() { periodicSyncEnabled = true; syncRequested = true; @@ -1115,6 +1154,7 @@ public synchronized void close() { } sealedSegments.clear(); sealedHead = 0; + firstNonDurableSealed = 0; for (int i = 0, n = pendingTrims.size(); i < n; i++) { pendingTrims.getQuick(i).close(); } @@ -1211,7 +1251,6 @@ public synchronized MmapSegment firstTrimmable() { return lastSeq <= ackedFsn ? segment : null; } - /** Active segment -- exposed for the I/O thread's "send next batch" path. */ /** * Walks every published frame in the ring (sealed segments plus the active * segment) and returns the FSN of the LAST frame whose payload does NOT @@ -1419,6 +1458,20 @@ public synchronized int getPendingTrimCount() { return pendingTrims.size(); } + /** + * Number of live segments the periodic path would barrier this tick, + * advancing the durability frontier exactly as a real tick does. In the + * steady state (every sealed segment proven durable) this collapses to 1 + * -- the active segment -- proving the proven-durable sealed prefix is no + * longer copied/scanned under the monitor. + */ + @TestOnly + public synchronized int pendingSyncSegmentCountForTest() { + ObjList scratch = new ObjList<>(); + copyPendingSyncSegments(scratch); + return scratch.size(); + } + @TestOnly public synchronized MmapSegment pinSegmentContainingForTest(long fsn) { return pinSegmentContaining(fsn); @@ -1618,16 +1671,30 @@ private void compactSealedSegments() { int liveCount = sealedSegments.size() - sealedHead; trimMovedReferences += liveCount; sealedSegments.remove(0, sealedHead - 1); + // The durable-prefix frontier lives in the same index space as the + // entries we just shifted down by sealedHead; move it with them. + // The invariant firstNonDurableSealed >= sealedHead keeps this >= 0. + firstNonDurableSealed -= sealedHead; + if (firstNonDurableSealed < 0) { + firstNonDurableSealed = 0; + } sealedHead = 0; } } private void removeSealedHead() { sealedSegments.setQuick(sealedHead++, null); + // If the removed head WAS the frontier (a non-durable but already-ACKed + // recovery-resumed segment can be trimmed before its first barrier), + // keep the frontier at or ahead of the live head. + if (firstNonDurableSealed < sealedHead) { + firstNonDurableSealed = sealedHead; + } int size = sealedSegments.size(); if (sealedHead == size) { sealedSegments.clear(); sealedHead = 0; + firstNonDurableSealed = 0; } else if (sealedHead >= 64 && sealedHead >= size - sealedHead) { compactSealedSegments(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java index 83604657..967e6420 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java @@ -305,6 +305,87 @@ public void testSyncPassStopsAtFirstFailureThenRetryCoversAllSegments() throws E }); } + @Test + public void testPeriodicFrontierSkipsSealedPrefixOnceDurable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Recovery resumes a non-durable sealed segment (durableCursor at + // the header) plus a non-durable active. The first periodic copy + // must offer BOTH -- the sealed one still needs a barrier. Once a + // pass has made them durable, the durability frontier advances and + // later copies must offer ONLY the active: the proven-durable + // sealed prefix is never re-copied/re-scanned under the ring + // monitor. This guards the O(1)-steady-state contract that + // replaced the old O(live-sealed) copyLiveSegmentsForSync pass. + final long intervalNanos = 100L; + final long segmentSize = MmapSegment.HEADER_SIZE + + 2 * (MmapSegment.FRAME_HEADER_SIZE + 16); + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-frontier-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + // Sealed chain member (full: FSNs 0..1) + active (FSN 2). + try (MmapSegment s0 = MmapSegment.create( + filesFacade, dir + "/r0.sfa", 0L, segmentSize)) { + s0.tryAppend(payload, 16); + s0.tryAppend(payload, 16); + s0.msync(); + } + try (MmapSegment s1 = MmapSegment.create( + filesFacade, dir + "/r1.sfa", 2L, segmentSize)) { + s1.tryAppend(payload, 16); + s1.msync(); + } + ring = SegmentRing.openExisting(filesFacade, dir, segmentSize); + assertTrue(ring != null); + assertEquals(1, ring.getSealedSegments().size()); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 8L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // Before any successful barrier the recovered sealed segment is + // non-durable, so the frontier cannot advance past it: the copy + // must offer sealed + active. + assertEquals("non-durable recovered sealed segment must be offered for sync", + 2, ring.pendingSyncSegmentCountForTest()); + + // Run the pass (enablePeriodicSync requested it): both segments + // are barriered and become durable. + manager.serviceRingForTesting(ring); + + // The sealed segment is still live (nothing ACKed, so no trim) + // but now durable, so the frontier skips it: only the active is + // offered. count == 1 with a still-present sealed segment proves + // the skip. + assertEquals("durable sealed segment must remain live (no trim without ACKs)", + 1, ring.getSealedSegments().size()); + assertEquals("durable sealed prefix must be skipped by the periodic copy", + 1, ring.pendingSyncSegmentCountForTest()); + // Idempotent: re-running the frontier advance changes nothing. + assertEquals(1, ring.pendingSyncSegmentCountForTest()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + @Test public void testTransientSyncFailureClearsOnNextSuccess() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index d2aa903f..d000cd3a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -39,6 +39,7 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -232,6 +233,64 @@ public void testRotationRearmsHighWaterBackupWakeup() throws Exception { }); } + @Test + public void testDurabilityFrontierSurvivesSealedListCompaction() throws Exception { + // The periodic durability frontier (firstNonDurableSealed) is an + // absolute index into sealedSegments, so the prefix compaction that + // fires once the dead head grows past 64 -- shifting every live entry + // down by sealedHead -- must shift the frontier with it. Drive >64 + // rotations, then trim a 64-segment prefix to force at least one + // compaction, and assert the periodic copy still offers EVERY still- + // live (here non-durable) sealed segment plus the active. A frontier + // not shifted with the compaction would drift past the live entries + // and under-count -- silently skipping un-fsynced segments. + TestUtils.assertMemoryLeak(() -> { + // One frame per segment: each retried append rotates. + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 16); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment seg0 = MmapSegment.create(tmpDir + "/c0.sfa", 0, segSize); + try (SegmentRing ring = new SegmentRing(seg0, segSize)) { + final int sealedTarget = 68; + for (int i = 0; i < sealedTarget; i++) { + long r; + while ((r = ring.appendOrFsn(buf, 16)) >= 0) { + // fill the active until it is full + } + assertEquals(SegmentRing.BACKPRESSURE_NO_SPARE, r); + ring.installHotSpare(MmapSegment.create( + tmpDir + "/c" + (i + 1) + ".sfa", ring.nextSeqHint(), segSize)); + assertTrue("retry must rotate", ring.appendOrFsn(buf, 16) >= 0); + } + assertEquals(sealedTarget, ring.getSealedSegments().size()); + + // Trim the FSN 0..63 prefix (one frame per segment) so + // sealedHead crosses the 64-entry compaction threshold. + SegmentRing.resetTrimMovedReferences(); + ring.acknowledge(63); + ObjList trimmed = ring.drainTrimmable(); + assertNotNull(trimmed); + for (int i = 0, n = trimmed.size(); i < n; i++) { + trimmed.getQuick(i).close(); + } + assertTrue("compaction must have fired", + SegmentRing.getTrimMovedReferences() > 0); + + int liveSealed = ring.getSealedSegments().size(); + assertTrue("some sealed segments must remain live", + liveSealed > 0 && liveSealed < sealedTarget); + // Load-bearing: the frontier tracked the compaction shift, + // so the periodic copy still covers every live sealed + // segment plus the active. Under-counting here would mean a + // live segment is never fsynced by the periodic path. + assertEquals(liveSealed + 1, ring.pendingSyncSegmentCountForTest()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testAcknowledgeAndDrainTrimsOldestFirstUntilUnackedFound() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -645,8 +704,7 @@ public void testRingRecoverySanitizesResumedActiveTornTail() throws Exception { } try (SegmentRing ring = SegmentRing.openExisting(tmpDir, 4096)) { assertNotNull(ring); - assertEquals("the observation must survive for diagnostics", - true, ring.getActive().tornTailBytes() > 0); + assertTrue("the observation must survive for diagnostics", ring.getActive().tornTailBytes() > 0); } // No appends happened; the zeroing must already be durable. try (MmapSegment seg = MmapSegment.openExisting(path)) { @@ -1052,9 +1110,7 @@ public void testAdversarialSegmentOrdersSortLogLinear() throws Exception { assertAdversarialSortWithinBound("organ-pipe", organPipe, bound); long[] allDuplicates = new long[n]; - for (int i = 0; i < n; i++) { - allDuplicates[i] = 42; - } + Arrays.fill(allDuplicates, 42); assertAdversarialSortWithinBound("all-duplicates", allDuplicates, bound); long[] fewDistinct = new long[n];