From de86197b3fb19d9dc274408ae08f7967b468e456 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 02:45:30 +0100 Subject: [PATCH 01/36] Delta-encode the QWP symbol dictionary Previously every QWP ingress message re-sent the entire symbol dictionary, so a connection with many distinct symbols paid to retransmit the whole dictionary on every message. The client now sends each symbol id to the server only once per connection. Memory mode: - The producer keeps a monotonic "sent" watermark and each frame carries only the ids above it (a delta section), instead of the full dictionary from id 0. - On reconnect or failover the fresh server has an empty dictionary, so the I/O thread replays the whole dictionary as a catch-up frame before any post-reconnect traffic, keeping the producer's monotonic baseline valid across the wire boundary. Store-and-forward (file mode): - Each slot persists its dictionary to a dot-prefixed side-file (PersistedSymbolDict) using write-ahead ordering: new symbols are appended before the referencing frame is published, so a recovered or orphan-drained slot on a fresh process can always rebuild the dictionary that a delta frame references. - The persistence does not fsync, matching the rest of store-and-forward, which is process-crash durable (the page cache survives) but not host-crash durable. A host crash that tears the dictionary is caught at replay by a guard that fails the send cleanly ("resend required") instead of transmitting a gapped frame that would corrupt the table. Catch-up split: - The reconnect/recovery catch-up splits across as many frames as the server's advertised batch cap requires, so a dictionary larger than the cap is re-registered without any single frame exceeding it. The frames carry contiguous id ranges and reassemble on the server exactly as the original per-frame deltas would. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 144 ++++++- .../client/sf/cursor/CursorSendEngine.java | 70 +++- .../sf/cursor/CursorWebSocketSendLoop.java | 308 +++++++++++++- .../client/sf/cursor/PersistedSymbolDict.java | 379 ++++++++++++++++++ .../qwp/client/DeltaDictCatchUpTest.java | 325 +++++++++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 352 ++++++++++++++++ .../cutlass/qwp/client/ReconnectTest.java | 7 +- .../qwp/client/SelfSufficientFramesTest.java | 151 +++++-- .../sf/cursor/PersistedSymbolDictTest.java | 219 ++++++++++ .../qwp/websocket/TestWebSocketServer.java | 17 + 10 files changed, 1910 insertions(+), 62 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..286596c4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -46,6 +46,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; @@ -221,6 +222,14 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled only in memory-mode, where a reconnect + // replays from the in-process ring and the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying. File-mode store-and-forward + // keeps full self-sufficient frames (recovery/orphan-drain can replay mid-stream + // to a fresh server, where a partial delta would leave gaps). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -313,6 +322,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -2215,6 +2230,18 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + if (deltaDictEnabled && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } } /** @@ -3423,14 +3450,15 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode only) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3456,10 +3484,18 @@ private void flushPendingRows(boolean deferCommit) { return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { @@ -3514,8 +3550,12 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean deferThis = deferCommit || !isLast; encoder.setDeferCommit(deferThis); + // Each split frame emits the delta above sentMaxSymbolId; the first + // frame ships the whole batch's new ids and advances the baseline, so + // the remaining frames carry an empty delta and just reference ids the + // first frame already registered. encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); encoder.addTable(tableBuffer); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); @@ -3528,11 +3568,18 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); } + // Write-ahead persist before publish (see flushPendingRows). The + // first split frame carries the batch's new symbols; the rest are + // no-ops once the baseline has advanced past them. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3573,8 +3620,10 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows and no new symbols; in delta mode its empty + // delta simply starts at the server's current dictionary size. encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3586,15 +3635,84 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + int from = sentMaxSymbolId + 1; + int to = currentBatchMaxSymbolId; + if (to < from) { + return; + } + for (int id = from; id <= to; id++) { + pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} from + * the slot's persisted dictionary (ids assigned in the same ascending order, + * so they match the recovered frames) and resumes the delta baseline at the + * recovered tip, so newly ingested symbols continue above the recovered ids. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (pd == null || pd.size() == 0) { + return; + } + ObjList symbols = pd.readLoadedSymbols(); + for (int i = 0, n = symbols.size(); i < n; i++) { + globalSymbolDictionary.getOrAddSymbol(symbols.getQuick(i)); + } + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..85015a73 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -110,6 +110,13 @@ public final class CursorSendEngine implements QuietCloseable { // in the constructor, closed by {@link #close()}. The segment manager // writes through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -199,6 +206,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; try { // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine @@ -257,6 +265,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. watermarkInProgress = AckWatermark.open(sfDir); + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Null on open failure -> delta disabled. + persistedDictInProgress = PersistedSymbolDict.open(sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -309,6 +321,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); + // Same stale-side-file hygiene for the symbol dictionary: a + // fresh slot starts with an empty dictionary. + PersistedSymbolDict.removeOrphan(sfDir); + persistedDictInProgress = PersistedSymbolDict.open(sfDir); } MmapSegment initial; String initialPath = null; @@ -333,10 +349,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded — commit the ring, watermark and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -362,6 +379,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -541,6 +564,12 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -550,6 +579,11 @@ public synchronized void close() { AckWatermark.removeOrphan(sfDir); } catch (Throwable ignored) { } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } } } finally { if (slotLock != null) { @@ -586,6 +620,38 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * True when this engine has no store-and-forward directory: the ring lives + * only in malloc'd memory, so it cannot be recovered after a crash and no + * orphan drainer ever replays it. Only in-process reconnect/failover replays + * its frames, which is what makes send-time symbol-dict catch-up (rather than + * fully self-sufficient frames) safe. + */ + public boolean isMemoryMode() { + return sfDir == null; + } + + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..cbf948b0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -36,9 +36,12 @@ import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; @@ -177,6 +180,20 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); + // Delta symbol dictionary catch-up state (memory-mode only; see swapClient). + // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every + // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in + // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that + // on reconnect it can re-register the whole dictionary on the fresh server + // (which discards its dictionary on every disconnect) before replaying frames + // whose deltas start above id 0. All of this is touched only by the I/O thread. + private final boolean deltaDictEnabled; + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + private int sentDictCount; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private final CountDownLatch shutdownLatch = new CountDownLatch(1); private final AtomicLong totalAcks = new AtomicLong(); // Counters for observability of the durable-ack path. Both are zero @@ -489,6 +506,23 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.deltaDictEnabled = engine.isDeltaDictEnabled(); + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + if (deltaDictEnabled) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + ensureSentDictCapacity(len); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + sentDictBytesLen = len; + } + sentDictCount = pd.size(); + } + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -1669,6 +1703,14 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + } shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -1849,8 +1891,6 @@ private void swapClient(WebSocketClient newClient) { // past the tail instead of replaying into it. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; // Snapshot publishedFsn at swap time — frames at FSN ≤ this value // were already on the wire before the drop and will be replayed. // trySendOne resets replayTargetFsn to -1 once we cross the boundary. @@ -1862,9 +1902,240 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + if (client != null && deltaDictEnabled && sentDictCount > 0) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + */ + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + return (int) readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + private void accumulateSentDict(long payloadAddr, int payloadLen) { + long limit = payloadAddr + payloadLen; + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return; + } + long p = payloadAddr + QwpConstants.HEADER_SIZE; + long deltaStart = readVarintAt(p, limit); + p = varintEnd; + long deltaCount = readVarintAt(p, limit); + p = varintEnd; + // Only a delta that extends exactly from our current tip introduces new + // symbols. deltaStart < sentDictCount is a replay/overlap we already hold; + // deltaStart > sentDictCount is rejected before send by the torn-dictionary + // guard in trySendOne, so it never reaches here. + if (deltaCount <= 0 || deltaStart != sentDictCount) { + return; + } + long regionStart = p; + for (long i = 0; i < deltaCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + } + int regionBytes = (int) (p - regionStart); + ensureSentDictCapacity(sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) deltaCount; + } + + private void ensureSentDictCapacity(int required) { + if (sentDictBytesCapacity >= required) { + return; + } + int newCap = Math.max(sentDictBytesCapacity * 2, Math.max(4096, required)); + sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + sentDictBytesCapacity = newCap; + } + + private long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + long cur = p; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(cur++); + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + varintEnd = cur; + return value; + } + + /** + * Builds a table-less catch-up frame carrying the full symbol dictionary + * ({@code deltaStart=0, deltaCount=sentDictCount}) and sends it to the freshly + * connected server, then consumes wire seq 0 for it. {@code sendBinary} copies + * the bytes synchronously, so the temporary frame buffer is freed immediately. + * On send failure it fails the loop, which recycles and retries on the next + * connection. + */ + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Stops early and fails + * the loop on a send error or an entry too large for the cap. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + // Symbol-bytes budget per frame, leaving room for the 12-byte header and + // the two delta-section varints. cap <= 0 means the server advertised no + // limit -> send the whole dictionary in one frame. + int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : Integer.MAX_VALUE; + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + long p = sentDictBytesAddr; + long limit = sentDictBytesAddr + sentDictBytesLen; + while (p < limit) { + long entryStart = p; + long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix + long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] + long entryBytes = entryEnd - entryStart; + if (entryBytes > budget) { + fail(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "entryBytes=" + entryBytes + ", budget=" + budget + ']')); + return framesSent; + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + if (!sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + return framesSent; + } + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + p = entryEnd; + } + if (chunkSymbols > 0 && sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + framesSent++; + } + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Returns false and fails the + * loop on a send error. + */ + private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + int payloadLen = NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + int frameLen = QwpConstants.HEADER_SIZE + payloadLen; + long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = writeVarintAt(q, deltaCount); + Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); + client.sendBinary(frame, frameLen); + } catch (Throwable t) { + fail(t); + return false; + } finally { + Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + return true; + } + + private static long writeVarintAt(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + private boolean tryReceiveAcks() { boolean any = false; try { @@ -1949,12 +2220,38 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + if (deltaDictEnabled) { + // Torn-dictionary guard. In normal operation a delta frame's start id + // never exceeds the dictionary coverage established so far (replayed + // frames overlap the catch-up dict; fresh frames extend it + // contiguously). A gap here means the persisted dictionary was torn -- + // almost always by a host/power crash, which leaves segment frames on + // disk but loses recently-written dictionary entries (SF, like the rest + // of the store, is process-crash durable but not host-crash durable). + // Sending the frame would corrupt the table (the server would null-pad + // the missing ids), so fail terminally instead; the unreplayable data + // must be resent. + int deltaStart = frameDeltaStart(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } + } try { client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); } catch (Throwable t) { fail(t); return false; } + if (deltaDictEnabled) { + // Mirror the symbols this frame introduced so a later reconnect can + // rebuild the whole dictionary. Idempotent on replay: a frame whose + // delta we already hold advances nothing. + accumulateSentDict(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -1984,8 +2281,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java new file mode 100644 index 00000000..4c2674b9 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -0,0 +1,379 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Append-only, per-slot persistence of the global symbol dictionary that a + * store-and-forward sender ships to the server with delta encoding. Lives at + * {@code /.symbol-dict} alongside the segment files, the slot lock and + * {@code .ack-watermark}. + *

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the + * symbols it introduces, so recovering (process restart) or draining (orphan + * adoption) a slot requires re-registering the whole dictionary on the fresh + * server before those frames replay. This file is that dictionary. Unlike + * {@link AckWatermark} -- a discardable optimization protected by a + * {@code max()} clamp -- this file is load-bearing: a surviving frame + * that references an id missing from it is unrecoverable. It is therefore held + * to a stronger durability contract. + *

+ * Layout (little-endian): + * 

+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 1
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: entries, each [len: varint][utf8 bytes], in ascending global-id order
+ *
+ * Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned + * sequentially from 0), so no id needs to be stored. + *

+ * Durability / write-ahead ordering: the producer appends the symbols a + * frame introduces BEFORE that frame is published to the ring, but does NOT + * fsync -- matching the rest of store-and-forward, which is page-cache (not + * disk) durable. This ordering is sufficient for a process/JVM crash: the + * page cache survives, so both the dictionary and the frames survive and the + * dictionary is a superset of every recoverable frame's references. It is NOT + * sufficient for a host/power crash, where unflushed pages can be lost out + * of order and the dictionary may end up torn relative to the frames it serves -- + * exactly as the segment frames themselves may be lost on a host crash. A torn + * dictionary is caught at replay by the send loop's guard, which fails loudly + * (the unreplayable data must be resent) rather than corrupting the target table. + *

+ * A torn trailing entry from a crash mid-append is self-healing: {@link #open} + * stops parsing at the first incomplete entry and the next append overwrites it. + *

+ * Lifecycle: single-writer (the producer / user thread). Read once at + * {@link #open} to seed in-memory state on recovery or orphan-drain. Owner + * (the engine) closes it. Not thread-safe for concurrent writers. + */ +public final class PersistedSymbolDict implements QuietCloseable { + + /** + * Filename within the slot directory. Dot-prefixed so directory + * enumerators that filter by the {@code .sfa} suffix (segment recovery, + * OrphanScanner, trim) skip it automatically. + */ + public static final String FILE_NAME = ".symbol-dict"; + static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian + static final int HEADER_SIZE = 8; + static final byte VERSION = 1; + // Guards against a hostile/corrupt varint length driving a huge allocation + // or a runaway parse. Symbols are short; this is a generous ceiling. + private static final int MAX_ENTRY_LEN = 1 << 20; + private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); + private final int fd; + // In-memory copy of the entry region [len][utf8]... exactly as on disk, + // populated only when open() recovered existing entries (recovery / + // orphan-drain). Zero/empty for a freshly created file. Consumed once to + // seed the send loop's catch-up mirror and the producer's id map. + private final long loadedEntriesAddr; + private final int loadedEntriesLen; + private long appendOffset; + private boolean closed; + private long scratchAddr; + private int scratchCap; + private int size; + + private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + this.fd = fd; + this.appendOffset = appendOffset; + this.size = size; + this.loadedEntriesAddr = loadedEntriesAddr; + this.loadedEntriesLen = loadedEntriesLen; + } + + /** + * Opens (creating if absent) the dictionary file in {@code slotDir}. An + * existing file is parsed and its complete entries are loaded into memory + * (see {@link #loadedEntriesAddr()}); a missing or invalid file is (re)created + * with a fresh header. Returns {@code null} on any I/O failure -- the caller + * then falls back to full-dictionary (self-sufficient) frames for this slot, + * so a broken side-file degrades gracefully rather than aborting the sender. + */ + public static PersistedSymbolDict open(String slotDir) { + String filePath = slotDir + "/" + FILE_NAME; + long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + if (existing >= HEADER_SIZE) { + PersistedSymbolDict d = openExisting(filePath, existing); + if (d != null) { + return d; + } + // Fall through to a clean re-create: a header/parse failure on an + // existing file means it cannot be trusted for delta replay. + } + return openFresh(filePath); + } + + /** + * Best-effort removal of a stale dictionary file. Used at fresh-start (a + * stale dict with no segments behind it is meaningless) and at fully-drained + * close (the slot is empty, nothing references the dictionary any more), + * mirroring {@link AckWatermark#removeOrphan}. + */ + public static void removeOrphan(String slotDir) { + Files.remove(slotDir + "/" + FILE_NAME); + } + + /** + * Appends one symbol, extending the on-disk dictionary. The caller appends a + * frame's new symbols BEFORE publishing that frame, so the write ordering + * (dictionary entry before referencing frame) holds; no fsync is performed + * (see the class-level durability note). Assigns the next dense id implicitly + * (the entry's position). + */ + public void appendSymbol(CharSequence symbol) { + if (closed) { + return; + } + int utf8Len = Utf8s.utf8Bytes(symbol); + int varLen = varintSize(utf8Len); + int recLen = varLen + utf8Len; + ensureScratch(recLen); + long p = writeVarint(scratchAddr, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + long written = Files.write(fd, scratchAddr, recLen, appendOffset); + if (written != recLen) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + recLen + ", actual=" + written + ']'); + } + appendOffset += recLen; + size++; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + if (loadedEntriesAddr != 0L) { + Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT); + } + if (scratchAddr != 0L) { + Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); + scratchAddr = 0L; + scratchCap = 0; + } + if (fd >= 0) { + Files.close(fd); + } + } + + /** + * Base address of the loaded entry region -- the concatenated + * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly + * as a delta section carries them. Zero when nothing was recovered. + */ + public long loadedEntriesAddr() { + return loadedEntriesAddr; + } + + /** + * Byte length of {@link #loadedEntriesAddr()}. + */ + public int loadedEntriesLen() { + return loadedEntriesLen; + } + + /** + * Materialises the loaded entries as symbol strings in ascending-id order + * (entry {@code i} is symbol id {@code i}). Used once on recovery to + * repopulate the producer's global dictionary. Empty when nothing was + * recovered. + */ + public ObjList readLoadedSymbols() { + ObjList out = new ObjList<>(Math.max(size, 1)); + long p = loadedEntriesAddr; + long limit = p + loadedEntriesLen; + for (int i = 0; i < size && p < limit; i++) { + long len = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + len |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + if (p + len > limit) { + break; // defensive: torn tail (should not happen past parse in open) + } + out.add(Utf8s.stringFromUtf8Bytes(p, p + len)); + p += len; + } + return out; + } + + /** + * Number of symbols the dictionary holds (highest id + 1). + */ + public int size() { + return size; + } + + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { + int fd = Files.openRW(filePath); + if (fd < 0) { + LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); + return null; + } + long buf = 0L; + try { + int len = (int) fileLen; + buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + long read = Files.read(fd, buf, len, 0); + if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC) { + LOG.warn("symbol dict {} unreadable or bad magic; recreating", filePath); + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + return null; + } + // Parse complete entries starting after the header; stop at the + // first torn/incomplete trailing entry. + int pos = HEADER_SIZE; + int count = 0; + while (pos < len) { + long[] vr = decodeVarint(buf, pos, len); + if (vr == null) { + break; // torn length varint + } + int entryLen = (int) vr[0]; + int next = (int) vr[1]; + if (entryLen < 0 || entryLen > MAX_ENTRY_LEN || (long) next + entryLen > len) { + break; // torn/oversized entry -- self-healing tail + } + pos = next + entryLen; + count++; + } + int entriesLen = pos - HEADER_SIZE; + long entriesAddr = 0L; + if (entriesLen > 0) { + entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); + } + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + buf = 0L; + // appendOffset lands just past the last complete entry, so the next + // append overwrites any torn trailing bytes. + return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); + } catch (Throwable t) { + if (buf != 0L) { + Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); + } + Files.close(fd); + LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); + return null; + } + } + + private static PersistedSymbolDict openFresh(String filePath) { + int fd = Files.openCleanRW(filePath); + if (fd < 0) { + LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); + return null; + } + long hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC); + Unsafe.getUnsafe().putByte(hdr + 4, VERSION); + Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); + Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0); + Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); + long written = Files.write(fd, hdr, HEADER_SIZE, 0); + if (written != HEADER_SIZE) { + Files.close(fd); + Files.remove(filePath); + LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); + return null; + } + } finally { + Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + } + return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); + } + + /** + * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns + * {@code [value, newPos]} or {@code null} if the varint is truncated + * (torn tail). + */ + private static long[] decodeVarint(long buf, int pos, int limit) { + long value = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new long[]{value, cur}; + } + shift += 7; + if (shift > 35) { + return null; // implausible for an entry length; treat as torn + } + } + return null; + } + + private static int varintSize(long value) { + int n = 1; + while (value > 0x7F) { + value >>>= 7; + n++; + } + return n; + } + + private static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + + private void ensureScratch(int required) { + if (scratchCap >= required) { + return; + } + int newCap = Math.max(required, Math.max(256, scratchCap * 2)); + scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, newCap, MemoryTag.NATIVE_DEFAULT); + scratchCap = newCap; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java new file mode 100644 index 00000000..5e9af978 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -0,0 +1,325 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode). + *

+ * When a memory-mode sender reconnects, the server it lands on has an empty + * dictionary (the server discards it on every disconnect). Because the producer + * ships monotonic deltas -- each symbol id once -- a naive replay would leave the + * fresh server with a dictionary gap. The I/O thread prevents this by sending a + * full-dictionary catch-up frame before any post-reconnect traffic. This test + * reconstructs the server's per-connection dictionary from the captured wire + * bytes and asserts it stays complete and gap-free across the reconnect. + */ +public class DeltaDictCatchUpTest { + + @Test + public void testReconnectCatchUpRebuildsDictionary() throws Exception { + // Connection 1: send "alpha" (id 0), ACK it, then drop the socket so the + // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1). + // Without catch-up, connection 2's first data frame would carry + // deltaStart=1 and the fresh server would never learn id 0. + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + + // Let the I/O loop observe the server-side close before the next + // batch, so batch 2 is what drives the reconnect + catch-up. + Thread.sleep(200); + + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // The fresh (2nd) connection's dictionary, rebuilt purely from the + // frames it received, must hold both symbols contiguously with no + // null gap -- exactly what the catch-up frame guarantees. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals("alpha", conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + } + + @Test + public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception { + // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes), + // then drops once the server has learned them all. On reconnect the fresh + // server has an empty dictionary, so the I/O thread must replay all 40 as a + // catch-up -- but the server advertises a 160-byte batch cap, so the whole + // dictionary cannot fit in a single frame. The catch-up therefore splits + // into several contiguous zero-table frames that the fresh server stitches + // back into a complete, gap-free dictionary. + final int symbolCount = 40; + SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // One row per flush so each frame stays under the 160-byte cap; the + // sent dictionary still accumulates all 40 symbols on connection 1. + for (int i = 0; i < symbolCount; i++) { + sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + sender.flush(); + } + // Wait until connection 1 has learned every symbol, so the sender's + // sent-dictionary mirror (the catch-up source) holds all of them. + waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); + + // Let the I/O loop observe the server-side close before the next + // batch, so batch 2 drives the reconnect + split catch-up. + Thread.sleep(200); + + sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= symbolCount + 1, 10_000); + } + + // Connection 2's dictionary, rebuilt purely from the frames it received, + // must hold every symbol contiguously with no null gap -- the split + // catch-up frames reassemble exactly. + List conn2 = handler.dictFor(2); + Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); + for (int i = 0; i <= symbolCount; i++) { + Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + } + // The catch-up had to span more than one zero-table frame to stay under + // the advertised cap -- that split is the behaviour under test. + Assert.assertTrue("catch-up split into multiple frames (saw " + + handler.zeroTableFramesOnConn2 + ")", + handler.zeroTableFramesOnConn2 >= 2); + } + } + + private static String symbolName(int i) { + // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap. + return "symbol" + (1000 + i); + } + + private static int readVarint(byte[] buf, int[] pos) { + int result = 0; + int shift = 0; + while (pos[0] < buf.length) { + int b = buf[pos[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) return result; + shift += 7; + if (shift > 28) throw new IllegalStateException("varint too long"); + } + throw new IllegalStateException("varint truncated"); + } + + private static void waitFor(BoolCondition cond, long timeoutMillis) { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (cond.test()) return; + try { + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Assert.fail("interrupted"); + } + } + Assert.fail("waitFor timed out"); + } + + @FunctionalInterface + private interface BoolCondition { + boolean test(); + } + + /** + * Mirrors the server's per-connection delta-dictionary accumulation: the + * dictionary resets on every new connection, and each frame's delta section + * ({@code setQuick(deltaStart + i)}, null-padding to reach deltaStart) extends + * or overwrites it. A missing catch-up would show up here as a null gap. + */ + private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + volatile boolean sawZeroTableFrameOnConn2; + private final List> dictsByConn = new CopyOnWriteArrayList<>(); + private TestWebSocketServer.ClientHandler currentClient; + private final AtomicLong nextSeq = new AtomicLong(0); + + synchronized List dictFor(int connNumber) { + return connNumber <= dictsByConn.size() + ? dictsByConn.get(connNumber - 1) + : new ArrayList<>(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + boolean newConnection = currentClient != client; + if (newConnection) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + } + int connNumber = dictsByConn.size(); + List dict = dictsByConn.get(connNumber - 1); + accumulate(data, dict); + if (connNumber == 2 && tableCount(data) == 0) { + sawZeroTableFrameOnConn2 = true; + } + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + // Drop the first connection right after ACKing its only frame, + // forcing the sender to reconnect onto a fresh dictionary. + if (connNumber == 1) { + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static void accumulate(byte[] frame, List dict) { + final byte FLAG_DELTA_SYMBOL_DICT = 0x08; + if (frame.length < 12 || (frame[5] & FLAG_DELTA_SYMBOL_DICT) == 0) { + return; + } + int[] pos = {12}; // just past the 12-byte QWP header + int deltaStart = readVarint(frame, pos); + int deltaCount = readVarint(frame, pos); + while (dict.size() < deltaStart) { + dict.add(null); // null-pad, mirroring the server + } + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(frame, pos); + String symbol = new String(frame, pos[0], len, StandardCharsets.UTF_8); + pos[0] += len; + int idx = deltaStart + i; + while (dict.size() <= idx) { + dict.add(null); + } + dict.set(idx, symbol); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + } + + /** + * Like {@link CatchUpHandler}, but drops connection 1 only after it has + * learned the whole batch, and counts the zero-table catch-up frames on + * connection 2 so a test can assert the dictionary replay split across + * several frames to respect the advertised batch cap. + */ + private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + volatile int zeroTableFramesOnConn2; + private final List> dictsByConn = new CopyOnWriteArrayList<>(); + private final int dropConn1AtDictSize; + private final AtomicLong nextSeq = new AtomicLong(0); + private boolean conn1Dropped; + private TestWebSocketServer.ClientHandler currentClient; + + SplitCatchUpHandler(int dropConn1AtDictSize) { + this.dropConn1AtDictSize = dropConn1AtDictSize; + } + + synchronized List dictFor(int connNumber) { + return connNumber <= dictsByConn.size() + ? dictsByConn.get(connNumber - 1) + : new ArrayList<>(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + boolean newConnection = currentClient != client; + if (newConnection) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + } + int connNumber = dictsByConn.size(); + List dict = dictsByConn.get(connNumber - 1); + CatchUpHandler.accumulate(data, dict); + if (connNumber == 2 && CatchUpHandler.tableCount(data) == 0) { + zeroTableFramesOnConn2++; + } + try { + client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + // Drop connection 1 only once it has learned the entire batch, so + // the sender's sent-dictionary mirror is complete and the reconnect + // catch-up must replay a dictionary larger than the batch cap. + if (connNumber == 1 && !conn1Dropped && dict.size() >= dropConn1AtDictSize) { + conn1Dropped = true; + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java new file mode 100644 index 00000000..88afc3ef --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -0,0 +1,352 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.Files; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * End-to-end recovery for the delta symbol dictionary in store-and-forward mode. + *

+ * A file-mode sender writes delta-encoded SYMBOL frames (each frame carries only + * the ids it introduces) to a slot but never drains it -- simulating a crash. A + * fresh sender then recovers the slot and replays those non-self-sufficient + * frames to a brand-new server whose dictionary starts empty. Correctness hinges + * on the persisted {@code .symbol-dict}: the recovering sender loads it, the I/O + * thread re-registers the whole dictionary via a catch-up frame, and only then do + * the delta frames replay. This test reconstructs the server-side dictionary from + * the wire and asserts it comes out complete and gap-free. + */ +public class DeltaDictRecoveryTest { + + private static final int DISTINCT_SYMBOLS = 8; + private static final int ROWS = 40; + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-delta-recov-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir != null) { + rmDirRec(sfDir); + } + } + + @Test + public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception { + // Phase 1: silent server (no acks). Sender 1 writes symbol rows and + // close-fast (no drain), leaving unacked delta frames + a persisted + // dictionary in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String pad = TestUtils.repeat("x", 64); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .stringColumn("p", pad) + .longColumn("v", i) + .atNow(); + s1.flush(); + } + } + } + + // Phase 2: fresh server that reconstructs its per-connection dictionary + // from the delta sections. Sender 2 recovers the slot and replays. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } + } + + // The recovering sender must have re-registered the dictionary via a + // catch-up (0-table) frame before replaying delta frames. + Assert.assertTrue("recovery sent a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + // The reconstructed dictionary must be complete and gap-free: exactly + // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. + List dict = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + } + } + } + + @Test + public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { + // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. + // Silent server + close-fast leaves all frames unacked in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Simulate a host/power crash: the segment frames survive but the persisted + // dictionary is lost, and the ack watermark was left mid-stream. Truncate + // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at + // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The replay guard must + // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + Thread.sleep(1_000); // let the I/O loop attempt replay and hit the guard + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + terminal = e; + } + } + Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", + 0, handler.frames.get()); + Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); + } + } + + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { + byte[] buf = new byte[16]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.putInt(0x31574B41); // 'AKW1' + bb.putInt(0); // reserved + bb.putLong(fsn); + java.nio.file.Files.write(path, buf); + } + + private static int readVarint(byte[] buf, int[] pos) { + int result = 0; + int shift = 0; + while (pos[0] < buf.length) { + int b = buf[pos[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift > 28) { + throw new IllegalStateException("varint too long"); + } + } + throw new IllegalStateException("varint truncated"); + } + + private static void rmDirRec(String dir) { + if (!Files.exists(dir)) { + return; + } + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRec(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + /** + * Reconstructs the per-connection symbol dictionary from delta sections, + * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. + */ + private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler { + volatile boolean sawCatchUpFrame; + private final List dict = new ArrayList<>(); + private final AtomicLong nextSeq = new AtomicLong(0); + private TestWebSocketServer.ClientHandler currentClient; + + synchronized List dictSnapshot() { + return new ArrayList<>(dict); + } + + synchronized int maxDictSize() { + return dict.size(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + dict.clear(); // fresh server dictionary per connection + } + accumulate(data); + if (tableCount(data) == 0 && hasDelta(data)) { + sawCatchUpFrame = true; + } + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static boolean hasDelta(byte[] frame) { + return frame.length >= 12 && (frame[5] & 0x08) != 0; + } + + private static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + + private void accumulate(byte[] frame) { + if (!hasDelta(frame)) { + return; + } + int[] pos = {12}; + int deltaStart = readVarint(frame, pos); + int deltaCount = readVarint(frame, pos); + while (dict.size() < deltaStart) { + dict.add(null); + } + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(frame, pos); + String sym = new String(frame, pos[0], len, StandardCharsets.UTF_8); + pos[0] += len; + int idx = deltaStart + i; + while (dict.size() <= idx) { + dict.add(null); + } + dict.set(idx, sym); + } + } + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- sender leaves everything unacked in the slot + } + } + + /** Counts every binary frame it receives and acks it. */ + private static class CountingHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger frames = new AtomicInteger(); + private final AtomicLong nextSeq = new AtomicLong(0); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + frames.incrementAndGet(); + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java index 36553388..7ba7a1fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java @@ -52,9 +52,10 @@ * disconnect = sender broken, every subsequent batch threw. Reconnect * machinery now handles transient drops: detect, build a fresh client * via the registered factory, reset wire state, and reposition the replay - * cursor at {@code engine.ackedFsn() + 1}. Cursor frames are self-sufficient - * (every frame carries full schema + full symbol-dict delta), so post-reconnect - * replay needs no producer-side schema-reset signal. + * cursor at {@code engine.ackedFsn() + 1}. Every frame carries its full schema + * inline; the symbol dictionary is either shipped in full per frame (file-mode + * store-and-forward) or re-registered by an I/O-thread catch-up frame before + * replay (memory-mode), so post-reconnect replay needs no producer-side reset. *

* This commit covers the mechanics with a single-attempt retry; backoff, * per-outage time cap, and auth-failure detection follow. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 45275528..b0a1b5a7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -32,23 +32,26 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; /** - * Pins down the "every frame on disk is self-sufficient" rule for the symbol - * dictionary. + * Pins down how the symbol dictionary is framed on the wire. *

- * The cursor SF path used to elide previously-sent symbols on subsequent - * batches over the same connection, emitting a delta-dict that carried only - * the new entries. That's wrong for SF: the bytes survive process restarts and - * replay against fresh server connections (post-reconnect, or via a background - * drainer adopting an orphan slot). A delta that references symbol ids the new - * server has never seen is unrecoverable. + * Both engine modes ship monotonic deltas -- each symbol id travels once, + * not the whole dictionary per message -- which is the bandwidth win this feature + * adds. The I/O thread re-registers the dictionary with a catch-up frame whenever + * it (re)connects, so a fresh server can resolve the non-self-sufficient delta + * frames that follow. *

- * Today every frame must carry a complete symbol-dict delta starting at id 0 - * (column schemas travel inline on the first batch too). This test asserts the - * symbol-dict invariant on the wire. + * The modes differ only in where the catch-up's dictionary comes from: memory + * mode keeps it in an in-process mirror; file-backed store-and-forward keeps it in + * a per-slot {@code .symbol-dict} file so a recovered or orphan-drained slot (a + * fresh process with no in-memory mirror) can rebuild it. This test asserts the + * monotonic wire framing in both modes and the presence of that dictionary file. */ public class SelfSufficientFramesTest { @@ -56,12 +59,72 @@ public class SelfSufficientFramesTest { private static final int DELTA_START_OFFSET = 12; @Test - public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception { - // Send two batches against the same connection, each with a - // distinct symbol value. With the old schema-ref/delta encoding, - // batch 2 would emit deltaStart=1, deltaCount=1 — only the new - // symbol. With self-sufficient frames, batch 2 must emit - // deltaStart=0 covering BOTH symbols. + public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { + // File-backed SF also ships monotonic deltas now: batch 2 carries only + // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict + // so a recovered/orphan-drained slot can rebuild it. + Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); + } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } finally { + rmDir(sfDir); + } + } + + private static boolean containsUtf8(byte[] haystack, String needle) { + byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8); + outer: + for (int i = 0; i + n.length <= haystack.length; i++) { + for (int j = 0; j < n.length; j++) { + if (haystack[i + j] != n[j]) { + continue outer; + } + } + return true; + } + return false; + } + + @Test + public void testMemoryModeShipsMonotonicDelta() throws Exception { + // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries + // only "beta" as a delta starting at id 1, not the whole dictionary. CapturingHandler handler = new CapturingHandler(); try (TestWebSocketServer server = new TestWebSocketServer(handler)) { int port = server.getPort(); @@ -82,29 +145,17 @@ public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception { byte[] b1 = handler.batches.get(0); byte[] b2 = handler.batches.get(1); - // The deltaStart varint sits right after the 12-byte header. - // For self-sufficient frames it must be 0 (single byte 0x00) - // in BOTH batches — regardless of how many symbols the prior - // batch already shipped. - int deltaStart1 = readVarint(b1, DELTA_START_OFFSET); - int deltaStart2 = readVarint(b2, DELTA_START_OFFSET); - Assert.assertEquals("batch 1 deltaStart must be 0", 0, deltaStart1); - Assert.assertEquals("batch 2 deltaStart must be 0 (self-sufficient)", - 0, deltaStart2); - - // batch 2 must include >= 2 symbols in its delta dict (alpha - // from the prior batch + beta from this one). The varint at - // DELTA_START_OFFSET+1 is deltaCount. - int deltaCount2 = readVarint(b2, DELTA_START_OFFSET + 1); - Assert.assertTrue("batch 2 must redefine at least 2 symbols, got " + deltaCount2, - deltaCount2 >= 2); - - // Sanity: batch 2 should NOT be much smaller than batch 1 — - // with schema-ref/delta encoding it would have been; with - // self-sufficient frames the size is in the same ballpark. - Assert.assertTrue("batch 2 (" + b2.length + " bytes) must not be drastically smaller than batch 1 (" - + b1.length + ")", - b2.length >= b1.length / 2); + // Batch 1 introduces alpha at id 0. + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + + // Batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); } } @@ -122,6 +173,25 @@ private static int readVarint(byte[] buf, int offset) { throw new IllegalStateException("varint truncated"); } + private static void rmDir(Path dir) { + try { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } catch (IOException ignored) { + // best-effort + } + } + private static void waitFor(BoolCondition cond, long timeoutMillis) { long deadline = System.currentTimeMillis() + timeoutMillis; while (System.currentTimeMillis() < deadline) { @@ -157,6 +227,7 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat } } + // Mirrors WebSocketResponse STATUS_OK layout: status u8 | sequence u64 | table_count u16 static byte[] buildAck(long seq) { byte[] buf = new byte[1 + 8 + 2]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java new file mode 100644 index 00000000..10148468 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -0,0 +1,219 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.ObjList; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class PersistedSymbolDictTest { + + @Test + public void testAppendPersistsAcrossReopen() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals(0, d.size()); + d.appendSymbol("AAPL"); + d.appendSymbol("GOOG"); + d.appendSymbol("MSFT"); + Assert.assertEquals(3, d.size()); + } finally { + d.close(); + } + + // Reopen: entries recovered in id order. + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(reopened); + try { + Assert.assertEquals(3, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(3, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertTrue(reopened.loadedEntriesLen() > 0); + + // Appending after recovery continues from the recovered tip. + reopened.appendSymbol("TSLA"); + Assert.assertEquals(4, reopened.size()); + } finally { + reopened.close(); + } + + PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(4, third.size()); + Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); + } finally { + third.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testBadMagicIsRecreatedEmpty() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // A file with the right size but garbage content (bad magic). + Path f = dir.resolve(".symbol-dict"); + Files.write(f, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("bad-magic file recreated empty", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testEmptySymbolRoundTrips() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol(""); + d.appendSymbol("nonempty"); + } finally { + d.close(); + } + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("", s.getQuick(0)); + Assert.assertEquals("nonempty", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testRemoveOrphanDeletesFile() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("A"); + d.close(); + Path f = dir.resolve(".symbol-dict"); + Assert.assertTrue(Files.exists(f)); + PersistedSymbolDict.removeOrphan(dir.toString()); + Assert.assertFalse(Files.exists(f)); + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testTornTrailingEntrySelfHeals() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Write two complete entries, then a torn trailing record: a + // length prefix of 5 followed by only 2 bytes (crash mid-append). + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + Path f = dir.resolve(".symbol-dict"); + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, + StandardOpenOption.APPEND); + + // Reopen: the torn tail is ignored; only the two complete entries load. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + // The next append overwrites the torn tail, keeping the file consistent. + re.appendSymbol("three"); + Assert.assertEquals(3, re.size()); + } finally { + re.close(); + } + + PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(3, re2.size()); + Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); + } finally { + re2.close(); + } + } finally { + rmDir(dir); + } + }); + } + + private static void rmDir(Path dir) { + try { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + } + }); + } catch (IOException ignored) { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..5e009d67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -108,6 +108,11 @@ public class TestWebSocketServer implements Closeable { // 401, 403, 404, 426, 503, etc. that the failover loop should // classify per failover.md §6. private volatile String rejectingStatusReason; + // When > 0, 101 upgrade responses advertise this value as the + // X-QWP-Max-Batch-Size header, capping the QWP message size the client + // builds. Lets a test force the delta-dictionary catch-up to split across + // several frames. Live-updatable via setAdvertisedMaxBatchSize(). + private volatile int advertisedMaxBatchSize; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -221,6 +226,14 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Advertises {@code X-QWP-Max-Batch-Size: } on subsequent + * handshakes (live update). Pass {@code 0} to stop advertising a cap. + */ + public void setAdvertisedMaxBatchSize(int maxBatchSize) { + this.advertisedMaxBatchSize = maxBatchSize; + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -603,6 +616,10 @@ private boolean performHandshake() throws IOException { if (role != null) { sb.append("X-QuestDB-Role: ").append(role).append("\r\n"); } + int maxBatch = advertisedMaxBatchSize; + if (maxBatch > 0) { + sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n"); + } sb.append("\r\n"); out.write(sb.toString().getBytes(StandardCharsets.US_ASCII)); out.flush(); From 30922301ef77d53ab68dab3da1d8f4e9dc4db1f3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:10:41 +0100 Subject: [PATCH 02/36] Contain QWP dict catch-up send failures in the reconnect loop The symbol-dictionary catch-up called fail() on a send error, but the catch-up runs inside connectLoop (via swapClient) and, on the initial connect, on the caller thread (via start() -> positionCursorForStart). Calling fail() there re-entered connectLoop. On a reconnect this corrupted the wire mapping: the outer setWireBaselineWithCatchUp overwrote fsnAtZero while nextWireSeq kept the nested attempt's value, so a later ACK translated through engine.acknowledge(fsnAtZero + wireSeq) and trimmed un-acked frames from the store-and-forward log -- silent data loss. A flapping connection recursed connectLoop until the stack overflowed into a terminal, turning a transient outage into a hard failure (breaking Invariant B). On the initial connect the same fail() ran connectLoop on the caller thread and blocked Sender construction forever. sendDictCatchUp and sendCatchUpChunk now throw CatchUpSendException instead of calling fail(). connectLoop's own retry catch handles the swapClient path (one non-re-entrant reconnect with backoff); trySendOne's orphan-retire re-anchor turns it into a fresh fail() from the I/O loop body; start() drops the dead client so the I/O thread reconnects and re-sends the catch-up off the caller thread. A single dictionary entry too large for the server batch cap is non-retriable, so it latches a terminal (recordFatal) rather than looping -- also removing the oversized-entry reconnect livelock. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 109 ++++++++++++++---- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index cbf948b0..25ebcadd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1030,7 +1030,28 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { @@ -2028,14 +2049,6 @@ private long readVarintAt(long p, long limit) { return value; } - /** - * Builds a table-less catch-up frame carrying the full symbol dictionary - * ({@code deltaStart=0, deltaCount=sentDictCount}) and sends it to the freshly - * connected server, then consumes wire seq 0 for it. {@code sendBinary} copies - * the bytes synchronously, so the temporary frame buffer is freed immediately. - * On send failure it fails the loop, which recycles and retries on the next - * connection. - */ /** * Re-registers the whole symbol dictionary on a fresh connection, split into * as many table-less frames as the server's advertised batch cap requires so @@ -2043,8 +2056,10 @@ private long readVarintAt(long p, long limit) { * Each chunk carries a contiguous id range {@code [start .. start+count)}, in * order, so the server accumulates them exactly as it would the original * per-frame deltas. Returns the number of frames sent (each consumed a wire - * sequence), so the caller can align {@code fsnAtZero}. Stops early and fails - * the loop on a send error or an entry too large for the cap. + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); @@ -2065,15 +2080,19 @@ private int sendDictCatchUp() { long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] long entryBytes = entryEnd - entryStart; if (entryBytes > budget) { - fail(new LineSenderException( + // Non-retriable: the entry will not shrink and the same cluster + // re-advertises the same cap, so reconnecting would livelock. + // Latch a terminal (the data must be resent after the cap is + // raised) rather than calling fail() -- which, from inside the + // catch-up, would re-enter connectLoop (see CatchUpSendException). + LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "entryBytes=" + entryBytes + ", budget=" + budget + ']')); - return framesSent; + + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); + recordFatal(err); + throw new CatchUpSendException(err); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { - if (!sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { - return framesSent; - } + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; chunkStartId += chunkSymbols; chunkStartAddr = entryStart; @@ -2084,7 +2103,8 @@ private int sendDictCatchUp() { chunkBytes += entryBytes; p = entryEnd; } - if (chunkSymbols > 0 && sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; } return framesSent; @@ -2092,10 +2112,12 @@ private int sendDictCatchUp() { /** * Sends one table-less catch-up frame carrying dictionary ids - * {@code [deltaStart .. deltaStart+deltaCount)}. Returns false and fails the - * loop on a send error. + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. */ - private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { int payloadLen = NativeBufferWriter.varintSize(deltaStart) + NativeBufferWriter.varintSize(deltaCount) + symbolsLen; @@ -2116,15 +2138,20 @@ private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAdd Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); client.sendBinary(frame, frameLen); } catch (Throwable t) { - fail(t); - return false; + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); } finally { Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); } nextWireSeq++; // this catch-up chunk consumed a wire sequence lastFrameOrPingNanos = System.nanoTime(); totalFramesSent.incrementAndGet(); - return true; } private static long writeVarintAt(long addr, long value) { @@ -2169,7 +2196,15 @@ private boolean trySendOne() { // Nothing sent on this connection yet: re-anchor in place past // the retired tail. The wireSeq<->FSN mapping is untouched // because no wire sequence has been consumed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + fail(e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -2336,6 +2371,30 @@ public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. + */ + private static final class CatchUpSendException extends RuntimeException { + CatchUpSendException(Throwable cause) { + super(cause); + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) From 9f3f7bdc8b8f1109781d6f92196d283068d15eb9 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:24:13 +0100 Subject: [PATCH 03/36] Resume the QWP symbol-dict write-ahead from the durable size persistNewSymbolsBeforePublish keyed the append range off sentMaxSymbolId+1. That watermark only advances after the whole frame is published, whereas PersistedSymbolDict.size() advances per persisted entry. If a mid-batch appendSymbol threw (a short write on a full disk), the symbols before the failing one were already durable but the frame was not published, so sentMaxSymbolId stayed put. A retry then re-keyed from sentMaxSymbolId+1 and re-appended that already-persisted prefix, duplicating entries and breaking the dense id->symbol mapping recovery relies on (entry i must be symbol id i) -- a torn dictionary that re-registers the wrong symbols on the fresh server, or diverges the producer's watermark from the I/O thread's mirror. Resume from pd.size() instead: it is exactly the count already durable, so the retry continues past the persisted prefix (the next append overwrites any torn trailing bytes) without duplicating. In the happy path pd.size() equals sentMaxSymbolId+1, so behaviour is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 286596c4..216794ff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3665,12 +3665,22 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - int from = sentMaxSymbolId + 1; + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1. + // appendSymbol advances pd.size() per persisted entry, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior + // appendSymbol threw mid-batch (a short write -- disk full/quota), the + // frame was not published and sentMaxSymbolId stayed put, while the + // symbols before the failing one are already on disk. Keying the resume + // point off sentMaxSymbolId+1 would then re-append that persisted prefix + // on the retry, duplicating entries and corrupting the dense id->symbol + // mapping recovery relies on (position i must be symbol id i). pd.size() + // resumes exactly past what is already durable -- the next appendSymbol + // overwrites any torn trailing bytes at appendOffset -- so the + // write-ahead is idempotent under retry. In the happy path pd.size() + // equals sentMaxSymbolId+1, so behaviour is unchanged. int to = currentBatchMaxSymbolId; - if (to < from) { - return; - } - for (int id = from; id <= to; id++) { + for (int id = pd.size(); id <= to; id++) { pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); } } From 09cd706af158f97ecb1299bd14024f52a5a6cb56 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:24:23 +0100 Subject: [PATCH 04/36] Guard the oversized catch-up entry terminal Add a regression test: a dictionary entry larger than the reconnect server's per-chunk catch-up budget must latch a clean terminal, not reconnect-loop. Connection 1 advertises no cap so a ~200-byte symbol registers into the sent-dictionary mirror; the handler then shrinks the advertised cap and drops the socket, so the reconnect's catch-up cannot re-ship the entry. The test asserts the surfaced terminal names the catch-up path ("... during catch-up"). Reverting the fix (entry-too-large calling fail() again) fails this test with a StackOverflowError on the I/O thread -- the catch-up re-entering connectLoop -- confirming the guard bites both ways. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictCatchUpTest.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 5e9af978..1af55bc3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -25,7 +25,9 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -92,6 +94,65 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { } } + @Test + public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { + // A dictionary entry that exceeds the reconnect server's per-chunk budget + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. + // sendDictCatchUp must latch a clean terminal ("... during catch-up") + // rather than call fail(): pre-fix the oversized entry drove an endless + // reconnect loop (the entry never shrinks and the same cluster + // re-advertises the same cap) and re-entered connectLoop from the catch-up. + // + // Connection 1 advertises no cap, so the ~202-byte symbol registers and + // enters the sent-dictionary mirror. The handler then shrinks the + // advertised cap to 160 (catch-up budget 132) and drops the socket, so the + // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do + // this: the client refuses to SEND a single-table frame over the cap, and + // that data frame is always larger than the bare catch-up entry.) + CapShrinkHandler handler = new CapShrinkHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + handler.setServer(server); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry + LineSenderException terminal = null; + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + try { + sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); + sender.flush(); + // The terminal latches on the I/O thread once the reconnect's + // catch-up hits the oversized entry; it surfaces to the producer + // on a subsequent flush. Poll a bounded time for it. The polling + // rows use a small symbol that fits the shrunk cap, so the + // producer-side cap check never fires and flush() surfaces the + // I/O thread's catch-up terminal via checkError. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + sender.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); + Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up")); + } + } + @Test public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception { // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes), @@ -267,6 +328,47 @@ private static int tableCount(byte[] frame) { } } + /** + * ACKs connection 1's frames with no advertised cap (so an oversized symbol + * registers), then -- once connection 1 has sent something -- shrinks the + * advertised batch cap and drops the socket. The reconnect (connection 2) + * therefore advertises a cap whose catch-up budget is too small for the + * symbol, exercising the oversized-entry terminal in sendDictCatchUp. + */ + private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + private final AtomicLong nextSeq = new AtomicLong(0); + private TestWebSocketServer.ClientHandler currentClient; + private volatile TestWebSocketServer server; + + void setServer(TestWebSocketServer server) { + this.server = server; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + } + try { + client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + if (connectionsAccepted.get() == 1) { + // Connection 1 registered the big symbol. Shrink the cap so the + // reconnect's catch-up budget can't fit it, then drop to force + // the reconnect. Setting the cap before the close (not from the + // test thread after it) removes the set-vs-reconnect race. + server.setAdvertisedMaxBatchSize(160); // catch-up budget = 132 + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + /** * Like {@link CatchUpHandler}, but drops connection 1 only after it has * learned the whole batch, and counts the zero-table catch-up frames on From 18dd545381d0f5c8a0a5fb547a26447b1fd32362 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:25:36 +0100 Subject: [PATCH 05/36] Persist the QWP symbol-dict batch in a single write persistNewSymbolsBeforePublish appended each new symbol with its own PersistedSymbolDict.appendSymbol call, and each appendSymbol issues one positioned write. A high-cardinality batch -- one new symbol per row, which is exactly the store-and-forward workload delta encoding targets -- therefore stalled the producer thread with up to one pwrite syscall per row per flush. Add PersistedSymbolDict.appendSymbols(dict, from, to): it encodes the whole [from..to] entry region into scratch once and issues a single positioned write, so a flush that introduces N symbols costs one syscall instead of N. It keeps appendSymbol's durability and idempotency contract -- no fsync, and a short write throws without advancing size, so a retry keyed off size() re-encodes and overwrites at the same offset. PersistedSymbolDictTest.testAppendSymbolsBatchWritesDenseRange checks the batched write produces the same dense, id-ordered file (including an empty symbol mid-range), that an empty range is a no-op, and that a follow-on batch keyed off the recovered size continues without a gap or duplicate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 34 +++++------ .../client/sf/cursor/PersistedSymbolDict.java | 41 +++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 59 +++++++++++++++++++ 3 files changed, 117 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 216794ff..54be9ea0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3665,24 +3665,24 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1. - // appendSymbol advances pd.size() per persisted entry, whereas + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // appendSymbols advances pd.size() only after a full write, whereas // sentMaxSymbolId only advances after the WHOLE frame is published (via - // advanceSentMaxSymbolId, after activeBuffer.write). If a prior - // appendSymbol threw mid-batch (a short write -- disk full/quota), the - // frame was not published and sentMaxSymbolId stayed put, while the - // symbols before the failing one are already on disk. Keying the resume - // point off sentMaxSymbolId+1 would then re-append that persisted prefix - // on the retry, duplicating entries and corrupting the dense id->symbol - // mapping recovery relies on (position i must be symbol id i). pd.size() - // resumes exactly past what is already durable -- the next appendSymbol - // overwrites any torn trailing bytes at appendOffset -- so the - // write-ahead is idempotent under retry. In the happy path pd.size() - // equals sentMaxSymbolId+1, so behaviour is unchanged. - int to = currentBatchMaxSymbolId; - for (int id = pd.size(); id <= to; id++) { - pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); - } + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (a short write -- disk full/quota), the frame was not published + // and sentMaxSymbolId stayed put, while the symbols before the failing + // one are already on disk. Keying the resume point off sentMaxSymbolId+1 + // would then re-append that persisted prefix on the retry, duplicating + // entries and corrupting the dense id->symbol mapping recovery relies on + // (position i must be symbol id i). pd.size() resumes exactly past what is + // already durable (the write overwrites any torn trailing bytes at + // appendOffset), so the write-ahead is idempotent under retry. In the + // happy path pd.size() equals sentMaxSymbolId+1, so the range -- and the + // behaviour -- are unchanged; the single positioned write just replaces + // the previous one-syscall-per-new-symbol loop. + pd.appendSymbols(globalSymbolDictionary, pd.size(), currentBatchMaxSymbolId); } private void resetSymbolDictStateForNewConnection() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 4c2674b9..92a0cc78 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -173,6 +174,46 @@ public void appendSymbol(CharSequence symbol) { size++; } + /** + * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes + * the whole {@code [len varint][utf8]...} region into scratch first, then + * issues one positioned write -- versus one {@code pwrite(2)} per symbol via + * {@link #appendSymbol}. That per-symbol syscall count is the hot-path cost + * on a high-cardinality batch (one new symbol per row), which is exactly the + * store-and-forward workload delta encoding targets. Callers pass the + * dictionary and the range so the ids resolve to their symbol strings. + *

+ * Same durability and idempotency contract as {@link #appendSymbol}: no + * fsync, and a short write throws WITHOUT advancing {@code size}/{@code + * appendOffset}, so a retry keyed off {@link #size()} re-encodes the same + * range and overwrites at the same offset. No-op when the range is empty or + * the dictionary is closed. + */ + public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { + if (closed || to < from) { + return; + } + int len = 0; + for (int id = from; id <= to; id++) { + CharSequence symbol = dict.getSymbol(id); + int utf8Len = Utf8s.utf8Bytes(symbol); + int recLen = varintSize(utf8Len) + utf8Len; + ensureScratch(len + recLen); + long p = writeVarint(scratchAddr + len, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + len += recLen; + } + long written = Files.write(fd, scratchAddr, len, appendOffset); + if (written != len) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + len + ", actual=" + written + ']'); + } + appendOffset += len; + size += to - from + 1; + } + @Override public void close() { if (closed) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 10148468..73549689 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.ObjList; import org.junit.Assert; @@ -89,6 +90,64 @@ public void testAppendPersistsAcrossReopen() throws Exception { }); } + @Test + public void testAppendSymbolsBatchWritesDenseRange() throws Exception { + // appendSymbols persists a whole id range in one write (the hot-path + // syscall reduction). It must produce the same dense, id-ordered file a + // per-symbol loop would, including an empty symbol mid-range, and a second + // batched call keyed off size() must continue densely (the resume-from- + // durable-size contract the producer relies on). + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + dict.getOrAddSymbol("TSLA"); // id 3 + + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbols(dict, 0, 3); // one write for all four ids + Assert.assertEquals(4, d.size()); + d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op + Assert.assertEquals(4, d.size()); + } finally { + d.close(); + } + + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(4, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(4, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertEquals("TSLA", symbols.getQuick(3)); + + // A follow-on batch keyed off the recovered size continues + // the dense sequence without a gap or duplicate. + dict.getOrAddSymbol("NVDA"); // id 4 + reopened.appendSymbols(dict, reopened.size(), 4); + Assert.assertEquals(5, reopened.size()); + } finally { + reopened.close(); + } + + PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(5, third.size()); + Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); + } finally { + third.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testBadMagicIsRecreatedEmpty() throws Exception { assertMemoryLeak(() -> { From 2a1a2d636b4fa31dc264d923f9f285b0189cf0f7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:25:49 +0100 Subject: [PATCH 06/36] Free the recovery-seeded dict mirror in close() On recovery / orphan-drain the CursorWebSocketSendLoop constructor seeds a native mirror (sentDictBytesAddr) from the slot's persisted dictionary so the first connection can re-register it. That mirror is freed only on ioLoop's exit path, so a loop that is constructed but never runs -- start() never called, or Thread.start() failing before the loop runs, or a close() racing an unstarted loop -- leaked it. close() already safety-nets the client for that same "loop never started" case; the mirror was missed. close() now frees the mirror when the loop never ran (ioThread was null on entry). It does NOT free it when the loop ran: ioLoop's exit owns the free there, and on the failed-stop path the thread may still be mid-send, so touching the mirror would race; a duplicate close observes a zero address and skips. CursorWebSocketSendLoopMirrorLeakTest populates a recoverable slot, then leak-checks constructing an engine + loop over it and closing WITHOUT start(). Reverting the free fails it with a 4096-byte NATIVE_DEFAULT leak. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 17 +++ ...CursorWebSocketSendLoopMirrorLeakTest.java | 141 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 25ebcadd..2e6b826c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -788,6 +788,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -846,6 +852,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + } } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java new file mode 100644 index 00000000..19626ba4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Guards the recovery-seeded symbol-dictionary mirror against leaking when the + * I/O loop is constructed but never run. + *

+ * On recovery / orphan-drain the {@link CursorWebSocketSendLoop} constructor + * seeds a native mirror ({@code sentDictBytesAddr}) from the slot's persisted + * dictionary so the first connection can re-register it. That mirror is freed on + * the I/O thread's exit path -- so if the loop is closed WITHOUT ever starting + * (start() never called, or Thread.start() failing before the loop runs), the + * free never happens. {@code close()} must free it in that case. + */ +public class CursorWebSocketSendLoopMirrorLeakTest { + + private static final int DISTINCT_SYMBOLS = 8; + private static final int ROWS = 40; + + @Test + public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { + Path sfDir = Files.createTempDirectory("qwp-mirror-leak"); + try { + // Populate a slot with delta frames + a non-empty .symbol-dict, then + // abandon it (silent server, close-fast) -- outside assertMemoryLeak, + // because a full Sender+server round trip is not net-zero on its own. + populateRecoverableSlot(sfDir); + + Path slot = sfDir.resolve("default"); + Assert.assertTrue("populate must leave a persisted dictionary", + Files.exists(slot.resolve(".symbol-dict"))); + + // Only the recovery construct + close is leak-checked: the engine + // recovers (loading the dict), the loop ctor seeds the mirror from it, + // and close() -- with NO start() -- must free every native allocation. + // Pre-fix the seeded mirror leaks here and this assertion fails. + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("disk-mode engine must open a persisted dict", pd); + Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)", + pd.size() > 0 && pd.loadedEntriesLen() > 0); + + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + // Close without start(): the ctor-seeded mirror is this + // thread's to free, since the I/O loop never ran. + loop.close(); + } + }); + } finally { + rmDir(sfDir); + } + } + + private static void populateRecoverableSlot(Path sfDir) throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .longColumn("v", i) + .atNow(); + s1.flush(); + } + } + } + } + + private static void rmDir(Path dir) throws IOException { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- the sender leaves everything unacked in the slot + } + } +} From e130da922fd35e981be682ea9609dbfb774f746d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:42:03 +0100 Subject: [PATCH 07/36] Pin the recovery test on the catch-up frame testRecoveredSlotReplaysDeltaFramesAgainstFreshServer never acked in phase 1, so recovery replayed from the very first frame -- whose delta already starts at id 0. The replayed frames were thus self-sufficient from 0, and the reconstructed-dictionary assertions passed whether or not the seeded catch-up carried the right symbols (or any at all). Only the sawCatchUpFrame existence check was load-bearing. Stamp the ack watermark at FSN DISTINCT_SYMBOLS-1 between the phases so recovery replays from the first frame past the symbol-introducing cycle: a frame with deltaStart=DISTINCT_SYMBOLS carrying no new symbols. The early ids it references now exist only in the persisted dictionary, so the reconstructed dictionary is complete solely because the catch-up re-registered them. Verified both ways: with a catch-up that sends a table-less frame but no symbols, the pre-change test still passes (the head frames carry the dictionary) while the stamped test fails at "dictionary id 0 expected sym-0 but was null". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/DeltaDictRecoveryTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 88afc3ef..46c2a66b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -104,6 +104,20 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep } } + // Ack a prefix so recovery does NOT replay from the self-sufficient head. + // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the + // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN + // DISTINCT_SYMBOLS onward -- frames whose delta starts at + // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle + // reuse existing ids). The early ids those frames reference then exist + // ONLY in the persisted dictionary, so the reconstructed dictionary below + // is complete solely because the catch-up frame re-registered them. That + // pins the content assertions to the catch-up: without it (or with a + // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 + // and the per-id checks would fail. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); + // Phase 2: fresh server that reconstructs its per-connection dictionary // from the delta sections. Sender 2 recovers the slot and replays. DictReconstructingHandler handler = new DictReconstructingHandler(); From 2b78b0e5e0dd80ad7170cc16cea2e731e75a7331 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:42:13 +0100 Subject: [PATCH 08/36] Cover the disk-mode full-dict fallback When a disk-mode slot's .symbol-dict cannot be opened, the engine reports delta encoding as unavailable and the sender must fall back to self-sufficient frames -- every batch re-ships the whole dictionary from id 0 -- because a recovered slot would have no dictionary to rebuild non-self-sufficient deltas from. Nothing exercised that path. Add a test that plants a directory where the dictionary file belongs, so openRW / openCleanRW fail and open() returns null. It then asserts both batches ship deltaStart=0 and that batch 2 re-ships the whole dictionary (deltaCount=2), rather than the monotonic delta (deltaStart=1, deltaCount=1) the enabled path emits. Verified it bites: forcing isDeltaDictEnabled() to stay true regresses batch 2 to deltaStart=1 and the test fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index b0a1b5a7..10bd1e6b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -107,6 +107,62 @@ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { } } + @Test + public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Exception { + // When the per-slot .symbol-dict cannot be opened in disk mode, + // isDeltaDictEnabled() is false and the sender must fall back to + // self-sufficient frames: every batch re-ships the WHOLE dictionary from + // id 0. A recovered / orphan-drained slot then has no dictionary to + // rebuild deltas from, so a monotonic delta would dangle ids on the fresh + // server -- the full-dict frame is the safe degradation. Force the open + // failure by planting a DIRECTORY where the dictionary file belongs: + // openRW / openCleanRW on a directory fails, so open() returns null. + Path sfDir = Files.createTempDirectory("qwp-sf-fallback"); + Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); + Files.createDirectories(dictPath); // a directory, not a file + Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + // The planted directory is untouched -- the dictionary never opened, + // so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 re-ships + // the WHOLE dictionary (alpha + beta), NOT a monotonic delta (which + // would be deltaStart=1, deltaCount=1 as in the test above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } finally { + rmDir(sfDir); + } + } + private static boolean containsUtf8(byte[] haystack, String needle) { byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8); outer: From 282ac09fbc3ffe920435964780506816d02f9899 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:52:59 +0100 Subject: [PATCH 09/36] Truncate the torn tail when reopening the symbol dict openExisting parsed complete entries and set appendOffset past the last one, but left the file at its full length. A crash mid-append leaves a torn trailing record; if the next append after recovery is SHORTER than that torn tail, it overwrites only the tail's prefix and leaves residue beyond its own end. A later recovery can then mis-parse that residue as a ghost symbol, shifting every subsequent dense id -- so the "self-healing tail" guarantee was not actually airtight. open() now truncates the file to the end of the last complete entry (ftruncate) so nothing survives past appendOffset. Best-effort: a failed truncate falls back to the prior overwrite-from-appendOffset behaviour. testTornTrailingEntrySelfHeals now asserts the file returns to its clean length after the reopen; reverting the truncate fails it (19 vs 16 bytes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/sf/cursor/PersistedSymbolDict.java | 14 ++++++++++++-- .../client/sf/cursor/PersistedSymbolDictTest.java | 9 ++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 92a0cc78..15179e89 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -328,8 +328,18 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { } Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; - // appendOffset lands just past the last complete entry, so the next - // append overwrites any torn trailing bytes. + // Physically drop any torn/stale trailing bytes (a crash mid-append) + // so a LATER, shorter append cannot leave residue past its own end + // that a future recovery mis-parses as a ghost symbol -- which would + // shift every subsequent dense id. Without this, appendOffset already + // lands just past the last complete entry, so the next append + // overwrites FROM there, but only truncation guarantees nothing + // survives BEYOND it. Best-effort: a failed truncate just forgoes the + // hardening and falls back to the overwrite-from-appendOffset behaviour. + long validLen = HEADER_SIZE + entriesLen; + if (validLen < len) { + Files.truncate(fd, validLen); + } return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 73549689..df6bab21 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -229,17 +229,24 @@ public void testTornTrailingEntrySelfHeals() throws Exception { d.close(); Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); // Reopen: the torn tail is ignored; only the two complete entries load. PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); try { + // open() physically truncates the torn tail: the file returns + // to its clean length, so a later SHORTER append can never + // leave residue past its end that a future recovery mis-parses + // as a ghost symbol. + Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); Assert.assertEquals(2, re.size()); ObjList s = re.readLoadedSymbols(); Assert.assertEquals("one", s.getQuick(0)); Assert.assertEquals("two", s.getQuick(1)); - // The next append overwrites the torn tail, keeping the file consistent. + // The next append continues from the truncated tail cleanly. re.appendSymbol("three"); Assert.assertEquals(3, re.size()); } finally { From 754eb3d642ce991755bef93dc833b0be114fa7a0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:53:09 +0100 Subject: [PATCH 10/36] Guard the sent-dict mirror against int overflow The I/O thread's lifetime-monotonic symbol-dictionary mirror is sized with int math: accumulateSentDict passed sentDictBytesLen + regionBytes (an int sum) to ensureSentDictCapacity, and the grow step doubled capacity*2, also int. On a pathological, very-high-cardinality connection the sum overflows negative -- so the capacity check passes and copyMemory scribbles past the buffer (silent heap corruption) -- and capacity*2 overflows negative near 1 GB, degrading the doubling to exact-fit reallocs. Reaching this needs ~200M+ distinct symbols on one connection, far past any real workload, but the failure mode is silent corruption. ensureSentDictCapacity now takes a long, the caller passes a long sum, and the method throws a LineSenderException above an int-addressable ceiling (Integer.MAX_VALUE - 8) instead of overflowing, growing in long math clamped to that ceiling. Defensive only -- not reachable at realistic symbol cardinality, so there is no scale test. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2e6b826c..3c7a0421 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,6 +133,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ @@ -2035,19 +2041,32 @@ private void accumulateSentDict(long payloadAddr, int payloadLen) { } } int regionBytes = (int) (p - regionStart); - ensureSentDictCapacity(sentDictBytesLen + regionBytes); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; sentDictCount += (int) deltaCount; } - private void ensureSentDictCapacity(int required) { + private void ensureSentDictCapacity(long required) { if (sentDictBytesCapacity >= required) { return; } - int newCap = Math.max(sentDictBytesCapacity * 2, Math.max(4096, required)); - sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); - sentDictBytesCapacity = newCap; + if (required > MAX_SENT_DICT_BYTES) { + throw new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, (int) newCap, MemoryTag.NATIVE_DEFAULT); + sentDictBytesCapacity = (int) newCap; } private long readVarintAt(long p, long limit) { From 4f44fb3b8d880e416044b41cf1ebde0e348503e7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:09:25 +0100 Subject: [PATCH 11/36] Parse the delta header once per frame on send trySendOne decoded a frame's delta header twice: the pre-send torn-dictionary guard called frameDeltaStart (magic/flags check + start-id varint), then post-send accumulateSentDict re-ran isDeltaFrame and re-read the start id before reading deltaCount. Both run on every delta frame on the I/O send path. Decode the start id once in the guard, hoist the frame address into a local, and pass the start id into accumulateSentDict, which now locates deltaCount just past the canonical start-id encoding (via NativeBufferWriter.varintSize) instead of re-parsing the header. The non-delta-frame case is carried by the same start id (-1), so the post- send mirror update runs exactly when it did before. Also move the accumulateSentDict javadoc onto accumulateSentDict: it had drifted above frameDeltaStart (which kept its own doc), leaving accumulateSentDict undocumented. The per-entry region walk (to size the mirror copy) remains; eliminating it needs a wire-level deltaBytes field, a server-side change out of scope for this client fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3c7a0421..9f41e1e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1978,17 +1978,6 @@ private void setWireBaselineWithCatchUp(long replayStart) { } } - /** - * Copies the symbol-dictionary delta a just-sent frame carries into the - * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can - * re-register it. Frames are sent in FSN order carrying monotonically - * extending deltas, so a frame whose delta starts exactly at - * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame - * (nothing new) is skipped. Only ever called in delta mode. - * - * @param payloadAddr address of the QWP message (12-byte header first) - * @param payloadLen message length in bytes - */ /** * Returns the symbol-dictionary delta start id of a frame, or -1 when the * frame carries no delta section. Used by the pre-send torn-dictionary guard. @@ -2013,14 +2002,27 @@ private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; } - private void accumulateSentDict(long payloadAddr, int payloadLen) { + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { long limit = payloadAddr + payloadLen; - if (!isDeltaFrame(payloadAddr, payloadLen)) { - return; - } - long p = payloadAddr + QwpConstants.HEADER_SIZE; - long deltaStart = readVarintAt(p, limit); - p = varintEnd; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); long deltaCount = readVarintAt(p, limit); p = varintEnd; // Only a delta that extends exactly from our current tip introduces new @@ -2291,6 +2293,11 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // -1 unless this is a delta frame; the guard decodes it once here and + // accumulateSentDict reuses it post-send, so the delta header is parsed + // once per frame rather than twice. + int deltaStart = -1; if (deltaDictEnabled) { // Torn-dictionary guard. In normal operation a delta frame's start id // never exceeds the dictionary coverage established so far (replayed @@ -2302,7 +2309,7 @@ private boolean trySendOne() { // Sending the frame would corrupt the table (the server would null-pad // the missing ids), so fail terminally instead; the unreplayable data // must be resent. - int deltaStart = frameDeltaStart(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + deltaStart = frameDeltaStart(frameAddr, payloadLen); if (deltaStart > sentDictCount) { recordFatal(new LineSenderException( "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " @@ -2312,16 +2319,16 @@ private boolean trySendOne() { } } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } - if (deltaDictEnabled) { + if (deltaStart >= 0) { // Mirror the symbols this frame introduced so a later reconnect can // rebuild the whole dictionary. Idempotent on replay: a frame whose // delta we already hold advances nothing. - accumulateSentDict(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + accumulateSentDict(frameAddr, payloadLen, deltaStart); } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; From 0f9d88f1c31befe03c4702cd08965585f6b96724 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:09:35 +0100 Subject: [PATCH 12/36] Fix stale self-sufficient-frame comments Several comments predated file-mode delta encoding and claimed every cursor frame is self-sufficient with a "symbol-dict delta from id 0". That is now only the fallback: in delta mode (memory mode, and file mode when the persisted dictionary opened) frames carry monotonic deltas that are NOT self-sufficient, and the fresh server's dictionary is re-established by an I/O-thread catch-up frame before replay. The worst offender was the deltaDictEnabled field doc ("Enabled only in memory-mode ... File-mode keeps full self-sufficient frames"), which directly contradicted the feature. Corrected it plus the two ensureConnected call-site comments, the append-failed-path comment, and the wasRecoveredFromDisk field doc (schema stays self-sufficient per frame; the dictionary does not). No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 31 ++++++++++--------- .../client/sf/cursor/CursorSendEngine.java | 6 ++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 54be9ea0..77e85392 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -224,11 +224,14 @@ public class QwpWebSocketSender implements Sender { private boolean deferCommit; // True when the sender emits incremental (delta) symbol dictionaries: each // message carries only symbol ids not yet sent on the wire, rather than the - // full dictionary from id 0. Enabled only in memory-mode, where a reconnect - // replays from the in-process ring and the I/O thread re-registers the whole - // dictionary via a catch-up frame before replaying. File-mode store-and-forward - // keeps full self-sufficient frames (recovery/orphan-drain can replay mid-stream - // to a fresh server, where a partial delta would leave gaps). Set in setCursorEngine. + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after @@ -3386,18 +3389,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3784,7 +3787,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 85015a73..3bc0a185 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -84,9 +84,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered From d89f14f6186e4ca8d96ce1cf250836af3fe7406d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:35:19 +0100 Subject: [PATCH 13/36] Make the delta-dict tests deterministic and leak-checked Two robustness fixes to the delta symbol-dictionary tests. Deterministic synchronization (replaces fixed sleeps): - DeltaDictCatchUpTest waited a fixed 200 ms for the server to close connection 1 before sending batch 2. On a loaded machine that could under-wait and let batch 2 race into connection 1's pre-close window, changing which connection the catch-up lands on. The handler now sets a conn1Closed flag after it closes the socket, and the test waits on that. - DeltaDictRecoveryTest's torn-dictionary test slept a fixed 1 s to let the replay guard fire before close(). It now polls flush() for the latched terminal (close() remains the fallback), so it captures the terminal as soon as it fires -- the run dropped from ~1 s to ~0.3 s. Leak checks: the Sender-based tests allocate native memory (the send-loop mirror, persisted-dict buffers, segment mmaps) but were not wrapped in assertMemoryLeak, unlike the rest of the suite. Wrap all eight methods across the three classes; every one is balanced (they already cleaned up via try-with-resources -- the wrapper now guards against future leaks). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictCatchUpTest.java | 227 +++++++++-------- .../qwp/client/DeltaDictRecoveryTest.java | 238 ++++++++++-------- .../qwp/client/SelfSufficientFramesTest.java | 220 ++++++++-------- 3 files changed, 367 insertions(+), 318 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 1af55bc3..0d2ff9f7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -42,6 +42,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode). *

@@ -61,37 +63,40 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1). // Without catch-up, connection 2's first data frame would carry // deltaStart=1 and the fresh server would never learn id 0. - CatchUpHandler handler = new CatchUpHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); - - // Let the I/O loop observe the server-side close before the next - // batch, so batch 2 is what drives the reconnect + catch-up. - Thread.sleep(200); - - sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.connectionsAccepted.get() >= 2 - && handler.dictFor(2).size() >= 2, 5_000); - } + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); - // The fresh (2nd) connection's dictionary, rebuilt purely from the - // frames it received, must hold both symbols contiguously with no - // null gap -- exactly what the catch-up frame guarantees. - List conn2 = handler.dictFor(2); - Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", - handler.sawZeroTableFrameOnConn2); - Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); - Assert.assertEquals("alpha", conn2.get(0)); - Assert.assertEquals("beta", conn2.get(1)); - } + // Wait until the server has actually closed connection 1 before + // sending batch 2, so batch 2 cannot race into connection 1 and + // must drive the reconnect + catch-up. + waitFor(() -> handler.conn1Closed, 5_000); + + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // The fresh (2nd) connection's dictionary, rebuilt purely from the + // frames it received, must hold both symbols contiguously with no + // null gap -- exactly what the catch-up frame guarantees. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals("alpha", conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + }); } @Test @@ -109,48 +114,50 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do // this: the client refuses to SEND a single-table frame over the cap, and // that data frame is always larger than the bare catch-up entry.) - CapShrinkHandler handler = new CapShrinkHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - handler.setServer(server); - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry - LineSenderException terminal = null; - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); - try { - sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); - sender.flush(); - // The terminal latches on the I/O thread once the reconnect's - // catch-up hits the oversized entry; it surfaces to the producer - // on a subsequent flush. Poll a bounded time for it. The polling - // rows use a small symbol that fits the shrunk cap, so the - // producer-side cap check never fires and flush() surfaces the - // I/O thread's catch-up terminal via checkError. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { + assertMemoryLeak(() -> { + CapShrinkHandler handler = new CapShrinkHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + handler.setServer(server); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry + LineSenderException terminal = null; + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + try { + sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); + sender.flush(); + // The terminal latches on the I/O thread once the reconnect's + // catch-up hits the oversized entry; it surfaces to the producer + // on a subsequent flush. Poll a bounded time for it. The polling + // rows use a small symbol that fits the shrunk cap, so the + // producer-side cap check never fires and flush() surfaces the + // I/O thread's catch-up terminal via checkError. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { try { - sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); - sender.flush(); - Thread.sleep(20); + sender.close(); } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - sender.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; + if (terminal == null) { + terminal = e; + } } } + Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); + Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up")); } - Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); - Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), - terminal.getMessage().contains("during catch-up")); - } + }); } @Test @@ -163,48 +170,50 @@ public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Excep // into several contiguous zero-table frames that the fresh server stitches // back into a complete, gap-free dictionary. final int symbolCount = 40; - SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - // One row per flush so each frame stays under the 160-byte cap; the - // sent dictionary still accumulates all 40 symbols on connection 1. - for (int i = 0; i < symbolCount; i++) { - sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + assertMemoryLeak(() -> { + SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // One row per flush so each frame stays under the 160-byte cap; the + // sent dictionary still accumulates all 40 symbols on connection 1. + for (int i = 0; i < symbolCount; i++) { + sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + sender.flush(); + } + // Wait until connection 1 has learned every symbol, so the sender's + // sent-dictionary mirror (the catch-up source) holds all of them. + waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); + + // Wait until the server has actually closed connection 1 before + // sending batch 2, so batch 2 drives the reconnect + split catch-up. + waitFor(() -> handler.conn1Closed, 5_000); + + sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= symbolCount + 1, 10_000); } - // Wait until connection 1 has learned every symbol, so the sender's - // sent-dictionary mirror (the catch-up source) holds all of them. - waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); - - // Let the I/O loop observe the server-side close before the next - // batch, so batch 2 drives the reconnect + split catch-up. - Thread.sleep(200); - - sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); - sender.flush(); - waitFor(() -> handler.connectionsAccepted.get() >= 2 - && handler.dictFor(2).size() >= symbolCount + 1, 10_000); - } - // Connection 2's dictionary, rebuilt purely from the frames it received, - // must hold every symbol contiguously with no null gap -- the split - // catch-up frames reassemble exactly. - List conn2 = handler.dictFor(2); - Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); - for (int i = 0; i <= symbolCount; i++) { - Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + // Connection 2's dictionary, rebuilt purely from the frames it received, + // must hold every symbol contiguously with no null gap -- the split + // catch-up frames reassemble exactly. + List conn2 = handler.dictFor(2); + Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); + for (int i = 0; i <= symbolCount; i++) { + Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + } + // The catch-up had to span more than one zero-table frame to stay under + // the advertised cap -- that split is the behaviour under test. + Assert.assertTrue("catch-up split into multiple frames (saw " + + handler.zeroTableFramesOnConn2 + ")", + handler.zeroTableFramesOnConn2 >= 2); } - // The catch-up had to span more than one zero-table frame to stay under - // the advertised cap -- that split is the behaviour under test. - Assert.assertTrue("catch-up split into multiple frames (saw " - + handler.zeroTableFramesOnConn2 + ")", - handler.zeroTableFramesOnConn2 >= 2); - } + }); } private static String symbolName(int i) { @@ -252,6 +261,10 @@ private interface BoolCondition { */ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + // Set once the server has closed connection 1. A test waits on this + // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot + // race into connection 1's pre-close window and must land on the reconnect. + volatile boolean conn1Closed; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -284,6 +297,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (connNumber == 1) { Thread.sleep(50); client.close(); + conn1Closed = true; } } catch (IOException | InterruptedException e) { Thread.currentThread().interrupt(); @@ -377,6 +391,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien */ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed). + volatile boolean conn1Closed; volatile int zeroTableFramesOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private final int dropConn1AtDictSize; @@ -417,6 +433,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien conn1Dropped = true; Thread.sleep(50); client.close(); + conn1Closed = true; } } catch (IOException | InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 46c2a66b..8d1bd24a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -46,6 +46,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * End-to-end recovery for the delta symbol dictionary in store-and-forward mode. *

@@ -79,131 +81,149 @@ public void tearDown() { @Test public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception { - // Phase 1: silent server (no acks). Sender 1 writes symbol rows and - // close-fast (no drain), leaving unacked delta frames + a persisted - // dictionary in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - - String pad = TestUtils.repeat("x", 64); - String cfg = "ws::addr=localhost:" + port - + ";sf_dir=" + sfDir - + ";sf_max_bytes=4096" - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < ROWS; i++) { - s1.table("m") - .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) - .stringColumn("p", pad) - .longColumn("v", i) - .atNow(); - s1.flush(); + assertMemoryLeak(() -> { + // Phase 1: silent server (no acks). Sender 1 writes symbol rows and + // close-fast (no drain), leaving unacked delta frames + a persisted + // dictionary in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String pad = TestUtils.repeat("x", 64); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .stringColumn("p", pad) + .longColumn("v", i) + .atNow(); + s1.flush(); + } } } - } - // Ack a prefix so recovery does NOT replay from the self-sufficient head. - // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the - // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN - // DISTINCT_SYMBOLS onward -- frames whose delta starts at - // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle - // reuse existing ids). The early ids those frames reference then exist - // ONLY in the persisted dictionary, so the reconstructed dictionary below - // is complete solely because the catch-up frame re-registered them. That - // pins the content assertions to the catch-up: without it (or with a - // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 - // and the per-id checks would fail. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); - - // Phase 2: fresh server that reconstructs its per-connection dictionary - // from the delta sections. Sender 2 recovers the slot and replays. - DictReconstructingHandler handler = new DictReconstructingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender ignored = Sender.fromConfig(cfg)) { - long deadline = System.currentTimeMillis() + 5_000; - while (System.currentTimeMillis() < deadline - && handler.maxDictSize() < DISTINCT_SYMBOLS) { - Thread.sleep(20); + // Ack a prefix so recovery does NOT replay from the self-sufficient head. + // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the + // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN + // DISTINCT_SYMBOLS onward -- frames whose delta starts at + // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle + // reuse existing ids). The early ids those frames reference then exist + // ONLY in the persisted dictionary, so the reconstructed dictionary below + // is complete solely because the catch-up frame re-registered them. That + // pins the content assertions to the catch-up: without it (or with a + // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 + // and the per-id checks would fail. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); + + // Phase 2: fresh server that reconstructs its per-connection dictionary + // from the delta sections. Sender 2 recovers the slot and replays. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } } - } - // The recovering sender must have re-registered the dictionary via a - // catch-up (0-table) frame before replaying delta frames. - Assert.assertTrue("recovery sent a full-dictionary catch-up frame", - handler.sawCatchUpFrame); - // The reconstructed dictionary must be complete and gap-free: exactly - // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. - List dict = handler.dictSnapshot(); - Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); - for (int i = 0; i < DISTINCT_SYMBOLS; i++) { - Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + // The recovering sender must have re-registered the dictionary via a + // catch-up (0-table) frame before replaying delta frames. + Assert.assertTrue("recovery sent a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + // The reconstructed dictionary must be complete and gap-free: exactly + // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. + List dict = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + } } - } + }); } @Test public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { - // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. - // Silent server + close-fast leaves all frames unacked in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. + // Silent server + close-fast leaves all frames unacked in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } } } - } - // Simulate a host/power crash: the segment frames survive but the persisted - // dictionary is lost, and the ack watermark was left mid-stream. Truncate - // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at - // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); - writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. The replay guard must - // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { - Thread.sleep(1_000); // let the I/O loop attempt replay and hit the guard - } finally { + // Simulate a host/power crash: the segment frames survive but the persisted + // dictionary is lost, and the ack watermark was left mid-stream. Truncate + // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at + // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The replay guard must + // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); try { - s2.close(); - } catch (LineSenderException e) { - terminal = e; + // Poll for the replay guard to fire (it recordFatal's on the I/O + // thread); flush() surfaces the latched terminal to the producer. + // A bounded poll replaces a fixed sleep and captures it as soon as + // it fires; close() below is the fallback if it surfaces only there. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } } + Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", + 0, handler.frames.get()); + Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); } - Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", - 0, handler.frames.get()); - Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); - } + }); } private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 10bd1e6b..af30da83 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -38,6 +38,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Pins down how the symbol dictionary is framed on the wire. *

@@ -64,44 +66,48 @@ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict // so a recovered/orphan-drained slot can rebuild it. Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - // The engine places slot files under sf_dir/ (default "default"). - Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - - // Check the persisted dictionary while the sender is live: a - // fully-drained close intentionally unlinks it (slot cleanup). - Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); - byte[] dict = Files.readAllBytes(dictFile); - Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); - Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); - } + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); + } - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); - // batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } finally { rmDir(sfDir); } @@ -121,43 +127,47 @@ public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); Files.createDirectories(dictPath); // a directory, not a file Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } - // The planted directory is untouched -- the dictionary never opened, - // so delta encoding stayed disabled. - Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", - Files.isDirectory(dictPath)); - - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Full-dict fallback: BOTH batches start at id 0, and batch 2 re-ships - // the WHOLE dictionary (alpha + beta), NOT a monotonic delta (which - // would be deltaStart=1, deltaCount=1 as in the test above). - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", - 0, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", - 2, readVarint(b2, DELTA_START_OFFSET + 1)); + // The planted directory is untouched -- the dictionary never + // opened, so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 + // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic + // delta (which would be deltaStart=1, deltaCount=1 as above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } finally { rmDir(sfDir); } @@ -181,38 +191,40 @@ private static boolean containsUtf8(byte[] haystack, String needle) { public void testMemoryModeShipsMonotonicDelta() throws Exception { // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries // only "beta" as a delta starting at id 1, not the whole dictionary. - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Batch 1 introduces alpha at id 0. - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - - // Batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); - } + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Batch 1 introduces alpha at id 0. + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + + // Batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } private static int readVarint(byte[] buf, int offset) { From 27aad9edc4d4254ff5ce87bf66c75735cc9d8d0b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:38:08 +0100 Subject: [PATCH 14/36] Cover the split-batch delta contract flushPendingRowsSplit fires when one flush's encoded size exceeds the server's batch cap: it emits one frame per table. The first frame must carry the whole batch's symbol-dict delta and advance the baseline, and the remaining frames must carry an empty delta that only references ids the first frame already registered -- otherwise a fresh server would see dangling symbol ids. No test drove that producer-side split. Add a test that buffers two padded tables into one flush under a small advertised cap, so the batch splits, and asserts the first frame ships deltaStart=0/deltaCount=2 while the second ships deltaStart=2/deltaCount=0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index af30da83..cb2b0118 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -227,6 +227,52 @@ public void testMemoryModeShipsMonotonicDelta() throws Exception { }); } + @Test + public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception { + // A single flush whose encoded size exceeds the server's batch cap is + // split into one frame per table (flushPendingRowsSplit). The FIRST split + // frame carries the whole batch's symbol-dict delta and advances the + // baseline; the remaining frames carry an EMPTY delta and only reference + // ids the first frame already registered. A regression that shipped each + // table's own symbols (wrong deltaStart) would dangle ids on the server. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Padding inflates each table past half the cap, so the combined + // two-table message exceeds it while each single-table frame fits. + String pad = new String(new char[60]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Buffer TWO tables (symbols "a" id 0, "b" id 1), then ONE flush. + sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("the oversized two-table batch must split into 2 frames", + 2, handler.batches.size()); + byte[] f1 = handler.batches.get(0); + byte[] f2 = handler.batches.get(1); + + // First split frame ships the whole batch's dictionary (a + b). + Assert.assertEquals("first split frame deltaStart must be 0", + 0, readVarint(f1, DELTA_START_OFFSET)); + Assert.assertEquals("first split frame ships both new symbols", + 2, readVarint(f1, DELTA_START_OFFSET + 1)); + // Second split frame carries an empty delta above the advanced baseline. + Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)", + 2, readVarint(f2, DELTA_START_OFFSET)); + Assert.assertEquals("second split frame carries no new symbols", + 0, readVarint(f2, DELTA_START_OFFSET + 1)); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From 034d8f30c780692c612dd6b93498eb4b36d2e8ad Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 23:12:35 +0100 Subject: [PATCH 15/36] Accumulate the tail of a partial-overlap delta into the mirror accumulateSentDict dropped a frame entirely whenever deltaStart != sentDictCount. A delta that overlaps the mirror tip and extends past it (deltaStart < sentDictCount < deltaStart+deltaCount) was therefore discarded whole -- the new tail symbols never entered the mirror, which would leave a later reconnect catch-up incomplete and shift server-side ids. The producer only ever emits strictly contiguous, non-overlapping deltas, so this is currently unreachable, but it is load-bearing correctness resting on an invariant enforced elsewhere. Handle the overlap: skip the already-held prefix [deltaStart, sentDictCount) and copy only the new tail [sentDictCount, deltaStart+deltaCount). The steady-state case (deltaStart == sentDictCount) has skip == 0, so it is unchanged and free. A gap (deltaStart > sentDictCount, which the torn-dictionary guard rejects before send) now bails explicitly rather than implicitly. Also document that the I/O-thread mirror is a second, native copy of the dictionary (the producer's GlobalSymbolDictionary already holds the same symbols as Java Strings) -- so a memory-mode connection's steady-state dictionary footprint is ~2x the symbol set, an intentional cost of the reconnect-catch-up capability. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 9f41e1e6..3e14de02 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -193,6 +193,13 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // on reconnect it can re-register the whole dictionary on the fresh server // (which discards its dictionary on every disconnect) before replaying frames // whose deltas start above id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed + // for the connection's lifetime (a reconnect may need the whole dictionary at + // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; private long sentDictBytesAddr; private int sentDictBytesCapacity; @@ -2025,15 +2032,38 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); long deltaCount = readVarintAt(p, limit); p = varintEnd; - // Only a delta that extends exactly from our current tip introduces new - // symbols. deltaStart < sentDictCount is a replay/overlap we already hold; - // deltaStart > sentDictCount is rejected before send by the torn-dictionary - // guard in trySendOne, so it never reaches here. - if (deltaCount <= 0 || deltaStart != sentDictCount) { + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { return; } + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + return; // malformed -- bail rather than corrupt the mirror + } + } long regionStart = p; - for (long i = 0; i < deltaCount; i++) { + long newCount = deltaEnd - sentDictCount; + for (long i = 0; i < newCount; i++) { long len = readVarintAt(p, limit); p = varintEnd + len; if (p > limit) { @@ -2050,7 +2080,7 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; - sentDictCount += (int) deltaCount; + sentDictCount += (int) newCount; } private void ensureSentDictCapacity(long required) { From 41fddb11cd409af8a7452c46a220d702be909c80 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 23:12:46 +0100 Subject: [PATCH 16/36] Guard against persisted-dict duplication on a failed publish Regression test for the write-ahead persist path: persistNewSymbolsBefore- Publish runs before the frame is published (sealAndSwapBuffer -> appendBlocking). If publish fails after the persist -- here PAYLOAD_TOO_LARGE (a frame bigger than the SF segment), a backpressure deadline in production -- the symbols are already on disk but sentMaxSymbolId is not advanced and the rows stay buffered, so a retry re-runs the persist. The fix keys the persist range off pd.size() (idempotent); this pins it. The test drives one new-symbol row whose padded frame exceeds a 1 KB segment, flushes it twice (both fail to publish), then asserts the persisted .symbol-dict holds the symbol exactly once. Reverting the fix to sentMaxSymbolId+1 fails it with size 2 -- the duplicate that shifts every later global id and silently misattributes symbol values on recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictRecoveryTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8d1bd24a..143686b4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -26,6 +26,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -226,6 +227,70 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { + // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs + // BEFORE the frame is published (sealAndSwapBuffer -> appendBlocking). If + // publish fails (here PAYLOAD_TOO_LARGE, a frame bigger than the SF + // segment; a backpressure deadline in production), the frame's symbols are + // already on disk but sentMaxSymbolId is NOT advanced and the rows stay + // buffered -- so a retry re-runs the persist. Keying the persist range off + // pd.size() (not sentMaxSymbolId+1) makes it idempotent. Before that fix, + // the retry appended the symbol a SECOND time, breaking the dense + // id->position invariant; on recovery every later global id shifts by one + // and symbol column values are silently misattributed. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + // Small segment; a heavily padded row's frame cannot fit, so + // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no + // backpressure timing needed. The server never acks (SilentHandler). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;"; + String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment + Sender sender = Sender.fromConfig(cfg); + try { + // Buffer ONE new-symbol row, then flush it TWICE. Each flush + // runs the write-ahead persist and then fails to publish; the + // failed flush leaves the row buffered, so the second flush is + // the retry that (pre-fix) duplicated the persisted symbol. + sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow(); + for (int attempt = 0; attempt < 2; attempt++) { + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + // frame too large -- expected on every attempt + } + } + + // The persisted dictionary must hold "s0" EXACTLY ONCE. + // Pre-fix, the retry duplicated it (size == 2). + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", + 1, pd.size()); + Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized row and + // fails again (PAYLOAD_TOO_LARGE); expected here and not + // what we assert. close() still runs its resource cleanup, + // so no native memory leaks. + } + } + } + }); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); From 0ef96afbcf840540c8b45a7ba07c7a33cf750a1e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:43:35 +0100 Subject: [PATCH 17/36] Cover the reconnect catch-up ACK alignment setWireBaselineWithCatchUp anchors fsnAtZero = replayStart - catchUpFrames so every catch-up frame maps to an already-acked FSN. Dropping the - catchUpFrames term is silent data loss: a server ACK for a catch-up frame then translates to an FSN at or above replayStart and trims a not-yet-delivered data frame from the store-and-forward log. The existing catch-up tests reconstruct the dictionary from wire bytes and never assert ACK/trim accounting, so they were blind to this line; the enterprise SqlFailoverQwpClientLosslessTest ingests no symbols and never enters the catch-up path at all. CursorWebSocketSendLoopCatchUpAlignmentTest drives the catch-up against a stub client and asserts the catch-up frame's OK leaves the real engine's ackedFsn untouched, for both a single catch-up frame and a split (multi-frame) catch-up. Reverting the - catchUpFrames term fails both. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 324 ++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java new file mode 100644 index 00000000..ed337f00 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -0,0 +1,324 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +/** + * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in + * {@link CursorWebSocketSendLoop#setWireBaselineWithCatchUp}. + *

+ * On a fresh connection the loop re-registers the whole dictionary with a + * catch-up frame BEFORE replaying data frames. Each catch-up frame consumes a + * wire sequence, so the loop anchors {@code fsnAtZero = replayStart - catchUpFrames} + * to keep every catch-up frame mapped to an already-acked FSN. Dropping the + * {@code - catchUpFrames} term is silent data loss: a server ACK for a catch-up + * frame then translates through {@code engine.acknowledge(fsnAtZero + wireSeq)} + * to an FSN at or above {@code replayStart}, trimming a not-yet-delivered data + * frame from the store-and-forward log. + *

+ * The loop is constructed but never {@link CursorWebSocketSendLoop#start started}; + * the catch-up runs against a stub {@link WebSocketClient} that counts frames, and + * the OK is delivered straight into the inner {@code ResponseHandler} -- the same + * white-box idiom {@code CursorWebSocketSendLoopDurableAckTest} uses, because + * {@code setWireBaselineWithCatchUp} and the wire ports have no public entry point. + * {@link CursorSendEngine#ackedFsn()} is the authoritative trim watermark asserted + * against. + */ +public class CursorWebSocketSendLoopCatchUpAlignmentTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-cursor-catchup-" + System.nanoTime()).toString(); + assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + long find = Files.findFirst(tmpDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(tmpDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(tmpDir); + } + + @Test + public void testCatchUpFrameAckDoesNotAdvanceTrimWatermark() throws Exception { + // Single catch-up frame (server advertises no cap). Two frames were + // acked before the reconnect (ackedFsn=1), FSN 2 is unacked. The catch-up + // frame's OK must NOT advance the watermark past 1 -- it carries no data, + // only the dictionary the fresh server needs before replay. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); // 0 => no cap => one frame + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 3); // FSN 0,1,2 published + engine.acknowledge(1); // ackedFsn=1 => replayStart=2, FSN 2 still unacked + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "s0", "s1"); // sentDictCount=2 => catch-up fires + long replayStart = engine.ackedFsn() + 1L; // = 2 + + invokeSetWireBaselineWithCatchUp(loop, replayStart); + + assertEquals("whole dictionary fits one frame under no cap", + 1, client.framesSent); + + // Behavioural (the harm): the catch-up frame (wire seq 0) is + // OK'd by the fresh server. It carries no data, so it must + // resolve to an already-acked FSN and leave the trim watermark + // untouched -- advancing it would trim the undelivered FSN 2. + deliverOk(loop, 0); + assertEquals("catch-up frame ACK must not advance the trim watermark " + + "(would trim an undelivered data frame -> silent data loss)", + 1L, engine.ackedFsn()); + // Mechanism: the catch-up frames are anchored below replayStart. + assertEquals("fsnAtZero must be anchored catchUpFrames below replayStart", + replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + } finally { + loop.close(); // frees the seeded mirror + the stub client's buffers + } + } + }); + } + + @Test + public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Exception { + // A small advertised cap splits the dictionary across several catch-up + // frames, so the fsnAtZero offset must subtract the full frame count. Ack + // the LAST catch-up wire sequence: it still maps below replayStart. With + // the offset dropped it would translate to replayStart+1 and over-trim. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(40); // budget 12 => one 11-byte symbol per frame + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 5); // FSN 0..4 published + engine.acknowledge(2); // ackedFsn=2 => replayStart=3, FSN 3,4 unacked + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "symbol0000", "symbol0001"); // 11 bytes each -> two frames + long replayStart = engine.ackedFsn() + 1L; // = 3 + + invokeSetWireBaselineWithCatchUp(loop, replayStart); + + assertEquals("cap must split the two symbols across two frames", + 2, client.framesSent); + + // ACK the highest catch-up wire sequence (the last catch-up + // frame). It too must map below replayStart -- with the offset + // dropped it translates to replayStart+1 and over-trims. + deliverOk(loop, client.framesSent - 1); + assertEquals("no catch-up frame ACK may advance the trim watermark", + 2L, engine.ackedFsn()); + assertEquals("fsnAtZero must subtract the full split frame count", + replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + } finally { + loop.close(); + } + } + }); + } + + private static void appendFrames(CursorSendEngine engine, int count) { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + byte[] payload = "frame-bytes-padd".getBytes(StandardCharsets.US_ASCII); + for (int i = 0; i < payload.length; i++) { + Unsafe.getUnsafe().putByte(buf + i, payload[i]); + } + for (int i = 0; i < count; i++) { + engine.appendBlocking(buf, 16); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response + // handler, mimicking the server acking a catch-up frame (which carries no tables). + private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception { + int size = 11; // status(1) + sequence(8) + tableCount(2) + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK); + Unsafe.getUnsafe().putLong(ptr + 1, wireSeq); + Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); + Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); + f.setAccessible(true); + Object handler = f.get(loop); + Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class); + m.setAccessible(true); + m.invoke(handler, ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("setWireBaselineWithCatchUp", long.class); + m.setAccessible(true); + m.invoke(loop, replayStart); + } + + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return new CursorWebSocketSendLoop( + client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + throw new UnsupportedOperationException("test loop is never started"); + }, + 5_000L, 100L, 5_000L, false); + } + + private CursorSendEngine newEngine() { + return new CursorSendEngine(tmpDir, 16384); + } + + private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(loop); + } + + // Populates the loop's native sent-dictionary mirror with {@code symbols} in + // the on-wire [len varint][utf8] layout, so setWireBaselineWithCatchUp sees a + // non-empty dictionary to re-register. loop.close() frees it. + private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) throws Exception { + int total = 0; + for (String s : symbols) { + int len = s.getBytes(StandardCharsets.UTF_8).length; + total += varintSize(len) + len; + } + long addr = Unsafe.malloc(total, MemoryTag.NATIVE_DEFAULT); + long p = addr; + for (String s : symbols) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + p = writeVarint(p, bytes.length); + for (byte b : bytes) { + Unsafe.getUnsafe().putByte(p++, b); + } + } + setField(loop, "sentDictBytesAddr", addr); + setIntField(loop, "sentDictBytesCapacity", total); + setIntField(loop, "sentDictBytesLen", total); + setIntField(loop, "sentDictCount", symbols.length); + } + + private static void setField(Object target, String name, long value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + + private static void setIntField(Object target, String name, int value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setInt(target, value); + } + + private static int varintSize(long value) { + int n = 1; + while (value > 0x7F) { + value >>>= 7; + n++; + } + return n; + } + + private static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + + // Stub transport: completes no real I/O. getServerMaxBatchSize drives the + // catch-up split; sendBinary counts the frames the catch-up emitted. + private static final class CatchUpCapturingClient extends WebSocketClient { + private final int cap; + private int framesSent; + + CatchUpCapturingClient(int cap) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.cap = cap; + } + + @Override + public int getServerMaxBatchSize() { + return cap; + } + + @Override + public int getServerQwpVersion() { + return 1; + } + + @Override + public void sendBinary(long dataPtr, int length) { + framesSent++; + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } +} From ff79b72f9c798b79e8a558314b34932277209d8c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:47:08 +0100 Subject: [PATCH 18/36] Cover the retriable catch-up send containment sendCatchUpChunk throws CatchUpSendException on a transient wire failure instead of calling fail(). From inside the catch-up fail() re-enters connectLoop -- desyncing the fsnAtZero/nextWireSeq wire mapping (a later ACK then trims un-acked store-and-forward frames), or overflowing the stack on a flapping connection -- turning a transient outage into a hard failure. Only the oversized-entry (non-retriable) terminal was covered; the retriable path had no test. testTransientCatchUpSendFailureIsRetriableNotTerminal drives the catch-up against a stub whose sendBinary throws, and asserts the failure surfaces as a retriable CatchUpSendException and leaves the producer-facing error latch clear. Reverting the throw to fail() fails it. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index ed337f00..9b2195d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -39,11 +39,13 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in @@ -173,6 +175,42 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti }); } + @Test + public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception { + // A transient wire failure WHILE shipping the catch-up (the fresh + // connection drops mid-handshake) must surface as a retriable + // CatchUpSendException for the reconnect loop to handle -- it must NOT + // call fail(). From inside the catch-up fail() re-enters connectLoop + // (corrupting the fsnAtZero/nextWireSeq mapping, or overflowing the stack + // on a flapping connection) or, with no reconnect attempt reachable, + // latches a terminal -- turning a transient outage into a hard failure and + // breaking store-and-forward. Only the oversized-entry (non-retriable) + // terminal was covered; this pins the retriable path. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); // sendBinary throws + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + engine.acknowledge(0); // ackedFsn=0 => a real unacked frame exists behind the catch-up + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "s0", "s1"); // non-empty dict => catch-up fires and hits the failing send + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("a transient catch-up send failure must raise a retriable " + + "CatchUpSendException, not be swallowed into fail()/a terminal"); + } catch (InvocationTargetException e) { + assertEquals("transient catch-up send failure must surface as CatchUpSendException", + "CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + // Retriable, not terminal: the producer-facing error latch stays clear. + loop.checkError(); + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { @@ -288,14 +326,22 @@ private static long writeVarint(long addr, long value) { } // Stub transport: completes no real I/O. getServerMaxBatchSize drives the - // catch-up split; sendBinary counts the frames the catch-up emitted. + // catch-up split; sendBinary counts the frames the catch-up emitted, or -- + // when throwOnSend is set -- raises a transient wire error to model the fresh + // connection dropping mid-catch-up. private static final class CatchUpCapturingClient extends WebSocketClient { private final int cap; + private final boolean throwOnSend; private int framesSent; CatchUpCapturingClient(int cap) { + this(cap, false); + } + + CatchUpCapturingClient(int cap, boolean throwOnSend) { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); this.cap = cap; + this.throwOnSend = throwOnSend; } @Override @@ -310,6 +356,9 @@ public int getServerQwpVersion() { @Override public void sendBinary(long dataPtr, int length) { + if (throwOnSend) { + throw new RuntimeException("transient wire failure during catch-up"); + } framesSent++; } From 8fd9ad19a2860b47097b8fe861df2f41c5027474 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:55:59 +0100 Subject: [PATCH 19/36] Tidy catch-up comments and harden two edges Four minor cleanups on the delta symbol-dictionary catch-up, all behaviour-preserving on every reachable path: - The sentDict* field comment said the catch-up mirror is memory-mode only; it is also seeded and used in disk mode on a recovered / orphan-drained slot. Corrected. - positionCursorAt's javadoc said it runs after nextWireSeq was reset to 0, but the catch-up path leaves nextWireSeq past the frames it emitted. Corrected to describe setWireBaselineWithCatchUp anchoring the wire baseline; the method only moves the byte cursor. - The recovery-seed constructor set sentDictCount = pd.size() outside the loadedEntriesLen > 0 block. A recovered slot always has entries when size > 0, so the result is unchanged, but coupling the count to the mirror bytes stops sentDictCount ever claiming symbols the mirror does not hold. - sendDictCatchUp used Integer.MAX_VALUE as the no-cap per-frame budget, so sendCatchUpChunk's int frameLen could overflow on a multi-GB dictionary. Bound it by MAX_SENT_DICT_BYTES, the same ceiling ensureSentDictCapacity enforces. Unreachable at real cardinality (~200M+ symbols); defensive. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3e14de02..7a9086d8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -186,7 +186,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - // Delta symbol dictionary catch-up state (memory-mode only; see swapClient). + // Delta symbol dictionary catch-up state (see swapClient). Active in memory + // mode, and in disk mode on a recovered / orphan-drained slot (the constructor + // seeds sentDict* from the persisted dictionary there). // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that @@ -532,8 +534,12 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, ensureSentDictCapacity(len); Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); } - sentDictCount = pd.size(); } } this.fsnAtZero = fsnAtZero; @@ -1784,8 +1790,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -2133,8 +2142,13 @@ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); // Symbol-bytes budget per frame, leaving room for the 12-byte header and // the two delta-section varints. cap <= 0 means the server advertised no - // limit -> send the whole dictionary in one frame. - int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : Integer.MAX_VALUE; + // limit -- send the whole dictionary in one frame, but still bound the + // budget by MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen + // (HEADER_SIZE + varints + symbolsLen) cannot overflow on a pathological + // multi-GB dictionary (unreachable at real cardinality; defensive). + int budget = cap > 0 + ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) + : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; int framesSent = 0; int chunkStartId = 0; long chunkStartAddr = sentDictBytesAddr; From 8fcd8359089e9b177fc24706a72644a15f72def6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 02:46:13 +0100 Subject: [PATCH 20/36] Fix delta symbol-dict recovery and reconnect bugs Close three ways the delta symbol-dictionary feature could lose or corrupt data on the reconnect and store-and-forward recovery paths. Run the torn-dictionary guard unconditionally. trySendOne gated the guard on deltaDictEnabled, which CursorSendEngine reports false when a recovered disk slot cannot open its persisted dictionary (fd exhaustion, a read-only remount, ENOSPC). The recorded frames are still delta frames, so replaying them against a fresh empty-dictionary server null-padded the missing ids and silently corrupted the table. The guard now decodes the delta start for every frame and fails terminally on a gap regardless of the flag; only the sent-dictionary mirror stays gated. Stop treating a catch-up frame as the head data frame. sendCatchUpChunk advances nextWireSeq, but onClose's poison-strike gate and handleServerRejection's pre-send gate read nextWireSeq > 0 as "a data frame was sent". A transient non-orderly close or NACK after the catch-up but before the first replay frame then charged a poison strike on a frame that never left, and after a few flaps escalated a transient outage to a PROTOCOL_VIOLATION terminal that quarantines an orphan drainer. A new dataFrameSentThisConnection flag, set only after a real ring frame sends, now gates both decisions, so the drainer keeps retrying as Invariant B requires. Bound the commit message's dictionary delta to the sent watermark. sendCommitMessage skips the write-ahead persist yet encoded a delta up to currentBatchMaxSymbolId, so a symbol left in the batch by a cancelled row (cancelRow rolls back neither currentBatchMaxSymbolId nor the global registration) rode out on the commit frame without being persisted. A recovered slot then under-seeded the producer against the surviving frame and misattributed the reused id. The commit now caps the delta at sentMaxSymbolId in delta mode, giving an empty delta. Each fix carries a regression test proven to fail when the fix is reverted: a directory-shadowed .symbol-dict (guard), a close after only the catch-up (poison gate), and a cancelled-row symbol on a transactional commit (delta bound). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 19 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 100 ++++++++----- .../qwp/client/DeltaDictRecoveryTest.java | 133 ++++++++++++++++++ ...ursorWebSocketSendLoopPoisonFrameTest.java | 51 +++++++ 4 files changed, 264 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 77e85392..4a499300 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3623,10 +3623,23 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); - // A commit carries no rows and no new symbols; in delta mode its empty - // delta simply starts at the server's current dictionary size. + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId so the frame stays self-sufficient. Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - symbolDeltaBaseline(), currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 7a9086d8..2772cdf8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -203,6 +203,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up + // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies + // "the head frame was sent": onClose's poison-strike gate and + // handleServerRejection's pre-send gate key off THIS instead. Without it, a + // transient outage AFTER the catch-up but BEFORE the first data frame (a + // flapping LB/middlebox that accepts the upgrade + catch-up then closes) would + // be mistaken for a deterministic head-frame rejection and escalate to a + // PROTOCOL_VIOLATION terminal -- breaking the store-and-forward "retry a + // transient outage forever" contract. Reset per connection in + // setWireBaselineWithCatchUp; set in trySendOne after a successful send. + private boolean dataFrameSentThisConnection; private long sentDictBytesAddr; private int sentDictBytesCapacity; private int sentDictBytesLen; @@ -1980,6 +1992,11 @@ private void swapClient(WebSocketClient newClient) { * first real connection via swapClient. */ private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; if (client != null && deltaDictEnabled && sentDictCount > 0) { this.nextWireSeq = 0L; // The catch-up may span several frames when the dictionary exceeds the @@ -2338,29 +2355,31 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; - // -1 unless this is a delta frame; the guard decodes it once here and - // accumulateSentDict reuses it post-send, so the delta header is parsed - // once per frame rather than twice. - int deltaStart = -1; - if (deltaDictEnabled) { - // Torn-dictionary guard. In normal operation a delta frame's start id - // never exceeds the dictionary coverage established so far (replayed - // frames overlap the catch-up dict; fresh frames extend it - // contiguously). A gap here means the persisted dictionary was torn -- - // almost always by a host/power crash, which leaves segment frames on - // disk but loses recently-written dictionary entries (SF, like the rest - // of the store, is process-crash durable but not host-crash durable). - // Sending the frame would corrupt the table (the server would null-pad - // the missing ids), so fail terminally instead; the unreplayable data - // must be resent. - deltaStart = frameDeltaStart(frameAddr, payloadLen); - if (deltaStart > sentDictCount) { - recordFatal(new LineSenderException( - "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " - + "frame delta start " + deltaStart + " exceeds recovered dictionary size " - + sentDictCount + "; cannot replay without corrupting data -- resend required")); - return false; - } + // Torn-dictionary guard. Decode the delta start unconditionally (-1 for a + // non-delta frame); the guard MUST run even when deltaDictEnabled is false. + // A disk slot recovered with its persisted dictionary unavailable + // (PersistedSymbolDict.open() returned null -- fd exhaustion, a read-only + // remount, ENOSPC) reports deltaDictEnabled=false, yet its recorded frames + // are still DELTA frames (deltaStart > 0). Replaying those against a fresh + // empty-dictionary server would null-pad the missing ids and SILENTLY + // corrupt the table -- precisely what this guard exists to prevent -- so it + // cannot be gated on the very flag that goes false in that failure mode. In + // normal operation a delta frame's start id never exceeds the dictionary + // coverage established so far (replayed frames overlap the catch-up dict; + // fresh frames extend it contiguously), so a gap here means the recovered + // dictionary is incomplete (a host/power crash that lost recently-written + // entries, SF being process-crash but not host-crash durable). Fail + // terminally; the unreplayable data must be resent. Full-dict / fallback + // frames carry deltaStart=0 with sentDictCount=0, so 0 > 0 never + // false-positives; only the sent-dictionary mirror below stays gated on + // deltaDictEnabled. + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; } try { client.sendBinary(frameAddr, payloadLen); @@ -2368,7 +2387,12 @@ private boolean trySendOne() { fail(t); return false; } - if (deltaStart >= 0) { + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaDictEnabled && deltaStart >= 0) { // Mirror the symbols this frame introduced so a later reconnect can // rebuild the whole dictionary. Idempotent on replay: a frame whose // delta we already hold advances nothing. @@ -2619,7 +2643,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2726,17 +2750,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 143686b4..81fd8676 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,139 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { + // C1 regression: when a recovered disk slot's persisted dictionary cannot be + // OPENED (fd exhaustion, a read-only remount, ENOSPC -- simulated here by a + // .symbol-dict that is a DIRECTORY, so both openRW and openCleanRW fail), + // CursorSendEngine.isDeltaDictEnabled() returns false. The recorded frames + // are still DELTA frames, and replaying them against a fresh + // empty-dictionary server would null-pad the missing ids and SILENTLY + // corrupt the table. The torn-dictionary guard must fire regardless of + // deltaDictEnabled -- pre-fix it was gated on that very flag, so the + // corrupting frame sailed through unguarded. Unlike + // testTornDictionaryFailsCleanlyInsteadOfCorrupting (dict present but empty, + // deltaDictEnabled=true), here the dict is UNOPENABLE (deltaDictEnabled=false). + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Make the persisted dictionary UNOPENABLE: replace the .symbol-dict file + // with a directory of the same name so PersistedSymbolDict.open() returns + // null (both openRW and openCleanRW fail) and the engine reports + // deltaDictEnabled=false. Stamp the watermark at FSN 2 so replay starts + // at FSN 3 -- a frame whose delta starts at id 3, with ids 0..2 living + // only in the now-unreadable dictionary. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The guard must fire + // (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertEquals("no delta frame may be replayed when the persisted dictionary is unopenable", + 0, handler.frames.get()); + Assert.assertNotNull("an unopenable dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); + } + }); + } + + @Test + public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception { + // C3 regression: sendCommitMessage does NOT write-ahead-persist the + // dictionary, so its frame must carry NO new symbol. A symbol left in the + // batch by a cancelled row -- cancelRow rolls back neither + // currentBatchMaxSymbolId nor the global-dictionary registration -- must not + // ride out on the commit frame: doing so puts an id on the wire that a + // recovered slot cannot rebuild from .symbol-dict, diverging the producer + // dictionary from the surviving frames and silently misattributing reused + // ids after a crash. The commit's delta must be bounded by sentMaxSymbolId + // (empty here), not currentBatchMaxSymbolId. Memory mode suffices to observe + // the wire behaviour; close() drains every frame to the server first. + assertMemoryLeak(() -> { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Transactional + autoFlushRows=1: every completed row auto-flushes + // as a DEFERRED batch (setting hasDeferredMessages); an explicit + // flush() then emits the commit message. + Sender sender = Sender.builder("ws::addr=localhost:" + port + ";") + .transactional(true) + .autoFlushRows(1) + .build(); + try { + // Row 1 registers "a"@0 and auto-flushes it deferred. + sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow(); + // Register "b"@1 on a row that is then cancelled: "b" stays in + // the global dictionary and currentBatchMaxSymbolId advances to + // 1, but nothing persists or sends it. + sender.table("m").symbol("s", "b"); + sender.cancelRow(); + // Commit the deferred batch. The commit frame must carry an + // EMPTY delta -- NOT "b"@1. + sender.flush(); + } finally { + sender.close(); // drains every frame (incl. the commit) to the server + } + + // The server's reconstructed dictionary must hold ONLY "a". Pre-fix + // the commit shipped "b"@1, so the server saw a second symbol. + List dict = handler.dictSnapshot(); + Assert.assertEquals("commit frame must not ship the cancelled row's leaked symbol " + + "(recovery would then diverge from the persisted dictionary): " + dict, + 1, dict.size()); + Assert.assertEquals("a", dict.get(0)); + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 34ae5518..6a3d8079 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -244,6 +244,41 @@ public void testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine() throws Exception }); } + @Test + public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception { + // C2 regression: the dictionary catch-up advances nextWireSeq WITHOUT + // sending a data frame. A non-orderly close in that window -- a flapping + // LB/middlebox that completes the upgrade, accepts the catch-up, then drops + // before the first replay frame -- must be strike-EXEMPT. Keying the + // poison-strike gate off nextWireSeq > 0 (rather than + // dataFrameSentThisConnection) charges a strike on a frame that was never + // sent; MAX_REJECTIONS such closes then escalate a TRANSIENT outage to a + // PROTOCOL_VIOLATION terminal, hard-failing the producer and quarantining an + // orphan drainer -- exactly what store-and-forward's retry-forever contract + // forbids. Mirror image of testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine, + // which sends a real data frame (setSentCount) and DOES escalate. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + for (int i = 0; i < MAX_REJECTIONS + 2; i++) { + // Model the catch-up: nextWireSeq advanced, but NO data frame + // sent. swapClient resets both on every recycle, so re-apply + // before each close. Pre-fix, each of these lands a strike on the + // never-sent head frame and the loop terminals by now. + setCatchUpWireSeqOnly(loop, 2); + deliverNonOrderlyClose(loop); + } + // No strike was ever charged, so nothing escalated: the loop stays + // retriable and the producer-facing error latch is clear. + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // A reachable, healthy server that NACKs the head frame (RETRIABLE) @@ -969,6 +1004,22 @@ private static void setSentCount(CursorWebSocketSendLoop loop, long count) throw Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); f.setAccessible(true); f.setLong(loop, count); + // A non-zero sent count models DATA frames sent on this connection. The + // poison-strike / pre-send gates key off dataFrameSentThisConnection, not + // nextWireSeq (the dictionary catch-up advances nextWireSeq without sending + // a data frame), so keep the two in sync for these white-box scenarios. + Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); + d.setAccessible(true); + d.setBoolean(loop, count > 0); + } + + // Sets ONLY nextWireSeq -- deliberately NOT dataFrameSentThisConnection -- to + // model the dictionary catch-up having advanced the wire sequence with no data + // frame sent yet. Contrast setSentCount, which sets both. + private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long count) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); + f.setAccessible(true); + f.setLong(loop, count); } private static long[] txns(long... v) { From fc8b4ba64dd0e7a52957891bcf44217e361f7774 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 03:06:18 +0100 Subject: [PATCH 21/36] Adopt recovered dictionary into the send mirror On recovery the send loop copied the persisted dictionary's loaded-entries buffer into a fresh mirror allocation and left PersistedSymbolDict holding a second copy for the engine's lifetime -- roughly twice the dictionary size in native memory on a high-cardinality recovered slot, retained long after the one-time seed. The loop now adopts that buffer as its mirror backing via takeLoadedEntries(), which transfers ownership so the dictionary no longer retains or frees it. The producer's readLoadedSymbols() is the only other consumer and runs first (setCursorEngine seeds the producer before the loop is built; the drainer has no producer consumer), guarded by an assert. Add a recover-then-continue-ingest test. A file-mode sender writes symbols and crashes; a fresh sender recovers the slot and ingests a NEW symbol. It asserts the producer continues the dictionary from the recovered size instead of colliding at id 0, exercising seedGlobalDictionaryFromPersisted, which no prior test drove past recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 12 +++- .../client/sf/cursor/PersistedSymbolDict.java | 37 ++++++++++-- .../qwp/client/DeltaDictRecoveryTest.java | 56 +++++++++++++++++++ 3 files changed, 97 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2772cdf8..f5f21207 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -543,8 +543,16 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, if (pd != null && pd.size() > 0) { int len = pd.loadedEntriesLen(); if (len > 0) { - ensureSentDictCapacity(len); - Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + // Adopt the persisted dictionary's loaded-entries buffer as the + // mirror's initial backing instead of copying it and leaving the + // dictionary to retain a second copy for the engine's lifetime. + // The producer's readLoadedSymbols() -- the only other consumer -- + // runs earlier in setCursorEngine; the drainer path has no + // producer consumer. takeLoadedEntries() transfers ownership, so + // this loop's mirror lifecycle frees it (not pd.close()). The + // buffer's allocated size equals loadedEntriesLen. + sentDictBytesAddr = pd.takeLoadedEntries(); + sentDictBytesCapacity = len; sentDictBytesLen = len; // Set the count only alongside the bytes so sentDictCount can // never claim symbols the mirror does not hold. A recovered slot diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 15179e89..dbb3593c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -94,14 +94,19 @@ public final class PersistedSymbolDict implements QuietCloseable { private static final int MAX_ENTRY_LEN = 1 << 20; private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; - // In-memory copy of the entry region [len][utf8]... exactly as on disk, - // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. Consumed once to - // seed the send loop's catch-up mirror and the producer's id map. - private final long loadedEntriesAddr; - private final int loadedEntriesLen; private long appendOffset; private boolean closed; + // In-memory copy of the entry region [len][utf8]... exactly as on disk, + // populated only when open() recovered existing entries (recovery / + // orphan-drain). Zero/empty for a freshly created file. Consumed once to seed + // the producer's id map (readLoadedSymbols) and then ADOPTED by the send loop's + // catch-up mirror (takeLoadedEntries), which takes ownership so this file no + // longer retains a second copy of the dictionary for the engine's lifetime. + private long loadedEntriesAddr; + private int loadedEntriesLen; + // True once takeLoadedEntries() handed loadedEntriesAddr to the send-loop + // mirror; guards a stale readLoadedSymbols() call (which must run first). + private boolean loadedEntriesTaken; private long scratchAddr; private int scratchCap; private int size; @@ -256,6 +261,7 @@ public int loadedEntriesLen() { * recovered. */ public ObjList readLoadedSymbols() { + assert !loadedEntriesTaken : "readLoadedSymbols() must run before takeLoadedEntries()"; ObjList out = new ObjList<>(Math.max(size, 1)); long p = loadedEntriesAddr; long limit = p + loadedEntriesLen; @@ -286,6 +292,25 @@ public int size() { return size; } + /** + * Transfers ownership of the loaded-entries buffer to the caller and returns + * its base address (0 when nothing was recovered). The send loop adopts it as + * the initial backing of its catch-up mirror rather than copying it, so this + * file stops retaining a second copy of the dictionary for the engine's + * lifetime. After this call {@link #close()} no longer frees the buffer -- the + * caller owns it. The producer's {@link #readLoadedSymbols()} is the only other + * consumer and MUST run first: {@code setCursorEngine} seeds the producer + * before the send loop is constructed, and the drainer path has no producer + * consumer. The buffer was allocated with {@link MemoryTag#NATIVE_DEFAULT}. + */ + public long takeLoadedEntries() { + long addr = loadedEntriesAddr; + loadedEntriesAddr = 0L; + loadedEntriesLen = 0; + loadedEntriesTaken = true; + return addr; + } + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 81fd8676..8174ed24 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,62 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { + // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's + // dictionary and delta baseline from the persisted .symbol-dict, so a + // recovered sender that continues ingesting assigns the NEXT id, not a + // colliding low one. Without it the new symbol reuses a recovered id and the + // fresh server sees a redefinition -> silent misattribution. No prior test + // ingests on the recovered sender. Replay is from FSN 0 (no acks), so the + // recovered frames legitimately overlap the seeded dictionary -- this also + // pins that the redefinition guard does not false-positive on normal + // recovery. + assertMemoryLeak(() -> { + // Phase 1: ingest DISTINCT_SYMBOLS symbols, silent server, close-fast -> + // unacked frames + a full persisted dictionary. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096;close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Phase 2: recover against a fresh server, then ingest a genuinely NEW + // symbol. The producer must continue at id DISTINCT_SYMBOLS. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "brand-new").longColumn("v", 99L).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS + 1) { + Thread.sleep(20); + } + } + List dict = handler.dictSnapshot(); + Assert.assertEquals("recovered sender must continue the dictionary, not collide: " + dict, + DISTINCT_SYMBOLS + 1, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("sym-" + i, dict.get(i)); + } + Assert.assertEquals("brand-new", dict.get(DISTINCT_SYMBOLS)); + } + }); + } + @Test public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { // C1 regression: when a recovered disk slot's persisted dictionary cannot be From 8dd7723789615b679b5047ad74778c16567a264e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 09:54:44 +0100 Subject: [PATCH 22/36] Harden delta symbol-dict recovery and NACK gating Two fixes surfaced by review of the delta symbol-dictionary change. Recover oversized persisted symbols (C1). PersistedSymbolDict.openExisting rejected any entry larger than a fixed 1 MB ceiling as "oversized" and truncated the dictionary at that id, but the write path (appendSymbol/appendSymbols) caps nothing. A symbol value over 1 MB therefore persisted fine, yet a normal process-crash recovery dropped it and every higher id; the send loop's replay guard then fired a spurious "host crash / resend required" terminal -- a store-and-forward process-crash-durability violation on a reachable input (nothing caps symbol value length). Drop the fixed per-entry ceiling and bound entries only by the file length, which is the actual corruption guard and was already present. Keep the length in a long so a corrupt multi-gigabyte varint cannot wrap an int back under the check. The write and read paths now agree, so a legitimately persisted large symbol recovers. Guard the catch-up NACK pre-send gate (C2). handleServerRejection keys its pre-send branch off dataFrameSentThisConnection, because the symbol catch-up advances nextWireSeq without sending a data frame. That branch had no regression test: every existing NACK test sets both flags together, so reverting the gate to the old nextWireSeq-based predicate left the suite green. Add the server-NACK twin of testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike -- a WRITE_ERROR NACK of a catch-up frame must take the pre-send path (surface and recycle, no strike), not escalate a transient outage to a producer-fatal PROTOCOL_VIOLATION terminal. Both regression tests fail with their production line reverted and pass with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 19 +++++--- ...ursorWebSocketSendLoopPoisonFrameTest.java | 36 +++++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 45 +++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index dbb3593c..ca769181 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -89,9 +89,6 @@ public final class PersistedSymbolDict implements QuietCloseable { static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; static final byte VERSION = 1; - // Guards against a hostile/corrupt varint length driving a huge allocation - // or a runaway parse. Symbols are short; this is a generous ceiling. - private static final int MAX_ENTRY_LEN = 1 << 20; private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; private long appendOffset; @@ -337,12 +334,20 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (vr == null) { break; // torn length varint } - int entryLen = (int) vr[0]; + long entryLen = vr[0]; int next = (int) vr[1]; - if (entryLen < 0 || entryLen > MAX_ENTRY_LEN || (long) next + entryLen > len) { - break; // torn/oversized entry -- self-healing tail + // The file length is the sole sanity bound: a length that runs past + // the file is a torn/incomplete trailing entry and stops the parse + // (self-healing tail). There is deliberately NO fixed per-entry + // ceiling -- the write path (appendSymbol/appendSymbols) applies none + // either, so a legitimately persisted large symbol must recover here, + // not be truncated and then trip the send loop's "resend required" + // guard on a normal process-crash recovery. entryLen stays a long so a + // corrupt multi-gigabyte length cannot wrap an int back under the check. + if ((long) next + entryLen > len) { + break; // torn/incomplete trailing entry -- self-healing tail } - pos = next + entryLen; + pos = next + (int) entryLen; count++; } int entriesLen = pos - HEADER_SIZE; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 6a3d8079..ab200e80 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -279,6 +279,42 @@ public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception }); } + @Test + public void testNonOrderlyRejectionAfterOnlyCatchUpDoesNotStrike() throws Exception { + // C2 regression: the server-NACK twin of + // testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike. handleServerRejection's + // pre-send gate keys off dataFrameSentThisConnection, NOT nextWireSeq -- the + // dictionary catch-up advances nextWireSeq WITHOUT sending a data frame. A + // server NACK of a catch-up frame (nextWireSeq>0, no data frame sent) must + // take the pre-send path (surface + recycle, no strike), not the post-send + // poison-strike path. Pre-fix (gate on highestSent >= 0, i.e. nextWireSeq>0) + // each catch-up NACK strikes the never-sent head frame; MAX_REJECTIONS such + // strikes escalate a TRANSIENT outage to a producer-fatal PROTOCOL_VIOLATION + // terminal -- exactly what store-and-forward's retry-forever contract forbids. + // WRITE_ERROR (RETRIABLE) is the discriminating category: it DOES accrue + // strikes on the post-send path (see testNackRecycleIsPacedAgainstHealthyServer), + // so a regressed gate escalates here and checkError() throws. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + for (int i = 0; i < MAX_REJECTIONS + 2; i++) { + // Model the catch-up: nextWireSeq advanced, but NO data frame + // sent. The pre-send recycle (swapClient) resets nextWireSeq, so + // re-apply before each NACK (mirrors the onClose twin). + setCatchUpWireSeqOnly(loop, 2); + deliverRetriableNack(loop, 1, "disk full"); + } + // No strike was ever charged, so nothing escalated: the loop stays + // retriable and the producer-facing error latch is clear. + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // A reachable, healthy server that NACKs the head frame (RETRIABLE) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index df6bab21..d580f886 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.ObjList; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -198,6 +199,50 @@ public void testEmptySymbolRoundTrips() throws Exception { }); } + @Test + public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { + // C1 regression: the write path caps nothing, so a symbol larger than the + // old fixed 1 MB read ceiling must still recover intact. Before the fix, + // appendSymbol wrote the oversized entry but openExisting rejected it as + // "oversized", truncated the dictionary at that id (dropping it and every + // higher id), and a normal process-crash recovery then hard-failed with a + // spurious "host crash / resend required" terminal -- defeating store-and- + // forward's process-crash durability for large symbols. The file length is + // now the only bound, so the write and read paths agree. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Just over the old 1 << 20 (1 MB) ceiling. + String big = TestUtils.repeat("x", (1 << 20) + 17); + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol("before"); + d.appendSymbol(big); + d.appendSymbol("after"); + Assert.assertEquals(3, d.size()); + } finally { + d.close(); + } + + // Recovery must load ALL three; pre-fix the reopen truncated at the + // big entry and came back with size 1 (only "before" survived). + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals("large entry must survive recovery, not be truncated", + 3, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("before", s.getQuick(0)); + Assert.assertEquals(big, s.getQuick(1)); + Assert.assertEquals("after", s.getQuick(2)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { From 8853d8fd5fe922ce6b27b475b84e315dac9271e1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 10:01:30 +0100 Subject: [PATCH 23/36] Persist symbol delta from frame, skip re-encode M1: persistNewSymbolsBeforePublish re-encoded each new symbol into a scratch buffer even though beginMessage had just written the identical [len][utf8] bytes into the frame's symbol-dict delta section. The encoder now records that entry region's offset and length; the producer writes those bytes straight to the slot .symbol-dict via a new PersistedSymbolDict.appendRawEntries, so a high-cardinality batch no longer re-encodes what the wire frame already carries. The common path (durable size == the frame's delta start id) takes the direct copy; a retry after a failed publish, where the durable size has run ahead of the wire baseline, falls back to re-encoding just the remaining suffix. The persisted bytes are byte-identical either way. M2: document, on the sendDictCatchUp oversized-entry terminal, that in a heterogeneous or rolling-cap cluster a symbol accepted under a larger cap can hit this hard stop on failover to a smaller-cap node and will not self-recover if a later node advertises a larger cap. Behavior is unchanged -- the terminal keeps the homogeneous common case livelock-free; this only records the tradeoff and the two ways to relax it (an ingest-side symbol-size bound, or a settle budget across reconnects before latching). Adds testAppendRawEntriesMatchesAppendSymbols; the file-mode persist, recovery, and failed-publish suites cover both the direct-copy and re-encode branches end to end. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketEncoder.java | 26 +++++++++ .../qwp/client/QwpWebSocketSender.java | 45 ++++++++++----- .../sf/cursor/CursorWebSocketSendLoop.java | 9 +++ .../client/sf/cursor/PersistedSymbolDict.java | 24 ++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 56 +++++++++++++++++++ 5 files changed, 146 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..8013330a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -39,6 +39,14 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaEntriesEnd; + private int deltaEntriesStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -75,10 +83,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -122,6 +132,22 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4a499300..942af99b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3681,24 +3681,41 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write. + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. // // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: - // appendSymbols advances pd.size() only after a full write, whereas + // the persist advances pd.size() only after a full write, whereas // sentMaxSymbolId only advances after the WHOLE frame is published (via // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist - // threw (a short write -- disk full/quota), the frame was not published - // and sentMaxSymbolId stayed put, while the symbols before the failing - // one are already on disk. Keying the resume point off sentMaxSymbolId+1 - // would then re-append that persisted prefix on the retry, duplicating - // entries and corrupting the dense id->symbol mapping recovery relies on - // (position i must be symbol id i). pd.size() resumes exactly past what is - // already durable (the write overwrites any torn trailing bytes at - // appendOffset), so the write-ahead is idempotent under retry. In the - // happy path pd.size() equals sentMaxSymbolId+1, so the range -- and the - // behaviour -- are unchanged; the single positioned write just replaces - // the previous one-syscall-per-new-symbol loop. - pd.appendSymbols(globalSymbolDictionary, pd.size(), currentBatchMaxSymbolId); + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } } private void resetSymbolDictStateForNewConnection() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index f5f21207..6b9eee60 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2192,6 +2192,15 @@ private int sendDictCatchUp() { // Latch a terminal (the data must be resent after the cap is // raised) rather than calling fail() -- which, from inside the // catch-up, would re-enter connectLoop (see CatchUpSendException). + // + // Tradeoff (heterogeneous / rolling-cap clusters): a symbol + // accepted under a larger/absent cap can hit this on failover to a + // smaller-cap node, and the hard terminal does NOT self-recover if + // a later node advertises a larger cap -- the producer must be + // resumed after the data is resent (or the cap raised). Bounding + // symbol size at ingest, or a settle budget across reconnects + // before latching, would relax this, but both are larger changes; + // the terminal keeps the homogeneous common case livelock-free. LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index ca769181..55c04ba7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -148,6 +148,30 @@ public static void removeOrphan(String slotDir) { Files.remove(slotDir + "/" + FILE_NAME); } + /** + * Appends {@code count} already-encoded entries -- {@code [len varint][utf8]...}, + * the exact layout {@link #appendSymbols} produces -- verbatim from {@code addr} + * in a single write. The producer uses this when it already holds those bytes + * (the symbol-dict delta section the frame encoder just wrote), so it does not + * re-encode the same symbols. Writes {@code len} bytes and advances {@code size} + * by {@code count}. Same durability/idempotency contract as {@link #appendSymbols}: + * no fsync, and a short write throws WITHOUT advancing {@code size}/{@code + * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range + * at the same offset. No-op when the range is empty or the dictionary is closed. + */ + public void appendRawEntries(long addr, int len, int count) { + if (closed || count <= 0 || len <= 0) { + return; + } + long written = Files.write(fd, addr, len, appendOffset); + if (written != len) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + len + ", actual=" + written + ']'); + } + appendOffset += len; + size += count; + } + /** * Appends one symbol, extending the on-disk dictionary. The caller appends a * frame's new symbols BEFORE publishing that frame, so the write ordering diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index d580f886..67fd7438 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -91,6 +91,62 @@ public void testAppendPersistsAcrossReopen() throws Exception { }); } + @Test + public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { + // M1: the producer persists the frame's already-encoded delta bytes via + // appendRawEntries instead of re-encoding the symbols. Those bytes are the + // same [len][utf8]... layout appendSymbols writes, so both must produce an + // identical, recoverable dictionary. Encode a range with appendSymbols, + // reopen to grab its on-disk entry bytes, replay them through + // appendRawEntries into a fresh dict, and assert the recovered symbols + // match -- including an empty entry mid-range. + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 2); + encoded.close(); + + // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, + // then replay it byte-for-byte into a fresh dict via appendRawEntries. + PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + try { + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); + try { + raw.appendRawEntries(reopened.loadedEntriesAddr(), + reopened.loadedEntriesLen(), reopened.size()); + Assert.assertEquals(3, raw.size()); + } finally { + raw.close(); + } + } finally { + reopened.close(); + } + + // The raw-appended dict must recover the same dense symbols. + PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString()); + try { + Assert.assertEquals(3, recovered.size()); + ObjList symbols = recovered.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + } finally { + recovered.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // appendSymbols persists a whole id range in one write (the hot-path From 10a125c05d169dcb87dc38eeeab73d28621add70 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 11:10:00 +0100 Subject: [PATCH 24/36] Guard catch-up frame overflow, add biting tests Two defensive spots in the delta symbol-dict path had no biting test, and one guard was incomplete (M3). Complete the catch-up frame-size guard (M3b). The int-overflow hardening covered ensureSentDictCapacity (the mirror growth) but not the single catch-up frame: with no server cap, frameLen = HEADER + varints + symbolsLen is an int and would wrap negative as symbolsLen approaches the mirror ceiling, feeding a bad Unsafe.malloc. sendDictCatchUp's budget already keeps each chunk under that bound, so this is unreachable at real cardinality -- but the guard must be local so a future caller cannot overflow it silently. Compute the size in long and fail loud (CatchUpSendException) before the malloc, matching the mirror-side guard. Cover accumulateSentDict's partial-overlap tail (M3a). A delta that straddles the mirror tip (deltaStart < sentDictCount < deltaEnd) must copy only the new tail, not drop the whole frame. The monotonic producer never emits a straddling delta in steady state, so reverting to the pre-fix drop-whole-frame guard passed every test; it is reachable on a torn-dict replay (mirror seeded smaller than a frame's coverage), where dropping the tail would leave the reconnect catch-up incomplete and shift server ids. Add a white-box test that drives the straddle directly. Both new tests fail with their production line reverted (the mirror stays at 1 id; the frame guard falls through to a negative malloc that throws IllegalArgumentException, not the clean terminal) and pass with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 18 ++- ...WebSocketSendLoopCatchUpAlignmentTest.java | 132 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6b9eee60..6d8ed64b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2234,10 +2234,24 @@ private int sendDictCatchUp() { * the caller turns it into a single, non-re-entrant reconnect. */ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { - int payloadLen = NativeBufferWriter.varintSize(deltaStart) + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + NativeBufferWriter.varintSize(deltaCount) + symbolsLen; - int frameLen = QwpConstants.HEADER_SIZE + payloadLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int frameLen = (int) frameLenL; long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); try { Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 9b2195d9..79640f25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -43,8 +43,12 @@ import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -211,6 +215,76 @@ public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Excep }); } + @Test + public void testAccumulateSentDictPartialOverlapExtendsMirror() throws Exception { + // M3: accumulateSentDict must handle a delta that STRADDLES the mirror tip + // (deltaStart < sentDictCount < deltaStart+deltaCount) by copying only the + // new tail, not dropping the whole frame. The monotonic producer never emits + // a straddling delta in steady state (so the pre-fix drop-whole-frame guard + // passed every test), but a torn-dict replay can seed the mirror smaller than + // a frame's coverage. Seed the mirror with 1 symbol, feed a [0..2] delta, and + // assert the mirror extends to all 3 -- pre-fix it stayed at 1, leaving the + // reconnect catch-up incomplete and shifting server-side ids. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "aa"); // sentDictCount = 1, mirror holds "aa" + int[] frameLen = new int[1]; + long frame = buildDeltaFrame(0, new String[]{"aa", "bb", "cc"}, frameLen); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "accumulateSentDict", long.class, int.class, int.class); + m.setAccessible(true); + m.invoke(loop, frame, frameLen[0], 0); + } finally { + Unsafe.free(frame, frameLen[0], MemoryTag.NATIVE_DEFAULT); + } + assertEquals("straddling delta must extend the mirror to all 3 ids", + 3, readInt(loop, "sentDictCount")); + assertEquals("mirror must hold the two new tail symbols after the " + + "already-held prefix, gap-free", + Arrays.asList("aa", "bb", "cc"), readMirrorSymbols(loop)); + } finally { + loop.close(); + } + } + }); + } + + @Test + public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { + // M3: sendDictCatchUp caps each chunk under the budget, so the single-frame + // catch-up path cannot overflow its int frameLen at any real cardinality. The + // guard must still be LOCAL -- a future caller must not be able to feed a + // wrapped-negative frameLen to Unsafe.malloc. An oversized symbolsLen must + // fail loud (CatchUpSendException) BEFORE the malloc; the guard fires before + // symbolsAddr is read, so a dummy address is fine. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "sendCatchUpChunk", int.class, int.class, long.class, int.class); + m.setAccessible(true); + // symbolsLen past the mirror ceiling: HEADER + varints + symbolsLen + // overflows an int, so the guard must reject it before malloc. + m.invoke(loop, 0, 1, 0L, Integer.MAX_VALUE - 4); + fail("an overflowing catch-up frame size must fail loud, not malloc negative"); + } catch (InvocationTargetException e) { + assertEquals("overflow must surface as CatchUpSendException", + "CatchUpSendException", e.getCause().getClass().getSimpleName()); + assertTrue("message must name the frame-size guard: " + e.getCause().getMessage(), + e.getCause().getMessage().contains("catch-up frame exceeds the maximum size")); + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { @@ -226,6 +300,64 @@ private static void appendFrames(CursorSendEngine engine, int count) { } } + // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount + // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict + // skips the header, so its content is irrelevant; the caller frees the frame. + private static long buildDeltaFrame(int deltaStart, String[] symbols, int[] outLen) { + int deltaCount = symbols.length; + int size = 12 + varintSize(deltaStart) + varintSize(deltaCount); + for (String s : symbols) { + size += varintSize(s.getBytes(StandardCharsets.UTF_8).length) + + s.getBytes(StandardCharsets.UTF_8).length; + } + long addr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + long p = writeVarint(addr + 12, deltaStart); + p = writeVarint(p, deltaCount); + for (String s : symbols) { + byte[] b = s.getBytes(StandardCharsets.UTF_8); + p = writeVarint(p, b.length); + for (byte x : b) { + Unsafe.getUnsafe().putByte(p++, x); + } + } + outLen[0] = size; + return addr; + } + + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(loop); + } + + // Parses the loop's native sent-dictionary mirror ([len varint][utf8]...) back + // into the symbol strings a reconnect catch-up would re-register. + private static List readMirrorSymbols(CursorWebSocketSendLoop loop) throws Exception { + long addr = readLong(loop, "sentDictBytesAddr"); + int len = readInt(loop, "sentDictBytesLen"); + List out = new ArrayList<>(); + long p = addr; + long limit = addr + len; + while (p < limit) { + long l = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + l |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + byte[] bytes = new byte[(int) l]; + for (int i = 0; i < l; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(p++); + } + out.add(new String(bytes, StandardCharsets.UTF_8)); + } + return out; + } + // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response // handler, mimicking the server acking a catch-up frame (which carries no tables). private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception { From fa50e81f0288077d2e0040314fb77a0ce5a780b7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 12:26:08 +0100 Subject: [PATCH 25/36] Fix orphan-drain losing symbol-dict mirror seed The send loop seeded its reconnect catch-up mirror by calling PersistedSymbolDict.takeLoadedEntries(), a one-shot ownership transfer that zeroed the dictionary's loaded-entries buffer. The foreground sender builds a single loop, so that was safe. But BackgroundDrainer builds a fresh send loop per wire session against the SAME, persistent engine when a durable-ack capability gap forces a mid-drain recycle. The second loop then found an empty loaded-entries buffer, seeded an empty mirror (sentDictCount = 0), sent no reconnect catch-up, and the first replayed delta frame (deltaStart > 0) tripped the torn-dict guard -- falsely quarantining a healthy slot with a bogus "resend required" terminal and defeating the capability-gap settle-retry for delta slots. The send loop now COPIES the loaded entries into its own mirror and leaves the dictionary owning its buffer for the engine's lifetime, so every recycled loop re-seeds. PersistedSymbolDict.close() frees the dictionary's copy; each loop frees its own copy on exit. Peak footprint is unchanged (the drainer closes loop N before building loop N+1). Removes the now-unused takeLoadedEntries() and loadedEntriesTaken, and adds a regression test that builds two loops against one recovered engine and asserts the second re-seeds (pre-fix its mirror was empty). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 24 ++++--- .../client/sf/cursor/PersistedSymbolDict.java | 33 ++------- ...CursorWebSocketSendLoopMirrorLeakTest.java | 70 +++++++++++++++++++ 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6d8ed64b..d910a6ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -543,15 +543,21 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, if (pd != null && pd.size() > 0) { int len = pd.loadedEntriesLen(); if (len > 0) { - // Adopt the persisted dictionary's loaded-entries buffer as the - // mirror's initial backing instead of copying it and leaving the - // dictionary to retain a second copy for the engine's lifetime. - // The producer's readLoadedSymbols() -- the only other consumer -- - // runs earlier in setCursorEngine; the drainer path has no - // producer consumer. takeLoadedEntries() transfers ownership, so - // this loop's mirror lifecycle frees it (not pd.close()). The - // buffer's allocated size equals loadedEntriesLen. - sentDictBytesAddr = pd.takeLoadedEntries(); + // COPY the persisted dictionary's loaded-entries buffer into this + // loop's own mirror rather than taking ownership of it. The engine + // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan + // drainer path: BackgroundDrainer builds a fresh send loop per wire + // session against the same engine on a durable-ack capability-gap + // recycle. A one-shot ownership transfer would leave every loop + // after the first with an EMPTY mirror -- it would then send no + // reconnect catch-up, and the first replayed delta frame + // (deltaStart > 0) would trip the torn-dict guard, falsely + // quarantining a healthy slot. Copying keeps the dictionary's + // loaded entries intact for the engine's lifetime so every + // recycled loop re-seeds; pd.close() (at engine close) frees the + // dictionary's copy, this loop frees its own copy on exit. + sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); sentDictBytesCapacity = len; sentDictBytesLen = len; // Set the count only alongside the bytes so sentDictCount can diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 55c04ba7..bf5fc39f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -95,15 +95,14 @@ public final class PersistedSymbolDict implements QuietCloseable { private boolean closed; // In-memory copy of the entry region [len][utf8]... exactly as on disk, // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. Consumed once to seed - // the producer's id map (readLoadedSymbols) and then ADOPTED by the send loop's - // catch-up mirror (takeLoadedEntries), which takes ownership so this file no - // longer retains a second copy of the dictionary for the engine's lifetime. + // orphan-drain). Zero/empty for a freshly created file. READ (not consumed) to + // seed the producer's id map (readLoadedSymbols) and to seed the send loop's + // catch-up mirror, which COPIES it. This file retains ownership for the engine's + // lifetime -- the orphan drainer builds a fresh send loop per wire session + // against the same engine, and each must re-seed its mirror -- and frees this + // buffer in close(). private long loadedEntriesAddr; private int loadedEntriesLen; - // True once takeLoadedEntries() handed loadedEntriesAddr to the send-loop - // mirror; guards a stale readLoadedSymbols() call (which must run first). - private boolean loadedEntriesTaken; private long scratchAddr; private int scratchCap; private int size; @@ -282,7 +281,6 @@ public int loadedEntriesLen() { * recovered. */ public ObjList readLoadedSymbols() { - assert !loadedEntriesTaken : "readLoadedSymbols() must run before takeLoadedEntries()"; ObjList out = new ObjList<>(Math.max(size, 1)); long p = loadedEntriesAddr; long limit = p + loadedEntriesLen; @@ -313,25 +311,6 @@ public int size() { return size; } - /** - * Transfers ownership of the loaded-entries buffer to the caller and returns - * its base address (0 when nothing was recovered). The send loop adopts it as - * the initial backing of its catch-up mirror rather than copying it, so this - * file stops retaining a second copy of the dictionary for the engine's - * lifetime. After this call {@link #close()} no longer frees the buffer -- the - * caller owns it. The producer's {@link #readLoadedSymbols()} is the only other - * consumer and MUST run first: {@code setCursorEngine} seeds the producer - * before the send loop is constructed, and the drainer path has no producer - * consumer. The buffer was allocated with {@link MemoryTag#NATIVE_DEFAULT}. - */ - public long takeLoadedEntries() { - long addr = loadedEntriesAddr; - loadedEntriesAddr = 0L; - loadedEntriesLen = 0; - loadedEntriesTaken = true; - return addr; - } - private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 19626ba4..7869882d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -33,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.Comparator; @@ -96,6 +97,69 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { } } + @Test + public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { + // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW + // CursorWebSocketSendLoop per wire session against the SAME, persistent + // engine when a durable-ack capability gap forces a mid-drain recycle. The + // recovery mirror seed must survive that recycle. If the first loop CONSUMED + // the persisted dictionary's loaded entries (a one-shot ownership transfer), + // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no + // reconnect catch-up, and the first replayed delta frame (deltaStart > 0) + // trips the torn-dict guard -- falsely quarantining a healthy slot with a + // bogus "resend required" terminal. Copying the entries (leaving the + // dictionary intact for the engine's lifetime) lets every recycled loop + // re-seed. Pre-fix, loop2's sentDictCount is 0 and this assertion fails. + Path sfDir = Files.createTempDirectory("qwp-mirror-reseed"); + try { + populateRecoverableSlot(sfDir); + Path slot = sfDir.resolve("default"); + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull(pd); + int dictSize = pd.size(); + Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0); + + // Session 1 seeds its mirror from the persisted dictionary. + CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine); + try { + Assert.assertEquals("session-1 mirror must seed from the persisted dict", + dictSize, readInt(loop1, "sentDictCount")); + } finally { + loop1.close(); + } + + // Session 2 against the SAME engine (the drainer recycle): the + // seed must NOT have been consumed -- the mirror must re-seed to + // the full dictionary so the reconnect catch-up is complete. + CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine); + try { + Assert.assertEquals("recycled session-2 mirror must re-seed from the " + + "persisted dict (pre-fix it was 0)", + dictSize, readInt(loop2, "sentDictCount")); + } finally { + loop2.close(); + } + } + }); + } finally { + rmDir(sfDir); + } + } + + // Constructs a recovery send loop but does NOT start it: the ctor seeds the + // catch-up mirror synchronously, which is all these tests observe. The + // reconnect factory is never invoked. + private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine) { + return new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + } + private static void populateRecoverableSlot(Path sfDir) throws Exception { try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); @@ -117,6 +181,12 @@ private static void populateRecoverableSlot(Path sfDir) throws Exception { } } + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(loop); + } + private static void rmDir(Path dir) throws IOException { if (dir == null || !Files.exists(dir)) { return; From 444199ea881da44aa9ee988df1b220ff5b568d20 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 12:32:53 +0100 Subject: [PATCH 26/36] Fix symbol-dict durability doc; add recovery test The PersistedSymbolDict class doc claimed a host-crash-torn dictionary is "caught at replay by the send loop's guard ... rather than corrupting the target table." The guard is tail-only: it fires only when a frame's delta start id exceeds the recovered dictionary size. It does NOT catch an interior page lost out of order (reads back as zeroes that parse as empty-string entries) or a failed best-effort truncate that leaves a stale trailing entry -- either shifts the dense id->symbol mapping without changing the entry count the guard checks, so a replay silently misattributes symbols. Correct the class doc and the truncate comment to describe the actual, tail-only protection and name the residual host-crash exposure (within the already-documented "not host-crash durable" boundary); note that a per-entry or running CRC would close it. Add testRecoveryAfterFailedPublishReplaysGapFree: a failed publish persists a frame's symbol without recording the frame, leaving the persisted dictionary a strict superset of the recorded frames. The test recovers the slot on a fresh sender against a dictionary-reconstructing server and asserts the catch-up re-registers the whole superset (the unrecorded symbol included) and the replay is gap-free -- the end-to-end fail -> recover -> replay chain the existing no-duplicate test omitted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 23 ++++- .../qwp/client/DeltaDictRecoveryTest.java | 94 +++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index bf5fc39f..986039c2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -67,9 +67,18 @@ * dictionary is a superset of every recoverable frame's references. It is NOT * sufficient for a host/power crash, where unflushed pages can be lost out * of order and the dictionary may end up torn relative to the frames it serves -- - * exactly as the segment frames themselves may be lost on a host crash. A torn - * dictionary is caught at replay by the send loop's guard, which fails loudly - * (the unreplayable data must be resent) rather than corrupting the target table. + * exactly as the segment frames themselves may be lost on a host crash. The send + * loop's replay guard catches the COMMON host-crash outcome -- a truncated tail, + * where a frame's delta start id exceeds the recovered dictionary size -- and + * fails loudly (the unreplayable data must be resent) rather than sending a + * gapped frame. It does NOT catch every host-crash tear: an interior page lost + * out of order reads back as zeroes that parse as empty-string entries, and a + * failed best-effort truncate (see {@link #open}) can leave a stale trailing + * entry -- either SHIFTS the dense id->symbol mapping without changing the entry + * count the guard checks, so a replay silently misattributes symbols. That + * residual exposure sits within the "not host-crash durable" boundary above; a + * per-entry or running CRC would be needed to close it (the 3 reserved header + * bytes leave room for a future integrity field). *

* A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. @@ -367,8 +376,12 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // shift every subsequent dense id. Without this, appendOffset already // lands just past the last complete entry, so the next append // overwrites FROM there, but only truncation guarantees nothing - // survives BEYOND it. Best-effort: a failed truncate just forgoes the - // hardening and falls back to the overwrite-from-appendOffset behaviour. + // survives BEYOND it. Best-effort: a failed truncate (rare -- an I/O + // error) leaves a latent silent-corruption path, not merely a lost + // optimization: if a later, shorter append then leaves stale bytes past + // its end, a subsequent recovery mis-parses them as a ghost symbol and + // shifts the dense ids, and the count-based replay guard does not catch + // it (see the class-level durability note). long validLen = HEADER_SIZE + entriesLen; if (validLen < len) { Files.truncate(fd, validLen); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8174ed24..222eeebe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -480,6 +480,100 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception }); } + @Test + public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { + // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay. + // A failed publish persists the frame's symbol (write-ahead) but does NOT + // record the frame, so the persisted dictionary becomes a strict SUPERSET of + // the recorded frames' references. A recovering sender must still replay + // gap-free: it re-registers the whole (superset) dictionary via the catch-up + // -- including the symbol whose frame never reached disk -- so the fresh + // server reconstructs a complete, gap-free dictionary. The sibling + // testFailedPublishDoesNotDuplicatePersistedSymbols proves the dict has no + // duplicate after the failed publish; this proves the resulting slot then + // recovers and replays end-to-end against a real server. + assertMemoryLeak(() -> { + // Phase 1: a silent server (no acks) + a small SF segment. Four small + // rows register sym-0..sym-3 and their frames are recorded; a fifth, + // oversized row registers (persists) sym-4 but its frame is too large for + // the segment, so appendBlocking throws and the frame is NOT recorded. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + for (int i = 0; i < 4; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + // Oversized frame: sym-4 is persisted (write-ahead) before the + // publish fails, so the dictionary runs one ahead of the frames. + s1.table("m").symbol("s", "sym-4") + .stringColumn("p", TestUtils.repeat("x", 8000)) + .longColumn("v", 4).atNow(); + try { + s1.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + // PAYLOAD_TOO_LARGE -- frame not recorded, sym-4 stays persisted + } + } finally { + try { + // Re-flushes the still-buffered oversized row and fails again + // (expected); resources are still released, and the idempotent + // write-ahead does not re-append sym-4. + s1.close(); + } catch (LineSenderException ignored) { + } + } + } + + // The persisted dictionary must hold the superset: sym-0..sym-4 (5 ids), + // one more than the four recorded frames reference, with no duplicate. + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("failed publish must leave the dict a superset (sym-0..sym-4)", + 5, pd.size()); + } finally { + pd.close(); + } + + // Phase 2: recover against a fresh server that reconstructs its + // dictionary from the wire. The recovering sender must re-register all 5 + // symbols via a catch-up (sym-4 exists ONLY in the dictionary -- no frame + // carries it) and replay the 4 recorded frames, leaving a complete, + // gap-free server dictionary. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 5) { + Thread.sleep(20); + } + } + Assert.assertTrue("recovery must send a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + List dict = handler.dictSnapshot(); + Assert.assertEquals("recovered dictionary must include the failed-publish symbol", + 5, dict.size()); + for (int i = 0; i < 5; i++) { + Assert.assertEquals("dictionary id " + i + " must be gap-free", + "sym-" + i, dict.get(i)); + } + } + }); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); From 7a9565397c18aaa5411434fd8a27f0f49317b416 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 13:48:20 +0100 Subject: [PATCH 27/36] Discard a surviving symbol dict on fresh start A fresh store-and-forward slot inherited a stale .symbol-dict when the best-effort removeOrphan failed to delete it (e.g. a Windows share lock) or a crash landed in the close window: open() parses and TRUSTS any survivor. Because the fresh-start producer is not seeded from the dictionary (that is gated on wasRecoveredFromDisk, while the send-loop mirror and the persist resume point key off pd.size()), the producer's ids then diverged from the dictionary the send loop replays, silently misattributing symbol values after a reconnect. The dictionary is load-bearing, unlike the max()-clamped ack watermark whose removeOrphan+open pattern it had copied. Enforce "fresh start -> empty dictionary" at the source: openClean() truncates any survivor via openCleanRW instead of trusting it, and if the clean open itself fails the sender falls back to full self-sufficient frames, which is also safe. The recovery path keeps open(); removeOrphan stays for the fully-drained close. Both regression tests fail without the fix (dict size 2, not 0): - PersistedSymbolDictTest.testOpenCleanDiscardsSurvivingDictionary - EmptyOrphanSlotChurnTest.testFreshStartDiscardsSurvivingStaleDictionary Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 17 +++++-- .../client/sf/cursor/PersistedSymbolDict.java | 29 +++++++++-- .../sf/cursor/EmptyOrphanSlotChurnTest.java | 44 +++++++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 49 +++++++++++++++++++ 4 files changed, 131 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 3bc0a185..4b692288 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -321,10 +321,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); - // Same stale-side-file hygiene for the symbol dictionary: a - // fresh slot starts with an empty dictionary. - PersistedSymbolDict.removeOrphan(sfDir); - persistedDictInProgress = PersistedSymbolDict.open(sfDir); + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock); if the clean open itself fails, + // persistedSymbolDict stays null and the sender falls back to full + // self-sufficient frames, which is also safe. + persistedDictInProgress = PersistedSymbolDict.openClean(sfDir); } MmapSegment initial; String initialPath = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 986039c2..3fc76fd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -147,10 +147,31 @@ public static PersistedSymbolDict open(String slotDir) { } /** - * Best-effort removal of a stale dictionary file. Used at fresh-start (a - * stale dict with no segments behind it is meaningless) and at fully-drained - * close (the slot is empty, nothing references the dictionary any more), - * mirroring {@link AckWatermark#removeOrphan}. + * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding + * any surviving content. This is the fresh-start counterpart to {@link #open}: + * a slot with no recovered segments must start with an empty dictionary, so a + * dictionary left by a prior lifecycle -- a fully-drained slot whose + * best-effort delete failed, or a crash in the close window -- must NOT be + * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for + * recovery/orphan-drain replay, this truncates it: the fresh-start producer is + * not seeded from the dictionary, so trusting a survivor would leave the + * producer's ids diverged from the dictionary the send loop replays and + * silently misattribute symbols on the next reconnect. Truncating (rather than + * relying on an unlink succeeding first) closes the gap even when the delete is + * refused -- e.g. a Windows share lock. Returns {@code null} on I/O failure, so + * the caller falls back to full self-sufficient frames exactly as {@link #open} + * does. + */ + public static PersistedSymbolDict openClean(String slotDir) { + return openFresh(slotDir + "/" + FILE_NAME); + } + + /** + * Best-effort removal of a stale dictionary file. Used at fully-drained close + * (the slot is empty, nothing references the dictionary any more), mirroring + * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT + * use this -- it opens a clean dictionary via {@link #openClean} instead, so a + * failed delete cannot leave a stale dictionary a new session would trust. */ public static void removeOrphan(String slotDir) { Files.remove(slotDir + "/" + FILE_NAME); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java index 5c7607a6..2efee2e4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.Files; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -35,6 +36,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; /** * Regression test for M6 — drainer adopting an empty orphan slot would @@ -87,6 +89,48 @@ public void tearDown() { Files.remove(sfDir); } + @Test + public void testFreshStartDiscardsSurvivingStaleDictionary() throws Exception { + // Regression: a prior fully-drained lifecycle can leave a stale + // .symbol-dict behind (a best-effort delete that failed, or a crash in the + // close window) with NO segments. A fresh start must DISCARD it -- the + // dictionary is load-bearing and the fresh-start producer is not seeded + // from it, so trusting a survivor would diverge the producer ids from the + // dictionary the send loop replays and misattribute symbols on reconnect. + TestUtils.assertMemoryLeak(() -> { + // Pre-seed a stale dictionary in the slot, with no segments behind it. + PersistedSymbolDict stale = PersistedSymbolDict.open(sfDir); + assertNotNull(stale); + try { + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + assertEquals(2, stale.size()); + } finally { + stale.close(); + } + + // A fresh start (no recovered segments) must open a CLEAN, empty + // dictionary -- not inherit the survivor. + try (CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024)) { + assertFalse("fresh start must not report a disk recovery", + engine.wasRecoveredFromDisk()); + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + assertNotNull(pd); + assertEquals("fresh start must discard the surviving stale dictionary", + 0, pd.size()); + } + + // The survivor's bytes are physically gone, not just hidden. + PersistedSymbolDict reopened = PersistedSymbolDict.open(sfDir); + assertNotNull(reopened); + try { + assertEquals(0, reopened.size()); + } finally { + reopened.close(); + } + }); + } + @Test public void testNeverPublishedCloseLeavesNoSfaFiles() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 67fd7438..cc45c23b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -299,6 +299,55 @@ public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { }); } + @Test + public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { + // A fresh start must NOT inherit a dictionary left by a prior lifecycle: + // openClean() truncates any survivor to empty, where open() would recover + // (and TRUST) it. Trusting a survivor whose segments are gone -- the + // fresh-start producer is not seeded from it -- shifts the dense id->symbol + // mapping and misattributes symbols on the next reconnect. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict-clean"); + try { + PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(stale); + try { + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + Assert.assertEquals(2, stale.size()); + } finally { + stale.close(); + } + + // Fresh start: openClean yields an EMPTY dictionary regardless of + // the survivor, and appends from id 0 again. + PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString()); + Assert.assertNotNull(fresh); + try { + Assert.assertEquals(0, fresh.size()); + Assert.assertEquals(0, fresh.readLoadedSymbols().size()); + fresh.appendSymbol("freshA"); + Assert.assertEquals(1, fresh.size()); + } finally { + fresh.close(); + } + + // The survivor's bytes are physically gone, not just hidden: a + // subsequent recovery open() sees only the post-clean content. + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(reopened); + try { + Assert.assertEquals(1, reopened.size()); + Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); + } finally { + reopened.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { From 6d14727421438637ed9e3be5391d34a19e94c567 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 14:05:04 +0100 Subject: [PATCH 28/36] Harden delta symbol-dict send and persist paths Four follow-ups from the delta symbol-dictionary review. Catch-up NACK no longer launders the poison detector. After a reconnect the loop ships the head data frame before tryReceiveAcks reads the server's NACK of a dictionary catch-up frame, so handleServerRejection saw dataFrameSentThisConnection=true and attributed the NACK a wire seq below the replay head -- fsn = fsnAtZero+cappedSeq, negative when replayStart < catchUpFrames. recordHeadRejectionStrike then set poisonFsn to that value (often -1, the "no suspect" sentinel), wiping a genuine in-progress poison run and reporting a bogus fsn. Route an fsn <= ackedFsn rejection to the pre-send path (surface + recycle, no strike, no watermark advance), symmetric with the success path's engine.acknowledge() no-op below ackedFsn. A real replayed data frame is at fsn > ackedFsn, so it is never caught. Regression test: testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState. Persist short-write now surfaces as LineSenderException. A disk-full during the write-ahead persist threw a raw IllegalStateException that escaped flush() unwrapped, unlike every other flush-path failure (e.g. the cursor append in sealAndSwapBuffer). Wrap it so a caller catching LineSenderException around flush() sees a persist disk-full too; a JVM Error still propagates. PersistedSymbolDict close() and the append methods are synchronized. close() is callable from any thread (a shutdown hook); without mutual exclusion a close racing an in-flight append could free the scratch buffer or close the fd mid-write and let the write land on a descriptor the OS has reused for another file (silent cross-file corruption). The appendSymbols re-encode branch now has a regression test. After a failed publish leaves the durable dictionary size ahead of the wire baseline, a later flush introducing a new symbol re-encodes only the [pd.size() .. currentBatchMaxSymbolId] suffix; keying that off pd.size() (not sentMaxSymbolId+1) keeps it idempotent. Regression test: testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating. Both new tests fail with their production hunk reverted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 32 +++++++--- .../sf/cursor/CursorWebSocketSendLoop.java | 19 ++++++ .../client/sf/cursor/PersistedSymbolDict.java | 19 ++++-- .../qwp/client/DeltaDictRecoveryTest.java | 64 +++++++++++++++++++ ...ursorWebSocketSendLoopPoisonFrameTest.java | 62 ++++++++++++++++++ 5 files changed, 181 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 942af99b..bb1256dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3707,14 +3707,30 @@ private void persistNewSymbolsBeforePublish() { // failed publish the durable size has run ahead of the wire baseline, so // the frame's delta covers MORE than remains to persist; then re-encode // just the [from .. currentBatchMaxSymbolId] suffix. - if (from == sentMaxSymbolId + 1) { - QwpBufferWriter buffer = encoder.getBuffer(); - pd.appendRawEntries( - buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), - encoder.getDeltaEntriesLen(), - currentBatchMaxSymbolId - from + 1); - } else { - pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A short write (disk full / quota) to the persisted dictionary throws + // a low-level IllegalStateException. Surface it as a LineSenderException + // -- like every other flush-path failure, e.g. the cursor append in + // sealAndSwapBuffer -- so a caller catching LineSenderException around + // flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the short + // write, so the still-buffered rows re-persist the same range + // idempotently on retry. A JVM Error is never a persist failure; let it + // propagate. + if (t instanceof Error) { + throw (Error) t; + } + throw new LineSenderException("failed to persist symbol dictionary before publish", t); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index d910a6ca..478d33c5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2811,6 +2811,25 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 3fc76fd4..d2651923 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -83,9 +83,14 @@ * A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. *

- * Lifecycle: single-writer (the producer / user thread). Read once at - * {@link #open} to seed in-memory state on recovery or orphan-drain. Owner - * (the engine) closes it. Not thread-safe for concurrent writers. + * Lifecycle: single-writer (the producer / user thread) for appends. Read + * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The + * owner (the engine) closes it, and {@code close()} is callable from any thread + * (a shutdown hook, test cleanup). {@code close()} and the append methods are + * therefore {@code synchronized}: without that, a close racing an in-flight append + * could free the scratch buffer or close the fd mid-write and let the write land + * on a descriptor the OS has reused for another file (silent cross-file + * corruption). Not thread-safe for concurrent writers. */ public final class PersistedSymbolDict implements QuietCloseable { @@ -188,7 +193,7 @@ public static void removeOrphan(String slotDir) { * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range * at the same offset. No-op when the range is empty or the dictionary is closed. */ - public void appendRawEntries(long addr, int len, int count) { + public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } @@ -208,7 +213,7 @@ public void appendRawEntries(long addr, int len, int count) { * (see the class-level durability note). Assigns the next dense id implicitly * (the entry's position). */ - public void appendSymbol(CharSequence symbol) { + public synchronized void appendSymbol(CharSequence symbol) { if (closed) { return; } @@ -244,7 +249,7 @@ public void appendSymbol(CharSequence symbol) { * range and overwrites at the same offset. No-op when the range is empty or * the dictionary is closed. */ - public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { + public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { if (closed || to < from) { return; } @@ -270,7 +275,7 @@ public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { } @Override - public void close() { + public synchronized void close() { if (closed) { return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 222eeebe..58f9cfa9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -480,6 +480,70 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception }); } + @Test + public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() throws Exception { + // Regression for the re-encode (else) branch of persistNewSymbolsBeforePublish. + // After a failed publish leaves the durable dictionary size ahead of the wire + // baseline (pd.size() > sentMaxSymbolId+1), a later flush that introduces a NEW + // symbol cannot reuse the frame's already-encoded fast-path bytes -- it + // re-encodes just the [pd.size() .. currentBatchMaxSymbolId] suffix via + // appendSymbols. Keying that off pd.size() (not sentMaxSymbolId+1) keeps it + // idempotent: the already-persisted prefix is NOT re-appended. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segment: any frame carrying the padded row fails to publish with + // PAYLOAD_TOO_LARGE deterministically (no backpressure timing). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;"; + String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment + Sender sender = Sender.fromConfig(cfg); + try { + // Flush 1: s0 is persisted (write-ahead) before the oversized frame + // fails to publish. pd.size()=1, sentMaxSymbolId stays -1. + sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow(); + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + } + + // Add a NEW symbol s1 (id 1). The failed s0 row is still buffered, so + // the batch is {s0, s1} and the durable size (1) has run ahead of the + // wire baseline (-1) -- the state that selects the appendSymbols branch. + sender.table("m").symbol("s", "s1").stringColumn("p", pad).longColumn("v", 2L).atNow(); + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + } + + // The else branch persisted ONLY s1 (the suffix). The dictionary holds + // s0, s1 exactly once each. Pre-fix (appendSymbols from + // sentMaxSymbolId+1) re-appended s0, giving size 3. + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", + 2, pd.size()); + Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); + Assert.assertEquals("s1", pd.readLoadedSymbols().getQuick(1)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized rows and fails + // again (PAYLOAD_TOO_LARGE); expected, resources still released. + } + } + } + }); + } + @Test public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index ab200e80..d413d82e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -656,6 +656,46 @@ public void testPoisonDwellHoldsEscalationUntilWallClockWindowElapses() throws E }); } + @Test + public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Exception { + // Regression: after a reconnect the loop ships the head DATA frame + // (dataFrameSentThisConnection=true) BEFORE tryReceiveAcks reads the server's + // NACK of a dictionary CATCH-UP frame in the same loop iteration. That NACK + // names a wire seq below the replay head, so its fsn = fsnAtZero+cappedSeq is + // at or below ackedFsn -- negative when replayStart < catchUpFrames. It must + // NOT charge a poison strike: recordHeadRejectionStrike(fsn) would set + // poisonFsn to that value (the common shape yields -1, the "no suspect" + // sentinel), laundering any genuine in-progress poison run and reporting a + // bogus fsn. The fix routes an fsn <= ackedFsn rejection to the pre-send path + // (surface + recycle, no strike), symmetric with the success path's + // engine.acknowledge() no-op below ackedFsn. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + // Model: 2 catch-up frames (wire seq 0,1) + 1 data frame (wire seq 2) + // sent, nothing acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - + // catchUpFrames(2) = -2. A NACK of catch-up wire seq 0 maps to fsn -2, + // strictly below the -1 sentinel so the assertion discriminates. + setPostCatchUpDataFrameBaseline(loop, -2L, 3L); + assertEquals("precondition: no poison suspect yet", + -1L, getLongField(loop, "poisonFsn")); + + deliverRetriableNack(loop, 0, "disk full"); + + // The catch-up NACK was surfaced + recycled but charged NO strike, so + // the sentinel is intact. Pre-fix it was set to -2, laundering the + // poison detector's state. + assertEquals("catch-up NACK must not set/launder the poison sentinel", + -1L, getLongField(loop, "poisonFsn")); + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception { // RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame @@ -1058,6 +1098,28 @@ private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long cou f.setLong(loop, count); } + // Models a fresh connection that has shipped the dictionary catch-up (advancing + // fsnAtZero below the replay head) AND the head data frame: sets fsnAtZero, + // nextWireSeq, and dataFrameSentThisConnection=true together. + private static void setPostCatchUpDataFrameBaseline(CursorWebSocketSendLoop loop, + long fsnAtZero, long nextWireSeq) throws Exception { + Field z = CursorWebSocketSendLoop.class.getDeclaredField("fsnAtZero"); + z.setAccessible(true); + z.setLong(loop, fsnAtZero); + Field w = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); + w.setAccessible(true); + w.setLong(loop, nextWireSeq); + Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); + d.setAccessible(true); + d.setBoolean(loop, true); + } + + private static long getLongField(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(loop); + } + private static long[] txns(long... v) { return v; } From fdd7141f246839f9df002d2a8e5b0c919bce02ba Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 14:37:42 +0100 Subject: [PATCH 29/36] Tidy delta symbol-dict dead code and edges Minor follow-ups from the review. Remove the unused CursorSendEngine.isMemoryMode() -- it had no callers; isDeltaDictEnabled() tests sfDir == null directly. Sort the NativeBufferWriter import into its alphabetical place among the qwp.client imports in CursorWebSocketSendLoop. Harden PersistedSymbolDict.openExisting: hoist entriesAddr / entriesLen so the catch frees the loaded-entries buffer too. It is unreachable today (nothing between that malloc and the return throws, and on success the buffer is transferred to the returned dict), but the error path no longer leaks if a future edit adds a throwing step. Return a defensive copy from DeltaDictCatchUpTest's dictFor() instead of the live inner list: the caller iterates it unlocked while the server thread may still be appending, matching the sibling dictSnapshot(). Add testTornDictionaryOneIdGapFailsCleanly -- the tightest torn-dictionary boundary (deltaStart == recoveredSize + 1), the common one-entry-short host-crash tail. The existing torn test uses a 3-id gap, so it does not pin the guard's false-negative edge; a "deltaStart > size + 1" mutation passes it but ships the gapped frame here (the new test fails on it). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 11 --- .../sf/cursor/CursorWebSocketSendLoop.java | 2 +- .../client/sf/cursor/PersistedSymbolDict.java | 12 ++- .../qwp/client/DeltaDictCatchUpTest.java | 8 +- .../qwp/client/DeltaDictRecoveryTest.java | 75 +++++++++++++++++++ 5 files changed, 92 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 4b692288..c962a56d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -629,17 +629,6 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } - /** - * True when this engine has no store-and-forward directory: the ring lives - * only in malloc'd memory, so it cannot be recovered after a crash and no - * orphan drainer ever replays it. Only in-process reconnect/failover replays - * its frames, which is what makes send-time symbol-dict catch-up (rather than - * fully self-sufficient frames) safe. - */ - public boolean isMemoryMode() { - return sfDir == null; - } - /** * Whether the sender may delta-encode symbol dictionaries on this engine. * Always true in memory mode (the send loop keeps an in-process catch-up diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 478d33c5..acb56cb7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,12 +31,12 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; -import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index d2651923..889104ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -353,6 +353,8 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { return null; } long buf = 0L; + long entriesAddr = 0L; + int entriesLen = 0; try { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); @@ -388,8 +390,7 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { pos = next + (int) entryLen; count++; } - int entriesLen = pos - HEADER_SIZE; - long entriesAddr = 0L; + entriesLen = pos - HEADER_SIZE; if (entriesLen > 0) { entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); @@ -417,6 +418,13 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); } + // entriesAddr is transferred to the returned dict on the success path, + // and the catch only runs before that return, so freeing it here cannot + // double-free. Unreachable today (nothing between its malloc and the + // return throws), but keeps the error path leak-free against a future edit. + if (entriesAddr != 0L) { + Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + } Files.close(fd); LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 0d2ff9f7..d35a648e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -272,7 +272,9 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() - ? dictsByConn.get(connNumber - 1) + // Copy under the lock: the caller iterates it unlocked while the + // server thread may still be appending to the live inner list. + ? new ArrayList<>(dictsByConn.get(connNumber - 1)) : new ArrayList<>(); } @@ -406,7 +408,9 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() - ? dictsByConn.get(connNumber - 1) + // Copy under the lock: the caller iterates it unlocked while the + // server thread may still be appending to the live inner list. + ? new ArrayList<>(dictsByConn.get(connNumber - 1)) : new ArrayList<>(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 58f9cfa9..1439ac20 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,81 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { + // Boundary variant of testTornDictionaryFailsCleanlyInsteadOfCorrupting: the + // recovered dictionary is short by EXACTLY ONE id -- the tightest gap the + // guard must still reject (deltaStart == recoveredSize + 1). A one-entry-short + // tail is the common host-crash outcome, so this pins the guard's + // false-negative edge: a "deltaStart > size + 1" mutation would let this + // single-id gap through and null-pad the missing symbol on the server. + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Lose the whole dictionary (truncate to the 8-byte header, size 0) but + // stamp the watermark at FSN 0, so recovery replays from FSN 1 -- a frame + // with deltaStart=1. The dictionary covers no ids, so id 0 is the single + // missing entry: deltaStart(1) == recoveredSize(0) + 1. Because size is 0 + // no catch-up is sent, so any counted frame would be the gapped data frame. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 0); + + // Phase 2: recover against a fresh counting server. The guard must fire on + // the very first replay frame (deltaStart 1 > recovered size 0) and fail + // terminally rather than send a frame that null-pads id 0. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertEquals("a one-id gap must still block replay to a fresh server", + 0, handler.frames.get()); + Assert.assertNotNull("a one-id-short dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("delta start 1 exceeds recovered dictionary size 0")); + } + }); + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's From 2fdbbf969e10bf7cef3986633bba599d8510e504 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 17:46:16 +0100 Subject: [PATCH 30/36] Fix catch-up terminal on a homogeneous batch cap The reconnect symbol-dictionary catch-up latched a terminal for any entry exceeding the packing budget (cap - HEADER_SIZE - 16). That reserve is larger than the minimal data-frame overhead, so a symbol the producer had already shipped under a given cap could exceed the budget and trip the terminal on reconnect -- permanently hard-failing a running producer on a homogeneous single-cap cluster, the exact failure the store-and-forward path exists to survive. sendDictCatchUp now tests the actual solo catch-up frame (header + the entry's delta varints + the entry bytes) against the cap. That frame is always smaller than the data frame that already carried the entry, so an accepted symbol always re-registers on the same cap; a genuinely oversized entry on a shrunk (heterogeneous) cap still terminates. The conservative packing budget stays, used only for multi-entry chunk splitting. Add a fixed-cap regression test: a 173-char symbol under a fixed cap of 200 is accepted, then re-registers gap-free across a reconnect. It fails before this change, where the reverted condition terminates a frame that fits the cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 38 +++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 61 +++++++++++++++++++ 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index acb56cb7..bd9a54bc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2171,12 +2171,21 @@ private long readVarintAt(long p, long limit) { */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); - // Symbol-bytes budget per frame, leaving room for the 12-byte header and - // the two delta-section varints. cap <= 0 means the server advertised no - // limit -- send the whole dictionary in one frame, but still bound the - // budget by MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen - // (HEADER_SIZE + varints + symbolsLen) cannot overflow on a pathological - // multi-GB dictionary (unreachable at real cardinality; defensive). + // The frame ceiling a catch-up chunk must not exceed: the server's + // advertised cap, or -- when the server advertises none (cap <= 0) -- + // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + + // varints + symbolsLen) cannot overflow on a pathological multi-GB + // dictionary (unreachable at real cardinality; defensive). Used by the + // single-entry terminal below, which measures the real solo frame. + int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the cap. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against frameLimit instead. int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; @@ -2192,7 +2201,20 @@ private int sendDictCatchUp() { long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] long entryBytes = entryEnd - entryStart; - if (entryBytes > budget) { + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. Terminal only when even that + // solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > frameLimit) { // Non-retriable: the entry will not shrink and the same cluster // re-advertises the same cap, so reconnecting would livelock. // Latch a terminal (the data must be resent after the cap is @@ -2209,7 +2231,7 @@ private int sendDictCatchUp() { // the terminal keeps the homogeneous common case livelock-free. LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']'); recordFatal(err); throw new CatchUpSendException(err); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index d35a648e..811c0c67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -99,6 +99,67 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { }); } + @Test + public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception { + // Regression (homogeneous single cap): a symbol whose length sits just below + // the advertised cap is ACCEPTED into a data frame (messageSize <= cap) and + // enters the sent-dictionary mirror. On reconnect the catch-up must + // re-register it under the SAME cap -- the bare catch-up frame (header + two + // varints + the entry) is smaller than the data frame that already shipped + // it (which also carried the table schema + a row), so it fits. + // + // Pre-fix, the single-entry terminal used the conservative PACKING budget + // (cap - HEADER_SIZE - 16), which is stricter than the producer's publish + // gate (messageSize <= cap) by more than the minimal data-frame overhead. So + // a symbol accepted onto the wire under cap C could exceed that budget and + // trip a spurious "during catch-up" terminal, permanently hard-failing a + // running producer on its first transient reconnect. Concretely at cap=200: + // table("t").symbol("s", <173 chars>).atNow() encodes to 198 bytes (<=200, + // accepted), its dict entry is 2+173=175 bytes (> old budget 172 -> old + // terminal), while the real solo catch-up frame is 12+1+1+175=189 (<=200 -> + // fits). Unlike testCatchUpEntryTooLargeForCapFailsTerminally (a genuinely + // oversized entry on a shrunk cap, which MUST still terminate), this entry + // is legally shippable and must NOT terminate. + final int cap = 200; + final String nearCapSymbol = TestUtils.repeat("x", 173); + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(cap); // same cap on every handshake (homogeneous) + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Symbol-only row so the near-cap symbol drives the frame size. + sender.table("t").symbol("s", nearCapSymbol).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + waitFor(() -> handler.conn1Closed, 5_000); + + // The reconnect runs the catch-up under the SAME cap. Pre-fix + // this latched a terminal (surfacing on this flush); post-fix the + // catch-up ships the near-cap symbol and the flush goes through. + sender.table("t").symbol("s", "beta").atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // Connection 2's dictionary, rebuilt purely from the frames it + // received, must hold the near-cap symbol (re-registered by the + // catch-up) and beta, gap-free -- proving the catch-up SHIPPED rather + // than terminated. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals(nearCapSymbol, conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + }); + } + @Test public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget From 8cfd7bdc86796c6a5c976d8da3d780a19e3d957d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 17:54:43 +0100 Subject: [PATCH 31/36] Fix recovery id desync on UTF-8-colliding symbols Two DISTINCT source symbols that collapse to the same UTF-8 bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get distinct producer ids and persist as separate .symbol-dict entries. On recovery, seedGlobalDictionaryFromPersisted replayed them through getOrAddSymbol, which de-dups the decoded "?" strings, leaving the producer dictionary (and sentMaxSymbolId) one short of pd.size(). That desynced the delta baseline from the send-loop catch-up mirror (which uses pd.size()) and, after a reconnect, silently misattributed later well-formed symbols. Add GlobalSymbolDictionary.addRecoveredSymbol, which appends at the next id WITHOUT de-duplicating, and seed recovery through it so the producer id space always matches the persisted entry count. The reverse lookup keeps the highest id for a colliding string, which is harmless -- both ids encode to identical bytes. A GlobalSymbolDictionary unit test pins the no-dedup contract, and a new recovery test seeds a slot with two lone-surrogate symbols and asserts the producer dictionary and sentMaxSymbolId match the persisted entry count. It fails before this change (dictionary size 1 vs the file's 2). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/GlobalSymbolDictionary.java | 31 ++++++++ .../qwp/client/QwpWebSocketSender.java | 12 +++- .../qwp/client/DeltaDictRecoveryTest.java | 71 +++++++++++++++++++ .../client/GlobalSymbolDictionaryTest.java | 30 ++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bb1256dd..23d1240f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3750,6 +3750,16 @@ private void resetSymbolDictStateForNewConnection() { * the slot's persisted dictionary (ids assigned in the same ascending order, * so they match the recovered frames) and resumes the delta baseline at the * recovered tip, so newly ingested symbols continue above the recovered ids. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): + * the persisted dictionary, the on-wire delta and the send-loop catch-up mirror + * all key on the entry POSITION (id), so the producer id space must match the + * persisted entry count exactly. {@code getOrAddSymbol} would collapse two + * source strings that decode to the same characters -- only malformed lone + * UTF-16 surrogates, which UTF-8-encode to {@code '?'} -- leaving this + * dictionary shorter than {@code pd.size()} and desyncing + * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, + * which silently misattributes later symbols after a reconnect. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { if (pd == null || pd.size() == 0) { @@ -3757,7 +3767,7 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { - globalSymbolDictionary.getOrAddSymbol(symbols.getQuick(i)); + globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); } sentMaxSymbolId = globalSymbolDictionary.size() - 1; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 1439ac20..e46d0148 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -36,6 +36,7 @@ import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; @@ -713,6 +714,76 @@ public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { }); } + @Test + public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Exception { + // M2 regression: two DISTINCT source symbols that collapse to the SAME UTF-8 + // bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get + // distinct producer ids and persist as separate entries. On recovery the + // producer must rebuild its id space to match the persisted entry count + // exactly. Pre-fix, seedGlobalDictionaryFromPersisted used getOrAddSymbol, + // which de-duped the two decoded "?" strings, leaving the producer + // dictionary (and sentMaxSymbolId) one short of pd.size() -- desyncing from + // the send-loop catch-up mirror (which uses pd.size()) and silently + // misattributing later symbols after a reconnect. addRecoveredSymbol appends + // without de-duping, keeping producer and mirror in lockstep. + assertMemoryLeak(() -> { + // Phase 1: ingest two lone-surrogate symbols in file mode, close-fast. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + s1.table("m").symbol("s", "\uD800").longColumn("v", 1L).atNow(); // lone surrogate -> '?' + s1.flush(); + s1.table("m").symbol("s", "\uD801").longColumn("v", 2L).atNow(); // a DIFFERENT one -> '?' + s1.flush(); + } + } + + // The persisted dictionary holds TWO entries (both encode to '?'). + int persistedSize; + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + persistedSize = pd.size(); + } finally { + pd.close(); + } + Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize); + + // Phase 2: recover. The seeded producer id space must match pd.size(). + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + Assert.assertEquals("recovered producer dictionary must match the persisted " + + "entry count (not de-duped), else the delta baseline desyncs from the mirror", + persistedSize, globalDictSize(s2)); + Assert.assertEquals("delta baseline must resume at the persisted tip", + persistedSize - 1, intField(s2, "sentMaxSymbolId")); + } + } + }); + } + + private static int globalDictSize(Sender sender) throws Exception { + Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); + f.setAccessible(true); + Object dict = f.get(sender); + return (int) dict.getClass().getMethod("size").invoke(dict); + } + + private static int intField(Sender sender, String name) throws Exception { + Field f = sender.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.getInt(sender); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java index b8945d40..2f38ab02 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -31,6 +31,36 @@ public class GlobalSymbolDictionaryTest { + @Test + public void testAddRecoveredSymbol_appendsWithoutDeduplicating() { + // Recovery replays persisted entries in id order. Distinct source strings + // that decode to the same characters -- lone UTF-16 surrogates both + // UTF-8-encode to '?', so they read back as the string "?" -- must keep + // DISTINCT ids, so the producer id space matches the persisted entry count. + // getOrAddSymbol de-dups them; addRecoveredSymbol must not. + GlobalSymbolDictionary dedup = new GlobalSymbolDictionary(); + dedup.getOrAddSymbol("?"); + dedup.getOrAddSymbol("?"); + assertEquals("getOrAddSymbol de-dups colliding strings", 1, dedup.size()); + + GlobalSymbolDictionary recovered = new GlobalSymbolDictionary(); + assertEquals(0, recovered.addRecoveredSymbol("?")); + assertEquals(1, recovered.addRecoveredSymbol("?")); + assertEquals(2, recovered.addRecoveredSymbol("nvda")); + assertEquals("addRecoveredSymbol keeps colliding entries distinct", 3, recovered.size()); + + // Dense id -> symbol mapping is preserved position-for-position. + assertEquals("?", recovered.getSymbol(0)); + assertEquals("?", recovered.getSymbol(1)); + assertEquals("nvda", recovered.getSymbol(2)); + + // A later ingest of a colliding string reuses the highest recovered id + // (harmless -- both encode to identical bytes), and a genuinely new symbol + // continues past the recovered tip. + assertEquals(1, recovered.getOrAddSymbol("?")); + assertEquals(3, recovered.getOrAddSymbol("brand-new")); + } + @Test public void testAddSymbol_assignsSequentialIds() { GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); From 0b4ab3ff1c0ac5a044da8eb49c6a69fbb7528a72 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 18:06:40 +0100 Subject: [PATCH 32/36] Harden symbol-dict resource teardown paths Three defensive fixes on currently-unreachable teardown paths, each guarding a future edit from a native-memory hazard: - PersistedSymbolDict.close() now nulls loadedEntriesAddr/Len after freeing them (like scratchAddr), so a post-close read of the non-closed-guarded getters cannot dereference freed memory. - CursorWebSocketSendLoop resets sentDictCount alongside the buffer in both mirror-free sites, keeping the mirror all-or-nothing. start() has no closed guard, so a hypothetical close()-then-start() would otherwise observe a non-zero count against a freed buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. - PersistedSymbolDict.openFresh() now closes the fd on any throw (the header malloc moves inside the try, plus a catch), mirroring openExisting; previously a throw in the header-write body freed the scratch buffer but leaked the fd. Tests: testCloseNullsLoadedEntries asserts the getters read 0 after close; the mirror-leak test now also asserts sentDictCount is reset. Both fail before their fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 6 +++++ .../client/sf/cursor/PersistedSymbolDict.java | 20 ++++++++++++-- ...CursorWebSocketSendLoopMirrorLeakTest.java | 8 ++++++ .../sf/cursor/PersistedSymbolDictTest.java | 26 +++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index bd9a54bc..7593cf47 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -907,6 +907,11 @@ public synchronized void close() { sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; + // Reset the count alongside the buffer so the mirror stays all-or- + // nothing: a hypothetical close()-then-start() (start() has no closed + // guard) must not observe a non-zero sentDictCount against a freed + // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. + sentDictCount = 0; } } @@ -1793,6 +1798,7 @@ private void ioLoop() { sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; + sentDictCount = 0; // keep the mirror all-or-nothing (see close()) } shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 889104ec..7f557708 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -282,6 +282,11 @@ public synchronized void close() { closed = true; if (loadedEntriesAddr != 0L) { Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT); + // Null after freeing (like scratchAddr below) so a future accessor that + // reads loadedEntriesAddr()/loadedEntriesLen() post-close cannot + // dereference freed native memory; the getters are not closed-guarded. + loadedEntriesAddr = 0L; + loadedEntriesLen = 0; } if (scratchAddr != 0L) { Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); @@ -437,8 +442,9 @@ private static PersistedSymbolDict openFresh(String filePath) { LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); return null; } - long hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + long hdr = 0L; try { + hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC); Unsafe.getUnsafe().putByte(hdr + 4, VERSION); Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); @@ -451,8 +457,18 @@ private static PersistedSymbolDict openFresh(String filePath) { LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } + } catch (Throwable t) { + // Unreachable today (Files.write is native and returns -1 rather than + // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte + // malloc cannot realistically OOM), but close the fd against a future + // edit so it cannot leak -- mirroring openExisting's error handling. + Files.close(fd); + LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); + return null; } finally { - Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + if (hdr != 0L) { + Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + } } return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 7869882d..057e1a5f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -89,7 +89,15 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { 0, 0, 1); // Close without start(): the ctor-seeded mirror is this // thread's to free, since the I/O loop never ran. + Assert.assertTrue("precondition: the ctor seeded a non-empty mirror", + readInt(loop, "sentDictCount") > 0); loop.close(); + // close() must reset sentDictCount alongside freeing the buffer, + // so the mirror stays all-or-nothing: a hypothetical post-close + // start() (no closed guard) cannot read a stale count against a + // freed buffer and drive a null-mirror catch-up. + Assert.assertEquals("close() must reset sentDictCount to 0", + 0, readInt(loop, "sentDictCount")); } }); } finally { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index cc45c23b..aa37031b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -228,6 +228,32 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { }); } + @Test + public void testCloseNullsLoadedEntries() throws Exception { + // close() must null loadedEntriesAddr/Len after freeing them (like + // scratchAddr), so an accidental post-close read of the getters cannot + // dereference freed native memory. Pre-fix the pointer survived close() + // non-zero. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("AAPL"); + d.close(); + + // Reopen so recovery loads the entries into native memory. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertTrue("recovery must load entries into native memory", + re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); + re.close(); + Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); + Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); + } finally { + rmDir(dir); + } + }); + } + @Test public void testEmptySymbolRoundTrips() throws Exception { assertMemoryLeak(() -> { From 186ae10dec23ff02e982b87a49047b33764d1a24 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 18:15:51 +0100 Subject: [PATCH 33/36] Defer catch-up commit; fix stale delta comments Three delta symbol-dict cleanups (a fourth reported item, a dead CursorSendEngine.isMemoryMode(), was already removed earlier on this branch, so nothing to do there): - sendCatchUpChunk now sets FLAG_DEFER_COMMIT on the catch-up frame. It carries dictionary entries but no rows, so it must never trigger a server-side commit. Today it is always the first frame on a fresh (empty-transaction) connection, so committing nothing is a no-op -- but that invariant is load-bearing and was unasserted. Deferring the empty commit removes the dependency, so a future mid-stream catch-up cannot prematurely commit an in-flight deferred transaction. The dictionary delta still registers, as any deferred data frame's does. - flushPendingRows' comment claimed delta mode is "memory-mode only"; file-mode store-and-forward ships monotonic deltas too once the persisted dictionary opened. Corrected. - The deltaDictEnabled field comment implied delta mode only applies to recovered / orphan-drained disk slots; it applies to fresh disk slots too. Clarified that recovery additionally seeds the mirror. testReconnectCatchUpRebuildsDictionary now asserts the catch-up frame carries FLAG_DEFER_COMMIT; it fails without the flag. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 3 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 19 ++++++++++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 23d1240f..5915f0fb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3455,7 +3455,8 @@ private void flushPendingRows(boolean deferCommit) { ensureActiveBufferReady(); // In full-dict mode every frame is self-sufficient: it carries the whole // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh - // server never dangles a symbol id. In delta mode (memory-mode only) each + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each // frame carries only ids above sentMaxSymbolId; a reconnect re-registers // the dictionary via an I/O-thread catch-up frame before replay, so the // producer's monotonic baseline stays valid across the wire boundary. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 7593cf47..87d1dab0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -187,8 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); // Delta symbol dictionary catch-up state (see swapClient). Active in memory - // mode, and in disk mode on a recovered / orphan-drained slot (the constructor - // seeds sentDict* from the persisted dictionary there). + // mode, and in disk mode whenever the per-slot persisted dictionary opened -- + // fresh slots included, not just recovered / orphan-drained ones. On a + // recovered / orphan-drained slot the constructor additionally SEEDS sentDict* + // from that persisted dictionary; a fresh slot starts with an empty mirror and + // grows it as frames are sent. // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that @@ -2293,8 +2296,18 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, - (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT)); + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 811c0c67..e7f27db1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -92,6 +92,9 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { List conn2 = handler.dictFor(2); Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", handler.sawZeroTableFrameOnConn2); + Assert.assertTrue("the catch-up frame carries no rows, so it must defer its " + + "(empty) commit -- FLAG_DEFER_COMMIT set", + handler.catchUpDeferredOnConn2); Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals("alpha", conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); @@ -326,6 +329,9 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot // race into connection 1's pre-close window and must land on the reconnect. volatile boolean conn1Closed; + // Set from the flags byte of the zero-table catch-up frame on connection 2: + // the catch-up carries no rows and must defer its (empty) commit. + volatile boolean catchUpDeferredOnConn2; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -352,6 +358,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien accumulate(data, dict); if (connNumber == 2 && tableCount(data) == 0) { sawZeroTableFrameOnConn2 = true; + // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). + catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; } try { client.sendBinary(buildAck(nextSeq.getAndIncrement())); From fef7203593c13a1c8308b91afe081d91ee0779dd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 19:34:11 +0100 Subject: [PATCH 34/36] Plug ensureConnected leak; consolidate varint helpers Review cleanups: - ensureConnected's start()-failure catch now closes cursorSendLoop before nulling client. Previously it closed only the client; because the connected flag is set past the catch, a caller that retries -- re-entering ensureConnected and reassigning cursorSendLoop -- orphaned a recovered slot's ctor-seeded native mirror. That is a reachable leak on the recovered-slot + start()/dispatcher-OOM + retry path. close() is idempotent and frees the mirror via its loopNeverRan path, and also closes the shared client, so the following client.close() is a no-op. - Consolidated the triplicated raw-address varint writer into NativeBufferWriter.writeVarint (writeVarintDirect now delegates to it); PersistedSymbolDict and the catch-up frame builder in CursorWebSocketSendLoop use it, and PersistedSymbolDict.varintSize now reuses NativeBufferWriter.varintSize. - Alphabetized CursorSendEngine (getPersistedSymbolDict before getTotalBackpressureStalls before isDeltaDictEnabled) and moved PersistedSymbolDict.decodeVarint to the front of its private-static cluster. - readVarintAt gained the shift > 35 defensive bound that decodeVarint already has (unreachable today, all inputs are freshly-encoded or validated varints, but consistent). - CursorWebSocketSendLoopCatchUpAlignmentTest now uses the existing TestUtils.createTmpDir/removeTmpDir instead of reinventing them. Not changed: the reported dead CursorSendEngine.isMemoryMode() and a PersistedSymbolDict MAX_ENTRY_LEN constant do not exist at HEAD (the former removed earlier on this branch, the latter never present), and the NativeBufferWriter import is already correctly ordered. The buildAck/waitFor/rmDir/readVarint test helpers are duplicated across ~19 pre-existing test files, so consolidating them belongs in a dedicated repo-wide test-hygiene change rather than this feature PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/NativeBufferWriter.java | 21 ++++-- .../qwp/client/QwpWebSocketSender.java | 11 +++ .../client/sf/cursor/CursorSendEngine.java | 18 ++--- .../sf/cursor/CursorWebSocketSendLoop.java | 21 +++--- .../client/sf/cursor/PersistedSymbolDict.java | 75 +++++++------------ ...WebSocketSendLoopCatchUpAlignmentTest.java | 24 +----- 6 files changed, 77 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..cb6ebb92 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,21 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + */ + public static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -336,11 +351,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 5915f0fb..d00ab872 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3362,6 +3362,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index c962a56d..6c34124d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -619,6 +619,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -641,15 +650,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** - * The engine's persisted symbol dictionary, or {@code null} in memory mode - * (and in disk mode if it failed to open). The producer appends new symbols - * to it; recovery / orphan-drain read its loaded entries to seed catch-up. - */ - public PersistedSymbolDict getPersistedSymbolDict() { - return persistedSymbolDict; - } - /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 87d1dab0..a855cc50 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2161,6 +2161,14 @@ private long readVarintAt(long p, long limit) { break; } shift += 7; + if (shift > 35) { + // Defensive bound, matching PersistedSymbolDict.decodeVarint: a + // canonical entry-length / delta varint is <= 5 bytes. Every caller + // reads freshly-encoded, CRC- or openExisting-validated bytes, so + // this is unreachable, but it stops a corrupt continuation run from + // over-shifting into a garbage length. + break; + } } varintEnd = cur; return value; @@ -2310,8 +2318,8 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, | QwpConstants.FLAG_DEFER_COMMIT)); Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); - long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); - q = writeVarintAt(q, deltaCount); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = NativeBufferWriter.writeVarint(q, deltaCount); Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); client.sendBinary(frame, frameLen); } catch (Throwable t) { @@ -2331,15 +2339,6 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, totalFramesSent.incrementAndGet(); } - private static long writeVarintAt(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr++, (byte) value); - return addr; - } - private boolean tryReceiveAcks() { boolean any = false; try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 7f557708..f00acbd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -218,10 +219,10 @@ public synchronized void appendSymbol(CharSequence symbol) { return; } int utf8Len = Utf8s.utf8Bytes(symbol); - int varLen = varintSize(utf8Len); + int varLen = NativeBufferWriter.varintSize(utf8Len); int recLen = varLen + utf8Len; ensureScratch(recLen); - long p = writeVarint(scratchAddr, utf8Len); + long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } @@ -257,9 +258,9 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); - int recLen = varintSize(utf8Len) + utf8Len; + int recLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; ensureScratch(len + recLen); - long p = writeVarint(scratchAddr + len, utf8Len); + long p = NativeBufferWriter.writeVarint(scratchAddr + len, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } @@ -351,6 +352,30 @@ public int size() { return size; } + /** + * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns + * {@code [value, newPos]} or {@code null} if the varint is truncated + * (torn tail). + */ + private static long[] decodeVarint(long buf, int pos, int limit) { + long value = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new long[]{value, cur}; + } + shift += 7; + if (shift > 35) { + return null; // implausible for an entry length; treat as torn + } + } + return null; + } + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { @@ -473,48 +498,6 @@ private static PersistedSymbolDict openFresh(String filePath) { return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); } - /** - * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns - * {@code [value, newPos]} or {@code null} if the varint is truncated - * (torn tail). - */ - private static long[] decodeVarint(long buf, int pos, int limit) { - long value = 0; - int shift = 0; - int cur = pos; - while (cur < limit) { - byte b = Unsafe.getUnsafe().getByte(buf + cur); - cur++; - value |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return new long[]{value, cur}; - } - shift += 7; - if (shift > 35) { - return null; // implausible for an entry length; treat as torn - } - } - return null; - } - - private static int varintSize(long value) { - int n = 1; - while (value > 0x7F) { - value >>>= 7; - n++; - } - return n; - } - - private static long writeVarint(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr++, (byte) value); - return addr; - } - private void ensureScratch(int required) { if (scratchCap >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 79640f25..3814d826 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -30,7 +30,6 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.network.PlainSocketFactory; -import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; @@ -42,7 +41,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -78,30 +76,12 @@ public class CursorWebSocketSendLoopCatchUpAlignmentTest { @Before public void setUp() { - tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), - "qdb-cursor-catchup-" + System.nanoTime()).toString(); - assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + tmpDir = TestUtils.createTmpDir("qdb-cursor-catchup-"); } @After public void tearDown() { - if (tmpDir == null) return; - long find = Files.findFirst(tmpDir); - if (find > 0) { - try { - int rc = 1; - while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - if (name != null && !".".equals(name) && !"..".equals(name)) { - Files.remove(tmpDir + "/" + name); - } - rc = Files.findNext(find); - } - } finally { - Files.findClose(find); - } - } - Files.remove(tmpDir); + TestUtils.removeTmpDir(tmpDir); } @Test From cbaf474c270d16a45f8041cfdc03fe40855f11a6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 21:18:12 +0100 Subject: [PATCH 35/36] Add a per-entry CRC to the SF symbol dictionary The .symbol-dict side-file was page-cache durable but carried no integrity check, so a host-crash interior tear -- a lost page reading back as zeroes, or a stale entry left past the end by a failed truncate -- could shift the dense id->symbol map without changing the entry count the send-loop replay guard checks, silently misattributing symbol-column values on recovery. The SF segment frames are CRC-32C protected; the dictionary they depend on was not. Each entry now carries a trailing CRC-32C over its [len][utf8] bytes, the same checksum MmapSegment uses. openExisting verifies every entry and stops at the first torn or mismatched one, so recovery trusts only the intact prefix and the guard then forces a resend of the rest -- turning a silent misattribution into a fail-clean "resend required". open() strips the CRCs into the wire-shaped loaded-entries buffer, so the catch-up mirror seed and appendRawEntries keep the file==wire contract and the send loop stays untouched. The tail truncate is now failure-checked: a file open() cannot trim is untrusted and recreated empty rather than left exposing stale bytes. Bumps the on-disk format to v2 with a version check; existing files are recreated. A tear that leaves a CRC-matching byte run is still undetected, a 1-in-2^32 collision per entry, no weaker than the frames. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 246 ++++++++++++------ .../sf/cursor/PersistedSymbolDictTest.java | 56 ++++ 2 files changed, 222 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index f00acbd4..6bdc8162 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.std.Crc32c; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -53,12 +54,15 @@ * Layout (little-endian): * 

  *   offset 0: u32 magic = 'SYD1'
- *   offset 4: u8  version = 1
+ *   offset 4: u8  version = 2
  *   offset 5: 3 bytes reserved (zero)
- *   offset 8: entries, each [len: varint][utf8 bytes], in ascending global-id order
+ *   offset 8: entries, each [len: varint][utf8 bytes][crc32c: u32], in ascending global-id order
  *
* Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned - * sequentially from 0), so no id needs to be stored. + * sequentially from 0), so no id needs to be stored. Each entry carries a + * CRC-32C over its {@code [len][utf8]} bytes (the same checksum the SF segment + * frames use), so a torn or stale entry is detected on recovery instead of + * being silently mis-parsed. *

* Durability / write-ahead ordering: the producer appends the symbols a * frame introduces BEFORE that frame is published to the ring, but does NOT @@ -68,18 +72,27 @@ * dictionary is a superset of every recoverable frame's references. It is NOT * sufficient for a host/power crash, where unflushed pages can be lost out * of order and the dictionary may end up torn relative to the frames it serves -- - * exactly as the segment frames themselves may be lost on a host crash. The send - * loop's replay guard catches the COMMON host-crash outcome -- a truncated tail, - * where a frame's delta start id exceeds the recovered dictionary size -- and - * fails loudly (the unreplayable data must be resent) rather than sending a - * gapped frame. It does NOT catch every host-crash tear: an interior page lost - * out of order reads back as zeroes that parse as empty-string entries, and a - * failed best-effort truncate (see {@link #open}) can leave a stale trailing - * entry -- either SHIFTS the dense id->symbol mapping without changing the entry - * count the guard checks, so a replay silently misattributes symbols. That - * residual exposure sits within the "not host-crash durable" boundary above; a - * per-entry or running CRC would be needed to close it (the 3 reserved header - * bytes leave room for a future integrity field). + * exactly as the segment frames themselves may be lost on a host crash. Two + * layers keep a host-crash tear from silently corrupting data: + *

    + *
  • The per-entry CRC-32C: {@link #open} verifies every entry and stops at + * the first one whose checksum fails, so an interior page lost out of + * order (reading back as zeroes) or a stale entry left past the end by a + * failed truncate is DETECTED and the trusted region ends before it -- + * recovery never mis-parses a corrupt entry as a real symbol nor shifts + * the dense id->symbol map. The truncate that drops a torn/stale tail is + * now failure-checked (see {@link #open}): a file that cannot be trimmed + * is untrusted and recreated empty rather than left exposing stale bytes.
    • + *
  • The send loop's replay guard: once recovery trusts only the intact + * prefix, a surviving frame whose delta start id exceeds that prefix + * fails loudly (the unreplayable data must be resent) rather than sending + * a gapped frame.
    • + *
+ * Together these turn every detectable host-crash tear into a fail-clean + * "resend required" instead of a silent symbol misattribution -- the same + * CRC-32C protection the segment frames carry. A tear that happened to leave a + * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32 + * collision per corrupted entry, no weaker than the frames' own checksum. *

* A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. @@ -101,9 +114,10 @@ public final class PersistedSymbolDict implements QuietCloseable { * OrphanScanner, trim) skip it automatically. */ public static final String FILE_NAME = ".symbol-dict"; + static final int CRC_SIZE = 4; // u32 CRC-32C trailing every entry static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; - static final byte VERSION = 1; + static final byte VERSION = 2; // v2 appended the per-entry CRC-32C private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; private long appendOffset; @@ -184,26 +198,56 @@ public static void removeOrphan(String slotDir) { } /** - * Appends {@code count} already-encoded entries -- {@code [len varint][utf8]...}, - * the exact layout {@link #appendSymbols} produces -- verbatim from {@code addr} - * in a single write. The producer uses this when it already holds those bytes - * (the symbol-dict delta section the frame encoder just wrote), so it does not - * re-encode the same symbols. Writes {@code len} bytes and advances {@code size} - * by {@code count}. Same durability/idempotency contract as {@link #appendSymbols}: - * no fsync, and a short write throws WITHOUT advancing {@code size}/{@code - * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range - * at the same offset. No-op when the range is empty or the dictionary is closed. + * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the + * symbol-dict delta section the frame encoder just wrote -- to the on-disk + * dictionary, computing and appending a per-entry CRC-32C as it copies so the + * producer does not re-encode the symbols. The on-disk layout is + * {@code [len varint][utf8][crc32c]} per entry (see the class layout note); the + * {@code addr}/{@code len} bytes carry no CRC, so this walks the {@code count} + * entries to insert one. Advances {@code size} by {@code count}. Same + * durability/idempotency contract as {@link #appendSymbols}: no fsync, and a + * short write throws WITHOUT advancing {@code size}/{@code appendOffset}, so a + * retry keyed off {@link #size()} re-persists the same range at the same + * offset. No-op when the range is empty or the dictionary is closed. */ public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } - long written = Files.write(fd, addr, len, appendOffset); - if (written != len) { + int outLen = len + count * CRC_SIZE; + ensureScratch(outLen); + long src = addr; + long srcLimit = addr + len; + long dst = scratchAddr; + for (int i = 0; i < count; i++) { + long entryStart = src; + long symLen = 0; + int shift = 0; + while (src < srcLimit) { + byte b = Unsafe.getUnsafe().getByte(src++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + long entryEnd = src + symLen; // src is just past the len varint + if (entryEnd > srcLimit) { + throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + + " [entry=" + i + ", count=" + count + ']'); + } + int wireSpan = (int) (entryEnd - entryStart); // [len][utf8] + Unsafe.getUnsafe().copyMemory(entryStart, dst, wireSpan); + Unsafe.getUnsafe().putInt(dst + wireSpan, Crc32c.update(Crc32c.INIT, entryStart, wireSpan)); + dst += wireSpan + CRC_SIZE; + src = entryEnd; + } + long written = Files.write(fd, scratchAddr, outLen, appendOffset); + if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + len + ", actual=" + written + ']'); + + " [expected=" + outLen + ", actual=" + written + ']'); } - appendOffset += len; + appendOffset += outLen; size += count; } @@ -212,7 +256,9 @@ public synchronized void appendRawEntries(long addr, int len, int count) { * frame's new symbols BEFORE publishing that frame, so the write ordering * (dictionary entry before referencing frame) holds; no fsync is performed * (see the class-level durability note). Assigns the next dense id implicitly - * (the entry's position). + * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC + * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on + * recovery. */ public synchronized void appendSymbol(CharSequence symbol) { if (closed) { @@ -220,12 +266,14 @@ public synchronized void appendSymbol(CharSequence symbol) { } int utf8Len = Utf8s.utf8Bytes(symbol); int varLen = NativeBufferWriter.varintSize(utf8Len); - int recLen = varLen + utf8Len; + int wireLen = varLen + utf8Len; // [len][utf8] + int recLen = wireLen + CRC_SIZE; // + trailing crc ensureScratch(recLen); long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } + Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); long written = Files.write(fd, scratchAddr, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME @@ -237,11 +285,12 @@ public synchronized void appendSymbol(CharSequence symbol) { /** * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes - * the whole {@code [len varint][utf8]...} region into scratch first, then - * issues one positioned write -- versus one {@code pwrite(2)} per symbol via - * {@link #appendSymbol}. That per-symbol syscall count is the hot-path cost - * on a high-cardinality batch (one new symbol per row), which is exactly the - * store-and-forward workload delta encoding targets. Callers pass the + * the whole {@code [len varint][utf8][crc32c]...} region into scratch first, + * then issues one positioned write -- versus one {@code pwrite(2)} per symbol + * via {@link #appendSymbol}. That per-symbol syscall count is the hot-path + * cost on a high-cardinality batch (one new symbol per row), which is exactly + * the store-and-forward workload delta encoding targets. Each entry carries a + * trailing CRC-32C over its {@code [len][utf8]} bytes. Callers pass the * dictionary and the range so the ids resolve to their symbol strings. *

* Same durability and idempotency contract as {@link #appendSymbol}: no @@ -258,13 +307,15 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); - int recLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; - ensureScratch(len + recLen); - long p = NativeBufferWriter.writeVarint(scratchAddr + len, utf8Len); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + ensureScratch(len + wireLen + CRC_SIZE); + long entryStart = scratchAddr + len; + long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - len += recLen; + Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); + len += wireLen + CRC_SIZE; } long written = Files.write(fd, scratchAddr, len, appendOffset); if (written != len) { @@ -389,61 +440,96 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); long read = Files.read(fd, buf, len, 0); - if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC) { - LOG.warn("symbol dict {} unreadable or bad magic; recreating", filePath); + if (read != len + || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC + || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { + LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); Files.close(fd); return null; } - // Parse complete entries starting after the header; stop at the - // first torn/incomplete trailing entry. - int pos = HEADER_SIZE; + // Parse complete, CRC-valid entries after the header; stop at the first + // torn/incomplete OR crc-mismatched entry. The CRC turns an interior + // tear (a lost page reading back as zeroes) or a stale entry left past + // the end by a failed truncate into a clean stop point, so recovery + // trusts only the intact prefix instead of silently mis-parsing a + // corrupt entry and shifting the dense id->symbol map. + int diskPos = HEADER_SIZE; // walks the on-disk [len][utf8][crc] entries int count = 0; - while (pos < len) { - long[] vr = decodeVarint(buf, pos, len); + int wireLen = 0; // running size of the crc-stripped copy + while (diskPos < len) { + long[] vr = decodeVarint(buf, diskPos, len); if (vr == null) { break; // torn length varint } - long entryLen = vr[0]; - int next = (int) vr[1]; - // The file length is the sole sanity bound: a length that runs past - // the file is a torn/incomplete trailing entry and stops the parse - // (self-healing tail). There is deliberately NO fixed per-entry - // ceiling -- the write path (appendSymbol/appendSymbols) applies none - // either, so a legitimately persisted large symbol must recover here, - // not be truncated and then trip the send loop's "resend required" - // guard on a normal process-crash recovery. entryLen stays a long so a - // corrupt multi-gigabyte length cannot wrap an int back under the check. - if ((long) next + entryLen > len) { - break; // torn/incomplete trailing entry -- self-healing tail + long symLen = vr[0]; + int afterVar = (int) vr[1]; + // [len varint][utf8] then a u32 CRC. symLen stays a long so a + // corrupt multi-gigabyte length cannot wrap an int back under the + // bound check. No fixed per-entry ceiling -- the write path applies + // none, so a legitimately large symbol must recover here. + long wireEnd = (long) afterVar + symLen; // end of [len][utf8] + if (wireEnd + CRC_SIZE > len) { + break; // torn/incomplete trailing entry (its CRC doesn't fit) + } + int wireEndI = (int) wireEnd; + int wireSpan = wireEndI - diskPos; + int crcStored = Unsafe.getUnsafe().getInt(buf + wireEndI); + int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, wireSpan); + if (crcCalc != crcStored) { + break; // corrupt/stale entry -- stop before it (fail-clean) } - pos = next + (int) entryLen; + diskPos = wireEndI + CRC_SIZE; + wireLen += wireSpan; count++; } - entriesLen = pos - HEADER_SIZE; - if (entriesLen > 0) { - entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); - Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); + int diskConsumed = diskPos - HEADER_SIZE; // valid entries incl. their CRCs + // Materialise the trusted entries as WIRE bytes ([len][utf8]..., no + // CRC) so loadedEntries*/readLoadedSymbols and the send-loop catch-up + // mirror stay wire-shaped -- the on-disk CRC is stripped here, once, at + // open. A second no-alloc walk over the already-validated region. + if (wireLen > 0) { + entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); + long dst = entriesAddr; + int p = HEADER_SIZE; + for (int i = 0; i < count; i++) { + int vp = p; + long symLen = 0; + int shift = 0; + while (true) { + byte b = Unsafe.getUnsafe().getByte(buf + vp++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC + Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); + dst += wireSpan; + p += wireSpan + CRC_SIZE; // skip the entry's CRC + } } + entriesLen = wireLen; Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; - // Physically drop any torn/stale trailing bytes (a crash mid-append) - // so a LATER, shorter append cannot leave residue past its own end - // that a future recovery mis-parses as a ghost symbol -- which would - // shift every subsequent dense id. Without this, appendOffset already - // lands just past the last complete entry, so the next append - // overwrites FROM there, but only truncation guarantees nothing - // survives BEYOND it. Best-effort: a failed truncate (rare -- an I/O - // error) leaves a latent silent-corruption path, not merely a lost - // optimization: if a later, shorter append then leaves stale bytes past - // its end, a subsequent recovery mis-parses them as a ghost symbol and - // shifts the dense ids, and the count-based replay guard does not catch - // it (see the class-level durability note). - long validLen = HEADER_SIZE + entriesLen; - if (validLen < len) { - Files.truncate(fd, validLen); + // Drop any torn/stale trailing bytes so a LATER, shorter append cannot + // leave residue past its own end. Unlike before, the truncate result IS + // checked: a file we cannot trim could still expose stale post-end bytes + // whose (self-consistent) per-entry CRC the parse would accept at a + // shifted position, so a failed truncate makes the file untrusted -- + // open() then recreates it empty (fail-clean) rather than risk a silent + // misattribution. + long validLen = HEADER_SIZE + diskConsumed; + if (validLen < len && !Files.truncate(fd, validLen)) { + LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); + if (entriesAddr != 0L) { + Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + } + Files.close(fd); + return null; } - return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); + return new PersistedSymbolDict(fd, validLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index aa37031b..cfcf4cf5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -281,6 +281,62 @@ public void testEmptySymbolRoundTrips() throws Exception { }); } + @Test + public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception { + // A host-crash interior tear (a lost page reading back as zeroes) or a + // stale entry left past the end by a failed truncate can change the bytes + // of a NON-trailing entry. Without the per-entry CRC the parse would + // accept those bytes, shifting the dense id->symbol map and silently + // misattributing symbol-column values on replay. With the CRC the corrupt + // entry fails verification and the parse stops there, so recovery trusts + // only the intact prefix (fail-clean: the send loop's torn-dict guard then + // forces a resend of the rest). + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol("s0"); + d.appendSymbol("s1"); + d.appendSymbol("s2"); + d.appendSymbol("s3"); + d.appendSymbol("s4"); + Assert.assertEquals(5, d.size()); + } finally { + d.close(); + } + + // Corrupt one byte inside the 3rd entry's UTF-8 (id 2). On-disk + // entry layout is [len varint][utf8][crc32c u32]; a 2-byte ASCII + // symbol is 1 + 2 + 4 = 7 bytes, after the 8-byte header: + // header[0,8) e0[8,15) e1[15,22) e2[22,29) ... + // Offset 23 is "s2"'s first UTF-8 byte; flipping it leaves e2's + // stored CRC stale. + Path f = dir.resolve(".symbol-dict"); + byte[] bytes = Files.readAllBytes(f); + bytes[23] ^= 0x7F; + Files.write(f, bytes); + + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + // Only the intact prefix [s0, s1] is trusted; the corrupt e2 + // and everything after it are dropped. No recovered symbol is + // the corrupted string -- the tear is DETECTED, never silently + // misattributed. + Assert.assertEquals("parse must stop at the corrupt interior entry", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("s0", s.getQuick(0)); + Assert.assertEquals("s1", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { // C1 regression: the write path caps nothing, so a symbol larger than the From 6cc1307e4b1421e422be8f853c612d4059a4d7d0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 21:48:52 +0100 Subject: [PATCH 36/36] Document split-flush and dict-getter contracts Two documentation-only additions to the QWP store-and-forward client, making previously implicit contracts explicit. No behavior change. flushPendingRowsSplit: add a "Not atomic across frames" note. A sealAndSwapBuffer() failure on split frame k>1 leaves frames 1..k-1 published as deferred; the throw skips the trailing table-buffer reset, so the next flush re-emits the whole batch and the eventual commit commits the already-published prefix twice. This is pre-existing and within store-and-forward's at-least-once contract (absorbed by a DEDUP table or a durable-ack await); the note names the atomic-split fix (roll back or skip the published prefix on retry) as the larger follow-up. PersistedSymbolDict: mark loadedEntriesAddr(), loadedEntriesLen() and readLoadedSymbols() as construction-phase only. They read the native entry region that close() frees and nulls, with no closed-guard, so they are safe only before the slot's I/O thread and any producer append start; reading them from a running thread would risk a use-after-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/QwpWebSocketSender.java | 16 ++++++++++++++++ .../client/sf/cursor/PersistedSymbolDict.java | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d00ab872..39db7d41 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3526,6 +3526,22 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 6bdc8162..bb7884bd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -354,13 +354,22 @@ public synchronized void close() { * Base address of the loaded entry region -- the concatenated * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly * as a delta section carries them. Zero when nothing was recovered. + *

+ * Construction-phase only. This hands out a raw pointer into native + * memory that {@link #close()} frees and nulls, with no closed-guard and no + * synchronization. It is safe to read only BEFORE the slot's I/O thread and + * any producer append start -- i.e. while the send loop is being constructed + * or an orphan-drain is seeding its mirror, both of which happen-before those + * threads. A caller that reads it from a running thread races {@code close()} + * and can dereference freed memory (use-after-free). */ public long loadedEntriesAddr() { return loadedEntriesAddr; } /** - * Byte length of {@link #loadedEntriesAddr()}. + * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for + * the same reason -- see {@link #loadedEntriesAddr()}. */ public int loadedEntriesLen() { return loadedEntriesLen; @@ -371,6 +380,10 @@ public int loadedEntriesLen() { * (entry {@code i} is symbol id {@code i}). Used once on recovery to * repopulate the producer's global dictionary. Empty when nothing was * recovered. + *

+ * Construction-phase only -- like {@link #loadedEntriesAddr()}, this + * walks the native entry region {@link #close()} frees, with no closed-guard, + * so it must run before the I/O thread and any producer append start. */ public ObjList readLoadedSymbols() { ObjList out = new ObjList<>(Math.max(size, 1));