From f711868fb6185e5c84f1c01747bd995b6149c3a9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 2 Jun 2026 21:37:15 -0700 Subject: [PATCH] fix(log_buffer): re-check buffer before bailing with ResumeFromDiskError (#9804) ReadFromBuffer and HasData() take the read lock separately, so a write that lands between them can make a subscriber which just read a momentarily empty buffer return ResumeFromDiskError even though the data is now servable from memory. Re-read under a fresh lock and only bail when the position is genuinely behind the in-memory window (flushed to disk); otherwise loop back and read it. --- weed/util/log_buffer/log_read.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/weed/util/log_buffer/log_read.go b/weed/util/log_buffer/log_read.go index d62eea428..364047f74 100644 --- a/weed/util/log_buffer/log_read.go +++ b/weed/util/log_buffer/log_read.go @@ -147,7 +147,17 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition glog.V(4).Infof("%s: Caught up to disk head, backing off to %s poll", readerName, caughtUpDiskPollInterval) } } else if logBuffer.HasData() { - return lastReadPosition, isDone, ResumeFromDiskError + // HasData() and ReadFromBuffer lock separately, so a racing write can make + // HasData() see data the empty-buffer read missed. Re-read; only bail if the + // position is genuinely behind the in-memory window (flushed to disk). + reBuf, _, reErr := logBuffer.ReadFromBuffer(lastReadPosition) + if reErr == ResumeFromDiskError { + return lastReadPosition, isDone, ResumeFromDiskError + } + if reBuf != nil { + logBuffer.ReleaseMemory(reBuf) + } + continue } // CRITICAL: Check if client is still connected @@ -362,7 +372,17 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star glog.V(4).Infof("%s: Caught up to disk head, backing off to %s poll", readerName, caughtUpDiskPollInterval) } } else if logBuffer.HasData() { - return lastReadPosition, isDone, ResumeFromDiskError + // HasData() and ReadFromBuffer lock separately, so a racing write can make + // HasData() see data the empty-buffer read missed. Re-read; only bail if the + // position is genuinely behind the in-memory window (flushed to disk). + reBuf, _, reErr := logBuffer.ReadFromBuffer(lastReadPosition) + if reErr == ResumeFromDiskError { + return lastReadPosition, isDone, ResumeFromDiskError + } + if reBuf != nil { + logBuffer.ReleaseMemory(reBuf) + } + continue } // CRITICAL: Check if client is still connected after disk read