mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-17 01:05:29 +01:00
[filer] fix log buffer idle polling (#9012)
* fix log buffer idle polling * log_buffer: document notificationHealthCheckInterval tradeoffs Explain that notifyChan is the primary wakeup path and this interval only bounds the fallback / state-recheck cadence, so future maintainers don't tune it without understanding the implications for client-disconnect detection latency. * log_buffer: rename waitForNotification to awaitNotificationOrTimeout The helper returns after either a notification or the health-check timeout; the old name read like it blocked indefinitely. No behavior change. * log_buffer: wake blocked subscribers on shutdown awaitNotificationOrTimeout previously only returned on notifyChan or the health-check timeout, so ShutdownLogBuffer on an idle buffer (where copyToFlush returns nil and loopFlush never fires the post-flush notification) would leave subscribers parked for up to 250ms before they noticed IsStopping. Add an internal shutdownCh closed by ShutdownLogBuffer and select on it from awaitNotificationOrTimeout, which is now a method on *LogBuffer. Subscribers wake immediately, re-check IsStopping, and exit. No change to LoopProcessLogData signatures or any caller (filer metadata subscribers, MQ broker, local partition subscribe). * log_buffer: regression tests for flush-notify wake-up TestLoopFlush_NotifiesSubscribersAfterFlush directly verifies that loopFlush calls notifySubscribers after processing a flush, so a reader parked on notifyChan wakes promptly when a flush lands. Verified to fail if that notification is removed. TestLoopProcessLogDataWithOffset_WakesOnDataArrival is the end-to-end counterpart: a real LoopProcessLogDataWithOffset reader parks on notifyChan via the ResumeFromDiskError branch, then wakes and processes the entry well under the 250ms fallback once data arrives. * log_buffer: keep notification-timeout logs at V(4) Revert the V(4)->V(5) demotion. Now that the shutdown wake-up path exists and (with the follow-up fix) idle-polling CPU churn is bounded by the 250ms health check, these timeout logs no longer flood at V=4 the way they did on the 10ms fallback, so the previous verbosity is appropriate again. * log_buffer: exit reader loops cleanly on shutdown awaitNotificationOrTimeout returns true on both data notifications and shutdown (shutdownCh closed). Without an explicit IsStopping() guard, the ResumeFromDiskError, offset-based no-data, empty-buffer, and timestamp-wait paths would either tight-spin against a closed shutdownCh or, in the offset-based case, return ResumeFromDiskError to the caller instead of exiting. Add an IsStopping() check after each awaitNotificationOrTimeout call that previously continued or returned ResumeFromDiskError, so subscribers exit promptly with isDone=true and err=nil when ShutdownLogBuffer is called. * log_buffer: regression test for shutdown wake-up Park a real LoopProcessLogDataWithOffset reader on notifyChan via the ResumeFromDiskError branch, call ShutdownLogBuffer, and assert the reader exits with isDone=true and err=nil well under the 250ms fallback. Verified to fail (timeout) if the IsStopping() guards added in the prior commit are removed. * log_buffer: bump reader-park sleep to 50ms with rationale Both wake-path tests use a sleep to give the goroutine time to reach awaitNotificationOrTimeout before the test triggers the wake-up. Bump from 20ms to 50ms and document the timing assumption to reduce flakiness on slow CI. Both paths are race-free either way (a buffered notification or a closed shutdownCh stays valid until consumed), so this is purely about exercising the park-then-wake path rather than the already-pending fast path.
This commit is contained in:
1 parent
546f255b46
commit
eb5624233d
3 files changed
+287
-47
No files matched your search
@@ -18,6 +18,15 @@ var (
|
||||
ResumeFromDiskError = fmt.Errorf("resumeFromDisk")
|
||||
)
|
||||
|
||||
// notificationHealthCheckInterval bounds how long an idle subscriber blocks
|
||||
// on the notification channel before re-checking state (client disconnect via
|
||||
// waitForDataFn, LogBuffer shutdown, timestamp advancement). notifyChan is the
|
||||
// primary wakeup path when new data arrives or a flush lands; this timeout is
|
||||
// the safety net for any missed notification and also caps the latency to
|
||||
// notice that the subscriber should exit. Balances idle CPU and log noise
|
||||
// against client-disconnect detection latency.
|
||||
const notificationHealthCheckInterval = 250 * time.Millisecond
|
||||
|
||||
type MessagePosition struct {
|
||||
Time time.Time // timestamp of the message
|
||||
Offset int64 // Kafka offset for offset-based positioning, or batch index for timestamp-based
|
||||
@@ -49,6 +58,26 @@ func (mp MessagePosition) GetOffset() int64 {
|
||||
return mp.Offset // Offset is stored directly
|
||||
}
|
||||
|
||||
// awaitNotificationOrTimeout blocks until one of:
|
||||
// - a new-data / flush notification arrives on notifyChan (returns true)
|
||||
// - the LogBuffer is shut down via ShutdownLogBuffer (returns true; callers
|
||||
// re-check IsStopping() and exit)
|
||||
// - notificationHealthCheckInterval elapses (returns false; caller
|
||||
// re-checks client-disconnect and other state)
|
||||
func (logBuffer *LogBuffer) awaitNotificationOrTimeout(notifyChan <-chan struct{}) bool {
|
||||
timer := time.NewTimer(notificationHealthCheckInterval)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-notifyChan:
|
||||
return true
|
||||
case <-logBuffer.shutdownCh:
|
||||
return true
|
||||
case <-timer.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition MessagePosition, stopTsNs int64,
|
||||
waitForDataFn func() bool, eachLogDataFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
|
||||
|
||||
@@ -100,13 +129,19 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition
|
||||
}
|
||||
|
||||
// Wait for notification or timeout (instant wake-up when data arrives)
|
||||
select {
|
||||
case <-notifyChan:
|
||||
// New data available, retry immediately
|
||||
if logBuffer.awaitNotificationOrTimeout(notifyChan) {
|
||||
glog.V(3).Infof("%s: Woke up from notification after ResumeFromDiskError", readerName)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
// Timeout, retry anyway (fallback for edge cases)
|
||||
glog.V(4).Infof("%s: Notification timeout after ResumeFromDiskError, polling", readerName)
|
||||
} else {
|
||||
glog.V(4).Infof("%s: Notification timeout after ResumeFromDiskError, rechecking state", readerName)
|
||||
}
|
||||
|
||||
// If the LogBuffer is shutting down, exit cleanly instead of looping
|
||||
// on ResumeFromDiskError. awaitNotificationOrTimeout returns true on
|
||||
// shutdown (shutdownCh closed), which would otherwise spin here
|
||||
// because ReadFromBuffer keeps returning ResumeFromDiskError.
|
||||
if logBuffer.IsStopping() {
|
||||
isDone = true
|
||||
return
|
||||
}
|
||||
|
||||
// Continue to next iteration (don't return ResumeFromDiskError)
|
||||
@@ -143,17 +178,18 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition
|
||||
return
|
||||
}
|
||||
// Wait for notification or timeout (instant wake-up when data arrives)
|
||||
select {
|
||||
case <-notifyChan:
|
||||
// New data available, break and retry read
|
||||
if logBuffer.awaitNotificationOrTimeout(notifyChan) {
|
||||
glog.V(3).Infof("%s: Woke up from notification (LoopProcessLogData)", readerName)
|
||||
} else if lastTsNs != logBuffer.LastTsNs.Load() {
|
||||
break
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
// Timeout, check if timestamp changed
|
||||
if lastTsNs != logBuffer.LastTsNs.Load() {
|
||||
break
|
||||
}
|
||||
glog.V(4).Infof("%s: Notification timeout (LoopProcessLogData), polling", readerName)
|
||||
} else {
|
||||
glog.V(4).Infof("%s: Notification timeout (LoopProcessLogData), rechecking state", readerName)
|
||||
}
|
||||
// Exit the wait loop on shutdown so we don't spin against a
|
||||
// closed shutdownCh.
|
||||
if logBuffer.IsStopping() {
|
||||
isDone = true
|
||||
return
|
||||
}
|
||||
}
|
||||
if logBuffer.IsStopping() {
|
||||
@@ -294,13 +330,15 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
|
||||
}
|
||||
|
||||
// Wait for notification or timeout (instant wake-up when data arrives)
|
||||
select {
|
||||
case <-notifyChan:
|
||||
// New data available, retry immediately
|
||||
if logBuffer.awaitNotificationOrTimeout(notifyChan) {
|
||||
glog.V(3).Infof("%s: Woke up from notification after disk read", readerName)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
// Timeout, retry anyway (fallback for edge cases)
|
||||
glog.V(4).Infof("%s: Notification timeout, polling", readerName)
|
||||
} else {
|
||||
glog.V(4).Infof("%s: Notification timeout, rechecking state", readerName)
|
||||
}
|
||||
|
||||
// Exit cleanly on shutdown so we don't loop on ResumeFromDiskError.
|
||||
if logBuffer.IsStopping() {
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
|
||||
// Continue to next iteration (don't return ResumeFromDiskError)
|
||||
@@ -338,13 +376,14 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
// Wait for notification or timeout (instant wake-up when data arrives)
|
||||
select {
|
||||
case <-notifyChan:
|
||||
// New data available, retry immediately
|
||||
if logBuffer.awaitNotificationOrTimeout(notifyChan) {
|
||||
glog.V(3).Infof("%s: Woke up from notification for offset-based read", readerName)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
// Timeout, retry anyway (fallback for edge cases)
|
||||
glog.V(4).Infof("%s: Notification timeout for offset-based, polling", readerName)
|
||||
} else {
|
||||
glog.V(4).Infof("%s: Notification timeout for offset-based, rechecking state", readerName)
|
||||
}
|
||||
// On shutdown, exit cleanly instead of returning ResumeFromDiskError.
|
||||
if logBuffer.IsStopping() {
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
return lastReadPosition, isDone, ResumeFromDiskError
|
||||
}
|
||||
@@ -357,17 +396,18 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
// Wait for notification or timeout (instant wake-up when data arrives)
|
||||
select {
|
||||
case <-notifyChan:
|
||||
// New data available, break and retry read
|
||||
if logBuffer.awaitNotificationOrTimeout(notifyChan) {
|
||||
glog.V(3).Infof("%s: Woke up from notification (main loop)", readerName)
|
||||
} else if lastTsNs != logBuffer.LastTsNs.Load() {
|
||||
break
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
// Timeout, check if timestamp changed
|
||||
if lastTsNs != logBuffer.LastTsNs.Load() {
|
||||
break
|
||||
}
|
||||
glog.V(4).Infof("%s: Notification timeout (main loop), polling", readerName)
|
||||
} else {
|
||||
glog.V(4).Infof("%s: Notification timeout (main loop), rechecking state", readerName)
|
||||
}
|
||||
// Exit the wait loop on shutdown so we don't spin against a
|
||||
// closed shutdownCh.
|
||||
if logBuffer.IsStopping() {
|
||||
glog.V(4).Infof("%s: LogBuffer is stopping", readerName)
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
}
|
||||
if logBuffer.IsStopping() {
|
||||
@@ -388,8 +428,15 @@ func (logBuffer *LogBuffer) LoopProcessLogDataWithOffset(readerName string, star
|
||||
glog.V(4).Infof("%s: Client disconnected on empty buffer", readerName)
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
// Sleep to avoid busy-wait on empty buffer
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if logBuffer.awaitNotificationOrTimeout(notifyChan) {
|
||||
glog.V(3).Infof("%s: Woke up from notification on empty buffer", readerName)
|
||||
} else {
|
||||
glog.V(4).Infof("%s: Empty buffer timeout, rechecking state", readerName)
|
||||
}
|
||||
// Exit cleanly on shutdown to avoid an idle spin on the empty buffer.
|
||||
if logBuffer.IsStopping() {
|
||||
return lastReadPosition, true, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user