mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
filer: pin-aware reader cache eviction and stream release (#11503)
* filer: synchronize stream pins and release them on transitions Guard chunkStream.cacher with the ReaderCache lock everywhere: mount sections share one ChunkReadAt across concurrent reads, and unsynchronized release could double-unpin. Reads served from the chunk cache now detach the stream's pin instead of retaining the previous chunk. Eviction prefers unpinned downloaders so a pinned buffer is not dropped mid-stream. A new ReleaseStream lets callers drop their pin without destroying the shared cache; S3 and WebDAV readers use it. lastChunkFid becomes atomic since concurrent mount reads can update it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer: keep eviction bounded when every downloader is pinned Both eviction paths still fall back to a pinned victim when no unpinned one exists, so abandoned stream pins cannot bypass the downloader limit or stall the memory budget. Budget eviction also rechecks the pin under the ReaderCache lock at removal time: a stream that pinned the selected victim in between keeps it mapped and the selection retries. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer: restore budget bookkeeping when a victim gets pinned mid-eviction removeUnpinned losing the pin race left the victim out of the idle list while still holding its reservation, making it unevictable even as the pinned fallback. Push it back when the reservation is still live. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
3abdef3202
commit
9b3b12c607
6 files changed
+149
-7
No files matched your search
@@ -169,14 +169,26 @@ retry:
|
||||
}
|
||||
}
|
||||
|
||||
// clean up old downloaders
|
||||
// clean up old downloaders; prefer one no stream is positioned in, but
|
||||
// fall back to a pinned one so abandoned pins cannot bypass the limit
|
||||
if len(rc.downloaders) >= rc.limit {
|
||||
oldestFid, oldestTime := "", time.Now().UnixNano()
|
||||
pinnedFid, pinnedTime := "", int64(0)
|
||||
for fid, downloader := range rc.downloaders {
|
||||
completedTime := atomic.LoadInt64(&downloader.completedTimeNew)
|
||||
if completedTime > 0 && completedTime < oldestTime {
|
||||
oldestFid, oldestTime = fid, completedTime
|
||||
if completedTime <= 0 {
|
||||
continue
|
||||
}
|
||||
if atomic.LoadInt32(&downloader.pins) == 0 {
|
||||
if completedTime < oldestTime {
|
||||
oldestFid, oldestTime = fid, completedTime
|
||||
}
|
||||
} else if pinnedFid == "" || completedTime < pinnedTime {
|
||||
pinnedFid, pinnedTime = fid, completedTime
|
||||
}
|
||||
}
|
||||
if oldestFid == "" {
|
||||
oldestFid = pinnedFid
|
||||
}
|
||||
if oldestFid != "" {
|
||||
oldDownloader := rc.downloaders[oldestFid]
|
||||
@@ -290,6 +302,23 @@ func (rc *ReaderCache) remove(downloader *SingleChunkCacher) {
|
||||
}
|
||||
}
|
||||
|
||||
// removeUnpinned drops a cacher only while no stream is positioned in it.
|
||||
// Budget eviction picks its victim under the budget lock, so the pin check
|
||||
// and the map removal must happen together under the ReaderCache lock.
|
||||
func (rc *ReaderCache) removeUnpinned(downloader *SingleChunkCacher) (removed bool) {
|
||||
rc.Lock()
|
||||
removed = rc.downloaders[downloader.chunkFileId] == downloader &&
|
||||
atomic.LoadInt32(&downloader.pins) == 0
|
||||
if removed {
|
||||
delete(rc.downloaders, downloader.chunkFileId)
|
||||
}
|
||||
rc.Unlock()
|
||||
if removed {
|
||||
downloader.destroy()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// removeConsumed drops a cacher once its buffer was fully read, or the
|
||||
// streams positioned in it have left, and no readers remain attached or
|
||||
// pinned. The checks run under the ReaderCache lock so a reader attaching
|
||||
|
||||
Reference in new issue
Block a user