mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-12 14:58:44 +01:00
feat(mount): singleflight dedup for concurrent chunk reads (#9100)
* feat(mount): add singleflight deduplication for concurrent chunk reads When multiple FUSE readers request the same uncached chunk concurrently, only one network fetch is performed. Other readers wait and share the downloaded data, reducing redundant volume server traffic under parallel read workloads. * fix(util): make singleflight panic-safe with defer cleanup If the provided function panics, the WaitGroup and map entry are now cleaned up via defer, preventing other waiters from hanging forever. * fix(filer): remove singleflight from reader_cache to fix buffer ownership The singleflight wrapper around chunk fetches returned the same []byte buffer to concurrent callers. Since each SingleChunkCacher owns and frees its data buffer in destroy(), sharing the same slice would cause a use-after-free or double-free with the mem allocator. The downloaders map already deduplicates in-flight downloads for the same fileId, so the singleflight was redundant at this layer. The SingleFlightGroup utility is retained for use elsewhere.
This commit is contained in:
1 parent
e1fa4ec756
commit
886d50a6a5
4 files changed
+258
No files matched your search
@@ -433,6 +433,46 @@ func TestSingleChunkCacherMultipleReadersWaitForDownload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderCacheDownloaderDedup tests that concurrent ReadChunkAt calls for
|
||||
// the same fileId result in only one network fetch (lookup call), because
|
||||
// the downloaders map deduplicates in-flight downloads.
|
||||
func TestReaderCacheDownloaderDedup(t *testing.T) {
|
||||
cache := newMockChunkCacheForReaderCache()
|
||||
var lookupCount int32
|
||||
lookupGate := make(chan struct{})
|
||||
|
||||
lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
|
||||
atomic.AddInt32(&lookupCount, 1)
|
||||
<-lookupGate
|
||||
// Return an error so we don't need to mock the HTTP fetch.
|
||||
return nil, fmt.Errorf("simulated lookup for %s", fileId)
|
||||
}
|
||||
|
||||
rc := NewReaderCache(10, cache, lookupFn)
|
||||
defer rc.destroy()
|
||||
|
||||
const numReaders = 10
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(numReaders)
|
||||
|
||||
for i := 0; i < numReaders; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
buffer := make([]byte, 100)
|
||||
rc.ReadChunkAt(context.Background(), buffer, "dedup-file", nil, false, 0, 100, false)
|
||||
}()
|
||||
}
|
||||
|
||||
// Allow downloads to proceed.
|
||||
close(lookupGate)
|
||||
wg.Wait()
|
||||
|
||||
count := atomic.LoadInt32(&lookupCount)
|
||||
if count != 1 {
|
||||
t.Errorf("expected exactly 1 lookup call, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleChunkCacherOneReaderCancelsOthersContinue tests that when one reader
|
||||
// cancels, other readers waiting on the same chunk continue to wait.
|
||||
func TestSingleChunkCacherOneReaderCancelsOthersContinue(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user