mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 14:31:57 +02:00
8db41d0217
* cache resolved chunk manifests for Mount * Address PR review: per-mount cache, singleflight, reuse ResolveOneChunkManifest - Own the manifest cache per WFS mount instead of a process-global variable, so manifests from one filer backend are never served to another (Devin/CodeRabbit major bug). - Coalesce concurrent cold misses via singleflight so only one fetch runs during a cold burst (Greptile P2). - Copy cached data after releasing the mutex so a large copy does not block concurrent hits, inserts, and evictions (CodeRabbit nitpick). - Reuse the existing ResolveOneChunkManifest function name instead of introducing a new resolveOneChunkManifest wrapper. - Validate (unmarshal) manifest bytes before caching so malformed manifests do not poison the cache. - Add TestChunkGroupManifestResolutionCoalescesColdMisses covering the singleflight cold-miss path. * Address round 2 review: coalesced-miss cancellation, test overlap - Use singleflight.DoChan in fetchOrLoad and select on ctx.Done() so a caller whose context is canceled while waiting for an in-flight fetch returns ctx.Err() promptly instead of blocking for the leader's result (Devin BUG). - Add TestResolveOneChunkManifestCanceledWaiterReturnsDuringCoalescedMiss covering the canceled-waiter path. - Delay the cold-miss fixture response so the leader's fetch is still in flight when concurrent opens join the singleflight, making the one-fetch assertions reliable (CodeRabbit Minor). * Address review: keep ResolveOneChunkManifest four-argument Restore the exported ResolveOneChunkManifest to its original four-argument signature so external callers keep compiling. Move the cache-aware resolution into an unexported resolveOneChunkManifest helper that accepts the per-mount ChunkManifestCache. The exported function delegates to the helper with a nil cache, preserving the historical uncached behavior for every non-Mount caller. The Mount path (ChunkGroup.SetChunks) now calls the unexported helper with the mount-owned cache. Tests and benchmarks that exercise the cache path call the unexported helper directly. --------- Co-authored-by: Chris Lu <chris.lu@gmail.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
73 lines
2.4 KiB
Go
73 lines
2.4 KiB
Go
package filer
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
func BenchmarkManifestResolutionRepeatedOpen(b *testing.B) {
|
|
for _, delay := range []time.Duration{0, 5 * time.Millisecond, 20 * time.Millisecond} {
|
|
b.Run(fmt.Sprintf("latency=%s", delay), func(b *testing.B) {
|
|
fixture := newManifestReadFixture(b, map[string][]*filer_pb.FileChunk{
|
|
"benchmark-cached": {resolveTestData("benchmark-data", 0)},
|
|
}, map[string]time.Duration{"benchmark-cached": delay})
|
|
var lookups atomic.Int32
|
|
lookup := func(ctx context.Context, fileID string) ([]string, error) {
|
|
lookups.Add(1)
|
|
return fixture.lookup(ctx, fileID)
|
|
}
|
|
chunk := newManifestCacheTestChunk("benchmark-cached")
|
|
cache := NewChunkManifestCache(MaxMountChunkManifestCacheEntries, MaxMountChunkManifestCacheBytes)
|
|
_, err := resolveOneChunkManifest(context.Background(), lookup, chunk, nil, cache)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
fixture.loads.Store(0)
|
|
lookups.Store(0)
|
|
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
if _, err := resolveOneChunkManifest(context.Background(), lookup, chunk, nil, cache); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
b.StopTimer()
|
|
b.ReportMetric(float64(fixture.loads.Load())/float64(b.N), "fetches/op")
|
|
b.ReportMetric(float64(lookups.Load())/float64(b.N), "lookups/op")
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkManifestResolutionWithoutCache(b *testing.B) {
|
|
for _, delay := range []time.Duration{0, 5 * time.Millisecond, 20 * time.Millisecond} {
|
|
b.Run(fmt.Sprintf("latency=%s", delay), func(b *testing.B) {
|
|
fixture := newManifestReadFixture(b, map[string][]*filer_pb.FileChunk{
|
|
"benchmark-uncached": {resolveTestData("benchmark-data", 0)},
|
|
}, map[string]time.Duration{"benchmark-uncached": delay})
|
|
var lookups atomic.Int32
|
|
lookup := func(ctx context.Context, fileID string) ([]string, error) {
|
|
lookups.Add(1)
|
|
return fixture.lookup(ctx, fileID)
|
|
}
|
|
chunk := newManifestCacheTestChunk("benchmark-uncached")
|
|
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
if _, err := ResolveOneChunkManifest(context.Background(), lookup, chunk, nil); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
b.StopTimer()
|
|
b.ReportMetric(float64(fixture.loads.Load())/float64(b.N), "fetches/op")
|
|
b.ReportMetric(float64(lookups.Load())/float64(b.N), "lookups/op")
|
|
})
|
|
}
|
|
}
|