mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 12:34:29 +02:00
storage: hash index-metric bloom filter with xxhash
The reverse index walk dedupes seen NeedleIds with a BoomFilters bloom filter. Its default kernel is FNV-1, whose final step only XORs the last input byte into the hash's low bits, so keys sharing the high 7 bytes of a big-endian NeedleId land on correlated probe positions and false positives arrive in bursts of hundreds instead of the ~0.001 target rate. Each false positive hides a live-latest key and inflates the rebuilt deletion counters, which is what made TestFastLoadingNeedleMapMetrics flake past its +256 bound in CI (observed DeletedCount drift of 267). Swap the kernel for xxhash, which fully diffuses the key bytes; the observed false-positive count in 300 local runs of the test drops to max 2 (avg 0.23).
This commit is contained in:
1 parent
d04d838ebd
commit
ab559bca76
1 file changed
+7
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
@@ -155,6 +156,12 @@ func needleMapMetricFromIndexFile(r *os.File, mm *mapMetric, version needle.Vers
|
||||
var liveKeyBytes uint64
|
||||
err := reverseWalkIndexFile(r, func(entryCount int64) {
|
||||
bf = boom.NewBloomFilter(uint(entryCount), 0.001)
|
||||
// The default FNV-1 kernel leaves the last byte of a big-endian
|
||||
// NeedleId as the final XOR, so keys in the same 256-block get
|
||||
// correlated probe positions and false positives arrive in bursts
|
||||
// of hundreds. xxhash diffuses the input fully and keeps the
|
||||
// observed rate near the 0.001 target.
|
||||
bf.SetHash(xxhash.New())
|
||||
}, func(key NeedleId, offset Offset, size Size) error {
|
||||
|
||||
mm.MaybeSetMaxFileKey(key)
|
||||
|
||||
Reference in new issue
Block a user