storage: hold the data-file read lock across the append-time search (#12789)

* storage: red test for the append-time search racing a vacuum commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: hold the data-file read lock across the append-time search

Fixes #12785

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: Version() writes the superblock version only when it changes

Version() stored the .vif version into SuperBlock.Version on every call, so
each call was a write that raced with readers of the superblock that do not
take superBlockAccessLock, such as the append-time search and the
compaction setup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: harden the append-time search race test; add a lock-hold budget test

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: Linux-only lock budget; stop before reading search failures

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 15:01:47 -08:00
1 parent f7323f2b63
commit 05427e8108
3 files changed
+262 -7

No files matched your search

+1 -1
View File
@@ -185,7 +185,7 @@ func (v *Volume) RelocateIndexTo(newIdxDir string) error {
func (v *Volume) Version() needle.Version {
v.superBlockAccessLock.Lock()
defer v.superBlockAccessLock.Unlock()
if v.volumeInfo.Version != 0 {
if v.volumeInfo.Version != 0 && v.SuperBlock.Version != needle.Version(v.volumeInfo.Version) {
v.SuperBlock.Version = needle.Version(v.volumeInfo.Version)
}
return v.SuperBlock.Version
+25 -6
View File
@@ -382,10 +382,18 @@ func (v *Volume) ResumeByLastEntry(entryCount uint64, key NeedleId, cookie uint3
return
}
// on server side
// on server side. The read lock is held for the whole search: it takes the
// index size once and then probes .idx rows and .dat records, so a vacuum
// commit (which swaps both files under the write lock) must not land
// mid-search.
func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast bool, err error) {
v.dataFileAccessLock.RLock()
defer v.dataFileAccessLock.RUnlock()
if v.nm == nil {
return Offset{}, false, io.EOF
}
fileSize := int64(v.IndexFileSize())
fileSize := int64(v.nm.IndexFileSize())
if fileSize%NeedleMapEntrySize != 0 {
err = fmt.Errorf("unexpected file %s.idx size: %d", v.IndexFileName(), fileSize)
return
@@ -404,7 +412,7 @@ func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast
}
// read the appendAtNs for entry m
offset, err = v.readOffsetFromIndex(m)
offset, err = v.readOffsetFromIndexLocked(m)
if err != nil {
err = fmt.Errorf("read entry %d: %v", m, err)
return
@@ -458,7 +466,7 @@ func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast
return Offset{}, true, nil
}
offset, err = v.readOffsetFromIndex(l)
offset, err = v.readOffsetFromIndexLocked(l)
return offset, false, err
@@ -471,7 +479,7 @@ func (v *Volume) readRightNs(m, max int64) (index int64, offset Offset, ts uint6
if index >= max {
return
}
offset, err = v.readOffsetFromIndex(index)
offset, err = v.readOffsetFromIndexLocked(index)
if err != nil {
err = fmt.Errorf("read left entry at %d: %v", index, err)
return
@@ -490,7 +498,7 @@ func (v *Volume) readLeftNs(m int64) (index int64, offset Offset, ts uint64, err
if index < 0 {
return
}
offset, err = v.readOffsetFromIndex(index)
offset, err = v.readOffsetFromIndexLocked(index)
if err != nil {
err = fmt.Errorf("read right entry at %d: %v", index, err)
return
@@ -505,6 +513,11 @@ func (v *Volume) readLeftNs(m int64) (index int64, offset Offset, ts uint64, err
func (v *Volume) readIndexEntry(m int64) (key NeedleId, offset Offset, size Size, err error) {
v.dataFileAccessLock.RLock()
defer v.dataFileAccessLock.RUnlock()
return v.readIndexEntryLocked(m)
}
// readIndexEntryLocked is readIndexEntry for a caller that holds dataFileAccessLock.
func (v *Volume) readIndexEntryLocked(m int64) (key NeedleId, offset Offset, size Size, err error) {
if v.nm == nil {
return 0, Offset{}, 0, io.EOF
}
@@ -517,6 +530,12 @@ func (v *Volume) readOffsetFromIndex(m int64) (Offset, error) {
return offset, err
}
// readOffsetFromIndexLocked is readOffsetFromIndex for a caller that holds dataFileAccessLock.
func (v *Volume) readOffsetFromIndexLocked(m int64) (Offset, error) {
_, offset, _, err := v.readIndexEntryLocked(m)
return offset, err
}
// DatSizeCoveredByIndex returns the .dat end of the last record the index
// rows before indexSize cover, or the superblock size when no row inside the
// bound names a record. Rows are not always in .dat order — rebuilds can
@@ -0,0 +1,236 @@
package storage
import (
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// startSearcher runs fn in a loop until the returned stop is called. stop is
// failure-safe (it runs from a defer, also after t.Fatal) and joins the
// goroutine with a bound; it reports whether the goroutine ended.
func startSearcher(t *testing.T, fn func()) (stop func() bool) {
var halt atomic.Bool
done := make(chan struct{})
go func() {
defer close(done)
for !halt.Load() {
fn()
}
}()
return func() bool {
halt.Store(true)
select {
case <-done:
return true
case <-time.After(30 * time.Second):
t.Errorf("searcher did not stop within 30s")
return false
}
}
}
// The append-time search (tail and incremental copy) must stay correct while a
// vacuum commit swaps the volume's files: it takes the index size once and
// then probes .idx rows and .dat records, so a commit that lands mid-search
// must not turn into a read error (EOF past the new, smaller files) or a
// wrong answer.
func TestBinarySearchByAppendAtNsDuringCompactionCommit(t *testing.T) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
joined := true
defer func() {
if joined { // never close the volume under a searcher that is still running
v.Close()
}
}()
next := uint64(1)
write := func(n int) {
for i := 0; i < n; i++ {
if _, _, _, err := v.writeNeedle2(newRandomNeedle(next), true, false, false); err != nil {
t.Fatalf("write %d: %v", next, err)
}
next++
}
}
write(2000)
var (
inCommit atomic.Bool
commitsBegun atomic.Int64
searches atomic.Int64
overlapped atomic.Int64
kind atomic.Int64
mu sync.Mutex
failures []string
)
fail := func(format string) {
mu.Lock()
failures = append(failures, format)
mu.Unlock()
}
stop := startSearcher(t, func() {
before, busy := commitsBegun.Load(), inCommit.Load()
k := kind.Add(1) % 3
var since uint64
switch k {
case 0:
since = 0 // before every record: the first record, never "last"
case 1:
since = uint64(time.Now().Add(time.Hour).UnixNano()) // after every record: "last"
default:
since = uint64(time.Now().UnixNano()) - uint64(time.Second) // inside the history
}
offset, isLast, err := v.BinarySearchByAppendAtNs(since)
if busy || inCommit.Load() || commitsBegun.Load() != before {
overlapped.Add(1)
}
searches.Add(1)
switch {
case err != nil:
fail("search error: " + err.Error())
case k == 0 && (isLast || offset.IsZero()):
fail("search from 0 did not return the first record")
case k == 1 && !isLast:
fail("search from the future did not report the end")
}
})
defer func() { joined = stop() }()
deadline := time.Now().Add(3 * time.Second)
rounds := 0
for time.Now().Before(deadline) {
// shrink the volume on every commit: delete the newest live needles, add a few
for id := next - 1; id > 0 && id+400 > next; id-- {
if _, err := v.deleteNeedle2(newEmptyNeedle(id), false); err != nil {
t.Fatalf("delete %d: %v", id, err)
}
}
write(400)
if err := v.CompactByIndex(nil); err != nil {
t.Fatalf("compact round %d: %v", rounds, err)
}
inCommit.Store(true)
commitsBegun.Add(1)
err := v.CommitCompact()
inCommit.Store(false)
if err != nil {
t.Fatalf("commit round %d: %v", rounds, err)
}
rounds++
}
if joined = stop(); !joined {
return // the searcher may still be appending to failures; stop already reported it
}
mu.Lock()
failed := append([]string(nil), failures...)
mu.Unlock()
t.Logf("%d compaction commits, %d searches (%d overlapping a commit), %d failed", rounds, searches.Load(), overlapped.Load(), len(failed))
if overlapped.Load() == 0 {
t.Fatalf("no search overlapped a commit; the test did not exercise the race")
}
if len(failed) > 0 {
t.Fatalf("append-time search failed during compaction commits (first of %d): %s", len(failed), failed[0])
}
}
// The search holds the data-file read lock for its whole run, so its length
// bounds how long a foreground write can wait. The slowest search is one whose
// probe lands in a long run of index-only tombstones (offset zero): it scans
// left and right row by row. With such a run of searchLockRunRows rows, a
// search and a foreground write must stay within searchLockBudget. The budget
// is a Linux figure (about 5 ms measured there); other platforms only log the
// times, since per-row file reads are much slower there (about 0.8 s per search
// measured on Windows). Correctness (no search error) is checked everywhere.
const (
searchLockRunRows = 20000
searchLockBudget = 250 * time.Millisecond
)
func TestBinarySearchByAppendAtNsLockHoldBudget(t *testing.T) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
joined := true
defer func() {
if joined {
v.Close()
}
}()
next := uint64(1)
write := func(n int) {
for i := 0; i < n; i++ {
if _, _, _, err := v.writeNeedle2(newRandomNeedle(next), true, false, false); err != nil {
t.Fatalf("write %d: %v", next, err)
}
next++
}
}
write(500)
mid := uint64(time.Now().UnixNano())
// a long run of index-only tombstones (as remote-tier deletes write)
for i := 0; i < searchLockRunRows; i++ {
if err := v.nm.Delete(NeedleId(1_000_000+i), Offset{}); err != nil {
t.Fatalf("index-only tombstone %d: %v", i, err)
}
}
write(500)
var (
maxSearch atomic.Int64
searches atomic.Int64
)
stop := startSearcher(t, func() {
start := time.Now()
if _, _, err := v.BinarySearchByAppendAtNs(mid); err != nil {
t.Errorf("search: %v", err)
}
if d := int64(time.Since(start)); d > maxSearch.Load() {
maxSearch.Store(d)
}
searches.Add(1)
})
defer func() { joined = stop() }()
var maxWrite time.Duration
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
start := time.Now()
write(1)
if d := time.Since(start); d > maxWrite {
maxWrite = d
}
}
if joined = stop(); !joined {
return
}
t.Logf("%d searches over a %d-row tombstone run: slowest search %v, slowest foreground write %v (budget %v)",
searches.Load(), searchLockRunRows, time.Duration(maxSearch.Load()), maxWrite, searchLockBudget)
if searches.Load() == 0 {
t.Fatal("no search ran")
}
if runtime.GOOS != "linux" {
return
}
if s := time.Duration(maxSearch.Load()); s > searchLockBudget {
t.Errorf("a search held the read lock for %v, over the %v budget", s, searchLockBudget)
}
if maxWrite > searchLockBudget {
t.Errorf("a foreground write waited %v, over the %v budget", maxWrite, searchLockBudget)
}
}