mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
Chris Lu
1c130c2d47
* fix(mount): close inodeLocks cleanup race that allowed two flock holders
PosixLockTable.getOrCreateInodeLocks released plt.mu before the caller
acquired il.mu. A concurrent maybeCleanupInode could delete the map
entry in that window; the first caller would then insert its lock into
the orphaned inodeLocks while a later caller created a fresh entry in
the map, so findConflict never observed the orphaned lock and two
owners could simultaneously believe they held the same exclusive flock.
This matches the flaky CI failure seen in
TestPosixFileLocking/ConcurrentLockContention:
Error: Should be empty, but was [worker N: flock overlap detected with 2 holders]
Mark removed inodeLocks as dead under plt.mu+il.mu, and have SetLk /
SetLkw recheck the flag after locking il.mu, refetching the live entry
from the map when orphaned. Also delete the map entry only if it still
points to this il, so a racing recreate is not clobbered.
Adds TestConcurrentFlockChurnPreservesMutualExclusion: 16 goroutines x
500 flock/unflock iterations on one inode. Reliably reports 500+
overlaps per run before the fix; clean across 100 race-enabled runs
after.
* fix(mount): extend dead-flag contract to GetLk and self-heal primitives
Address review feedback on the initial cleanup-race fix:
1. GetLk had the same stale-pointer bug as SetLk. A caller could grab
an inodeLocks pointer, have cleanup orphan it and a replacement il
receive a conflicting lock, then answer F_UNLCK off the empty dead
pointer. Add the same dead recheck + refetch loop.
2. getOrCreateInodeLocks and getInodeLocks now treat a dead map entry
as defective: the former replaces it with a fresh inodeLocks, the
latter drops it and returns nil. Production cannot reach that state
(maybeCleanupInode atomically deletes under plt.mu when it sets
dead), but the hardening guarantees the SetLk / SetLkw / GetLk
retry loops always make progress even if a future refactor reorders
those operations, and it lets the white-box tests set up a stale
dead entry without spinning.
3. Strengthen the regression suite:
- TestSetLkRetriesPastDeadInodeLocks: deterministic white-box test
that installs a dead il in the map and asserts SetLk routes the
new lock into a fresh il (not the orphan), that GetLk reports the
resulting conflict, and that a different-owner acquire is rejected
with EAGAIN.
- TestGetInodeLocksEvictsDeadEntry: verifies both map-read primitives
drop or replace dead entries.
- TestConcurrentFlockChurnPreservesMutualExclusion: replace the
timing-fragile Add(1)-and-check counter with a Swap+CAS detector.
Each worker claims a slot after SetLk OK and releases it before
UN, flagging both an observed predecessor and a lost CAS on
release. Against a reverted fix the detector fires 1000+ times per
run; with the fix clean across 100 race-enabled iterations.
* test(mount): fail fast on unexpected SetLk statuses in churn loop
The stress test blindly spun on any non-OK SetLk status and discarded
the unlock return. If SetLk ever returns something other than OK or
EAGAIN (e.g. after a future refactor introduces a new error), the
acquire loop would spin forever and an unlock failure would be
silently swallowed.
Capture the acquire status, retry only on the expected EAGAIN, and
assert unlock returns OK. Use t.Errorf + return (not t.Fatalf) because
the checks run on worker goroutines where FailNow is unsafe. The
Swap+CAS overlap detector is unchanged.
825 lines
26 KiB
Go
825 lines
26 KiB
Go
package mount
|
|
|
|
import (
|
|
"math"
|
|
"runtime"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/go-fuse/v2/fuse"
|
|
)
|
|
|
|
func TestNonOverlappingLocksFromDifferentOwners(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
s1 := plt.SetLk(inode, lockRange{Start: 0, End: 49, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
if s1 != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s1)
|
|
}
|
|
s2 := plt.SetLk(inode, lockRange{Start: 50, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
|
|
if s2 != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s2)
|
|
}
|
|
}
|
|
|
|
func TestOverlappingReadLocksFromDifferentOwners(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
s1 := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
|
|
if s1 != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s1)
|
|
}
|
|
s2 := plt.SetLk(inode, lockRange{Start: 50, End: 149, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
|
|
if s2 != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s2)
|
|
}
|
|
}
|
|
|
|
func TestOverlappingWriteReadConflict(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
s := plt.SetLk(inode, lockRange{Start: 50, End: 149, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
|
|
if s != fuse.EAGAIN {
|
|
t.Fatalf("expected EAGAIN, got %v", s)
|
|
}
|
|
}
|
|
|
|
func TestOverlappingWriteWriteConflict(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
s := plt.SetLk(inode, lockRange{Start: 50, End: 149, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
|
|
if s != fuse.EAGAIN {
|
|
t.Fatalf("expected EAGAIN, got %v", s)
|
|
}
|
|
}
|
|
|
|
func TestSameOwnerUpgradeReadToWrite(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
|
|
s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK for same-owner upgrade, got %v", s)
|
|
}
|
|
|
|
// Verify the lock is now a write lock.
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, &out)
|
|
if out.Lk.Typ != syscall.F_WRLCK {
|
|
t.Fatalf("expected conflicting write lock, got type %d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
func TestSameOwnerDowngradeWriteToRead(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK for same-owner downgrade, got %v", s)
|
|
}
|
|
|
|
// Another owner should now be able to get a read lock.
|
|
s2 := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
|
|
if s2 != fuse.OK {
|
|
t.Fatalf("expected OK for shared read lock, got %v", s2)
|
|
}
|
|
}
|
|
|
|
func TestLockCoalescing(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 9, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.SetLk(inode, lockRange{Start: 10, End: 19, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
il := plt.getInodeLocks(inode)
|
|
il.mu.Lock()
|
|
ownerLocks := 0
|
|
for _, lk := range il.locks {
|
|
if lk.Owner == 1 {
|
|
ownerLocks++
|
|
if lk.Start != 0 || lk.End != 19 {
|
|
t.Errorf("expected coalesced lock [0,19], got [%d,%d]", lk.Start, lk.End)
|
|
}
|
|
}
|
|
}
|
|
il.mu.Unlock()
|
|
if ownerLocks != 1 {
|
|
t.Fatalf("expected 1 coalesced lock, got %d", ownerLocks)
|
|
}
|
|
}
|
|
|
|
func TestLockSplitting(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
// Unlock the middle portion.
|
|
plt.SetLk(inode, lockRange{Start: 40, End: 59, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
|
|
|
|
il := plt.getInodeLocks(inode)
|
|
il.mu.Lock()
|
|
ownerLocks := 0
|
|
for _, lk := range il.locks {
|
|
if lk.Owner == 1 {
|
|
ownerLocks++
|
|
}
|
|
}
|
|
if ownerLocks != 2 {
|
|
il.mu.Unlock()
|
|
t.Fatalf("expected 2 locks after split, got %d", ownerLocks)
|
|
}
|
|
// Check the ranges.
|
|
if il.locks[0].Start != 0 || il.locks[0].End != 39 {
|
|
t.Errorf("expected left lock [0,39], got [%d,%d]", il.locks[0].Start, il.locks[0].End)
|
|
}
|
|
if il.locks[1].Start != 60 || il.locks[1].End != 99 {
|
|
t.Errorf("expected right lock [60,99], got [%d,%d]", il.locks[1].Start, il.locks[1].End)
|
|
}
|
|
il.mu.Unlock()
|
|
}
|
|
|
|
func TestGetLkConflict(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 10, End: 50, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 30, End: 70, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20}, &out)
|
|
if out.Lk.Typ != syscall.F_WRLCK {
|
|
t.Fatalf("expected conflicting write lock, got type %d", out.Lk.Typ)
|
|
}
|
|
if out.Lk.Pid != 10 {
|
|
t.Fatalf("expected holder PID 10, got %d", out.Lk.Pid)
|
|
}
|
|
if out.Lk.Start != 10 || out.Lk.End != 50 {
|
|
t.Fatalf("expected conflict [10,50], got [%d,%d]", out.Lk.Start, out.Lk.End)
|
|
}
|
|
}
|
|
|
|
func TestGetLkNoConflict(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 10, End: 50, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
|
|
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 30, End: 70, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20}, &out)
|
|
if out.Lk.Typ != syscall.F_UNLCK {
|
|
t.Fatalf("expected F_UNLCK (no conflict), got type %d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
func TestGetLkSameOwnerNoConflict(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10}, &out)
|
|
if out.Lk.Typ != syscall.F_UNLCK {
|
|
t.Fatalf("same owner should not conflict with itself, got type %d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
func TestReleaseOwner(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 49, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.SetLk(inode, lockRange{Start: 50, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.SetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
|
|
|
|
plt.ReleaseOwner(inode, 1)
|
|
|
|
// Owner 1's locks should be gone.
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, &out)
|
|
if out.Lk.Typ != syscall.F_UNLCK {
|
|
t.Fatalf("expected no conflict after ReleaseOwner, got type %d", out.Lk.Typ)
|
|
}
|
|
|
|
// Owner 2's lock should still exist.
|
|
plt.GetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, &out)
|
|
if out.Lk.Typ != syscall.F_RDLCK {
|
|
t.Fatalf("expected owner 2's read lock to remain, got type %d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
func TestDifferentLockKindsDoNotConflict(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
s1 := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
if s1 != fuse.OK {
|
|
t.Fatalf("expected POSIX lock OK, got %v", s1)
|
|
}
|
|
|
|
s2 := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20, IsFlock: true})
|
|
if s2 != fuse.OK {
|
|
t.Fatalf("expected flock lock OK in separate namespace, got %v", s2)
|
|
}
|
|
}
|
|
|
|
func TestReleasePosixOwnerReleasesPosixLocksAndWakesWaiters(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
done := make(chan fuse.Status, 1)
|
|
go func() {
|
|
cancel := make(chan struct{})
|
|
done <- plt.SetLkw(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
|
|
}()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
plt.ReleasePosixOwner(inode, 1)
|
|
|
|
select {
|
|
case s := <-done:
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK after ReleasePosixOwner, got %v", s)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("SetLkw did not unblock after ReleasePosixOwner")
|
|
}
|
|
}
|
|
|
|
func TestReleasePosixOwnerDoesNotReleaseFlockLocks(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10, IsFlock: true})
|
|
plt.ReleasePosixOwner(inode, 1)
|
|
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20, IsFlock: true}, &out)
|
|
if out.Lk.Typ != syscall.F_WRLCK {
|
|
t.Fatalf("expected flock lock to remain after ReleasePosixOwner, got type %d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
func TestWakeEligibleWaitersKeepsInodeUntilWakeRefReleased(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
il := plt.getOrCreateInodeLocks(inode)
|
|
waiter := &lockWaiter{
|
|
requested: lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20},
|
|
ch: make(chan struct{}),
|
|
}
|
|
|
|
il.mu.Lock()
|
|
il.waiters = append(il.waiters, waiter)
|
|
il.mu.Unlock()
|
|
|
|
plt.releaseMatching(inode, func(lockRange) bool { return false })
|
|
|
|
select {
|
|
case <-waiter.ch:
|
|
// Expected.
|
|
default:
|
|
t.Fatal("expected waiter to be woken")
|
|
}
|
|
|
|
plt.mu.Lock()
|
|
_, exists := plt.inodes[inode]
|
|
plt.mu.Unlock()
|
|
if !exists {
|
|
t.Fatal("inodeLocks should remain while a woken waiter still holds a wake ref")
|
|
}
|
|
|
|
il.mu.Lock()
|
|
releaseWakeRef(il, waiter)
|
|
il.mu.Unlock()
|
|
plt.maybeCleanupInode(inode, il)
|
|
|
|
plt.mu.Lock()
|
|
_, exists = plt.inodes[inode]
|
|
plt.mu.Unlock()
|
|
if exists {
|
|
t.Fatal("inodeLocks should be cleaned up after the final wake ref is released")
|
|
}
|
|
}
|
|
|
|
func TestReleaseFlockOwnerDoesNotReleasePosixLocks(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 10, IsFlock: true})
|
|
|
|
plt.ReleaseFlockOwner(inode, 2)
|
|
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, &out)
|
|
if out.Lk.Typ != syscall.F_WRLCK {
|
|
t.Fatalf("expected POSIX lock to remain after ReleaseFlockOwner, got type %d", out.Lk.Typ)
|
|
}
|
|
|
|
plt.GetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 4, Pid: 40, IsFlock: true}, &out)
|
|
if out.Lk.Typ != syscall.F_UNLCK {
|
|
t.Fatalf("expected flock lock to be removed after ReleaseFlockOwner, got type %d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
func TestReleaseOwnerWakesWaiters(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
done := make(chan fuse.Status, 1)
|
|
go func() {
|
|
cancel := make(chan struct{})
|
|
s := plt.SetLkw(inode, lockRange{Start: 50, End: 60, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
|
|
done <- s
|
|
}()
|
|
|
|
// Give the goroutine time to block.
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
plt.ReleaseOwner(inode, 1)
|
|
|
|
select {
|
|
case s := <-done:
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK after ReleaseOwner woke waiter, got %v", s)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("SetLkw did not unblock after ReleaseOwner")
|
|
}
|
|
}
|
|
|
|
func TestSetLkwBlocksAndSucceeds(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
done := make(chan fuse.Status, 1)
|
|
go func() {
|
|
cancel := make(chan struct{})
|
|
s := plt.SetLkw(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
|
|
done <- s
|
|
}()
|
|
|
|
// Give the goroutine time to block.
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
// Release the conflicting lock.
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
|
|
|
|
select {
|
|
case s := <-done:
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("SetLkw did not unblock after conflicting lock was released")
|
|
}
|
|
}
|
|
|
|
func TestSetLkwCancellation(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
cancel := make(chan struct{})
|
|
done := make(chan fuse.Status, 1)
|
|
go func() {
|
|
s := plt.SetLkw(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20}, cancel)
|
|
done <- s
|
|
}()
|
|
|
|
// Give the goroutine time to block.
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
close(cancel)
|
|
|
|
select {
|
|
case s := <-done:
|
|
if s != fuse.EINTR {
|
|
t.Fatalf("expected EINTR on cancel, got %v", s)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("SetLkw did not unblock after cancel")
|
|
}
|
|
}
|
|
|
|
func TestWholeFileLock(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
// Simulate flock() — whole-file exclusive lock.
|
|
s1 := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
if s1 != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s1)
|
|
}
|
|
|
|
// Second owner should be blocked.
|
|
s2 := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
|
|
if s2 != fuse.EAGAIN {
|
|
t.Fatalf("expected EAGAIN, got %v", s2)
|
|
}
|
|
|
|
// Even a partial overlap should fail.
|
|
s3 := plt.SetLk(inode, lockRange{Start: 100, End: 200, Typ: syscall.F_RDLCK, Owner: 2, Pid: 20})
|
|
if s3 != fuse.EAGAIN {
|
|
t.Fatalf("expected EAGAIN for partial overlap with whole-file lock, got %v", s3)
|
|
}
|
|
}
|
|
|
|
func TestUnlockNoExistingLocks(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
// Unlock on an inode with no locks should succeed silently.
|
|
s := plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK for unlock with no existing locks, got %v", s)
|
|
}
|
|
}
|
|
|
|
func TestMultipleInodesIndependent(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
|
|
// Write lock on inode 1 should not affect inode 2.
|
|
plt.SetLk(1, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
s := plt.SetLk(2, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
|
|
if s != fuse.OK {
|
|
t.Fatalf("locks on different inodes should be independent, got %v", s)
|
|
}
|
|
}
|
|
|
|
func TestMemoryCleanup(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.ReleaseOwner(inode, 1)
|
|
|
|
plt.mu.Lock()
|
|
_, exists := plt.inodes[inode]
|
|
plt.mu.Unlock()
|
|
if exists {
|
|
t.Fatal("expected inode entry to be cleaned up after all locks released")
|
|
}
|
|
}
|
|
|
|
func TestSelectiveWaking(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
// Owner 1 holds write lock on [0, 99], owner 2 holds write lock on [200, 299].
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.SetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_WRLCK, Owner: 2, Pid: 20})
|
|
|
|
// Owner 3 waits for [50, 60] (blocked by owner 1).
|
|
done3 := make(chan fuse.Status, 1)
|
|
go func() {
|
|
cancel := make(chan struct{})
|
|
s := plt.SetLkw(inode, lockRange{Start: 50, End: 60, Typ: syscall.F_WRLCK, Owner: 3, Pid: 30}, cancel)
|
|
done3 <- s
|
|
}()
|
|
// Owner 4 waits for [250, 260] (blocked by owner 2).
|
|
done4 := make(chan fuse.Status, 1)
|
|
go func() {
|
|
cancel := make(chan struct{})
|
|
s := plt.SetLkw(inode, lockRange{Start: 250, End: 260, Typ: syscall.F_WRLCK, Owner: 4, Pid: 40}, cancel)
|
|
done4 <- s
|
|
}()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
// Release owner 1's lock. Only owner 3 should be woken; owner 4 is still blocked.
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_UNLCK, Owner: 1, Pid: 10})
|
|
|
|
select {
|
|
case s := <-done3:
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK for owner 3, got %v", s)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("owner 3 was not woken after owner 1 released")
|
|
}
|
|
|
|
// Owner 4 should still be blocked.
|
|
select {
|
|
case s := <-done4:
|
|
t.Fatalf("owner 4 should still be blocked, but got %v", s)
|
|
case <-time.After(100 * time.Millisecond):
|
|
// Expected — still blocked.
|
|
}
|
|
|
|
// Now release owner 2's lock. Owner 4 should wake.
|
|
plt.SetLk(inode, lockRange{Start: 200, End: 299, Typ: syscall.F_UNLCK, Owner: 2, Pid: 20})
|
|
|
|
select {
|
|
case s := <-done4:
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK for owner 4, got %v", s)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("owner 4 was not woken after owner 2 released")
|
|
}
|
|
}
|
|
|
|
func TestSameOwnerReplaceDifferentType(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
// Lock [0, 99] as write.
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 99, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
// Replace middle portion [30, 60] with read lock.
|
|
plt.SetLk(inode, lockRange{Start: 30, End: 60, Typ: syscall.F_RDLCK, Owner: 1, Pid: 10})
|
|
|
|
il := plt.getInodeLocks(inode)
|
|
il.mu.Lock()
|
|
defer il.mu.Unlock()
|
|
|
|
// Should have 3 locks: write [0,29], read [30,60], write [61,99].
|
|
if len(il.locks) != 3 {
|
|
t.Fatalf("expected 3 locks after partial type change, got %d", len(il.locks))
|
|
}
|
|
if il.locks[0].Typ != syscall.F_WRLCK || il.locks[0].Start != 0 || il.locks[0].End != 29 {
|
|
t.Errorf("expected write [0,29], got type=%d [%d,%d]", il.locks[0].Typ, il.locks[0].Start, il.locks[0].End)
|
|
}
|
|
if il.locks[1].Typ != syscall.F_RDLCK || il.locks[1].Start != 30 || il.locks[1].End != 60 {
|
|
t.Errorf("expected read [30,60], got type=%d [%d,%d]", il.locks[1].Typ, il.locks[1].Start, il.locks[1].End)
|
|
}
|
|
if il.locks[2].Typ != syscall.F_WRLCK || il.locks[2].Start != 61 || il.locks[2].End != 99 {
|
|
t.Errorf("expected write [61,99], got type=%d [%d,%d]", il.locks[2].Typ, il.locks[2].Start, il.locks[2].End)
|
|
}
|
|
}
|
|
|
|
func TestNonAdjacentRangesNotCoalesced(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
// Lock [5, MaxUint64] then [0, 2] — gap at [3,4] must prevent coalescing.
|
|
plt.SetLk(inode, lockRange{Start: 5, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
s := plt.SetLk(inode, lockRange{Start: 0, End: 2, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
if s != fuse.OK {
|
|
t.Fatalf("expected OK, got %v", s)
|
|
}
|
|
|
|
il := plt.getInodeLocks(inode)
|
|
il.mu.Lock()
|
|
defer il.mu.Unlock()
|
|
|
|
if len(il.locks) != 2 {
|
|
t.Fatalf("expected 2 separate locks (gap [3,4] prevents coalescing), got %d", len(il.locks))
|
|
}
|
|
if il.locks[0].Start != 0 || il.locks[0].End != 2 {
|
|
t.Errorf("expected first lock [0,2], got [%d,%d]", il.locks[0].Start, il.locks[0].End)
|
|
}
|
|
if il.locks[1].Start != 5 || il.locks[1].End != math.MaxUint64 {
|
|
t.Errorf("expected second lock [5,MaxUint64], got [%d,%d]", il.locks[1].Start, il.locks[1].End)
|
|
}
|
|
}
|
|
|
|
func TestAdjacencyNoOverflowAtMaxUint64(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(1)
|
|
|
|
// Lock to EOF (End = MaxUint64), then lock [0, 0] same type.
|
|
// Without the overflow guard, MaxUint64+1 wraps to 0, falsely merging.
|
|
plt.SetLk(inode, lockRange{Start: 100, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
plt.SetLk(inode, lockRange{Start: 0, End: 0, Typ: syscall.F_WRLCK, Owner: 1, Pid: 10})
|
|
|
|
il := plt.getInodeLocks(inode)
|
|
il.mu.Lock()
|
|
defer il.mu.Unlock()
|
|
|
|
// Should remain 2 separate locks, not merged.
|
|
ownerLocks := 0
|
|
for _, lk := range il.locks {
|
|
if lk.Owner == 1 {
|
|
ownerLocks++
|
|
}
|
|
}
|
|
if ownerLocks != 2 {
|
|
t.Fatalf("expected 2 separate locks (no overflow merge), got %d", ownerLocks)
|
|
}
|
|
}
|
|
|
|
// TestSetLkRetriesPastDeadInodeLocks deterministically exercises the
|
|
// getOrCreateInodeLocks vs maybeCleanupInode race: a caller holding a
|
|
// pointer to an inodeLocks that is concurrently marked dead must refetch
|
|
// from the map instead of mutating the orphaned instance (which would be
|
|
// invisible to subsequent callers and let two exclusive flock holders
|
|
// coexist). The test bypasses scheduling by hand-installing a dead il into
|
|
// the table and asserting that the next SetLk routes around it.
|
|
func TestSetLkRetriesPastDeadInodeLocks(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(42)
|
|
|
|
// Acquire and release a lock so maybeCleanupInode marks the il dead and
|
|
// removes it from the map.
|
|
lock := lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, IsFlock: true}
|
|
if s := plt.SetLk(inode, lock); s != fuse.OK {
|
|
t.Fatalf("prime SetLk: got %v", s)
|
|
}
|
|
dead := plt.getInodeLocks(inode)
|
|
unlock := lock
|
|
unlock.Typ = syscall.F_UNLCK
|
|
if s := plt.SetLk(inode, unlock); s != fuse.OK {
|
|
t.Fatalf("prime unlock: got %v", s)
|
|
}
|
|
if !dead.dead {
|
|
t.Fatal("expected il to be marked dead after unlock+cleanup")
|
|
}
|
|
plt.mu.Lock()
|
|
_, stillMapped := plt.inodes[inode]
|
|
plt.mu.Unlock()
|
|
if stillMapped {
|
|
t.Fatal("expected the dead il to be removed from the map")
|
|
}
|
|
|
|
// Simulate the race: the next caller's getOrCreateInodeLocks races with
|
|
// the cleanup and ends up holding a pointer to the dead il. We force that
|
|
// state by re-publishing `dead` into the map.
|
|
plt.mu.Lock()
|
|
plt.inodes[inode] = dead
|
|
plt.mu.Unlock()
|
|
|
|
// SetLk must notice dead, refetch, and install the new lock in a fresh il.
|
|
if s := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 2, IsFlock: true}); s != fuse.OK {
|
|
t.Fatalf("SetLk after dead: got %v", s)
|
|
}
|
|
|
|
dead.mu.Lock()
|
|
if n := len(dead.locks); n != 0 {
|
|
t.Fatalf("dead il should not have accepted the insert, found %d locks", n)
|
|
}
|
|
dead.mu.Unlock()
|
|
|
|
plt.mu.Lock()
|
|
live := plt.inodes[inode]
|
|
plt.mu.Unlock()
|
|
if live == nil || live == dead {
|
|
t.Fatalf("expected a fresh live il, got %v", live)
|
|
}
|
|
|
|
// A conflicting owner must see the new lock and be rejected.
|
|
if s := plt.SetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 3, IsFlock: true}); s != fuse.EAGAIN {
|
|
t.Fatalf("second owner should conflict with owner 2, got %v", s)
|
|
}
|
|
|
|
// GetLk must report the conflict as well: without the dead-recheck the
|
|
// GetLk path would answer F_UNLCK off the orphaned il.
|
|
var out fuse.LkOut
|
|
plt.GetLk(inode, lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 4, IsFlock: true}, &out)
|
|
if out.Lk.Typ != syscall.F_WRLCK {
|
|
t.Fatalf("GetLk should report the live conflict, got Typ=%d", out.Lk.Typ)
|
|
}
|
|
}
|
|
|
|
// TestGetInodeLocksEvictsDeadEntry verifies that a dead inodeLocks which
|
|
// somehow ends up in the map (e.g. through a future refactor that reorders
|
|
// delete and dead=true) is dropped on read so callers never observe one.
|
|
// This is the backstop that lets GetLk's and SetLk's retry loops terminate.
|
|
func TestGetInodeLocksEvictsDeadEntry(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
inode := uint64(42)
|
|
|
|
lock := lockRange{Start: 0, End: math.MaxUint64, Typ: syscall.F_WRLCK, Owner: 1, IsFlock: true}
|
|
if s := plt.SetLk(inode, lock); s != fuse.OK {
|
|
t.Fatalf("prime SetLk: got %v", s)
|
|
}
|
|
dead := plt.getInodeLocks(inode)
|
|
unlock := lock
|
|
unlock.Typ = syscall.F_UNLCK
|
|
if s := plt.SetLk(inode, unlock); s != fuse.OK {
|
|
t.Fatalf("prime unlock: got %v", s)
|
|
}
|
|
if !dead.dead {
|
|
t.Fatal("expected dead after cleanup")
|
|
}
|
|
|
|
// Force the broken state that production cannot reach but tests and
|
|
// future refactors might: dead entry still in the map.
|
|
plt.mu.Lock()
|
|
plt.inodes[inode] = dead
|
|
plt.mu.Unlock()
|
|
|
|
if il := plt.getInodeLocks(inode); il != nil {
|
|
t.Fatalf("getInodeLocks should drop a dead map entry, got %p", il)
|
|
}
|
|
plt.mu.Lock()
|
|
_, stillMapped := plt.inodes[inode]
|
|
plt.mu.Unlock()
|
|
if stillMapped {
|
|
t.Fatal("expected dead entry to be removed from the map")
|
|
}
|
|
|
|
// getOrCreateInodeLocks must also self-heal (replace the dead entry with
|
|
// a fresh live one) so SetLk's retry path cannot spin.
|
|
plt.mu.Lock()
|
|
plt.inodes[inode] = dead
|
|
plt.mu.Unlock()
|
|
fresh := plt.getOrCreateInodeLocks(inode)
|
|
if fresh == dead {
|
|
t.Fatal("getOrCreateInodeLocks should not return a dead entry")
|
|
}
|
|
if fresh.dead {
|
|
t.Fatal("fresh entry should not be dead")
|
|
}
|
|
}
|
|
|
|
// TestConcurrentFlockChurnPreservesMutualExclusion is a stress companion to
|
|
// the deterministic tests above. It uses a Swap+CAS detector that flags
|
|
// overlap at two points (on acquire and on release), so a second granted
|
|
// holder is caught even if it sneaks in after the first goroutine's claim
|
|
// but before its release. 16 goroutines churn whole-file exclusive flock on
|
|
// one inode; with the race the detector fires hundreds of times per run,
|
|
// with the fix it stays at zero.
|
|
func TestConcurrentFlockChurnPreservesMutualExclusion(t *testing.T) {
|
|
plt := NewPosixLockTable()
|
|
const (
|
|
inode = uint64(42)
|
|
numWorkers = 16
|
|
iterations = 500
|
|
)
|
|
var (
|
|
wg sync.WaitGroup
|
|
holder atomic.Int64 // 0 = nobody; otherwise = holder's claim token
|
|
overlapSeen atomic.Int32
|
|
)
|
|
|
|
for w := 0; w < numWorkers; w++ {
|
|
wg.Add(1)
|
|
go func(id int) {
|
|
defer wg.Done()
|
|
owner := uint64(100 + id)
|
|
lock := lockRange{
|
|
Start: 0,
|
|
End: math.MaxUint64,
|
|
Typ: syscall.F_WRLCK,
|
|
Owner: owner,
|
|
Pid: uint32(id + 1),
|
|
IsFlock: true,
|
|
}
|
|
unlock := lock
|
|
unlock.Typ = syscall.F_UNLCK
|
|
token := int64(id + 1)
|
|
for i := 0; i < iterations; i++ {
|
|
// SetLk(WRLCK) may only return OK (granted) or EAGAIN
|
|
// (conflict); anything else indicates a bug and the test
|
|
// must fail rather than spin. Use Errorf + return because
|
|
// Fatalf is not safe from a non-test goroutine.
|
|
for {
|
|
s := plt.SetLk(inode, lock)
|
|
if s == fuse.OK {
|
|
break
|
|
}
|
|
if s != fuse.EAGAIN {
|
|
t.Errorf("worker %d iter %d: unexpected SetLk(WRLCK) status %v", id, i, s)
|
|
return
|
|
}
|
|
runtime.Gosched()
|
|
}
|
|
// Claim the slot. If Swap observes a non-zero predecessor,
|
|
// another goroutine already believes it holds the lock.
|
|
if prev := holder.Swap(token); prev != 0 {
|
|
overlapSeen.Add(1)
|
|
}
|
|
// Widen the window so a concurrently-granted peer has a
|
|
// chance to race into its own Swap before we release.
|
|
runtime.Gosched()
|
|
runtime.Gosched()
|
|
// Release the slot. If CAS fails someone else overwrote our
|
|
// claim, which only happens when two holders raced.
|
|
if !holder.CompareAndSwap(token, 0) {
|
|
overlapSeen.Add(1)
|
|
}
|
|
if s := plt.SetLk(inode, unlock); s != fuse.OK {
|
|
t.Errorf("worker %d iter %d: unexpected SetLk(UNLCK) status %v", id, i, s)
|
|
return
|
|
}
|
|
}
|
|
}(w)
|
|
}
|
|
wg.Wait()
|
|
|
|
if n := overlapSeen.Load(); n != 0 {
|
|
t.Fatalf("flock overlap detected %d times: two owners simultaneously granted the same exclusive lock", n)
|
|
}
|
|
}
|