Compare commits

...
Author SHA1 Message Date
user.emailandDevin 7ab73b914e dlm: advance the mutation seq on a lost-renewal replay
The replayed lease extension kept the old seq, so InsertBackupLock
dropped it as stale and a promoted backup would have expired the
recovered lock early.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-26 19:49:33 -07:00
user.emailandDevin f7684c4478 dlm: replay the current token on a lost renewal's retry
A renewal rotates the token server-side before the response is sent. When
that response is lost the client only knows the superseded token: retries
mismatched, the token was cleared, and the holder conflicted with its own
lease until it expired. The lock now remembers the superseded token, so a
retry from the same owner replays the current token, and a superseded
token can still release its own lock.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-26 19:30:07 -07:00
user.emailandDevin db02eea349 cluster: bound connection acquisition inside the lock attempt
A blocking dial ran under context.Background(), so a caller supplying
grpc.WithBlock() could wait past the attempt window before the RPC
deadline even started. The attempt context now covers both
getOrCreateConnection and the DistributedLock call, and WithGrpcClient
threads it into GrpcDial on both the cached and streaming paths.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-26 18:42:49 -07:00
user.email dbf2c3d30f cluster: keep the renew token across a failed attempt
A renewal can rotate the token server-side before its response is lost,
and clearing the token on any RPC failure then abandons a live lease the
client can neither renew nor unlock. Keep the token so the next attempt
retries with it: it still validates when the request never landed, and a
definitive rejection clears it through the response's empty RenewToken.
2026-10-26 17:49:33 -07:00
user.email ad73b66200 cluster: bound the whole lock-acquisition window
A single bounded attempt can take most of noLockServerRetryPeriod before
the retry clock even starts, so an unreachable owner stalls acquisition
roughly two attempts past the intended window. Start the clock before
the first attempt and cap each attempt by what remains; contention still
waits it out.
2026-10-26 17:49:18 -07:00
user.email fcc7bd04c6 cluster: bound each DistributedLock attempt
A lock-ring member reachable from the masters but not from this filer
makes one DistributedLock attempt wait out the TCP connect timeout, so a
write whose key hashes to it stalls far past noLockServerRetryPeriod
before failing. Give the attempt a 10s context deadline so the retry
window can take effect; a timed-out attempt keeps no renew token, so the
lock server still grants the key to at most one owner.
2026-10-26 17:38:19 -07:00
6 changed files with 200 additions and 39 deletions

No files matched your search

+41 -16
View File
@@ -45,6 +45,12 @@ type LockClient struct {
// snapshot) but short enough that a write fails instead of hanging
// forever.
noLockServerRetryPeriod time.Duration
// lockRpcTimeout bounds a single DistributedLock attempt: a ring member
// reachable from the masters but not from this filer would otherwise
// block an attempt on the TCP connect timeout, far past
// noLockServerRetryPeriod.
lockRpcTimeout time.Duration
}
func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
@@ -55,6 +61,7 @@ func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *
seedFiler: seedFiler,
priorWindow: 5 * time.Second,
noLockServerRetryPeriod: 15 * time.Second,
lockRpcTimeout: 10 * time.Second,
}
}
@@ -304,26 +311,34 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
// waiter must pick it up promptly, otherwise cross-mount write handoff stalls
// long enough to time out clients.
func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) error {
var unavailableSince time.Time
// The retry window bounds acquisition as a whole: the clock starts
// before the first attempt and each attempt gets what is left of it, so
// a bounded attempt cannot burn the window inside one call.
unavailableSince := time.Now()
for lock.renewToken == "" {
err := lock.AttemptToLock(lockDuration)
attemptTimeout := lock.lc.lockRpcTimeout
if remaining := lock.lc.noLockServerRetryPeriod - time.Since(unavailableSince); remaining < attemptTimeout {
if remaining <= 0 {
return fmt.Errorf("no lock server within %s", lock.lc.noLockServerRetryPeriod)
}
attemptTimeout = remaining
}
err := lock.attemptToLockWithin(lockDuration, attemptTimeout)
if err == nil {
unavailableSince = time.Time{}
unavailableSince = time.Now()
continue
}
glog.V(1).Infof("create lock %s: %v", lock.key, err)
if strings.Contains(err.Error(), "lock already owned") {
// Ordinary contention: a reachable server holds the lock, so
// waiting is the point and has no bound.
unavailableSince = time.Time{}
unavailableSince = time.Now()
continue
}
// Anything else — "no lock server found", a dead ring member refusing
// connections — is a systemic fault waiting cannot fix; give up once
// it persists past the retry period.
if unavailableSince.IsZero() {
unavailableSince = time.Now()
} else if time.Since(unavailableSince) > lock.lc.noLockServerRetryPeriod {
if time.Since(unavailableSince) > lock.lc.noLockServerRetryPeriod {
return err
}
}
@@ -331,8 +346,12 @@ func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) error {
}
func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
return lock.attemptToLockWithin(lockDuration, lock.lc.lockRpcTimeout)
}
func (lock *LiveLock) attemptToLockWithin(lockDuration, attemptTimeout time.Duration) error {
glog.V(4).Infof("LOCK: AttemptToLock key=%s owner=%s", lock.key, lock.self)
errorMessage, err := lock.doLock(lockDuration)
errorMessage, err := lock.doLock(lockDuration, attemptTimeout)
if err != nil {
glog.V(1).Infof("LOCK: doLock failed for key=%s: %v", lock.key, err)
time.Sleep(time.Second)
@@ -409,15 +428,20 @@ func (lock *LiveLock) Stop() error {
return lock.StopShortLivedLock()
}
func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
func (lock *LiveLock) doLock(lockDuration, attemptTimeout time.Duration) (errorMessage string, err error) {
glog.V(4).Infof("LOCK: doLock calling DistributedLock - key=%s filer=%s owner=%s",
lock.key, lock.hostFiler, lock.self)
previousHostFiler := lock.hostFiler
previousOwner := lock.owner
err = pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
// Bound the whole attempt — connection acquisition included — so an
// unreachable owner fails within the retry window.
ctx, cancel := context.WithTimeout(context.Background(), attemptTimeout)
defer cancel()
err = pb.WithGrpcClient(ctx, false, 0, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
resp, err := client.DistributedLock(ctx, &filer_pb.LockRequest{
Name: lock.key,
SecondsToLock: int64(lockDuration.Seconds()),
RenewToken: lock.renewToken,
@@ -433,9 +457,11 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
lock.consecutiveFailures = 0 // Reset failure counter on success
glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
} else {
//this can be retried. Need to remember the last valid renewToken
lock.renewToken = ""
glog.V(1).Infof("LOCK: Cleared renewToken for key=%s (err=%v)", lock.key, err)
// the request may have renewed the lease server-side before its
// response was lost: the last token can still be valid, so the
// next attempt retries with it. A definitive rejection clears it
// through the response's empty RenewToken.
glog.V(1).Infof("LOCK: no response for key=%s (err=%v)", lock.key, err)
}
if resp != nil {
errorMessage = resp.Error
@@ -459,7 +485,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
}
}
return err
})
}, lock.hostFiler.ToGrpcAddress(), false, lock.grpcDialOption)
if err != nil && !lock.hostFiler.Equals(lock.lc.seedFiler) {
lock.consecutiveFailures++
@@ -469,7 +495,6 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
lock.consecutiveFailures, lock.key, lock.hostFiler, lock.lc.seedFiler)
lock.hostFiler = lock.lc.seedFiler
lock.consecutiveFailures = 0
lock.renewToken = ""
}
}
+79
View File
@@ -286,6 +286,85 @@ func TestNewShortLivedLockFailsFastOnUnreachableFiler(t *testing.T) {
}
}
type hangingLockFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
}
func (s *hangingLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(30 * time.Second):
return nil, fmt.Errorf("test: server never answered")
}
}
// A failed attempt keeps the token it sent: the request may have renewed the
// lease server-side before its response was lost, and a retry with the last
// token can still succeed. Only a definitive rejection may clear it.
func TestDoLockKeepsRenewTokenOnError(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
addr := listener.Addr().String()
listener.Close()
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
host, port, err := net.SplitHostPort(addr)
if err != nil {
t.Fatalf("split host port: %v", err)
}
lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
lock := &LiveLock{
key: "test-key",
renewToken: "tok",
hostFiler: pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)),
grpcDialOption: dialOption,
lc: lc,
}
if _, err := lock.doLock(time.Second, 50*time.Millisecond); err == nil {
t.Fatal("expected the attempt to fail against a dead member")
}
if lock.renewToken != "tok" {
t.Errorf("renewToken = %q, want it kept for the next attempt", lock.renewToken)
}
}
// A ring member that accepts connections but never answers stalls the caller
// the same way a partially partitioned peer does: each attempt must fail
// within the per-attempt bound so the retry window can give up.
func TestNewShortLivedLockBoundsHangingAttempt(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
grpcServer := grpc.NewServer()
filer_pb.RegisterSeaweedFilerServer(grpcServer, &hangingLockFiler{})
go grpcServer.Serve(listener)
defer grpcServer.Stop()
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
host, port, err := net.SplitHostPort(listener.Addr().String())
if err != nil {
t.Fatalf("split host port: %v", err)
}
lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
lc.noLockServerRetryPeriod = 200 * time.Millisecond
lc.lockRpcTimeout = 50 * time.Millisecond
start := time.Now()
lock := lc.NewShortLivedLock("test-key", "test-owner")
elapsed := time.Since(start)
if lock != nil {
t.Fatal("expected nil lock when the ring member never answers")
}
if elapsed > 5*time.Second {
t.Fatalf("lock acquisition took %v, expected it bounded by the per-attempt timeout", elapsed)
}
}
// LiveLock.generation is a fencing token written and read with 64-bit atomic
// operations. On 32-bit platforms (GOARCH=386 and GOARCH=arm) a 64-bit atomic
// op requires an 8-byte-aligned address, which Go only guarantees for the
@@ -46,7 +46,7 @@ func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64
// ring update and lock transfer completion — the old primary remains
// authoritative for locks it still holds.
if token != "" {
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && (lock.Token == token || lock.PrevToken == token) {
var seq int64
lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
if err == nil && renewToken != "" {
@@ -95,7 +95,7 @@ func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.
}
if primary != dlm.Host {
// If we still hold this lock locally, serve the unlock here
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && token != "" && (lock.Token == token || lock.PrevToken == token) {
var isUnlocked bool
var generation int64
var seq int64
+23 -12
View File
@@ -24,6 +24,7 @@ type LockManager struct {
}
type Lock struct {
Token string
PrevToken string // token superseded by the latest renewal; a holder whose renewal response was lost still presents it
ExpiredAtNs int64
Key string // only used for moving locks
Owner string
@@ -102,19 +103,29 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
generation = oldValue.Generation // keep same generation on renewal
seq = oldValue.Seq + 1
glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
return
} else {
if token == "" {
// new lock
glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
return
}
glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
err = fmt.Errorf("lock: token mismatch")
lm.locks[path] = &Lock{Token: renewToken, PrevToken: oldValue.Token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
return
}
if token != "" && token == oldValue.PrevToken && owner == oldValue.Owner {
// The latest renewal already rotated the token but its response
// was lost: replay the current token so the holder recovers it
// instead of conflicting with its own lease until it expires.
renewToken = oldValue.Token
generation = oldValue.Generation
seq = oldValue.Seq + 1
oldValue.ExpiredAtNs = expiredAtNs
oldValue.Seq = seq
return
}
if token == "" {
// new lock
glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
return
}
glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
err = fmt.Errorf("lock: token mismatch")
return
} else {
glog.V(4).Infof("key %s no lock owner %v", path, owner)
if token == "" {
@@ -148,7 +159,7 @@ func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, gener
delete(lm.locks, path)
return
}
if oldValue.Token == token {
if oldValue.Token == token || (token != "" && token == oldValue.PrevToken) {
isUnlocked = true
generation = oldValue.Generation
seq = oldValue.Seq + 1
@@ -84,3 +84,49 @@ func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
assert.Equal(t, int64(8), lock.Generation)
assert.Equal(t, int64(1), lock.Seq)
}
func TestLockManager_LostRenewalResponseReplaysToken(t *testing.T) {
lm := NewLockManager()
expires := time.Now().Add(30 * time.Second).UnixNano()
_, token1, _, _, err := lm.Lock("key1", expires, "", "owner1")
assert.NoError(t, err)
// The renewal rotated token1 to token2; the client never learned it.
_, token2, generation, seq2, err := lm.Lock("key1", expires, token1, "owner1")
assert.NoError(t, err)
assert.NotEqual(t, token1, token2)
// Retrying with the superseded token replays the current one instead of
// conflicting with its own lease until it expires. The replayed mutation
// still advances seq so backups accept the extended expiry as fresh.
later := time.Now().Add(60 * time.Second).UnixNano()
owner, replayed, _, seq3, err := lm.Lock("key1", later, token1, "owner1")
assert.NoError(t, err)
assert.Equal(t, "owner1", owner)
assert.Equal(t, token2, replayed)
assert.Greater(t, seq3, seq2)
backup := NewLockManager()
backup.InsertBackupLock("key1", expires, token2, "owner1", generation, seq2)
backup.InsertBackupLock("key1", later, token2, "owner1", generation, seq3)
backupLock, found := backup.GetLock("key1")
assert.True(t, found)
assert.Equal(t, later, backupLock.ExpiredAtNs)
// A replay from a different owner still mismatches.
_, _, _, _, err = lm.Lock("key1", expires, token1, "intruder")
assert.Error(t, err)
// The next rotation retires the superseded token for good.
_, token3, _, _, err := lm.Lock("key1", expires, token2, "owner1")
assert.NoError(t, err)
assert.NotEqual(t, token2, token3)
_, _, _, _, err = lm.Lock("key1", expires, token1, "owner1")
assert.Error(t, err)
// A superseded token still releases its own lock.
unlocked, _, _, err := lm.Unlock("key1", token2)
assert.NoError(t, err)
assert.True(t, unlocked)
}
+9 -9
View File
@@ -272,7 +272,7 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr
return grpc.DialContext(ctx, address, options...)
}
func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
func getOrCreateConnection(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
grpcClientsLock.Lock()
defer grpcClientsLock.Unlock()
@@ -284,7 +284,6 @@ func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialO
}
glog.V(2).Infof("Creating new gRPC connection to %s", address)
ctx := context.Background()
grpcConnection, err := GrpcDial(ctx, address, waitForReady, opts...)
if err != nil {
return nil, fmt.Errorf("fail to dial %s: %v", address, err)
@@ -468,15 +467,16 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool {
}
// WithGrpcClient In streamingMode, always use a fresh connection. Otherwise, try to reuse an existing connection.
// ctx must be the context that bounds the RPC fn issues, so that a
// Canceled/DeadlineExceeded fired by ctx itself does not tear down the shared
// cached ClientConn out from under other concurrent callers. Pass
// context.Background() when fn picks its own context: a Canceled/DeadlineExceeded
// then never invalidates, since nothing here can attribute it to the channel.
// ctx must be the context that bounds this call's connection acquisition and
// the RPC fn issues, so that a Canceled/DeadlineExceeded fired by ctx itself
// does not tear down the shared cached ClientConn out from under other
// concurrent callers. Pass context.Background() when fn picks its own
// context: a Canceled/DeadlineExceeded then never invalidates, since nothing
// here can attribute it to the channel.
func WithGrpcClient(ctx context.Context, streamingMode bool, signature int32, fn func(*grpc.ClientConn) error, address string, waitForReady bool, opts ...grpc.DialOption) error {
if !streamingMode {
vgc, err := getOrCreateConnection(address, waitForReady, opts...)
vgc, err := getOrCreateConnection(ctx, address, waitForReady, opts...)
if err != nil {
return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
}
@@ -498,7 +498,7 @@ func WithGrpcClient(ctx context.Context, streamingMode bool, signature int32, fn
}
// Streaming mode: dedicate a fresh ClientConn to this call.
dialCtx := context.Background()
dialCtx := ctx
if signature != 0 {
// Optimize: Use AppendToOutgoingContext instead of creating new map
dialCtx = metadata.AppendToOutgoingContext(dialCtx, "sw-client-id", fmt.Sprintf("%d", signature))