mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 19:33:41 +01:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7ab73b914e | ||
|
|
f7684c4478 | ||
|
|
db02eea349 | ||
|
|
dbf2c3d30f | ||
|
|
ad73b66200 | ||
|
|
fcc7bd04c6 |
No files matched your search
+41
-16
@@ -45,6 +45,12 @@ type LockClient struct {
|
||||
// snapshot) but short enough that a write fails instead of hanging
|
||||
// forever.
|
||||
noLockServerRetryPeriod time.Duration
|
||||
|
||||
// lockRpcTimeout bounds a single DistributedLock attempt: a ring member
|
||||
// reachable from the masters but not from this filer would otherwise
|
||||
// block an attempt on the TCP connect timeout, far past
|
||||
// noLockServerRetryPeriod.
|
||||
lockRpcTimeout time.Duration
|
||||
}
|
||||
|
||||
func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
|
||||
@@ -55,6 +61,7 @@ func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *
|
||||
seedFiler: seedFiler,
|
||||
priorWindow: 5 * time.Second,
|
||||
noLockServerRetryPeriod: 15 * time.Second,
|
||||
lockRpcTimeout: 10 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,26 +311,34 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
|
||||
// waiter must pick it up promptly, otherwise cross-mount write handoff stalls
|
||||
// long enough to time out clients.
|
||||
func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) error {
|
||||
var unavailableSince time.Time
|
||||
// The retry window bounds acquisition as a whole: the clock starts
|
||||
// before the first attempt and each attempt gets what is left of it, so
|
||||
// a bounded attempt cannot burn the window inside one call.
|
||||
unavailableSince := time.Now()
|
||||
for lock.renewToken == "" {
|
||||
err := lock.AttemptToLock(lockDuration)
|
||||
attemptTimeout := lock.lc.lockRpcTimeout
|
||||
if remaining := lock.lc.noLockServerRetryPeriod - time.Since(unavailableSince); remaining < attemptTimeout {
|
||||
if remaining <= 0 {
|
||||
return fmt.Errorf("no lock server within %s", lock.lc.noLockServerRetryPeriod)
|
||||
}
|
||||
attemptTimeout = remaining
|
||||
}
|
||||
err := lock.attemptToLockWithin(lockDuration, attemptTimeout)
|
||||
if err == nil {
|
||||
unavailableSince = time.Time{}
|
||||
unavailableSince = time.Now()
|
||||
continue
|
||||
}
|
||||
glog.V(1).Infof("create lock %s: %v", lock.key, err)
|
||||
if strings.Contains(err.Error(), "lock already owned") {
|
||||
// Ordinary contention: a reachable server holds the lock, so
|
||||
// waiting is the point and has no bound.
|
||||
unavailableSince = time.Time{}
|
||||
unavailableSince = time.Now()
|
||||
continue
|
||||
}
|
||||
// Anything else — "no lock server found", a dead ring member refusing
|
||||
// connections — is a systemic fault waiting cannot fix; give up once
|
||||
// it persists past the retry period.
|
||||
if unavailableSince.IsZero() {
|
||||
unavailableSince = time.Now()
|
||||
} else if time.Since(unavailableSince) > lock.lc.noLockServerRetryPeriod {
|
||||
if time.Since(unavailableSince) > lock.lc.noLockServerRetryPeriod {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -331,8 +346,12 @@ func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) error {
|
||||
}
|
||||
|
||||
func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
|
||||
return lock.attemptToLockWithin(lockDuration, lock.lc.lockRpcTimeout)
|
||||
}
|
||||
|
||||
func (lock *LiveLock) attemptToLockWithin(lockDuration, attemptTimeout time.Duration) error {
|
||||
glog.V(4).Infof("LOCK: AttemptToLock key=%s owner=%s", lock.key, lock.self)
|
||||
errorMessage, err := lock.doLock(lockDuration)
|
||||
errorMessage, err := lock.doLock(lockDuration, attemptTimeout)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("LOCK: doLock failed for key=%s: %v", lock.key, err)
|
||||
time.Sleep(time.Second)
|
||||
@@ -409,15 +428,20 @@ func (lock *LiveLock) Stop() error {
|
||||
return lock.StopShortLivedLock()
|
||||
}
|
||||
|
||||
func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
|
||||
func (lock *LiveLock) doLock(lockDuration, attemptTimeout time.Duration) (errorMessage string, err error) {
|
||||
glog.V(4).Infof("LOCK: doLock calling DistributedLock - key=%s filer=%s owner=%s",
|
||||
lock.key, lock.hostFiler, lock.self)
|
||||
|
||||
previousHostFiler := lock.hostFiler
|
||||
previousOwner := lock.owner
|
||||
|
||||
err = pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
|
||||
// Bound the whole attempt — connection acquisition included — so an
|
||||
// unreachable owner fails within the retry window.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), attemptTimeout)
|
||||
defer cancel()
|
||||
err = pb.WithGrpcClient(ctx, false, 0, func(grpcConnection *grpc.ClientConn) error {
|
||||
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
|
||||
resp, err := client.DistributedLock(ctx, &filer_pb.LockRequest{
|
||||
Name: lock.key,
|
||||
SecondsToLock: int64(lockDuration.Seconds()),
|
||||
RenewToken: lock.renewToken,
|
||||
@@ -433,9 +457,11 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
lock.consecutiveFailures = 0 // Reset failure counter on success
|
||||
glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
|
||||
} else {
|
||||
//this can be retried. Need to remember the last valid renewToken
|
||||
lock.renewToken = ""
|
||||
glog.V(1).Infof("LOCK: Cleared renewToken for key=%s (err=%v)", lock.key, err)
|
||||
// the request may have renewed the lease server-side before its
|
||||
// response was lost: the last token can still be valid, so the
|
||||
// next attempt retries with it. A definitive rejection clears it
|
||||
// through the response's empty RenewToken.
|
||||
glog.V(1).Infof("LOCK: no response for key=%s (err=%v)", lock.key, err)
|
||||
}
|
||||
if resp != nil {
|
||||
errorMessage = resp.Error
|
||||
@@ -459,7 +485,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
}, lock.hostFiler.ToGrpcAddress(), false, lock.grpcDialOption)
|
||||
|
||||
if err != nil && !lock.hostFiler.Equals(lock.lc.seedFiler) {
|
||||
lock.consecutiveFailures++
|
||||
@@ -469,7 +495,6 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
lock.consecutiveFailures, lock.key, lock.hostFiler, lock.lc.seedFiler)
|
||||
lock.hostFiler = lock.lc.seedFiler
|
||||
lock.consecutiveFailures = 0
|
||||
lock.renewToken = ""
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -286,6 +286,85 @@ func TestNewShortLivedLockFailsFastOnUnreachableFiler(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type hangingLockFiler struct {
|
||||
filer_pb.UnimplementedSeaweedFilerServer
|
||||
}
|
||||
|
||||
func (s *hangingLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(30 * time.Second):
|
||||
return nil, fmt.Errorf("test: server never answered")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed attempt keeps the token it sent: the request may have renewed the
|
||||
// lease server-side before its response was lost, and a retry with the last
|
||||
// token can still succeed. Only a definitive rejection may clear it.
|
||||
func TestDoLockKeepsRenewTokenOnError(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
addr := listener.Addr().String()
|
||||
listener.Close()
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("split host port: %v", err)
|
||||
}
|
||||
lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
|
||||
lock := &LiveLock{
|
||||
key: "test-key",
|
||||
renewToken: "tok",
|
||||
hostFiler: pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)),
|
||||
grpcDialOption: dialOption,
|
||||
lc: lc,
|
||||
}
|
||||
if _, err := lock.doLock(time.Second, 50*time.Millisecond); err == nil {
|
||||
t.Fatal("expected the attempt to fail against a dead member")
|
||||
}
|
||||
if lock.renewToken != "tok" {
|
||||
t.Errorf("renewToken = %q, want it kept for the next attempt", lock.renewToken)
|
||||
}
|
||||
}
|
||||
|
||||
// A ring member that accepts connections but never answers stalls the caller
|
||||
// the same way a partially partitioned peer does: each attempt must fail
|
||||
// within the per-attempt bound so the retry window can give up.
|
||||
func TestNewShortLivedLockBoundsHangingAttempt(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
grpcServer := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(grpcServer, &hangingLockFiler{})
|
||||
go grpcServer.Serve(listener)
|
||||
defer grpcServer.Stop()
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
host, port, err := net.SplitHostPort(listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("split host port: %v", err)
|
||||
}
|
||||
lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
|
||||
lc.noLockServerRetryPeriod = 200 * time.Millisecond
|
||||
lc.lockRpcTimeout = 50 * time.Millisecond
|
||||
|
||||
start := time.Now()
|
||||
lock := lc.NewShortLivedLock("test-key", "test-owner")
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if lock != nil {
|
||||
t.Fatal("expected nil lock when the ring member never answers")
|
||||
}
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("lock acquisition took %v, expected it bounded by the per-attempt timeout", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// LiveLock.generation is a fencing token written and read with 64-bit atomic
|
||||
// operations. On 32-bit platforms (GOARCH=386 and GOARCH=arm) a 64-bit atomic
|
||||
// op requires an 8-byte-aligned address, which Go only guarantees for the
|
||||
|
||||
@@ -46,7 +46,7 @@ func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64
|
||||
// ring update and lock transfer completion — the old primary remains
|
||||
// authoritative for locks it still holds.
|
||||
if token != "" {
|
||||
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
|
||||
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && (lock.Token == token || lock.PrevToken == token) {
|
||||
var seq int64
|
||||
lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
|
||||
if err == nil && renewToken != "" {
|
||||
@@ -95,7 +95,7 @@ func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.
|
||||
}
|
||||
if primary != dlm.Host {
|
||||
// If we still hold this lock locally, serve the unlock here
|
||||
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
|
||||
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && token != "" && (lock.Token == token || lock.PrevToken == token) {
|
||||
var isUnlocked bool
|
||||
var generation int64
|
||||
var seq int64
|
||||
|
||||
@@ -24,6 +24,7 @@ type LockManager struct {
|
||||
}
|
||||
type Lock struct {
|
||||
Token string
|
||||
PrevToken string // token superseded by the latest renewal; a holder whose renewal response was lost still presents it
|
||||
ExpiredAtNs int64
|
||||
Key string // only used for moving locks
|
||||
Owner string
|
||||
@@ -102,19 +103,29 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
|
||||
generation = oldValue.Generation // keep same generation on renewal
|
||||
seq = oldValue.Seq + 1
|
||||
glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
|
||||
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
|
||||
return
|
||||
} else {
|
||||
if token == "" {
|
||||
// new lock
|
||||
glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
|
||||
err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
|
||||
return
|
||||
}
|
||||
glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
|
||||
err = fmt.Errorf("lock: token mismatch")
|
||||
lm.locks[path] = &Lock{Token: renewToken, PrevToken: oldValue.Token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
|
||||
return
|
||||
}
|
||||
if token != "" && token == oldValue.PrevToken && owner == oldValue.Owner {
|
||||
// The latest renewal already rotated the token but its response
|
||||
// was lost: replay the current token so the holder recovers it
|
||||
// instead of conflicting with its own lease until it expires.
|
||||
renewToken = oldValue.Token
|
||||
generation = oldValue.Generation
|
||||
seq = oldValue.Seq + 1
|
||||
oldValue.ExpiredAtNs = expiredAtNs
|
||||
oldValue.Seq = seq
|
||||
return
|
||||
}
|
||||
if token == "" {
|
||||
// new lock
|
||||
glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
|
||||
err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
|
||||
return
|
||||
}
|
||||
glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
|
||||
err = fmt.Errorf("lock: token mismatch")
|
||||
return
|
||||
} else {
|
||||
glog.V(4).Infof("key %s no lock owner %v", path, owner)
|
||||
if token == "" {
|
||||
@@ -148,7 +159,7 @@ func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, gener
|
||||
delete(lm.locks, path)
|
||||
return
|
||||
}
|
||||
if oldValue.Token == token {
|
||||
if oldValue.Token == token || (token != "" && token == oldValue.PrevToken) {
|
||||
isUnlocked = true
|
||||
generation = oldValue.Generation
|
||||
seq = oldValue.Seq + 1
|
||||
|
||||
@@ -84,3 +84,49 @@ func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
|
||||
assert.Equal(t, int64(8), lock.Generation)
|
||||
assert.Equal(t, int64(1), lock.Seq)
|
||||
}
|
||||
|
||||
func TestLockManager_LostRenewalResponseReplaysToken(t *testing.T) {
|
||||
lm := NewLockManager()
|
||||
expires := time.Now().Add(30 * time.Second).UnixNano()
|
||||
|
||||
_, token1, _, _, err := lm.Lock("key1", expires, "", "owner1")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// The renewal rotated token1 to token2; the client never learned it.
|
||||
_, token2, generation, seq2, err := lm.Lock("key1", expires, token1, "owner1")
|
||||
assert.NoError(t, err)
|
||||
assert.NotEqual(t, token1, token2)
|
||||
|
||||
// Retrying with the superseded token replays the current one instead of
|
||||
// conflicting with its own lease until it expires. The replayed mutation
|
||||
// still advances seq so backups accept the extended expiry as fresh.
|
||||
later := time.Now().Add(60 * time.Second).UnixNano()
|
||||
owner, replayed, _, seq3, err := lm.Lock("key1", later, token1, "owner1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "owner1", owner)
|
||||
assert.Equal(t, token2, replayed)
|
||||
assert.Greater(t, seq3, seq2)
|
||||
|
||||
backup := NewLockManager()
|
||||
backup.InsertBackupLock("key1", expires, token2, "owner1", generation, seq2)
|
||||
backup.InsertBackupLock("key1", later, token2, "owner1", generation, seq3)
|
||||
backupLock, found := backup.GetLock("key1")
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, later, backupLock.ExpiredAtNs)
|
||||
|
||||
// A replay from a different owner still mismatches.
|
||||
_, _, _, _, err = lm.Lock("key1", expires, token1, "intruder")
|
||||
assert.Error(t, err)
|
||||
|
||||
// The next rotation retires the superseded token for good.
|
||||
_, token3, _, _, err := lm.Lock("key1", expires, token2, "owner1")
|
||||
assert.NoError(t, err)
|
||||
assert.NotEqual(t, token2, token3)
|
||||
_, _, _, _, err = lm.Lock("key1", expires, token1, "owner1")
|
||||
assert.Error(t, err)
|
||||
|
||||
// A superseded token still releases its own lock.
|
||||
unlocked, _, _, err := lm.Unlock("key1", token2)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, unlocked)
|
||||
}
|
||||
@@ -272,7 +272,7 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr
|
||||
return grpc.DialContext(ctx, address, options...)
|
||||
}
|
||||
|
||||
func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
|
||||
func getOrCreateConnection(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
|
||||
|
||||
grpcClientsLock.Lock()
|
||||
defer grpcClientsLock.Unlock()
|
||||
@@ -284,7 +284,6 @@ func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialO
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Creating new gRPC connection to %s", address)
|
||||
ctx := context.Background()
|
||||
grpcConnection, err := GrpcDial(ctx, address, waitForReady, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fail to dial %s: %v", address, err)
|
||||
@@ -468,15 +467,16 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool {
|
||||
}
|
||||
|
||||
// WithGrpcClient In streamingMode, always use a fresh connection. Otherwise, try to reuse an existing connection.
|
||||
// ctx must be the context that bounds the RPC fn issues, so that a
|
||||
// Canceled/DeadlineExceeded fired by ctx itself does not tear down the shared
|
||||
// cached ClientConn out from under other concurrent callers. Pass
|
||||
// context.Background() when fn picks its own context: a Canceled/DeadlineExceeded
|
||||
// then never invalidates, since nothing here can attribute it to the channel.
|
||||
// ctx must be the context that bounds this call's connection acquisition and
|
||||
// the RPC fn issues, so that a Canceled/DeadlineExceeded fired by ctx itself
|
||||
// does not tear down the shared cached ClientConn out from under other
|
||||
// concurrent callers. Pass context.Background() when fn picks its own
|
||||
// context: a Canceled/DeadlineExceeded then never invalidates, since nothing
|
||||
// here can attribute it to the channel.
|
||||
func WithGrpcClient(ctx context.Context, streamingMode bool, signature int32, fn func(*grpc.ClientConn) error, address string, waitForReady bool, opts ...grpc.DialOption) error {
|
||||
|
||||
if !streamingMode {
|
||||
vgc, err := getOrCreateConnection(address, waitForReady, opts...)
|
||||
vgc, err := getOrCreateConnection(ctx, address, waitForReady, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
|
||||
}
|
||||
@@ -498,7 +498,7 @@ func WithGrpcClient(ctx context.Context, streamingMode bool, signature int32, fn
|
||||
}
|
||||
|
||||
// Streaming mode: dedicate a fresh ClientConn to this call.
|
||||
dialCtx := context.Background()
|
||||
dialCtx := ctx
|
||||
if signature != 0 {
|
||||
// Optimize: Use AppendToOutgoingContext instead of creating new map
|
||||
dialCtx = metadata.AppendToOutgoingContext(dialCtx, "sw-client-id", fmt.Sprintf("%d", signature))
|
||||
|
||||
Reference in new issue
Block a user