mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-09 13:28:20 +01:00
grpc: don't tear down the shared master connection on a caller's own timeout (#9775)
A Canceled/DeadlineExceeded from the caller's per-request context was treated like a dead channel: it closed the shared cached ClientConn and cancelled every other in-flight RPC on it with "the client connection is closing". Under a burst of concurrent chunk assigns (e.g. a large S3 multipart upload) one slow assign hitting its 10s attempt timeout could poison the connection for all the rest, cascading into a flood of 500s. Thread the caller's context into shouldInvalidateConnection and only invalidate on Canceled/DeadlineExceeded while that context is still live, which isolates the genuine stale-channel signal (a peer restart behind a k8s Service VIP). To carry the context, add a ctx parameter to the existing WithGrpcClient, WithMasterClient, and WithMasterServerClient; the master assign and volume-lookup paths pass their per-attempt context and every other caller passes context.Background().
This commit is contained in:
1 parent
dfa86b4313
commit
2386fa550a
31 files changed
+129
-63
No files matched your search
@@ -222,7 +222,7 @@ func (ms *MasterServer) Ping(ctx context.Context, req *master_pb.PingRequest) (r
|
||||
})
|
||||
}
|
||||
if req.TargetType == cluster.MasterType {
|
||||
pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
pingErr = pb.WithMasterClient(context.Background(), false, pb.ServerAddress(req.Target), ms.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
|
||||
Reference in new issue
Block a user