mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
2386fa550a
A Canceled/DeadlineExceeded from the caller's per-request context was treated like a dead channel: it closed the shared cached ClientConn and cancelled every other in-flight RPC on it with "the client connection is closing". Under a burst of concurrent chunk assigns (e.g. a large S3 multipart upload) one slow assign hitting its 10s attempt timeout could poison the connection for all the rest, cascading into a flood of 500s. Thread the caller's context into shouldInvalidateConnection and only invalidate on Canceled/DeadlineExceeded while that context is still live, which isolates the genuine stale-channel signal (a peer restart behind a k8s Service VIP). To carry the context, add a ctx parameter to the existing WithGrpcClient, WithMasterClient, and WithMasterServerClient; the master assign and volume-lookup paths pass their per-attempt context and every other caller passes context.Background().
58 lines
1.3 KiB
Go
58 lines
1.3 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"sync/atomic"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
var _ = filer_pb.FilerClient(&WFS{})
|
|
|
|
func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
|
|
|
|
return util.Retry("filer grpc", func() error {
|
|
|
|
i := atomic.LoadInt32(&wfs.option.filerIndex)
|
|
n := len(wfs.option.FilerAddresses)
|
|
for x := 0; x < n; x++ {
|
|
|
|
filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
|
|
err = pb.WithGrpcClient(context.Background(), streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
|
|
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
|
|
return fn(client)
|
|
}, filerGrpcAddress, false, wfs.option.GrpcDialOption)
|
|
|
|
if err != nil {
|
|
// glog.V(0).Infof("WithFilerClient %d %v: %v", x, filerGrpcAddress, err)
|
|
} else {
|
|
atomic.StoreInt32(&wfs.option.filerIndex, i)
|
|
return nil
|
|
}
|
|
|
|
i++
|
|
if i >= int32(n) {
|
|
i = 0
|
|
}
|
|
|
|
}
|
|
return err
|
|
})
|
|
|
|
}
|
|
|
|
func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string {
|
|
if wfs.option.VolumeServerAccess == "publicUrl" {
|
|
return location.PublicUrl
|
|
}
|
|
return location.Url
|
|
}
|
|
|
|
func (wfs *WFS) GetDataCenter() string {
|
|
return wfs.option.DataCenter
|
|
}
|