diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index e046eba54c..f3eb715a3f 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/server/constants" "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/util" @@ -152,8 +153,54 @@ func reportsPriorClusterState(heartbeat *master_pb.Heartbeat, topologyId string, return false } +// volumeHeartbeatTimeout is how long SendHeartbeat waits for the next +// heartbeat before it ends the stream and unregisters the volume server. +var volumeHeartbeatTimeout = 3 * constants.VolumePulsePeriod + +// heartbeatReceiver returns a function that receives the next heartbeat from +// stream, or fails with DeadlineExceeded once none has arrived for timeout, +// and a function that releases the receiving goroutine. A volume server sends +// a heartbeat every VolumePulsePeriod. One that goes silent without closing +// its connection (a hung host, a network partition) leaves Recv blocked until +// gRPC keepalive closes the connection, which takes 80 s or more, and until +// then the master keeps assigning writes to its volumes. +func heartbeatReceiver(stream master_pb.Seaweed_SendHeartbeatServer, timeout time.Duration) (recv func() (*master_pb.Heartbeat, error), stop func()) { + type received struct { + heartbeat *master_pb.Heartbeat + err error + } + ch := make(chan received) + done := make(chan struct{}) + go func() { + for { + heartbeat, err := stream.Recv() + select { + case ch <- received{heartbeat, err}: + case <-done: + return + } + if err != nil { + return + } + } + }() + recv = func() (*master_pb.Heartbeat, error) { + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case r := <-ch: + return r.heartbeat, r.err + case <-timer.C: + return nil, status.Errorf(codes.DeadlineExceeded, "no heartbeat for %v", timeout) + } + } + return recv, func() { close(done) } +} + func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error { var dn *topology.DataNode + recv, stopRecv := heartbeatReceiver(stream, volumeHeartbeatTimeout) + defer stopRecv() defer func() { if dn != nil { @@ -191,7 +238,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ var echoedVid needle.VolumeId var echoedTopoId string for { - heartbeat, err := stream.Recv() + heartbeat, err := recv() if err != nil { // Graceful shutdown on either side cancels the stream; don't warn. canceled := errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled diff --git a/weed/server/master_grpc_server_heartbeat_timeout_test.go b/weed/server/master_grpc_server_heartbeat_timeout_test.go new file mode 100644 index 0000000000..01abb06a01 --- /dev/null +++ b/weed/server/master_grpc_server_heartbeat_timeout_test.go @@ -0,0 +1,121 @@ +package weed_server + +import ( + "context" + "io" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// silentHeartbeatStream hands out its heartbeats and then blocks in Recv until +// its context ends, as a stream from a volume server that went silent. +type silentHeartbeatStream struct { + grpc.ServerStream + ctx context.Context + heartbeats chan *master_pb.Heartbeat +} + +func (s *silentHeartbeatStream) Context() context.Context { return s.ctx } + +func (s *silentHeartbeatStream) Recv() (*master_pb.Heartbeat, error) { + select { + case hb := <-s.heartbeats: + return hb, nil + case <-s.ctx.Done(): + return nil, s.ctx.Err() + } +} + +func (s *silentHeartbeatStream) Send(*master_pb.HeartbeatResponse) error { return nil } + +func newSilentHeartbeatStream(t *testing.T, heartbeats ...*master_pb.Heartbeat) *silentHeartbeatStream { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + ch := make(chan *master_pb.Heartbeat, len(heartbeats)) + for _, hb := range heartbeats { + ch <- hb + } + return &silentHeartbeatStream{ctx: ctx, heartbeats: ch} +} + +func TestHeartbeatReceiverPassesHeartbeatsAndErrors(t *testing.T) { + stream := &fakeHeartbeatStream{heartbeats: []*master_pb.Heartbeat{{Ip: "10.0.0.1"}, {Ip: "10.0.0.2"}}} + recv, stop := heartbeatReceiver(stream, time.Second) + defer stop() + + for _, ip := range []string{"10.0.0.1", "10.0.0.2"} { + hb, err := recv() + require.NoError(t, err) + assert.Equal(t, ip, hb.Ip) + } + _, err := recv() + assert.ErrorIs(t, err, io.EOF) +} + +func TestHeartbeatReceiverTimesOutOnSilentStream(t *testing.T) { + stream := newSilentHeartbeatStream(t, &master_pb.Heartbeat{Ip: "10.0.0.1"}) + recv, stop := heartbeatReceiver(stream, 50*time.Millisecond) + defer stop() + + _, err := recv() + require.NoError(t, err) + errc := make(chan error, 1) + go func() { + _, err := recv() + errc <- err + }() + select { + case err = <-errc: + assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) + case <-time.After(5 * time.Second): + t.Fatal("recv still waiting on a silent stream") + } +} + +// A volume server that stops sending heartbeats is unregistered once +// volumeHeartbeatTimeout passes, although its stream is still open. +func TestSendHeartbeatUnregistersSilentVolumeServer(t *testing.T) { + saved := volumeHeartbeatTimeout + volumeHeartbeatTimeout = 500 * time.Millisecond + defer func() { volumeHeartbeatTimeout = saved }() + + ms := newLeaderMaster() + ms.clientChans = make(map[string]chan *master_pb.KeepConnectedResponse) + _, clientChan := ms.addClient("", "filer", "10.0.0.9:8888") + stream := newSilentHeartbeatStream(t, &master_pb.Heartbeat{ + Ip: "10.0.0.1", + Port: 8080, + MaxVolumeCounts: map[string]uint32{"": 10}, + Volumes: []*master_pb.VolumeInformationMessage{{Id: 7, Version: 3}}, + }) + + errc := make(chan error, 1) + go func() { errc <- ms.SendHeartbeat(stream) }() + require.Eventually(t, func() bool { return ms.Topo.DataNodeExists("", "", "10.0.0.1:8080") }, + 5*time.Second, 10*time.Millisecond, "the first heartbeat registers the server") + select { + case err := <-errc: + assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) + case <-time.After(10 * time.Second): + t.Fatal("SendHeartbeat still waiting on a silent stream") + } + assert.False(t, ms.Topo.DataNodeExists("", "", "10.0.0.1:8080")) + for { + select { + case message := <-clientChan: + if location := message.GetVolumeLocation(); len(location.GetDeletedVids()) > 0 { + assert.Equal(t, []uint32{7}, location.DeletedVids) + return + } + default: + t.Fatal("clients were not told the silent server's volumes are gone") + } + } +}