mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
master: unregister a volume server whose heartbeats stop (#12697)
* master: unregister a volume server whose heartbeats stop A volume server sends a heartbeat every 5 s. When its host hangs or its network drops packets, its SendHeartbeat stream stays open, and the master unregisters it only once gRPC keepalive closes the connection, 60 s plus 20 s after the last frame. Until then the master keeps assigning writes to volumes with a copy on it, and those writes hang. SendHeartbeat now ends the stream when no heartbeat arrives for three pulses (15 s), which unregisters the server through the existing deferred path and tells clients its locations are gone. The server reconnects and registers again once it can reach the master. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: test that clients learn a silent server's volumes are gone Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: fail the silent-stream receive test instead of hanging Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
2 files changed
+169
-1
No files matched your search
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/server/constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -152,8 +153,54 @@ func reportsPriorClusterState(heartbeat *master_pb.Heartbeat, topologyId string,
|
||||
return false
|
||||
}
|
||||
|
||||
// volumeHeartbeatTimeout is how long SendHeartbeat waits for the next
|
||||
// heartbeat before it ends the stream and unregisters the volume server.
|
||||
var volumeHeartbeatTimeout = 3 * constants.VolumePulsePeriod
|
||||
|
||||
// heartbeatReceiver returns a function that receives the next heartbeat from
|
||||
// stream, or fails with DeadlineExceeded once none has arrived for timeout,
|
||||
// and a function that releases the receiving goroutine. A volume server sends
|
||||
// a heartbeat every VolumePulsePeriod. One that goes silent without closing
|
||||
// its connection (a hung host, a network partition) leaves Recv blocked until
|
||||
// gRPC keepalive closes the connection, which takes 80 s or more, and until
|
||||
// then the master keeps assigning writes to its volumes.
|
||||
func heartbeatReceiver(stream master_pb.Seaweed_SendHeartbeatServer, timeout time.Duration) (recv func() (*master_pb.Heartbeat, error), stop func()) {
|
||||
type received struct {
|
||||
heartbeat *master_pb.Heartbeat
|
||||
err error
|
||||
}
|
||||
ch := make(chan received)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
heartbeat, err := stream.Recv()
|
||||
select {
|
||||
case ch <- received{heartbeat, err}:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
recv = func() (*master_pb.Heartbeat, error) {
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r.heartbeat, r.err
|
||||
case <-timer.C:
|
||||
return nil, status.Errorf(codes.DeadlineExceeded, "no heartbeat for %v", timeout)
|
||||
}
|
||||
}
|
||||
return recv, func() { close(done) }
|
||||
}
|
||||
|
||||
func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
|
||||
var dn *topology.DataNode
|
||||
recv, stopRecv := heartbeatReceiver(stream, volumeHeartbeatTimeout)
|
||||
defer stopRecv()
|
||||
|
||||
defer func() {
|
||||
if dn != nil {
|
||||
@@ -191,7 +238,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
||||
var echoedVid needle.VolumeId
|
||||
var echoedTopoId string
|
||||
for {
|
||||
heartbeat, err := stream.Recv()
|
||||
heartbeat, err := recv()
|
||||
if err != nil {
|
||||
// Graceful shutdown on either side cancels the stream; don't warn.
|
||||
canceled := errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// silentHeartbeatStream hands out its heartbeats and then blocks in Recv until
|
||||
// its context ends, as a stream from a volume server that went silent.
|
||||
type silentHeartbeatStream struct {
|
||||
grpc.ServerStream
|
||||
ctx context.Context
|
||||
heartbeats chan *master_pb.Heartbeat
|
||||
}
|
||||
|
||||
func (s *silentHeartbeatStream) Context() context.Context { return s.ctx }
|
||||
|
||||
func (s *silentHeartbeatStream) Recv() (*master_pb.Heartbeat, error) {
|
||||
select {
|
||||
case hb := <-s.heartbeats:
|
||||
return hb, nil
|
||||
case <-s.ctx.Done():
|
||||
return nil, s.ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *silentHeartbeatStream) Send(*master_pb.HeartbeatResponse) error { return nil }
|
||||
|
||||
func newSilentHeartbeatStream(t *testing.T, heartbeats ...*master_pb.Heartbeat) *silentHeartbeatStream {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
ch := make(chan *master_pb.Heartbeat, len(heartbeats))
|
||||
for _, hb := range heartbeats {
|
||||
ch <- hb
|
||||
}
|
||||
return &silentHeartbeatStream{ctx: ctx, heartbeats: ch}
|
||||
}
|
||||
|
||||
func TestHeartbeatReceiverPassesHeartbeatsAndErrors(t *testing.T) {
|
||||
stream := &fakeHeartbeatStream{heartbeats: []*master_pb.Heartbeat{{Ip: "10.0.0.1"}, {Ip: "10.0.0.2"}}}
|
||||
recv, stop := heartbeatReceiver(stream, time.Second)
|
||||
defer stop()
|
||||
|
||||
for _, ip := range []string{"10.0.0.1", "10.0.0.2"} {
|
||||
hb, err := recv()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ip, hb.Ip)
|
||||
}
|
||||
_, err := recv()
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
}
|
||||
|
||||
func TestHeartbeatReceiverTimesOutOnSilentStream(t *testing.T) {
|
||||
stream := newSilentHeartbeatStream(t, &master_pb.Heartbeat{Ip: "10.0.0.1"})
|
||||
recv, stop := heartbeatReceiver(stream, 50*time.Millisecond)
|
||||
defer stop()
|
||||
|
||||
_, err := recv()
|
||||
require.NoError(t, err)
|
||||
errc := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := recv()
|
||||
errc <- err
|
||||
}()
|
||||
select {
|
||||
case err = <-errc:
|
||||
assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("recv still waiting on a silent stream")
|
||||
}
|
||||
}
|
||||
|
||||
// A volume server that stops sending heartbeats is unregistered once
|
||||
// volumeHeartbeatTimeout passes, although its stream is still open.
|
||||
func TestSendHeartbeatUnregistersSilentVolumeServer(t *testing.T) {
|
||||
saved := volumeHeartbeatTimeout
|
||||
volumeHeartbeatTimeout = 500 * time.Millisecond
|
||||
defer func() { volumeHeartbeatTimeout = saved }()
|
||||
|
||||
ms := newLeaderMaster()
|
||||
ms.clientChans = make(map[string]chan *master_pb.KeepConnectedResponse)
|
||||
_, clientChan := ms.addClient("", "filer", "10.0.0.9:8888")
|
||||
stream := newSilentHeartbeatStream(t, &master_pb.Heartbeat{
|
||||
Ip: "10.0.0.1",
|
||||
Port: 8080,
|
||||
MaxVolumeCounts: map[string]uint32{"": 10},
|
||||
Volumes: []*master_pb.VolumeInformationMessage{{Id: 7, Version: 3}},
|
||||
})
|
||||
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- ms.SendHeartbeat(stream) }()
|
||||
require.Eventually(t, func() bool { return ms.Topo.DataNodeExists("", "", "10.0.0.1:8080") },
|
||||
5*time.Second, 10*time.Millisecond, "the first heartbeat registers the server")
|
||||
select {
|
||||
case err := <-errc:
|
||||
assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("SendHeartbeat still waiting on a silent stream")
|
||||
}
|
||||
assert.False(t, ms.Topo.DataNodeExists("", "", "10.0.0.1:8080"))
|
||||
for {
|
||||
select {
|
||||
case message := <-clientChan:
|
||||
if location := message.GetVolumeLocation(); len(location.GetDeletedVids()) > 0 {
|
||||
assert.Equal(t, []uint32{7}, location.DeletedVids)
|
||||
return
|
||||
}
|
||||
default:
|
||||
t.Fatal("clients were not told the silent server's volumes are gone")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user