master: unregister a volume server whose heartbeats stop (#12697)

* master: unregister a volume server whose heartbeats stop

A volume server sends a heartbeat every 5 s. When its host hangs or its
network drops packets, its SendHeartbeat stream stays open, and the
master unregisters it only once gRPC keepalive closes the connection,
60 s plus 20 s after the last frame. Until then the master keeps
assigning writes to volumes with a copy on it, and those writes hang.

SendHeartbeat now ends the stream when no heartbeat arrives for three
pulses (15 s), which unregisters the server through the existing
deferred path and tells clients its locations are gone. The server
reconnects and registers again once it can reach the master.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: test that clients learn a silent server's volumes are gone

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: fail the silent-stream receive test instead of hanging

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-17 16:59:40 -08:00
1 parent dbb2fdb93b
commit d8a47042c5
2 files changed
+169 -1

No files matched your search

+48 -1
View File
@@ -14,6 +14,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/server/constants"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -152,8 +153,54 @@ func reportsPriorClusterState(heartbeat *master_pb.Heartbeat, topologyId string,
return false
}
// volumeHeartbeatTimeout is how long SendHeartbeat waits for the next
// heartbeat before it ends the stream and unregisters the volume server.
var volumeHeartbeatTimeout = 3 * constants.VolumePulsePeriod
// heartbeatReceiver returns a function that receives the next heartbeat from
// stream, or fails with DeadlineExceeded once none has arrived for timeout,
// and a function that releases the receiving goroutine. A volume server sends
// a heartbeat every VolumePulsePeriod. One that goes silent without closing
// its connection (a hung host, a network partition) leaves Recv blocked until
// gRPC keepalive closes the connection, which takes 80 s or more, and until
// then the master keeps assigning writes to its volumes.
func heartbeatReceiver(stream master_pb.Seaweed_SendHeartbeatServer, timeout time.Duration) (recv func() (*master_pb.Heartbeat, error), stop func()) {
type received struct {
heartbeat *master_pb.Heartbeat
err error
}
ch := make(chan received)
done := make(chan struct{})
go func() {
for {
heartbeat, err := stream.Recv()
select {
case ch <- received{heartbeat, err}:
case <-done:
return
}
if err != nil {
return
}
}
}()
recv = func() (*master_pb.Heartbeat, error) {
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case r := <-ch:
return r.heartbeat, r.err
case <-timer.C:
return nil, status.Errorf(codes.DeadlineExceeded, "no heartbeat for %v", timeout)
}
}
return recv, func() { close(done) }
}
func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
var dn *topology.DataNode
recv, stopRecv := heartbeatReceiver(stream, volumeHeartbeatTimeout)
defer stopRecv()
defer func() {
if dn != nil {
@@ -191,7 +238,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
var echoedVid needle.VolumeId
var echoedTopoId string
for {
heartbeat, err := stream.Recv()
heartbeat, err := recv()
if err != nil {
// Graceful shutdown on either side cancels the stream; don't warn.
canceled := errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled
@@ -0,0 +1,121 @@
package weed_server
import (
"context"
"io"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// silentHeartbeatStream hands out its heartbeats and then blocks in Recv until
// its context ends, as a stream from a volume server that went silent.
type silentHeartbeatStream struct {
grpc.ServerStream
ctx context.Context
heartbeats chan *master_pb.Heartbeat
}
func (s *silentHeartbeatStream) Context() context.Context { return s.ctx }
func (s *silentHeartbeatStream) Recv() (*master_pb.Heartbeat, error) {
select {
case hb := <-s.heartbeats:
return hb, nil
case <-s.ctx.Done():
return nil, s.ctx.Err()
}
}
func (s *silentHeartbeatStream) Send(*master_pb.HeartbeatResponse) error { return nil }
func newSilentHeartbeatStream(t *testing.T, heartbeats ...*master_pb.Heartbeat) *silentHeartbeatStream {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
ch := make(chan *master_pb.Heartbeat, len(heartbeats))
for _, hb := range heartbeats {
ch <- hb
}
return &silentHeartbeatStream{ctx: ctx, heartbeats: ch}
}
func TestHeartbeatReceiverPassesHeartbeatsAndErrors(t *testing.T) {
stream := &fakeHeartbeatStream{heartbeats: []*master_pb.Heartbeat{{Ip: "10.0.0.1"}, {Ip: "10.0.0.2"}}}
recv, stop := heartbeatReceiver(stream, time.Second)
defer stop()
for _, ip := range []string{"10.0.0.1", "10.0.0.2"} {
hb, err := recv()
require.NoError(t, err)
assert.Equal(t, ip, hb.Ip)
}
_, err := recv()
assert.ErrorIs(t, err, io.EOF)
}
func TestHeartbeatReceiverTimesOutOnSilentStream(t *testing.T) {
stream := newSilentHeartbeatStream(t, &master_pb.Heartbeat{Ip: "10.0.0.1"})
recv, stop := heartbeatReceiver(stream, 50*time.Millisecond)
defer stop()
_, err := recv()
require.NoError(t, err)
errc := make(chan error, 1)
go func() {
_, err := recv()
errc <- err
}()
select {
case err = <-errc:
assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
case <-time.After(5 * time.Second):
t.Fatal("recv still waiting on a silent stream")
}
}
// A volume server that stops sending heartbeats is unregistered once
// volumeHeartbeatTimeout passes, although its stream is still open.
func TestSendHeartbeatUnregistersSilentVolumeServer(t *testing.T) {
saved := volumeHeartbeatTimeout
volumeHeartbeatTimeout = 500 * time.Millisecond
defer func() { volumeHeartbeatTimeout = saved }()
ms := newLeaderMaster()
ms.clientChans = make(map[string]chan *master_pb.KeepConnectedResponse)
_, clientChan := ms.addClient("", "filer", "10.0.0.9:8888")
stream := newSilentHeartbeatStream(t, &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
MaxVolumeCounts: map[string]uint32{"": 10},
Volumes: []*master_pb.VolumeInformationMessage{{Id: 7, Version: 3}},
})
errc := make(chan error, 1)
go func() { errc <- ms.SendHeartbeat(stream) }()
require.Eventually(t, func() bool { return ms.Topo.DataNodeExists("", "", "10.0.0.1:8080") },
5*time.Second, 10*time.Millisecond, "the first heartbeat registers the server")
select {
case err := <-errc:
assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
case <-time.After(10 * time.Second):
t.Fatal("SendHeartbeat still waiting on a silent stream")
}
assert.False(t, ms.Topo.DataNodeExists("", "", "10.0.0.1:8080"))
for {
select {
case message := <-clientChan:
if location := message.GetVolumeLocation(); len(location.GetDeletedVids()) > 0 {
assert.Equal(t, []uint32{7}, location.DeletedVids)
return
}
default:
t.Fatal("clients were not told the silent server's volumes are gone")
}
}
}