filer: rebuild peer metadata subscriptions after a master reconnect (#10648)

* filer: keep the existing peer subscription on a repeated add

A cluster node add for a peer that is already followed restarted the
subscription, dropping the metadata events between the two runs.

* master: tell a connecting client the current cluster membership

Cluster node updates are only broadcast to the clients connected at that
moment. A filer that lost its master stream while a peer came back never
learned about the peer, and stopped replicating its metadata for good.

* test: a filer joining the master learns about the filers already there

* test: a filer resubscribes to a peer that registered while it was disconnected

Runs the reported sequence against real processes: filer2 leaves, filer1
is paused and its master stream is broken, filer2 registers again, and
filer1 has to replicate from it after reconnecting.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-08-08 10:28:25 -07:00
1 parent 37f3dff677
commit 3a61debaa5
8 files changed
+432 -3

No files matched your search

+10
View File
@@ -138,6 +138,16 @@ func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType stri
return
}
// ListClusterNodeUpdates reports the current members as add updates, so a
// client that just connected can rebuild the membership it missed while it was
// away.
func (cluster *Cluster) ListClusterNodeUpdates(filerGroup FilerGroupName, nodeType string) (updates []*master_pb.KeepConnectedResponse) {
for _, node := range cluster.ListClusterNode(filerGroup, nodeType) {
updates = append(updates, buildClusterNodeUpdateMessage(true, filerGroup, nodeType, node.Address)...)
}
return
}
// IsKnownNode reports whether address is currently registered under nodeType
// in any filer group. The lookup is intentionally group-agnostic because callers
// (e.g. Ping admission) only know the target address, not the group it joined.
+21
View File
@@ -40,6 +40,27 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
wg.Wait()
}
func TestListClusterNodeUpdates(t *testing.T) {
c := NewCluster()
filer := pb.ServerAddress("10.0.0.20:8888")
c.AddClusterNode("group", FilerType, "dc1", "rack1", filer, "test")
c.AddClusterNode("group", BrokerType, "dc1", "rack1", pb.ServerAddress("10.0.0.20:17777"), "test")
updates := c.ListClusterNodeUpdates("group", FilerType)
if len(updates) != 1 {
t.Fatalf("expecting one filer update, got %d", len(updates))
}
update := updates[0].ClusterNodeUpdate
if update.Address != string(filer) || !update.IsAdd || update.FilerGroup != "group" {
t.Fatalf("unexpected update %+v", update)
}
c.RemoveClusterNode("group", FilerType, filer)
if updates := c.ListClusterNodeUpdates("group", FilerType); len(updates) != 0 {
t.Fatalf("expecting no update for a removed filer, got %d", len(updates))
}
}
func TestIsKnownNode(t *testing.T) {
c := NewCluster()
filer := pb.ServerAddress("10.0.0.20:8888")
+4 -3
View File
@@ -53,9 +53,10 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
address := pb.ServerAddress(update.Address)
if update.IsAdd {
// cancel previous subscription if any
if prevChan, found := ma.peerChans[address]; found {
close(prevChan)
// the peer is already followed, restarting would only lose the events
// in between
if _, found := ma.peerChans[address]; found {
return
}
stopChan := make(chan struct{})
ma.peerChans[address] = stopChan
+36
View File
@@ -0,0 +1,36 @@
package filer
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func TestOnPeerUpdateRepeatedAdd(t *testing.T) {
peer := pb.ServerAddress("127.0.0.1:1")
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
add := &master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}
ma.OnPeerUpdate(add, time.Now())
first, found := ma.peerChans[peer]
if !found {
t.Fatal("expecting a subscription after the first add")
}
ma.OnPeerUpdate(add, time.Now())
if ma.peerChans[peer] != first {
t.Fatal("expecting the same subscription after a repeated add")
}
select {
case <-first:
t.Fatal("expecting the subscription to stay alive after a repeated add")
default:
}
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
if _, found := ma.peerChans[peer]; found {
t.Fatal("expecting the subscription to be removed")
}
}
+8
View File
@@ -437,6 +437,14 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
}
}
// Cluster node changes are only broadcast to the clients connected at that
// moment, so a client that reconnects has to be told who is around now.
for _, update := range ms.Cluster.ListClusterNodeUpdates(cluster.FilerGroupName(req.FilerGroup), cluster.FilerType) {
if sendErr := stream.Send(update); sendErr != nil {
return sendErr
}
}
if initialLockRingUpdate := ms.initialLockRingUpdate(req.ClientType, req.FilerGroup); initialLockRingUpdate != nil {
if sendErr := stream.Send(initialLockRingUpdate); sendErr != nil {
return sendErr