mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
filer: rebuild peer metadata subscriptions after a master reconnect (#10648)
* filer: keep the existing peer subscription on a repeated add A cluster node add for a peer that is already followed restarted the subscription, dropping the metadata events between the two runs. * master: tell a connecting client the current cluster membership Cluster node updates are only broadcast to the clients connected at that moment. A filer that lost its master stream while a peer came back never learned about the peer, and stopped replicating its metadata for good. * test: a filer joining the master learns about the filers already there * test: a filer resubscribes to a peer that registered while it was disconnected Runs the reported sequence against real processes: filer2 leaves, filer1 is paused and its master stream is broken, filer2 registers again, and filer1 has to replicate from it after reconnecting.
This commit is contained in:
1 parent
37f3dff677
commit
3a61debaa5
8 files changed
+432
-3
No files matched your search
@@ -138,6 +138,16 @@ func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType stri
|
||||
return
|
||||
}
|
||||
|
||||
// ListClusterNodeUpdates reports the current members as add updates, so a
|
||||
// client that just connected can rebuild the membership it missed while it was
|
||||
// away.
|
||||
func (cluster *Cluster) ListClusterNodeUpdates(filerGroup FilerGroupName, nodeType string) (updates []*master_pb.KeepConnectedResponse) {
|
||||
for _, node := range cluster.ListClusterNode(filerGroup, nodeType) {
|
||||
updates = append(updates, buildClusterNodeUpdateMessage(true, filerGroup, nodeType, node.Address)...)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// IsKnownNode reports whether address is currently registered under nodeType
|
||||
// in any filer group. The lookup is intentionally group-agnostic because callers
|
||||
// (e.g. Ping admission) only know the target address, not the group it joined.
|
||||
|
||||
@@ -40,6 +40,27 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestListClusterNodeUpdates(t *testing.T) {
|
||||
c := NewCluster()
|
||||
filer := pb.ServerAddress("10.0.0.20:8888")
|
||||
c.AddClusterNode("group", FilerType, "dc1", "rack1", filer, "test")
|
||||
c.AddClusterNode("group", BrokerType, "dc1", "rack1", pb.ServerAddress("10.0.0.20:17777"), "test")
|
||||
|
||||
updates := c.ListClusterNodeUpdates("group", FilerType)
|
||||
if len(updates) != 1 {
|
||||
t.Fatalf("expecting one filer update, got %d", len(updates))
|
||||
}
|
||||
update := updates[0].ClusterNodeUpdate
|
||||
if update.Address != string(filer) || !update.IsAdd || update.FilerGroup != "group" {
|
||||
t.Fatalf("unexpected update %+v", update)
|
||||
}
|
||||
|
||||
c.RemoveClusterNode("group", FilerType, filer)
|
||||
if updates := c.ListClusterNodeUpdates("group", FilerType); len(updates) != 0 {
|
||||
t.Fatalf("expecting no update for a removed filer, got %d", len(updates))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsKnownNode(t *testing.T) {
|
||||
c := NewCluster()
|
||||
filer := pb.ServerAddress("10.0.0.20:8888")
|
||||
|
||||
@@ -53,9 +53,10 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
|
||||
|
||||
address := pb.ServerAddress(update.Address)
|
||||
if update.IsAdd {
|
||||
// cancel previous subscription if any
|
||||
if prevChan, found := ma.peerChans[address]; found {
|
||||
close(prevChan)
|
||||
// the peer is already followed, restarting would only lose the events
|
||||
// in between
|
||||
if _, found := ma.peerChans[address]; found {
|
||||
return
|
||||
}
|
||||
stopChan := make(chan struct{})
|
||||
ma.peerChans[address] = stopChan
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
func TestOnPeerUpdateRepeatedAdd(t *testing.T) {
|
||||
peer := pb.ServerAddress("127.0.0.1:1")
|
||||
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
|
||||
|
||||
add := &master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}
|
||||
ma.OnPeerUpdate(add, time.Now())
|
||||
first, found := ma.peerChans[peer]
|
||||
if !found {
|
||||
t.Fatal("expecting a subscription after the first add")
|
||||
}
|
||||
|
||||
ma.OnPeerUpdate(add, time.Now())
|
||||
if ma.peerChans[peer] != first {
|
||||
t.Fatal("expecting the same subscription after a repeated add")
|
||||
}
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatal("expecting the subscription to stay alive after a repeated add")
|
||||
default:
|
||||
}
|
||||
|
||||
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
|
||||
if _, found := ma.peerChans[peer]; found {
|
||||
t.Fatal("expecting the subscription to be removed")
|
||||
}
|
||||
}
|
||||
@@ -437,6 +437,14 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
|
||||
}
|
||||
}
|
||||
|
||||
// Cluster node changes are only broadcast to the clients connected at that
|
||||
// moment, so a client that reconnects has to be told who is around now.
|
||||
for _, update := range ms.Cluster.ListClusterNodeUpdates(cluster.FilerGroupName(req.FilerGroup), cluster.FilerType) {
|
||||
if sendErr := stream.Send(update); sendErr != nil {
|
||||
return sendErr
|
||||
}
|
||||
}
|
||||
|
||||
if initialLockRingUpdate := ms.initialLockRingUpdate(req.ClientType, req.FilerGroup); initialLockRingUpdate != nil {
|
||||
if sendErr := stream.Send(initialLockRingUpdate); sendErr != nil {
|
||||
return sendErr
|
||||
|
||||
Reference in new issue
Block a user