filer: stop following a peer that stays unreachable (#12725)

* filer: stop following a peer that stays unreachable

* filer: re-track an answering peer only while it is still followed

A signature read that succeeds just before the master removes the peer
called initPeerWatermark after the removal, clearing its mark. The loop
then exits, so the watermark it re-tracked never advances or drops and
pins aggregated readers at the settled horizon.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: forget a peer's unreachable clock when the master removes it

Only a successful connection cleared the clock, so every departed address
that once failed stayed in the map for the filer's lifetime, and a re-added
peer inherited the old outage. The removal now drops it, and the clock is
guarded by peerChansLock so the removed peer's exiting loop cannot start a
new one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: report the unreachable mark only when it is set

Once peerResumeRetention forgets the mark of a peer that stays
unreachable, markPeerWatermarkRemoved no longer marks it (its watermarks
are gone), but notePeerUnreachable still saw no mark and logged
"treating it as removed" on every retry. markPeerWatermarkRemoved now
reports whether it set a new mark.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: fix the unreachable-peer subscriber's start before the baseline write

The subscriber goroutine picked its start time itself, so a baseline write
that finished first fell before the subscription and the test failed after
20 seconds with correct delivery.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-18 11:00:15 -08:00
1 parent 72ac9e96b2
commit b64895ed0a
4 files changed
+358 -5

No files matched your search

@@ -0,0 +1,116 @@
//go:build !windows
package metadata_subscribe
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// A peer removal is broadcast only to the filers connected to the master that
// saw the peer go. A peer that dies while no master is up (or across a leader
// change) is never removed: the filer keeps retrying its address, the peer's
// delivery watermark stays at zero, and every aggregated subscriber of that
// filer is held at the settled horizon (2 x LogFlushInterval) for good.
func TestAggregatedSubscriberNotHeldByPeerThatDiedUnnoticed(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
weedBinary := findWeedBinary()
require.NotEmpty(t, weedBinary, "weed binary not found")
testDir, err := os.MkdirTemp("", "seaweedfs_peer_unreachable_")
require.NoError(t, err)
t.Cleanup(func() {
if t.Failed() {
t.Logf("logs kept at %s", testDir)
return
}
os.RemoveAll(testDir)
})
ports, err := testutil.AllocateMiniPorts(3)
require.NoError(t, err)
masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2]
master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort))
filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port)
masterArgs := []string{"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(masterPort),
"-mdir=" + mkdir(t, testDir, "master"),
"-peers=none"}
masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort)
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second))
filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port)
// filer1 follows filer2: an entry created on filer2 reaches filer1
createPeerEntry(t, filer2Address, "from-filer2")
require.NoError(t, waitForPeerEntry(filer1Address, "from-filer2", 60*time.Second),
"filer1 never followed filer2")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
events := make(chan *filer_pb.SubscribeMetadataResponse, 1000)
sinceNs := time.Now().UnixNano()
go subscribeToMetadataWithOptions(ctx, filer1Address, peerEntryDir, sinceNs, events)
arrived := make(map[string]time.Time)
waitArrival := func(name string, within time.Duration) bool {
deadline := time.After(within)
for {
if _, ok := arrived[name]; ok {
return true
}
select {
case ev := <-events:
if n := ev.GetEventNotification().GetNewEntry(); n != nil {
arrived[n.Name] = time.Now()
}
case <-deadline:
return false
}
}
}
createPeerEntry(t, filer1Address, "baseline")
require.True(t, waitArrival("baseline", 20*time.Second), "the baseline event never reached the subscriber")
// filer2 dies while no master is up, so no removal is ever broadcast
stopProcess(masterProcess)
stopProcess(filer2)
startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second))
diedAt := time.Now()
// Unpatched, every event from here on arrives 2 x LogFlushInterval late.
// The dead peer may hold reads for its unreachable bound plus the removal
// grace (2 x LogFlushInterval each); after that delivery must be prompt.
for i := 0; ; i++ {
name := fmt.Sprintf("probe-%d", i)
createPeerEntry(t, filer1Address, name)
if waitArrival(name, 10*time.Second) {
t.Logf("%s delivered promptly %v after the peer died", name, time.Since(diedAt).Round(time.Second))
return
}
if time.Since(diedAt) > 6*time.Minute {
t.Fatalf("aggregated delivery still held %v after the peer died unnoticed", time.Since(diedAt).Round(time.Second))
}
}
}
+59 -5
View File
@@ -51,6 +51,10 @@ type MetaAggregator struct {
// gone past the grace is dropped so it cannot pin the low-watermarks.
// The mark itself stays until the peer's resume point is forgotten.
peerRemovedAtNs map[pb.ServerAddress]int64
// peerUnreachableSinceNs is when each followed peer's subscription last
// started failing to connect; see notePeerUnreachable. Guarded by
// peerChansLock, so a removed peer's exiting loop cannot recreate it.
peerUnreachableSinceNs map[pb.ServerAddress]int64
// lowWatermarkTsNs and lowFlushWatermarkTsNs are the last minima signalled
// on deliveryAdvanced and flushAdvanced, each closed and replaced when its
// own minimum rises. Held readers park on the one that bounds them:
@@ -95,6 +99,8 @@ func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.
peerFlushWatermarks: make(map[pb.ServerAddress]int64),
peerRemovedAtNs: make(map[pb.ServerAddress]int64),
peerResumeTsNs: make(map[pb.ServerAddress]int64),
peerUnreachableSinceNs: make(map[pb.ServerAddress]int64),
}
t.peerSubscribeLoop = t.loopSubscribeToOneFiler
// nil notifyFn: aggregated subscribers wake through the buffer's
@@ -140,6 +146,7 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
close(prevChan)
delete(ma.peerChans, address)
}
delete(ma.peerUnreachableSinceNs, address)
// Only mark: dropping the watermarks at once would let subscribers
// advance past a flapping peer's unflushed events (see peerRemovedAtNs).
ma.markPeerWatermarkRemoved(address)
@@ -156,6 +163,46 @@ const peerWatermarkRemovalGrace = 2 * LogFlushInterval
// position.
const peerResumeRetention = time.Hour
// peerUnreachableRemovalAfter is how long a peer may refuse connections
// before it is treated as removed.
const peerUnreachableRemovalAfter = 2 * LogFlushInterval
// errPeerUnreachable marks a subscription attempt that never reached the peer.
var errPeerUnreachable = errors.New("connecting to peer filer")
// notePeerUnreachable records a failed connection to peer at nowNs and marks
// the peer removed once the failures have lasted peerUnreachableRemovalAfter.
// The master's removal notice reaches only the clients attached to the master
// that saw the peer go, so a peer that died across a leader change is
// otherwise followed forever and its zero watermark holds every aggregated
// subscriber at the settled horizon. Reports true the one time it marks.
func (ma *MetaAggregator) notePeerUnreachable(peer pb.ServerAddress, stopChan <-chan struct{}, nowNs int64) bool {
ma.peerChansLock.Lock()
defer ma.peerChansLock.Unlock()
if ma.peerChans[peer] != stopChan {
return false
}
since, found := ma.peerUnreachableSinceNs[peer]
if !found {
ma.peerUnreachableSinceNs[peer] = nowNs
since = nowNs
}
return nowNs-since >= int64(peerUnreachableRemovalAfter) && ma.markPeerWatermarkRemoved(peer)
}
// notePeerReachable tracks peer again after a successful connection, from a
// zero watermark if the removal grace already dropped it. A subscription the
// master has removed meanwhile must not undo that removal.
func (ma *MetaAggregator) notePeerReachable(peer pb.ServerAddress, stopChan <-chan struct{}) {
ma.peerChansLock.Lock()
defer ma.peerChansLock.Unlock()
if ma.peerChans[peer] != stopChan {
return
}
delete(ma.peerUnreachableSinceNs, peer)
ma.initPeerWatermark(peer)
}
// initPeerWatermark adds the peer with a zero (unknown) watermark; a re-added
// peer keeps its prior values and stops being considered removed.
func (ma *MetaAggregator) initPeerWatermark(peer pb.ServerAddress) {
@@ -171,16 +218,19 @@ func (ma *MetaAggregator) initPeerWatermark(peer pb.ServerAddress) {
ma.noteLowWatermarksLocked()
}
func (ma *MetaAggregator) markPeerWatermarkRemoved(peer pb.ServerAddress) {
// markPeerWatermarkRemoved reports whether it set a new mark.
func (ma *MetaAggregator) markPeerWatermarkRemoved(peer pb.ServerAddress) bool {
ma.peerWatermarksLock.Lock()
defer ma.peerWatermarksLock.Unlock()
if _, found := ma.peerWatermarks[peer]; !found {
return
return false
}
// First removal time wins: duplicate removals must not refresh the grace.
if _, marked := ma.peerRemovedAtNs[peer]; !marked {
ma.peerRemovedAtNs[peer] = time.Now().UnixNano()
if _, marked := ma.peerRemovedAtNs[peer]; marked {
return false
}
ma.peerRemovedAtNs[peer] = time.Now().UnixNano()
return true
}
// dropExpiredRemovedPeersLocked drops the watermarks of peers whose removal
@@ -494,6 +544,9 @@ func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddres
errLvl = glog.Level(4)
}
glog.V(errLvl).Infof("subscribing remote %s meta change: %v", peer, err)
if errors.Is(err, errPeerUnreachable) && ma.notePeerUnreachable(peer, stopChan, time.Now().UnixNano()) {
glog.V(0).Infof("peer %s unreachable for %v, treating it as removed until it answers", peer, peerUnreachableRemovalAfter)
}
}
if lastTsNs < nextLastTsNs {
lastTsNs = nextLastTsNs
@@ -548,8 +601,9 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
peerSignature, err := ma.readFilerStoreSignature(ctx, peer)
if err != nil {
return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err)
return lastTsNs, fmt.Errorf("%w %s: %v", errPeerUnreachable, peer, err)
}
ma.notePeerReachable(peer, stopChan)
// when filer store is not shared by multiple filers
if peerSignature != f.Signature {
+57
View File
@@ -1,9 +1,15 @@
package filer
import (
"fmt"
"net"
"strconv"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
@@ -36,6 +42,57 @@ func TestOnPeerUpdateRepeatedAdd(t *testing.T) {
}
}
// A peer the subscription loop cannot connect to starts the unreachable
// clock, so a peer that died without a removal notice stops being followed
// as if it were alive.
func TestPeerSubscribeLoopNotesUnreachablePeer(t *testing.T) {
// A listener that accepts then immediately closes every connection makes
// the gRPC dial fail fast and portably: a closed port behaves differently
// per OS (Linux sends RST at once, some stacks drop the SYN and make the
// dial wait out grpc's 20s connect timeout), which this test must not rely
// on. The filer HTTP address is the gRPC port minus 10000.
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
conn.Close()
}
}()
_, portStr, err := net.SplitHostPort(ln.Addr().String())
if err != nil {
t.Fatal(err)
}
grpcPort, err := strconv.Atoi(portStr)
if err != nil || grpcPort <= 10000 {
t.Fatalf("unexpected listener port %q", portStr)
}
peer := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", grpcPort-10000))
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), grpc.WithTransportCredentials(insecure.NewCredentials()))
defer ma.Shutdown()
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}, time.Now())
deadline := time.Now().Add(10 * time.Second)
for {
ma.peerChansLock.Lock()
_, noted := ma.peerUnreachableSinceNs[peer]
ma.peerChansLock.Unlock()
if noted {
return
}
if time.Now().After(deadline) {
t.Fatal("a refused connection never started the unreachable clock")
}
time.Sleep(50 * time.Millisecond)
}
}
// Shutdown must close every peer's stopChan and wait for the loops, which is
// what lets each loop's deferred offset save run before the store closes.
func TestMetaAggregatorShutdownStopsPeerLoops(t *testing.T) {
@@ -5,14 +5,18 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func newTestAggregator() *MetaAggregator {
ma := &MetaAggregator{
peerChans: make(map[pb.ServerAddress]chan struct{}),
peerWatermarks: make(map[pb.ServerAddress]int64),
peerFlushWatermarks: make(map[pb.ServerAddress]int64),
peerRemovedAtNs: make(map[pb.ServerAddress]int64),
peerResumeTsNs: make(map[pb.ServerAddress]int64),
peerUnreachableSinceNs: make(map[pb.ServerAddress]int64),
}
ma.peerSubscribeLoop = ma.loopSubscribeToOneFiler
return ma
@@ -217,3 +221,125 @@ func TestWatermarkAdvancedChansAreScoped(t *testing.T) {
t.Fatal("the trailing peer catching up did not signal a rise")
}
}
// TestUnreachablePeerStopsPinningAfterTheGrace pins the rule for a peer whose
// removal never reached this filer (it died across a master leader change):
// its zero watermark holds the low-watermark only until the connection
// failures have lasted peerUnreachableRemovalAfter; it is then marked removed,
// drops after the usual grace, and is tracked again the moment it answers.
func TestUnreachablePeerStopsPinningAfterTheGrace(t *testing.T) {
ma := newTestAggregator()
a, b := pb.ServerAddress("filer-a:8888"), pb.ServerAddress("filer-b:8888")
ma.initPeerWatermark(a)
ma.initPeerWatermark(b)
ma.advancePeerWatermark(a, 100)
stopB := make(chan struct{})
ma.peerChans[b] = stopB
t0 := time.Now().UnixNano()
if ma.notePeerUnreachable(b, stopB, t0) || ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)-1) {
t.Fatal("a peer unreachable for less than the bound must not be marked")
}
if !ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)) {
t.Fatal("a peer unreachable for the bound must be marked removed")
}
if ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)+1) {
t.Fatal("the mark must be reported once")
}
if got := ma.PeerLowWatermarkTsNs(); got != 0 {
t.Fatalf("within the removal grace: low=%d want 0", got)
}
ma.peerWatermarksLock.Lock()
ma.peerRemovedAtNs[b] = time.Now().UnixNano() - int64(peerWatermarkRemovalGrace) - 1
ma.peerWatermarksLock.Unlock()
if got := ma.PeerLowWatermarkTsNs(); got != 100 {
t.Fatalf("past the grace the unreachable peer must not pin: low=%d want 100", got)
}
// It answers again: tracked from zero until it signals.
ma.notePeerReachable(b, stopB)
if got := ma.PeerLowWatermarkTsNs(); got != 0 {
t.Fatalf("re-tracked peer before its first claim: low=%d want 0", got)
}
ma.advancePeerWatermark(b, 70)
if got := ma.PeerLowWatermarkTsNs(); got != 70 {
t.Fatalf("after its claim: low=%d want 70", got)
}
if ma.notePeerUnreachable(b, stopB, t0+10*int64(peerUnreachableRemovalAfter)) {
t.Fatal("a fresh outage must start a fresh clock")
}
}
// A subscription whose signature read succeeded just before the master
// removed the peer must not clear the removal: its loop is about to exit, so
// nothing would advance or drop the watermark it re-tracked.
func TestStaleSubscriptionDoesNotUndoMasterRemoval(t *testing.T) {
ma := newTestAggregator()
ma.peerSubscribeLoop = func(_ *Filer, _, _ pb.ServerAddress, _ time.Time, _ chan struct{}) {}
a, b := pb.ServerAddress("filer-a:8888"), pb.ServerAddress("filer-b:8888")
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(a), IsAdd: true}, time.Now())
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: true}, time.Now())
ma.advancePeerWatermark(a, 100)
ma.advancePeerWatermark(b, 50)
stopB := ma.peerChans[b]
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: false}, time.Now())
ma.notePeerReachable(b, stopB)
ma.peerWatermarksLock.Lock()
_, marked := ma.peerRemovedAtNs[b]
ma.peerRemovedAtNs[b] = time.Now().UnixNano() - int64(peerWatermarkRemovalGrace) - 1
ma.peerWatermarksLock.Unlock()
if !marked {
t.Fatal("a stale subscription cleared the master's removal mark")
}
if got := ma.PeerLowWatermarkTsNs(); got != 100 {
t.Fatalf("the removed peer still pins past the grace: low=%d want 100", got)
}
}
// A master removal forgets the peer's unreachable clock, and the removed
// peer's exiting loop cannot start a new one: departed addresses do not
// accumulate, and a re-added peer starts a fresh outage clock.
func TestMasterRemovalForgetsUnreachableClock(t *testing.T) {
ma := newTestAggregator()
ma.peerSubscribeLoop = func(_ *Filer, _, _ pb.ServerAddress, _ time.Time, _ chan struct{}) {}
b := pb.ServerAddress("filer-b:8888")
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: true}, time.Now())
stopB := ma.peerChans[b]
t0 := time.Now().UnixNano()
ma.notePeerUnreachable(b, stopB, t0)
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: false}, time.Now())
ma.notePeerUnreachable(b, stopB, t0+1)
if n := len(ma.peerUnreachableSinceNs); n != 0 {
t.Fatalf("removed peer still has an unreachable clock: %d entries", n)
}
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: true}, time.Now())
if ma.notePeerUnreachable(b, ma.peerChans[b], t0+int64(peerUnreachableRemovalAfter)) {
t.Fatal("a re-added peer inherited the previous outage clock")
}
}
// A peer still unreachable once its removal mark has been forgotten is no
// longer tracked, so it is not reported as newly marked on every retry.
func TestLongUnreachablePeerIsReportedOnce(t *testing.T) {
ma := newTestAggregator()
b := pb.ServerAddress("filer-b:8888")
ma.initPeerWatermark(b)
stopB := make(chan struct{})
ma.peerChans[b] = stopB
t0 := time.Now().UnixNano()
ma.notePeerUnreachable(b, stopB, t0)
if !ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)) {
t.Fatal("a peer unreachable for the bound must be marked removed")
}
ma.peerWatermarksLock.Lock()
ma.peerRemovedAtNs[b] = time.Now().UnixNano() - int64(peerResumeRetention) - 1
ma.dropExpiredRemovedPeersLocked()
ma.peerWatermarksLock.Unlock()
if ma.notePeerUnreachable(b, stopB, t0+2*int64(peerUnreachableRemovalAfter)) {
t.Fatal("an untracked peer was reported as newly marked")
}
}