diff --git a/test/metadata_subscribe/peer_unreachable_test.go b/test/metadata_subscribe/peer_unreachable_test.go new file mode 100644 index 0000000000..278dbcbbe6 --- /dev/null +++ b/test/metadata_subscribe/peer_unreachable_test.go @@ -0,0 +1,116 @@ +//go:build !windows + +package metadata_subscribe + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/test/testutil" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +// A peer removal is broadcast only to the filers connected to the master that +// saw the peer go. A peer that dies while no master is up (or across a leader +// change) is never removed: the filer keeps retrying its address, the peer's +// delivery watermark stays at zero, and every aggregated subscriber of that +// filer is held at the settled horizon (2 x LogFlushInterval) for good. +func TestAggregatedSubscriberNotHeldByPeerThatDiedUnnoticed(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + weedBinary := findWeedBinary() + require.NotEmpty(t, weedBinary, "weed binary not found") + + testDir, err := os.MkdirTemp("", "seaweedfs_peer_unreachable_") + require.NoError(t, err) + t.Cleanup(func() { + if t.Failed() { + t.Logf("logs kept at %s", testDir) + return + } + os.RemoveAll(testDir) + }) + + ports, err := testutil.AllocateMiniPorts(3) + require.NoError(t, err) + masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2] + master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort)) + filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port) + + masterArgs := []string{"master", + "-ip=127.0.0.1", + "-port=" + strconv.Itoa(masterPort), + "-mdir=" + mkdir(t, testDir, "master"), + "-peers=none"} + masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...) + require.NoError(t, waitForLeader(masterPort, 60*time.Second)) + startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort) + require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second)) + filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort) + require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second)) + filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port) + + // filer1 follows filer2: an entry created on filer2 reaches filer1 + createPeerEntry(t, filer2Address, "from-filer2") + require.NoError(t, waitForPeerEntry(filer1Address, "from-filer2", 60*time.Second), + "filer1 never followed filer2") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events := make(chan *filer_pb.SubscribeMetadataResponse, 1000) + sinceNs := time.Now().UnixNano() + go subscribeToMetadataWithOptions(ctx, filer1Address, peerEntryDir, sinceNs, events) + arrived := make(map[string]time.Time) + waitArrival := func(name string, within time.Duration) bool { + deadline := time.After(within) + for { + if _, ok := arrived[name]; ok { + return true + } + select { + case ev := <-events: + if n := ev.GetEventNotification().GetNewEntry(); n != nil { + arrived[n.Name] = time.Now() + } + case <-deadline: + return false + } + } + } + + createPeerEntry(t, filer1Address, "baseline") + require.True(t, waitArrival("baseline", 20*time.Second), "the baseline event never reached the subscriber") + + // filer2 dies while no master is up, so no removal is ever broadcast + stopProcess(masterProcess) + stopProcess(filer2) + startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...) + require.NoError(t, waitForLeader(masterPort, 60*time.Second)) + require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second)) + diedAt := time.Now() + + // Unpatched, every event from here on arrives 2 x LogFlushInterval late. + // The dead peer may hold reads for its unreachable bound plus the removal + // grace (2 x LogFlushInterval each); after that delivery must be prompt. + for i := 0; ; i++ { + name := fmt.Sprintf("probe-%d", i) + createPeerEntry(t, filer1Address, name) + if waitArrival(name, 10*time.Second) { + t.Logf("%s delivered promptly %v after the peer died", name, time.Since(diedAt).Round(time.Second)) + return + } + if time.Since(diedAt) > 6*time.Minute { + t.Fatalf("aggregated delivery still held %v after the peer died unnoticed", time.Since(diedAt).Round(time.Second)) + } + } +} diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index d4244060e5..83fe9db3c5 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -51,6 +51,10 @@ type MetaAggregator struct { // gone past the grace is dropped so it cannot pin the low-watermarks. // The mark itself stays until the peer's resume point is forgotten. peerRemovedAtNs map[pb.ServerAddress]int64 + // peerUnreachableSinceNs is when each followed peer's subscription last + // started failing to connect; see notePeerUnreachable. Guarded by + // peerChansLock, so a removed peer's exiting loop cannot recreate it. + peerUnreachableSinceNs map[pb.ServerAddress]int64 // lowWatermarkTsNs and lowFlushWatermarkTsNs are the last minima signalled // on deliveryAdvanced and flushAdvanced, each closed and replaced when its // own minimum rises. Held readers park on the one that bounds them: @@ -95,6 +99,8 @@ func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc. peerFlushWatermarks: make(map[pb.ServerAddress]int64), peerRemovedAtNs: make(map[pb.ServerAddress]int64), peerResumeTsNs: make(map[pb.ServerAddress]int64), + + peerUnreachableSinceNs: make(map[pb.ServerAddress]int64), } t.peerSubscribeLoop = t.loopSubscribeToOneFiler // nil notifyFn: aggregated subscribers wake through the buffer's @@ -140,6 +146,7 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star close(prevChan) delete(ma.peerChans, address) } + delete(ma.peerUnreachableSinceNs, address) // Only mark: dropping the watermarks at once would let subscribers // advance past a flapping peer's unflushed events (see peerRemovedAtNs). ma.markPeerWatermarkRemoved(address) @@ -156,6 +163,46 @@ const peerWatermarkRemovalGrace = 2 * LogFlushInterval // position. const peerResumeRetention = time.Hour +// peerUnreachableRemovalAfter is how long a peer may refuse connections +// before it is treated as removed. +const peerUnreachableRemovalAfter = 2 * LogFlushInterval + +// errPeerUnreachable marks a subscription attempt that never reached the peer. +var errPeerUnreachable = errors.New("connecting to peer filer") + +// notePeerUnreachable records a failed connection to peer at nowNs and marks +// the peer removed once the failures have lasted peerUnreachableRemovalAfter. +// The master's removal notice reaches only the clients attached to the master +// that saw the peer go, so a peer that died across a leader change is +// otherwise followed forever and its zero watermark holds every aggregated +// subscriber at the settled horizon. Reports true the one time it marks. +func (ma *MetaAggregator) notePeerUnreachable(peer pb.ServerAddress, stopChan <-chan struct{}, nowNs int64) bool { + ma.peerChansLock.Lock() + defer ma.peerChansLock.Unlock() + if ma.peerChans[peer] != stopChan { + return false + } + since, found := ma.peerUnreachableSinceNs[peer] + if !found { + ma.peerUnreachableSinceNs[peer] = nowNs + since = nowNs + } + return nowNs-since >= int64(peerUnreachableRemovalAfter) && ma.markPeerWatermarkRemoved(peer) +} + +// notePeerReachable tracks peer again after a successful connection, from a +// zero watermark if the removal grace already dropped it. A subscription the +// master has removed meanwhile must not undo that removal. +func (ma *MetaAggregator) notePeerReachable(peer pb.ServerAddress, stopChan <-chan struct{}) { + ma.peerChansLock.Lock() + defer ma.peerChansLock.Unlock() + if ma.peerChans[peer] != stopChan { + return + } + delete(ma.peerUnreachableSinceNs, peer) + ma.initPeerWatermark(peer) +} + // initPeerWatermark adds the peer with a zero (unknown) watermark; a re-added // peer keeps its prior values and stops being considered removed. func (ma *MetaAggregator) initPeerWatermark(peer pb.ServerAddress) { @@ -171,16 +218,19 @@ func (ma *MetaAggregator) initPeerWatermark(peer pb.ServerAddress) { ma.noteLowWatermarksLocked() } -func (ma *MetaAggregator) markPeerWatermarkRemoved(peer pb.ServerAddress) { +// markPeerWatermarkRemoved reports whether it set a new mark. +func (ma *MetaAggregator) markPeerWatermarkRemoved(peer pb.ServerAddress) bool { ma.peerWatermarksLock.Lock() defer ma.peerWatermarksLock.Unlock() if _, found := ma.peerWatermarks[peer]; !found { - return + return false } // First removal time wins: duplicate removals must not refresh the grace. - if _, marked := ma.peerRemovedAtNs[peer]; !marked { - ma.peerRemovedAtNs[peer] = time.Now().UnixNano() + if _, marked := ma.peerRemovedAtNs[peer]; marked { + return false } + ma.peerRemovedAtNs[peer] = time.Now().UnixNano() + return true } // dropExpiredRemovedPeersLocked drops the watermarks of peers whose removal @@ -494,6 +544,9 @@ func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddres errLvl = glog.Level(4) } glog.V(errLvl).Infof("subscribing remote %s meta change: %v", peer, err) + if errors.Is(err, errPeerUnreachable) && ma.notePeerUnreachable(peer, stopChan, time.Now().UnixNano()) { + glog.V(0).Infof("peer %s unreachable for %v, treating it as removed until it answers", peer, peerUnreachableRemovalAfter) + } } if lastTsNs < nextLastTsNs { lastTsNs = nextLastTsNs @@ -548,8 +601,9 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peerSignature, err := ma.readFilerStoreSignature(ctx, peer) if err != nil { - return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err) + return lastTsNs, fmt.Errorf("%w %s: %v", errPeerUnreachable, peer, err) } + ma.notePeerReachable(peer, stopChan) // when filer store is not shared by multiple filers if peerSignature != f.Signature { diff --git a/weed/filer/meta_aggregator_test.go b/weed/filer/meta_aggregator_test.go index 2b7b7f7e5b..fea3786590 100644 --- a/weed/filer/meta_aggregator_test.go +++ b/weed/filer/meta_aggregator_test.go @@ -1,9 +1,15 @@ package filer import ( + "fmt" + "net" + "strconv" "testing" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" ) @@ -36,6 +42,57 @@ func TestOnPeerUpdateRepeatedAdd(t *testing.T) { } } +// A peer the subscription loop cannot connect to starts the unreachable +// clock, so a peer that died without a removal notice stops being followed +// as if it were alive. +func TestPeerSubscribeLoopNotesUnreachablePeer(t *testing.T) { + // A listener that accepts then immediately closes every connection makes + // the gRPC dial fail fast and portably: a closed port behaves differently + // per OS (Linux sends RST at once, some stacks drop the SYN and make the + // dial wait out grpc's 20s connect timeout), which this test must not rely + // on. The filer HTTP address is the gRPC port minus 10000. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + conn.Close() + } + }() + _, portStr, err := net.SplitHostPort(ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + grpcPort, err := strconv.Atoi(portStr) + if err != nil || grpcPort <= 10000 { + t.Fatalf("unexpected listener port %q", portStr) + } + peer := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", grpcPort-10000)) + ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), grpc.WithTransportCredentials(insecure.NewCredentials())) + defer ma.Shutdown() + + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}, time.Now()) + deadline := time.Now().Add(10 * time.Second) + for { + ma.peerChansLock.Lock() + _, noted := ma.peerUnreachableSinceNs[peer] + ma.peerChansLock.Unlock() + if noted { + return + } + if time.Now().After(deadline) { + t.Fatal("a refused connection never started the unreachable clock") + } + time.Sleep(50 * time.Millisecond) + } +} + // Shutdown must close every peer's stopChan and wait for the loops, which is // what lets each loop's deferred offset save run before the store closes. func TestMetaAggregatorShutdownStopsPeerLoops(t *testing.T) { diff --git a/weed/filer/meta_aggregator_watermark_test.go b/weed/filer/meta_aggregator_watermark_test.go index 5edd2fb1e1..4a5ec547f0 100644 --- a/weed/filer/meta_aggregator_watermark_test.go +++ b/weed/filer/meta_aggregator_watermark_test.go @@ -5,14 +5,18 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" ) func newTestAggregator() *MetaAggregator { ma := &MetaAggregator{ + peerChans: make(map[pb.ServerAddress]chan struct{}), peerWatermarks: make(map[pb.ServerAddress]int64), peerFlushWatermarks: make(map[pb.ServerAddress]int64), peerRemovedAtNs: make(map[pb.ServerAddress]int64), peerResumeTsNs: make(map[pb.ServerAddress]int64), + + peerUnreachableSinceNs: make(map[pb.ServerAddress]int64), } ma.peerSubscribeLoop = ma.loopSubscribeToOneFiler return ma @@ -217,3 +221,125 @@ func TestWatermarkAdvancedChansAreScoped(t *testing.T) { t.Fatal("the trailing peer catching up did not signal a rise") } } + +// TestUnreachablePeerStopsPinningAfterTheGrace pins the rule for a peer whose +// removal never reached this filer (it died across a master leader change): +// its zero watermark holds the low-watermark only until the connection +// failures have lasted peerUnreachableRemovalAfter; it is then marked removed, +// drops after the usual grace, and is tracked again the moment it answers. +func TestUnreachablePeerStopsPinningAfterTheGrace(t *testing.T) { + ma := newTestAggregator() + a, b := pb.ServerAddress("filer-a:8888"), pb.ServerAddress("filer-b:8888") + ma.initPeerWatermark(a) + ma.initPeerWatermark(b) + ma.advancePeerWatermark(a, 100) + stopB := make(chan struct{}) + ma.peerChans[b] = stopB + + t0 := time.Now().UnixNano() + if ma.notePeerUnreachable(b, stopB, t0) || ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)-1) { + t.Fatal("a peer unreachable for less than the bound must not be marked") + } + if !ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)) { + t.Fatal("a peer unreachable for the bound must be marked removed") + } + if ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)+1) { + t.Fatal("the mark must be reported once") + } + if got := ma.PeerLowWatermarkTsNs(); got != 0 { + t.Fatalf("within the removal grace: low=%d want 0", got) + } + ma.peerWatermarksLock.Lock() + ma.peerRemovedAtNs[b] = time.Now().UnixNano() - int64(peerWatermarkRemovalGrace) - 1 + ma.peerWatermarksLock.Unlock() + if got := ma.PeerLowWatermarkTsNs(); got != 100 { + t.Fatalf("past the grace the unreachable peer must not pin: low=%d want 100", got) + } + + // It answers again: tracked from zero until it signals. + ma.notePeerReachable(b, stopB) + if got := ma.PeerLowWatermarkTsNs(); got != 0 { + t.Fatalf("re-tracked peer before its first claim: low=%d want 0", got) + } + ma.advancePeerWatermark(b, 70) + if got := ma.PeerLowWatermarkTsNs(); got != 70 { + t.Fatalf("after its claim: low=%d want 70", got) + } + if ma.notePeerUnreachable(b, stopB, t0+10*int64(peerUnreachableRemovalAfter)) { + t.Fatal("a fresh outage must start a fresh clock") + } +} + +// A subscription whose signature read succeeded just before the master +// removed the peer must not clear the removal: its loop is about to exit, so +// nothing would advance or drop the watermark it re-tracked. +func TestStaleSubscriptionDoesNotUndoMasterRemoval(t *testing.T) { + ma := newTestAggregator() + ma.peerSubscribeLoop = func(_ *Filer, _, _ pb.ServerAddress, _ time.Time, _ chan struct{}) {} + a, b := pb.ServerAddress("filer-a:8888"), pb.ServerAddress("filer-b:8888") + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(a), IsAdd: true}, time.Now()) + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: true}, time.Now()) + ma.advancePeerWatermark(a, 100) + ma.advancePeerWatermark(b, 50) + stopB := ma.peerChans[b] + + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: false}, time.Now()) + ma.notePeerReachable(b, stopB) + + ma.peerWatermarksLock.Lock() + _, marked := ma.peerRemovedAtNs[b] + ma.peerRemovedAtNs[b] = time.Now().UnixNano() - int64(peerWatermarkRemovalGrace) - 1 + ma.peerWatermarksLock.Unlock() + if !marked { + t.Fatal("a stale subscription cleared the master's removal mark") + } + if got := ma.PeerLowWatermarkTsNs(); got != 100 { + t.Fatalf("the removed peer still pins past the grace: low=%d want 100", got) + } +} + +// A master removal forgets the peer's unreachable clock, and the removed +// peer's exiting loop cannot start a new one: departed addresses do not +// accumulate, and a re-added peer starts a fresh outage clock. +func TestMasterRemovalForgetsUnreachableClock(t *testing.T) { + ma := newTestAggregator() + ma.peerSubscribeLoop = func(_ *Filer, _, _ pb.ServerAddress, _ time.Time, _ chan struct{}) {} + b := pb.ServerAddress("filer-b:8888") + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: true}, time.Now()) + stopB := ma.peerChans[b] + t0 := time.Now().UnixNano() + ma.notePeerUnreachable(b, stopB, t0) + + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: false}, time.Now()) + ma.notePeerUnreachable(b, stopB, t0+1) + if n := len(ma.peerUnreachableSinceNs); n != 0 { + t.Fatalf("removed peer still has an unreachable clock: %d entries", n) + } + + ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(b), IsAdd: true}, time.Now()) + if ma.notePeerUnreachable(b, ma.peerChans[b], t0+int64(peerUnreachableRemovalAfter)) { + t.Fatal("a re-added peer inherited the previous outage clock") + } +} + +// A peer still unreachable once its removal mark has been forgotten is no +// longer tracked, so it is not reported as newly marked on every retry. +func TestLongUnreachablePeerIsReportedOnce(t *testing.T) { + ma := newTestAggregator() + b := pb.ServerAddress("filer-b:8888") + ma.initPeerWatermark(b) + stopB := make(chan struct{}) + ma.peerChans[b] = stopB + t0 := time.Now().UnixNano() + ma.notePeerUnreachable(b, stopB, t0) + if !ma.notePeerUnreachable(b, stopB, t0+int64(peerUnreachableRemovalAfter)) { + t.Fatal("a peer unreachable for the bound must be marked removed") + } + ma.peerWatermarksLock.Lock() + ma.peerRemovedAtNs[b] = time.Now().UnixNano() - int64(peerResumeRetention) - 1 + ma.dropExpiredRemovedPeersLocked() + ma.peerWatermarksLock.Unlock() + if ma.notePeerUnreachable(b, stopB, t0+2*int64(peerUnreachableRemovalAfter)) { + t.Fatal("an untracked peer was reported as newly marked") + } +}