Files
seaweedfs/weed/filer/meta_aggregator_test.go
T
0f0f3811bf filer: serve caught-up aggregated subscribers from memory, not a held disk pass (#12674)
* filer: serve caught-up aggregated subscribers from memory, not a held disk pass

An aggregated SubscribeMetadata stream that does not take metadata chunks
(filer.sync, filer.backup, filer.meta.backup, filer.meta.tail, the
filer.remote.* syncs, the S3 gateway's subscriptions, and mounts that
predate chunk support) re-ran the persisted-log pass whenever its cursor
moved: after every memory hold at the delivery watermark and on every
2 s re-probe. The pass holds at the peers' flush low-watermark. Peers
flush on their own one-minute schedules, so on a cluster that keeps
writing some peer has always flushed past a caught-up cursor while the
slowest has not: the pass held at its first entry, the loop came back
to it, and memory was never read. The stream got each change only once
the slowest peer's flush covered it, in bursts up to a flush interval
late, and every pass listed and re-read the newest log files.

When the aggregated ring was already filling at the cursor and evicted
nothing after it, memory holds everything the persisted logs could add,
since every peer's log reaches the ring through the aggregator. Skip the
pass and read memory, held at the delivery watermark as before. The
skipped pass proves nothing past the cursor, and the next-day scan for
an empty pass does not apply. An empty ring, a ring that starts past the
cursor, and a cursor memory just refused still take the pass. Chunk
subscribers already skip it for a caught-up cursor: their ref listing
stops below the flush hold.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: keep a new peer's pre-start history on the disk pass

The ring only receives a peer's events from the start of its subscription.
A peer followed for the first time after this filer started (a filer added
to the cluster, a restarted pod with a new address, or the late-arriving
peers bootstrap) starts at its notice time, so what it logged before then
is only in the persisted logs. A cursor below that start passed the ring
check and skipped the disk pass, losing those events for subscribers that
do not take metadata chunks.

Record the newest start of a peer subscription that had no resume point and
send cursors below it through the disk pass as before. Peers that rejoin with
a resume point (a restart under the same address) do not move it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2026-11-16 16:12:06 -08:00

176 lines
5.9 KiB
Go

package filer
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func TestOnPeerUpdateRepeatedAdd(t *testing.T) {
peer := pb.ServerAddress("127.0.0.1:1")
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
defer ma.Shutdown()
add := &master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}
ma.OnPeerUpdate(add, time.Now())
first, found := ma.peerChans[peer]
if !found {
t.Fatal("expecting a subscription after the first add")
}
ma.OnPeerUpdate(add, time.Now())
if ma.peerChans[peer] != first {
t.Fatal("expecting the same subscription after a repeated add")
}
select {
case <-first:
t.Fatal("expecting the subscription to stay alive after a repeated add")
default:
}
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
if _, found := ma.peerChans[peer]; found {
t.Fatal("expecting the subscription to be removed")
}
}
// Shutdown must close every peer's stopChan and wait for the loops, which is
// what lets each loop's deferred offset save run before the store closes.
func TestMetaAggregatorShutdownStopsPeerLoops(t *testing.T) {
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
stopChan := make(chan struct{})
peer := pb.ServerAddress("127.0.0.1:1")
ma.peerChans[peer] = stopChan
ma.peerWg.Add(1)
exited := make(chan struct{})
go func() {
defer ma.peerWg.Done()
<-stopChan
close(exited)
}()
done := make(chan struct{})
go func() {
ma.Shutdown()
close(done)
}()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("Shutdown did not return")
}
select {
case <-exited:
default:
t.Fatal("peer loop was not stopped")
}
}
// A peer removed and re-added must resume where its previous subscription
// was delivered through: subscribing from the update's arrival time drops
// whatever the peer logged while its stream was down.
func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
peer := pb.ServerAddress("127.0.0.1:1")
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
defer ma.Shutdown()
subscribeStarts := make(chan int64, 3)
ma.peerSubscribeLoop = func(_ *Filer, _ pb.ServerAddress, _ pb.ServerAddress, startFrom time.Time, stopChan chan struct{}) {
subscribeStarts <- startFrom.UnixNano()
<-stopChan
}
nextSubscribeStart := func() int64 {
select {
case tsNs := <-subscribeStarts:
return tsNs
case <-time.After(3 * time.Second):
t.Fatal("no peer subscription started")
return 0
}
}
add := &master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}
firstStart := time.Now()
ma.OnPeerUpdate(add, firstStart)
if got := nextSubscribeStart(); got != firstStart.UnixNano() {
t.Fatalf("expecting first subscribe at %d, got %d", firstStart.UnixNano(), got)
}
if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() {
t.Fatalf("expecting a first-seen peer to mark its start %d, got %d", firstStart.UnixNano(), got)
}
deliveredThrough := time.Now().Add(-time.Hour).UnixNano()
ma.advancePeerWatermark(peer, deliveredThrough)
if got := ma.peerResumeStartFrom(peer, time.Now()); got.UnixNano() != deliveredThrough {
t.Fatalf("expecting resume at %d, got %d", deliveredThrough, got.UnixNano())
}
// Removal and even the watermark-grace expiry keep the resume position.
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
ma.peerWatermarksLock.Lock()
ma.peerRemovedAtNs[peer] = time.Now().Add(-2 * peerWatermarkRemovalGrace).UnixNano()
ma.dropExpiredRemovedPeersLocked()
ma.peerWatermarksLock.Unlock()
ma.OnPeerUpdate(add, time.Now())
if got := nextSubscribeStart(); got != deliveredThrough {
t.Fatalf("expecting re-added peer to resume at %d, got %d", deliveredThrough, got)
}
if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() {
t.Fatalf("expecting a resumed peer to leave the new-peer start at %d, got %d", firstStart.UnixNano(), got)
}
// A peer gone past the retention is forgotten and starts afresh.
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
ma.peerWatermarksLock.Lock()
ma.peerRemovedAtNs[peer] = time.Now().Add(-2 * peerResumeRetention).UnixNano()
ma.dropExpiredRemovedPeersLocked()
_, retained := ma.peerResumeTsNs[peer]
ma.peerWatermarksLock.Unlock()
if retained {
t.Fatal("expecting the resume point to be forgotten past the retention")
}
freshStart := time.Now()
ma.OnPeerUpdate(add, freshStart)
if got := nextSubscribeStart(); got != freshStart.UnixNano() {
t.Fatalf("expecting forgotten peer to start at %d, got %d", freshStart.UnixNano(), got)
}
if got := ma.NewPeerStartTsNs(); got != freshStart.UnixNano() {
t.Fatalf("expecting a forgotten peer to mark its start %d, got %d", freshStart.UnixNano(), got)
}
// A first-seen peer, or a caller start already behind the recorded
// position, keeps the caller's start.
other := pb.ServerAddress("127.0.0.1:3")
start := time.Now().Add(-time.Minute)
if got := ma.peerResumeStartFrom(other, start); !got.Equal(start) {
t.Fatalf("expecting caller start %v for unseen peer, got %v", start, got)
}
early := time.Unix(0, deliveredThrough).Add(-time.Second)
if got := ma.peerResumeStartFrom(peer, early); !got.Equal(early) {
t.Fatalf("expecting caller start %v behind resume, got %v", early, got)
}
}
// After Shutdown begins, a peer-add update arriving on the still-connected
// master watch must not start a subscription: it would have no stop signal
// and could run past the store close.
func TestOnPeerUpdateAfterShutdownRejected(t *testing.T) {
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
ma.Shutdown()
peer := pb.ServerAddress("127.0.0.1:1")
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}, time.Now())
ma.peerChansLock.Lock()
_, found := ma.peerChans[peer]
ma.peerChansLock.Unlock()
if found {
t.Fatal("peer subscription started after shutdown")
}
}