filer: serve caught-up aggregated subscribers from memory, not a held disk pass (#12674)

* filer: serve caught-up aggregated subscribers from memory, not a held disk pass

An aggregated SubscribeMetadata stream that does not take metadata chunks
(filer.sync, filer.backup, filer.meta.backup, filer.meta.tail, the
filer.remote.* syncs, the S3 gateway's subscriptions, and mounts that
predate chunk support) re-ran the persisted-log pass whenever its cursor
moved: after every memory hold at the delivery watermark and on every
2 s re-probe. The pass holds at the peers' flush low-watermark. Peers
flush on their own one-minute schedules, so on a cluster that keeps
writing some peer has always flushed past a caught-up cursor while the
slowest has not: the pass held at its first entry, the loop came back
to it, and memory was never read. The stream got each change only once
the slowest peer's flush covered it, in bursts up to a flush interval
late, and every pass listed and re-read the newest log files.

When the aggregated ring was already filling at the cursor and evicted
nothing after it, memory holds everything the persisted logs could add,
since every peer's log reaches the ring through the aggregator. Skip the
pass and read memory, held at the delivery watermark as before. The
skipped pass proves nothing past the cursor, and the next-day scan for
an empty pass does not apply. An empty ring, a ring that starts past the
cursor, and a cursor memory just refused still take the pass. Chunk
subscribers already skip it for a caught-up cursor: their ref listing
stops below the flush hold.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: keep a new peer's pre-start history on the disk pass

The ring only receives a peer's events from the start of its subscription.
A peer followed for the first time after this filer started (a filer added
to the cluster, a restarted pod with a new address, or the late-arriving
peers bootstrap) starts at its notice time, so what it logged before then
is only in the persisted logs. A cursor below that start passed the ring
check and skipped the disk pass, losing those events for subscribers that
do not take metadata chunks.

Record the newest start of a peer subscription that had no resume point and
send cursors below it through the disk pass as before. Peers that rejoin with
a resume point (a restart under the same address) do not move it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-16 16:12:06 -08:00
1 parent 53b234558e
commit 0f0f3811bf
6 files changed
+166 -2

No files matched your search

+16 -1
View File
@@ -68,7 +68,10 @@ type MetaAggregator struct {
// the grace, but a rejoining peer still needs it to resume. Forgotten
// peerResumeRetention after the removal so addresses that never return,
// such as restarted pods' IPs, do not accumulate.
peerResumeTsNs map[pb.ServerAddress]int64
peerResumeTsNs map[pb.ServerAddress]int64
// newPeerStartTsNs is the newest start of a peer subscription that had no
// resume point: the ring holds nothing that peer logged at or before it.
newPeerStartTsNs int64
peerWatermarksLock sync.Mutex
peerWg sync.WaitGroup
// peerSubscribeLoop is replaceable so a test can observe the subscribe
@@ -245,9 +248,21 @@ func (ma *MetaAggregator) peerResumeStartFrom(peer pb.ServerAddress, startFrom t
if prev := ma.peerResumeTsNs[peer]; prev > 0 && prev < startFrom.UnixNano() {
return time.Unix(0, prev)
}
if startTsNs := startFrom.UnixNano(); startTsNs > ma.newPeerStartTsNs {
ma.newPeerStartTsNs = startTsNs
}
return startFrom
}
// NewPeerStartTsNs returns the newest start of a peer subscription that had
// no resume point. What that peer logged at or before it is only on disk, so
// a cursor below it cannot be served from the ring alone.
func (ma *MetaAggregator) NewPeerStartTsNs() int64 {
ma.peerWatermarksLock.Lock()
defer ma.peerWatermarksLock.Unlock()
return ma.newPeerStartTsNs
}
// advancePeerFlushWatermark records the peer's reported flush watermark.
// Monotonic, and only for tracked peers (see advancePeerWatermark).
func (ma *MetaAggregator) advancePeerFlushWatermark(peer pb.ServerAddress, tsNs int64) {
+9
View File
@@ -99,6 +99,9 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
if got := nextSubscribeStart(); got != firstStart.UnixNano() {
t.Fatalf("expecting first subscribe at %d, got %d", firstStart.UnixNano(), got)
}
if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() {
t.Fatalf("expecting a first-seen peer to mark its start %d, got %d", firstStart.UnixNano(), got)
}
deliveredThrough := time.Now().Add(-time.Hour).UnixNano()
ma.advancePeerWatermark(peer, deliveredThrough)
@@ -117,6 +120,9 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
if got := nextSubscribeStart(); got != deliveredThrough {
t.Fatalf("expecting re-added peer to resume at %d, got %d", deliveredThrough, got)
}
if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() {
t.Fatalf("expecting a resumed peer to leave the new-peer start at %d, got %d", firstStart.UnixNano(), got)
}
// A peer gone past the retention is forgotten and starts afresh.
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
@@ -133,6 +139,9 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
if got := nextSubscribeStart(); got != freshStart.UnixNano() {
t.Fatalf("expecting forgotten peer to start at %d, got %d", freshStart.UnixNano(), got)
}
if got := ma.NewPeerStartTsNs(); got != freshStart.UnixNano() {
t.Fatalf("expecting a forgotten peer to mark its start %d, got %d", freshStart.UnixNano(), got)
}
// A first-seen peer, or a caller start already behind the recorded
// position, keeps the caller's start.
+9
View File
@@ -1,6 +1,8 @@
package filer
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
)
@@ -17,6 +19,13 @@ func (ma *MetaAggregator) TrackPeerForTesting(peer pb.ServerAddress) {
ma.initPeerWatermark(peer)
}
// StartPeerForTesting is TrackPeerForTesting for a peer first followed from
// startFrom, as OnPeerUpdate records it. Test support only.
func (ma *MetaAggregator) StartPeerForTesting(peer pb.ServerAddress, startFrom time.Time) {
ma.TrackPeerForTesting(peer)
ma.peerResumeStartFrom(peer, startFrom)
}
// ReportPeerWatermarksForTesting stands in for what a peer's stream reports:
// its delivery watermark (events and idle heartbeats) and its flush
// watermark. Test support only.
+38 -1
View File
@@ -301,6 +301,19 @@ func memoryHoldsGap(currentTsNs, lastEvictedTsNs int64) bool {
return currentTsNs >= lastEvictedTsNs
}
// ringServesCursor reports whether the aggregated ring alone can serve a read
// from the cursor: it was already filling at the cursor, evicted nothing
// after it (lastEvictedTsNs in pre-bump timestamps, as the gap checks take it),
// and every peer was streaming into it by then (newPeerStartTsNs). An empty
// ring, or one that starts past the cursor, proves nothing about what only the
// persisted logs hold.
func ringServesCursor(cursorTsNs int64, earliest time.Time, lastEvictedTsNs, newPeerStartTsNs int64) bool {
if earliest.IsZero() || earliest.UnixNano() > cursorTsNs || cursorTsNs < newPeerStartTsNs {
return false
}
return memoryHoldsGap(cursorTsNs, lastEvictedTsNs)
}
// errHeldByPeerWatermark aborts a read at an entry beyond the hold point; the
// caller rewinds to the last delivered entry, waits, and re-reads (the
// re-listing is what picks up a late-landing log file). Holding is the normal
@@ -835,6 +848,9 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
time.Since(lastDiskPassAt) >= aggDiskReprobeInterval
diskAdvanced := false
// Set when the persisted logs were not read because memory holds
// everything after the cursor: nothing on disk was proven empty.
memoryServesCursor := false
if shouldReadFromDisk {
lastCheckedFlushTsNs = currentFlushTsNs
lastCheckedFlushLowTsNs = diskPassFlushLowTsNs
@@ -855,6 +871,27 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
} else {
processedTsNs, isDone, readPersistedLogErr = 0, false, nil
}
} else if cursorTsNs := lastReadTime.Time.UnixNano(); !errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) &&
ringServesCursor(cursorTsNs, aggBuffer.GetEarliestTime(), aggBuffer.GetLastEvictedOriginalTsNs(), fs.filer.MetaAggregator.NewPeerStartTsNs()) {
// The ring holds everything after the cursor (every peer's log
// reaches it through this aggregator), so read it there, held at
// the delivery low-watermark. The persisted logs are complete only
// through the peers' flush low-watermark, and the peers that flushed
// after the slowest one already have files past it: a pass from a
// caught-up cursor holds on the first of them. Each cursor move
// (every memory hold, every re-probe) started such a pass, so on a
// cluster that keeps writing the stream stayed in the disk hold and
// got each event only once the slowest peer flushed it, up to a
// flush interval late. Chunk subscribers are served the same way:
// their ref listing stops below the flush hold.
//
// A cursor memory just refused still takes the pass; the gap checks
// below re-arm it from there.
if cursorTsNs < diskPassProvenTsNs {
diskPassProvenTsNs = cursorTsNs
}
memoryServesCursor = true
processedTsNs, isDone, readPersistedLogErr = 0, false, nil
} else {
processedTsNs, isDone, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(ctx, lastReadTime, req.UntilNs, diskEachLogEntryFn)
}
@@ -896,7 +933,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
if processedTsNs > diskAnchorTsNs {
diskAnchorTsNs = processedTsNs
}
} else if readInMemoryLogErr == nil {
} else if readInMemoryLogErr == nil && !memoryServesCursor {
// Nothing on disk and memory never spoke: scan forward for the next
// day that has logs.
nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano())
@@ -640,3 +640,35 @@ func TestRefNeedsReship(t *testing.T) {
})
}
}
// TestRingServesCursor pins when an aggregated subscriber may skip the
// persisted logs: only when the ring was already filling at its cursor,
// evicted nothing after it, and no peer started streaming into it after it.
func TestRingServesCursor(t *testing.T) {
cursor := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC).UnixNano()
before := time.Unix(0, cursor-int64(time.Minute))
cases := []struct {
name string
earliest time.Time
evictedTsNs int64
peerStartTsNs int64
want bool
}{
{"empty ring", time.Time{}, 0, 0, false},
{"ring starting past the cursor", time.Unix(0, cursor+1), 0, 0, false},
{"ring starting at the cursor", time.Unix(0, cursor), 0, 0, true},
{"ring holding the cursor", before, 0, 0, true},
{"evicted through the cursor", before, cursor, 0, true},
{"evicted past the cursor", before, cursor + 1, 0, false},
{"new peer streaming from the cursor", before, 0, cursor, true},
{"new peer streaming from past the cursor", before, 0, cursor + 1, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := ringServesCursor(cursor, tc.earliest, tc.evictedTsNs, tc.peerStartTsNs); got != tc.want {
t.Fatalf("ringServesCursor(%d, %v, %d, %d) = %v, want %v",
cursor, tc.earliest, tc.evictedTsNs, tc.peerStartTsNs, got, tc.want)
}
})
}
}
+62
View File
@@ -316,3 +316,65 @@ func TestSubscribeLoop_AggregatedDiskHoldIgnoresDeliveryProgress(t *testing.T) {
reportPeersAt(ma, 0, onDisk)
waitForEvents(t, r, []int64{onDisk}, 3*time.Second)
}
// TestSubscribeLoop_AggregatedRingServesPastDiskHold: peers flush on their own
// schedules, so the persisted logs hold files past the slowest peer's flush
// watermark, and a disk pass from a caught-up cursor holds on the first of
// them. Every cursor move used to start such a pass, so a subscriber that does
// not take metadata chunks stayed in that hold and got each change only once
// the slowest peer flushed it. The ring has everything after the cursor: read
// it there.
func TestSubscribeLoop_AggregatedRingServesPastDiskHold(t *testing.T) {
h := newSubscribeHarness(t)
// Far longer than this test runs: a disk hold cannot time out into the
// ring here.
prevRetry := unflushedGapRetryInterval
unflushedGapRetryInterval = 30 * time.Second
t.Cleanup(func() { unflushedGapRetryInterval = prevRetry })
now := time.Now()
onDisk := now.Add(-20 * time.Second).UnixNano()
h.append(onDisk)
h.f.LocalMetaLogBuffer.ForceFlush()
waitForFlushedFiles(t, h, onDisk)
ma := h.startAggregator()
// The ring took the same entry, and was filling before the cursor.
h.appendAggregated(now.Add(-30 * time.Second).UnixNano())
h.appendAggregated(onDisk)
live := now.Add(-10 * time.Second).UnixNano()
h.appendAggregated(live)
// Delivered through the newest entry, flushed only below the one on disk.
reportPeersAt(ma, live, onDisk-int64(time.Second))
r := h.subscribeAggregated(now.Add(-25 * time.Second).UnixNano())
waitForEvents(t, r, []int64{onDisk, live}, 3*time.Second)
}
// TestSubscribeLoop_AggregatedNewPeerHistoryReadFromDisk: a peer first
// followed after the cursor streams into the ring only from its start, so what
// it logged before then is on disk alone and the ring cannot serve the cursor.
func TestSubscribeLoop_AggregatedNewPeerHistoryReadFromDisk(t *testing.T) {
h := newSubscribeHarness(t)
prevRetry := unflushedGapRetryInterval
unflushedGapRetryInterval = 30 * time.Second
t.Cleanup(func() { unflushedGapRetryInterval = prevRetry })
now := time.Now()
onDisk := now.Add(-20 * time.Second).UnixNano()
h.append(onDisk)
h.f.LocalMetaLogBuffer.ForceFlush()
waitForFlushedFiles(t, h, onDisk)
ma := h.startAggregator()
newPeer := pb.ServerAddress("new-peer:8888")
ma.StartPeerForTesting(newPeer, now.Add(-15*time.Second))
h.appendAggregated(now.Add(-30 * time.Second).UnixNano())
live := now.Add(-10 * time.Second).UnixNano()
h.appendAggregated(live)
reportPeersAt(ma, live, live)
ma.ReportPeerWatermarksForTesting(newPeer, live, live)
r := h.subscribeAggregated(now.Add(-25 * time.Second).UnixNano())
waitForEvents(t, r, []int64{onDisk, live}, 3*time.Second)
}