diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index ac736946ed..d4244060e5 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -68,7 +68,10 @@ type MetaAggregator struct { // the grace, but a rejoining peer still needs it to resume. Forgotten // peerResumeRetention after the removal so addresses that never return, // such as restarted pods' IPs, do not accumulate. - peerResumeTsNs map[pb.ServerAddress]int64 + peerResumeTsNs map[pb.ServerAddress]int64 + // newPeerStartTsNs is the newest start of a peer subscription that had no + // resume point: the ring holds nothing that peer logged at or before it. + newPeerStartTsNs int64 peerWatermarksLock sync.Mutex peerWg sync.WaitGroup // peerSubscribeLoop is replaceable so a test can observe the subscribe @@ -245,9 +248,21 @@ func (ma *MetaAggregator) peerResumeStartFrom(peer pb.ServerAddress, startFrom t if prev := ma.peerResumeTsNs[peer]; prev > 0 && prev < startFrom.UnixNano() { return time.Unix(0, prev) } + if startTsNs := startFrom.UnixNano(); startTsNs > ma.newPeerStartTsNs { + ma.newPeerStartTsNs = startTsNs + } return startFrom } +// NewPeerStartTsNs returns the newest start of a peer subscription that had +// no resume point. What that peer logged at or before it is only on disk, so +// a cursor below it cannot be served from the ring alone. +func (ma *MetaAggregator) NewPeerStartTsNs() int64 { + ma.peerWatermarksLock.Lock() + defer ma.peerWatermarksLock.Unlock() + return ma.newPeerStartTsNs +} + // advancePeerFlushWatermark records the peer's reported flush watermark. // Monotonic, and only for tracked peers (see advancePeerWatermark). func (ma *MetaAggregator) advancePeerFlushWatermark(peer pb.ServerAddress, tsNs int64) { diff --git a/weed/filer/meta_aggregator_test.go b/weed/filer/meta_aggregator_test.go index 0655ed0037..2b7b7f7e5b 100644 --- a/weed/filer/meta_aggregator_test.go +++ b/weed/filer/meta_aggregator_test.go @@ -99,6 +99,9 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) { if got := nextSubscribeStart(); got != firstStart.UnixNano() { t.Fatalf("expecting first subscribe at %d, got %d", firstStart.UnixNano(), got) } + if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() { + t.Fatalf("expecting a first-seen peer to mark its start %d, got %d", firstStart.UnixNano(), got) + } deliveredThrough := time.Now().Add(-time.Hour).UnixNano() ma.advancePeerWatermark(peer, deliveredThrough) @@ -117,6 +120,9 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) { if got := nextSubscribeStart(); got != deliveredThrough { t.Fatalf("expecting re-added peer to resume at %d, got %d", deliveredThrough, got) } + if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() { + t.Fatalf("expecting a resumed peer to leave the new-peer start at %d, got %d", firstStart.UnixNano(), got) + } // A peer gone past the retention is forgotten and starts afresh. ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now()) @@ -133,6 +139,9 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) { if got := nextSubscribeStart(); got != freshStart.UnixNano() { t.Fatalf("expecting forgotten peer to start at %d, got %d", freshStart.UnixNano(), got) } + if got := ma.NewPeerStartTsNs(); got != freshStart.UnixNano() { + t.Fatalf("expecting a forgotten peer to mark its start %d, got %d", freshStart.UnixNano(), got) + } // A first-seen peer, or a caller start already behind the recorded // position, keeps the caller's start. diff --git a/weed/filer/meta_aggregator_testing.go b/weed/filer/meta_aggregator_testing.go index 1c805f5840..7c9de9a9a4 100644 --- a/weed/filer/meta_aggregator_testing.go +++ b/weed/filer/meta_aggregator_testing.go @@ -1,6 +1,8 @@ package filer import ( + "time" + "github.com/seaweedfs/seaweedfs/weed/pb" ) @@ -17,6 +19,13 @@ func (ma *MetaAggregator) TrackPeerForTesting(peer pb.ServerAddress) { ma.initPeerWatermark(peer) } +// StartPeerForTesting is TrackPeerForTesting for a peer first followed from +// startFrom, as OnPeerUpdate records it. Test support only. +func (ma *MetaAggregator) StartPeerForTesting(peer pb.ServerAddress, startFrom time.Time) { + ma.TrackPeerForTesting(peer) + ma.peerResumeStartFrom(peer, startFrom) +} + // ReportPeerWatermarksForTesting stands in for what a peer's stream reports: // its delivery watermark (events and idle heartbeats) and its flush // watermark. Test support only. diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index d3ba194aae..f96a6a1787 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -301,6 +301,19 @@ func memoryHoldsGap(currentTsNs, lastEvictedTsNs int64) bool { return currentTsNs >= lastEvictedTsNs } +// ringServesCursor reports whether the aggregated ring alone can serve a read +// from the cursor: it was already filling at the cursor, evicted nothing +// after it (lastEvictedTsNs in pre-bump timestamps, as the gap checks take it), +// and every peer was streaming into it by then (newPeerStartTsNs). An empty +// ring, or one that starts past the cursor, proves nothing about what only the +// persisted logs hold. +func ringServesCursor(cursorTsNs int64, earliest time.Time, lastEvictedTsNs, newPeerStartTsNs int64) bool { + if earliest.IsZero() || earliest.UnixNano() > cursorTsNs || cursorTsNs < newPeerStartTsNs { + return false + } + return memoryHoldsGap(cursorTsNs, lastEvictedTsNs) +} + // errHeldByPeerWatermark aborts a read at an entry beyond the hold point; the // caller rewinds to the last delivered entry, waits, and re-reads (the // re-listing is what picks up a late-landing log file). Holding is the normal @@ -835,6 +848,9 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, time.Since(lastDiskPassAt) >= aggDiskReprobeInterval diskAdvanced := false + // Set when the persisted logs were not read because memory holds + // everything after the cursor: nothing on disk was proven empty. + memoryServesCursor := false if shouldReadFromDisk { lastCheckedFlushTsNs = currentFlushTsNs lastCheckedFlushLowTsNs = diskPassFlushLowTsNs @@ -855,6 +871,27 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, } else { processedTsNs, isDone, readPersistedLogErr = 0, false, nil } + } else if cursorTsNs := lastReadTime.Time.UnixNano(); !errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) && + ringServesCursor(cursorTsNs, aggBuffer.GetEarliestTime(), aggBuffer.GetLastEvictedOriginalTsNs(), fs.filer.MetaAggregator.NewPeerStartTsNs()) { + // The ring holds everything after the cursor (every peer's log + // reaches it through this aggregator), so read it there, held at + // the delivery low-watermark. The persisted logs are complete only + // through the peers' flush low-watermark, and the peers that flushed + // after the slowest one already have files past it: a pass from a + // caught-up cursor holds on the first of them. Each cursor move + // (every memory hold, every re-probe) started such a pass, so on a + // cluster that keeps writing the stream stayed in the disk hold and + // got each event only once the slowest peer flushed it, up to a + // flush interval late. Chunk subscribers are served the same way: + // their ref listing stops below the flush hold. + // + // A cursor memory just refused still takes the pass; the gap checks + // below re-arm it from there. + if cursorTsNs < diskPassProvenTsNs { + diskPassProvenTsNs = cursorTsNs + } + memoryServesCursor = true + processedTsNs, isDone, readPersistedLogErr = 0, false, nil } else { processedTsNs, isDone, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(ctx, lastReadTime, req.UntilNs, diskEachLogEntryFn) } @@ -896,7 +933,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, if processedTsNs > diskAnchorTsNs { diskAnchorTsNs = processedTsNs } - } else if readInMemoryLogErr == nil { + } else if readInMemoryLogErr == nil && !memoryServesCursor { // Nothing on disk and memory never spoke: scan forward for the next // day that has logs. nextDayTs := util.GetNextDayTsNano(lastReadTime.Time.UnixNano()) diff --git a/weed/server/filer_grpc_server_sub_meta_gap_test.go b/weed/server/filer_grpc_server_sub_meta_gap_test.go index 9628719747..b6ec8b78ad 100644 --- a/weed/server/filer_grpc_server_sub_meta_gap_test.go +++ b/weed/server/filer_grpc_server_sub_meta_gap_test.go @@ -640,3 +640,35 @@ func TestRefNeedsReship(t *testing.T) { }) } } + +// TestRingServesCursor pins when an aggregated subscriber may skip the +// persisted logs: only when the ring was already filling at its cursor, +// evicted nothing after it, and no peer started streaming into it after it. +func TestRingServesCursor(t *testing.T) { + cursor := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC).UnixNano() + before := time.Unix(0, cursor-int64(time.Minute)) + cases := []struct { + name string + earliest time.Time + evictedTsNs int64 + peerStartTsNs int64 + want bool + }{ + {"empty ring", time.Time{}, 0, 0, false}, + {"ring starting past the cursor", time.Unix(0, cursor+1), 0, 0, false}, + {"ring starting at the cursor", time.Unix(0, cursor), 0, 0, true}, + {"ring holding the cursor", before, 0, 0, true}, + {"evicted through the cursor", before, cursor, 0, true}, + {"evicted past the cursor", before, cursor + 1, 0, false}, + {"new peer streaming from the cursor", before, 0, cursor, true}, + {"new peer streaming from past the cursor", before, 0, cursor + 1, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ringServesCursor(cursor, tc.earliest, tc.evictedTsNs, tc.peerStartTsNs); got != tc.want { + t.Fatalf("ringServesCursor(%d, %v, %d, %d) = %v, want %v", + cursor, tc.earliest, tc.evictedTsNs, tc.peerStartTsNs, got, tc.want) + } + }) + } +} diff --git a/weed/server/filer_subscribe_agg_test.go b/weed/server/filer_subscribe_agg_test.go index 9d7a422f28..bf23759e77 100644 --- a/weed/server/filer_subscribe_agg_test.go +++ b/weed/server/filer_subscribe_agg_test.go @@ -316,3 +316,65 @@ func TestSubscribeLoop_AggregatedDiskHoldIgnoresDeliveryProgress(t *testing.T) { reportPeersAt(ma, 0, onDisk) waitForEvents(t, r, []int64{onDisk}, 3*time.Second) } + +// TestSubscribeLoop_AggregatedRingServesPastDiskHold: peers flush on their own +// schedules, so the persisted logs hold files past the slowest peer's flush +// watermark, and a disk pass from a caught-up cursor holds on the first of +// them. Every cursor move used to start such a pass, so a subscriber that does +// not take metadata chunks stayed in that hold and got each change only once +// the slowest peer flushed it. The ring has everything after the cursor: read +// it there. +func TestSubscribeLoop_AggregatedRingServesPastDiskHold(t *testing.T) { + h := newSubscribeHarness(t) + // Far longer than this test runs: a disk hold cannot time out into the + // ring here. + prevRetry := unflushedGapRetryInterval + unflushedGapRetryInterval = 30 * time.Second + t.Cleanup(func() { unflushedGapRetryInterval = prevRetry }) + + now := time.Now() + onDisk := now.Add(-20 * time.Second).UnixNano() + h.append(onDisk) + h.f.LocalMetaLogBuffer.ForceFlush() + waitForFlushedFiles(t, h, onDisk) + + ma := h.startAggregator() + // The ring took the same entry, and was filling before the cursor. + h.appendAggregated(now.Add(-30 * time.Second).UnixNano()) + h.appendAggregated(onDisk) + live := now.Add(-10 * time.Second).UnixNano() + h.appendAggregated(live) + // Delivered through the newest entry, flushed only below the one on disk. + reportPeersAt(ma, live, onDisk-int64(time.Second)) + + r := h.subscribeAggregated(now.Add(-25 * time.Second).UnixNano()) + waitForEvents(t, r, []int64{onDisk, live}, 3*time.Second) +} + +// TestSubscribeLoop_AggregatedNewPeerHistoryReadFromDisk: a peer first +// followed after the cursor streams into the ring only from its start, so what +// it logged before then is on disk alone and the ring cannot serve the cursor. +func TestSubscribeLoop_AggregatedNewPeerHistoryReadFromDisk(t *testing.T) { + h := newSubscribeHarness(t) + prevRetry := unflushedGapRetryInterval + unflushedGapRetryInterval = 30 * time.Second + t.Cleanup(func() { unflushedGapRetryInterval = prevRetry }) + + now := time.Now() + onDisk := now.Add(-20 * time.Second).UnixNano() + h.append(onDisk) + h.f.LocalMetaLogBuffer.ForceFlush() + waitForFlushedFiles(t, h, onDisk) + + ma := h.startAggregator() + newPeer := pb.ServerAddress("new-peer:8888") + ma.StartPeerForTesting(newPeer, now.Add(-15*time.Second)) + h.appendAggregated(now.Add(-30 * time.Second).UnixNano()) + live := now.Add(-10 * time.Second).UnixNano() + h.appendAggregated(live) + reportPeersAt(ma, live, live) + ma.ReportPeerWatermarksForTesting(newPeer, live, live) + + r := h.subscribeAggregated(now.Add(-25 * time.Second).UnixNano()) + waitForEvents(t, r, []int64{onDisk, live}, 3*time.Second) +}