fix(filer.sync): keep sync_offset fresh through filtered-event markers (#9733)

On a read-only watched path the idle heartbeat keeps sync_offset fresh,
but a busy source filer still emits a MaxUnsyncedEvents marker after many
filtered events. The marker has a non-nil but empty EventNotification, so
the client routed it to the event path, where it advanced no real
watermark yet drove offsetFunc to republish the stale processed
watermark — regressing the gauge between heartbeats and spiking the
derived lag every time a filtered-event burst landed.

Route the empty marker through OnIdleHeartbeat like the idle heartbeat so
its fresh timestamp keeps the gauge current; it still advances the
in-stream resume cursor.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-05-28 23:29:59 -07:00
1 parent 5955972fe6
commit c9623007a2
4 files changed
+207 -5

No files matched your search

+28
View File
@@ -4,6 +4,7 @@ import (
"container/heap"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -582,3 +583,30 @@ func BenchmarkConflictCheck(b *testing.B) {
})
}
}
// TestMetadataProcessorEmptyMarkerKeepsWatermarkStale: the MaxUnsyncedEvents
// marker (empty EventNotification, fresh timestamp) is dropped by AddSyncJob and
// does NOT advance processedTsWatermark, so offsetFunc keeps publishing the stale
// offset. This is why the client must not drive sync_offset off the watermark
// for these markers.
func TestMetadataProcessorEmptyMarkerKeepsWatermarkStale(t *testing.T) {
const staleOffset = int64(1_000_000_000)
freshTs := staleOffset + int64(time.Hour) // a "now"-ish source timestamp
p := NewMetadataProcessor(func(*filer_pb.SubscribeMetadataResponse) error { return nil }, 4, staleOffset)
marker := &filer_pb.SubscribeMetadataResponse{
TsNs: freshTs,
EventNotification: &filer_pb.EventNotification{},
}
if !filer_pb.IsEmpty(marker) {
t.Fatal("marker should be IsEmpty")
}
p.AddSyncJob(marker)
if got := p.processedTsWatermark.Load(); got != staleOffset {
t.Fatalf("empty marker advanced watermark to %d; want it to stay stale at %d", got, staleOffset)
}
t.Logf("marker carried fresh ts %d but watermark stayed stale at %d", freshTs, staleOffset)
}