filer: metrics for metadata subscription delivery (lag, holds, ring fall-offs, claim and watermark ages) (#12773)

* filer: metrics for metadata subscription delivery (lag, holds, ring fall-offs, claim and watermark ages)

* filer: measure subscribe delivery lag after the real send using the mutation stamp, and record a watermark hold once per hold

* filer: skip empty progress markers when sampling subscribe delivery lag

* filer: assert subscribe delivery lag is sampled only after the stream send succeeds

* grafana: panels for the metadata subscription delivery metrics

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: record a watermark hold when the read passes the held entry, not when the next hold starts

* filer: use a monotonic clock for the subscribe hold duration

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 12:43:34 -08:00
1 parent 05a1463855
commit ed7324611c
8 files changed
+660 -21

No files matched your search

+69 -4
View File
@@ -116,21 +116,47 @@ type pipelinedSender struct {
sendCh chan *filer_pb.SubscribeMetadataResponse
errCh chan error
done chan struct{}
canBatch bool // true only if client set ClientSupportsBatching
canBatch bool // true only if client set ClientSupportsBatching
scope string // local or aggregated, for the delivery-lag metric
}
func newPipelinedSender(ctx context.Context, stream metadataStreamSender, bufSize int, clientSupportsBatching bool) *pipelinedSender {
func newPipelinedSender(ctx context.Context, stream metadataStreamSender, bufSize int, clientSupportsBatching bool, scope string) *pipelinedSender {
s := &pipelinedSender{
ctx: ctx,
sendCh: make(chan *filer_pb.SubscribeMetadataResponse, bufSize),
errCh: make(chan error, 1),
done: make(chan struct{}),
canBatch: clientSupportsBatching,
scope: scope,
}
go s.sendLoop(stream)
return s
}
// observeSubscribeDeliveryLag records, for a real metadata event that was just
// sent, the seconds from its original mutation stamp until delivery. It skips
// control messages (heartbeats, flush reports) and log-file-ref envelopes, which
// carry no mutation. MutationTsNs is the original stamp (TsNs is the stream
// position, which the buffer can move forward past a late event); payloads older
// than the MutationTsNs field still carry the stamp in TsNs.
func observeSubscribeDeliveryLag(scope string, msg *filer_pb.SubscribeMetadataResponse) {
if msg == nil || msg.EventNotification == nil || len(msg.LogFileRefs) > 0 || filer_pb.IsEmpty(msg) {
// Skip control messages, log-file-ref envelopes, and the empty progress
// markers sent after MaxUnsyncedEvents filtered entries: none is a real
// delivery, and the marker carries the filtered entry's position (TsNs),
// not a mutation.
return
}
stampNs := msg.MutationTsNs
if stampNs == 0 {
stampNs = msg.TsNs
}
if stampNs <= 0 {
return
}
stats.FilerSubscribeDeliveryLagSeconds.WithLabelValues(scope).Observe(float64(time.Now().UnixNano()-stampNs) / 1e9)
}
func (s *pipelinedSender) sendLoop(stream metadataStreamSender) {
defer close(s.done)
// No Send may start after the subscription context ends: the handler is
@@ -146,6 +172,14 @@ func (s *pipelinedSender) sendLoop(stream metadataStreamSender) {
}
return false
}
// Record delivery lag only after the event actually reached the client
// over the stream: Send() above is the real gRPC send (this loop runs
// off the queue), and a batched envelope carries its extra events in
// Events, so each one is sampled here too.
observeSubscribeDeliveryLag(s.scope, msg)
for _, ev := range msg.Events {
observeSubscribeDeliveryLag(s.scope, ev)
}
return true
}
for msg := range s.sendCh {
@@ -695,7 +729,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
// had already arrived and been delivered).
diskAnchorTsNs := req.SinceNs
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching)
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching, "aggregated")
defer sender.Close()
// Register for instant notification when new data arrives in the aggregated log buffer.
@@ -728,6 +762,23 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
// diskHeldAtTsNs is the same marker for the disk pass alone: a pending
// disk hold keeps the pass re-reading until the entry is served.
var heldAtTsNs, diskHeldAtTsNs int64
// A watermark hold on one entry spans many waitHeld retries; waitHeld starts
// the timing and the read records the whole hold once -- when it passes the
// held entry, or the subscription ends -- not per retry, which would turn a
// long hold into many short samples. holdScope is memory or disk.
var holdEntryTsNs int64
var holdStart time.Time
var holdScope string
flushHold := func() {
if holdEntryTsNs != 0 {
// time.Since uses the monotonic clock, so a wall-clock step back
// mid-hold cannot produce a negative duration.
stats.FilerSubscribeHoldSeconds.WithLabelValues(holdScope).Observe(time.Since(holdStart).Seconds())
holdEntryTsNs = 0
}
}
defer flushHold()
// Each read path holds at its own watermark: persisted logs are complete
// only up to every peer's flush watermark, the ring only up to every
// peer's delivery watermark.
@@ -744,6 +795,12 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
heldAtTsNs = logEntry.TsNs
return false, errHeldByPeerWatermark
}
if holdEntryTsNs != 0 && logEntry.TsNs >= holdEntryTsNs {
// The read passed the held entry: the hold is over, so record
// its duration now. >= also covers a cursor the gap machinery
// moved past the held entry.
flushHold()
}
lastSeenTsNs = logEntry.TsNs
deliveredUpToTsNs = logEntry.TsNs
return baseEachLogEntryFn(logEntry)
@@ -778,6 +835,12 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
// False: context ended.
waitHeld := func(scope string, watermarkChan <-chan struct{}) bool {
stats.FilerSubscribeWatermarkHolds.WithLabelValues(scope).Inc()
if heldAtTsNs != holdEntryTsNs {
// A different entry is now held: the previous hold (if any) was
// released, so record it and start timing this one.
flushHold()
holdEntryTsNs, holdStart, holdScope = heldAtTsNs, time.Now(), scope
}
glog.V(3).Infof("held at %v (deliveredUpTo %v, flushLow %v, deliveryLow %v) for %v",
time.Unix(0, heldAtTsNs), time.Unix(0, deliveredUpToTsNs),
time.Unix(0, fs.filer.MetaAggregator.PeerLowFlushWatermarkTsNs()),
@@ -1041,6 +1104,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
continue
}
if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) {
stats.FilerSubscribeResumeFromDisk.WithLabelValues("aggregated").Inc()
// Fell off the ring: resume the disk pass from the anchor, not
// the (possibly bumped) cursor - redelivery is within the
// at-least-once contract, skipping is not. For an anchored
@@ -1178,7 +1242,7 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
lastReadTime := log_buffer.NewMessagePosition(req.SinceNs, gapResumeCursorOffset)
glog.V(0).Infof(" + %v local subscribe %s from %+v clientId:%d", clientName, req.PathPrefix, lastReadTime, req.ClientId)
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching)
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching, "local")
defer sender.Close()
// Bounded gap waits use the buffer's subscriber notification plus a retry
@@ -1349,6 +1413,7 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
return errAggregationUpgrade
}
if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) {
stats.FilerSubscribeResumeFromDisk.WithLabelValues("local").Inc()
// Fell behind the ring: back to the disk pass (it re-runs when
// the flush or the cursor moved), and from there to the gap
// resolution above if the disk has nothing either.
@@ -133,7 +133,7 @@ func TestPipelinedSenderThroughput(t *testing.T) {
var batchedRate float64
t.Run("pipelined_batched_send", func(t *testing.T) {
stream := &slowStream{sendDelay: sendDelay}
sender := newPipelinedSender(context.Background(), stream, 1024, true)
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
start := time.Now()
for _, file := range files {
@@ -220,7 +220,7 @@ func TestBatchingAdaptive(t *testing.T) {
t.Run("old_events_are_batched", func(t *testing.T) {
stream := &slowStream{sendDelay: 10 * time.Microsecond}
sender := newPipelinedSender(context.Background(), stream, 1024, true)
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
// Push all events at once (no read delay) so the sender can batch aggressively
for _, ev := range makeOldEvents(numEvents) {
@@ -240,7 +240,7 @@ func TestBatchingAdaptive(t *testing.T) {
t.Run("recent_events_sent_individually", func(t *testing.T) {
stream := &slowStream{sendDelay: 10 * time.Microsecond}
sender := newPipelinedSender(context.Background(), stream, 1024, true)
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
for _, ev := range makeRecentEvents(numEvents) {
sender.Send(ev)
@@ -284,7 +284,7 @@ func TestPipelinedSenderErrorPropagation(t *testing.T) {
t.Run("send_returns_error", func(t *testing.T) {
// Stream fails after 5 successful sends
stream := &errorStreamImpl{failAfter: 5, err: sendErr}
sender := newPipelinedSender(context.Background(), stream, 4, true)
sender := newPipelinedSender(context.Background(), stream, 4, true, "local")
var lastErr error
for i := 0; i < 100; i++ {
@@ -306,7 +306,7 @@ func TestPipelinedSenderErrorPropagation(t *testing.T) {
// since Send may have already returned before the sender goroutine
// processes the message.
stream := &errorStreamImpl{failAfter: 0, err: sendErr}
sender := newPipelinedSender(context.Background(), stream, 1024, true)
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
ev := makeOldEvents(1)[0]
sender.Send(ev)
@@ -355,7 +355,7 @@ func TestPipelinedSingleVsParallelStreams(t *testing.T) {
// simulatePipeline: read files with I/O delay, push events, send via pipelinedSender
simulatePipeline := func(files []logFile) (eventsSent, sends int64, elapsed time.Duration, err error) {
stream := &slowStream{sendDelay: sendDelay}
sender := newPipelinedSender(context.Background(), stream, 1024, true)
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
start := time.Now()
outer:
@@ -57,7 +57,7 @@ func (s *gatedRecordingStream) Send(msg *filer_pb.SubscribeMetadataResponse) err
// reads past it.
func TestPipelinedSenderControlMessagesNeverNested(t *testing.T) {
stream := &gatedRecordingStream{gate: make(chan struct{})}
sender := newPipelinedSender(context.Background(), stream, 16, true)
sender := newPipelinedSender(context.Background(), stream, 16, true, "local")
oldTs := time.Now().Add(-time.Hour).UnixNano()
if err := sender.Send(makeEvent("/d", "e1", oldTs)); err != nil {
+94 -3
View File
@@ -9,7 +9,10 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
type recordingStream struct {
@@ -49,7 +52,7 @@ func (s *recordingStream) snapshot() []*filer_pb.SubscribeMetadataResponse {
// event. Everything must arrive, in order, whatever the batcher does.
func TestPipelinedSenderRefsNeverBatched(t *testing.T) {
stream := &recordingStream{slow: 2 * time.Millisecond} // let the queue back up so batching engages
sender := newPipelinedSender(context.Background(), stream, 64, true)
sender := newPipelinedSender(context.Background(), stream, 64, true, "local")
oldTs := time.Now().Add(-time.Hour).UnixNano() // far behind: the batch heuristic fires
var wantOrder []string
@@ -129,7 +132,7 @@ func TestPipelinedSenderUnblocksOnCancel(t *testing.T) {
stream := &blockedStream{release: make(chan struct{})}
defer close(stream.release)
ctx, cancel := context.WithCancel(context.Background())
sender := newPipelinedSender(ctx, stream, 1, false)
sender := newPipelinedSender(ctx, stream, 1, false, "local")
msg := &filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}}
if err := sender.Send(msg); err != nil { // consumed by the blocked sendLoop
@@ -163,7 +166,7 @@ func TestPipelinedSenderUnblocksOnCancel(t *testing.T) {
func TestPipelinedSenderStopsSendingOnCancel(t *testing.T) {
stream := &blockedStream{release: make(chan struct{}), entered: make(chan struct{}, 1)}
ctx, cancel := context.WithCancel(context.Background())
sender := newPipelinedSender(ctx, stream, 4, false)
sender := newPipelinedSender(ctx, stream, 4, false, "local")
msg := &filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}}
if err := sender.Send(msg); err != nil {
@@ -191,3 +194,91 @@ func TestPipelinedSenderStopsSendingOnCancel(t *testing.T) {
t.Fatalf("stream.Send ran %d times, want 1; a new send started after cancel", n)
}
}
// deliveryLagSnapshot returns the delivery-lag histogram's sample count and sum
// for a scope, so a test can assert what one send added.
func deliveryLagSnapshot(t *testing.T, scope string) (uint64, float64) {
t.Helper()
var m dto.Metric
if err := stats.FilerSubscribeDeliveryLagSeconds.WithLabelValues(scope).(prometheus.Metric).Write(&m); err != nil {
t.Fatalf("read delivery-lag metric: %v", err)
}
return m.GetHistogram().GetSampleCount(), m.GetHistogram().GetSampleSum()
}
// gatedStream blocks Send until release is closed, signalling on entered the
// first time Send is called, so a test can inspect state while a send is in
// flight (i.e. before it has succeeded).
type gatedStream struct {
recordingStream
entered chan struct{}
release chan struct{}
}
func (s *gatedStream) Send(m *filer_pb.SubscribeMetadataResponse) error {
select {
case s.entered <- struct{}{}:
default:
}
<-s.release
return s.recordingStream.Send(m)
}
// TestPipelinedSenderObservesDeliveryLagAfterSend proves delivery lag is sampled
// after the real stream send (through the queue + Close), measured from the
// event's mutation stamp (not the moved stream position), and that control
// messages and empty progress markers are not sampled.
func TestPipelinedSenderObservesDeliveryLagAfterSend(t *testing.T) {
const scope = "delivery-lag-test"
beforeCount, beforeSum := deliveryLagSnapshot(t, scope)
stream := &gatedStream{entered: make(chan struct{}, 1), release: make(chan struct{})}
sender := newPipelinedSender(context.Background(), stream, 64, false, scope)
mutationNs := time.Now().Add(-5 * time.Second).UnixNano()
// A real event whose stream position (TsNs) has moved well past its original
// mutation stamp; lag must be measured from the mutation stamp.
event := &filer_pb.SubscribeMetadataResponse{
TsNs: time.Now().UnixNano(),
MutationTsNs: mutationNs,
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "e"}},
}
// A control message (heartbeat): no EventNotification -> must not be sampled.
control := &filer_pb.SubscribeMetadataResponse{TsNs: time.Now().UnixNano()}
// A progress marker (sent after MaxUnsyncedEvents filtered entries): a
// non-nil but empty EventNotification carrying the filtered entry's TsNs ->
// must not be sampled either.
marker := &filer_pb.SubscribeMetadataResponse{
TsNs: time.Now().UnixNano(),
EventNotification: &filer_pb.EventNotification{},
}
if err := sender.Send(event); err != nil {
t.Fatalf("send event: %v", err)
}
// Wait until the send loop is blocked inside stream.Send for the event, then
// confirm nothing was sampled yet: the observation must come after the send
// succeeds, not when the event entered the queue.
<-stream.entered
if midCount, _ := deliveryLagSnapshot(t, scope); midCount != beforeCount {
t.Fatalf("delivery-lag sampled %d while the send was still blocked; it must observe only after stream.Send succeeds", midCount-beforeCount)
}
close(stream.release)
if err := sender.Send(control); err != nil {
t.Fatalf("send control: %v", err)
}
if err := sender.Send(marker); err != nil {
t.Fatalf("send marker: %v", err)
}
if err := sender.Close(); err != nil {
t.Fatalf("close: %v", err)
}
afterCount, afterSum := deliveryLagSnapshot(t, scope)
if afterCount != beforeCount+1 {
t.Fatalf("delivery-lag samples added = %d, want 1 (only the real event; control and progress markers skipped)", afterCount-beforeCount)
}
if lag := afterSum - beforeSum; lag < 4 || lag > 60 {
t.Fatalf("delivery lag = %.2fs, want ~5s from the mutation stamp (not ~0 from the moved TsNs)", lag)
}
}