From ed7324611cc813eb75d50b875eef49cea38e4e27 Mon Sep 17 00:00:00 2001 From: Carlos Leyva <66577002+carlos-leyva-guerrero@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:43:34 +0100 Subject: [PATCH] filer: metrics for metadata subscription delivery (lag, holds, ring fall-offs, claim and watermark ages) (#12773) * filer: metrics for metadata subscription delivery (lag, holds, ring fall-offs, claim and watermark ages) * filer: measure subscribe delivery lag after the real send using the mutation stamp, and record a watermark hold once per hold * filer: skip empty progress markers when sampling subscribe delivery lag * filer: assert subscribe delivery lag is sampled only after the stream send succeeds * grafana: panels for the metadata subscription delivery metrics Co-Authored-By: Claude Opus 5.5 * filer: record a watermark hold when the read passes the held entry, not when the next hold starts * filer: use a monotonic clock for the subscribe hold duration --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- other/metrics/grafana_seaweedfs.json | 425 +++++++++++++++++- weed/filer/filer_notify.go | 7 +- weed/filer/meta_aggregator.go | 15 +- weed/server/filer_grpc_server_sub_meta.go | 73 ++- .../server/filer_grpc_server_sub_meta_test.go | 12 +- ...ler_grpc_server_sub_meta_watermark_test.go | 2 +- weed/server/filer_pipelined_sender_test.go | 97 +++- weed/stats/metrics.go | 50 +++ 8 files changed, 660 insertions(+), 21 deletions(-) diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index 21cc57ac89..c267483820 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -6736,6 +6736,423 @@ ], "pluginVersion": "10.3.1" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Metadata Subscribe Delivery Lag", + "description": "Seconds from a metadata event's mutation until the filer sends it to a subscriber, by stream scope", + "type": "timeseries", + "id": 263, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))", + "range": true, + "refId": "A", + "legendFormat": "p99 {{scope}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))", + "range": true, + "refId": "B", + "legendFormat": "p50 {{scope}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Metadata Watermark Hold Duration", + "description": "How long an aggregated read stayed held at one entry until the peers' watermark released it (memory: delivery watermark, disk: flush watermark)", + "type": "timeseries", + "id": 264, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))", + "range": true, + "refId": "A", + "legendFormat": "p99 {{scope}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))", + "range": true, + "refId": "B", + "legendFormat": "p50 {{scope}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Metadata Delivery Claim / Peer Low-Watermark Age", + "description": "Seconds the filer's delivery claim and the peers' delivery and flush low-watermarks trail the clock; -1 while a tracked peer has not signalled", + "type": "timeseries", + "id": 265, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (instance) (SeaweedFS_filer_meta_delivery_claim_age_seconds{cluster=~\"$cluster\"})", + "range": true, + "refId": "A", + "legendFormat": "claim {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (instance, kind) (SeaweedFS_filer_meta_peer_low_watermark_age_seconds{cluster=~\"$cluster\"})", + "range": true, + "refId": "B", + "legendFormat": "peer low {{kind}} {{instance}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Metadata Subscribers Resumed from Disk", + "description": "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log", + "type": "timeseries", + "id": 266, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (instance, scope) (rate(SeaweedFS_filer_subscribe_resume_from_disk{cluster=~\"$cluster\"}[$__rate_interval]))", + "range": true, + "refId": "A", + "legendFormat": "{{scope}} {{instance}}" + } + ], + "pluginVersion": "10.3.1" + }, { "datasource": { "type": "prometheus", @@ -6749,7 +7166,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 87 }, "fieldConfig": { "defaults": { @@ -6845,7 +7262,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 87 }, "fieldConfig": { "defaults": { @@ -6941,7 +7358,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 95 }, "fieldConfig": { "defaults": { @@ -7037,7 +7454,7 @@ "h": 8, "w": 12, "x": 12, - "y": 79 + "y": 95 }, "fieldConfig": { "defaults": { diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index baf646537d..7b7de3c2f8 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -19,6 +19,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/notification" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) @@ -212,7 +213,11 @@ func (f *Filer) LocalFlushedThroughTsNs(nowNs int64) int64 { // write certain to be visible yet, and a peer aggregator turns the claim into // its delivery low-watermark. func (f *Filer) LocalDeliveredThroughTsNs(nowNs int64) int64 { - return f.metaLogInflight.claimThrough(nowNs) + claim := f.metaLogInflight.claimThrough(nowNs) + // export how far this filer's delivery claim trails the clock (peer + // aggregators turn the claim into their delivery low-watermark) + stats.FilerMetaDeliveryClaimAgeSeconds.Set(float64(nowNs-claim) / 1e9) + return claim } // StampMetaLogInflightForTest and DoneMetaLogInflightForTest let tests in diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index 4494bdf2ac..5e7739776d 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -332,7 +332,7 @@ func (ma *MetaAggregator) PeerLowFlushWatermarkTsNs() int64 { ma.peerWatermarksLock.Lock() defer ma.peerWatermarksLock.Unlock() ma.dropExpiredRemovedPeersLocked() - return lowWatermarkOf(ma.peerFlushWatermarks) + return observeLowWatermark("flush", lowWatermarkOf(ma.peerFlushWatermarks)) } // PeerLowWatermarkTsNs returns the minimum received-through timestamp across @@ -343,7 +343,18 @@ func (ma *MetaAggregator) PeerLowWatermarkTsNs() int64 { ma.peerWatermarksLock.Lock() defer ma.peerWatermarksLock.Unlock() ma.dropExpiredRemovedPeersLocked() - return lowWatermarkOf(ma.peerWatermarks) + return observeLowWatermark("delivery", lowWatermarkOf(ma.peerWatermarks)) +} + +// observeLowWatermark exports how far a low-watermark trails the clock (-1 +// while a tracked peer has not signalled) and hands the value back. +func observeLowWatermark(kind string, lowTsNs int64) int64 { + age := -1.0 + if lowTsNs > 0 { + age = float64(time.Now().UnixNano()-lowTsNs) / 1e9 + } + stats.FilerMetaPeerLowWatermarkAgeSeconds.WithLabelValues(kind).Set(age) + return lowTsNs } // DeliveryWatermarkAdvancedChan returns a channel closed the next time the diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index c9ac2fddd0..109b7b0c65 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -116,21 +116,47 @@ type pipelinedSender struct { sendCh chan *filer_pb.SubscribeMetadataResponse errCh chan error done chan struct{} - canBatch bool // true only if client set ClientSupportsBatching + canBatch bool // true only if client set ClientSupportsBatching + scope string // local or aggregated, for the delivery-lag metric } -func newPipelinedSender(ctx context.Context, stream metadataStreamSender, bufSize int, clientSupportsBatching bool) *pipelinedSender { +func newPipelinedSender(ctx context.Context, stream metadataStreamSender, bufSize int, clientSupportsBatching bool, scope string) *pipelinedSender { s := &pipelinedSender{ ctx: ctx, sendCh: make(chan *filer_pb.SubscribeMetadataResponse, bufSize), errCh: make(chan error, 1), done: make(chan struct{}), canBatch: clientSupportsBatching, + scope: scope, } go s.sendLoop(stream) return s } +// observeSubscribeDeliveryLag records, for a real metadata event that was just +// sent, the seconds from its original mutation stamp until delivery. It skips +// control messages (heartbeats, flush reports) and log-file-ref envelopes, which +// carry no mutation. MutationTsNs is the original stamp (TsNs is the stream +// position, which the buffer can move forward past a late event); payloads older +// than the MutationTsNs field still carry the stamp in TsNs. +func observeSubscribeDeliveryLag(scope string, msg *filer_pb.SubscribeMetadataResponse) { + if msg == nil || msg.EventNotification == nil || len(msg.LogFileRefs) > 0 || filer_pb.IsEmpty(msg) { + // Skip control messages, log-file-ref envelopes, and the empty progress + // markers sent after MaxUnsyncedEvents filtered entries: none is a real + // delivery, and the marker carries the filtered entry's position (TsNs), + // not a mutation. + return + } + stampNs := msg.MutationTsNs + if stampNs == 0 { + stampNs = msg.TsNs + } + if stampNs <= 0 { + return + } + stats.FilerSubscribeDeliveryLagSeconds.WithLabelValues(scope).Observe(float64(time.Now().UnixNano()-stampNs) / 1e9) +} + func (s *pipelinedSender) sendLoop(stream metadataStreamSender) { defer close(s.done) // No Send may start after the subscription context ends: the handler is @@ -146,6 +172,14 @@ func (s *pipelinedSender) sendLoop(stream metadataStreamSender) { } return false } + // Record delivery lag only after the event actually reached the client + // over the stream: Send() above is the real gRPC send (this loop runs + // off the queue), and a batched envelope carries its extra events in + // Events, so each one is sampled here too. + observeSubscribeDeliveryLag(s.scope, msg) + for _, ev := range msg.Events { + observeSubscribeDeliveryLag(s.scope, ev) + } return true } for msg := range s.sendCh { @@ -695,7 +729,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, // had already arrived and been delivered). diskAnchorTsNs := req.SinceNs - sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching) + sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching, "aggregated") defer sender.Close() // Register for instant notification when new data arrives in the aggregated log buffer. @@ -728,6 +762,23 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, // diskHeldAtTsNs is the same marker for the disk pass alone: a pending // disk hold keeps the pass re-reading until the entry is served. var heldAtTsNs, diskHeldAtTsNs int64 + + // A watermark hold on one entry spans many waitHeld retries; waitHeld starts + // the timing and the read records the whole hold once -- when it passes the + // held entry, or the subscription ends -- not per retry, which would turn a + // long hold into many short samples. holdScope is memory or disk. + var holdEntryTsNs int64 + var holdStart time.Time + var holdScope string + flushHold := func() { + if holdEntryTsNs != 0 { + // time.Since uses the monotonic clock, so a wall-clock step back + // mid-hold cannot produce a negative duration. + stats.FilerSubscribeHoldSeconds.WithLabelValues(holdScope).Observe(time.Since(holdStart).Seconds()) + holdEntryTsNs = 0 + } + } + defer flushHold() // Each read path holds at its own watermark: persisted logs are complete // only up to every peer's flush watermark, the ring only up to every // peer's delivery watermark. @@ -744,6 +795,12 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, heldAtTsNs = logEntry.TsNs return false, errHeldByPeerWatermark } + if holdEntryTsNs != 0 && logEntry.TsNs >= holdEntryTsNs { + // The read passed the held entry: the hold is over, so record + // its duration now. >= also covers a cursor the gap machinery + // moved past the held entry. + flushHold() + } lastSeenTsNs = logEntry.TsNs deliveredUpToTsNs = logEntry.TsNs return baseEachLogEntryFn(logEntry) @@ -778,6 +835,12 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, // False: context ended. waitHeld := func(scope string, watermarkChan <-chan struct{}) bool { stats.FilerSubscribeWatermarkHolds.WithLabelValues(scope).Inc() + if heldAtTsNs != holdEntryTsNs { + // A different entry is now held: the previous hold (if any) was + // released, so record it and start timing this one. + flushHold() + holdEntryTsNs, holdStart, holdScope = heldAtTsNs, time.Now(), scope + } glog.V(3).Infof("held at %v (deliveredUpTo %v, flushLow %v, deliveryLow %v) for %v", time.Unix(0, heldAtTsNs), time.Unix(0, deliveredUpToTsNs), time.Unix(0, fs.filer.MetaAggregator.PeerLowFlushWatermarkTsNs()), @@ -1041,6 +1104,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, continue } if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) { + stats.FilerSubscribeResumeFromDisk.WithLabelValues("aggregated").Inc() // Fell off the ring: resume the disk pass from the anchor, not // the (possibly bumped) cursor - redelivery is within the // at-least-once contract, skipping is not. For an anchored @@ -1178,7 +1242,7 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq lastReadTime := log_buffer.NewMessagePosition(req.SinceNs, gapResumeCursorOffset) glog.V(0).Infof(" + %v local subscribe %s from %+v clientId:%d", clientName, req.PathPrefix, lastReadTime, req.ClientId) - sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching) + sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching, "local") defer sender.Close() // Bounded gap waits use the buffer's subscriber notification plus a retry @@ -1349,6 +1413,7 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq return errAggregationUpgrade } if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) { + stats.FilerSubscribeResumeFromDisk.WithLabelValues("local").Inc() // Fell behind the ring: back to the disk pass (it re-runs when // the flush or the cursor moved), and from there to the gap // resolution above if the disk has nothing either. diff --git a/weed/server/filer_grpc_server_sub_meta_test.go b/weed/server/filer_grpc_server_sub_meta_test.go index 8091da8856..4fd3fb981d 100644 --- a/weed/server/filer_grpc_server_sub_meta_test.go +++ b/weed/server/filer_grpc_server_sub_meta_test.go @@ -133,7 +133,7 @@ func TestPipelinedSenderThroughput(t *testing.T) { var batchedRate float64 t.Run("pipelined_batched_send", func(t *testing.T) { stream := &slowStream{sendDelay: sendDelay} - sender := newPipelinedSender(context.Background(), stream, 1024, true) + sender := newPipelinedSender(context.Background(), stream, 1024, true, "local") start := time.Now() for _, file := range files { @@ -220,7 +220,7 @@ func TestBatchingAdaptive(t *testing.T) { t.Run("old_events_are_batched", func(t *testing.T) { stream := &slowStream{sendDelay: 10 * time.Microsecond} - sender := newPipelinedSender(context.Background(), stream, 1024, true) + sender := newPipelinedSender(context.Background(), stream, 1024, true, "local") // Push all events at once (no read delay) so the sender can batch aggressively for _, ev := range makeOldEvents(numEvents) { @@ -240,7 +240,7 @@ func TestBatchingAdaptive(t *testing.T) { t.Run("recent_events_sent_individually", func(t *testing.T) { stream := &slowStream{sendDelay: 10 * time.Microsecond} - sender := newPipelinedSender(context.Background(), stream, 1024, true) + sender := newPipelinedSender(context.Background(), stream, 1024, true, "local") for _, ev := range makeRecentEvents(numEvents) { sender.Send(ev) @@ -284,7 +284,7 @@ func TestPipelinedSenderErrorPropagation(t *testing.T) { t.Run("send_returns_error", func(t *testing.T) { // Stream fails after 5 successful sends stream := &errorStreamImpl{failAfter: 5, err: sendErr} - sender := newPipelinedSender(context.Background(), stream, 4, true) + sender := newPipelinedSender(context.Background(), stream, 4, true, "local") var lastErr error for i := 0; i < 100; i++ { @@ -306,7 +306,7 @@ func TestPipelinedSenderErrorPropagation(t *testing.T) { // since Send may have already returned before the sender goroutine // processes the message. stream := &errorStreamImpl{failAfter: 0, err: sendErr} - sender := newPipelinedSender(context.Background(), stream, 1024, true) + sender := newPipelinedSender(context.Background(), stream, 1024, true, "local") ev := makeOldEvents(1)[0] sender.Send(ev) @@ -355,7 +355,7 @@ func TestPipelinedSingleVsParallelStreams(t *testing.T) { // simulatePipeline: read files with I/O delay, push events, send via pipelinedSender simulatePipeline := func(files []logFile) (eventsSent, sends int64, elapsed time.Duration, err error) { stream := &slowStream{sendDelay: sendDelay} - sender := newPipelinedSender(context.Background(), stream, 1024, true) + sender := newPipelinedSender(context.Background(), stream, 1024, true, "local") start := time.Now() outer: diff --git a/weed/server/filer_grpc_server_sub_meta_watermark_test.go b/weed/server/filer_grpc_server_sub_meta_watermark_test.go index 6b76a3da23..50a6103f81 100644 --- a/weed/server/filer_grpc_server_sub_meta_watermark_test.go +++ b/weed/server/filer_grpc_server_sub_meta_watermark_test.go @@ -57,7 +57,7 @@ func (s *gatedRecordingStream) Send(msg *filer_pb.SubscribeMetadataResponse) err // reads past it. func TestPipelinedSenderControlMessagesNeverNested(t *testing.T) { stream := &gatedRecordingStream{gate: make(chan struct{})} - sender := newPipelinedSender(context.Background(), stream, 16, true) + sender := newPipelinedSender(context.Background(), stream, 16, true, "local") oldTs := time.Now().Add(-time.Hour).UnixNano() if err := sender.Send(makeEvent("/d", "e1", oldTs)); err != nil { diff --git a/weed/server/filer_pipelined_sender_test.go b/weed/server/filer_pipelined_sender_test.go index aeada1b67c..025c08226d 100644 --- a/weed/server/filer_pipelined_sender_test.go +++ b/weed/server/filer_pipelined_sender_test.go @@ -9,7 +9,10 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" ) type recordingStream struct { @@ -49,7 +52,7 @@ func (s *recordingStream) snapshot() []*filer_pb.SubscribeMetadataResponse { // event. Everything must arrive, in order, whatever the batcher does. func TestPipelinedSenderRefsNeverBatched(t *testing.T) { stream := &recordingStream{slow: 2 * time.Millisecond} // let the queue back up so batching engages - sender := newPipelinedSender(context.Background(), stream, 64, true) + sender := newPipelinedSender(context.Background(), stream, 64, true, "local") oldTs := time.Now().Add(-time.Hour).UnixNano() // far behind: the batch heuristic fires var wantOrder []string @@ -129,7 +132,7 @@ func TestPipelinedSenderUnblocksOnCancel(t *testing.T) { stream := &blockedStream{release: make(chan struct{})} defer close(stream.release) ctx, cancel := context.WithCancel(context.Background()) - sender := newPipelinedSender(ctx, stream, 1, false) + sender := newPipelinedSender(ctx, stream, 1, false, "local") msg := &filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}} if err := sender.Send(msg); err != nil { // consumed by the blocked sendLoop @@ -163,7 +166,7 @@ func TestPipelinedSenderUnblocksOnCancel(t *testing.T) { func TestPipelinedSenderStopsSendingOnCancel(t *testing.T) { stream := &blockedStream{release: make(chan struct{}), entered: make(chan struct{}, 1)} ctx, cancel := context.WithCancel(context.Background()) - sender := newPipelinedSender(ctx, stream, 4, false) + sender := newPipelinedSender(ctx, stream, 4, false, "local") msg := &filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}} if err := sender.Send(msg); err != nil { @@ -191,3 +194,91 @@ func TestPipelinedSenderStopsSendingOnCancel(t *testing.T) { t.Fatalf("stream.Send ran %d times, want 1; a new send started after cancel", n) } } + +// deliveryLagSnapshot returns the delivery-lag histogram's sample count and sum +// for a scope, so a test can assert what one send added. +func deliveryLagSnapshot(t *testing.T, scope string) (uint64, float64) { + t.Helper() + var m dto.Metric + if err := stats.FilerSubscribeDeliveryLagSeconds.WithLabelValues(scope).(prometheus.Metric).Write(&m); err != nil { + t.Fatalf("read delivery-lag metric: %v", err) + } + return m.GetHistogram().GetSampleCount(), m.GetHistogram().GetSampleSum() +} + +// gatedStream blocks Send until release is closed, signalling on entered the +// first time Send is called, so a test can inspect state while a send is in +// flight (i.e. before it has succeeded). +type gatedStream struct { + recordingStream + entered chan struct{} + release chan struct{} +} + +func (s *gatedStream) Send(m *filer_pb.SubscribeMetadataResponse) error { + select { + case s.entered <- struct{}{}: + default: + } + <-s.release + return s.recordingStream.Send(m) +} + +// TestPipelinedSenderObservesDeliveryLagAfterSend proves delivery lag is sampled +// after the real stream send (through the queue + Close), measured from the +// event's mutation stamp (not the moved stream position), and that control +// messages and empty progress markers are not sampled. +func TestPipelinedSenderObservesDeliveryLagAfterSend(t *testing.T) { + const scope = "delivery-lag-test" + beforeCount, beforeSum := deliveryLagSnapshot(t, scope) + + stream := &gatedStream{entered: make(chan struct{}, 1), release: make(chan struct{})} + sender := newPipelinedSender(context.Background(), stream, 64, false, scope) + + mutationNs := time.Now().Add(-5 * time.Second).UnixNano() + // A real event whose stream position (TsNs) has moved well past its original + // mutation stamp; lag must be measured from the mutation stamp. + event := &filer_pb.SubscribeMetadataResponse{ + TsNs: time.Now().UnixNano(), + MutationTsNs: mutationNs, + EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "e"}}, + } + // A control message (heartbeat): no EventNotification -> must not be sampled. + control := &filer_pb.SubscribeMetadataResponse{TsNs: time.Now().UnixNano()} + // A progress marker (sent after MaxUnsyncedEvents filtered entries): a + // non-nil but empty EventNotification carrying the filtered entry's TsNs -> + // must not be sampled either. + marker := &filer_pb.SubscribeMetadataResponse{ + TsNs: time.Now().UnixNano(), + EventNotification: &filer_pb.EventNotification{}, + } + if err := sender.Send(event); err != nil { + t.Fatalf("send event: %v", err) + } + // Wait until the send loop is blocked inside stream.Send for the event, then + // confirm nothing was sampled yet: the observation must come after the send + // succeeds, not when the event entered the queue. + <-stream.entered + if midCount, _ := deliveryLagSnapshot(t, scope); midCount != beforeCount { + t.Fatalf("delivery-lag sampled %d while the send was still blocked; it must observe only after stream.Send succeeds", midCount-beforeCount) + } + close(stream.release) + + if err := sender.Send(control); err != nil { + t.Fatalf("send control: %v", err) + } + if err := sender.Send(marker); err != nil { + t.Fatalf("send marker: %v", err) + } + if err := sender.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + afterCount, afterSum := deliveryLagSnapshot(t, scope) + if afterCount != beforeCount+1 { + t.Fatalf("delivery-lag samples added = %d, want 1 (only the real event; control and progress markers skipped)", afterCount-beforeCount) + } + if lag := afterSum - beforeSum; lag < 4 || lag > 60 { + t.Fatalf("delivery lag = %.2fs, want ~5s from the mutation stamp (not ~0 from the moved TsNs)", lag) + } +} diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 7c3cb35a0f..12ea6e2e8e 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -282,6 +282,51 @@ var ( Help: "Number of metadata subscribers currently parked waiting to read past a gap in the metadata log.", }, []string{"scope"}) + // Delivery observability for the metadata subscription paths: lag at send + // time, hold durations, ring fall-offs, the delivery claim and the peer + // low-watermarks, so a late delivery can be attributed to a mechanism. + FilerSubscribeDeliveryLagSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: subsystemFiler, + Name: "subscribe_delivery_lag_seconds", + Help: "Seconds from a metadata event's mutation until the filer sends it to the subscriber's stream, measured after the send-queue wait and the actual gRPC send (excludes client receipt): scope=local on the filer's own stream, scope=aggregated on the multi-filer stream.", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 180, 300}, + }, []string{"scope"}) + + FilerSubscribeHoldSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: subsystemFiler, + Name: "subscribe_hold_seconds", + Help: "How long an aggregated metadata read stayed held at one entry before the peers' watermark released it: scope=memory at the delivery watermark, scope=disk at the flush watermark.", + Buckets: []float64{0.05, 0.2, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300}, + }, []string{"scope"}) + + FilerSubscribeResumeFromDisk = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystemFiler, + Name: "subscribe_resume_from_disk", + Help: "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log (delivery then follows the per-minute flush).", + }, []string{"scope"}) + + FilerMetaDeliveryClaimAgeSeconds = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: subsystemFiler, + Name: "meta_delivery_claim_age_seconds", + Help: "Seconds the filer's delivery claim (what it tells peer aggregators it has delivered through) trails the clock: capped by the oldest commit stamp still in flight.", + }) + + FilerMetaPeerLowWatermarkAgeSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: subsystemFiler, + Name: "meta_peer_low_watermark_age_seconds", + Help: "Seconds the peers' low-watermark trails the clock (kind=delivery bounds in-memory aggregated reads, kind=flush bounds persisted-log reads); -1 while a tracked peer has not signalled yet.", + }, []string{"kind"}) + FilerMetaAggregatorReplayFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -1089,6 +1134,11 @@ func init() { Gather.MustRegister(FilerSubscribeGapStalledGauge) Gather.MustRegister(FilerSubscribeUnprovenGapCrossings) Gather.MustRegister(FilerSubscribeWatermarkHolds) + Gather.MustRegister(FilerSubscribeDeliveryLagSeconds) + Gather.MustRegister(FilerSubscribeHoldSeconds) + Gather.MustRegister(FilerSubscribeResumeFromDisk) + Gather.MustRegister(FilerMetaDeliveryClaimAgeSeconds) + Gather.MustRegister(FilerMetaPeerLowWatermarkAgeSeconds) Gather.MustRegister(FilerMetaAggregatorReplayFailures) Gather.MustRegister(FilerObjectSizeBytesHistogram) Gather.MustRegister(collectors.NewGoCollector())