mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
filer: metrics for metadata subscription delivery (lag, holds, ring fall-offs, claim and watermark ages) (#12773)
* filer: metrics for metadata subscription delivery (lag, holds, ring fall-offs, claim and watermark ages) * filer: measure subscribe delivery lag after the real send using the mutation stamp, and record a watermark hold once per hold * filer: skip empty progress markers when sampling subscribe delivery lag * filer: assert subscribe delivery lag is sampled only after the stream send succeeds * grafana: panels for the metadata subscription delivery metrics Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: record a watermark hold when the read passes the held entry, not when the next hold starts * filer: use a monotonic clock for the subscribe hold duration --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
8 files changed
+660
-21
No files matched your search
@@ -6736,6 +6736,423 @@
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Subscribe Delivery Lag",
|
||||
"description": "Seconds from a metadata event's mutation until the filer sends it to a subscriber, by stream scope",
|
||||
"type": "timeseries",
|
||||
"id": 263,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{scope}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p50 {{scope}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Watermark Hold Duration",
|
||||
"description": "How long an aggregated read stayed held at one entry until the peers' watermark released it (memory: delivery watermark, disk: flush watermark)",
|
||||
"type": "timeseries",
|
||||
"id": 264,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{scope}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p50 {{scope}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Delivery Claim / Peer Low-Watermark Age",
|
||||
"description": "Seconds the filer's delivery claim and the peers' delivery and flush low-watermarks trail the clock; -1 while a tracked peer has not signalled",
|
||||
"type": "timeseries",
|
||||
"id": 265,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (instance) (SeaweedFS_filer_meta_delivery_claim_age_seconds{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "claim {{instance}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (instance, kind) (SeaweedFS_filer_meta_peer_low_watermark_age_seconds{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "peer low {{kind}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Subscribers Resumed from Disk",
|
||||
"description": "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log",
|
||||
"type": "timeseries",
|
||||
"id": 266,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (instance, scope) (rate(SeaweedFS_filer_subscribe_resume_from_disk{cluster=~\"$cluster\"}[$__rate_interval]))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{scope}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -6749,7 +7166,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
"y": 87
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -6845,7 +7262,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 71
|
||||
"y": 87
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -6941,7 +7358,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
"y": 95
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -7037,7 +7454,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
"y": 95
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/notification"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
@@ -212,7 +213,11 @@ func (f *Filer) LocalFlushedThroughTsNs(nowNs int64) int64 {
|
||||
// write certain to be visible yet, and a peer aggregator turns the claim into
|
||||
// its delivery low-watermark.
|
||||
func (f *Filer) LocalDeliveredThroughTsNs(nowNs int64) int64 {
|
||||
return f.metaLogInflight.claimThrough(nowNs)
|
||||
claim := f.metaLogInflight.claimThrough(nowNs)
|
||||
// export how far this filer's delivery claim trails the clock (peer
|
||||
// aggregators turn the claim into their delivery low-watermark)
|
||||
stats.FilerMetaDeliveryClaimAgeSeconds.Set(float64(nowNs-claim) / 1e9)
|
||||
return claim
|
||||
}
|
||||
|
||||
// StampMetaLogInflightForTest and DoneMetaLogInflightForTest let tests in
|
||||
|
||||
@@ -332,7 +332,7 @@ func (ma *MetaAggregator) PeerLowFlushWatermarkTsNs() int64 {
|
||||
ma.peerWatermarksLock.Lock()
|
||||
defer ma.peerWatermarksLock.Unlock()
|
||||
ma.dropExpiredRemovedPeersLocked()
|
||||
return lowWatermarkOf(ma.peerFlushWatermarks)
|
||||
return observeLowWatermark("flush", lowWatermarkOf(ma.peerFlushWatermarks))
|
||||
}
|
||||
|
||||
// PeerLowWatermarkTsNs returns the minimum received-through timestamp across
|
||||
@@ -343,7 +343,18 @@ func (ma *MetaAggregator) PeerLowWatermarkTsNs() int64 {
|
||||
ma.peerWatermarksLock.Lock()
|
||||
defer ma.peerWatermarksLock.Unlock()
|
||||
ma.dropExpiredRemovedPeersLocked()
|
||||
return lowWatermarkOf(ma.peerWatermarks)
|
||||
return observeLowWatermark("delivery", lowWatermarkOf(ma.peerWatermarks))
|
||||
}
|
||||
|
||||
// observeLowWatermark exports how far a low-watermark trails the clock (-1
|
||||
// while a tracked peer has not signalled) and hands the value back.
|
||||
func observeLowWatermark(kind string, lowTsNs int64) int64 {
|
||||
age := -1.0
|
||||
if lowTsNs > 0 {
|
||||
age = float64(time.Now().UnixNano()-lowTsNs) / 1e9
|
||||
}
|
||||
stats.FilerMetaPeerLowWatermarkAgeSeconds.WithLabelValues(kind).Set(age)
|
||||
return lowTsNs
|
||||
}
|
||||
|
||||
// DeliveryWatermarkAdvancedChan returns a channel closed the next time the
|
||||
|
||||
@@ -116,21 +116,47 @@ type pipelinedSender struct {
|
||||
sendCh chan *filer_pb.SubscribeMetadataResponse
|
||||
errCh chan error
|
||||
done chan struct{}
|
||||
canBatch bool // true only if client set ClientSupportsBatching
|
||||
canBatch bool // true only if client set ClientSupportsBatching
|
||||
scope string // local or aggregated, for the delivery-lag metric
|
||||
}
|
||||
|
||||
func newPipelinedSender(ctx context.Context, stream metadataStreamSender, bufSize int, clientSupportsBatching bool) *pipelinedSender {
|
||||
func newPipelinedSender(ctx context.Context, stream metadataStreamSender, bufSize int, clientSupportsBatching bool, scope string) *pipelinedSender {
|
||||
s := &pipelinedSender{
|
||||
ctx: ctx,
|
||||
sendCh: make(chan *filer_pb.SubscribeMetadataResponse, bufSize),
|
||||
errCh: make(chan error, 1),
|
||||
done: make(chan struct{}),
|
||||
canBatch: clientSupportsBatching,
|
||||
scope: scope,
|
||||
}
|
||||
go s.sendLoop(stream)
|
||||
return s
|
||||
}
|
||||
|
||||
// observeSubscribeDeliveryLag records, for a real metadata event that was just
|
||||
// sent, the seconds from its original mutation stamp until delivery. It skips
|
||||
// control messages (heartbeats, flush reports) and log-file-ref envelopes, which
|
||||
// carry no mutation. MutationTsNs is the original stamp (TsNs is the stream
|
||||
// position, which the buffer can move forward past a late event); payloads older
|
||||
// than the MutationTsNs field still carry the stamp in TsNs.
|
||||
func observeSubscribeDeliveryLag(scope string, msg *filer_pb.SubscribeMetadataResponse) {
|
||||
if msg == nil || msg.EventNotification == nil || len(msg.LogFileRefs) > 0 || filer_pb.IsEmpty(msg) {
|
||||
// Skip control messages, log-file-ref envelopes, and the empty progress
|
||||
// markers sent after MaxUnsyncedEvents filtered entries: none is a real
|
||||
// delivery, and the marker carries the filtered entry's position (TsNs),
|
||||
// not a mutation.
|
||||
return
|
||||
}
|
||||
stampNs := msg.MutationTsNs
|
||||
if stampNs == 0 {
|
||||
stampNs = msg.TsNs
|
||||
}
|
||||
if stampNs <= 0 {
|
||||
return
|
||||
}
|
||||
stats.FilerSubscribeDeliveryLagSeconds.WithLabelValues(scope).Observe(float64(time.Now().UnixNano()-stampNs) / 1e9)
|
||||
}
|
||||
|
||||
func (s *pipelinedSender) sendLoop(stream metadataStreamSender) {
|
||||
defer close(s.done)
|
||||
// No Send may start after the subscription context ends: the handler is
|
||||
@@ -146,6 +172,14 @@ func (s *pipelinedSender) sendLoop(stream metadataStreamSender) {
|
||||
}
|
||||
return false
|
||||
}
|
||||
// Record delivery lag only after the event actually reached the client
|
||||
// over the stream: Send() above is the real gRPC send (this loop runs
|
||||
// off the queue), and a batched envelope carries its extra events in
|
||||
// Events, so each one is sampled here too.
|
||||
observeSubscribeDeliveryLag(s.scope, msg)
|
||||
for _, ev := range msg.Events {
|
||||
observeSubscribeDeliveryLag(s.scope, ev)
|
||||
}
|
||||
return true
|
||||
}
|
||||
for msg := range s.sendCh {
|
||||
@@ -695,7 +729,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
|
||||
// had already arrived and been delivered).
|
||||
diskAnchorTsNs := req.SinceNs
|
||||
|
||||
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching)
|
||||
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching, "aggregated")
|
||||
defer sender.Close()
|
||||
|
||||
// Register for instant notification when new data arrives in the aggregated log buffer.
|
||||
@@ -728,6 +762,23 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
|
||||
// diskHeldAtTsNs is the same marker for the disk pass alone: a pending
|
||||
// disk hold keeps the pass re-reading until the entry is served.
|
||||
var heldAtTsNs, diskHeldAtTsNs int64
|
||||
|
||||
// A watermark hold on one entry spans many waitHeld retries; waitHeld starts
|
||||
// the timing and the read records the whole hold once -- when it passes the
|
||||
// held entry, or the subscription ends -- not per retry, which would turn a
|
||||
// long hold into many short samples. holdScope is memory or disk.
|
||||
var holdEntryTsNs int64
|
||||
var holdStart time.Time
|
||||
var holdScope string
|
||||
flushHold := func() {
|
||||
if holdEntryTsNs != 0 {
|
||||
// time.Since uses the monotonic clock, so a wall-clock step back
|
||||
// mid-hold cannot produce a negative duration.
|
||||
stats.FilerSubscribeHoldSeconds.WithLabelValues(holdScope).Observe(time.Since(holdStart).Seconds())
|
||||
holdEntryTsNs = 0
|
||||
}
|
||||
}
|
||||
defer flushHold()
|
||||
// Each read path holds at its own watermark: persisted logs are complete
|
||||
// only up to every peer's flush watermark, the ring only up to every
|
||||
// peer's delivery watermark.
|
||||
@@ -744,6 +795,12 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
|
||||
heldAtTsNs = logEntry.TsNs
|
||||
return false, errHeldByPeerWatermark
|
||||
}
|
||||
if holdEntryTsNs != 0 && logEntry.TsNs >= holdEntryTsNs {
|
||||
// The read passed the held entry: the hold is over, so record
|
||||
// its duration now. >= also covers a cursor the gap machinery
|
||||
// moved past the held entry.
|
||||
flushHold()
|
||||
}
|
||||
lastSeenTsNs = logEntry.TsNs
|
||||
deliveredUpToTsNs = logEntry.TsNs
|
||||
return baseEachLogEntryFn(logEntry)
|
||||
@@ -778,6 +835,12 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
|
||||
// False: context ended.
|
||||
waitHeld := func(scope string, watermarkChan <-chan struct{}) bool {
|
||||
stats.FilerSubscribeWatermarkHolds.WithLabelValues(scope).Inc()
|
||||
if heldAtTsNs != holdEntryTsNs {
|
||||
// A different entry is now held: the previous hold (if any) was
|
||||
// released, so record it and start timing this one.
|
||||
flushHold()
|
||||
holdEntryTsNs, holdStart, holdScope = heldAtTsNs, time.Now(), scope
|
||||
}
|
||||
glog.V(3).Infof("held at %v (deliveredUpTo %v, flushLow %v, deliveryLow %v) for %v",
|
||||
time.Unix(0, heldAtTsNs), time.Unix(0, deliveredUpToTsNs),
|
||||
time.Unix(0, fs.filer.MetaAggregator.PeerLowFlushWatermarkTsNs()),
|
||||
@@ -1041,6 +1104,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
|
||||
continue
|
||||
}
|
||||
if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) {
|
||||
stats.FilerSubscribeResumeFromDisk.WithLabelValues("aggregated").Inc()
|
||||
// Fell off the ring: resume the disk pass from the anchor, not
|
||||
// the (possibly bumped) cursor - redelivery is within the
|
||||
// at-least-once contract, skipping is not. For an anchored
|
||||
@@ -1178,7 +1242,7 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
|
||||
lastReadTime := log_buffer.NewMessagePosition(req.SinceNs, gapResumeCursorOffset)
|
||||
glog.V(0).Infof(" + %v local subscribe %s from %+v clientId:%d", clientName, req.PathPrefix, lastReadTime, req.ClientId)
|
||||
|
||||
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching)
|
||||
sender := newPipelinedSender(ctx, stream, 1024, req.ClientSupportsBatching, "local")
|
||||
defer sender.Close()
|
||||
|
||||
// Bounded gap waits use the buffer's subscriber notification plus a retry
|
||||
@@ -1349,6 +1413,7 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
|
||||
return errAggregationUpgrade
|
||||
}
|
||||
if errors.Is(readInMemoryLogErr, log_buffer.ResumeFromDiskError) {
|
||||
stats.FilerSubscribeResumeFromDisk.WithLabelValues("local").Inc()
|
||||
// Fell behind the ring: back to the disk pass (it re-runs when
|
||||
// the flush or the cursor moved), and from there to the gap
|
||||
// resolution above if the disk has nothing either.
|
||||
|
||||
@@ -133,7 +133,7 @@ func TestPipelinedSenderThroughput(t *testing.T) {
|
||||
var batchedRate float64
|
||||
t.Run("pipelined_batched_send", func(t *testing.T) {
|
||||
stream := &slowStream{sendDelay: sendDelay}
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
|
||||
|
||||
start := time.Now()
|
||||
for _, file := range files {
|
||||
@@ -220,7 +220,7 @@ func TestBatchingAdaptive(t *testing.T) {
|
||||
|
||||
t.Run("old_events_are_batched", func(t *testing.T) {
|
||||
stream := &slowStream{sendDelay: 10 * time.Microsecond}
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
|
||||
|
||||
// Push all events at once (no read delay) so the sender can batch aggressively
|
||||
for _, ev := range makeOldEvents(numEvents) {
|
||||
@@ -240,7 +240,7 @@ func TestBatchingAdaptive(t *testing.T) {
|
||||
|
||||
t.Run("recent_events_sent_individually", func(t *testing.T) {
|
||||
stream := &slowStream{sendDelay: 10 * time.Microsecond}
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
|
||||
|
||||
for _, ev := range makeRecentEvents(numEvents) {
|
||||
sender.Send(ev)
|
||||
@@ -284,7 +284,7 @@ func TestPipelinedSenderErrorPropagation(t *testing.T) {
|
||||
t.Run("send_returns_error", func(t *testing.T) {
|
||||
// Stream fails after 5 successful sends
|
||||
stream := &errorStreamImpl{failAfter: 5, err: sendErr}
|
||||
sender := newPipelinedSender(context.Background(), stream, 4, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 4, true, "local")
|
||||
|
||||
var lastErr error
|
||||
for i := 0; i < 100; i++ {
|
||||
@@ -306,7 +306,7 @@ func TestPipelinedSenderErrorPropagation(t *testing.T) {
|
||||
// since Send may have already returned before the sender goroutine
|
||||
// processes the message.
|
||||
stream := &errorStreamImpl{failAfter: 0, err: sendErr}
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
|
||||
|
||||
ev := makeOldEvents(1)[0]
|
||||
sender.Send(ev)
|
||||
@@ -355,7 +355,7 @@ func TestPipelinedSingleVsParallelStreams(t *testing.T) {
|
||||
// simulatePipeline: read files with I/O delay, push events, send via pipelinedSender
|
||||
simulatePipeline := func(files []logFile) (eventsSent, sends int64, elapsed time.Duration, err error) {
|
||||
stream := &slowStream{sendDelay: sendDelay}
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 1024, true, "local")
|
||||
|
||||
start := time.Now()
|
||||
outer:
|
||||
|
||||
@@ -57,7 +57,7 @@ func (s *gatedRecordingStream) Send(msg *filer_pb.SubscribeMetadataResponse) err
|
||||
// reads past it.
|
||||
func TestPipelinedSenderControlMessagesNeverNested(t *testing.T) {
|
||||
stream := &gatedRecordingStream{gate: make(chan struct{})}
|
||||
sender := newPipelinedSender(context.Background(), stream, 16, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 16, true, "local")
|
||||
|
||||
oldTs := time.Now().Add(-time.Hour).UnixNano()
|
||||
if err := sender.Send(makeEvent("/d", "e1", oldTs)); err != nil {
|
||||
|
||||
@@ -9,7 +9,10 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
)
|
||||
|
||||
type recordingStream struct {
|
||||
@@ -49,7 +52,7 @@ func (s *recordingStream) snapshot() []*filer_pb.SubscribeMetadataResponse {
|
||||
// event. Everything must arrive, in order, whatever the batcher does.
|
||||
func TestPipelinedSenderRefsNeverBatched(t *testing.T) {
|
||||
stream := &recordingStream{slow: 2 * time.Millisecond} // let the queue back up so batching engages
|
||||
sender := newPipelinedSender(context.Background(), stream, 64, true)
|
||||
sender := newPipelinedSender(context.Background(), stream, 64, true, "local")
|
||||
|
||||
oldTs := time.Now().Add(-time.Hour).UnixNano() // far behind: the batch heuristic fires
|
||||
var wantOrder []string
|
||||
@@ -129,7 +132,7 @@ func TestPipelinedSenderUnblocksOnCancel(t *testing.T) {
|
||||
stream := &blockedStream{release: make(chan struct{})}
|
||||
defer close(stream.release)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
sender := newPipelinedSender(ctx, stream, 1, false)
|
||||
sender := newPipelinedSender(ctx, stream, 1, false, "local")
|
||||
|
||||
msg := &filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}}
|
||||
if err := sender.Send(msg); err != nil { // consumed by the blocked sendLoop
|
||||
@@ -163,7 +166,7 @@ func TestPipelinedSenderUnblocksOnCancel(t *testing.T) {
|
||||
func TestPipelinedSenderStopsSendingOnCancel(t *testing.T) {
|
||||
stream := &blockedStream{release: make(chan struct{}), entered: make(chan struct{}, 1)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
sender := newPipelinedSender(ctx, stream, 4, false)
|
||||
sender := newPipelinedSender(ctx, stream, 4, false, "local")
|
||||
|
||||
msg := &filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}}
|
||||
if err := sender.Send(msg); err != nil {
|
||||
@@ -191,3 +194,91 @@ func TestPipelinedSenderStopsSendingOnCancel(t *testing.T) {
|
||||
t.Fatalf("stream.Send ran %d times, want 1; a new send started after cancel", n)
|
||||
}
|
||||
}
|
||||
|
||||
// deliveryLagSnapshot returns the delivery-lag histogram's sample count and sum
|
||||
// for a scope, so a test can assert what one send added.
|
||||
func deliveryLagSnapshot(t *testing.T, scope string) (uint64, float64) {
|
||||
t.Helper()
|
||||
var m dto.Metric
|
||||
if err := stats.FilerSubscribeDeliveryLagSeconds.WithLabelValues(scope).(prometheus.Metric).Write(&m); err != nil {
|
||||
t.Fatalf("read delivery-lag metric: %v", err)
|
||||
}
|
||||
return m.GetHistogram().GetSampleCount(), m.GetHistogram().GetSampleSum()
|
||||
}
|
||||
|
||||
// gatedStream blocks Send until release is closed, signalling on entered the
|
||||
// first time Send is called, so a test can inspect state while a send is in
|
||||
// flight (i.e. before it has succeeded).
|
||||
type gatedStream struct {
|
||||
recordingStream
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (s *gatedStream) Send(m *filer_pb.SubscribeMetadataResponse) error {
|
||||
select {
|
||||
case s.entered <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-s.release
|
||||
return s.recordingStream.Send(m)
|
||||
}
|
||||
|
||||
// TestPipelinedSenderObservesDeliveryLagAfterSend proves delivery lag is sampled
|
||||
// after the real stream send (through the queue + Close), measured from the
|
||||
// event's mutation stamp (not the moved stream position), and that control
|
||||
// messages and empty progress markers are not sampled.
|
||||
func TestPipelinedSenderObservesDeliveryLagAfterSend(t *testing.T) {
|
||||
const scope = "delivery-lag-test"
|
||||
beforeCount, beforeSum := deliveryLagSnapshot(t, scope)
|
||||
|
||||
stream := &gatedStream{entered: make(chan struct{}, 1), release: make(chan struct{})}
|
||||
sender := newPipelinedSender(context.Background(), stream, 64, false, scope)
|
||||
|
||||
mutationNs := time.Now().Add(-5 * time.Second).UnixNano()
|
||||
// A real event whose stream position (TsNs) has moved well past its original
|
||||
// mutation stamp; lag must be measured from the mutation stamp.
|
||||
event := &filer_pb.SubscribeMetadataResponse{
|
||||
TsNs: time.Now().UnixNano(),
|
||||
MutationTsNs: mutationNs,
|
||||
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "e"}},
|
||||
}
|
||||
// A control message (heartbeat): no EventNotification -> must not be sampled.
|
||||
control := &filer_pb.SubscribeMetadataResponse{TsNs: time.Now().UnixNano()}
|
||||
// A progress marker (sent after MaxUnsyncedEvents filtered entries): a
|
||||
// non-nil but empty EventNotification carrying the filtered entry's TsNs ->
|
||||
// must not be sampled either.
|
||||
marker := &filer_pb.SubscribeMetadataResponse{
|
||||
TsNs: time.Now().UnixNano(),
|
||||
EventNotification: &filer_pb.EventNotification{},
|
||||
}
|
||||
if err := sender.Send(event); err != nil {
|
||||
t.Fatalf("send event: %v", err)
|
||||
}
|
||||
// Wait until the send loop is blocked inside stream.Send for the event, then
|
||||
// confirm nothing was sampled yet: the observation must come after the send
|
||||
// succeeds, not when the event entered the queue.
|
||||
<-stream.entered
|
||||
if midCount, _ := deliveryLagSnapshot(t, scope); midCount != beforeCount {
|
||||
t.Fatalf("delivery-lag sampled %d while the send was still blocked; it must observe only after stream.Send succeeds", midCount-beforeCount)
|
||||
}
|
||||
close(stream.release)
|
||||
|
||||
if err := sender.Send(control); err != nil {
|
||||
t.Fatalf("send control: %v", err)
|
||||
}
|
||||
if err := sender.Send(marker); err != nil {
|
||||
t.Fatalf("send marker: %v", err)
|
||||
}
|
||||
if err := sender.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
afterCount, afterSum := deliveryLagSnapshot(t, scope)
|
||||
if afterCount != beforeCount+1 {
|
||||
t.Fatalf("delivery-lag samples added = %d, want 1 (only the real event; control and progress markers skipped)", afterCount-beforeCount)
|
||||
}
|
||||
if lag := afterSum - beforeSum; lag < 4 || lag > 60 {
|
||||
t.Fatalf("delivery lag = %.2fs, want ~5s from the mutation stamp (not ~0 from the moved TsNs)", lag)
|
||||
}
|
||||
}
|
||||
@@ -282,6 +282,51 @@ var (
|
||||
Help: "Number of metadata subscribers currently parked waiting to read past a gap in the metadata log.",
|
||||
}, []string{"scope"})
|
||||
|
||||
// Delivery observability for the metadata subscription paths: lag at send
|
||||
// time, hold durations, ring fall-offs, the delivery claim and the peer
|
||||
// low-watermarks, so a late delivery can be attributed to a mechanism.
|
||||
FilerSubscribeDeliveryLagSeconds = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemFiler,
|
||||
Name: "subscribe_delivery_lag_seconds",
|
||||
Help: "Seconds from a metadata event's mutation until the filer sends it to the subscriber's stream, measured after the send-queue wait and the actual gRPC send (excludes client receipt): scope=local on the filer's own stream, scope=aggregated on the multi-filer stream.",
|
||||
Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 180, 300},
|
||||
}, []string{"scope"})
|
||||
|
||||
FilerSubscribeHoldSeconds = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemFiler,
|
||||
Name: "subscribe_hold_seconds",
|
||||
Help: "How long an aggregated metadata read stayed held at one entry before the peers' watermark released it: scope=memory at the delivery watermark, scope=disk at the flush watermark.",
|
||||
Buckets: []float64{0.05, 0.2, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300},
|
||||
}, []string{"scope"})
|
||||
|
||||
FilerSubscribeResumeFromDisk = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemFiler,
|
||||
Name: "subscribe_resume_from_disk",
|
||||
Help: "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log (delivery then follows the per-minute flush).",
|
||||
}, []string{"scope"})
|
||||
|
||||
FilerMetaDeliveryClaimAgeSeconds = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemFiler,
|
||||
Name: "meta_delivery_claim_age_seconds",
|
||||
Help: "Seconds the filer's delivery claim (what it tells peer aggregators it has delivered through) trails the clock: capped by the oldest commit stamp still in flight.",
|
||||
})
|
||||
|
||||
FilerMetaPeerLowWatermarkAgeSeconds = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemFiler,
|
||||
Name: "meta_peer_low_watermark_age_seconds",
|
||||
Help: "Seconds the peers' low-watermark trails the clock (kind=delivery bounds in-memory aggregated reads, kind=flush bounds persisted-log reads); -1 while a tracked peer has not signalled yet.",
|
||||
}, []string{"kind"})
|
||||
|
||||
FilerMetaAggregatorReplayFailures = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
@@ -1089,6 +1134,11 @@ func init() {
|
||||
Gather.MustRegister(FilerSubscribeGapStalledGauge)
|
||||
Gather.MustRegister(FilerSubscribeUnprovenGapCrossings)
|
||||
Gather.MustRegister(FilerSubscribeWatermarkHolds)
|
||||
Gather.MustRegister(FilerSubscribeDeliveryLagSeconds)
|
||||
Gather.MustRegister(FilerSubscribeHoldSeconds)
|
||||
Gather.MustRegister(FilerSubscribeResumeFromDisk)
|
||||
Gather.MustRegister(FilerMetaDeliveryClaimAgeSeconds)
|
||||
Gather.MustRegister(FilerMetaPeerLowWatermarkAgeSeconds)
|
||||
Gather.MustRegister(FilerMetaAggregatorReplayFailures)
|
||||
Gather.MustRegister(FilerObjectSizeBytesHistogram)
|
||||
Gather.MustRegister(collectors.NewGoCollector())
|
||||
|
||||
Reference in new issue
Block a user