mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
feat(metrics): Add EC rebuild/reconstruct Prometheus metrics (#10124)
* Review comment removed unnecessary success and failure count * fix: use Gather.Gather() with seeded counter for EC rebuild registration test - Restore Gather.Gather() to verify MustRegister calls as requested in review - Seed VolumeServerECRebuildCounter before gathering because CounterVec only appears after at least one label value is observed - Use correct fully-qualified metric names (SeaweedFS_volumeServer_*) * fix: remove preflight checkEcVolumeStatus failure from ec_rebuild_total counter ec_rebuild_total should only reflect actual rebuild execution failures (from RebuildEcFiles / RebuildEcxFile), not scan/precheck failures in the volume status loop. The error is still returned to the caller; only the misleading counter increment was removed. * Review comment removed unnecessary observe * label EC rebuild duration histogram by result Without a result label, fast failures pull down the success-latency quantiles shown on the EC Rebuild Duration panel. Make the histogram a HistogramVec keyed by result, record success/failure through one recordEcRebuild helper, and split the Grafana quantiles by (le, result). * reset EC rebuild metric vecs in registration test The HistogramVec needs a child before Gather emits it, so the test must observe once; reset both vecs in cleanup so that sample doesn't leak into other tests. --------- Co-authored-by: Ubuntu User <ubuntu@example.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
Rushikesh Deshpande
co-authored by
Ubuntu User
Chris Lu
Ubuntu User
Chris Lu
parent
57ffef8543
commit
d0db94c34a
@@ -4362,6 +4362,220 @@
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "EC Rebuild Duration",
|
||||
"description": "p50/p95/p99 duration of EC shard rebuild operations",
|
||||
"type": "timeseries",
|
||||
"id": 217,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 126
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_ec_rebuild_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, result))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{result}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_volumeServer_ec_rebuild_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, result))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p95 {{result}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(SeaweedFS_volumeServer_ec_rebuild_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, result))",
|
||||
"range": true,
|
||||
"refId": "C",
|
||||
"legendFormat": "p50 {{result}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "EC Rebuild Operations",
|
||||
"description": "Rate of EC shard rebuild operations by result",
|
||||
"type": "timeseries",
|
||||
"id": 218,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 126
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (result) (rate(SeaweedFS_volumeServer_ec_rebuild_total{cluster=~\"$cluster\"}[5m]))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{result}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
@@ -174,6 +175,11 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
|
||||
return &volume_server_pb.VolumeEcShardsGenerateResponse{}, nil
|
||||
}
|
||||
|
||||
func recordEcRebuild(result string, d time.Duration) {
|
||||
stats.VolumeServerECRebuildHistogram.WithLabelValues(result).Observe(d.Seconds())
|
||||
stats.VolumeServerECRebuildCounter.WithLabelValues(result).Inc()
|
||||
}
|
||||
|
||||
// VolumeEcShardsRebuild generates the any of the missing .ec00 ~ .ec13 files
|
||||
func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_server_pb.VolumeEcShardsRebuildRequest) (*volume_server_pb.VolumeEcShardsRebuildResponse, error) {
|
||||
if err := vs.CheckMaintenanceMode(); err != nil {
|
||||
@@ -235,21 +241,26 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
|
||||
// Rebuild missing EC files, searching all disk locations for input shards.
|
||||
// Present input shards are verified against the bitrot sidecar (when present)
|
||||
// and corrupt ones are regenerated; unsafe_ignore_sidecar bypasses the guard.
|
||||
start := time.Now()
|
||||
dataBaseFileName := path.Join(rebuildDataDir, baseFileName)
|
||||
if generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, erasure_coding.BackgroundECContext(), req.UnsafeIgnoreSidecar, additionalDirs...); err != nil {
|
||||
generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, erasure_coding.BackgroundECContext(), req.UnsafeIgnoreSidecar, additionalDirs...)
|
||||
if err != nil {
|
||||
recordEcRebuild("failure", time.Since(start))
|
||||
return nil, fmt.Errorf("RebuildEcFiles %s: %v", dataBaseFileName, err)
|
||||
} else {
|
||||
rebuiltShardIds = generatedShardIds
|
||||
}
|
||||
rebuiltShardIds = generatedShardIds
|
||||
|
||||
indexBaseFileName := path.Join(rebuildLocation.IdxDirectory, baseFileName)
|
||||
if !util.FileExists(indexBaseFileName+".ecx") && rebuildLocation.IdxDirectory != rebuildLocation.Directory {
|
||||
indexBaseFileName = path.Join(rebuildLocation.Directory, baseFileName)
|
||||
}
|
||||
if err := erasure_coding.RebuildEcxFile(indexBaseFileName); err != nil {
|
||||
recordEcRebuild("failure", time.Since(start))
|
||||
return nil, fmt.Errorf("RebuildEcxFile %s: %v", indexBaseFileName, err)
|
||||
}
|
||||
|
||||
recordEcRebuild("success", time.Since(start))
|
||||
|
||||
// Opportunistic bitrot backfill: if protection is enabled, no sidecar exists
|
||||
// yet (a volume encoded before this feature), and this rebuilder can reach
|
||||
// every shard, compute and write a generation-0 sidecar. The TOFU baseline
|
||||
|
||||
@@ -512,6 +512,25 @@ var (
|
||||
Help: "Counter of replication failures by operation and reason (timeout, connection_refused, context_cancelled, server_error).",
|
||||
}, []string{"operation", "reason"})
|
||||
|
||||
// VolumeServerECRebuildHistogram records the duration of EC shard rebuild operations by result (success, failure).
|
||||
VolumeServerECRebuildHistogram = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemVolumeServer,
|
||||
Name: "ec_rebuild_seconds",
|
||||
Help: "Bucketed histogram of EC shard rebuild/reconstruct duration by result.",
|
||||
Buckets: prometheus.ExponentialBuckets(0.01, 2, 20),
|
||||
}, []string{"result"})
|
||||
|
||||
// VolumeServerECRebuildCounter counts EC shard rebuild operations by result (success, failure).
|
||||
VolumeServerECRebuildCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemVolumeServer,
|
||||
Name: "ec_rebuild_total",
|
||||
Help: "Counter of EC shard rebuild operations by result.",
|
||||
}, []string{"result"})
|
||||
|
||||
S3RequestCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
@@ -886,6 +905,8 @@ func init() {
|
||||
Gather.MustRegister(VolumeServerReplicationHistogram)
|
||||
Gather.MustRegister(VolumeServerReplicationTargets)
|
||||
Gather.MustRegister(VolumeServerReplicationFailures)
|
||||
Gather.MustRegister(VolumeServerECRebuildHistogram)
|
||||
Gather.MustRegister(VolumeServerECRebuildCounter)
|
||||
Gather.MustRegister(MasterUnderReplicatedVolumes)
|
||||
Gather.MustRegister(MasterVolumeCreationCounter)
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
func TestECRebuildMetricsRegistered(t *testing.T) {
|
||||
t.Cleanup(VolumeServerECRebuildCounter.Reset)
|
||||
t.Cleanup(VolumeServerECRebuildHistogram.Reset)
|
||||
|
||||
// A HistogramVec emits no metric family until it has a child, so observe once.
|
||||
VolumeServerECRebuildCounter.WithLabelValues("success").Inc()
|
||||
VolumeServerECRebuildHistogram.WithLabelValues("success").Observe(0.1)
|
||||
metrics, err := Gather.Gather()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to gather metrics: %v", err)
|
||||
}
|
||||
|
||||
counterFound := false
|
||||
histogramFound := false
|
||||
|
||||
for _, mf := range metrics {
|
||||
if mf.GetName() == "SeaweedFS_volumeServer_ec_rebuild_total" {
|
||||
counterFound = true
|
||||
}
|
||||
if mf.GetName() == "SeaweedFS_volumeServer_ec_rebuild_seconds" {
|
||||
histogramFound = true
|
||||
}
|
||||
}
|
||||
|
||||
if !counterFound {
|
||||
t.Errorf("VolumeServerECRebuildCounter: metric not registered with Gather")
|
||||
}
|
||||
if !histogramFound {
|
||||
t.Errorf("VolumeServerECRebuildHistogram: metric not registered with Gather")
|
||||
}
|
||||
}
|
||||
|
||||
func TestECRebuildCounterIncrement(t *testing.T) {
|
||||
VolumeServerECRebuildCounter.Reset()
|
||||
t.Cleanup(func() {
|
||||
VolumeServerECRebuildCounter.Reset()
|
||||
})
|
||||
|
||||
VolumeServerECRebuildCounter.WithLabelValues("success").Inc()
|
||||
VolumeServerECRebuildCounter.WithLabelValues("failure").Inc()
|
||||
|
||||
got := testutil.ToFloat64(VolumeServerECRebuildCounter.WithLabelValues("success"))
|
||||
if got != 1 {
|
||||
t.Errorf("expected 1.0, got %f", got)
|
||||
}
|
||||
got = testutil.ToFloat64(VolumeServerECRebuildCounter.WithLabelValues("failure"))
|
||||
if got != 1 {
|
||||
t.Errorf("expected 1.0, got %f", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user