diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index 10fd43b1e..660f79268 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -4362,6 +4362,220 @@ } ], "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "EC Rebuild Duration", + "description": "p50/p95/p99 duration of EC shard rebuild operations", + "type": "timeseries", + "id": 217, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 126 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_ec_rebuild_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, result))", + "range": true, + "refId": "A", + "legendFormat": "p99 {{result}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_volumeServer_ec_rebuild_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, result))", + "range": true, + "refId": "B", + "legendFormat": "p95 {{result}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(SeaweedFS_volumeServer_ec_rebuild_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, result))", + "range": true, + "refId": "C", + "legendFormat": "p50 {{result}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "EC Rebuild Operations", + "description": "Rate of EC shard rebuild operations by result", + "type": "timeseries", + "id": 218, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 126 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (result) (rate(SeaweedFS_volumeServer_ec_rebuild_total{cluster=~\"$cluster\"}[5m]))", + "range": true, + "refId": "A", + "legendFormat": "{{result}}" + } + ], + "pluginVersion": "10.3.1" } ] }, diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index 5362e0c7e..edce72d06 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -16,6 +16,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" @@ -174,6 +175,11 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_ return &volume_server_pb.VolumeEcShardsGenerateResponse{}, nil } +func recordEcRebuild(result string, d time.Duration) { + stats.VolumeServerECRebuildHistogram.WithLabelValues(result).Observe(d.Seconds()) + stats.VolumeServerECRebuildCounter.WithLabelValues(result).Inc() +} + // VolumeEcShardsRebuild generates the any of the missing .ec00 ~ .ec13 files func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_server_pb.VolumeEcShardsRebuildRequest) (*volume_server_pb.VolumeEcShardsRebuildResponse, error) { if err := vs.CheckMaintenanceMode(); err != nil { @@ -235,21 +241,26 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s // Rebuild missing EC files, searching all disk locations for input shards. // Present input shards are verified against the bitrot sidecar (when present) // and corrupt ones are regenerated; unsafe_ignore_sidecar bypasses the guard. + start := time.Now() dataBaseFileName := path.Join(rebuildDataDir, baseFileName) - if generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, erasure_coding.BackgroundECContext(), req.UnsafeIgnoreSidecar, additionalDirs...); err != nil { + generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, erasure_coding.BackgroundECContext(), req.UnsafeIgnoreSidecar, additionalDirs...) + if err != nil { + recordEcRebuild("failure", time.Since(start)) return nil, fmt.Errorf("RebuildEcFiles %s: %v", dataBaseFileName, err) - } else { - rebuiltShardIds = generatedShardIds } + rebuiltShardIds = generatedShardIds indexBaseFileName := path.Join(rebuildLocation.IdxDirectory, baseFileName) if !util.FileExists(indexBaseFileName+".ecx") && rebuildLocation.IdxDirectory != rebuildLocation.Directory { indexBaseFileName = path.Join(rebuildLocation.Directory, baseFileName) } if err := erasure_coding.RebuildEcxFile(indexBaseFileName); err != nil { + recordEcRebuild("failure", time.Since(start)) return nil, fmt.Errorf("RebuildEcxFile %s: %v", indexBaseFileName, err) } + recordEcRebuild("success", time.Since(start)) + // Opportunistic bitrot backfill: if protection is enabled, no sidecar exists // yet (a volume encoded before this feature), and this rebuilder can reach // every shard, compute and write a generation-0 sidecar. The TOFU baseline diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 329283bb3..0db62c38e 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -512,6 +512,25 @@ var ( Help: "Counter of replication failures by operation and reason (timeout, connection_refused, context_cancelled, server_error).", }, []string{"operation", "reason"}) + // VolumeServerECRebuildHistogram records the duration of EC shard rebuild operations by result (success, failure). + VolumeServerECRebuildHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: subsystemVolumeServer, + Name: "ec_rebuild_seconds", + Help: "Bucketed histogram of EC shard rebuild/reconstruct duration by result.", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 20), + }, []string{"result"}) + + // VolumeServerECRebuildCounter counts EC shard rebuild operations by result (success, failure). + VolumeServerECRebuildCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystemVolumeServer, + Name: "ec_rebuild_total", + Help: "Counter of EC shard rebuild operations by result.", + }, []string{"result"}) + S3RequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -886,6 +905,8 @@ func init() { Gather.MustRegister(VolumeServerReplicationHistogram) Gather.MustRegister(VolumeServerReplicationTargets) Gather.MustRegister(VolumeServerReplicationFailures) + Gather.MustRegister(VolumeServerECRebuildHistogram) + Gather.MustRegister(VolumeServerECRebuildCounter) Gather.MustRegister(MasterUnderReplicatedVolumes) Gather.MustRegister(MasterVolumeCreationCounter) diff --git a/weed/stats/metrics_ec_rebuild_test.go b/weed/stats/metrics_ec_rebuild_test.go new file mode 100644 index 000000000..e0e485679 --- /dev/null +++ b/weed/stats/metrics_ec_rebuild_test.go @@ -0,0 +1,58 @@ +package stats + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestECRebuildMetricsRegistered(t *testing.T) { + t.Cleanup(VolumeServerECRebuildCounter.Reset) + t.Cleanup(VolumeServerECRebuildHistogram.Reset) + + // A HistogramVec emits no metric family until it has a child, so observe once. + VolumeServerECRebuildCounter.WithLabelValues("success").Inc() + VolumeServerECRebuildHistogram.WithLabelValues("success").Observe(0.1) + metrics, err := Gather.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + + counterFound := false + histogramFound := false + + for _, mf := range metrics { + if mf.GetName() == "SeaweedFS_volumeServer_ec_rebuild_total" { + counterFound = true + } + if mf.GetName() == "SeaweedFS_volumeServer_ec_rebuild_seconds" { + histogramFound = true + } + } + + if !counterFound { + t.Errorf("VolumeServerECRebuildCounter: metric not registered with Gather") + } + if !histogramFound { + t.Errorf("VolumeServerECRebuildHistogram: metric not registered with Gather") + } +} + +func TestECRebuildCounterIncrement(t *testing.T) { + VolumeServerECRebuildCounter.Reset() + t.Cleanup(func() { + VolumeServerECRebuildCounter.Reset() + }) + + VolumeServerECRebuildCounter.WithLabelValues("success").Inc() + VolumeServerECRebuildCounter.WithLabelValues("failure").Inc() + + got := testutil.ToFloat64(VolumeServerECRebuildCounter.WithLabelValues("success")) + if got != 1 { + t.Errorf("expected 1.0, got %f", got) + } + got = testutil.ToFloat64(VolumeServerECRebuildCounter.WithLabelValues("failure")) + if got != 1 { + t.Errorf("expected 1.0, got %f", got) + } +}