From c84c43f3da1d89915503952d29ac85e23b2be175 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 6 Nov 2026 09:29:42 -0800 Subject: [PATCH] master: export Hashicorp raft metrics through the go-metrics global raft uses (#12448) * master: keep the hostname out of raft metric names go-metrics prefixes gauge names with the hostname by default, which gives every master its own metric family, e.g. SeaweedFS__raft_peers. Prometheus already tells masters apart by the scrape target. Co-Authored-By: Claude Opus 5.5 * master: send Hashicorp raft metrics through the go-metrics global raft uses hashicorp/raft and raft-boltdb report through github.com/hashicorp/go-metrics, but the master set up its prometheus sink on the armon/go-metrics global, so a -raftHashicorp master exported no raft metric families. The sink is now registered on the hashicorp/go-metrics global, and armon/go-metrics is no longer a dependency. Co-Authored-By: Claude Opus 5.5 * grafana: chart raft commit time and leader contact Co-Authored-By: Claude Opus 5.5 * grafana: keep the cluster in the raft panels Co-Authored-By: Claude Opus 5.5 * grafana: average raft commit time and leader contact over the rate window The go-metrics summaries keep quantiles for 10s, so a p99 scraped every 30s misses most observations. The _sum/_count rates count every sample. Co-Authored-By: Claude Opus 5.5 * grafana: leave raft panels empty when nothing was observed Clamping the count rate plotted 0 ms for windows without commits. Co-Authored-By: Claude Opus 5.5 * master: let the raft metrics test use its own registry Registering on stats.Gather made go test -count=2 fail with a duplicate collector registration. Co-Authored-By: Claude Opus 5.5 * master: register raft metrics before opening the raft stores raft.NewRaft starts its run loop before returning, so registering the sink afterwards could drop the first state and boltdb samples. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- go.mod | 3 +- go.sum | 1 - other/metrics/grafana_seaweedfs.json | 192 +++++++++++++++++++++++++++ weed/server/raft_hashicorp.go | 40 +++--- weed/server/raft_hashicorp_test.go | 26 ++++ 5 files changed, 244 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 2a8b343b83..58f86cc1f3 100644 --- a/go.mod +++ b/go.mod @@ -122,7 +122,6 @@ require ( github.com/apache/iceberg-go v0.7.0 github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3 github.com/arangodb/go-driver v1.6.9 - github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/aws/aws-sdk-go-v2/config v1.33.6 github.com/aws/aws-sdk-go-v2/credentials v1.20.6 @@ -134,6 +133,7 @@ require ( github.com/go-ldap/ldap/v3 v3.4.14 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 + github.com/hashicorp/go-metrics v0.7.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/raft v1.8.0 github.com/hashicorp/raft-boltdb/v2 v2.4.2 @@ -390,7 +390,6 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-metrics v0.7.0 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect diff --git a/go.sum b/go.sum index 0bec2ec331..b0230e59ab 100644 --- a/go.sum +++ b/go.sum @@ -961,7 +961,6 @@ github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZd github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index b9e7e64813..211bbd3033 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -2110,6 +2110,198 @@ } ], "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Raft Commit Time (avg)", + "description": "Time for the leader to commit a log entry (-raftHashicorp masters only)", + "type": "timeseries", + "id": 248, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (cluster, instance) (rate(SeaweedFS_raft_commitTime_sum{cluster=~\"$cluster\"}[$__rate_interval]) / (rate(SeaweedFS_raft_commitTime_count{cluster=~\"$cluster\"}[$__rate_interval]) > 0))", + "range": true, + "refId": "A", + "legendFormat": "{{cluster}} {{instance}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Raft Leader Last Contact (avg)", + "description": "Time since the leader last heard from its followers (-raftHashicorp masters only)", + "type": "timeseries", + "id": 249, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (cluster, instance) (rate(SeaweedFS_raft_leader_lastContact_sum{cluster=~\"$cluster\"}[$__rate_interval]) / (rate(SeaweedFS_raft_leader_lastContact_count{cluster=~\"$cluster\"}[$__rate_interval]) > 0))", + "range": true, + "refId": "A", + "legendFormat": "{{cluster}} {{instance}}" + } + ], + "pluginVersion": "10.3.1" } ] }, diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go index 9d1d601920..0c21eefb75 100644 --- a/weed/server/raft_hashicorp.go +++ b/weed/server/raft_hashicorp.go @@ -13,11 +13,12 @@ import ( "time" transport "github.com/Jille/raft-grpc-transport" - "github.com/armon/go-metrics" - "github.com/armon/go-metrics/prometheus" + "github.com/hashicorp/go-metrics" + "github.com/hashicorp/go-metrics/prometheus" "github.com/hashicorp/raft" hashicorpRaft "github.com/hashicorp/raft" boltdb "github.com/hashicorp/raft-boltdb/v2" + prom "github.com/prometheus/client_golang/prometheus" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/stats" @@ -146,6 +147,10 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { return nil, fmt.Errorf("raft.ValidateConfig: %w", err) } + if err := registerRaftMetrics(stats.Gather); err != nil { + return nil, err + } + if err := os.MkdirAll(path.Join(s.dataDir, "snapshots"), os.ModePerm); err != nil { return nil, err } @@ -199,18 +204,23 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { }() } - // Configure a prometheus sink as the raft metrics sink - if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{ - Registerer: stats.Gather, - }); err != nil { - return nil, fmt.Errorf("NewPrometheusSink: %w", err) - } else { - metricsConf := metrics.DefaultConfig(stats.Namespace) - metricsConf.EnableRuntimeMetrics = false - if _, err = metrics.NewGlobal(metricsConf, sink); err != nil { - return nil, fmt.Errorf("metrics.NewGlobal: %w", err) - } - } - return s, nil } + +// registerRaftMetrics points the hashicorp/go-metrics global, which raft and +// raft-boltdb report to, at a prometheus sink on reg. +func registerRaftMetrics(reg prom.Registerer) error { + sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{ + Registerer: reg, + }) + if err != nil { + return fmt.Errorf("NewPrometheusSink: %w", err) + } + metricsConf := metrics.DefaultConfig(stats.Namespace) + metricsConf.EnableRuntimeMetrics = false + metricsConf.EnableHostname = false + if _, err = metrics.NewGlobal(metricsConf, sink); err != nil { + return fmt.Errorf("metrics.NewGlobal: %w", err) + } + return nil +} diff --git a/weed/server/raft_hashicorp_test.go b/weed/server/raft_hashicorp_test.go index a33c93f3ed..8342a6749b 100644 --- a/weed/server/raft_hashicorp_test.go +++ b/weed/server/raft_hashicorp_test.go @@ -4,8 +4,11 @@ import ( "sort" "testing" + "github.com/hashicorp/go-metrics" "github.com/hashicorp/raft" + prom "github.com/prometheus/client_golang/prometheus" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/stats" ) func TestRaftServerID(t *testing.T) { @@ -72,3 +75,26 @@ func TestAddPeersConfigurationUsesCanonicalIDs(t *testing.T) { } } } + +func TestRaftMetricsReachPrometheus(t *testing.T) { + reg := prom.NewRegistry() + if err := registerRaftMetrics(reg); err != nil { + t.Fatal(err) + } + // raft reports through the hashicorp/go-metrics global + metrics.SetGauge([]string{"raft", "peers"}, 3) + + families, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + for _, f := range families { + if f.GetName() == stats.Namespace+"_raft_peers" { + if got := f.GetMetric()[0].GetGauge().GetValue(); got != 3 { + t.Fatalf("raft_peers = %v, want 3", got) + } + return + } + } + t.Fatalf("%s_raft_peers was not gathered", stats.Namespace) +}