mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-07 04:27:49 +01:00
master: export Hashicorp raft metrics through the go-metrics global raft uses (#12448)
* master: keep the hostname out of raft metric names go-metrics prefixes gauge names with the hostname by default, which gives every master its own metric family, e.g. SeaweedFS_<host>_raft_peers. Prometheus already tells masters apart by the scrape target. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: send Hashicorp raft metrics through the go-metrics global raft uses hashicorp/raft and raft-boltdb report through github.com/hashicorp/go-metrics, but the master set up its prometheus sink on the armon/go-metrics global, so a -raftHashicorp master exported no raft metric families. The sink is now registered on the hashicorp/go-metrics global, and armon/go-metrics is no longer a dependency. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: chart raft commit time and leader contact Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: keep the cluster in the raft panels Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: average raft commit time and leader contact over the rate window The go-metrics summaries keep quantiles for 10s, so a p99 scraped every 30s misses most observations. The _sum/_count rates count every sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: leave raft panels empty when nothing was observed Clamping the count rate plotted 0 ms for windows without commits. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: let the raft metrics test use its own registry Registering on stats.Gather made go test -count=2 fail with a duplicate collector registration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: register raft metrics before opening the raft stores raft.NewRaft starts its run loop before returning, so registering the sink afterwards could drop the first state and boltdb samples. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed
1 parent
7b3983f154
commit
c84c43f3da
5 files changed
+244
-18
No files matched your search
@@ -122,7 +122,6 @@ require (
|
||||
github.com/apache/iceberg-go v0.7.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
|
||||
@@ -134,6 +133,7 @@ require (
|
||||
github.com/go-ldap/ldap/v3 v3.4.14
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/go-metrics v0.7.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hashicorp/raft v1.8.0
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.4.2
|
||||
@@ -390,7 +390,6 @@ require (
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-metrics v0.7.0 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
|
||||
@@ -961,7 +961,6 @@ github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZd
|
||||
github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
|
||||
github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM=
|
||||
github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
|
||||
@@ -2110,6 +2110,198 @@
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Raft Commit Time (avg)",
|
||||
"description": "Time for the leader to commit a log entry (-raftHashicorp masters only)",
|
||||
"type": "timeseries",
|
||||
"id": 248,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 61
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (cluster, instance) (rate(SeaweedFS_raft_commitTime_sum{cluster=~\"$cluster\"}[$__rate_interval]) / (rate(SeaweedFS_raft_commitTime_count{cluster=~\"$cluster\"}[$__rate_interval]) > 0))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{cluster}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Raft Leader Last Contact (avg)",
|
||||
"description": "Time since the leader last heard from its followers (-raftHashicorp masters only)",
|
||||
"type": "timeseries",
|
||||
"id": 249,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 69
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (cluster, instance) (rate(SeaweedFS_raft_leader_lastContact_sum{cluster=~\"$cluster\"}[$__rate_interval]) / (rate(SeaweedFS_raft_leader_lastContact_count{cluster=~\"$cluster\"}[$__rate_interval]) > 0))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{cluster}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -13,11 +13,12 @@ import (
|
||||
"time"
|
||||
|
||||
transport "github.com/Jille/raft-grpc-transport"
|
||||
"github.com/armon/go-metrics"
|
||||
"github.com/armon/go-metrics/prometheus"
|
||||
"github.com/hashicorp/go-metrics"
|
||||
"github.com/hashicorp/go-metrics/prometheus"
|
||||
"github.com/hashicorp/raft"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
boltdb "github.com/hashicorp/raft-boltdb/v2"
|
||||
prom "github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
@@ -146,6 +147,10 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
return nil, fmt.Errorf("raft.ValidateConfig: %w", err)
|
||||
}
|
||||
|
||||
if err := registerRaftMetrics(stats.Gather); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(path.Join(s.dataDir, "snapshots"), os.ModePerm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -199,18 +204,23 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Configure a prometheus sink as the raft metrics sink
|
||||
if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
|
||||
Registerer: stats.Gather,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("NewPrometheusSink: %w", err)
|
||||
} else {
|
||||
metricsConf := metrics.DefaultConfig(stats.Namespace)
|
||||
metricsConf.EnableRuntimeMetrics = false
|
||||
if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
|
||||
return nil, fmt.Errorf("metrics.NewGlobal: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// registerRaftMetrics points the hashicorp/go-metrics global, which raft and
|
||||
// raft-boltdb report to, at a prometheus sink on reg.
|
||||
func registerRaftMetrics(reg prom.Registerer) error {
|
||||
sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
|
||||
Registerer: reg,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("NewPrometheusSink: %w", err)
|
||||
}
|
||||
metricsConf := metrics.DefaultConfig(stats.Namespace)
|
||||
metricsConf.EnableRuntimeMetrics = false
|
||||
metricsConf.EnableHostname = false
|
||||
if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
|
||||
return fmt.Errorf("metrics.NewGlobal: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -4,8 +4,11 @@ import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/go-metrics"
|
||||
"github.com/hashicorp/raft"
|
||||
prom "github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
)
|
||||
|
||||
func TestRaftServerID(t *testing.T) {
|
||||
@@ -72,3 +75,26 @@ func TestAddPeersConfigurationUsesCanonicalIDs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRaftMetricsReachPrometheus(t *testing.T) {
|
||||
reg := prom.NewRegistry()
|
||||
if err := registerRaftMetrics(reg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// raft reports through the hashicorp/go-metrics global
|
||||
metrics.SetGauge([]string{"raft", "peers"}, 3)
|
||||
|
||||
families, err := reg.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range families {
|
||||
if f.GetName() == stats.Namespace+"_raft_peers" {
|
||||
if got := f.GetMetric()[0].GetGauge().GetValue(); got != 3 {
|
||||
t.Fatalf("raft_peers = %v, want 3", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s_raft_peers was not gathered", stats.Namespace)
|
||||
}
|
||||
Reference in new issue
Block a user