Files
seaweedfs/weed/server/raft_hashicorp_test.go
T
Chris LuandClaude Opus 5.5 c84c43f3da master: export Hashicorp raft metrics through the go-metrics global raft uses (#12448)
* master: keep the hostname out of raft metric names

go-metrics prefixes gauge names with the hostname by default, which gives
every master its own metric family, e.g. SeaweedFS_<host>_raft_peers.
Prometheus already tells masters apart by the scrape target.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: send Hashicorp raft metrics through the go-metrics global raft uses

hashicorp/raft and raft-boltdb report through github.com/hashicorp/go-metrics,
but the master set up its prometheus sink on the armon/go-metrics global,
so a -raftHashicorp master exported no raft metric families. The sink is now
registered on the hashicorp/go-metrics global, and armon/go-metrics is no
longer a dependency.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: chart raft commit time and leader contact

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: keep the cluster in the raft panels

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: average raft commit time and leader contact over the rate window

The go-metrics summaries keep quantiles for 10s, so a p99 scraped every
30s misses most observations. The _sum/_count rates count every sample.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: leave raft panels empty when nothing was observed

Clamping the count rate plotted 0 ms for windows without commits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: let the raft metrics test use its own registry

Registering on stats.Gather made go test -count=2 fail with a duplicate
collector registration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: register raft metrics before opening the raft stores

raft.NewRaft starts its run loop before returning, so registering the
sink afterwards could drop the first state and boltdb samples.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 09:29:42 -08:00

101 lines
2.5 KiB
Go

package weed_server
import (
"sort"
"testing"
"github.com/hashicorp/go-metrics"
"github.com/hashicorp/raft"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
func TestRaftServerID(t *testing.T) {
tests := []struct {
name string
addr pb.ServerAddress
want string
}{
{
name: "without grpc suffix",
addr: pb.ServerAddress("master-0:9333"),
want: "master-0:9333",
},
{
name: "with grpc suffix",
addr: pb.NewServerAddress("master-0", 9333, 19333),
want: "master-0:9333",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := raftServerID(tt.addr); got != tt.want {
t.Fatalf("raftServerID(%q) = %q, want %q", tt.addr, got, tt.want)
}
})
}
}
func TestAddPeersConfigurationUsesCanonicalIDs(t *testing.T) {
rs := &RaftServer{
peers: map[string]pb.ServerAddress{
"master-0:9333": pb.ServerAddress("master-0:9333"),
"master-1:9333": pb.ServerAddress("master-1:9333"),
"master-2:9333": pb.ServerAddress("master-2:9333"),
},
}
cfg := rs.AddPeersConfiguration()
if len(cfg.Servers) != 3 {
t.Fatalf("len(cfg.Servers) = %d, want 3", len(cfg.Servers))
}
var ids []string
var addrs []string
for _, s := range cfg.Servers {
if s.Suffrage != raft.Voter {
t.Fatalf("server %q has suffrage %q, want %q", s.ID, s.Suffrage, raft.Voter)
}
ids = append(ids, string(s.ID))
addrs = append(addrs, string(s.Address))
}
sort.Strings(ids)
sort.Strings(addrs)
wantIDs := []string{"master-0:9333", "master-1:9333", "master-2:9333"}
wantAddrs := []string{"master-0:19333", "master-1:19333", "master-2:19333"}
for i := range wantIDs {
if ids[i] != wantIDs[i] {
t.Fatalf("ids[%d] = %q, want %q", i, ids[i], wantIDs[i])
}
if addrs[i] != wantAddrs[i] {
t.Fatalf("addrs[%d] = %q, want %q", i, addrs[i], wantAddrs[i])
}
}
}
func TestRaftMetricsReachPrometheus(t *testing.T) {
reg := prom.NewRegistry()
if err := registerRaftMetrics(reg); err != nil {
t.Fatal(err)
}
// raft reports through the hashicorp/go-metrics global
metrics.SetGauge([]string{"raft", "peers"}, 3)
families, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
for _, f := range families {
if f.GetName() == stats.Namespace+"_raft_peers" {
if got := f.GetMetric()[0].GetGauge().GetValue(); got != 3 {
t.Fatalf("raft_peers = %v, want 3", got)
}
return
}
}
t.Fatalf("%s_raft_peers was not gathered", stats.Namespace)
}