Files
seaweedfs/weed/server/master_writable_volumes_per_disk_test.go
T
Chris LuandClaude Opus 5.5 7b4bdb77bb master: export writable volumes per disk and reset leader-only layout gauges on followers (#12751)
* master: export writable volumes per disk for each volume layout

The layout gauges only report how many volumes of a collection take
writes, not where they sit, so two writable volumes sharing one disk
cannot be seen or alerted on. The leader now exports
writable_volumes_per_disk with the layout labels plus data center, rack,
node and disk id, counting the replicas of the layout's writable volumes
on each disk (state=writable) and those of them that are crowded
(state=crowded). Series for disks that no longer hold a writable volume of
the layout, and for layouts that are gone, are deleted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: reset the leader-only layout gauges on followers

The layout writable, crowded, under-replicated and per-disk gauges are only
set by the leader, so a master that lost leadership kept exporting the
values of its last term indefinitely, and dashboards that aggregate over
masters could show numbers days old. A master that is not the leader now
clears them on each periodic pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: chart disks holding several writable volumes of a collection

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: keep disks of different clusters apart in the per-disk panel

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 16:17:04 -08:00

121 lines
4.3 KiB
Go

package weed_server
import (
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/topology"
)
func writableVolumesPerDiskSeries(t *testing.T, collection string) map[string]float64 {
t.Helper()
ch := make(chan prometheus.Metric, 64)
go func() {
stats.MasterWritableVolumesPerDisk.Collect(ch)
close(ch)
}()
got := make(map[string]float64)
for m := range ch {
var metric dto.Metric
if err := m.Write(&metric); err != nil {
t.Fatal(err)
}
labels := make(map[string]string)
for _, l := range metric.GetLabel() {
labels[l.GetName()] = l.GetValue()
}
if labels["collection"] == collection {
got[labels["node"]+"/"+labels["disk_id"]+"/"+labels["state"]] = metric.GetGauge().GetValue()
}
}
return got
}
func TestWritableDiskSeriesDropsDisksWithoutWritableVolumes(t *testing.T) {
const collection = "perdisk"
topo := topology.NewTopology("perdisk", nil, 10000, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
{Id: 1, Collection: collection, Version: 3, DiskId: 0},
{Id: 2, Collection: collection, Version: 3, DiskId: 0},
{Id: 3, Collection: collection, Version: 3, DiskId: 1},
}, dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
vl.SetVolumeCrowded(needle.VolumeId(3))
node := string(dn.Id())
series := make(writableDiskSeries)
series.update(vl, collection, "hdd", "000", "")
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
node + "/0/writable": 2, node + "/0/crowded": 0,
node + "/1/writable": 1, node + "/1/crowded": 1,
})
vl.SetVolumeReadOnly(dn, needle.VolumeId(3))
series.update(vl, collection, "hdd", "000", "")
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
node + "/0/writable": 2, node + "/0/crowded": 0,
})
series.retain(nil)
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{})
if len(series) != 0 {
t.Errorf("series still tracks %d layouts", len(series))
}
}
func assertSeries(t *testing.T, got, want map[string]float64) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("series = %v, want %v", got, want)
}
for k, v := range want {
if g, ok := got[k]; !ok || g != v {
t.Errorf("series[%s] = %v (present %v), want %v", k, g, ok, v)
}
}
}
func TestResetLeaderLayoutGaugesClearsFollowerValues(t *testing.T) {
const collection = "follower"
topo := topology.NewTopology("follower", nil, 10000, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
{Id: 1, Collection: collection, Version: 3},
}, dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
series := make(writableDiskSeries)
series.update(vl, collection, "hdd", "000", "")
stats.MasterVolumeLayoutWritable.WithLabelValues(collection, "hdd", "000", "").Set(1)
stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, "hdd", "000", "").Set(1)
stats.MasterUnderReplicatedVolumes.WithLabelValues(collection, "hdd", "000", "").Set(1)
resetLeaderLayoutGauges(series)
for name, gauge := range map[string]*prometheus.GaugeVec{
"volume_layout_writable": stats.MasterVolumeLayoutWritable,
"volume_layout_crowded": stats.MasterVolumeLayoutCrowded,
"under_replicated_volumes": stats.MasterUnderReplicatedVolumes,
"writable_volumes_per_disk": stats.MasterWritableVolumesPerDisk,
} {
if n := testutil.CollectAndCount(gauge); n != 0 {
t.Errorf("%s still exports %d series after the reset", name, n)
}
}
if len(series) != 0 {
t.Errorf("series still tracks %d layouts", len(series))
}
}