master: export writable volumes per disk and reset leader-only layout gauges on followers (#12751)

* master: export writable volumes per disk for each volume layout

The layout gauges only report how many volumes of a collection take
writes, not where they sit, so two writable volumes sharing one disk
cannot be seen or alerted on. The leader now exports
writable_volumes_per_disk with the layout labels plus data center, rack,
node and disk id, counting the replicas of the layout's writable volumes
on each disk (state=writable) and those of them that are crowded
(state=crowded). Series for disks that no longer hold a writable volume of
the layout, and for layouts that are gone, are deleted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: reset the leader-only layout gauges on followers

The layout writable, crowded, under-replicated and per-disk gauges are only
set by the leader, so a master that lost leadership kept exporting the
values of its last term indefinitely, and dashboards that aggregate over
masters could show numbers days old. A master that is not the leader now
clears them on each periodic pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: chart disks holding several writable volumes of a collection

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* grafana: keep disks of different clusters apart in the per-disk panel

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-18 16:17:04 -08:00
1 parent 1fd5008d63
commit 7b4bdb77bb
6 files changed
+395 -1

No files matched your search

+57 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"math/rand/v2"
"strconv"
"strings"
"sync"
"time"
@@ -62,9 +63,60 @@ func periodicGrowCount(lastGrowCount uint32, writable, crowded int) (count uint3
return 0, false
}
// writableDiskSeries keeps the label values of the per-disk series last set
// for each layout, so disks that stop holding its writable volumes drop out.
type writableDiskSeries map[*topology.VolumeLayout]map[string][]string
func (s writableDiskSeries) update(vl *topology.VolumeLayout, layout ...string) {
writable, crowded := vl.WritableVolumesPerDisk()
current := make(map[string][]string, 2*len(writable))
for disk, count := range writable {
for _, state := range []struct {
name string
count int
}{{"writable", count}, {"crowded", crowded[disk]}} {
labels := append(append(make([]string, 0, len(layout)+5), layout...),
disk.DataCenter, disk.Rack, disk.Node, strconv.FormatUint(uint64(disk.DiskId), 10), state.name)
stats.MasterWritableVolumesPerDisk.WithLabelValues(labels...).Set(float64(state.count))
current[strings.Join(labels, "\x00")] = labels
}
}
for key, labels := range s[vl] {
if _, ok := current[key]; !ok {
stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...)
}
}
s[vl] = current
}
func (s writableDiskSeries) retain(layouts []*topology.VolumeLayoutCollection) {
listed := make(map[*topology.VolumeLayout]bool, len(layouts))
for _, vlc := range layouts {
listed[vlc.VolumeLayout] = true
}
for vl, series := range s {
if listed[vl] {
continue
}
for _, labels := range series {
stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...)
}
delete(s, vl)
}
}
func resetLeaderLayoutGauges(diskSeries writableDiskSeries) {
stats.MasterVolumeLayoutWritable.Reset()
stats.MasterVolumeLayoutCrowded.Reset()
stats.MasterUnderReplicatedVolumes.Reset()
stats.MasterWritableVolumesPerDisk.Reset()
clear(diskSeries)
}
func (ms *MasterServer) ProcessGrowRequest() {
go func() {
ctx := context.Background()
diskSeries := make(writableDiskSeries)
firstRun := true
for {
if firstRun {
@@ -73,10 +125,13 @@ func (ms *MasterServer) ProcessGrowRequest() {
time.Sleep(5*time.Minute + time.Duration(30*rand.Float32())*time.Second)
}
if !ms.Topo.IsLeader() {
resetLeaderLayoutGauges(diskSeries)
continue
}
dcs := ms.Topo.ListDCAndRacks()
for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
layouts := ms.Topo.ListVolumeLayoutCollections()
diskSeries.retain(layouts)
for _, vlc := range layouts {
var err error
vl := vlc.VolumeLayout
lastGrowCount := vl.GetLastGrowCount()
@@ -86,6 +141,7 @@ func (ms *MasterServer) ProcessGrowRequest() {
writable, crowded := vl.GetWritableVolumeCount()
vgr := vlc.ToVolumeGrowRequest()
underReplicated := vl.CountUnderReplicatedVolumes()
diskSeries.update(vl, vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl)
stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable))
stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded))
stats.MasterUnderReplicatedVolumes.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(underReplicated))
@@ -0,0 +1,120 @@
package weed_server
import (
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/topology"
)
func writableVolumesPerDiskSeries(t *testing.T, collection string) map[string]float64 {
t.Helper()
ch := make(chan prometheus.Metric, 64)
go func() {
stats.MasterWritableVolumesPerDisk.Collect(ch)
close(ch)
}()
got := make(map[string]float64)
for m := range ch {
var metric dto.Metric
if err := m.Write(&metric); err != nil {
t.Fatal(err)
}
labels := make(map[string]string)
for _, l := range metric.GetLabel() {
labels[l.GetName()] = l.GetValue()
}
if labels["collection"] == collection {
got[labels["node"]+"/"+labels["disk_id"]+"/"+labels["state"]] = metric.GetGauge().GetValue()
}
}
return got
}
func TestWritableDiskSeriesDropsDisksWithoutWritableVolumes(t *testing.T) {
const collection = "perdisk"
topo := topology.NewTopology("perdisk", nil, 10000, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
{Id: 1, Collection: collection, Version: 3, DiskId: 0},
{Id: 2, Collection: collection, Version: 3, DiskId: 0},
{Id: 3, Collection: collection, Version: 3, DiskId: 1},
}, dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
vl.SetVolumeCrowded(needle.VolumeId(3))
node := string(dn.Id())
series := make(writableDiskSeries)
series.update(vl, collection, "hdd", "000", "")
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
node + "/0/writable": 2, node + "/0/crowded": 0,
node + "/1/writable": 1, node + "/1/crowded": 1,
})
vl.SetVolumeReadOnly(dn, needle.VolumeId(3))
series.update(vl, collection, "hdd", "000", "")
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
node + "/0/writable": 2, node + "/0/crowded": 0,
})
series.retain(nil)
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{})
if len(series) != 0 {
t.Errorf("series still tracks %d layouts", len(series))
}
}
func assertSeries(t *testing.T, got, want map[string]float64) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("series = %v, want %v", got, want)
}
for k, v := range want {
if g, ok := got[k]; !ok || g != v {
t.Errorf("series[%s] = %v (present %v), want %v", k, g, ok, v)
}
}
}
func TestResetLeaderLayoutGaugesClearsFollowerValues(t *testing.T) {
const collection = "follower"
topo := topology.NewTopology("follower", nil, 10000, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
{Id: 1, Collection: collection, Version: 3},
}, dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
series := make(writableDiskSeries)
series.update(vl, collection, "hdd", "000", "")
stats.MasterVolumeLayoutWritable.WithLabelValues(collection, "hdd", "000", "").Set(1)
stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, "hdd", "000", "").Set(1)
stats.MasterUnderReplicatedVolumes.WithLabelValues(collection, "hdd", "000", "").Set(1)
resetLeaderLayoutGauges(series)
for name, gauge := range map[string]*prometheus.GaugeVec{
"volume_layout_writable": stats.MasterVolumeLayoutWritable,
"volume_layout_crowded": stats.MasterVolumeLayoutCrowded,
"under_replicated_volumes": stats.MasterUnderReplicatedVolumes,
"writable_volumes_per_disk": stats.MasterWritableVolumesPerDisk,
} {
if n := testutil.CollectAndCount(gauge); n != 0 {
t.Errorf("%s still exports %d series after the reset", name, n)
}
}
if len(series) != 0 {
t.Errorf("series still tracks %d layouts", len(series))
}
}
+10
View File
@@ -159,6 +159,14 @@ var (
Help: "Current number of volumes that do not have enough replicas per collection/layout. 0 = healthy.",
}, []string{"collection", "disk", "rp", "ttl"})
MasterWritableVolumesPerDisk = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: subsystemMaster,
Name: "writable_volumes_per_disk",
Help: "Number of writable volumes of a volume layout with a replica on the disk; state=crowded counts those past the grow threshold.",
}, []string{"collection", "disk", "rp", "ttl", "data_center", "rack", "node", "disk_id", "state"})
// MasterVolumeCreationCounter counts volume creation (growth) operations by result (success, failure).
// Volume growth is orchestrated by the master, so this metric is exported by the master process.
MasterVolumeCreationCounter = prometheus.NewCounterVec(
@@ -1121,6 +1129,7 @@ func init() {
Gather.MustRegister(VolumeServerECRebuildCounter)
Gather.MustRegister(VolumeServerECReconstructedIntervalCounter)
Gather.MustRegister(MasterUnderReplicatedVolumes)
Gather.MustRegister(MasterWritableVolumesPerDisk)
Gather.MustRegister(MasterVolumeCreationCounter)
Gather.MustRegister(S3RequestCounter)
@@ -1264,6 +1273,7 @@ func DeleteCollectionMetrics(collection string) {
c += MasterVolumeLayoutWritable.DeletePartialMatch(labels)
c += MasterVolumeLayoutCrowded.DeletePartialMatch(labels)
c += MasterUnderReplicatedVolumes.DeletePartialMatch(labels)
c += MasterWritableVolumesPerDisk.DeletePartialMatch(labels)
c += VolumeServerDiskSizeGauge.DeletePartialMatch(labels)
c += VolumeServerVolumeGauge.DeletePartialMatch(labels)
c += VolumeServerReadOnlyVolumeGauge.DeletePartialMatch(labels)
+45
View File
@@ -1140,6 +1140,51 @@ func (vl *VolumeLayout) CountUnderReplicatedVolumes() int {
return count
}
type WritableDisk struct {
DataCenter string
Rack string
Node string
DiskId uint32
}
// WritableVolumesPerDisk counts, per disk, the replicas of this layout's
// writable volumes, and how many of those are crowded.
func (vl *VolumeLayout) WritableVolumesPerDisk() (writable, crowded map[WritableDisk]int) {
type replicas struct {
vid needle.VolumeId
nodes []*DataNode
crowded bool
}
vl.accessLock.RLock()
snapshot := make([]replicas, 0, len(vl.writables))
for _, vid := range vl.writables {
if locations, ok := vl.vid2location[vid]; ok {
_, isCrowded := vl.crowded[vid]
snapshot = append(snapshot, replicas{vid, append([]*DataNode(nil), locations.list...), isCrowded})
}
}
vl.accessLock.RUnlock()
writable, crowded = make(map[WritableDisk]int), make(map[WritableDisk]int)
for _, r := range snapshot {
for _, dn := range r.nodes {
v, err := dn.GetVolumesById(r.vid)
if err != nil {
continue
}
disk := WritableDisk{DataCenter: dn.GetDataCenterId(), Node: string(dn.Id()), DiskId: v.DiskId}
if rack := dn.Parent(); rack != nil {
disk.Rack = string(rack.Id())
}
writable[disk]++
if r.crowded {
crowded[disk]++
}
}
}
return writable, crowded
}
func (vl *VolumeLayout) removeFromWritable(vid needle.VolumeId) bool {
if _, ok := vl.writableMembers[vid]; !ok {
return false
@@ -0,0 +1,56 @@
package topology
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func TestWritableVolumesPerDisk(t *testing.T) {
topo := NewTopology("perdisk", nil, 10000, 5, false)
dn1 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
dn2 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack2").
GetOrCreateDataNode("127.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100})
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
{Id: 1, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0},
{Id: 2, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0},
{Id: 3, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1, ReadOnly: true},
}, dn1)
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
{Id: 1, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0},
{Id: 2, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1},
{Id: 3, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1, ReadOnly: true},
}, dn2)
rp, _ := super_block.NewReplicaPlacementFromString("010")
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
vl.SetVolumeCrowded(needle.VolumeId(2))
writable, crowded := vl.WritableVolumesPerDisk()
d1 := WritableDisk{DataCenter: "dc1", Rack: "rack1", Node: string(dn1.Id()), DiskId: 0}
d20 := WritableDisk{DataCenter: "dc1", Rack: "rack2", Node: string(dn2.Id()), DiskId: 0}
d21 := WritableDisk{DataCenter: "dc1", Rack: "rack2", Node: string(dn2.Id()), DiskId: 1}
wantWritable := map[WritableDisk]int{d1: 2, d20: 1, d21: 1}
wantCrowded := map[WritableDisk]int{d1: 1, d21: 1}
if len(writable) != len(wantWritable) {
t.Fatalf("writable = %v, want %v", writable, wantWritable)
}
for d, n := range wantWritable {
if writable[d] != n {
t.Errorf("writable[%+v] = %d, want %d", d, writable[d], n)
}
}
if len(crowded) != len(wantCrowded) {
t.Fatalf("crowded = %v, want %v", crowded, wantCrowded)
}
for d, n := range wantCrowded {
if crowded[d] != n {
t.Errorf("crowded[%+v] = %d, want %d", d, crowded[d], n)
}
}
}