From 7b4bdb77bbcf03a56d1f885d3fd5e8aa68265e47 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2026 16:17:04 -0800 Subject: [PATCH] master: export writable volumes per disk and reset leader-only layout gauges on followers (#12751) * master: export writable volumes per disk for each volume layout The layout gauges only report how many volumes of a collection take writes, not where they sit, so two writable volumes sharing one disk cannot be seen or alerted on. The leader now exports writable_volumes_per_disk with the layout labels plus data center, rack, node and disk id, counting the replicas of the layout's writable volumes on each disk (state=writable) and those of them that are crowded (state=crowded). Series for disks that no longer hold a writable volume of the layout, and for layouts that are gone, are deleted. Co-Authored-By: Claude Opus 5.5 * master: reset the leader-only layout gauges on followers The layout writable, crowded, under-replicated and per-disk gauges are only set by the leader, so a master that lost leadership kept exporting the values of its last term indefinitely, and dashboards that aggregate over masters could show numbers days old. A master that is not the leader now clears them on each periodic pass. Co-Authored-By: Claude Opus 5.5 * grafana: chart disks holding several writable volumes of a collection Co-Authored-By: Claude Opus 5.5 * grafana: keep disks of different clusters apart in the per-disk panel Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- other/metrics/grafana_seaweedfs.json | 107 ++++++++++++++++ weed/server/master_grpc_server_volume.go | 58 ++++++++- .../master_writable_volumes_per_disk_test.go | 120 ++++++++++++++++++ weed/stats/metrics.go | 10 ++ weed/topology/volume_layout.go | 45 +++++++ .../writable_volumes_per_disk_test.go | 56 ++++++++ 6 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 weed/server/master_writable_volumes_per_disk_test.go create mode 100644 weed/topology/writable_volumes_per_disk_test.go diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index b4e0547bf4..21cc57ac89 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -2302,6 +2302,113 @@ } ], "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Writable Volumes per Disk", + "description": "Disks holding more than one writable volume of a collection, and the most writable volumes of a collection on one disk.", + "type": "timeseries", + "id": 262, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}) > 1)", + "range": true, + "refId": "A", + "legendFormat": "{{collection}} disks with 2+" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}))", + "range": true, + "refId": "B", + "legendFormat": "{{collection}} max on one disk" + } + ], + "pluginVersion": "10.3.1" } ] }, diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 099541c583..be306d9492 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand/v2" + "strconv" "strings" "sync" "time" @@ -62,9 +63,60 @@ func periodicGrowCount(lastGrowCount uint32, writable, crowded int) (count uint3 return 0, false } +// writableDiskSeries keeps the label values of the per-disk series last set +// for each layout, so disks that stop holding its writable volumes drop out. +type writableDiskSeries map[*topology.VolumeLayout]map[string][]string + +func (s writableDiskSeries) update(vl *topology.VolumeLayout, layout ...string) { + writable, crowded := vl.WritableVolumesPerDisk() + current := make(map[string][]string, 2*len(writable)) + for disk, count := range writable { + for _, state := range []struct { + name string + count int + }{{"writable", count}, {"crowded", crowded[disk]}} { + labels := append(append(make([]string, 0, len(layout)+5), layout...), + disk.DataCenter, disk.Rack, disk.Node, strconv.FormatUint(uint64(disk.DiskId), 10), state.name) + stats.MasterWritableVolumesPerDisk.WithLabelValues(labels...).Set(float64(state.count)) + current[strings.Join(labels, "\x00")] = labels + } + } + for key, labels := range s[vl] { + if _, ok := current[key]; !ok { + stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...) + } + } + s[vl] = current +} + +func (s writableDiskSeries) retain(layouts []*topology.VolumeLayoutCollection) { + listed := make(map[*topology.VolumeLayout]bool, len(layouts)) + for _, vlc := range layouts { + listed[vlc.VolumeLayout] = true + } + for vl, series := range s { + if listed[vl] { + continue + } + for _, labels := range series { + stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...) + } + delete(s, vl) + } +} + +func resetLeaderLayoutGauges(diskSeries writableDiskSeries) { + stats.MasterVolumeLayoutWritable.Reset() + stats.MasterVolumeLayoutCrowded.Reset() + stats.MasterUnderReplicatedVolumes.Reset() + stats.MasterWritableVolumesPerDisk.Reset() + clear(diskSeries) +} + func (ms *MasterServer) ProcessGrowRequest() { go func() { ctx := context.Background() + diskSeries := make(writableDiskSeries) firstRun := true for { if firstRun { @@ -73,10 +125,13 @@ func (ms *MasterServer) ProcessGrowRequest() { time.Sleep(5*time.Minute + time.Duration(30*rand.Float32())*time.Second) } if !ms.Topo.IsLeader() { + resetLeaderLayoutGauges(diskSeries) continue } dcs := ms.Topo.ListDCAndRacks() - for _, vlc := range ms.Topo.ListVolumeLayoutCollections() { + layouts := ms.Topo.ListVolumeLayoutCollections() + diskSeries.retain(layouts) + for _, vlc := range layouts { var err error vl := vlc.VolumeLayout lastGrowCount := vl.GetLastGrowCount() @@ -86,6 +141,7 @@ func (ms *MasterServer) ProcessGrowRequest() { writable, crowded := vl.GetWritableVolumeCount() vgr := vlc.ToVolumeGrowRequest() underReplicated := vl.CountUnderReplicatedVolumes() + diskSeries.update(vl, vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl) stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable)) stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded)) stats.MasterUnderReplicatedVolumes.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(underReplicated)) diff --git a/weed/server/master_writable_volumes_per_disk_test.go b/weed/server/master_writable_volumes_per_disk_test.go new file mode 100644 index 0000000000..7086e7cba8 --- /dev/null +++ b/weed/server/master_writable_volumes_per_disk_test.go @@ -0,0 +1,120 @@ +package weed_server + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/topology" +) + +func writableVolumesPerDiskSeries(t *testing.T, collection string) map[string]float64 { + t.Helper() + ch := make(chan prometheus.Metric, 64) + go func() { + stats.MasterWritableVolumesPerDisk.Collect(ch) + close(ch) + }() + got := make(map[string]float64) + for m := range ch { + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatal(err) + } + labels := make(map[string]string) + for _, l := range metric.GetLabel() { + labels[l.GetName()] = l.GetValue() + } + if labels["collection"] == collection { + got[labels["node"]+"/"+labels["disk_id"]+"/"+labels["state"]] = metric.GetGauge().GetValue() + } + } + return got +} + +func TestWritableDiskSeriesDropsDisksWithoutWritableVolumes(t *testing.T) { + const collection = "perdisk" + topo := topology.NewTopology("perdisk", nil, 10000, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + {Id: 1, Collection: collection, Version: 3, DiskId: 0}, + {Id: 2, Collection: collection, Version: 3, DiskId: 0}, + {Id: 3, Collection: collection, Version: 3, DiskId: 1}, + }, dn) + rp, _ := super_block.NewReplicaPlacementFromString("000") + vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType) + vl.SetVolumeCrowded(needle.VolumeId(3)) + node := string(dn.Id()) + + series := make(writableDiskSeries) + series.update(vl, collection, "hdd", "000", "") + assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{ + node + "/0/writable": 2, node + "/0/crowded": 0, + node + "/1/writable": 1, node + "/1/crowded": 1, + }) + + vl.SetVolumeReadOnly(dn, needle.VolumeId(3)) + series.update(vl, collection, "hdd", "000", "") + assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{ + node + "/0/writable": 2, node + "/0/crowded": 0, + }) + + series.retain(nil) + assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{}) + if len(series) != 0 { + t.Errorf("series still tracks %d layouts", len(series)) + } +} + +func assertSeries(t *testing.T, got, want map[string]float64) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("series = %v, want %v", got, want) + } + for k, v := range want { + if g, ok := got[k]; !ok || g != v { + t.Errorf("series[%s] = %v (present %v), want %v", k, g, ok, v) + } + } +} + +func TestResetLeaderLayoutGaugesClearsFollowerValues(t *testing.T) { + const collection = "follower" + topo := topology.NewTopology("follower", nil, 10000, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + {Id: 1, Collection: collection, Version: 3}, + }, dn) + rp, _ := super_block.NewReplicaPlacementFromString("000") + vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType) + + series := make(writableDiskSeries) + series.update(vl, collection, "hdd", "000", "") + stats.MasterVolumeLayoutWritable.WithLabelValues(collection, "hdd", "000", "").Set(1) + stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, "hdd", "000", "").Set(1) + stats.MasterUnderReplicatedVolumes.WithLabelValues(collection, "hdd", "000", "").Set(1) + + resetLeaderLayoutGauges(series) + + for name, gauge := range map[string]*prometheus.GaugeVec{ + "volume_layout_writable": stats.MasterVolumeLayoutWritable, + "volume_layout_crowded": stats.MasterVolumeLayoutCrowded, + "under_replicated_volumes": stats.MasterUnderReplicatedVolumes, + "writable_volumes_per_disk": stats.MasterWritableVolumesPerDisk, + } { + if n := testutil.CollectAndCount(gauge); n != 0 { + t.Errorf("%s still exports %d series after the reset", name, n) + } + } + if len(series) != 0 { + t.Errorf("series still tracks %d layouts", len(series)) + } +} diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 64ff6d373b..7c3cb35a0f 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -159,6 +159,14 @@ var ( Help: "Current number of volumes that do not have enough replicas per collection/layout. 0 = healthy.", }, []string{"collection", "disk", "rp", "ttl"}) + MasterWritableVolumesPerDisk = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: subsystemMaster, + Name: "writable_volumes_per_disk", + Help: "Number of writable volumes of a volume layout with a replica on the disk; state=crowded counts those past the grow threshold.", + }, []string{"collection", "disk", "rp", "ttl", "data_center", "rack", "node", "disk_id", "state"}) + // MasterVolumeCreationCounter counts volume creation (growth) operations by result (success, failure). // Volume growth is orchestrated by the master, so this metric is exported by the master process. MasterVolumeCreationCounter = prometheus.NewCounterVec( @@ -1121,6 +1129,7 @@ func init() { Gather.MustRegister(VolumeServerECRebuildCounter) Gather.MustRegister(VolumeServerECReconstructedIntervalCounter) Gather.MustRegister(MasterUnderReplicatedVolumes) + Gather.MustRegister(MasterWritableVolumesPerDisk) Gather.MustRegister(MasterVolumeCreationCounter) Gather.MustRegister(S3RequestCounter) @@ -1264,6 +1273,7 @@ func DeleteCollectionMetrics(collection string) { c += MasterVolumeLayoutWritable.DeletePartialMatch(labels) c += MasterVolumeLayoutCrowded.DeletePartialMatch(labels) c += MasterUnderReplicatedVolumes.DeletePartialMatch(labels) + c += MasterWritableVolumesPerDisk.DeletePartialMatch(labels) c += VolumeServerDiskSizeGauge.DeletePartialMatch(labels) c += VolumeServerVolumeGauge.DeletePartialMatch(labels) c += VolumeServerReadOnlyVolumeGauge.DeletePartialMatch(labels) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index f900d170fa..e915d4dea9 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -1140,6 +1140,51 @@ func (vl *VolumeLayout) CountUnderReplicatedVolumes() int { return count } +type WritableDisk struct { + DataCenter string + Rack string + Node string + DiskId uint32 +} + +// WritableVolumesPerDisk counts, per disk, the replicas of this layout's +// writable volumes, and how many of those are crowded. +func (vl *VolumeLayout) WritableVolumesPerDisk() (writable, crowded map[WritableDisk]int) { + type replicas struct { + vid needle.VolumeId + nodes []*DataNode + crowded bool + } + vl.accessLock.RLock() + snapshot := make([]replicas, 0, len(vl.writables)) + for _, vid := range vl.writables { + if locations, ok := vl.vid2location[vid]; ok { + _, isCrowded := vl.crowded[vid] + snapshot = append(snapshot, replicas{vid, append([]*DataNode(nil), locations.list...), isCrowded}) + } + } + vl.accessLock.RUnlock() + + writable, crowded = make(map[WritableDisk]int), make(map[WritableDisk]int) + for _, r := range snapshot { + for _, dn := range r.nodes { + v, err := dn.GetVolumesById(r.vid) + if err != nil { + continue + } + disk := WritableDisk{DataCenter: dn.GetDataCenterId(), Node: string(dn.Id()), DiskId: v.DiskId} + if rack := dn.Parent(); rack != nil { + disk.Rack = string(rack.Id()) + } + writable[disk]++ + if r.crowded { + crowded[disk]++ + } + } + } + return writable, crowded +} + func (vl *VolumeLayout) removeFromWritable(vid needle.VolumeId) bool { if _, ok := vl.writableMembers[vid]; !ok { return false diff --git a/weed/topology/writable_volumes_per_disk_test.go b/weed/topology/writable_volumes_per_disk_test.go new file mode 100644 index 0000000000..45abe015f1 --- /dev/null +++ b/weed/topology/writable_volumes_per_disk_test.go @@ -0,0 +1,56 @@ +package topology + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func TestWritableVolumesPerDisk(t *testing.T) { + topo := NewTopology("perdisk", nil, 10000, 5, false) + dn1 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + dn2 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack2"). + GetOrCreateDataNode("127.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100}) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + {Id: 1, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0}, + {Id: 2, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0}, + {Id: 3, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1, ReadOnly: true}, + }, dn1) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + {Id: 1, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0}, + {Id: 2, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1}, + {Id: 3, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1, ReadOnly: true}, + }, dn2) + + rp, _ := super_block.NewReplicaPlacementFromString("010") + vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType) + vl.SetVolumeCrowded(needle.VolumeId(2)) + + writable, crowded := vl.WritableVolumesPerDisk() + d1 := WritableDisk{DataCenter: "dc1", Rack: "rack1", Node: string(dn1.Id()), DiskId: 0} + d20 := WritableDisk{DataCenter: "dc1", Rack: "rack2", Node: string(dn2.Id()), DiskId: 0} + d21 := WritableDisk{DataCenter: "dc1", Rack: "rack2", Node: string(dn2.Id()), DiskId: 1} + wantWritable := map[WritableDisk]int{d1: 2, d20: 1, d21: 1} + wantCrowded := map[WritableDisk]int{d1: 1, d21: 1} + if len(writable) != len(wantWritable) { + t.Fatalf("writable = %v, want %v", writable, wantWritable) + } + for d, n := range wantWritable { + if writable[d] != n { + t.Errorf("writable[%+v] = %d, want %d", d, writable[d], n) + } + } + if len(crowded) != len(wantCrowded) { + t.Fatalf("crowded = %v, want %v", crowded, wantCrowded) + } + for d, n := range wantCrowded { + if crowded[d] != n { + t.Errorf("crowded[%+v] = %d, want %d", d, crowded[d], n) + } + } +}