mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
master: export writable volumes per disk and reset leader-only layout gauges on followers (#12751)
* master: export writable volumes per disk for each volume layout The layout gauges only report how many volumes of a collection take writes, not where they sit, so two writable volumes sharing one disk cannot be seen or alerted on. The leader now exports writable_volumes_per_disk with the layout labels plus data center, rack, node and disk id, counting the replicas of the layout's writable volumes on each disk (state=writable) and those of them that are crowded (state=crowded). Series for disks that no longer hold a writable volume of the layout, and for layouts that are gone, are deleted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: reset the leader-only layout gauges on followers The layout writable, crowded, under-replicated and per-disk gauges are only set by the leader, so a master that lost leadership kept exporting the values of its last term indefinitely, and dashboards that aggregate over masters could show numbers days old. A master that is not the leader now clears them on each periodic pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: chart disks holding several writable volumes of a collection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: keep disks of different clusters apart in the per-disk panel Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1fd5008d63
commit
7b4bdb77bb
6 files changed
+395
-1
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -62,9 +63,60 @@ func periodicGrowCount(lastGrowCount uint32, writable, crowded int) (count uint3
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// writableDiskSeries keeps the label values of the per-disk series last set
|
||||
// for each layout, so disks that stop holding its writable volumes drop out.
|
||||
type writableDiskSeries map[*topology.VolumeLayout]map[string][]string
|
||||
|
||||
func (s writableDiskSeries) update(vl *topology.VolumeLayout, layout ...string) {
|
||||
writable, crowded := vl.WritableVolumesPerDisk()
|
||||
current := make(map[string][]string, 2*len(writable))
|
||||
for disk, count := range writable {
|
||||
for _, state := range []struct {
|
||||
name string
|
||||
count int
|
||||
}{{"writable", count}, {"crowded", crowded[disk]}} {
|
||||
labels := append(append(make([]string, 0, len(layout)+5), layout...),
|
||||
disk.DataCenter, disk.Rack, disk.Node, strconv.FormatUint(uint64(disk.DiskId), 10), state.name)
|
||||
stats.MasterWritableVolumesPerDisk.WithLabelValues(labels...).Set(float64(state.count))
|
||||
current[strings.Join(labels, "\x00")] = labels
|
||||
}
|
||||
}
|
||||
for key, labels := range s[vl] {
|
||||
if _, ok := current[key]; !ok {
|
||||
stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...)
|
||||
}
|
||||
}
|
||||
s[vl] = current
|
||||
}
|
||||
|
||||
func (s writableDiskSeries) retain(layouts []*topology.VolumeLayoutCollection) {
|
||||
listed := make(map[*topology.VolumeLayout]bool, len(layouts))
|
||||
for _, vlc := range layouts {
|
||||
listed[vlc.VolumeLayout] = true
|
||||
}
|
||||
for vl, series := range s {
|
||||
if listed[vl] {
|
||||
continue
|
||||
}
|
||||
for _, labels := range series {
|
||||
stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...)
|
||||
}
|
||||
delete(s, vl)
|
||||
}
|
||||
}
|
||||
|
||||
func resetLeaderLayoutGauges(diskSeries writableDiskSeries) {
|
||||
stats.MasterVolumeLayoutWritable.Reset()
|
||||
stats.MasterVolumeLayoutCrowded.Reset()
|
||||
stats.MasterUnderReplicatedVolumes.Reset()
|
||||
stats.MasterWritableVolumesPerDisk.Reset()
|
||||
clear(diskSeries)
|
||||
}
|
||||
|
||||
func (ms *MasterServer) ProcessGrowRequest() {
|
||||
go func() {
|
||||
ctx := context.Background()
|
||||
diskSeries := make(writableDiskSeries)
|
||||
firstRun := true
|
||||
for {
|
||||
if firstRun {
|
||||
@@ -73,10 +125,13 @@ func (ms *MasterServer) ProcessGrowRequest() {
|
||||
time.Sleep(5*time.Minute + time.Duration(30*rand.Float32())*time.Second)
|
||||
}
|
||||
if !ms.Topo.IsLeader() {
|
||||
resetLeaderLayoutGauges(diskSeries)
|
||||
continue
|
||||
}
|
||||
dcs := ms.Topo.ListDCAndRacks()
|
||||
for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
|
||||
layouts := ms.Topo.ListVolumeLayoutCollections()
|
||||
diskSeries.retain(layouts)
|
||||
for _, vlc := range layouts {
|
||||
var err error
|
||||
vl := vlc.VolumeLayout
|
||||
lastGrowCount := vl.GetLastGrowCount()
|
||||
@@ -86,6 +141,7 @@ func (ms *MasterServer) ProcessGrowRequest() {
|
||||
writable, crowded := vl.GetWritableVolumeCount()
|
||||
vgr := vlc.ToVolumeGrowRequest()
|
||||
underReplicated := vl.CountUnderReplicatedVolumes()
|
||||
diskSeries.update(vl, vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl)
|
||||
stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable))
|
||||
stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded))
|
||||
stats.MasterUnderReplicatedVolumes.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(underReplicated))
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology"
|
||||
)
|
||||
|
||||
func writableVolumesPerDiskSeries(t *testing.T, collection string) map[string]float64 {
|
||||
t.Helper()
|
||||
ch := make(chan prometheus.Metric, 64)
|
||||
go func() {
|
||||
stats.MasterWritableVolumesPerDisk.Collect(ch)
|
||||
close(ch)
|
||||
}()
|
||||
got := make(map[string]float64)
|
||||
for m := range ch {
|
||||
var metric dto.Metric
|
||||
if err := m.Write(&metric); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
labels := make(map[string]string)
|
||||
for _, l := range metric.GetLabel() {
|
||||
labels[l.GetName()] = l.GetValue()
|
||||
}
|
||||
if labels["collection"] == collection {
|
||||
got[labels["node"]+"/"+labels["disk_id"]+"/"+labels["state"]] = metric.GetGauge().GetValue()
|
||||
}
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestWritableDiskSeriesDropsDisksWithoutWritableVolumes(t *testing.T) {
|
||||
const collection = "perdisk"
|
||||
topo := topology.NewTopology("perdisk", nil, 10000, 5, false)
|
||||
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
||||
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
||||
{Id: 1, Collection: collection, Version: 3, DiskId: 0},
|
||||
{Id: 2, Collection: collection, Version: 3, DiskId: 0},
|
||||
{Id: 3, Collection: collection, Version: 3, DiskId: 1},
|
||||
}, dn)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
|
||||
vl.SetVolumeCrowded(needle.VolumeId(3))
|
||||
node := string(dn.Id())
|
||||
|
||||
series := make(writableDiskSeries)
|
||||
series.update(vl, collection, "hdd", "000", "")
|
||||
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
|
||||
node + "/0/writable": 2, node + "/0/crowded": 0,
|
||||
node + "/1/writable": 1, node + "/1/crowded": 1,
|
||||
})
|
||||
|
||||
vl.SetVolumeReadOnly(dn, needle.VolumeId(3))
|
||||
series.update(vl, collection, "hdd", "000", "")
|
||||
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
|
||||
node + "/0/writable": 2, node + "/0/crowded": 0,
|
||||
})
|
||||
|
||||
series.retain(nil)
|
||||
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{})
|
||||
if len(series) != 0 {
|
||||
t.Errorf("series still tracks %d layouts", len(series))
|
||||
}
|
||||
}
|
||||
|
||||
func assertSeries(t *testing.T, got, want map[string]float64) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("series = %v, want %v", got, want)
|
||||
}
|
||||
for k, v := range want {
|
||||
if g, ok := got[k]; !ok || g != v {
|
||||
t.Errorf("series[%s] = %v (present %v), want %v", k, g, ok, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetLeaderLayoutGaugesClearsFollowerValues(t *testing.T) {
|
||||
const collection = "follower"
|
||||
topo := topology.NewTopology("follower", nil, 10000, 5, false)
|
||||
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
||||
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
||||
{Id: 1, Collection: collection, Version: 3},
|
||||
}, dn)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
|
||||
|
||||
series := make(writableDiskSeries)
|
||||
series.update(vl, collection, "hdd", "000", "")
|
||||
stats.MasterVolumeLayoutWritable.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
||||
stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
||||
stats.MasterUnderReplicatedVolumes.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
||||
|
||||
resetLeaderLayoutGauges(series)
|
||||
|
||||
for name, gauge := range map[string]*prometheus.GaugeVec{
|
||||
"volume_layout_writable": stats.MasterVolumeLayoutWritable,
|
||||
"volume_layout_crowded": stats.MasterVolumeLayoutCrowded,
|
||||
"under_replicated_volumes": stats.MasterUnderReplicatedVolumes,
|
||||
"writable_volumes_per_disk": stats.MasterWritableVolumesPerDisk,
|
||||
} {
|
||||
if n := testutil.CollectAndCount(gauge); n != 0 {
|
||||
t.Errorf("%s still exports %d series after the reset", name, n)
|
||||
}
|
||||
}
|
||||
if len(series) != 0 {
|
||||
t.Errorf("series still tracks %d layouts", len(series))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user