mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* master: export writable volumes per disk for each volume layout The layout gauges only report how many volumes of a collection take writes, not where they sit, so two writable volumes sharing one disk cannot be seen or alerted on. The leader now exports writable_volumes_per_disk with the layout labels plus data center, rack, node and disk id, counting the replicas of the layout's writable volumes on each disk (state=writable) and those of them that are crowded (state=crowded). Series for disks that no longer hold a writable volume of the layout, and for layouts that are gone, are deleted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: reset the leader-only layout gauges on followers The layout writable, crowded, under-replicated and per-disk gauges are only set by the leader, so a master that lost leadership kept exporting the values of its last term indefinitely, and dashboards that aggregate over masters could show numbers days old. A master that is not the leader now clears them on each periodic pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: chart disks holding several writable volumes of a collection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: keep disks of different clusters apart in the per-disk panel Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
121 lines
4.3 KiB
Go
121 lines
4.3 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
dto "github.com/prometheus/client_model/go"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/topology"
|
|
)
|
|
|
|
func writableVolumesPerDiskSeries(t *testing.T, collection string) map[string]float64 {
|
|
t.Helper()
|
|
ch := make(chan prometheus.Metric, 64)
|
|
go func() {
|
|
stats.MasterWritableVolumesPerDisk.Collect(ch)
|
|
close(ch)
|
|
}()
|
|
got := make(map[string]float64)
|
|
for m := range ch {
|
|
var metric dto.Metric
|
|
if err := m.Write(&metric); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
labels := make(map[string]string)
|
|
for _, l := range metric.GetLabel() {
|
|
labels[l.GetName()] = l.GetValue()
|
|
}
|
|
if labels["collection"] == collection {
|
|
got[labels["node"]+"/"+labels["disk_id"]+"/"+labels["state"]] = metric.GetGauge().GetValue()
|
|
}
|
|
}
|
|
return got
|
|
}
|
|
|
|
func TestWritableDiskSeriesDropsDisksWithoutWritableVolumes(t *testing.T) {
|
|
const collection = "perdisk"
|
|
topo := topology.NewTopology("perdisk", nil, 10000, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Collection: collection, Version: 3, DiskId: 0},
|
|
{Id: 2, Collection: collection, Version: 3, DiskId: 0},
|
|
{Id: 3, Collection: collection, Version: 3, DiskId: 1},
|
|
}, dn)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
vl.SetVolumeCrowded(needle.VolumeId(3))
|
|
node := string(dn.Id())
|
|
|
|
series := make(writableDiskSeries)
|
|
series.update(vl, collection, "hdd", "000", "")
|
|
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
|
|
node + "/0/writable": 2, node + "/0/crowded": 0,
|
|
node + "/1/writable": 1, node + "/1/crowded": 1,
|
|
})
|
|
|
|
vl.SetVolumeReadOnly(dn, needle.VolumeId(3))
|
|
series.update(vl, collection, "hdd", "000", "")
|
|
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
|
|
node + "/0/writable": 2, node + "/0/crowded": 0,
|
|
})
|
|
|
|
series.retain(nil)
|
|
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{})
|
|
if len(series) != 0 {
|
|
t.Errorf("series still tracks %d layouts", len(series))
|
|
}
|
|
}
|
|
|
|
func assertSeries(t *testing.T, got, want map[string]float64) {
|
|
t.Helper()
|
|
if len(got) != len(want) {
|
|
t.Fatalf("series = %v, want %v", got, want)
|
|
}
|
|
for k, v := range want {
|
|
if g, ok := got[k]; !ok || g != v {
|
|
t.Errorf("series[%s] = %v (present %v), want %v", k, g, ok, v)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResetLeaderLayoutGaugesClearsFollowerValues(t *testing.T) {
|
|
const collection = "follower"
|
|
topo := topology.NewTopology("follower", nil, 10000, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Collection: collection, Version: 3},
|
|
}, dn)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
|
|
series := make(writableDiskSeries)
|
|
series.update(vl, collection, "hdd", "000", "")
|
|
stats.MasterVolumeLayoutWritable.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
|
stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
|
stats.MasterUnderReplicatedVolumes.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
|
|
|
resetLeaderLayoutGauges(series)
|
|
|
|
for name, gauge := range map[string]*prometheus.GaugeVec{
|
|
"volume_layout_writable": stats.MasterVolumeLayoutWritable,
|
|
"volume_layout_crowded": stats.MasterVolumeLayoutCrowded,
|
|
"under_replicated_volumes": stats.MasterUnderReplicatedVolumes,
|
|
"writable_volumes_per_disk": stats.MasterWritableVolumesPerDisk,
|
|
} {
|
|
if n := testutil.CollectAndCount(gauge); n != 0 {
|
|
t.Errorf("%s still exports %d series after the reset", name, n)
|
|
}
|
|
}
|
|
if len(series) != 0 {
|
|
t.Errorf("series still tracks %d layouts", len(series))
|
|
}
|
|
}
|