Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-08 15:41:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

master: delete replica_placement_mismatch labels when volumes leave topology (#11062)

Browse Source 
* master: delete replica_placement_mismatch labels when volumes leave topology

Fixes #10804. Setting the gauge to 0 left stale Prometheus time series
that grew unbounded with volume churn; remove the label set on unregister
instead.

* master: delete replica_placement_mismatch only after last placement leaves

Unconditional DeleteLabelValues on UnRegisterVolumeLayout dropped the series
while other data nodes still held the volume, hiding under-replication until
the next collect cycle. Delete only when Lookup is empty, and cover the
two-copy case in a regression test.
This commit is contained in:
yanglongwei
2026-09-05 12:52:27 -07:00
committed by GitHub
parent 3e85d9ec8e
commit eb717199d0
2 changed files with 97 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/topology/topology.go
+6 -3
View File
@@ -567,12 +567,15 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
glog.Infof("removing volume info: %+v from %v", v, dn.id)
if v.ReplicaPlacement.GetCopyCount() > 1 {
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0)
}
diskType := types.ToDiskType(v.DiskType)
volumeLayout := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
volumeLayout.UnRegisterVolume(&v, dn)
// Drop the series only after the last placement is gone. Deleting while
// another data node still holds v would hide under-replication until the
// next CollectDeadNodeAndFullVolumes cycle recreates the label.
if v.ReplicaPlacement.GetCopyCount() > 1 && len(t.Lookup(v.Collection, v.Id)) == 0 {
stats.MasterReplicaPlacementMismatch.DeleteLabelValues(v.Collection, v.Id.String())
}
if volumeLayout.isEmpty() {
t.DeleteLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
}
weed/topology/topology_test.go
+91 -2
View File
@@ -2,17 +2,18 @@ package topology
import (
"reflect"
"testing"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"testing"
)
func TestRemoveDataCenter(t *testing.T) {
@@ -235,6 +236,94 @@ func TestAddRemoveVolume(t *testing.T) {
}
}
func TestUnRegisterVolumeLayoutClearsReplicaPlacementMismatchMetric(t *testing.T) {
stats.MasterReplicaPlacementMismatch.Reset()
t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := map[string]uint32{"": 25}
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
rp, err := super_block.NewReplicaPlacementFromString("001")
if err != nil {
t.Fatalf("NewReplicaPlacementFromString: %v", err)
}
v := storage.VolumeInfo{
Id: needle.VolumeId(42),
Size: 100,
Collection: "metrics-test",
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn)
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
}
topo.UnRegisterVolumeLayout(v, dn)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
t.Errorf("%d replica_placement_mismatch series left after volume left topology", n)
}
}
func TestUnRegisterVolumeLayoutKeepsReplicaPlacementMismatchMetricWhilePlacementsRemain(t *testing.T) {
stats.MasterReplicaPlacementMismatch.Reset()
t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := map[string]uint32{"": 25}
dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
dn2 := rack.GetOrCreateDataNode("127.0.0.1", 34535, 0, "127.0.0.1", "", maxVolumeCounts)
rp, err := super_block.NewReplicaPlacementFromString("001")
if err != nil {
t.Fatalf("NewReplicaPlacementFromString: %v", err)
}
v := storage.VolumeInfo{
Id: needle.VolumeId(42),
Size: 100,
Collection: "metrics-test",
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn1.UpdateVolumes([]storage.VolumeInfo{v})
dn2.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn1)
topo.RegisterVolumeLayout(v, dn2)
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
}
topo.UnRegisterVolumeLayout(v, dn1)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
t.Errorf("expected series to remain while %s still holds the volume, got %d", dn2.Id(), n)
}
if got := len(topo.Lookup(v.Collection, v.Id)); got != 1 {
t.Fatalf("expected 1 remaining placement, got %d", got)
}
topo.UnRegisterVolumeLayout(v, dn2)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
t.Errorf("%d replica_placement_mismatch series left after last placement left", n)
}
}
func TestVolumeReadOnlyStatusChange(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)