From eb717199d0915de8b46dff3e0e789bbc3c2b5e63 Mon Sep 17 00:00:00 2001 From: yanglongwei Date: Sun, 6 Sep 2026 03:52:27 +0800 Subject: [PATCH] master: delete replica_placement_mismatch labels when volumes leave topology (#11062) * master: delete replica_placement_mismatch labels when volumes leave topology Fixes #10804. Setting the gauge to 0 left stale Prometheus time series that grew unbounded with volume churn; remove the label set on unregister instead. * master: delete replica_placement_mismatch only after last placement leaves Unconditional DeleteLabelValues on UnRegisterVolumeLayout dropped the series while other data nodes still held the volume, hiding under-replication until the next collect cycle. Delete only when Lookup is empty, and cover the two-copy case in a regression test. --- weed/topology/topology.go | 9 ++-- weed/topology/topology_test.go | 93 +++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/weed/topology/topology.go b/weed/topology/topology.go index f1c56a05e..204661c06 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -567,12 +567,15 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { glog.Infof("removing volume info: %+v from %v", v, dn.id) - if v.ReplicaPlacement.GetCopyCount() > 1 { - stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0) - } diskType := types.ToDiskType(v.DiskType) volumeLayout := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) volumeLayout.UnRegisterVolume(&v, dn) + // Drop the series only after the last placement is gone. Deleting while + // another data node still holds v would hide under-replication until the + // next CollectDeadNodeAndFullVolumes cycle recreates the label. + if v.ReplicaPlacement.GetCopyCount() > 1 && len(t.Lookup(v.Collection, v.Id)) == 0 { + stats.MasterReplicaPlacementMismatch.DeleteLabelValues(v.Collection, v.Id.String()) + } if volumeLayout.isEmpty() { t.DeleteLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) } diff --git a/weed/topology/topology_test.go b/weed/topology/topology_test.go index a7177e676..0fbac10b5 100644 --- a/weed/topology/topology_test.go +++ b/weed/topology/topology_test.go @@ -2,17 +2,18 @@ package topology import ( "reflect" + "testing" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/sequence" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" - - "testing" ) func TestRemoveDataCenter(t *testing.T) { @@ -235,6 +236,94 @@ func TestAddRemoveVolume(t *testing.T) { } } +func TestUnRegisterVolumeLayoutClearsReplicaPlacementMismatchMetric(t *testing.T) { + stats.MasterReplicaPlacementMismatch.Reset() + t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + + dc := topo.GetOrCreateDataCenter("dc1") + rack := dc.GetOrCreateRack("rack1") + maxVolumeCounts := map[string]uint32{"": 25} + dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts) + + rp, err := super_block.NewReplicaPlacementFromString("001") + if err != nil { + t.Fatalf("NewReplicaPlacementFromString: %v", err) + } + v := storage.VolumeInfo{ + Id: needle.VolumeId(42), + Size: 100, + Collection: "metrics-test", + ReplicaPlacement: rp, + Ttl: needle.EMPTY_TTL, + } + + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + + stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1) + if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 { + t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n) + } + + topo.UnRegisterVolumeLayout(v, dn) + + if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 { + t.Errorf("%d replica_placement_mismatch series left after volume left topology", n) + } +} + +func TestUnRegisterVolumeLayoutKeepsReplicaPlacementMismatchMetricWhilePlacementsRemain(t *testing.T) { + stats.MasterReplicaPlacementMismatch.Reset() + t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + + dc := topo.GetOrCreateDataCenter("dc1") + rack := dc.GetOrCreateRack("rack1") + maxVolumeCounts := map[string]uint32{"": 25} + dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts) + dn2 := rack.GetOrCreateDataNode("127.0.0.1", 34535, 0, "127.0.0.1", "", maxVolumeCounts) + + rp, err := super_block.NewReplicaPlacementFromString("001") + if err != nil { + t.Fatalf("NewReplicaPlacementFromString: %v", err) + } + v := storage.VolumeInfo{ + Id: needle.VolumeId(42), + Size: 100, + Collection: "metrics-test", + ReplicaPlacement: rp, + Ttl: needle.EMPTY_TTL, + } + + dn1.UpdateVolumes([]storage.VolumeInfo{v}) + dn2.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn1) + topo.RegisterVolumeLayout(v, dn2) + + stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1) + if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 { + t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n) + } + + topo.UnRegisterVolumeLayout(v, dn1) + + if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 { + t.Errorf("expected series to remain while %s still holds the volume, got %d", dn2.Id(), n) + } + if got := len(topo.Lookup(v.Collection, v.Id)); got != 1 { + t.Fatalf("expected 1 remaining placement, got %d", got) + } + + topo.UnRegisterVolumeLayout(v, dn2) + + if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 { + t.Errorf("%d replica_placement_mismatch series left after last placement left", n) + } +} + func TestVolumeReadOnlyStatusChange(t *testing.T) { topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)