From 5681be241ebddaa329e2973d4a16aeb33e073733 Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Wed, 18 Nov 2026 06:32:01 +0530 Subject: [PATCH] topology: never report negative free volume slots (#12698) * topology: never report negative free volume slots Free volume slots are computed as max + remote - volume - ecSlots. When phantom or over-registered volumes push the count past the configured max, the result is negative and rolls up into the cluster total, causing volume grow and assign to fail with a negative 'volumes left' count even while healthy servers have capacity. Clamp every free-volume count at zero so an over-committed server contributes no slots instead of subtracting from the cluster. * topology: drop the issue reference from the free-slot test comment Co-Authored-By: Claude Opus 5.5 * topology: test free slots next to an over-committed server A server with more volumes than slots must not hide the free slots of the other servers in its rack, data center or topology. Co-Authored-By: Claude Opus 5.5 * topology: sum free slots per data node in racks, data centers and topology Clamping the rolled-up counters only clamped their sum, so one server with more volumes than slots still hid the free slots of every other server in its rack, data center and topology. Interior nodes now add up their children's free slots, so each data node counts as zero at worst. A data node in maintenance mode no longer adds slots to its parents either. Co-Authored-By: Claude Opus 5.5 * topology: test that disk info keeps an over-committed deficit Co-Authored-By: Claude Opus 5.5 * topology: keep the signed free volume count in disk info EC encode retries and the balance planners add refunds and planned moves to the published FreeVolumeCount. Clamping it at zero dropped an over-committed disk's deficit, so a refund could make a full disk look free. Volume growth already reads the clamped AvailableSpaceFor. Co-Authored-By: Claude Opus 5.5 * topology: test placement next to an over-committed server Co-Authored-By: Claude Opus 5.5 * topology: accept either data center order in the placement spread check Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- weed/topology/disk.go | 2 +- weed/topology/free_volume_count_test.go | 106 ++++++++++++++++++++++++ weed/topology/maintenance_mode_test.go | 7 +- weed/topology/node.go | 28 ++++--- 4 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 weed/topology/free_volume_count_test.go diff --git a/weed/topology/disk.go b/weed/topology/disk.go index 483c2f3679..85c97bc623 100644 --- a/weed/topology/disk.go +++ b/weed/topology/disk.go @@ -188,7 +188,7 @@ func (a *DiskUsageCounts) snapshot() DiskUsageCounts { func (a *DiskUsageCounts) FreeSpace() int64 { u := a.snapshot() - return u.maxVolumeCount + u.remoteVolumeCount - u.volumeCount - erasure_coding.VolumeSlots(u.ecShardCount) + return max(0, u.maxVolumeCount+u.remoteVolumeCount-u.volumeCount-erasure_coding.VolumeSlots(u.ecShardCount)) } func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts { diff --git a/weed/topology/free_volume_count_test.go b/weed/topology/free_volume_count_test.go new file mode 100644 index 0000000000..92a61837ca --- /dev/null +++ b/weed/topology/free_volume_count_test.go @@ -0,0 +1,106 @@ +package topology + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// An over-committed node (more volumes registered than it has slots for) must +// report zero free slots, not a negative count that drags the cluster total +// below the slots healthy servers can still offer. +func TestFreeVolumeCountNeverNegative(t *testing.T) { + topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5}) + // 8 volumes registered on a server configured for 5: 5 + 0 - 8 = -3. + dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 8}) + + option := &VolumeGrowOption{DiskType: types.HardDriveType} + if got := dn.AvailableSpaceFor(option); got != 0 { + t.Errorf("data node AvailableSpaceFor = %d, want 0 (over-committed slots must not go negative)", got) + } + if got := topo.AvailableSpaceFor(option); got != 0 { + t.Errorf("topology AvailableSpaceFor = %d, want 0", got) + } + if got := dn.diskUsages.FreeSpace(); got != 0 { + t.Errorf("diskUsages.FreeSpace = %d, want 0", got) + } + // Planners add refunds and moves to the published count, so it keeps the deficit. + if got := dn.diskUsages.ToDiskInfo()[""].FreeVolumeCount; got != -3 { + t.Errorf("disk info FreeVolumeCount = %d, want -3", got) + } +} + +func TestFreeVolumeCountPositiveCaseUnchanged(t *testing.T) { + topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5}) + dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 2}) + + option := &VolumeGrowOption{DiskType: types.HardDriveType} + if got := dn.AvailableSpaceFor(option); got != 3 { + t.Errorf("AvailableSpaceFor = %d, want 3", got) + } + if got := dn.diskUsages.ToDiskInfo()[""].FreeVolumeCount; got != 3 { + t.Errorf("disk info FreeVolumeCount = %d, want 3", got) + } +} + +func TestFreeVolumeCountSkipsOverCommittedServer(t *testing.T) { + topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false) + dc1 := topo.GetOrCreateDataCenter("dc1") + rack1 := dc1.GetOrCreateRack("rack1") + over := rack1.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5}) + over.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 20}) + healthy := rack1.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 10}) + dc2 := topo.GetOrCreateDataCenter("dc2") + other := dc2.GetOrCreateRack("rack1").GetOrCreateDataNode("10.0.0.3", 8080, 18080, "", "", map[string]uint32{"": 4}) + other.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 1}) + + option := &VolumeGrowOption{DiskType: types.HardDriveType} + for _, c := range []struct { + name string + node Node + want int64 + }{ + {"rack1", rack1, 10}, + {"dc1", dc1, 10}, + {"dc2", dc2, 3}, + {"topology", topo, 13}, + } { + if got := c.node.AvailableSpaceFor(option); got != c.want { + t.Errorf("%s AvailableSpaceFor = %d, want %d", c.name, got, c.want) + } + } + + for i := 0; i < 20; i++ { + first, rest, err := topo.PickNodesByWeight(2, option, func(Node) error { return nil }) + if err != nil { + t.Fatalf("topology PickNodesByWeight: %v", err) + } + if picked := map[Node]bool{first: true, rest[0]: true}; !picked[dc1] || !picked[dc2] { + t.Fatalf("topology picked %s and %s, want dc1 and dc2", first.Id(), rest[0].Id()) + } + first, _, err = rack1.PickNodesByWeight(1, option, func(Node) error { return nil }) + if err != nil { + t.Fatalf("rack1 PickNodesByWeight: %v", err) + } + if first != Node(healthy) { + t.Fatalf("rack1 picked %s, want %s", first.Id(), healthy.Id()) + } + } + + assigned := map[*DataNode]int{} + for r := int64(0); r < 13; r++ { + dn, err := topo.ReserveOneVolume(r, option) + if err != nil { + t.Fatalf("ReserveOneVolume(%d): %v", r, err) + } + assigned[dn]++ + } + if assigned[over] != 0 || assigned[healthy] == 0 { + t.Errorf("ReserveOneVolume spread over=%d healthy=%d other=%d, want the healthy servers only", + assigned[over], assigned[healthy], assigned[other]) + } +} diff --git a/weed/topology/maintenance_mode_test.go b/weed/topology/maintenance_mode_test.go index 8977efa12e..0003eea980 100644 --- a/weed/topology/maintenance_mode_test.go +++ b/weed/topology/maintenance_mode_test.go @@ -53,8 +53,11 @@ func TestMaintenanceModeReportsNoFreeSlots(t *testing.T) { if _, ok := dn.TryReserveCapacity(types.HardDriveType, 1); ok { t.Error("reserved capacity on a server in maintenance") } - if got := rack.AvailableSpaceFor(option); got != 10 { - t.Errorf("rack AvailableSpaceFor = %d, want 10: the node's capacity still rolls up to its parents", got) + if got := rack.CapacityFor(option); got != 10 { + t.Errorf("rack CapacityFor = %d, want 10: the node's capacity still rolls up to its parents", got) + } + if got := rack.AvailableSpaceFor(option); got != 0 { + t.Errorf("rack AvailableSpaceFor = %d, want 0: its only node is in maintenance", got) } if !dn.SetMaintenanceMode(false) { diff --git a/weed/topology/node.go b/weed/topology/node.go index be642027e7..4e2132c6a1 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -310,9 +310,7 @@ func (n *NodeImpl) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts { } // inMaintenanceMode is true for a data node whose volume server is in -// maintenance mode. Racks and data centers never are: their rolled-up counters -// still include such a node, so their free-slot totals may exceed what their -// children will actually hand out; see reserveOneVolumeInternal. +// maintenance mode. Racks and data centers never are. func (n *NodeImpl) inMaintenanceMode() bool { dn, ok := n.value.(*DataNode) return ok && dn.InMaintenanceMode() @@ -320,18 +318,29 @@ func (n *NodeImpl) inMaintenanceMode() bool { // AvailableSpaceFor is the free volume slots on this node for the option's disk // type. A data node in maintenance mode reports none: it is being drained, so -// it is neither a volume-growth candidate nor reservable. +// it is neither a volume-growth candidate nor reservable. A data node with more +// volumes than slots also reports none, and racks, data centers and the +// topology sum their children, so it cannot hide the free slots of others. func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 { if n.inMaintenanceMode() { return 0 } + if !n.IsDataNode() { + n.RLock() + defer n.RUnlock() + var free int64 + for _, child := range n.children { + free += child.AvailableSpaceFor(option) + } + return free + } t := n.diskUsages.getDisk(option.DiskType) if t == nil { return 0 } freeVolumeSlotCount := atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) - atomic.LoadInt64(&t.volumeCount) freeVolumeSlotCount -= erasure_coding.VolumeSlots(atomic.LoadInt64(&t.ecShardCount)) - return freeVolumeSlotCount + return max(0, freeVolumeSlotCount) } // CapacityFor is the total registered volume slots for the option's disk type; @@ -438,11 +447,10 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u } return node.AvailableSpaceFor(option) } - // The caller draws r from this node's rolled-up free slots, which still - // count children the walk below skips: a data node in maintenance mode - // reports no space, and an over-committed one reports less than zero. Fold - // r into the space that is actually on offer so it cannot walk off the end - // and fail with slots still free. + // The caller drew r from this node's free slots, which can exceed what the + // children offer once their reservations or a newer heartbeat are counted. + // Fold r into the space that is actually on offer so it cannot walk off the + // end and fail with slots still free. var eligible int64 for _, node := range n.children { if freeSpace := freeSpaceOf(node); freeSpace > 0 {