topology: never report negative free volume slots (#12698)

* topology: never report negative free volume slots

Free volume slots are computed as max + remote - volume - ecSlots. When phantom or over-registered volumes push the count past the configured max, the result is negative and rolls up into the cluster total, causing volume grow and assign to fail with a negative 'volumes left' count even while healthy servers have capacity. Clamp every free-volume count at zero so an over-committed server contributes no slots instead of subtracting from the cluster.

* topology: drop the issue reference from the free-slot test comment

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test free slots next to an over-committed server

A server with more volumes than slots must not hide the free slots of
the other servers in its rack, data center or topology.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: sum free slots per data node in racks, data centers and topology

Clamping the rolled-up counters only clamped their sum, so one server with
more volumes than slots still hid the free slots of every other server in
its rack, data center and topology. Interior nodes now add up their
children's free slots, so each data node counts as zero at worst. A data
node in maintenance mode no longer adds slots to its parents either.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test that disk info keeps an over-committed deficit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: keep the signed free volume count in disk info

EC encode retries and the balance planners add refunds and planned moves
to the published FreeVolumeCount. Clamping it at zero dropped an
over-committed disk's deficit, so a refund could make a full disk look
free. Volume growth already reads the clamped AvailableSpaceFor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test placement next to an over-committed server

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: accept either data center order in the placement spread check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-17 17:02:01 -08:00
1 parent 8ba33fae71
commit 5681be241e
4 files changed
+130 -13

No files matched your search

+1 -1
View File
@@ -188,7 +188,7 @@ func (a *DiskUsageCounts) snapshot() DiskUsageCounts {
func (a *DiskUsageCounts) FreeSpace() int64 {
u := a.snapshot()
return u.maxVolumeCount + u.remoteVolumeCount - u.volumeCount - erasure_coding.VolumeSlots(u.ecShardCount)
return max(0, u.maxVolumeCount+u.remoteVolumeCount-u.volumeCount-erasure_coding.VolumeSlots(u.ecShardCount))
}
func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts {
+106
View File
@@ -0,0 +1,106 @@
package topology
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// An over-committed node (more volumes registered than it has slots for) must
// report zero free slots, not a negative count that drags the cluster total
// below the slots healthy servers can still offer.
func TestFreeVolumeCountNeverNegative(t *testing.T) {
topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5})
// 8 volumes registered on a server configured for 5: 5 + 0 - 8 = -3.
dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 8})
option := &VolumeGrowOption{DiskType: types.HardDriveType}
if got := dn.AvailableSpaceFor(option); got != 0 {
t.Errorf("data node AvailableSpaceFor = %d, want 0 (over-committed slots must not go negative)", got)
}
if got := topo.AvailableSpaceFor(option); got != 0 {
t.Errorf("topology AvailableSpaceFor = %d, want 0", got)
}
if got := dn.diskUsages.FreeSpace(); got != 0 {
t.Errorf("diskUsages.FreeSpace = %d, want 0", got)
}
// Planners add refunds and moves to the published count, so it keeps the deficit.
if got := dn.diskUsages.ToDiskInfo()[""].FreeVolumeCount; got != -3 {
t.Errorf("disk info FreeVolumeCount = %d, want -3", got)
}
}
func TestFreeVolumeCountPositiveCaseUnchanged(t *testing.T) {
topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5})
dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 2})
option := &VolumeGrowOption{DiskType: types.HardDriveType}
if got := dn.AvailableSpaceFor(option); got != 3 {
t.Errorf("AvailableSpaceFor = %d, want 3", got)
}
if got := dn.diskUsages.ToDiskInfo()[""].FreeVolumeCount; got != 3 {
t.Errorf("disk info FreeVolumeCount = %d, want 3", got)
}
}
func TestFreeVolumeCountSkipsOverCommittedServer(t *testing.T) {
topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false)
dc1 := topo.GetOrCreateDataCenter("dc1")
rack1 := dc1.GetOrCreateRack("rack1")
over := rack1.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5})
over.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 20})
healthy := rack1.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 10})
dc2 := topo.GetOrCreateDataCenter("dc2")
other := dc2.GetOrCreateRack("rack1").GetOrCreateDataNode("10.0.0.3", 8080, 18080, "", "", map[string]uint32{"": 4})
other.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 1})
option := &VolumeGrowOption{DiskType: types.HardDriveType}
for _, c := range []struct {
name string
node Node
want int64
}{
{"rack1", rack1, 10},
{"dc1", dc1, 10},
{"dc2", dc2, 3},
{"topology", topo, 13},
} {
if got := c.node.AvailableSpaceFor(option); got != c.want {
t.Errorf("%s AvailableSpaceFor = %d, want %d", c.name, got, c.want)
}
}
for i := 0; i < 20; i++ {
first, rest, err := topo.PickNodesByWeight(2, option, func(Node) error { return nil })
if err != nil {
t.Fatalf("topology PickNodesByWeight: %v", err)
}
if picked := map[Node]bool{first: true, rest[0]: true}; !picked[dc1] || !picked[dc2] {
t.Fatalf("topology picked %s and %s, want dc1 and dc2", first.Id(), rest[0].Id())
}
first, _, err = rack1.PickNodesByWeight(1, option, func(Node) error { return nil })
if err != nil {
t.Fatalf("rack1 PickNodesByWeight: %v", err)
}
if first != Node(healthy) {
t.Fatalf("rack1 picked %s, want %s", first.Id(), healthy.Id())
}
}
assigned := map[*DataNode]int{}
for r := int64(0); r < 13; r++ {
dn, err := topo.ReserveOneVolume(r, option)
if err != nil {
t.Fatalf("ReserveOneVolume(%d): %v", r, err)
}
assigned[dn]++
}
if assigned[over] != 0 || assigned[healthy] == 0 {
t.Errorf("ReserveOneVolume spread over=%d healthy=%d other=%d, want the healthy servers only",
assigned[over], assigned[healthy], assigned[other])
}
}
+5 -2
View File
@@ -53,8 +53,11 @@ func TestMaintenanceModeReportsNoFreeSlots(t *testing.T) {
if _, ok := dn.TryReserveCapacity(types.HardDriveType, 1); ok {
t.Error("reserved capacity on a server in maintenance")
}
if got := rack.AvailableSpaceFor(option); got != 10 {
t.Errorf("rack AvailableSpaceFor = %d, want 10: the node's capacity still rolls up to its parents", got)
if got := rack.CapacityFor(option); got != 10 {
t.Errorf("rack CapacityFor = %d, want 10: the node's capacity still rolls up to its parents", got)
}
if got := rack.AvailableSpaceFor(option); got != 0 {
t.Errorf("rack AvailableSpaceFor = %d, want 0: its only node is in maintenance", got)
}
if !dn.SetMaintenanceMode(false) {
+18 -10
View File
@@ -310,9 +310,7 @@ func (n *NodeImpl) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts {
}
// inMaintenanceMode is true for a data node whose volume server is in
// maintenance mode. Racks and data centers never are: their rolled-up counters
// still include such a node, so their free-slot totals may exceed what their
// children will actually hand out; see reserveOneVolumeInternal.
// maintenance mode. Racks and data centers never are.
func (n *NodeImpl) inMaintenanceMode() bool {
dn, ok := n.value.(*DataNode)
return ok && dn.InMaintenanceMode()
@@ -320,18 +318,29 @@ func (n *NodeImpl) inMaintenanceMode() bool {
// AvailableSpaceFor is the free volume slots on this node for the option's disk
// type. A data node in maintenance mode reports none: it is being drained, so
// it is neither a volume-growth candidate nor reservable.
// it is neither a volume-growth candidate nor reservable. A data node with more
// volumes than slots also reports none, and racks, data centers and the
// topology sum their children, so it cannot hide the free slots of others.
func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 {
if n.inMaintenanceMode() {
return 0
}
if !n.IsDataNode() {
n.RLock()
defer n.RUnlock()
var free int64
for _, child := range n.children {
free += child.AvailableSpaceFor(option)
}
return free
}
t := n.diskUsages.getDisk(option.DiskType)
if t == nil {
return 0
}
freeVolumeSlotCount := atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) - atomic.LoadInt64(&t.volumeCount)
freeVolumeSlotCount -= erasure_coding.VolumeSlots(atomic.LoadInt64(&t.ecShardCount))
return freeVolumeSlotCount
return max(0, freeVolumeSlotCount)
}
// CapacityFor is the total registered volume slots for the option's disk type;
@@ -438,11 +447,10 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u
}
return node.AvailableSpaceFor(option)
}
// The caller draws r from this node's rolled-up free slots, which still
// count children the walk below skips: a data node in maintenance mode
// reports no space, and an over-committed one reports less than zero. Fold
// r into the space that is actually on offer so it cannot walk off the end
// and fail with slots still free.
// The caller drew r from this node's free slots, which can exceed what the
// children offer once their reservations or a newer heartbeat are counted.
// Fold r into the space that is actually on offer so it cannot walk off the
// end and fail with slots still free.
var eligible int64
for _, node := range n.children {
if freeSpace := freeSpaceOf(node); freeSpace > 0 {