diff --git a/weed/topology/disk.go b/weed/topology/disk.go index 483c2f3679..85c97bc623 100644 --- a/weed/topology/disk.go +++ b/weed/topology/disk.go @@ -188,7 +188,7 @@ func (a *DiskUsageCounts) snapshot() DiskUsageCounts { func (a *DiskUsageCounts) FreeSpace() int64 { u := a.snapshot() - return u.maxVolumeCount + u.remoteVolumeCount - u.volumeCount - erasure_coding.VolumeSlots(u.ecShardCount) + return max(0, u.maxVolumeCount+u.remoteVolumeCount-u.volumeCount-erasure_coding.VolumeSlots(u.ecShardCount)) } func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts { diff --git a/weed/topology/free_volume_count_test.go b/weed/topology/free_volume_count_test.go new file mode 100644 index 0000000000..92a61837ca --- /dev/null +++ b/weed/topology/free_volume_count_test.go @@ -0,0 +1,106 @@ +package topology + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// An over-committed node (more volumes registered than it has slots for) must +// report zero free slots, not a negative count that drags the cluster total +// below the slots healthy servers can still offer. +func TestFreeVolumeCountNeverNegative(t *testing.T) { + topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5}) + // 8 volumes registered on a server configured for 5: 5 + 0 - 8 = -3. + dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 8}) + + option := &VolumeGrowOption{DiskType: types.HardDriveType} + if got := dn.AvailableSpaceFor(option); got != 0 { + t.Errorf("data node AvailableSpaceFor = %d, want 0 (over-committed slots must not go negative)", got) + } + if got := topo.AvailableSpaceFor(option); got != 0 { + t.Errorf("topology AvailableSpaceFor = %d, want 0", got) + } + if got := dn.diskUsages.FreeSpace(); got != 0 { + t.Errorf("diskUsages.FreeSpace = %d, want 0", got) + } + // Planners add refunds and moves to the published count, so it keeps the deficit. + if got := dn.diskUsages.ToDiskInfo()[""].FreeVolumeCount; got != -3 { + t.Errorf("disk info FreeVolumeCount = %d, want -3", got) + } +} + +func TestFreeVolumeCountPositiveCaseUnchanged(t *testing.T) { + topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5}) + dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 2}) + + option := &VolumeGrowOption{DiskType: types.HardDriveType} + if got := dn.AvailableSpaceFor(option); got != 3 { + t.Errorf("AvailableSpaceFor = %d, want 3", got) + } + if got := dn.diskUsages.ToDiskInfo()[""].FreeVolumeCount; got != 3 { + t.Errorf("disk info FreeVolumeCount = %d, want 3", got) + } +} + +func TestFreeVolumeCountSkipsOverCommittedServer(t *testing.T) { + topo := NewTopology("free-slots", nil, 32*1024*1024*1024, 5, false) + dc1 := topo.GetOrCreateDataCenter("dc1") + rack1 := dc1.GetOrCreateRack("rack1") + over := rack1.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 5}) + over.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 20}) + healthy := rack1.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 10}) + dc2 := topo.GetOrCreateDataCenter("dc2") + other := dc2.GetOrCreateRack("rack1").GetOrCreateDataNode("10.0.0.3", 8080, 18080, "", "", map[string]uint32{"": 4}) + other.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 1}) + + option := &VolumeGrowOption{DiskType: types.HardDriveType} + for _, c := range []struct { + name string + node Node + want int64 + }{ + {"rack1", rack1, 10}, + {"dc1", dc1, 10}, + {"dc2", dc2, 3}, + {"topology", topo, 13}, + } { + if got := c.node.AvailableSpaceFor(option); got != c.want { + t.Errorf("%s AvailableSpaceFor = %d, want %d", c.name, got, c.want) + } + } + + for i := 0; i < 20; i++ { + first, rest, err := topo.PickNodesByWeight(2, option, func(Node) error { return nil }) + if err != nil { + t.Fatalf("topology PickNodesByWeight: %v", err) + } + if picked := map[Node]bool{first: true, rest[0]: true}; !picked[dc1] || !picked[dc2] { + t.Fatalf("topology picked %s and %s, want dc1 and dc2", first.Id(), rest[0].Id()) + } + first, _, err = rack1.PickNodesByWeight(1, option, func(Node) error { return nil }) + if err != nil { + t.Fatalf("rack1 PickNodesByWeight: %v", err) + } + if first != Node(healthy) { + t.Fatalf("rack1 picked %s, want %s", first.Id(), healthy.Id()) + } + } + + assigned := map[*DataNode]int{} + for r := int64(0); r < 13; r++ { + dn, err := topo.ReserveOneVolume(r, option) + if err != nil { + t.Fatalf("ReserveOneVolume(%d): %v", r, err) + } + assigned[dn]++ + } + if assigned[over] != 0 || assigned[healthy] == 0 { + t.Errorf("ReserveOneVolume spread over=%d healthy=%d other=%d, want the healthy servers only", + assigned[over], assigned[healthy], assigned[other]) + } +} diff --git a/weed/topology/maintenance_mode_test.go b/weed/topology/maintenance_mode_test.go index 8977efa12e..0003eea980 100644 --- a/weed/topology/maintenance_mode_test.go +++ b/weed/topology/maintenance_mode_test.go @@ -53,8 +53,11 @@ func TestMaintenanceModeReportsNoFreeSlots(t *testing.T) { if _, ok := dn.TryReserveCapacity(types.HardDriveType, 1); ok { t.Error("reserved capacity on a server in maintenance") } - if got := rack.AvailableSpaceFor(option); got != 10 { - t.Errorf("rack AvailableSpaceFor = %d, want 10: the node's capacity still rolls up to its parents", got) + if got := rack.CapacityFor(option); got != 10 { + t.Errorf("rack CapacityFor = %d, want 10: the node's capacity still rolls up to its parents", got) + } + if got := rack.AvailableSpaceFor(option); got != 0 { + t.Errorf("rack AvailableSpaceFor = %d, want 0: its only node is in maintenance", got) } if !dn.SetMaintenanceMode(false) { diff --git a/weed/topology/node.go b/weed/topology/node.go index be642027e7..4e2132c6a1 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -310,9 +310,7 @@ func (n *NodeImpl) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts { } // inMaintenanceMode is true for a data node whose volume server is in -// maintenance mode. Racks and data centers never are: their rolled-up counters -// still include such a node, so their free-slot totals may exceed what their -// children will actually hand out; see reserveOneVolumeInternal. +// maintenance mode. Racks and data centers never are. func (n *NodeImpl) inMaintenanceMode() bool { dn, ok := n.value.(*DataNode) return ok && dn.InMaintenanceMode() @@ -320,18 +318,29 @@ func (n *NodeImpl) inMaintenanceMode() bool { // AvailableSpaceFor is the free volume slots on this node for the option's disk // type. A data node in maintenance mode reports none: it is being drained, so -// it is neither a volume-growth candidate nor reservable. +// it is neither a volume-growth candidate nor reservable. A data node with more +// volumes than slots also reports none, and racks, data centers and the +// topology sum their children, so it cannot hide the free slots of others. func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 { if n.inMaintenanceMode() { return 0 } + if !n.IsDataNode() { + n.RLock() + defer n.RUnlock() + var free int64 + for _, child := range n.children { + free += child.AvailableSpaceFor(option) + } + return free + } t := n.diskUsages.getDisk(option.DiskType) if t == nil { return 0 } freeVolumeSlotCount := atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) - atomic.LoadInt64(&t.volumeCount) freeVolumeSlotCount -= erasure_coding.VolumeSlots(atomic.LoadInt64(&t.ecShardCount)) - return freeVolumeSlotCount + return max(0, freeVolumeSlotCount) } // CapacityFor is the total registered volume slots for the option's disk type; @@ -438,11 +447,10 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u } return node.AvailableSpaceFor(option) } - // The caller draws r from this node's rolled-up free slots, which still - // count children the walk below skips: a data node in maintenance mode - // reports no space, and an over-committed one reports less than zero. Fold - // r into the space that is actually on offer so it cannot walk off the end - // and fail with slots still free. + // The caller drew r from this node's free slots, which can exceed what the + // children offer once their reservations or a newer heartbeat are counted. + // Fold r into the space that is actually on offer so it cannot walk off the + // end and fail with slots still free. var eligible int64 for _, node := range n.children { if freeSpace := freeSpaceOf(node); freeSpace > 0 {