Files
seaweedfs/weed/topology/maintenance_mode_test.go
T
5681be241e topology: never report negative free volume slots (#12698)
* topology: never report negative free volume slots

Free volume slots are computed as max + remote - volume - ecSlots. When phantom or over-registered volumes push the count past the configured max, the result is negative and rolls up into the cluster total, causing volume grow and assign to fail with a negative 'volumes left' count even while healthy servers have capacity. Clamp every free-volume count at zero so an over-committed server contributes no slots instead of subtracting from the cluster.

* topology: drop the issue reference from the free-slot test comment

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test free slots next to an over-committed server

A server with more volumes than slots must not hide the free slots of
the other servers in its rack, data center or topology.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: sum free slots per data node in racks, data centers and topology

Clamping the rolled-up counters only clamped their sum, so one server with
more volumes than slots still hid the free slots of every other server in
its rack, data center and topology. Interior nodes now add up their
children's free slots, so each data node counts as zero at worst. A data
node in maintenance mode no longer adds slots to its parents either.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test that disk info keeps an over-committed deficit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: keep the signed free volume count in disk info

EC encode retries and the balance planners add refunds and planned moves
to the published FreeVolumeCount. Clamping it at zero dropped an
over-committed disk's deficit, so a refund could make a full disk look
free. Volume growth already reads the clamped AvailableSpaceFor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test placement next to an over-committed server

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: accept either data center order in the placement spread check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 17:02:01 -08:00

381 lines
15 KiB
Go

package topology
import (
"slices"
"testing"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// addMaintenanceTestNode links a data node with maxVol free slots under rack.
func addMaintenanceTestNode(rack *Rack, id string, maxVol int64) *DataNode {
dn := NewDataNode(id)
dn.Ip = id
rack.LinkChildNode(dn)
dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{maxVolumeCount: maxVol})
return dn
}
func newMaintenanceTestTopology() (*Topology, *DataCenter) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := NewDataCenter("dc1")
topo.LinkChildNode(dc)
return topo, dc
}
func TestMaintenanceModeReportsNoFreeSlots(t *testing.T) {
rack := NewRack("rack1")
dn := addMaintenanceTestNode(rack, "server1", 10)
option := &VolumeGrowOption{DiskType: types.HardDriveType}
if got := dn.AvailableSpaceFor(option); got != 10 {
t.Fatalf("AvailableSpaceFor = %d, want 10", got)
}
if !dn.SetMaintenanceMode(true) {
t.Fatal("first switch should report a change")
}
if dn.SetMaintenanceMode(true) {
t.Fatal("repeating the same value should not report a change")
}
if !dn.InMaintenanceMode() {
t.Fatal("InMaintenanceMode = false after switching on")
}
if got := dn.AvailableSpaceFor(option); got != 0 {
t.Errorf("AvailableSpaceFor in maintenance = %d, want 0: no new volumes go to a server in maintenance", got)
}
if got := dn.AvailableSpaceForReservation(option); got != 0 {
t.Errorf("AvailableSpaceForReservation in maintenance = %d, want 0", got)
}
if _, ok := dn.TryReserveCapacity(types.HardDriveType, 1); ok {
t.Error("reserved capacity on a server in maintenance")
}
if got := rack.CapacityFor(option); got != 10 {
t.Errorf("rack CapacityFor = %d, want 10: the node's capacity still rolls up to its parents", got)
}
if got := rack.AvailableSpaceFor(option); got != 0 {
t.Errorf("rack AvailableSpaceFor = %d, want 0: its only node is in maintenance", got)
}
if !dn.SetMaintenanceMode(false) {
t.Fatal("switching off should report a change")
}
if got := dn.AvailableSpaceFor(option); got != 10 {
t.Errorf("AvailableSpaceFor after maintenance = %d, want 10", got)
}
}
// A server in maintenance mode is the one being evacuated, so volume growth in
// its rack must land every replica on its siblings even when it has the most
// free slots by far.
func TestVolumeGrowthSkipsMaintenanceNodeInSameRack(t *testing.T) {
topo, dc := newMaintenanceTestTopology()
rack := NewRack("rack1")
dc.LinkChildNode(rack)
busy := addMaintenanceTestNode(rack, "busy", 1000)
busy.SetMaintenanceMode(true)
addMaintenanceTestNode(rack, "small1", 2)
addMaintenanceTestNode(rack, "small2", 2)
vg := NewDefaultVolumeGrowth()
rp, _ := super_block.NewReplicaPlacementFromString("001")
option := &VolumeGrowOption{ReplicaPlacement: rp, DiskType: types.HardDriveType}
for i := 0; i < 50; i++ {
servers, _, err := vg.findEmptySlotsForOneVolume(topo, option, false)
if err != nil {
t.Fatalf("iteration %d: %v", i, err)
}
if len(servers) != 2 {
t.Fatalf("iteration %d: got %d servers, want 2", i, len(servers))
}
for _, s := range servers {
if s.Id() == busy.Id() {
t.Fatalf("iteration %d: picked the server in maintenance mode", i)
}
}
}
// Pinning the request to the maintenance node must fail rather than
// create a volume there.
rp0, _ := super_block.NewReplicaPlacementFromString("000")
pinned := &VolumeGrowOption{ReplicaPlacement: rp0, DiskType: types.HardDriveType, DataNode: "busy"}
if _, _, err := vg.findEmptySlotsForOneVolume(topo, pinned, false); err == nil {
t.Fatal("a volume pinned to a server in maintenance mode must not be created")
}
}
// The replica on another rack is chosen by walking a random offset through the
// rack's rolled-up free slots, which still count a node in maintenance. The walk
// has to land on an eligible sibling every time, not fall off the end.
func TestVolumeGrowthSkipsMaintenanceNodeInOtherRack(t *testing.T) {
topo, dc := newMaintenanceTestTopology()
rack1 := NewRack("rack1")
dc.LinkChildNode(rack1)
addMaintenanceTestNode(rack1, "r1n1", 100)
rack2 := NewRack("rack2")
dc.LinkChildNode(rack2)
busy := addMaintenanceTestNode(rack2, "r2busy", 1000)
busy.SetMaintenanceMode(true)
ok := addMaintenanceTestNode(rack2, "r2ok", 1)
vg := NewDefaultVolumeGrowth()
rp, _ := super_block.NewReplicaPlacementFromString("010")
option := &VolumeGrowOption{ReplicaPlacement: rp, DiskType: types.HardDriveType}
for i := 0; i < 50; i++ {
servers, _, err := vg.findEmptySlotsForOneVolume(topo, option, false)
if err != nil {
t.Fatalf("iteration %d: %v", i, err)
}
if len(servers) != 2 {
t.Fatalf("iteration %d: got %d servers, want 2", i, len(servers))
}
onSibling := 0
for _, s := range servers {
if s.Id() == busy.Id() {
t.Fatalf("iteration %d: picked the server in maintenance mode", i)
}
if s.Id() == ok.Id() {
onSibling++
}
}
if onSibling != 1 {
t.Fatalf("iteration %d: the other-rack replica must land on the eligible sibling, got %v", i, servers)
}
}
}
// A replica on a server in maintenance mode takes no writes, so its volume
// leaves the writable list while the mode is on and returns once it is off.
func TestMaintenanceModeTogglesVolumeWritability(t *testing.T) {
topo, dc := newMaintenanceTestTopology()
rack := NewRack("rack1")
dc.LinkChildNode(rack)
dn1 := addMaintenanceTestNode(rack, "dn1", 10)
dn2 := addMaintenanceTestNode(rack, "dn2", 10)
rp, _ := super_block.NewReplicaPlacementFromString("001")
shared := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
dn1.AddOrUpdateVolume(shared)
dn2.AddOrUpdateVolume(shared)
topo.RegisterVolumeLayout(shared, dn1)
topo.RegisterVolumeLayout(shared, dn2)
rp0, _ := super_block.NewReplicaPlacementFromString("000")
alone := storage.VolumeInfo{Id: 2, Size: 100, ReplicaPlacement: rp0, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
dn2.AddOrUpdateVolume(alone)
topo.RegisterVolumeLayout(alone, dn2)
vlShared := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
vlAlone := topo.GetVolumeLayout("", rp0, needle.EMPTY_TTL, types.HardDriveType)
expectWritables := func(step string, vl *VolumeLayout, want ...needle.VolumeId) {
t.Helper()
got := vl.CloneWritableVolumes()
slices.Sort(got)
if !slices.Equal(got, want) {
t.Errorf("%s: writables = %v, want %v", step, got, want)
}
}
expectWritables("initial shared", vlShared, 1)
expectWritables("initial alone", vlAlone, 2)
if !topo.SetDataNodeMaintenanceMode(dn1, true) {
t.Fatal("switching dn1 on should report a change")
}
expectWritables("dn1 in maintenance: a volume with a replica there takes no writes", vlShared)
expectWritables("dn1 in maintenance: volumes elsewhere are unaffected", vlAlone, 2)
if locations := vlShared.Lookup(1); len(locations) != 2 {
t.Errorf("reads must still see every replica, got %v", locations)
}
if topo.SetDataNodeMaintenanceMode(dn1, true) {
t.Error("repeating the same state should not report a change")
}
if !topo.SetDataNodeMaintenanceMode(dn1, false) {
t.Fatal("switching dn1 off should report a change")
}
expectWritables("dn1 back: the volume is writable again", vlShared, 1)
topo.SetDataNodeMaintenanceMode(dn2, true)
expectWritables("dn2 in maintenance: shared", vlShared)
expectWritables("dn2 in maintenance: alone", vlAlone)
}
// A vacuum commit or a mark-writable that lands after the server entered
// maintenance mode must not put the volume back on the writable list. Both
// used to check only the replica count, and since heartbeats carry only changed
// volumes, nothing would have taken it off again.
func TestMaintenanceModeHoldsThroughVacuumAndMarkWritable(t *testing.T) {
topo, dc := newMaintenanceTestTopology()
rack := NewRack("rack1")
dc.LinkChildNode(rack)
dn1 := addMaintenanceTestNode(rack, "dn1", 10)
dn2 := addMaintenanceTestNode(rack, "dn2", 10)
rp, _ := super_block.NewReplicaPlacementFromString("001")
vi := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
dn1.AddOrUpdateVolume(vi)
dn2.AddOrUpdateVolume(vi)
topo.RegisterVolumeLayout(vi, dn1)
topo.RegisterVolumeLayout(vi, dn2)
vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
expectWritableCount := func(step string, want int) {
t.Helper()
if got, _ := vl.GetWritableVolumeCount(); got != want {
t.Errorf("%s: writable count = %d, want %d", step, got, want)
}
}
// vacuum: taken off the writable list before compaction, dn1 enters
// maintenance meanwhile, then every replica commits
vl.DrainAndRemoveFromWritable(1)
topo.SetDataNodeMaintenanceMode(dn1, true)
for _, dn := range []*DataNode{dn1, dn2} {
if vl.SetVolumeAvailable(dn, 1, 100, false, false, false) {
t.Errorf("vacuum commit on %s made the volume writable with a replica in maintenance", dn.Id())
}
}
expectWritableCount("after vacuum commit", 0)
topo.SetDataNodeMaintenanceMode(dn1, false)
expectWritableCount("after maintenance", 1)
// mark-writable from a vacuum worker, after dn1 entered maintenance
vl.SetVolumeReadOnly(dn2, 1)
topo.SetDataNodeMaintenanceMode(dn1, true)
if vl.SetVolumeWritable(dn2, 1) {
t.Error("mark-writable made the volume writable with a replica in maintenance")
}
expectWritableCount("after mark-writable", 0)
topo.SetDataNodeMaintenanceMode(dn1, false)
expectWritableCount("after maintenance", 1)
}
// A volume server notifies the master the moment it flips a volume between
// read-only and writable, ahead of the heartbeat that repeats the flag. The
// master must act on the notification, not on its heartbeat copy: mark-writable
// takes effect at once, and a maintenance toggle landing between a mark-readonly
// and its heartbeat must not put the volume back.
func TestMarkReadOnlyAndWritableAheadOfHeartbeat(t *testing.T) {
topo, dc := newMaintenanceTestTopology()
rack := NewRack("rack1")
dc.LinkChildNode(rack)
dn1 := addMaintenanceTestNode(rack, "dn1", 10)
dn2 := addMaintenanceTestNode(rack, "dn2", 10)
rp, _ := super_block.NewReplicaPlacementFromString("001")
vi := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
dn1.AddOrUpdateVolume(vi)
dn2.AddOrUpdateVolume(vi)
topo.RegisterVolumeLayout(vi, dn1)
topo.RegisterVolumeLayout(vi, dn2)
vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
expectWritableCount := func(step string, want int) {
t.Helper()
if got, _ := vl.GetWritableVolumeCount(); got != want {
t.Errorf("%s: writable count = %d, want %d", step, got, want)
}
}
// the heartbeat has reported dn2's replica read-only
readOnly := vi
readOnly.ReadOnly = true
dn2.AddOrUpdateVolume(readOnly)
vl.EnsureCorrectWritables(&readOnly)
expectWritableCount("heartbeat says read-only", 0)
// dn2 marks it writable and notifies, before the next heartbeat
if !vl.SetVolumeWritable(dn2, 1) {
t.Error("mark-writable from the replica's own server should take effect at once")
}
expectWritableCount("after mark-writable", 1)
if v, err := dn2.GetVolumesById(1); err != nil || v.ReadOnly {
t.Errorf("the node's record should say writable, got %+v, %v", v, err)
}
// dn2 marks it read-only and notifies; dn1 leaves maintenance before the
// heartbeat repeats the flag
vl.SetVolumeReadOnly(dn2, 1)
expectWritableCount("after mark-readonly", 0)
topo.SetDataNodeMaintenanceMode(dn1, true)
topo.SetDataNodeMaintenanceMode(dn1, false)
expectWritableCount("re-evaluated before the heartbeat", 0)
}
// SetDataNodeMaintenanceMode walks a snapshot of the node's volumes, so a
// concurrent disconnect can have removed one from its layout by the time it is
// re-evaluated. That must be a no-op, not a crash or a resurrected entry.
func TestMaintenanceModeAfterVolumeLeftLayout(t *testing.T) {
topo, dc := newMaintenanceTestTopology()
rack := NewRack("rack1")
dc.LinkChildNode(rack)
dn := addMaintenanceTestNode(rack, "dn1", 10)
rp, _ := super_block.NewReplicaPlacementFromString("000")
vi := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
dn.AddOrUpdateVolume(vi)
topo.RegisterVolumeLayout(vi, dn)
vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
vl.UnRegisterVolume(&vi, dn)
for _, on := range []bool{true, false} {
if !topo.SetDataNodeMaintenanceMode(dn, on) {
t.Fatalf("SetDataNodeMaintenanceMode(%v) should report a change", on)
}
if got, _ := vl.GetWritableVolumeCount(); got != 0 {
t.Errorf("maintenance %v: writable count = %d, want 0", on, got)
}
if locations := vl.Lookup(1); locations != nil {
t.Errorf("maintenance %v: re-evaluating a departed volume re-created its location list: %v", on, locations)
}
}
}
// GrowthBlocker reads room as well as shape: a location whose racks are all
// full cannot host the replica placement, and the rack or data node the
// option names must have room for the first replica itself. The assign
// layer, not this probe, makes the exception for a pending grow spending
// those slots on the request's own volume.
func TestGrowthBlockerNeedsRoom(t *testing.T) {
topo, dc1 := newMaintenanceTestTopology()
rack1 := NewRack("rack1")
dc1.LinkChildNode(rack1)
full1 := addMaintenanceTestNode(rack1, "s1", 5)
full1.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 8})
rack2 := NewRack("rack2")
dc1.LinkChildNode(rack2)
full2 := addMaintenanceTestNode(rack2, "s2", 3)
full2.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: 3})
rp010, _ := super_block.NewReplicaPlacementFromString("010")
if err := topo.GrowthBlocker(&VolumeGrowOption{ReplicaPlacement: rp010, DataCenter: "dc1"}); err == nil {
t.Fatal("two full racks can never place 010")
}
// One freed slot per rack puts the placement back in reach.
full1.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: -4})
full2.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{volumeCount: -3})
if err := topo.GrowthBlocker(&VolumeGrowOption{ReplicaPlacement: rp010, DataCenter: "dc1"}); err != nil {
t.Fatalf("one free slot per rack fits 010: %v", err)
}
if err := topo.GrowthBlocker(&VolumeGrowOption{ReplicaPlacement: rp010, Rack: "rack1"}); err != nil {
t.Fatalf("rack1 hosts the first 010 replica, rack2 the second: %v", err)
}
if err := topo.GrowthBlocker(&VolumeGrowOption{ReplicaPlacement: rp010, Rack: "rack3"}); err == nil {
t.Fatal("no rack named rack3: 010 can never place its first replica")
}
topo2, dc2 := newMaintenanceTestTopology()
rack := NewRack("rack1")
dc2.LinkChildNode(rack)
addMaintenanceTestNode(rack, "s1", 10)
if err := topo2.GrowthBlocker(&VolumeGrowOption{ReplicaPlacement: rp010, DataCenter: "dc1"}); err == nil {
t.Fatal("a single rack can never place 010")
}
if err := topo2.GrowthBlocker(&VolumeGrowOption{ReplicaPlacement: rp010, DataCenter: "dc9"}); err == nil {
t.Fatal("a missing data center can never place 010")
}
}