ecbalancer: cap total shards per rack in Plan (#11623)

* ecbalancer: cap total shards per rack in Plan Plan caps data and parity per rack separately (ceil(data/racks) and ceil(parity/racks)), so with 10+4 over 8 racks a rack can legally hold 2 data + 1 parity. When a rack is one disk, losing two such racks loses 6 of 14 shards and the volume can't be read. The cross-rack phase now also caps each rack's TOTAL shards of a volume, sized with Place's rackTotalCap: ceil(shards/racks) unless the racks lack room, counting a rack's own shards of the volume as room since Plan can move them. - A rack above the cap sheds parity until it fits. Those shards may go to a data-bearing rack, and when no rack is under the parity cap they fall back to a rack under the total cap. #11438's non-overflow candidates keep moving only to data-free racks. - No cross-rack move lands on a rack at the cap. - A rack above the cap triggers balancing regardless of the imbalance threshold. - The fallback applies only while the source rack is above the cap; otherwise the next Plan moves the shard back. - Options.RackTotalCapRaised reports volumes whose cap had to be raised above the even share; the worker and the shell log it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * ecbalancer: finish the rack cap in one Plan, count SameRackCount room Review follow-ups. - The data pass can run out of destinations before the parity pass frees slots elsewhere, which left a data-heavy rack above the cap after one Plan (a one-shot shell balance stops there). The cross-rack phase now repeats while a rack is above the cap and the last round moved something. A shard moves at most once per plan, since each move runs as its own task. - planRackTotalCap bounds each node's room by what SameRackCount still allows, as Place does. Counting the raw free slots sized the cap too low, so RackTotalCapRaised missed volumes it should report. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-09 21:38:19 +01:00 · 2026-10-07 20:35:25 +08:00
1 parent 576837f1b2
commit 0bcebf708c
@@ -106,6 +106,12 @@ type Options struct {
 	// heterogeneous-capacity racks; when false, by raw shard count. Both the worker
 	// and the shell enable it; the two metrics agree when capacities are uniform.
 	GlobalUtilizationBased bool
+	// RackTotalCapRaised, when set, is called for each volume whose
+	// total-shards-per-rack cap had to be raised above ceil(shards/racks)
+	// because the racks lack room for an even spread. Losing a rack then loses
+	// more shards than an even spread would, so callers log it as a degraded
+	// durability floor. It is called on every Plan while the condition holds.
+	RackTotalCapRaised func(collection string, vid uint32, rackCap, evenCap int)
 }

 // move is the internal form carrying node pointers; converted to Move at the end.
@@ -249,7 +255,7 @@ func Plan(topo *Topology, opts Options) []Move {
 			all = append(all, m...)
 		}
 		for _, vk := range byCollection[collection] {
-			m := detectCrossRackImbalance(vk, nodes, racks, opts.DiskType, opts.ImbalanceThreshold, dataShardsByVolume[vk], parityShardsByVolume[vk], opts.ReplicaPlacement)
+			m := detectCrossRackImbalance(vk, nodes, racks, opts.DiskType, opts.ImbalanceThreshold, dataShardsByVolume[vk], parityShardsByVolume[vk], opts.ReplicaPlacement, opts.RackTotalCapRaised)
 			applyMovesToTopology(m, racks)
 			all = append(all, m...)
 		}
@@ -343,42 +349,108 @@ func detectDuplicateShards(vk volKey, nodes map[string]*Node) []*move {
 }

 // detectCrossRackImbalance spreads a volume's shards across racks in two passes
-// (data, then parity with anti-affinity to data-bearing racks). Returns nil if
-// the overall distribution is below the imbalance threshold.
-func detectCrossRackImbalance(vk volKey, nodes map[string]*Node, racks map[string]*rack, diskType string, threshold float64, dataShards, parityShards int, rp *super_block.ReplicaPlacement) []*move {
+// (data, then parity with anti-affinity to data-bearing racks), keeping each
+// rack's total within planRackTotalCap. Returns nil if the overall distribution
+// is below the imbalance threshold and no rack is above the total cap.
+func detectCrossRackImbalance(vk volKey, nodes map[string]*Node, racks map[string]*rack, diskType string, threshold float64, dataShards, parityShards int, rp *super_block.ReplicaPlacement, onCapRaised func(collection string, vid uint32, rackCap, evenCap int)) []*move {
 	numRacks := len(racks)
 	if numRacks <= 1 {
 		return nil
 	}

+	// The per-type caps below spread data and parity independently, so together
+	// they can still stack e.g. 2 data + 1 parity on one rack. Cap the TOTAL per
+	// rack too, at the lowest value the racks' capacity allows.
+	rackShardCount := countShardsByRack(vk, nodes)
+	totalShards := 0
+	for _, n := range rackShardCount {
+		totalShards += n
+	}
+	totalCap := planRackTotalCap(vk, racks, rackShardCount, totalShards, rp)
+	if evenCap := ceilDivide(totalShards, numRacks); totalCap > evenCap && onCapRaised != nil {
+		onCapRaised(vk.collection, vk.vid, totalCap, evenCap)
+	}
+	aboveTotalCap := func() bool {
+		for _, n := range rackShardCount {
+			if n > totalCap {
+				return true
+			}
+		}
+		return false
+	}
+
 	// Gate on per-type spread: act when data OR parity shards are unevenly
 	// distributed across racks, even if the per-rack totals happen to be even.
+	// A rack above the total cap is a durability bound, not skew, so it bypasses
+	// the threshold.
 	gateData, gateParity := shardsByGroup(vk, nodes, dataShards, func(n *Node) string { return n.rack })
-	if !typeImbalanced(gateData, numRacks, threshold) && !typeImbalanced(gateParity, numRacks, threshold) {
+	if !aboveTotalCap() && !typeImbalanced(gateData, numRacks, threshold) && !typeImbalanced(gateParity, numRacks, threshold) {
 		return nil
 	}

-	rackShardCount := countShardsByRack(vk, nodes)
 	var moves []*move
+	// A shard moves at most once per plan: each move runs as its own task,
+	// possibly in parallel with the others, so a second hop could start before
+	// the first has landed.
+	moved := make(map[int]bool)
+	for {
+		before := len(moves)

-	dataPerRack, _ := shardsByGroup(vk, nodes, dataShards, func(n *Node) string { return n.rack })
-	moves = append(moves, balanceShardTypeAcrossRacks(vk, nodes, racks, diskType, dataShards,
-		dataPerRack, rackShardCount, ceilDivide(dataShards, numRacks), nil, rp)...)
+		// The data pass leaves the total cap out (0): ceil(data/racks) <= totalCap
+		// already bounds data, and rackShardCount still includes parity that the
+		// parity pass is about to shed, so a total bound here would block data
+		// moves that fit once that parity has gone.
+		dataPerRack, _ := shardsByGroup(vk, nodes, dataShards, func(n *Node) string { return n.rack })
+		moves = append(moves, balanceShardTypeAcrossRacks(vk, nodes, racks, diskType, dataShards,
+			dataPerRack, rackShardCount, ceilDivide(dataShards, numRacks), 0, nil, rp, moved)...)

-	dataPerRack, parityPerRack := shardsByGroup(vk, nodes, dataShards, func(n *Node) string { return n.rack })
-	antiAffinity := make(map[string]bool)
-	for rackID, shards := range dataPerRack {
-		if len(shards) > 0 {
-			antiAffinity[rackID] = true
+		dataPerRack, parityPerRack := shardsByGroup(vk, nodes, dataShards, func(n *Node) string { return n.rack })
+		antiAffinity := make(map[string]bool)
+		for rackID, shards := range dataPerRack {
+			if len(shards) > 0 {
+				antiAffinity[rackID] = true
+			}
+		}
+		moves = append(moves, balanceShardTypeAcrossRacks(vk, nodes, racks, diskType, dataShards,
+			parityPerRack, rackShardCount, ceilDivide(parityShards, numRacks), totalCap, antiAffinity, rp, moved)...)
+
+		// The data pass can run out of destinations before the parity pass frees
+		// slots on other racks. While a rack is still above the total cap, run
+		// another round over the shards not moved yet. A round that continues
+		// moved at least one more shard, so this ends.
+		if len(moves) == before || !aboveTotalCap() {
+			return moves
 		}
 	}
-	moves = append(moves, balanceShardTypeAcrossRacks(vk, nodes, racks, diskType, dataShards,
-		parityPerRack, rackShardCount, ceilDivide(parityShards, numRacks), antiAffinity, rp)...)
-
-	return moves
 }

-func balanceShardTypeAcrossRacks(vk volKey, nodes map[string]*Node, racks map[string]*rack, diskType string, dataShards int, shardsPerRack map[string][]int, rackShardCount map[string]int, maxPerRack int, antiAffinity map[string]bool, rp *super_block.ReplicaPlacement) []*move {
+// planRackTotalCap returns the lowest per-rack total that the racks can hold a
+// volume's totalShards under (see rackTotalCap). Plan may move any shard, so a
+// rack's shards of the volume count as room there rather than as held in place.
+// Moves of the volume shift slots between racks without changing a rack's
+// shards-plus-free-slots, so the cap holds for the whole cross-rack phase.
+// Under SameRackCount a node's free slots count only up to the shards of the
+// volume it may still take, the limit pickBestNodeForVolume enforces.
+func planRackTotalCap(vk volKey, racks map[string]*rack, rackShardCount map[string]int, totalShards int, rp *super_block.ReplicaPlacement) int {
+	room := make(map[string]int, len(racks))
+	for rk, r := range racks {
+		room[rk] = rackShardCount[rk]
+		if rp == nil || rp.SameRackCount <= 0 {
+			room[rk] += max(r.freeSlots, 0)
+			continue
+		}
+		for _, n := range r.nodes {
+			room[rk] += min(max(n.freeSlots, 0), max(rp.SameRackCount-volumeShardCount(n, vk), 0))
+		}
+	}
+	return rackTotalCap(sortedKeys(racks), nil, room, totalShards)
+}
+
+// balanceShardTypeAcrossRacks spreads one shard type across racks, at most
+// maxPerRack of the type per rack. A totalCap > 0 also bounds each rack's TOTAL
+// shards of the volume: racks above it shed this type, and no move lands on a
+// rack at it. Shards in moved stay put; the ones it moves are added to it.
+func balanceShardTypeAcrossRacks(vk volKey, nodes map[string]*Node, racks map[string]*rack, diskType string, dataShards int, shardsPerRack map[string][]int, rackShardCount map[string]int, maxPerRack, totalCap int, antiAffinity map[string]bool, rp *super_block.ReplicaPlacement, moved map[int]bool) []*move {
 	if maxPerRack < 1 {
 		maxPerRack = 1
 	}
@@ -392,14 +464,25 @@ func balanceShardTypeAcrossRacks(vk volKey, nodes map[string]*Node, racks map[st
 	var toMove []pending
 	for _, rackID := range rackKeys {
 		shards := append([]int(nil), shardsPerRack[rackID]...)
-		sort.Ints(shards)
+		// Shards this plan already moved go last, so the shed takes the others.
+		sort.Slice(shards, func(i, j int) bool {
+			if moved[shards[i]] != moved[shards[j]] {
+				return !moved[shards[i]]
+			}
+			return shards[i] < shards[j]
+		})
 		overflow := max(0, len(shards)-maxPerRack)
+		if totalCap > 0 {
+			// A rack above the total cap sheds this type until it fits, even
+			// when the type is within its own cap there.
+			overflow = max(overflow, min(len(shards), rackShardCount[rackID]-totalCap))
+		}
 		for i := 0; i < len(shards); i++ {
 			// A parity shard can fit the per-type cap yet share a rack with
 			// data while a data-free rack is empty; such candidates may only
 			// move to a rack without data.
 			avoidDataRack := i >= overflow
-			if avoidDataRack && !antiAffinity[rackID] {
+			if moved[shards[i]] || (avoidDataRack && !antiAffinity[rackID]) {
 				continue
 			}
 			if src := nodeInRackHoldingShard(nodes, rackID, vk, shards[i]); src != nil {
@@ -408,22 +491,32 @@ func balanceShardTypeAcrossRacks(vk volKey, nodes map[string]*Node, racks map[st
 		}
 	}

+	withinLimit := func(r string) bool {
+		if totalCap > 0 && rackShardCount[r] >= totalCap {
+			return false
+		}
+		if rp != nil && rp.DiffRackCount > 0 && rackShardCount[r] >= rp.DiffRackCount {
+			return false
+		}
+		return true
+	}
+
 	var moves []*move
 	for _, pm := range toMove {
-		destRack, ok := pickTarget(rackKeys, shardsPerRack, maxPerRack, antiAffinity,
-			func(r string) bool {
-				return r != pm.src.rack && racks[r].freeSlots > 0 &&
-					(!pm.avoidDataRack || !antiAffinity[r])
-			},
-			func(r string) bool {
-				if rp == nil {
-					return true
-				}
-				if rp.DiffRackCount > 0 && rackShardCount[r] >= rp.DiffRackCount {
-					return false
-				}
-				return true
-			})
+		hasRoom := func(r string) bool {
+			return r != pm.src.rack && racks[r].freeSlots > 0 &&
+				(!pm.avoidDataRack || !antiAffinity[r])
+		}
+		destRack, ok := pickTarget(rackKeys, shardsPerRack, maxPerRack, antiAffinity, hasRoom, withinLimit)
+		if !ok && totalCap > 0 && rackShardCount[pm.src.rack] > totalCap {
+			// No rack is under the per-type cap. A shard must not stay on a rack
+			// above the total cap while another rack is under it, so relax the
+			// per-type cap up to the total cap; anti-affinity stays a preference
+			// inside pickTarget. Only a source above the total cap may relax:
+			// otherwise the destination just takes over the per-type overflow
+			// and the next Plan moves it back.
+			destRack, ok = pickTarget(rackKeys, shardsPerRack, totalCap, antiAffinity, hasRoom, withinLimit)
+		}
 		if !ok {
 			continue
 		}
@@ -455,6 +548,7 @@ func balanceShardTypeAcrossRacks(vk volKey, nodes map[string]*Node, racks map[st
 		// limited-capacity destination across successive moves.
 		destNode.freeSlots--
 		pm.src.freeSlots++
+		moved[pm.shardID] = true
 	}
 	return moves
 }