mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
* shell: add volume.balance -byDiskUsage to balance by actual data The default balancer ranks servers by slot density, dividing used volumes by MaxVolumeCount. When MaxVolumeCount is configured higher than the disk can hold, a physically near-full server looks nearly empty and gets picked as the move target, so balancing drains less-full servers onto an already-full one. -byDiskUsage ranks servers by the actual data they hold (sum of volume sizes) instead, so the fullest-by-data server is treated as full and balancing drains it. It assumes comparable disk sizes per disk type and still respects each server's free volume slots. Default behavior is unchanged. * plumb physical disk usage into topology, gate volume.balance on it Volume servers now report each disk's filesystem total/free bytes in the heartbeat, and the master stores them in DiskInfo. volume.balance uses them to skip any move target whose disk is already near full (-maxDiskUsagePercent, default 90), so an over-configured maxVolumeCount can no longer make a physically full server look empty and get drained onto. The gate judges each server against its own disk, so heterogeneous disk sizes are fine; servers that do not report bytes fall back to slot-only behavior. Rust seaweed-volume mirrors the heartbeat reporting. * admin: report real physical disk capacity when volume servers provide it The dashboard estimated server capacity as maxVolumeCount * volumeSizeLimit, which overstates it when maxVolumeCount is set higher than the disk holds. Prefer the filesystem capacity now reported per disk, falling back to the estimate for servers that do not report it. * worker: gate automatic balance on physical disk fullness too The maintenance balance worker selects the least slot-utilized server as the move destination, so an over-configured maxVolumeCount makes a physically full server look empty and get drained onto — the same defect as the shell command. Now that DiskInfo carries real disk bytes, skip any destination whose disk is at/above 90% used (per server, against its own disk); a full server can still be a source. When every candidate destination is full, create no tasks. Servers that do not report disk bytes are not gated. * balance: share the physical-disk-fullness gate between shell and worker The shell volume.balance command and the maintenance balance worker each grew their own copy of the disk-fullness gate (targetDiskTooFull / destinationDiskTooFull) and a maxDiskUsagePercent=90 constant. Pull both into weed/topology/balancer (DiskTooFullAfter + DefaultMaxDiskUsagePercent) so the policy has one home and the two balancers can't drift. * balance: harden the physical-disk gate Guard against a nil DiskInfo in the byte/slot lookups. Let a zero disk-capacity report clear previously stored bytes (0 means "not reported" for bytes, unlike maxVolumeCount), so a server that stops reporting falls back to slot-only instead of trusting stale capacity. In the worker, charge each planned move's bytes to its destination within a detection cycle so the gate sees a target fill up rather than only its heartbeat-time free space. Note the per-location capacity summing assumes one location per filesystem (the used ratio the gate relies on stays correct regardless; absolute capacity can over-report).
This commit is contained in:
1 parent
41d6c821ba
commit
77bf2a3ab0
20 files changed
+825
-135
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
@@ -38,6 +39,11 @@ type commandVolumeBalance struct {
|
||||
applyBalancing bool
|
||||
volumesPerExec int
|
||||
movedCount int
|
||||
byDiskUsage bool
|
||||
|
||||
// diskUsageHighWaterPercent skips a move target whose physical disk used%
|
||||
// is at or above this mark. 0 or >=100 disables the gate.
|
||||
diskUsageHighWaterPercent int
|
||||
}
|
||||
|
||||
func (c *commandVolumeBalance) Name() string {
|
||||
@@ -47,7 +53,7 @@ func (c *commandVolumeBalance) Name() string {
|
||||
func (c *commandVolumeBalance) Help() string {
|
||||
return `balance all volumes among volume servers
|
||||
|
||||
volume.balance [-collection ALL_COLLECTIONS|EACH_COLLECTION|<collection_name>] [-apply] [-dataCenter=<data_center_name>] [-racks=rack_name_one,rack_name_two] [-nodes=192.168.0.1:8080,192.168.0.2:8080] [-volumesPerExec=5]
|
||||
volume.balance [-collection ALL_COLLECTIONS|EACH_COLLECTION|<collection_name>] [-apply] [-dataCenter=<data_center_name>] [-racks=rack_name_one,rack_name_two] [-nodes=192.168.0.1:8080,192.168.0.2:8080] [-volumesPerExec=5] [-byDiskUsage] [-maxDiskUsagePercent=90]
|
||||
|
||||
The -collection parameter supports:
|
||||
- ALL_COLLECTIONS: balance across all collections
|
||||
@@ -61,6 +67,19 @@ func (c *commandVolumeBalance) Help() string {
|
||||
If unset - the command will try to balance all volumes at once.
|
||||
It might be beneficial to set, if your cluster has lots of volumes growing and topology changes faster than balancing can occur.
|
||||
|
||||
The -maxDiskUsagePercent flag (default 90) skips any move target whose physical disk is already used at
|
||||
or above that percentage, using the real filesystem capacity each volume server reports. This is the
|
||||
default guard against an over-configured maxVolumeCount making a physically full disk look empty: such
|
||||
a server is never chosen as a move target, judged per server against its own disk so heterogeneous disk
|
||||
sizes are handled correctly. Set it to 0 (or >=100) to disable. Servers running an older build that does
|
||||
not report disk bytes are not gated, and balancing falls back to slot-only behavior for them.
|
||||
|
||||
The -byDiskUsage flag ranks servers by the actual data they hold (sum of volume sizes) instead of the
|
||||
default slot-density metric. The default metric normalizes by maxVolumeCount, so a server whose
|
||||
maxVolumeCount is configured too high for its disk looks nearly empty even when its disk is physically
|
||||
full, and balancing can drain less-full servers onto it. Use -byDiskUsage to balance actual data
|
||||
distribution instead. It assumes comparable disk sizes across servers of the same disk type.
|
||||
|
||||
Algorithm:
|
||||
|
||||
For each type of volume server (different max volume count limit){
|
||||
@@ -113,6 +132,8 @@ func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer
|
||||
// TODO: remove this alias
|
||||
applyBalancingAlias := balanceCommand.Bool("force", false, "apply the balancing plan (alias for -apply)")
|
||||
volumesPerExec := balanceCommand.Int("volumesPerExec", 0, "how many volumes to move in one run (default is 0 for unlimited)")
|
||||
byDiskUsage := balanceCommand.Bool("byDiskUsage", false, "rank servers by actual data held (sum of volume sizes) instead of slot density; use when maxVolumeCount is set too high for the disk. Assumes comparable disk sizes per disk type.")
|
||||
maxDiskUsagePercent := balanceCommand.Int("maxDiskUsagePercent", balancer.DefaultMaxDiskUsagePercent, "skip a move target whose physical disk used%% is at/above this; judged per server against its own disk, so heterogeneous disk sizes are fine. 0 or >=100 disables. Auto-skipped for servers that do not report disk bytes.")
|
||||
|
||||
balanceCommand.Func("volumeBy", "only apply the balancing for ALL volumes and ACTIVE or FULL", func(flagValue string) error {
|
||||
if flagValue == "" {
|
||||
@@ -136,6 +157,8 @@ func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer
|
||||
}
|
||||
c.volumesPerExec = *volumesPerExec
|
||||
c.movedCount = 0
|
||||
c.byDiskUsage = *byDiskUsage
|
||||
c.diskUsageHighWaterPercent = *maxDiskUsagePercent
|
||||
|
||||
infoAboutSimulationMode(writer, c.applyBalancing, "-apply")
|
||||
|
||||
@@ -320,6 +343,33 @@ func capacityByMinVolumeDensity(diskType types.DiskType, volumeSizeLimitMb uint6
|
||||
}
|
||||
}
|
||||
|
||||
// capacityByActualDataUsage ranks servers purely by how much actual data they
|
||||
// hold (sum of volume sizes), ignoring MaxVolumeCount. The slot-density metric
|
||||
// divides by MaxVolumeCount, so a server whose MaxVolumeCount was configured too
|
||||
// high for its disk looks nearly empty even when its disk is physically full and
|
||||
// gets picked as a move target. This function keeps the fullest-by-data server
|
||||
// ranked as full so balancing drains it instead of piling onto it. It assumes
|
||||
// comparable disk sizes across servers of the same disk type. Capacity is a
|
||||
// uniform constant so the density ratio is proportional to actual data; the
|
||||
// constant cancels out of every ratio comparison in balanceSelectedVolume.
|
||||
func capacityByActualDataUsage(diskType types.DiskType, volumeSizeLimitMb uint64) DensityFunc {
|
||||
return func(info *master_pb.DataNodeInfo) (float64, uint64) {
|
||||
diskInfo, found := info.DiskInfos[string(diskType)]
|
||||
if !found || diskInfo == nil {
|
||||
return 0, 0
|
||||
}
|
||||
var volumeSizes uint64
|
||||
for _, volumeInfo := range diskInfo.VolumeInfos {
|
||||
volumeSizes += volumeInfo.Size
|
||||
}
|
||||
if volumeSizeLimitMb == 0 {
|
||||
volumeSizeLimitMb = util.VolumeSizeLimitGB * util.KiByte
|
||||
}
|
||||
usedVolumeCount := volumeSizes / (volumeSizeLimitMb * util.MiByte)
|
||||
return 1, usedVolumeCount
|
||||
}
|
||||
}
|
||||
|
||||
func capacityByMaxVolumeCount(diskType types.DiskType) CapacityFunc {
|
||||
return func(info *master_pb.DataNodeInfo) float64 {
|
||||
diskInfo, found := info.DiskInfos[string(diskType)]
|
||||
@@ -371,6 +421,38 @@ func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 {
|
||||
return float64(len(n.selectedVolumes)) / capacityFunc(n.info)
|
||||
}
|
||||
|
||||
func (n *Node) hasFreeVolumeSlot(diskType types.DiskType) bool {
|
||||
diskInfo, found := n.info.DiskInfos[string(diskType)]
|
||||
if !found || diskInfo == nil {
|
||||
return false
|
||||
}
|
||||
return diskInfo.VolumeCount < diskInfo.MaxVolumeCount
|
||||
}
|
||||
|
||||
// diskBytes returns the node's physical disk capacity and free bytes for a disk
|
||||
// type. ok is false when the volume server did not report it (DiskTotalBytes==0),
|
||||
// which makes callers fall back to slot-only behavior.
|
||||
func (n *Node) diskBytes(diskType types.DiskType) (total, free uint64, ok bool) {
|
||||
diskInfo, found := n.info.DiskInfos[string(diskType)]
|
||||
if !found || diskInfo == nil || diskInfo.DiskTotalBytes == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
return diskInfo.DiskTotalBytes, diskInfo.DiskFreeBytes, true
|
||||
}
|
||||
|
||||
// targetDiskTooFull reports whether moving one more volume onto node would push
|
||||
// its physical disk used% at/above the high-water mark. It judges each server
|
||||
// against its own disk, so a larger disk holding more bytes is not unfairly
|
||||
// excluded. Returns false (no opinion) when the gate is disabled or the server
|
||||
// does not report disk bytes.
|
||||
func (c *commandVolumeBalance) targetDiskTooFull(node *Node, diskType types.DiskType, volumeSizeLimitMb uint64) bool {
|
||||
total, free, ok := node.diskBytes(diskType)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return balancer.DiskTooFullAfter(total, free, volumeSizeLimitMb*util.MiByte, c.diskUsageHighWaterPercent)
|
||||
}
|
||||
|
||||
func (n *Node) isOneVolumeOnly() bool {
|
||||
if len(n.selectedVolumes) != 1 {
|
||||
return false
|
||||
@@ -419,6 +501,9 @@ func (c *commandVolumeBalance) balanceSelectedVolume(diskType types.DiskType, vo
|
||||
volumeSizeLimitMb = util.VolumeSizeLimitGB * util.KiByte
|
||||
}
|
||||
capacityFunc := capacityByMinVolumeDensity(diskType, volumeSizeLimitMb)
|
||||
if c.byDiskUsage {
|
||||
capacityFunc = capacityByActualDataUsage(diskType, volumeSizeLimitMb)
|
||||
}
|
||||
for _, dn := range nodes {
|
||||
capacity, volumeCount := capacityFunc(dn.info)
|
||||
if capacity > 0 {
|
||||
@@ -475,6 +560,22 @@ func (c *commandVolumeBalance) balanceSelectedVolume(diskType types.DiskType, vo
|
||||
}
|
||||
sortCandidatesFn(candidateVolumes)
|
||||
for _, emptyNode := range nodesWithCapacity[:fullNodeIndex] {
|
||||
// In byte-usage mode capacity is a uniform constant, so a target's
|
||||
// free volume slots aren't reflected in its ranking; skip targets that
|
||||
// are already at MaxVolumeCount so balancing never exceeds the slot limit.
|
||||
if c.byDiskUsage && !emptyNode.hasFreeVolumeSlot(diskType) {
|
||||
continue
|
||||
}
|
||||
// Never move onto a server whose physical disk is already near full,
|
||||
// even if the slot-density metric ranks it as the emptiest node. This is
|
||||
// the root-cause guard for an over-configured maxVolumeCount making a
|
||||
// full disk look empty; it is judged per server against its own disk.
|
||||
if c.targetDiskTooFull(emptyNode, diskType, volumeSizeLimitMb) {
|
||||
if c.commandEnv != nil && c.commandEnv.verbose {
|
||||
fmt.Fprintf(os.Stdout, "skip target %s: disk used%% >= %d%%\n", emptyNode.info.Id, c.diskUsageHighWaterPercent)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !(fullNode.localVolumeDensityNextRatio(capacityFunc) > idealVolumeRatio && emptyNode.localVolumeDensityNextRatio(capacityFunc) <= idealVolumeRatio) {
|
||||
if c.commandEnv != nil && c.commandEnv.verbose {
|
||||
fmt.Printf("no more volume servers with empty slots %s, idealVolumeRatio %f\n", emptyNode.info.Id, idealVolumeRatio)
|
||||
@@ -597,6 +698,24 @@ func isGoodMove(placement *super_block.ReplicaPlacement, existingReplicas []*Vol
|
||||
return satisfyReplicaPlacement(placement, existingReplicasExceptSourceNode, targetLocation)
|
||||
}
|
||||
|
||||
// addDiskFreeBytes adjusts a disk's reported free bytes by delta (negative when a
|
||||
// volume lands on it), so the physical-fullness gate stays consistent as volumes
|
||||
// move within a single balance run. No-op when the disk reports no physical
|
||||
// capacity (DiskTotalBytes==0); clamps to [0, DiskTotalBytes].
|
||||
func addDiskFreeBytes(diskInfo *master_pb.DiskInfo, delta int64) {
|
||||
if diskInfo.DiskTotalBytes == 0 {
|
||||
return
|
||||
}
|
||||
free := int64(diskInfo.DiskFreeBytes) + delta
|
||||
if free < 0 {
|
||||
free = 0
|
||||
}
|
||||
if uint64(free) > diskInfo.DiskTotalBytes {
|
||||
free = int64(diskInfo.DiskTotalBytes)
|
||||
}
|
||||
diskInfo.DiskFreeBytes = uint64(free)
|
||||
}
|
||||
|
||||
func removeVolumeInfo(diskInfo *master_pb.DiskInfo, volumeId uint32) {
|
||||
for i, volumeInfo := range diskInfo.VolumeInfos {
|
||||
if volumeInfo.Id == volumeId {
|
||||
@@ -629,10 +748,12 @@ func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[u
|
||||
if fullDisk, found := fullNode.info.DiskInfos[v.DiskType]; found {
|
||||
removeVolumeInfo(fullDisk, v.Id)
|
||||
addVolumeCount(fullDisk, -1)
|
||||
addDiskFreeBytes(fullDisk, int64(v.Size))
|
||||
}
|
||||
if emptyDisk, found := emptyNode.info.DiskInfos[v.DiskType]; found {
|
||||
emptyDisk.VolumeInfos = append(emptyDisk.VolumeInfos, v)
|
||||
addVolumeCount(emptyDisk, 1)
|
||||
addDiskFreeBytes(emptyDisk, -int64(v.Size))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -356,6 +356,204 @@ func TestBalanceDoesNotDrainOntoOneNode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// byteFullNode physically holds twice the data of mediumNode but its
|
||||
// MaxVolumeCount was configured too high for its disk, so the default slot-density
|
||||
// metric ranks it as the emptiest server and drains mediumNode onto it (verified
|
||||
// by the default-mode sub-assertion below). With -byDiskUsage the ranking is by
|
||||
// actual data held, so the fuller server becomes the move source and the two end
|
||||
// up evenly distributed by data.
|
||||
func TestBalanceByDiskUsage(t *testing.T) {
|
||||
const mb = 1024 * 1024
|
||||
volumeSizeLimitMb := uint64(100)
|
||||
|
||||
makeNode := func(id string, maxVolumeCount int64, volumes []*master_pb.VolumeInformationMessage) *Node {
|
||||
return &Node{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: id,
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"": {
|
||||
MaxVolumeCount: maxVolumeCount,
|
||||
VolumeCount: int64(len(volumes)),
|
||||
VolumeInfos: volumes,
|
||||
},
|
||||
},
|
||||
},
|
||||
dc: "dc1",
|
||||
rack: "rack1",
|
||||
}
|
||||
}
|
||||
|
||||
mkVolumes := func(start, n uint32) []*master_pb.VolumeInformationMessage {
|
||||
var vs []*master_pb.VolumeInformationMessage
|
||||
for id := start; id < start+n; id++ {
|
||||
vs = append(vs, &master_pb.VolumeInformationMessage{Id: id, Size: 95 * mb})
|
||||
}
|
||||
return vs
|
||||
}
|
||||
|
||||
dataBytes := func(n *Node) uint64 {
|
||||
var sum uint64
|
||||
for _, v := range n.info.DiskInfos[""].VolumeInfos {
|
||||
sum += v.Size
|
||||
}
|
||||
return sum
|
||||
}
|
||||
volumeCount := func(n *Node) int { return len(n.info.DiskInfos[""].VolumeInfos) }
|
||||
|
||||
setup := func() (*Node, *Node, []*Node, map[uint32][]*VolumeReplica) {
|
||||
byteFullNode := makeNode("byte-full", 1000, mkVolumes(1, 20))
|
||||
mediumNode := makeNode("half-full", 30, mkVolumes(101, 10))
|
||||
nodes := []*Node{byteFullNode, mediumNode}
|
||||
volumeReplicas := map[uint32][]*VolumeReplica{}
|
||||
for _, n := range nodes {
|
||||
for _, v := range n.info.DiskInfos[""].VolumeInfos {
|
||||
loc := newLocation("dc1", "rack1", n.info)
|
||||
volumeReplicas[v.Id] = []*VolumeReplica{{location: &loc, info: v}}
|
||||
}
|
||||
n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool { return true })
|
||||
}
|
||||
return byteFullNode, mediumNode, nodes, volumeReplicas
|
||||
}
|
||||
|
||||
// Default mode: the over-configured MaxVolumeCount makes the fuller server the
|
||||
// target, so it gains even more data. This is the reported pathology.
|
||||
byteFullNode, _, nodes, volumeReplicas := setup()
|
||||
before := dataBytes(byteFullNode)
|
||||
c := &commandVolumeBalance{volumeSizeLimitMb: volumeSizeLimitMb}
|
||||
if err := c.balanceSelectedVolume(types.HardDriveType, volumeReplicas, nodes, sortWritableVolumes); err != nil {
|
||||
t.Fatalf("default balanceSelectedVolume: %v", err)
|
||||
}
|
||||
if dataBytes(byteFullNode) <= before {
|
||||
t.Fatalf("expected default mode to pile onto the fuller server (the bug), but it did not: %d MB -> %d MB", before/mb, dataBytes(byteFullNode)/mb)
|
||||
}
|
||||
|
||||
// -byDiskUsage: the fuller server is recognized as full, so it sheds data and
|
||||
// the two servers converge to an even data distribution.
|
||||
byteFullNode, mediumNode, nodes, volumeReplicas := setup()
|
||||
before = dataBytes(byteFullNode)
|
||||
c = &commandVolumeBalance{volumeSizeLimitMb: volumeSizeLimitMb, byDiskUsage: true}
|
||||
if err := c.balanceSelectedVolume(types.HardDriveType, volumeReplicas, nodes, sortWritableVolumes); err != nil {
|
||||
t.Fatalf("byDiskUsage balanceSelectedVolume: %v", err)
|
||||
}
|
||||
if got := dataBytes(byteFullNode); got >= before {
|
||||
t.Fatalf("-byDiskUsage should drain the fuller server, but it did not shrink: %d MB -> %d MB", before/mb, got/mb)
|
||||
}
|
||||
if diff := volumeCount(byteFullNode) - volumeCount(mediumNode); diff > 1 || diff < -1 {
|
||||
t.Fatalf("-byDiskUsage should even out the data, got byte-full=%d half-full=%d volumes",
|
||||
volumeCount(byteFullNode), volumeCount(mediumNode))
|
||||
}
|
||||
}
|
||||
|
||||
// makeByteNode builds a single-disk Node carrying physical disk bytes, for the
|
||||
// disk-fullness gate tests.
|
||||
func makeByteNode(id string, maxVolumeCount int64, totalBytes, freeBytes uint64, volumes []*master_pb.VolumeInformationMessage) *Node {
|
||||
return &Node{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: id,
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"": {
|
||||
MaxVolumeCount: maxVolumeCount,
|
||||
VolumeCount: int64(len(volumes)),
|
||||
VolumeInfos: volumes,
|
||||
DiskTotalBytes: totalBytes,
|
||||
DiskFreeBytes: freeBytes,
|
||||
},
|
||||
},
|
||||
},
|
||||
dc: "dc1",
|
||||
rack: "rack1",
|
||||
}
|
||||
}
|
||||
|
||||
func mkByteVolumes(start, n uint32, sizeMb uint64) []*master_pb.VolumeInformationMessage {
|
||||
const mb = 1024 * 1024
|
||||
var vs []*master_pb.VolumeInformationMessage
|
||||
for id := start; id < start+n; id++ {
|
||||
vs = append(vs, &master_pb.VolumeInformationMessage{Id: id, Size: sizeMb * mb})
|
||||
}
|
||||
return vs
|
||||
}
|
||||
|
||||
func runBalance(t *testing.T, c *commandVolumeBalance, nodes []*Node) {
|
||||
t.Helper()
|
||||
volumeReplicas := map[uint32][]*VolumeReplica{}
|
||||
for _, n := range nodes {
|
||||
for _, v := range n.info.DiskInfos[""].VolumeInfos {
|
||||
loc := newLocation("dc1", "rack1", n.info)
|
||||
volumeReplicas[v.Id] = []*VolumeReplica{{location: &loc, info: v}}
|
||||
}
|
||||
n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool { return true })
|
||||
}
|
||||
if err := c.balanceSelectedVolume(types.HardDriveType, volumeReplicas, nodes, sortWritableVolumes); err != nil {
|
||||
t.Fatalf("balanceSelectedVolume: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func volCount(n *Node) int { return len(n.info.DiskInfos[""].VolumeInfos) }
|
||||
|
||||
// Issue #10160 root-cause guard: a server whose physical disk is near full must
|
||||
// not be chosen as a move target, even when an over-configured maxVolumeCount
|
||||
// makes the slot-density metric rank it as the emptiest node. The gate is on by
|
||||
// default, disables at 0, and falls back to slot-only behavior when the server
|
||||
// does not report disk bytes.
|
||||
func TestBalanceSkipsPhysicallyFullTarget(t *testing.T) {
|
||||
const gb = 1024 * 1024 * 1024
|
||||
volumeSizeLimitMb := uint64(1024) // 1 GiB volumes
|
||||
|
||||
// source: tight slot budget, so it ranks as the move source.
|
||||
// fullDisk: huge maxVolumeCount (mis-set) but disk is physically 96% full.
|
||||
build := func(fullDiskTotal, fullDiskFree uint64) (*Node, *Node) {
|
||||
source := makeByteNode("source", 10, 1000*gb, 200*gb, mkByteVolumes(1, 8, 1000))
|
||||
fullDisk := makeByteNode("disk-full", 1000, fullDiskTotal, fullDiskFree, mkByteVolumes(101, 1, 1000))
|
||||
return source, fullDisk
|
||||
}
|
||||
|
||||
// Gate on (default 90%): the 96%-full server is never a target -> no move.
|
||||
source, fullDisk := build(1000*gb, 40*gb)
|
||||
c := &commandVolumeBalance{volumeSizeLimitMb: volumeSizeLimitMb, diskUsageHighWaterPercent: 90}
|
||||
runBalance(t, c, []*Node{source, fullDisk})
|
||||
if got := volCount(fullDisk); got != 1 {
|
||||
t.Fatalf("gate on: expected no move onto 96%%-full server, got %d volumes (was 1)", got)
|
||||
}
|
||||
|
||||
// Gate off (0): the old behavior piles onto the byte-full server.
|
||||
source, fullDisk = build(1000*gb, 40*gb)
|
||||
c = &commandVolumeBalance{volumeSizeLimitMb: volumeSizeLimitMb, diskUsageHighWaterPercent: 0}
|
||||
runBalance(t, c, []*Node{source, fullDisk})
|
||||
if got := volCount(fullDisk); got <= 1 {
|
||||
t.Fatalf("gate off: expected moves onto the byte-full server (the bug), got %d volumes", got)
|
||||
}
|
||||
|
||||
// Fallback: server reports no disk bytes (DiskTotalBytes==0) -> not gated.
|
||||
source, fullDisk = build(0, 0)
|
||||
c = &commandVolumeBalance{volumeSizeLimitMb: volumeSizeLimitMb, diskUsageHighWaterPercent: 90}
|
||||
runBalance(t, c, []*Node{source, fullDisk})
|
||||
if got := volCount(fullDisk); got <= 1 {
|
||||
t.Fatalf("fallback: expected slot-only behavior to move onto the server, got %d volumes", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the gate on, balancing steers moves to a physically empty disk and away
|
||||
// from a physically full one when both look equally empty by slot density.
|
||||
func TestBalanceGateSteersToEmptierDisk(t *testing.T) {
|
||||
const gb = 1024 * 1024 * 1024
|
||||
volumeSizeLimitMb := uint64(1024)
|
||||
|
||||
source := makeByteNode("source", 10, 1000*gb, 300*gb, mkByteVolumes(1, 8, 1000))
|
||||
fullDisk := makeByteNode("disk-full", 1000, 1000*gb, 40*gb, mkByteVolumes(101, 1, 1000)) // 96% used -> gated
|
||||
emptyDisk := makeByteNode("disk-empty", 1000, 1000*gb, 900*gb, mkByteVolumes(201, 1, 1000)) // 10% used -> ok
|
||||
|
||||
c := &commandVolumeBalance{volumeSizeLimitMb: volumeSizeLimitMb, diskUsageHighWaterPercent: 90}
|
||||
runBalance(t, c, []*Node{source, fullDisk, emptyDisk})
|
||||
|
||||
if got := volCount(fullDisk); got != 1 {
|
||||
t.Fatalf("expected no move onto the physically full disk, got %d volumes (was 1)", got)
|
||||
}
|
||||
if got := volCount(emptyDisk); got <= 1 {
|
||||
t.Fatalf("expected moves onto the physically empty disk, got %d volumes", got)
|
||||
}
|
||||
}
|
||||
|
||||
// volumesPerExec caps the number of moves performed in a single execution.
|
||||
func TestBalanceVolumesPerExec(t *testing.T) {
|
||||
const mb = 1024 * 1024
|
||||
|
||||
Reference in new issue
Block a user