Files
seaweedfs/weed/worker/tasks/balance/balance_task.go
T
Chris Lu 4f50c5b0d4 feat: throughput limits for replicate, EC shard, and worker-driven moves (#10749)
* feat: throughput limits for replicate, EC shard, and worker-driven moves

VolumeCopy was the only rate-limitable transfer; EC shard copies,
replica creation, and worker-driven moves all ran at whatever the
receiving server's maintenance rate allowed, with no per-operation
control.

- proto: VolumeEcShardsCopyRequest and the balance / ec_balance task
  params and configs gain io_byte_per_second; 0 keeps today's behavior
  (the volume server's own maintenance rate governs).
- volume server: VolumeEcShardsCopy throttles with one WriteThrottler
  per request, shared across the shard, .ecx, .ecj, .vif, and .ecsum
  copies so the limit caps the transfer as a whole - the same shape as
  VolumeCopy.
- volume_move: ReplicateVolume accepts the limit; EcMoveOptions carries
  it through MoveEcShards/CopyAndMountEcShards into the copy request,
  with fake-client tests asserting propagation.
- shell: ec.balance gains -ioBytePerSecond; volume.tier.move's
  replication top-up honors the command's existing -ioBytePerSecond
  instead of running unthrottled.
- worker: balance and ec_balance configs gain io_byte_per_second
  (surfaced in the admin config schema), carried through detection and
  plugin job parameters into task params and handed to the shared
  mover; batch balance jobs inherit the limit from their detection
  results.

The limit is per copy stream, so maxParallelization multiplies the
aggregate ceiling.

* worker plugins: expose io_byte_per_second in the plugin config and derive it

The plugin-driven detection path derives its task Config from the
plugin configuration values, and both balance and ec_balance left
IoBytePerSecond at zero there - a configured limit silently reverted
to the server maintenance rate. Both derive functions now read the
field (clamped at zero), and the plugin descriptors expose it with
defaults so the configuration form carries it.
2026-08-13 13:22:58 -07:00

136 lines
3.9 KiB
Go

package balance
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
"google.golang.org/grpc"
)
// BalanceTask implements the Task interface
type BalanceTask struct {
*base.BaseTask
server string
volumeID uint32
collection string
progress float64
grpcDialOption grpc.DialOption
}
// NewBalanceTask creates a new balance task instance
func NewBalanceTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *BalanceTask {
return &BalanceTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance),
server: server,
volumeID: volumeID,
collection: collection,
grpcDialOption: grpcDialOption,
}
}
// Execute implements the Task interface
func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
balanceParams := params.GetBalanceParams()
if balanceParams == nil {
return fmt.Errorf("balance parameters are required")
}
// Get source and destination from unified arrays
if len(params.Sources) == 0 {
return fmt.Errorf("source is required for balance task")
}
if len(params.Targets) == 0 {
return fmt.Errorf("target is required for balance task")
}
sourceNode := params.Sources[0].Node
destNode := params.Targets[0].Node
if sourceNode == "" {
return fmt.Errorf("source node is required for balance task")
}
if destNode == "" {
return fmt.Errorf("destination node is required for balance task")
}
t.GetLogger().WithFields(map[string]interface{}{
"volume_id": t.volumeID,
"source": sourceNode,
"destination": destNode,
"collection": t.collection,
}).Info("Starting balance task - moving volume")
// The move sequence — freeze the source, copy, tail, verify the target
// matches the source before the destructive delete — is shared with the
// shell's volume.move/volume.balance commands.
mover := volume_move.NewMover(t.grpcDialOption)
err := mover.LiveMoveVolume(ctx, needle.VolumeId(t.volumeID), pb.ServerAddress(sourceNode), pb.ServerAddress(destNode), volume_move.VolumeMoveOptions{
IdleTimeout: 60 * time.Second,
IoBytePerSecond: balanceParams.IoBytePerSecond,
Progress: func(percent float64, stage string) {
t.ReportProgress(percent)
t.GetLogger().Info(stage)
},
})
if err != nil {
return fmt.Errorf("move volume %d from %s to %s: %w", t.volumeID, sourceNode, destNode, err)
}
glog.Infof("Balance task completed successfully: volume %d moved from %s to %s",
t.volumeID, sourceNode, destNode)
return nil
}
// Validate implements the UnifiedTask interface
func (t *BalanceTask) Validate(params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
balanceParams := params.GetBalanceParams()
if balanceParams == nil {
return fmt.Errorf("balance parameters are required")
}
if params.VolumeId != t.volumeID {
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
}
// Validate that at least one source matches our server
found := false
for _, source := range params.Sources {
if source.Node == t.server {
found = true
break
}
}
if !found {
return fmt.Errorf("no source matches expected server %s", t.server)
}
return nil
}
// EstimateTime implements the UnifiedTask interface
func (t *BalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
// Basic estimate based on simulated steps
return 14 * time.Second // Sum of all step durations
}
// GetProgress returns current progress
func (t *BalanceTask) GetProgress() float64 {
return t.progress
}