Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-08 15:41:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: throughput limits for replicate, EC shard, and worker-driven moves (#10749)

Browse Source 
* feat: throughput limits for replicate, EC shard, and worker-driven moves

VolumeCopy was the only rate-limitable transfer; EC shard copies,
replica creation, and worker-driven moves all ran at whatever the
receiving server's maintenance rate allowed, with no per-operation
control.

- proto: VolumeEcShardsCopyRequest and the balance / ec_balance task
  params and configs gain io_byte_per_second; 0 keeps today's behavior
  (the volume server's own maintenance rate governs).
- volume server: VolumeEcShardsCopy throttles with one WriteThrottler
  per request, shared across the shard, .ecx, .ecj, .vif, and .ecsum
  copies so the limit caps the transfer as a whole - the same shape as
  VolumeCopy.
- volume_move: ReplicateVolume accepts the limit; EcMoveOptions carries
  it through MoveEcShards/CopyAndMountEcShards into the copy request,
  with fake-client tests asserting propagation.
- shell: ec.balance gains -ioBytePerSecond; volume.tier.move's
  replication top-up honors the command's existing -ioBytePerSecond
  instead of running unthrottled.
- worker: balance and ec_balance configs gain io_byte_per_second
  (surfaced in the admin config schema), carried through detection and
  plugin job parameters into task params and handed to the shared
  mover; batch balance jobs inherit the limit from their detection
  results.

The limit is per copy stream, so maxParallelization multiplies the
aggregate ceiling.

* worker plugins: expose io_byte_per_second in the plugin config and derive it

The plugin-driven detection path derives its task Config from the
plugin configuration values, and both balance and ec_balance left
IoBytePerSecond at zero there - a configured limit silently reverted
to the server maintenance rate. Both derive functions now read the
field (clamped at zero), and the plugin descriptors expose it with
defaults so the configuration form carries it.
This commit is contained in:
Chris Lu
2026-08-13 13:22:58 -07:00
committed by GitHub
parent 7d0fff32db
commit 4f50c5b0d4
32 changed files with 312 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/operation/volume_move/ec_move.go
+15 -11
View File
@@ -24,6 +24,9 @@ type EcShardMove struct {
// EcMoveOptions control MoveEcShards.
type EcMoveOptions struct {
// IoBytePerSecond limits the shard copy rate; 0 falls back to the volume
// server's maintenance rate.
IoBytePerSecond int64
// Writer receives human-readable progress lines (nil discards them).
Writer io.Writer
// Progress, when set, receives percent/stage callbacks as the move advances.
@@ -54,7 +57,7 @@ func (m *Mover) MoveEcShards(ctx context.Context, move EcShardMove, opts EcMoveO
}
progress(10, fmt.Sprintf("copying EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
if err := m.CopyAndMountEcShards(ctx, move.VolumeId, move.Collection, move.ShardIds, move.Source, move.Target, move.TargetDisk, writer); err != nil {
if err := m.CopyAndMountEcShards(ctx, move.VolumeId, move.Collection, move.ShardIds, move.Source, move.Target, move.TargetDisk, opts.IoBytePerSecond, writer); err != nil {
return err
}
@@ -83,7 +86,7 @@ func (m *Mover) MoveEcShards(ctx context.Context, move EcShardMove, opts EcMoveO
// .vif/.ecsum sidecars) from source and mount them. A same-address call skips
// the copy and just mounts — ec.encode uses that to bring freshly generated
// shards online in place.
func (m *Mover) CopyAndMountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, source, target pb.ServerAddress, targetDisk uint32, writer io.Writer) error {
func (m *Mover) CopyAndMountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, source, target pb.ServerAddress, targetDisk uint32, ioBytePerSecond int64, writer io.Writer) error {
if writer == nil {
writer = io.Discard
}
@@ -98,15 +101,16 @@ func (m *Mover) CopyAndMountEcShards(ctx context.Context, volumeId needle.Volume
if !SameServer(target, source) {
fmt.Fprintf(writer, "copy %d.%v %s => %s\n", volumeId, shardIds, source, target)
_, copyErr := client.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: uint32(volumeId),
Collection: collection,
ShardIds: erasure_coding.ShardIdsToUint32(shardIds),
CopyEcxFile: true,
CopyEcjFile: true,
CopyVifFile: true,
CopyEcsumFile: true, // propagate the bitrot sidecar with the shards (no-op if the source has none)
SourceDataNode: string(source),
DiskId: targetDisk,
VolumeId: uint32(volumeId),
Collection: collection,
ShardIds: erasure_coding.ShardIdsToUint32(shardIds),
CopyEcxFile: true,
CopyEcjFile: true,
CopyVifFile: true,
CopyEcsumFile: true, // propagate the bitrot sidecar with the shards (no-op if the source has none)
SourceDataNode: string(source),
DiskId: targetDisk,
IoBytePerSecond: ioBytePerSecond,
})
if copyErr != nil {
return fmt.Errorf("copy %d.%v %s => %s: %v", volumeId, shardIds, source, target, copyErr)
weed/operation/volume_move/ec_move_test.go
+3 -3
View File
@@ -33,7 +33,7 @@ func TestMoveEcShardsSequence(t *testing.T) {
cluster := newFakeCluster()
cluster.ecShards[string(dstAddr)] = dstShards(3, 4)
err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{})
err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{IoBytePerSecond: 77})
if err != nil {
t.Fatalf("MoveEcShards: %v", err)
}
@@ -50,7 +50,7 @@ func TestMoveEcShardsSequence(t *testing.T) {
if !copyReq.CopyEcxFile || !copyReq.CopyEcjFile || !copyReq.CopyVifFile || !copyReq.CopyEcsumFile {
t.Errorf("shard sidecars not all copied: %+v", copyReq)
}
if copyReq.DiskId != 2 || copyReq.SourceDataNode != string(srcAddr) || copyReq.Collection != "c1" {
if copyReq.DiskId != 2 || copyReq.SourceDataNode != string(srcAddr) || copyReq.Collection != "c1" || copyReq.IoBytePerSecond != 77 {
t.Errorf("copy request not propagated: %+v", copyReq)
}
}
@@ -106,7 +106,7 @@ func TestRemoveEcShards(t *testing.T) {
func TestCopyAndMountEcShardsSameAddressMountsOnly(t *testing.T) {
cluster := newFakeCluster()
err := cluster.mover().CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, srcAddr, srcAddr, 0, nil)
err := cluster.mover().CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, srcAddr, srcAddr, 0, 0, nil)
if err != nil {
t.Fatalf("CopyAndMountEcShards: %v", err)
}
weed/operation/volume_move/volume_move.go
+7 -5
View File
@@ -445,8 +445,9 @@ func (m *Mover) MarkVolumeWritable(ctx context.Context, volumeId needle.VolumeId
}
// ReplicateVolume copies a volume from source to target without touching the
// source — the replica-creation half of a move.
func (m *Mover) ReplicateVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, writer io.Writer) error {
// source — the replica-creation half of a move. ioBytePerSecond limits the
// copy rate; 0 falls back to the volume server's maintenance rate.
func (m *Mover) ReplicateVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, writer io.Writer) error {
if writer == nil {
writer = io.Discard
}
@@ -460,9 +461,10 @@ func (m *Mover) ReplicateVolume(ctx context.Context, volumeId needle.VolumeId, s
}
return m.withClient(false, target, func(client volume_server_pb.VolumeServerClient) error {
stream, replicateErr := client.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: uint32(volumeId),
SourceDataNode: string(source),
DiskType: diskType,
VolumeId: uint32(volumeId),
SourceDataNode: string(source),
DiskType: diskType,
IoBytePerSecond: ioBytePerSecond,
})
if replicateErr != nil {
return replicateErr
weed/operation/volume_move/volume_move_test.go
+14 -2
View File
@@ -80,13 +80,13 @@ func TestEmbeddedSourceAddressValidated(t *testing.T) {
ops := map[string]func(m *Mover) error{
"ReplicateVolume": func(m *Mover) error {
return m.ReplicateVolume(context.Background(), 7, bad, dstAddr, "", nil)
return m.ReplicateVolume(context.Background(), 7, bad, dstAddr, "", 0, nil)
},
"TailVolume": func(m *Mover) error {
return m.TailVolume(context.Background(), 7, bad, dstAddr, 0, time.Second)
},
"CopyAndMountEcShards": func(m *Mover) error {
return m.CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, bad, dstAddr, 0, nil)
return m.CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, bad, dstAddr, 0, 0, nil)
},
"MoveEcShards": func(m *Mover) error {
move := ecMove(3)
@@ -601,6 +601,18 @@ func TestLiveMoveVolumeReadonlyMarkFailureRestores(t *testing.T) {
}
}
func TestReplicateVolumePropagatesThroughput(t *testing.T) {
cluster := newFakeCluster()
if err := cluster.mover().ReplicateVolume(context.Background(), 7, srcAddr, dstAddr, "ssd", 55, nil); err != nil {
t.Fatalf("ReplicateVolume: %v", err)
}
copyReq := cluster.copyReqs[0]
if copyReq.IoBytePerSecond != 55 || copyReq.DiskType != "ssd" {
t.Errorf("replicate request not propagated: %+v", copyReq)
}
}
func TestCopyVolumeRestoreWritable(t *testing.T) {
for _, restoreWritable := range []bool{true, false} {
cluster := newFakeCluster()