feat: send EC shards in parallel per destination disk (#12050)

* feat: send EC shards in parallel per destination disk

One stream per destination disk. max_shard_transfers unset means no global cap, 0 sends shards sequentially, and N is a global cap that still keeps one stream per disk. The value is persisted through TaskPolicy. The maintenance form is a checkbox plus Max transfers because unset and 0 differ. The key is intentionally omitted from GetConfigSpec.

* ec shard distribution: treat negative max_shard_transfers as sequential

The config layer (normalizedMaxShardTransfers) clamps negative values to 0,
which selects sequential distribution. shardTransferCap diverged: a negative
cap that reached ShardDistributionOptions directly would have selected
uncapped parallel mode, the most aggressive setting. Align the options layer
with the same normalization so invalid input degrades to the conservative
mode.

* ec: serialize shard transfers to a server when a target leaves disk choice to the receiver

disk_id 0 is auto-placement on the receiving volume server, so an
auto-placed stream can land on a disk another queued stream is pinned
to. Grouping by (destNode, disk_id) alone let both run at once; collapse
all of that server's transfers under the destNode key instead.

* worker: clamp max_shard_transfers to the int32 wire range

ReadIntConfig returned the 0 fallback for values above MaxInt32, turning
a large positive cap into sequential mode. Read int64 and clamp into
range instead.

* worker: drop unused shard-transfer form helper

The maintenance form renders and collects max_shard_transfers in
plugin.templ JavaScript; the Go copy had no callers.

* worker: bound max_shard_transfers conversion under an in-range guard

---------

Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>

weed/storage/erasure_coding/shard_distribution.go

@@ -14,9 +14,66 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 )
 
+// ShardDistributionOptions controls DistributeEcShards concurrency.
+// MaxShardTransfers:
+//   - nil / unset: no global cap; one in-flight per (destNode, disk_id), or
+//     per destNode (host:port, not rack) when any target leaves disk choice
+//     to the receiver (disk_id 0)
+//   - N<=0: historical sequential nested loops
+//   - N>0: global cap of N, still at most one in-flight per key
+type ShardDistributionOptions struct {
+	MaxShardTransfers *int
+	send              shardSender
+}
+
+type shardSender func(ctx context.Context, volumeID uint32, collection string, dialOption grpc.DialOption, destServer string, diskID uint32, filePath, shardType string) error
+
+type shardJob struct {
+	destNode  string
+	shardType string
+	filePath  string
+	diskID    uint32
+	hasDisk   bool
+}
+
+func shardSenderOrDefault(opts *ShardDistributionOptions) shardSender {
+	if opts != nil && opts.send != nil {
+		return opts.send
+	}
+	return sendShardFileToDestination
+}
+
+func shardTransferCap(opts *ShardDistributionOptions) (sequential bool, capN int) {
+	if opts == nil || opts.MaxShardTransfers == nil {
+		return false, 0
+	}
+	n := *opts.MaxShardTransfers
+	if n <= 0 {
+		return true, 0
+	}
+	return false, n
+}
+
+func lookupShardDisk(shardDisks map[string]map[string]uint32, destNode, shardType string) (uint32, bool) {
+	if byShard, ok := shardDisks[destNode]; ok {
+		if id, exists := byShard[shardType]; exists {
+			return id, true
+		}
+	}
+	return 0, false
+}
+
+func shardTransferKey(j shardJob) string {
+	if j.hasDisk {
+		return fmt.Sprintf("%s#%d", j.destNode, j.diskID)
+	}
+	return j.destNode
+}
+
 func ensureLogger(logger logger) logger {
 	if logger == nil {
 		return &glogFallbackLogger{}
@@ -98,7 +155,8 @@ func (g *glogFallbackLogger) Error(msg string, args ...interface{}) {
 
 // DistributeEcShards distributes locally generated EC shards to destination servers.
 // Returns the shard assignment map used for mounting.
-func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb.TaskTarget, shardFiles map[string]string, dialOption grpc.DialOption, logger logger) (map[string][]string, error) {
+// opts nil is the default: one in-flight per dest disk/node, no global cap.
+func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb.TaskTarget, shardFiles map[string]string, dialOption grpc.DialOption, logger logger, opts *ShardDistributionOptions) (map[string][]string, error) {
 	if len(targets) == 0 {
 		return nil, fmt.Errorf("no targets specified for EC shard distribution")
 	}
@@ -173,6 +231,22 @@ func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb
 		return nil, fmt.Errorf("no shard assignments found from planning phase")
 	}
 
+	send := shardSenderOrDefault(opts)
+	sequential, capN := shardTransferCap(opts)
+	if sequential {
+		if err := distributeEcShardsSequential(volumeID, collection, shardAssignment, shardFiles, shardDisks, dialOption, log, send); err != nil {
+			return nil, err
+		}
+	} else if err := distributeEcShardsParallel(volumeID, collection, shardAssignment, shardFiles, shardDisks, dialOption, log, send, capN); err != nil {
+		return nil, err
+	}
+
+	glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment))
+	return shardAssignment, nil
+}
+
+func distributeEcShardsSequential(volumeID uint32, collection string, shardAssignment map[string][]string, shardFiles map[string]string, shardDisks map[string]map[string]uint32, dialOption grpc.DialOption, log logger, send shardSender) error {
+	ctx := context.Background()
 	for destNode, assignedShards := range shardAssignment {
 		withFields(log, map[string]interface{}{
 			"destination":     destNode,
@@ -184,7 +258,7 @@ func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb
 		for _, shardType := range assignedShards {
 			filePath, exists := shardFiles[shardType]
 			if !exists {
-				return nil, fmt.Errorf("shard file %s not found for destination %s", shardType, destNode)
+				return fmt.Errorf("shard file %s not found for destination %s", shardType, destNode)
 			}
 
 			if info, err := os.Stat(filePath); err == nil {
@@ -198,12 +272,9 @@ func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb
 				}).Info("Starting shard file transfer")
 			}
 
-			diskID := uint32(0)
-			if byShard, ok := shardDisks[destNode]; ok {
-				diskID = byShard[shardType]
-			}
-			if err := sendShardFileToDestination(volumeID, collection, dialOption, destNode, diskID, filePath, shardType); err != nil {
-				return nil, fmt.Errorf("failed to send %s to %s: %w", shardType, destNode, err)
+			diskID, _ := lookupShardDisk(shardDisks, destNode, shardType)
+			if err := send(ctx, volumeID, collection, dialOption, destNode, diskID, filePath, shardType); err != nil {
+				return fmt.Errorf("failed to send %s to %s: %w", shardType, destNode, err)
 			}
 
 			withFields(log, map[string]interface{}{
@@ -219,9 +290,139 @@ func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb
 			"total_mb":           float64(transferredBytes) / (1024 * 1024),
 		}).Info("All shards distributed to destination server")
 	}
+	return nil
+}
 
-	glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment))
-	return shardAssignment, nil
+func collectShardJobs(shardAssignment map[string][]string, shardFiles map[string]string, shardDisks map[string]map[string]uint32, log logger) (map[string][]shardJob, error) {
+	// disk_id 0 leaves placement to the receiver's auto-select, which can land
+	// on a disk another queued stream is pinned to — serialize every transfer
+	// to that server under one key.
+	autoPlacement := make(map[string]bool)
+	for destNode, assignedShards := range shardAssignment {
+		for _, shardType := range assignedShards {
+			if diskID, ok := lookupShardDisk(shardDisks, destNode, shardType); ok && diskID == 0 {
+				autoPlacement[destNode] = true
+				break
+			}
+		}
+	}
+
+	byKey := make(map[string][]shardJob)
+	for destNode, assignedShards := range shardAssignment {
+		withFields(log, map[string]interface{}{
+			"destination":     destNode,
+			"assigned_shards": len(assignedShards),
+			"shard_types":     assignedShards,
+		}).Info("Starting shard distribution to destination server")
+
+		for _, shardType := range assignedShards {
+			filePath, exists := shardFiles[shardType]
+			if !exists {
+				return nil, fmt.Errorf("shard file %s not found for destination %s", shardType, destNode)
+			}
+			diskID, hasDisk := lookupShardDisk(shardDisks, destNode, shardType)
+			job := shardJob{
+				destNode:  destNode,
+				shardType: shardType,
+				filePath:  filePath,
+				diskID:    diskID,
+				hasDisk:   hasDisk,
+			}
+			key := shardTransferKey(job)
+			if autoPlacement[destNode] {
+				key = destNode
+			}
+			byKey[key] = append(byKey[key], job)
+		}
+	}
+	return byKey, nil
+}
+
+func distributeEcShardsParallel(volumeID uint32, collection string, shardAssignment map[string][]string, shardFiles map[string]string, shardDisks map[string]map[string]uint32, dialOption grpc.DialOption, log logger, send shardSender, capN int) error {
+	byKey, err := collectShardJobs(shardAssignment, shardFiles, shardDisks, log)
+	if err != nil {
+		return err
+	}
+
+	g, ctx := errgroup.WithContext(context.Background())
+	var sem chan struct{}
+	if capN > 0 {
+		sem = make(chan struct{}, capN)
+	}
+
+	for _, jobs := range byKey {
+		jobs := jobs
+		g.Go(func() error {
+			for _, job := range jobs {
+				if err := ctx.Err(); err != nil {
+					return err
+				}
+				if sem != nil {
+					select {
+					case sem <- struct{}{}:
+					case <-ctx.Done():
+						return ctx.Err()
+					}
+					if err := ctx.Err(); err != nil {
+						<-sem
+						return err
+					}
+				}
+				err := transferOneShard(ctx, volumeID, collection, dialOption, log, send, job)
+				if err != nil {
+					// Hold the slot so another key cannot start a send in the
+					// window before errgroup cancels ctx.
+					return err
+				}
+				if sem != nil {
+					<-sem
+				}
+			}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	for destNode, assignedShards := range shardAssignment {
+		var transferredBytes int64
+		for _, shardType := range assignedShards {
+			if filePath, ok := shardFiles[shardType]; ok {
+				if info, statErr := os.Stat(filePath); statErr == nil {
+					transferredBytes += info.Size()
+				}
+			}
+		}
+		withFields(log, map[string]interface{}{
+			"destination":        destNode,
+			"shards_transferred": len(assignedShards),
+			"total_bytes":        transferredBytes,
+			"total_mb":           float64(transferredBytes) / (1024 * 1024),
+		}).Info("All shards distributed to destination server")
+	}
+	return nil
+}
+
+func transferOneShard(ctx context.Context, volumeID uint32, collection string, dialOption grpc.DialOption, log logger, send shardSender, job shardJob) error {
+	if info, err := os.Stat(job.filePath); err == nil {
+		withFields(log, map[string]interface{}{
+			"destination": job.destNode,
+			"shard_type":  job.shardType,
+			"file_path":   job.filePath,
+			"size_bytes":  info.Size(),
+			"size_kb":     float64(info.Size()) / 1024,
+		}).Info("Starting shard file transfer")
+	}
+	if err := send(ctx, volumeID, collection, dialOption, job.destNode, job.diskID, job.filePath, job.shardType); err != nil {
+		return fmt.Errorf("failed to send %s to %s: %w", job.shardType, job.destNode, err)
+	}
+	withFields(log, map[string]interface{}{
+		"destination": job.destNode,
+		"shard_type":  job.shardType,
+	}).Info("Shard file transfer completed")
+	return nil
 }
 
 // MountEcShards mounts EC shards on destination servers using an assignment map.
@@ -300,7 +501,10 @@ func MountEcShards(volumeID uint32, collection string, shardAssignment map[strin
 }
 
 // diskID=0 leaves disk placement to the server's auto-select.
-func sendShardFileToDestination(volumeID uint32, collection string, dialOption grpc.DialOption, destServer string, diskID uint32, filePath, shardType string) error {
+func sendShardFileToDestination(ctx context.Context, volumeID uint32, collection string, dialOption grpc.DialOption, destServer string, diskID uint32, filePath, shardType string) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), dialOption,
 		func(client volume_server_pb.VolumeServerClient) error {
 			file, err := os.Open(filePath)
@@ -335,7 +539,7 @@ func sendShardFileToDestination(volumeID uint32, collection string, dialOption g
 				return fmt.Errorf("unknown shard type: %s", shardType)
 			}
 
-			stream, err := client.ReceiveFile(context.Background())
+			stream, err := client.ReceiveFile(ctx)
 			if err != nil {
 				return fmt.Errorf("failed to create receive stream: %v", err)
 			}

weed/storage/erasure_coding/shard_distribution_test.go

@@ -0,0 +1,428 @@
+package erasure_coding
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"google.golang.org/grpc"
+)
+
+func intPtr(n int) *int { return &n }
+
+func writeShardFiles(t *testing.T, names ...string) map[string]string {
+	t.Helper()
+	dir := t.TempDir()
+	files := make(map[string]string, len(names))
+	for _, name := range names {
+		p := filepath.Join(dir, name)
+		if err := os.WriteFile(p, []byte(name), 0644); err != nil {
+			t.Fatal(err)
+		}
+		files[name] = p
+	}
+	return files
+}
+
+func shardTarget(node string, disk uint32, shards ...uint32) *worker_pb.TaskTarget {
+	return &worker_pb.TaskTarget{Node: node, DiskId: disk, ShardIds: shards}
+}
+
+type flightRecorder struct {
+	mu        sync.Mutex
+	inFlight  int
+	maxFlight int
+	keyFlight map[string]int
+	keyMax    int
+	attempts  []string
+	sent      []string
+	failOn    string
+	failFirst bool
+	enter     chan struct{}
+	goAhead   chan struct{}
+	noGate    atomic.Bool
+}
+
+func newFlightRecorder() *flightRecorder {
+	return &flightRecorder{
+		keyFlight: make(map[string]int),
+		enter:     make(chan struct{}, 32),
+		goAhead:   make(chan struct{}),
+	}
+}
+
+func (r *flightRecorder) release() {
+	r.noGate.Store(true)
+	close(r.goAhead)
+}
+
+func (r *flightRecorder) send(ctx context.Context, _ uint32, _ string, _ grpc.DialOption, destServer string, diskID uint32, _ string, shardType string) error {
+	id := destServer + "/" + shardType
+	key := shardTransferKey(shardJob{destNode: destServer, diskID: diskID, hasDisk: true})
+
+	r.mu.Lock()
+	r.inFlight++
+	if r.inFlight > r.maxFlight {
+		r.maxFlight = r.inFlight
+	}
+	r.keyFlight[key]++
+	if r.keyFlight[key] > r.keyMax {
+		r.keyMax = r.keyFlight[key]
+	}
+	r.attempts = append(r.attempts, id)
+	r.mu.Unlock()
+
+	defer func() {
+		r.mu.Lock()
+		r.keyFlight[key]--
+		r.inFlight--
+		r.mu.Unlock()
+	}()
+
+	if r.failOn == id || r.failFirst {
+		r.failFirst = false
+		return errors.New("mock send failed")
+	}
+	if !r.noGate.Load() {
+		select {
+		case r.enter <- struct{}{}:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+		select {
+		case <-r.goAhead:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+	r.mu.Lock()
+	r.sent = append(r.sent, id)
+	r.mu.Unlock()
+	return nil
+}
+
+func waitEnters(t *testing.T, rec *flightRecorder, n int) {
+	t.Helper()
+	deadline := time.After(2 * time.Second)
+	for i := 0; i < n; i++ {
+		select {
+		case <-rec.enter:
+		case <-deadline:
+			t.Fatalf("timed out waiting for %d in-flight sends (got %d)", n, i)
+		}
+	}
+}
+
+func assertNoExtraEnter(t *testing.T, rec *flightRecorder, wait time.Duration) {
+	t.Helper()
+	select {
+	case <-rec.enter:
+		t.Fatal("unexpected extra in-flight send")
+	case <-time.After(wait):
+	}
+}
+
+func runDistribute(t *testing.T, targets []*worker_pb.TaskTarget, files map[string]string, rec *flightRecorder, max *int) (map[string][]string, error) {
+	t.Helper()
+	opts := &ShardDistributionOptions{
+		MaxShardTransfers: max,
+		send:              rec.send,
+	}
+	return DistributeEcShards(1, "col", targets, files, nil, nil, opts)
+}
+
+func TestDistributeEcShardsZeroIsSequential(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01", "ec02", "ec03")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 0, 0),
+		shardTarget("full-01:8080", 1, 1),
+		shardTarget("full-02:8080", 0, 2),
+		shardTarget("full-02:8080", 1, 3),
+	}
+	rec := newFlightRecorder()
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := runDistribute(t, targets, files, rec, intPtr(0))
+		errCh <- err
+	}()
+
+	waitEnters(t, rec, 1)
+	assertNoExtraEnter(t, rec, 40*time.Millisecond)
+	rec.mu.Lock()
+	if rec.maxFlight != 1 {
+		rec.mu.Unlock()
+		t.Fatalf("max in-flight=%d, want 1 for max_shard_transfers=0", rec.maxFlight)
+	}
+	rec.mu.Unlock()
+	rec.release()
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if rec.maxFlight != 1 {
+		t.Fatalf("max in-flight=%d after release, want 1", rec.maxFlight)
+	}
+	if rec.keyMax != 1 {
+		t.Fatalf("per-key in-flight=%d, want 1", rec.keyMax)
+	}
+	if len(rec.sent) != 4 {
+		t.Fatalf("sent %d shards, want 4: %v", len(rec.sent), rec.sent)
+	}
+}
+
+func TestDistributeEcShardsNoCapParallelByKey(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01", "ec02", "ec03")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 1, 0),
+		shardTarget("full-01:8080", 2, 1),
+		shardTarget("full-02:8080", 0, 2),
+		shardTarget("full-02:8081", 0, 3), // same rack host, different destNode
+	}
+	rec := newFlightRecorder()
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := runDistribute(t, targets, files, rec, nil)
+		errCh <- err
+	}()
+
+	waitEnters(t, rec, 4)
+	rec.mu.Lock()
+	if rec.maxFlight != 4 {
+		rec.mu.Unlock()
+		t.Fatalf("no cap: max in-flight=%d, want 4 (one per dest disk/node)", rec.maxFlight)
+	}
+	if rec.keyMax != 1 {
+		rec.mu.Unlock()
+		t.Fatalf("per-key in-flight=%d, want 1", rec.keyMax)
+	}
+	rec.mu.Unlock()
+	rec.release()
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if len(rec.sent) != 4 {
+		t.Fatalf("sent %d, want 4: %v", len(rec.sent), rec.sent)
+	}
+}
+
+func TestDistributeEcShardsSameDiskQueued(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 0, 0, 1),
+	}
+	rec := newFlightRecorder()
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := runDistribute(t, targets, files, rec, nil)
+		errCh <- err
+	}()
+
+	waitEnters(t, rec, 1)
+	assertNoExtraEnter(t, rec, 40*time.Millisecond)
+	rec.release()
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if rec.maxFlight != 1 {
+		t.Fatalf("same disk: max in-flight=%d, want 1", rec.maxFlight)
+	}
+	if rec.sent[0] != "full-01:8080/ec00" || rec.sent[1] != "full-01:8080/ec01" {
+		t.Fatalf("same-disk order %v, want ec00 then ec01", rec.sent)
+	}
+}
+
+func TestDistributeEcShardsAutoDiskSerializesServer(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01", "ec02")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 0, 0), // disk picked by the receiver
+		shardTarget("full-01:8080", 1, 1),
+		shardTarget("full-02:8080", 1, 2),
+	}
+	rec := newFlightRecorder()
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := runDistribute(t, targets, files, rec, nil)
+		errCh <- err
+	}()
+
+	waitEnters(t, rec, 2)
+	rec.mu.Lock()
+	if rec.maxFlight != 2 {
+		rec.mu.Unlock()
+		t.Fatalf("auto-placement server must serialize: in-flight=%d, want 2 (one per server)", rec.maxFlight)
+	}
+	rec.mu.Unlock()
+	rec.release()
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if len(rec.sent) != 3 {
+		t.Fatalf("sent %d, want 3", len(rec.sent))
+	}
+}
+
+func TestDistributeEcShardsTwoDisksTwoStreams(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 1, 0),
+		shardTarget("full-01:8080", 2, 1),
+	}
+	rec := newFlightRecorder()
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := runDistribute(t, targets, files, rec, nil)
+		errCh <- err
+	}()
+
+	waitEnters(t, rec, 2)
+	rec.mu.Lock()
+	if rec.maxFlight != 2 {
+		rec.mu.Unlock()
+		t.Fatalf("two disks on one VS: in-flight=%d, want 2", rec.maxFlight)
+	}
+	rec.mu.Unlock()
+	rec.release()
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestDistributeEcShardsCapNeverExceedsN(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01", "ec02", "ec03")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 1, 0),
+		shardTarget("full-01:8080", 2, 1),
+		shardTarget("full-02:8080", 1, 2),
+		shardTarget("full-02:8080", 2, 3),
+	}
+	rec := newFlightRecorder()
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := runDistribute(t, targets, files, rec, intPtr(2))
+		errCh <- err
+	}()
+
+	waitEnters(t, rec, 2)
+	assertNoExtraEnter(t, rec, 40*time.Millisecond)
+	rec.mu.Lock()
+	if rec.maxFlight != 2 {
+		rec.mu.Unlock()
+		t.Fatalf("cap 2: in-flight=%d, want 2", rec.maxFlight)
+	}
+	rec.mu.Unlock()
+	rec.release()
+	if err := <-errCh; err != nil {
+		t.Fatal(err)
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if rec.maxFlight > 2 {
+		t.Fatalf("cap 2: max in-flight=%d", rec.maxFlight)
+	}
+	if rec.keyMax != 1 {
+		t.Fatalf("per-key in-flight=%d, want 1 under cap", rec.keyMax)
+	}
+	if len(rec.sent) != 4 {
+		t.Fatalf("sent %d, want 4", len(rec.sent))
+	}
+}
+
+func TestDistributeEcShardsErrorAbortSameKey(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01", "ec02")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 0, 0, 1, 2),
+	}
+	rec := newFlightRecorder()
+	rec.noGate.Store(true)
+	rec.failOn = "full-01:8080/ec01"
+
+	_, err := runDistribute(t, targets, files, rec, nil)
+	if err == nil {
+		t.Fatal("expected send error")
+	}
+	if !strings.Contains(err.Error(), "failed to send ec01 to full-01:8080") {
+		t.Fatalf("error wrap: %v", err)
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if got := strings.Join(rec.attempts, ","); got != "full-01:8080/ec00,full-01:8080/ec01" {
+		t.Fatalf("attempts %v, want ec00 then ec01 (ec02 aborted)", rec.attempts)
+	}
+	if len(rec.sent) != 1 || rec.sent[0] != "full-01:8080/ec00" {
+		t.Fatalf("successful sends %v, want only ec00", rec.sent)
+	}
+}
+
+func TestDistributeEcShardsErrorAbortCancelsOtherKeys(t *testing.T) {
+	files := writeShardFiles(t, "ec00", "ec01", "ec02")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 0, 0),
+		shardTarget("full-02:8080", 0, 1),
+		shardTarget("full-03:8080", 0, 2),
+	}
+	rec := newFlightRecorder()
+	rec.noGate.Store(true)
+	rec.failFirst = true
+
+	_, err := runDistribute(t, targets, files, rec, intPtr(1))
+	if err == nil {
+		t.Fatal("expected send error")
+	}
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	if len(rec.attempts) != 1 {
+		t.Fatalf("cap=1 abort: attempts %v, want only the failing send", rec.attempts)
+	}
+	if len(rec.sent) != 0 {
+		t.Fatalf("successful sends %v, want none", rec.sent)
+	}
+}
+
+func TestDistributeEcShardsMissingFile(t *testing.T) {
+	files := writeShardFiles(t, "ec00")
+	targets := []*worker_pb.TaskTarget{
+		shardTarget("full-01:8080", 0, 1),
+	}
+	for _, max := range []*int{intPtr(0), nil} {
+		_, err := DistributeEcShards(1, "col", targets, files, nil, nil, &ShardDistributionOptions{
+			MaxShardTransfers: max,
+			send: func(context.Context, uint32, string, grpc.DialOption, string, uint32, string, string) error {
+				t.Fatal("send should not run for missing file")
+				return nil
+			},
+		})
+		if err == nil || !strings.Contains(err.Error(), "shard file ec01 not found") {
+			t.Fatalf("missing file error (max=%v): %v", max, err)
+		}
+	}
+}
+
+func TestShardTransferKeyUsesDestNodeNotRack(t *testing.T) {
+	a := shardTransferKey(shardJob{destNode: "full-01:8080", diskID: 0, hasDisk: true})
+	b := shardTransferKey(shardJob{destNode: "full-01:8081", diskID: 0, hasDisk: true})
+	if a == b {
+		t.Fatalf("host:port keys collided: %s", a)
+	}
+	noDisk := shardTransferKey(shardJob{destNode: "full-01:8080", hasDisk: false})
+	if noDisk != "full-01:8080" {
+		t.Fatalf("no disk id key=%q, want destNode", noDisk)
+	}
+	if fmt.Sprint(a) == noDisk {
+		t.Fatalf("disk 0 key should differ from destNode-only when hasDisk")
+	}
+}