fix(ec): honor disk_id in ReceiveFile so EC shards respect admin placement (#9184) (#9185)

* test(volume_server): reproduce #9184 EC ReceiveFile disk-placement bug

The plugin-worker EC task sends shards via ReceiveFile, which picks
Locations[0] as the target directory regardless of the admin planner's
TargetDisk assignment. ReceiveFileInfo has no disk_id field, so there
is no wire channel to honor the plan.

Adds StartSingleVolumeClusterWithDataDirs to the integration framework
so tests can launch a volume server with N data directories. The new
repro asserts the current (buggy) behavior: sending three distinct EC
shards via ReceiveFile leaves all three files in dir[0] and the other
dirs empty. When the fix adds disk_id to ReceiveFileInfo, this
assertion must flip to verify the planned placement is respected.

* fix(ec): honor disk_id in ReceiveFile so EC shards respect admin placement

Before this change, VolumeServer.ReceiveFile for EC shards always
selected the first HDD location (Locations[0]). The plugin-worker EC
task had no way to pass the admin planner's per-shard disk
assignment — ReceiveFileInfo carried no disk_id field — so every
received EC shard piled onto a single disk per destination server.
On multi-disk servers this caused uneven load (one disk absorbing all
EC shard I/O), frequent ENOSPC retries, and a growing EC backlog
under sustained ingest (see issue #9184).

Changes:
- proto: add disk_id to ReceiveFileInfo, mirroring
  VolumeEcShardsCopyRequest.disk_id.
- worker: DistributeEcShards tracks the planner-assigned disk per
  shard; sendShardFileToDestination forwards that disk id. Metadata
  files (ecx/ecj/vif) inherit the disk of the first data shard
  targeting the same node so they land next to the shards.
- server: ReceiveFile honors disk_id when > 0 with bounds
  validation; disk_id=0 (unset) falls back to the same
  auto-selection pattern as VolumeEcShardsCopy (prefer disk that
  already has shards for this volume, then any HDD with free space,
  then any location with free space).

Tests updated:
- TestReceiveFileEcShardHonorsDiskID asserts three shards sent with
  disk_id={1,2,0} land on data dirs 1, 2, and 0 respectively.
- TestReceiveFileEcShardRejectsInvalidDiskID pins the out-of-range
  disk_id rejection path.

* fix(volume-rust): honor disk_id in ReceiveFile for EC shards

Mirror the Go-side change: when disk_id > 0 place the EC shard on the
requested disk; when unset, auto-select with the same preference order
as volume_ec_shards_copy (disk already holding shards, then any HDD,
then any disk).

* fix(volume): compare disk_id as uint32 to avoid 32-bit overflow

On 32-bit Go builds `int(fileInfo.DiskId) >= len(Locations)` can wrap a
high-bit uint32 to a negative int, bypassing the bounds check before the
index operation. Compare in the uint32 domain instead.

* test(ec): fail invalid-disk_id test on transport error

Previously a transport-level error from CloseAndRecv silently passed the
test by returning early, masking any real gRPC failure. Fail loudly so
only the structured ReceiveFileResponse rejection path counts as a pass.

* docs(test): explain why DiskId=0 auto-selects dir 0 in EC placement test

Documents the load-bearing assumption that shards are never mounted in
this test, so loc.FindEcVolume always returns false and auto-select
falls through to the first HDD. Saves future readers from re-deriving
the expected directory for the DiskId=0 case.

* fix(test): preserve baseDir/volume path for single-dir clusters

StartSingleVolumeClusterWithDataDirs started naming the data directory
volume0 even in the dataDirCount=1 case, which broke Scrub tests that
reach into baseDir/volume via CorruptDatFile / CorruptEcShardFile /
CorruptEcxFile. Keep the legacy name for single-dir clusters; only use
the indexed "volumeN" layout when multiple disks are requested.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-04-22 10:30:13 -07:00
1 parent 0f5e99f423
commit c4e1885053
8 files changed
+310 -33

No files matched your search

@@ -0,0 +1,178 @@
package volume_server_grpc_test
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
// Sends EC shards with disk_id={1, 2, 0} and verifies each lands on the
// requested disk (0 → auto-select → disk 0 for a fresh volume).
func TestReceiveFileEcShardHonorsDiskID(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const dataDirCount = 3
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
dataDirs := clusterHarness.VolumeDataDirs()
if len(dataDirs) != dataDirCount {
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
const volumeID = uint32(9184)
const collection = "ec-disk-placement"
// DiskId=0 triggers auto-select. Since this test never calls
// VolumeEcShardsMount, loc.FindEcVolume returns false on every disk, so
// the "already holds shards" preference is skipped and auto-select falls
// through to the first HDD (dir 0). If shards get mounted between uploads
// in a future revision, the expected dir for DiskId=0 becomes the first
// disk that already holds a shard.
shards := []struct {
shardID uint32
requestedDisk uint32
expectedDirIdx int
}{
{shardID: 0, requestedDisk: 1, expectedDirIdx: 1},
{shardID: 5, requestedDisk: 2, expectedDirIdx: 2},
{shardID: 13, requestedDisk: 0, expectedDirIdx: 0},
}
shardExt := func(id uint32) string { return fmt.Sprintf(".ec%02d", id) }
for _, s := range shards {
payload := []byte(fmt.Sprintf("ec-shard-payload-%02d", s.shardID))
stream, err := grpcClient.ReceiveFile(ctx)
if err != nil {
t.Fatalf("ReceiveFile stream create for shard %d: %v", s.shardID, err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: shardExt(s.shardID),
Collection: collection,
IsEcVolume: true,
ShardId: s.shardID,
FileSize: uint64(len(payload)),
DiskId: s.requestedDisk,
},
},
}); err != nil {
t.Fatalf("ReceiveFile send info for shard %d: %v", s.shardID, err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payload},
}); err != nil {
t.Fatalf("ReceiveFile send content for shard %d: %v", s.shardID, err)
}
resp, err := stream.CloseAndRecv()
if err != nil {
t.Fatalf("ReceiveFile close for shard %d: %v", s.shardID, err)
}
if resp.GetError() != "" {
t.Fatalf("ReceiveFile shard %d error response: %s", s.shardID, resp.GetError())
}
if resp.GetBytesWritten() != uint64(len(payload)) {
t.Fatalf("ReceiveFile shard %d bytes_written mismatch: got %d want %d",
s.shardID, resp.GetBytesWritten(), len(payload))
}
}
// Scan every data dir and record which shard files live where.
perDirShards := make(map[int][]uint32)
for dirIdx, dir := range dataDirs {
for _, s := range shards {
shardPath := filepath.Join(dir, fmt.Sprintf("%s_%d%s", collection, volumeID, shardExt(s.shardID)))
if _, err := os.Stat(shardPath); err == nil {
perDirShards[dirIdx] = append(perDirShards[dirIdx], s.shardID)
} else if !os.IsNotExist(err) {
t.Fatalf("unexpected stat error for %s: %v", shardPath, err)
}
}
}
for _, s := range shards {
found := false
for _, got := range perDirShards[s.expectedDirIdx] {
if got == s.shardID {
found = true
break
}
}
if !found {
t.Fatalf("shard %d (requestedDisk=%d) expected on dir[%d], full layout: %v",
s.shardID, s.requestedDisk, s.expectedDirIdx, perDirShards)
}
for dirIdx, ids := range perDirShards {
if dirIdx == s.expectedDirIdx {
continue
}
for _, id := range ids {
if id == s.shardID {
t.Fatalf("shard %d leaked onto dir[%d]: full layout: %v", s.shardID, dirIdx, perDirShards)
}
}
}
}
t.Logf("disk_id honored: shard placement = %v", perDirShards)
}
func TestReceiveFileEcShardRejectsInvalidDiskID(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const dataDirCount = 2
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
const volumeID = uint32(91840)
const collection = "ec-invalid-disk"
payload := []byte("invalid-disk-id-payload")
stream, err := grpcClient.ReceiveFile(ctx)
if err != nil {
t.Fatalf("ReceiveFile stream create: %v", err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: ".ec00",
Collection: collection,
IsEcVolume: true,
ShardId: 0,
FileSize: uint64(len(payload)),
DiskId: 99,
},
},
}); err != nil {
t.Fatalf("ReceiveFile send info: %v", err)
}
resp, err := stream.CloseAndRecv()
if err != nil {
t.Fatalf("ReceiveFile close: %v", err)
}
if resp.GetError() == "" {
t.Fatalf("expected invalid disk_id rejection, got success response: %+v", resp)
}
}