mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
* test(volume_server): reproduce #9184 EC ReceiveFile disk-placement bug The plugin-worker EC task sends shards via ReceiveFile, which picks Locations[0] as the target directory regardless of the admin planner's TargetDisk assignment. ReceiveFileInfo has no disk_id field, so there is no wire channel to honor the plan. Adds StartSingleVolumeClusterWithDataDirs to the integration framework so tests can launch a volume server with N data directories. The new repro asserts the current (buggy) behavior: sending three distinct EC shards via ReceiveFile leaves all three files in dir[0] and the other dirs empty. When the fix adds disk_id to ReceiveFileInfo, this assertion must flip to verify the planned placement is respected. * fix(ec): honor disk_id in ReceiveFile so EC shards respect admin placement Before this change, VolumeServer.ReceiveFile for EC shards always selected the first HDD location (Locations[0]). The plugin-worker EC task had no way to pass the admin planner's per-shard disk assignment — ReceiveFileInfo carried no disk_id field — so every received EC shard piled onto a single disk per destination server. On multi-disk servers this caused uneven load (one disk absorbing all EC shard I/O), frequent ENOSPC retries, and a growing EC backlog under sustained ingest (see issue #9184). Changes: - proto: add disk_id to ReceiveFileInfo, mirroring VolumeEcShardsCopyRequest.disk_id. - worker: DistributeEcShards tracks the planner-assigned disk per shard; sendShardFileToDestination forwards that disk id. Metadata files (ecx/ecj/vif) inherit the disk of the first data shard targeting the same node so they land next to the shards. - server: ReceiveFile honors disk_id when > 0 with bounds validation; disk_id=0 (unset) falls back to the same auto-selection pattern as VolumeEcShardsCopy (prefer disk that already has shards for this volume, then any HDD with free space, then any location with free space). Tests updated: - TestReceiveFileEcShardHonorsDiskID asserts three shards sent with disk_id={1,2,0} land on data dirs 1, 2, and 0 respectively. - TestReceiveFileEcShardRejectsInvalidDiskID pins the out-of-range disk_id rejection path. * fix(volume-rust): honor disk_id in ReceiveFile for EC shards Mirror the Go-side change: when disk_id > 0 place the EC shard on the requested disk; when unset, auto-select with the same preference order as volume_ec_shards_copy (disk already holding shards, then any HDD, then any disk). * fix(volume): compare disk_id as uint32 to avoid 32-bit overflow On 32-bit Go builds `int(fileInfo.DiskId) >= len(Locations)` can wrap a high-bit uint32 to a negative int, bypassing the bounds check before the index operation. Compare in the uint32 domain instead. * test(ec): fail invalid-disk_id test on transport error Previously a transport-level error from CloseAndRecv silently passed the test by returning early, masking any real gRPC failure. Fail loudly so only the structured ReceiveFileResponse rejection path counts as a pass. * docs(test): explain why DiskId=0 auto-selects dir 0 in EC placement test Documents the load-bearing assumption that shards are never mounted in this test, so loc.FindEcVolume always returns false and auto-select falls through to the first HDD. Saves future readers from re-deriving the expected directory for the DiskId=0 case. * fix(test): preserve baseDir/volume path for single-dir clusters StartSingleVolumeClusterWithDataDirs started naming the data directory volume0 even in the dataDirCount=1 case, which broke Scrub tests that reach into baseDir/volume via CorruptDatFile / CorruptEcShardFile / CorruptEcxFile. Keep the legacy name for single-dir clusters; only use the indexed "volumeN" layout when multiple disks are requested.
This commit is contained in:
@@ -54,12 +54,25 @@ type Cluster struct {
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmd *exec.Cmd
|
||||
|
||||
volumeDataDirs []string
|
||||
|
||||
cleanupOnce sync.Once
|
||||
}
|
||||
|
||||
// StartSingleVolumeCluster boots one master and one volume server.
|
||||
func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
|
||||
return StartSingleVolumeClusterWithDataDirs(t, profile, 1)
|
||||
}
|
||||
|
||||
// StartSingleVolumeClusterWithDataDirs boots one master and one volume server
|
||||
// with dataDirCount separate data directories (passed to -dir as a comma list).
|
||||
// Each directory becomes its own DiskLocation on the volume server, letting
|
||||
// tests exercise multi-disk EC placement paths.
|
||||
func StartSingleVolumeClusterWithDataDirs(t testing.TB, profile matrix.Profile, dataDirCount int) *Cluster {
|
||||
t.Helper()
|
||||
if dataDirCount < 1 {
|
||||
t.Fatalf("dataDirCount must be >= 1, got %d", dataDirCount)
|
||||
}
|
||||
|
||||
weedBinary, err := FindOrBuildWeedBinary()
|
||||
if err != nil {
|
||||
@@ -74,8 +87,19 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
|
||||
configDir := filepath.Join(baseDir, "config")
|
||||
logsDir := filepath.Join(baseDir, "logs")
|
||||
masterDataDir := filepath.Join(baseDir, "master")
|
||||
volumeDataDir := filepath.Join(baseDir, "volume")
|
||||
for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} {
|
||||
volumeDataDirs := make([]string, dataDirCount)
|
||||
// Single-dir layout stays at baseDir/volume so existing fixtures
|
||||
// (CorruptDatFile etc.) that hardcode that path keep working. Only
|
||||
// multi-dir clusters get the "volumeN" layout.
|
||||
for i := 0; i < dataDirCount; i++ {
|
||||
if dataDirCount == 1 {
|
||||
volumeDataDirs[i] = filepath.Join(baseDir, "volume")
|
||||
} else {
|
||||
volumeDataDirs[i] = filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
|
||||
}
|
||||
}
|
||||
setupDirs := append([]string{configDir, logsDir, masterDataDir}, volumeDataDirs...)
|
||||
for _, dir := range setupDirs {
|
||||
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
|
||||
t.Fatalf("create %s: %v", dir, mkErr)
|
||||
}
|
||||
@@ -120,11 +144,12 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
|
||||
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
|
||||
}
|
||||
|
||||
if err = c.startVolume(volumeDataDir); err != nil {
|
||||
if err = c.startVolume(volumeDataDirs); err != nil {
|
||||
masterLog := c.tailLog("master.log")
|
||||
c.Stop()
|
||||
t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog)
|
||||
}
|
||||
c.volumeDataDirs = volumeDataDirs
|
||||
if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
|
||||
volumeLog := c.tailLog("volume.log")
|
||||
c.Stop()
|
||||
@@ -184,12 +209,16 @@ func (c *Cluster) startMaster(dataDir string) error {
|
||||
return c.masterCmd.Start()
|
||||
}
|
||||
|
||||
func (c *Cluster) startVolume(dataDir string) error {
|
||||
func (c *Cluster) startVolume(dataDirs []string) error {
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
maxPerDir := make([]string, len(dataDirs))
|
||||
for i := range dataDirs {
|
||||
maxPerDir[i] = "16"
|
||||
}
|
||||
args := []string{
|
||||
"-config_dir=" + c.configDir,
|
||||
"volume",
|
||||
@@ -197,8 +226,8 @@ func (c *Cluster) startVolume(dataDir string) error {
|
||||
"-port=" + strconv.Itoa(c.volumePort),
|
||||
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
|
||||
"-port.public=" + strconv.Itoa(c.volumePubPort),
|
||||
"-dir=" + dataDir,
|
||||
"-max=16",
|
||||
"-dir=" + strings.Join(dataDirs, ","),
|
||||
"-max=" + strings.Join(maxPerDir, ","),
|
||||
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
|
||||
"-readMode=" + c.profile.ReadMode,
|
||||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
|
||||
@@ -415,3 +444,10 @@ func (c *Cluster) VolumePublicURL() string {
|
||||
func (c *Cluster) BaseDir() string {
|
||||
return c.baseDir
|
||||
}
|
||||
|
||||
// VolumeDataDirs returns the data directories the volume server was started with.
|
||||
// Index 0 corresponds to DiskLocation 0, index 1 to DiskLocation 1, and so on.
|
||||
// Tests can scan these directories to verify where files physically landed.
|
||||
func (c *Cluster) VolumeDataDirs() []string {
|
||||
return append([]string(nil), c.volumeDataDirs...)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user