From 6936bbdc3e518ec8dc180467808eeeddfcb8723c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 20 Oct 2026 07:40:43 -0700 Subject: [PATCH] rust volume: score EC shard placement with the volume's own ratio (#11862) The auto-select branches of ReceiveFile (is_ec_volume) and VolumeEcShardsCopy passed the build-default DATA_SHARDS_COUNT into find_ec_shard_target_location, so its free-slot maths scored every volume as 10+4 even when the volume's .vif declares a custom ratio (e.g. 6+2). A wider-ratio volume's shards could be steered onto a disk too full to hold them all, and a narrower one would be refused slots it actually has. Add Store::lookup_ec_data_shards(collection, vid), which answers from the mounted EC volume first and then from the .vif on whichever disk holds it (via read_ec_shard_config_across_dirs, so split -dir/-dir.idx and rebuild-staging layouts are covered), falling back to the default when no metadata exists yet. Mirrors the enterprise fix (seaweed-mono c22fa8a15). --- seaweed-volume/src/server/grpc_server.rs | 21 ++++---- seaweed-volume/src/storage/store.rs | 64 +++++++++++++++++++++++- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index efd86474aa..4536594200 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -16,7 +16,7 @@ use crate::pb::filer_pb; use crate::pb::master_pb; use crate::pb::volume_server_pb; use crate::pb::volume_server_pb::volume_server_server::VolumeServer; -use crate::storage::erasure_coding::ec_shard::{DATA_SHARDS_COUNT, ShardId, shard_id_try_from}; +use crate::storage::erasure_coding::ec_shard::{ShardId, shard_id_try_from}; use crate::storage::erasure_coding::ecj_merge::EcjIdDecoder; use crate::storage::needle::needle::{self, Needle}; use crate::storage::types::*; @@ -2317,9 +2317,9 @@ impl VolumeServer for VolumeGrpcService { // owns the .ecx on disk (volume not yet mounted — relevant // when shards stream in mid-rebuild before any // VolumeEcShardsMount has happened; see #9212), then any - // HDD, then any disk. Pass the build's default data-shard - // count for free-slot maths; the helper takes it as a - // parameter so custom-ratio builds can swap it. + // HDD, then any disk. The free-slot maths uses the + // volume's own data-shard count from its .vif so a + // custom-ratio volume is not scored as 10+4. let vid = VolumeId(info.volume_id); let dir = if info.disk_id > 0 { let count = store.locations.len(); @@ -2334,11 +2334,13 @@ impl VolumeServer for VolumeGrpcService { } else { // The mounted-volume refusal above means no disk // holds an in-memory claim here. + let data_shards = + store.lookup_ec_data_shards(&info.collection, vid); store .find_ec_shard_target_location( &info.collection, vid, - DATA_SHARDS_COUNT as u32, + data_shards, &[], ) .map(|i| store.locations[i].directory.clone()) @@ -3288,9 +3290,9 @@ impl VolumeServer for VolumeGrpcService { // volume mounted, then a disk that owns the .ecx on disk (volume // not yet mounted — relevant for ec.rebuild, where only the // first shard carries .ecx and subsequent shards must land on - // the same disk; see #9212), then any HDD, then any disk. Pass - // the build's default data-shard count; the helper takes it as a - // parameter so custom-ratio builds can swap it. + // the same disk; see #9212), then any HDD, then any disk. The + // free-slot maths uses the volume's own data-shard count from its + // .vif so a custom-ratio volume is not scored as 10+4. let (dest_dir, dest_idx_dir) = { let store = self.state.store.read().unwrap(); let count = store.locations.len(); @@ -3322,10 +3324,11 @@ impl VolumeServer for VolumeGrpcService { req.volume_id, shard_ids, dirs ))); } + let data_shards = store.lookup_ec_data_shards(&req.collection, vid); match store.find_ec_shard_target_location( &req.collection, vid, - DATA_SHARDS_COUNT as u32, + data_shards, &shard_ids, ) { Some(i) => { diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index a7f0ab5b4b..014705c9f7 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -14,7 +14,7 @@ use crate::pb::master_pb; use crate::storage::disk_location::DiskLocation; use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId}; use crate::storage::erasure_coding::ec_volume::{ - ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, + ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, read_ec_shard_config_across_dirs, }; use crate::storage::needle::needle::{Needle, get_actual_size}; use crate::storage::needle_map::NeedleMapKind; @@ -416,6 +416,39 @@ impl Store { .collect() } + /// Look up the per-volume EC data-shard count, preferring a mounted EC + /// volume's own config and then the `.vif` on whichever disk holds it. + /// Falls back to the default when neither exists yet (e.g. before the + /// first shard streams in during ec.rebuild), so + /// `find_ec_shard_target_location`'s free-slot maths scores custom-ratio + /// volumes against their real width instead of the build default. + pub fn lookup_ec_data_shards(&self, collection: &str, vid: VolumeId) -> u32 { + use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT; + + for loc in &self.locations { + if let Some(ec_vol) = loc.find_ec_volume(vid) { + return ec_vol.data_shards; + } + } + let Some(first) = self.locations.first() else { + return DATA_SHARDS_COUNT as u32; + }; + let other_dirs: Vec = self.locations[1..] + .iter() + .flat_map(|loc| [loc.directory.clone(), loc.idx_directory.clone()]) + .filter(|d| !d.is_empty()) + .collect(); + read_ec_shard_config_across_dirs( + &first.directory, + &first.idx_directory, + &other_dirs, + collection, + vid, + ) + .map(|(data_shards, _, _)| data_shards) + .unwrap_or(DATA_SHARDS_COUNT as u32) + } + /// Create a new volume, placing it on the location with the most free space. pub fn add_volume( &mut self, @@ -2980,6 +3013,35 @@ mod tests { ); } + /// A volume whose `.vif` declares a custom EC ratio must feed its own + /// data-shard count into the free-slot maths — not the build default — + /// wherever the `.vif` happens to sit. With no `.vif` anywhere (a + /// brand-new volume mid-ec.rebuild) the answer is the default. + #[test] + fn test_lookup_ec_data_shards_reads_vif_on_any_disk() { + use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT; + + let (store, _tmp) = make_ec_target_test_store(3); + let collection = "custom-ratio"; + let vid = VolumeId(4242); + + assert_eq!( + store.lookup_ec_data_shards(collection, vid), + DATA_SHARDS_COUNT as u32, + "no .vif anywhere must answer the default" + ); + + // A 6+2 .vif lands on disk 2 only — the rebuild staging disk. + let base = volume_file_name(&store.locations[2].directory, collection, vid); + std::fs::write( + format!("{}.vif", base), + br#"{"ecShardConfig":{"dataShards":6,"parityShards":2}}"#, + ) + .unwrap(); + + assert_eq!(store.lookup_ec_data_shards(collection, vid), 6); + } + /// Deleting a disk's last shard removes that disk's now-orphaned `.ecsum`; /// a disk this delete never changed keeps its sidecar. Disk 0 models /// ec.rebuild's index-prep state (`.ecsum` landed, no shards yet) — an