rust volume: score EC shard placement with the volume's own ratio (#11862)

The auto-select branches of ReceiveFile (is_ec_volume) and
VolumeEcShardsCopy passed the build-default DATA_SHARDS_COUNT into
find_ec_shard_target_location, so its free-slot maths scored every
volume as 10+4 even when the volume's .vif declares a custom ratio
(e.g. 6+2). A wider-ratio volume's shards could be steered onto a disk
too full to hold them all, and a narrower one would be refused slots
it actually has.

Add Store::lookup_ec_data_shards(collection, vid), which answers from
the mounted EC volume first and then from the .vif on whichever disk
holds it (via read_ec_shard_config_across_dirs, so split -dir/-dir.idx
and rebuild-staging layouts are covered), falling back to the default
when no metadata exists yet.

Mirrors the enterprise fix (seaweed-mono c22fa8a15).
This commit is contained in:
Chris Lu authored and GitHub committed 2026-10-20 07:40:43 -07:00
1 parent 4d06742664
commit 6936bbdc3e
2 files changed
+75 -10

No files matched your search

+12 -9
View File
@@ -16,7 +16,7 @@ use crate::pb::filer_pb;
use crate::pb::master_pb;
use crate::pb::volume_server_pb;
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
use crate::storage::erasure_coding::ec_shard::{DATA_SHARDS_COUNT, ShardId, shard_id_try_from};
use crate::storage::erasure_coding::ec_shard::{ShardId, shard_id_try_from};
use crate::storage::erasure_coding::ecj_merge::EcjIdDecoder;
use crate::storage::needle::needle::{self, Needle};
use crate::storage::types::*;
@@ -2317,9 +2317,9 @@ impl VolumeServer for VolumeGrpcService {
// owns the .ecx on disk (volume not yet mounted — relevant
// when shards stream in mid-rebuild before any
// VolumeEcShardsMount has happened; see #9212), then any
// HDD, then any disk. Pass the build's default data-shard
// count for free-slot maths; the helper takes it as a
// parameter so custom-ratio builds can swap it.
// HDD, then any disk. The free-slot maths uses the
// volume's own data-shard count from its .vif so a
// custom-ratio volume is not scored as 10+4.
let vid = VolumeId(info.volume_id);
let dir = if info.disk_id > 0 {
let count = store.locations.len();
@@ -2334,11 +2334,13 @@ impl VolumeServer for VolumeGrpcService {
} else {
// The mounted-volume refusal above means no disk
// holds an in-memory claim here.
let data_shards =
store.lookup_ec_data_shards(&info.collection, vid);
store
.find_ec_shard_target_location(
&info.collection,
vid,
DATA_SHARDS_COUNT as u32,
data_shards,
&[],
)
.map(|i| store.locations[i].directory.clone())
@@ -3288,9 +3290,9 @@ impl VolumeServer for VolumeGrpcService {
// volume mounted, then a disk that owns the .ecx on disk (volume
// not yet mounted — relevant for ec.rebuild, where only the
// first shard carries .ecx and subsequent shards must land on
// the same disk; see #9212), then any HDD, then any disk. Pass
// the build's default data-shard count; the helper takes it as a
// parameter so custom-ratio builds can swap it.
// the same disk; see #9212), then any HDD, then any disk. The
// free-slot maths uses the volume's own data-shard count from its
// .vif so a custom-ratio volume is not scored as 10+4.
let (dest_dir, dest_idx_dir) = {
let store = self.state.store.read().unwrap();
let count = store.locations.len();
@@ -3322,10 +3324,11 @@ impl VolumeServer for VolumeGrpcService {
req.volume_id, shard_ids, dirs
)));
}
let data_shards = store.lookup_ec_data_shards(&req.collection, vid);
match store.find_ec_shard_target_location(
&req.collection,
vid,
DATA_SHARDS_COUNT as u32,
data_shards,
&shard_ids,
) {
Some(i) => {
+63 -1
View File
@@ -14,7 +14,7 @@ use crate::pb::master_pb;
use crate::storage::disk_location::DiskLocation;
use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
use crate::storage::erasure_coding::ec_volume::{
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, read_ec_shard_config_across_dirs,
};
use crate::storage::needle::needle::{Needle, get_actual_size};
use crate::storage::needle_map::NeedleMapKind;
@@ -416,6 +416,39 @@ impl Store {
.collect()
}
/// Look up the per-volume EC data-shard count, preferring a mounted EC
/// volume's own config and then the `.vif` on whichever disk holds it.
/// Falls back to the default when neither exists yet (e.g. before the
/// first shard streams in during ec.rebuild), so
/// `find_ec_shard_target_location`'s free-slot maths scores custom-ratio
/// volumes against their real width instead of the build default.
pub fn lookup_ec_data_shards(&self, collection: &str, vid: VolumeId) -> u32 {
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
for loc in &self.locations {
if let Some(ec_vol) = loc.find_ec_volume(vid) {
return ec_vol.data_shards;
}
}
let Some(first) = self.locations.first() else {
return DATA_SHARDS_COUNT as u32;
};
let other_dirs: Vec<String> = self.locations[1..]
.iter()
.flat_map(|loc| [loc.directory.clone(), loc.idx_directory.clone()])
.filter(|d| !d.is_empty())
.collect();
read_ec_shard_config_across_dirs(
&first.directory,
&first.idx_directory,
&other_dirs,
collection,
vid,
)
.map(|(data_shards, _, _)| data_shards)
.unwrap_or(DATA_SHARDS_COUNT as u32)
}
/// Create a new volume, placing it on the location with the most free space.
pub fn add_volume(
&mut self,
@@ -2980,6 +3013,35 @@ mod tests {
);
}
/// A volume whose `.vif` declares a custom EC ratio must feed its own
/// data-shard count into the free-slot maths — not the build default —
/// wherever the `.vif` happens to sit. With no `.vif` anywhere (a
/// brand-new volume mid-ec.rebuild) the answer is the default.
#[test]
fn test_lookup_ec_data_shards_reads_vif_on_any_disk() {
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
let (store, _tmp) = make_ec_target_test_store(3);
let collection = "custom-ratio";
let vid = VolumeId(4242);
assert_eq!(
store.lookup_ec_data_shards(collection, vid),
DATA_SHARDS_COUNT as u32,
"no .vif anywhere must answer the default"
);
// A 6+2 .vif lands on disk 2 only — the rebuild staging disk.
let base = volume_file_name(&store.locations[2].directory, collection, vid);
std::fs::write(
format!("{}.vif", base),
br#"{"ecShardConfig":{"dataShards":6,"parityShards":2}}"#,
)
.unwrap();
assert_eq!(store.lookup_ec_data_shards(collection, vid), 6);
}
/// Deleting a disk's last shard removes that disk's now-orphaned `.ecsum`;
/// a disk this delete never changed keeps its sidecar. Disk 0 models
/// ec.rebuild's index-prep state (`.ecsum` landed, no shards yet) — an