mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-21 05:24:28 +02:00
rust volume: score EC shard placement with the volume's own ratio (#11862)
The auto-select branches of ReceiveFile (is_ec_volume) and VolumeEcShardsCopy passed the build-default DATA_SHARDS_COUNT into find_ec_shard_target_location, so its free-slot maths scored every volume as 10+4 even when the volume's .vif declares a custom ratio (e.g. 6+2). A wider-ratio volume's shards could be steered onto a disk too full to hold them all, and a narrower one would be refused slots it actually has. Add Store::lookup_ec_data_shards(collection, vid), which answers from the mounted EC volume first and then from the .vif on whichever disk holds it (via read_ec_shard_config_across_dirs, so split -dir/-dir.idx and rebuild-staging layouts are covered), falling back to the default when no metadata exists yet. Mirrors the enterprise fix (seaweed-mono c22fa8a15).
This commit is contained in:
1 parent
4d06742664
commit
6936bbdc3e
2 files changed
+75
-10
No files matched your search
@@ -16,7 +16,7 @@ use crate::pb::filer_pb;
|
||||
use crate::pb::master_pb;
|
||||
use crate::pb::volume_server_pb;
|
||||
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
|
||||
use crate::storage::erasure_coding::ec_shard::{DATA_SHARDS_COUNT, ShardId, shard_id_try_from};
|
||||
use crate::storage::erasure_coding::ec_shard::{ShardId, shard_id_try_from};
|
||||
use crate::storage::erasure_coding::ecj_merge::EcjIdDecoder;
|
||||
use crate::storage::needle::needle::{self, Needle};
|
||||
use crate::storage::types::*;
|
||||
@@ -2317,9 +2317,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// owns the .ecx on disk (volume not yet mounted — relevant
|
||||
// when shards stream in mid-rebuild before any
|
||||
// VolumeEcShardsMount has happened; see #9212), then any
|
||||
// HDD, then any disk. Pass the build's default data-shard
|
||||
// count for free-slot maths; the helper takes it as a
|
||||
// parameter so custom-ratio builds can swap it.
|
||||
// HDD, then any disk. The free-slot maths uses the
|
||||
// volume's own data-shard count from its .vif so a
|
||||
// custom-ratio volume is not scored as 10+4.
|
||||
let vid = VolumeId(info.volume_id);
|
||||
let dir = if info.disk_id > 0 {
|
||||
let count = store.locations.len();
|
||||
@@ -2334,11 +2334,13 @@ impl VolumeServer for VolumeGrpcService {
|
||||
} else {
|
||||
// The mounted-volume refusal above means no disk
|
||||
// holds an in-memory claim here.
|
||||
let data_shards =
|
||||
store.lookup_ec_data_shards(&info.collection, vid);
|
||||
store
|
||||
.find_ec_shard_target_location(
|
||||
&info.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
data_shards,
|
||||
&[],
|
||||
)
|
||||
.map(|i| store.locations[i].directory.clone())
|
||||
@@ -3288,9 +3290,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// volume mounted, then a disk that owns the .ecx on disk (volume
|
||||
// not yet mounted — relevant for ec.rebuild, where only the
|
||||
// first shard carries .ecx and subsequent shards must land on
|
||||
// the same disk; see #9212), then any HDD, then any disk. Pass
|
||||
// the build's default data-shard count; the helper takes it as a
|
||||
// parameter so custom-ratio builds can swap it.
|
||||
// the same disk; see #9212), then any HDD, then any disk. The
|
||||
// free-slot maths uses the volume's own data-shard count from its
|
||||
// .vif so a custom-ratio volume is not scored as 10+4.
|
||||
let (dest_dir, dest_idx_dir) = {
|
||||
let store = self.state.store.read().unwrap();
|
||||
let count = store.locations.len();
|
||||
@@ -3322,10 +3324,11 @@ impl VolumeServer for VolumeGrpcService {
|
||||
req.volume_id, shard_ids, dirs
|
||||
)));
|
||||
}
|
||||
let data_shards = store.lookup_ec_data_shards(&req.collection, vid);
|
||||
match store.find_ec_shard_target_location(
|
||||
&req.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
data_shards,
|
||||
&shard_ids,
|
||||
) {
|
||||
Some(i) => {
|
||||
|
||||
@@ -14,7 +14,7 @@ use crate::pb::master_pb;
|
||||
use crate::storage::disk_location::DiskLocation;
|
||||
use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
|
||||
use crate::storage::erasure_coding::ec_volume::{
|
||||
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
|
||||
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, read_ec_shard_config_across_dirs,
|
||||
};
|
||||
use crate::storage::needle::needle::{Needle, get_actual_size};
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
@@ -416,6 +416,39 @@ impl Store {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Look up the per-volume EC data-shard count, preferring a mounted EC
|
||||
/// volume's own config and then the `.vif` on whichever disk holds it.
|
||||
/// Falls back to the default when neither exists yet (e.g. before the
|
||||
/// first shard streams in during ec.rebuild), so
|
||||
/// `find_ec_shard_target_location`'s free-slot maths scores custom-ratio
|
||||
/// volumes against their real width instead of the build default.
|
||||
pub fn lookup_ec_data_shards(&self, collection: &str, vid: VolumeId) -> u32 {
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
|
||||
for loc in &self.locations {
|
||||
if let Some(ec_vol) = loc.find_ec_volume(vid) {
|
||||
return ec_vol.data_shards;
|
||||
}
|
||||
}
|
||||
let Some(first) = self.locations.first() else {
|
||||
return DATA_SHARDS_COUNT as u32;
|
||||
};
|
||||
let other_dirs: Vec<String> = self.locations[1..]
|
||||
.iter()
|
||||
.flat_map(|loc| [loc.directory.clone(), loc.idx_directory.clone()])
|
||||
.filter(|d| !d.is_empty())
|
||||
.collect();
|
||||
read_ec_shard_config_across_dirs(
|
||||
&first.directory,
|
||||
&first.idx_directory,
|
||||
&other_dirs,
|
||||
collection,
|
||||
vid,
|
||||
)
|
||||
.map(|(data_shards, _, _)| data_shards)
|
||||
.unwrap_or(DATA_SHARDS_COUNT as u32)
|
||||
}
|
||||
|
||||
/// Create a new volume, placing it on the location with the most free space.
|
||||
pub fn add_volume(
|
||||
&mut self,
|
||||
@@ -2980,6 +3013,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A volume whose `.vif` declares a custom EC ratio must feed its own
|
||||
/// data-shard count into the free-slot maths — not the build default —
|
||||
/// wherever the `.vif` happens to sit. With no `.vif` anywhere (a
|
||||
/// brand-new volume mid-ec.rebuild) the answer is the default.
|
||||
#[test]
|
||||
fn test_lookup_ec_data_shards_reads_vif_on_any_disk() {
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
|
||||
let (store, _tmp) = make_ec_target_test_store(3);
|
||||
let collection = "custom-ratio";
|
||||
let vid = VolumeId(4242);
|
||||
|
||||
assert_eq!(
|
||||
store.lookup_ec_data_shards(collection, vid),
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
"no .vif anywhere must answer the default"
|
||||
);
|
||||
|
||||
// A 6+2 .vif lands on disk 2 only — the rebuild staging disk.
|
||||
let base = volume_file_name(&store.locations[2].directory, collection, vid);
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
br#"{"ecShardConfig":{"dataShards":6,"parityShards":2}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(store.lookup_ec_data_shards(collection, vid), 6);
|
||||
}
|
||||
|
||||
/// Deleting a disk's last shard removes that disk's now-orphaned `.ecsum`;
|
||||
/// a disk this delete never changed keeps its sidecar. Disk 0 models
|
||||
/// ec.rebuild's index-prep state (`.ecsum` landed, no shards yet) — an
|
||||
|
||||
Reference in new issue
Block a user