diff --git a/seaweed-volume/src/main.rs b/seaweed-volume/src/main.rs index 3216e1ebeb..33a800a18f 100644 --- a/seaweed-volume/src/main.rs +++ b/seaweed-volume/src/main.rs @@ -450,6 +450,7 @@ async fn run( { let mut store = state.store.write().unwrap(); + store.store_dirs = config.folders.clone(); for (i, dir) in config.folders.iter().enumerate() { let idx_dir = if config.idx_folder.is_empty() { dir.as_str() diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index 28677bfa35..ad8578b7bf 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -58,6 +58,8 @@ pub struct DiskLocation { /// skips a claimed vid outright and claims any other through its load, /// so a VolumeCopy's .note sweep cannot unlink files it still writes. pub(crate) load_guard: Option>>>, + /// The data directory of every disk on this server. + pub(crate) store_dirs: Vec, } /// Keeps `vid` in the copy-claim set through one scan load so a VolumeCopy @@ -123,6 +125,7 @@ impl DiskLocation { disk_free_bytes: AtomicU64::new(0), min_free_space, load_guard: None, + store_dirs: Vec::new(), }) } @@ -486,8 +489,20 @@ impl DiskLocation { // Custom ratio comes from the volume's own .vif; the server holds no // cluster EC config in memory. - let data_shards = - ec_data_shards_from_vif(&self.directory, &self.idx_directory, collection, vid); + let data_shards = ec_data_shards_from_vif( + &self.directory, + &self.idx_directory, + &self.store_dirs, + collection, + vid, + ); + if data_shards == 0 { + warn!( + volume_id = vid.0, + "sibling .vif files disagree on the encode; keeping EC shards" + ); + return true; + } // On-disk .dat: an empty <= superblock one is a stub; a transient stat // error keeps the shards rather than deleting. @@ -1020,7 +1035,9 @@ impl DiskLocation { // propagate the error to the caller. let created = !self.ec_volumes.contains_key(&vid); if created { - let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid).map_err(VolumeError::Io)?; + let ec_vol = + EcVolume::new_with_vif_dirs(&dir, idx_dir, &self.store_dirs, collection, vid) + .map_err(VolumeError::Io)?; self.ec_volumes.insert(vid, ec_vol); } let ec_vol = self.ec_volumes.get_mut(&vid).expect("just inserted above"); @@ -1441,6 +1458,7 @@ fn rm_if_present(path: String) -> io::Result<()> { fn ec_data_shards_from_vif( directory: &str, idx_directory: &str, + store_dirs: &[String], collection: &str, vid: VolumeId, ) -> usize { @@ -1459,6 +1477,23 @@ fn ec_data_shards_from_vif( break; } } + match crate::storage::erasure_coding::ec_volume::sibling_ec_config_vif( + directory, + idx_directory, + store_dirs, + collection, + vid, + ) { + Err(()) => return 0, + Ok(Some(vi)) => { + if let Some(ds) = vi.ec_shard_config.map(|c| c.data_shards as usize) + && ds > 0 + { + return ds; + } + } + Ok(None) => {} + } DATA_SHARDS_COUNT } @@ -1542,7 +1577,7 @@ pub(crate) fn vif_references_remote_file(vif_path: &str) -> bool { /// True when a `.vif` records an EC shard config, i.e. the volume was /// erasure-coded. Such a volume keeps no local `.dat`, so an empty `.dat` /// alongside it is a leftover stub safe to remove. -fn vif_is_ec_volume(vif_path: &str) -> bool { +pub(crate) fn vif_is_ec_volume(vif_path: &str) -> bool { fs::read_to_string(vif_path) .ok() .and_then(|s| serde_json::from_str::(&s).ok()) diff --git a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs index 29433f052c..e3a8dd6d85 100644 --- a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs +++ b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs @@ -272,6 +272,96 @@ fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: Volume data_vif } +/// [`locate_vif_path`], except that a data-dir `.vif` recording no EC config +/// yields to the idx dir's when that one records it: a source volume's own +/// `.vif` stays on a disk that later receives shards of the same volume id, +/// while the encode's `.vif` lands with the index on a sibling. +fn resolve_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String { + let path = locate_vif_path(dir, dir_idx, collection, volume_id); + let data_vif = format!( + "{}.vif", + crate::storage::volume::volume_file_name(dir, collection, volume_id), + ); + if dir_idx == dir || path != data_vif { + return path; + } + let idx_vif = format!( + "{}.vif", + crate::storage::volume::volume_file_name(dir_idx, collection, volume_id), + ); + prefer_ec_config_vif(path, idx_vif) +} + +fn read_vif_file(path: &str) -> Result, ()> { + match std::fs::read_to_string(path) { + Ok(content) if content.trim().is_empty() => Ok(None), + Ok(content) => serde_json::from_str(&content).map(Some).map_err(|_| ()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(_) => Err(()), + } +} + +/// Returns `alt` when `local` records no EC config but `alt` does. +fn prefer_ec_config_vif(local: String, alt: String) -> String { + match read_vif_file(&local) { + Ok(Some(info)) if info.ec_shard_config.is_some() => return local, + Err(()) => return local, + _ => {} + } + match read_vif_file(&alt) { + Ok(Some(info)) if info.ec_shard_config.is_some() => alt, + _ => local, + } +} + +/// The `.vif` a disk mounting against a shared `-dir.idx` takes from a sibling +/// data dir in `vif_dirs`: its own records no EC config, the index is not its +/// own, and every sibling `.vif` recording an EC config names the same run. +/// `Err` when a sibling `.vif` cannot be read or the siblings name different runs. +pub(crate) fn sibling_ec_config_vif( + dir: &str, + dir_idx: &str, + vif_dirs: &[String], + collection: &str, + volume_id: VolumeId, +) -> Result, ()> { + let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id); + let data_vif = format!("{}.vif", data_base); + if dir_idx == dir + || vif_dirs.iter().any(|d| d == dir_idx) + || matches!(ecx_file_size(&format!("{}.ecx", data_base)), Some(n) if n > 0) + || resolve_vif_path(dir, dir_idx, collection, volume_id) != data_vif + { + return Ok(None); + } + match read_vif_file(&data_vif) { + Ok(Some(info)) if info.ec_shard_config.is_some() => return Ok(None), + Err(()) => return Ok(None), + _ => {} + } + let mut found: Option = None; + for other in vif_dirs.iter().filter(|d| d.as_str() != dir) { + let path = format!( + "{}.vif", + crate::storage::volume::volume_file_name(other, collection, volume_id) + ); + let info = match read_vif_file(&path) { + Ok(Some(info)) if info.ec_shard_config.is_some() => info, + Ok(_) => continue, + Err(()) => return Err(()), + }; + let run = |v: &crate::storage::volume::VifVolumeInfo| { + v.ec_shard_config.as_ref().map(|c| c.encode_ts_ns) + }; + match &found { + None => found = Some(info), + Some(f) if run(f) != run(&info) => return Err(()), + Some(_) => {} + } + } + Ok(found) +} + /// Load the volume's `.vif` (data dir first, then idx dir, then the active /// generation's versioned sidecar — see [`locate_vif_path`]). Absent /// everywhere is legal — legacy volumes predate the sidecar — and returns @@ -286,7 +376,7 @@ pub fn load_vif_info( collection: &str, volume_id: VolumeId, ) -> io::Result> { - let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id); + let vif_path = resolve_vif_path(dir, dir_idx, collection, volume_id); match std::fs::read_to_string(&vif_path) { Ok(content) if content.trim().is_empty() => Ok(None), Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| { @@ -336,15 +426,16 @@ pub fn load_vif_info_across_dirs( // pointed callers at the wrong disk whenever the vif lived with the // index — and a caller resolving the rest of the volume's metadata // against that answer would look in a directory holding none of it. - let data_vif = format!( + let idx_vif = format!( "{}.vif", - crate::storage::volume::volume_file_name(dir, collection, volume_id) + crate::storage::volume::volume_file_name(dir_idx, collection, volume_id) ); - let found_in = if std::path::Path::new(&data_vif).exists() { - dir.to_string() - } else { - dir_idx.to_string() - }; + let found_in = + if dir_idx != dir && resolve_vif_path(dir, dir_idx, collection, volume_id) == idx_vif { + dir_idx.to_string() + } else { + dir.to_string() + }; return Ok(Some((vif, found_in))); } for other in other_dirs { @@ -569,7 +660,25 @@ impl EcVolume { collection: &str, volume_id: VolumeId, ) -> io::Result { - Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL) + Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL, &[]) + } + + /// [`Self::new`] for a disk whose server's data directories are `vif_dirs`. + pub fn new_with_vif_dirs( + dir: &str, + dir_idx: &str, + vif_dirs: &[String], + collection: &str, + volume_id: VolumeId, + ) -> io::Result { + Self::open_with( + dir, + dir_idx, + collection, + volume_id, + EcjFsOps::REAL, + vif_dirs, + ) } /// [`Self::new`] with the journal-compaction filesystem steps supplied, so @@ -580,10 +689,15 @@ impl EcVolume { collection: &str, volume_id: VolumeId, ecj_ops: EcjFsOps, + vif_dirs: &[String], ) -> io::Result { // One load of the volume's `.vif`, used for both the shard config and // the version / dat-size fields below. let vif = load_vif_info(dir, dir_idx, collection, volume_id)?; + let vif = sibling_ec_config_vif(dir, dir_idx, vif_dirs, collection, volume_id) + .ok() + .flatten() + .or(vif); // Both directories: a split -dir/-dir.idx layout keeps the .ecsum with // the INDEX, and it is the only layout record a volume whose vif is // absent or config-free still has. @@ -3580,7 +3694,7 @@ mod tests { ); let before = std::fs::read(&ecj_path).unwrap(); ( - EcVolume::open_with(dir, dir, "", vid, ops), + EcVolume::open_with(dir, dir, "", vid, ops, &[]), ecj_path, before, ) @@ -3801,7 +3915,7 @@ mod tests { read_at: rewrite_after_last_read, ..EcjFsOps::REAL }; - let vol = EcVolume::open_with(dir, dir, "", vid, ops).unwrap(); + let vol = EcVolume::open_with(dir, dir, "", vid, ops, &[]).unwrap(); assert!( IN_PLACE_REWRITE.with(|r| r.borrow().is_none()), "the rewrite never ran" @@ -4108,6 +4222,143 @@ mod tests { assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume"); } + /// Shards placed on the disk that still holds the source volume's `.vif` + /// mount against an index on a sibling disk; the encode identity and + /// layout come from the sibling's `.vif`, not the source's config-free one. + #[test] + fn test_ec_volume_prefers_idx_vif_over_config_free_data_vif() { + let data_tmp = TempDir::new().unwrap(); + let idx_tmp = TempDir::new().unwrap(); + let dir = data_tmp.path().to_str().unwrap(); + let dir_idx = idx_tmp.path().to_str().unwrap(); + write_ecx_file(dir_idx, "pics", VolumeId(1), &[]); + let data_base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1)); + std::fs::write( + format!("{}.vif", data_base), + r#"{"version":3,"readOnly":true}"#, + ) + .unwrap(); + let vif = crate::storage::volume::VifVolumeInfo { + version: 3, + dat_file_size: 4096, + ec_shard_config: Some(crate::storage::volume::VifEcShardConfig { + data_shards: 12, + parity_shards: 4, + encode_ts_ns: 1_700_000_000_123_456_789, + block_size: 3 * 1024 * 1024, + }), + ..Default::default() + }; + let idx_base = crate::storage::volume::volume_file_name(dir_idx, "pics", VolumeId(1)); + std::fs::write( + format!("{}.vif", idx_base), + serde_json::to_string_pretty(&vif).unwrap(), + ) + .unwrap(); + + let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap(); + assert_eq!(vol.encode_ts_ns, 1_700_000_000_123_456_789); + assert_eq!((vol.data_shards, vol.parity_shards), (12, 4)); + assert_eq!(vol.block_size, 3 * 1024 * 1024); + assert_eq!(vol.dat_file_size, 4096); + let (_, found_in) = load_vif_info_across_dirs(dir, dir_idx, &[], "pics", VolumeId(1)) + .unwrap() + .unwrap(); + assert_eq!(found_in, dir_idx); + } + + /// A data-dir `.vif` that records an EC config stays authoritative. + #[test] + fn test_ec_volume_keeps_data_vif_with_ec_config() { + let data_tmp = TempDir::new().unwrap(); + let idx_tmp = TempDir::new().unwrap(); + let dir = data_tmp.path().to_str().unwrap(); + let dir_idx = idx_tmp.path().to_str().unwrap(); + write_ecx_file(dir_idx, "pics", VolumeId(1), &[]); + for (d, ts) in [(dir, 11), (dir_idx, 22)] { + let vif = crate::storage::volume::VifVolumeInfo { + version: 3, + ec_shard_config: Some(crate::storage::volume::VifEcShardConfig { + data_shards: 10, + parity_shards: 4, + encode_ts_ns: ts, + ..Default::default() + }), + ..Default::default() + }; + let base = crate::storage::volume::volume_file_name(d, "pics", VolumeId(1)); + std::fs::write( + format!("{}.vif", base), + serde_json::to_string_pretty(&vif).unwrap(), + ) + .unwrap(); + } + + let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap(); + assert_eq!(vol.encode_ts_ns, 11); + } + + /// With a shared `-dir.idx` the encode's `.vif` sits in the data dir of the + /// disk that received the index, so a disk holding a source volume's + /// config-free `.vif` takes it from that sibling, but only when the + /// siblings agree on one run. + #[test] + fn test_ec_volume_takes_sibling_vif_with_shared_index() { + const RUN_A: i64 = 1_700_000_000_000_000_011; + const RUN_B: i64 = 1_700_000_000_000_000_022; + let cases: [(&str, &[i64], bool, i64); 4] = [ + ("one sibling", &[RUN_A], false, RUN_A), + ("siblings agree", &[RUN_A, RUN_A], false, RUN_A), + ("siblings disagree", &[RUN_A, RUN_B], false, 0), + ("local index", &[RUN_A], true, 0), + ]; + for (name, sibling_ts, local_index, want) in cases { + let data_tmp = TempDir::new().unwrap(); + let idx_tmp = TempDir::new().unwrap(); + let dir = data_tmp.path().to_str().unwrap(); + let dir_idx = idx_tmp.path().to_str().unwrap(); + let entry = [(NeedleId(1), Offset::from_actual_offset(8), Size(10))]; + write_ecx_file(dir_idx, "c", VolumeId(1), &entry); + if local_index { + write_ecx_file(dir, "c", VolumeId(1), &entry); + } + let data_base = crate::storage::volume::volume_file_name(dir, "c", VolumeId(1)); + std::fs::write( + format!("{}.vif", data_base), + r#"{"version":3,"readOnly":true}"#, + ) + .unwrap(); + let mut dirs = vec![dir.to_string()]; + let mut siblings = Vec::new(); + for &ts in sibling_ts { + let sibling = TempDir::new().unwrap(); + let sibling_dir = sibling.path().to_str().unwrap().to_string(); + let vif = crate::storage::volume::VifVolumeInfo { + version: 3, + dat_file_size: 4096, + ec_shard_config: Some(crate::storage::volume::VifEcShardConfig { + data_shards: 10, + parity_shards: 4, + encode_ts_ns: ts, + ..Default::default() + }), + ..Default::default() + }; + let base = crate::storage::volume::volume_file_name(&sibling_dir, "c", VolumeId(1)); + std::fs::write( + format!("{}.vif", base), + serde_json::to_string(&vif).unwrap(), + ) + .unwrap(); + dirs.push(sibling_dir); + siblings.push(sibling); + } + + let vol = EcVolume::new_with_vif_dirs(dir, dir_idx, &dirs, "c", VolumeId(1)).unwrap(); + assert_eq!(vol.encode_ts_ns, want, "{}", name); + } + } + #[test] fn test_ec_volume_wide_ratio_vif_config() { let tmp = TempDir::new().unwrap(); diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 2ea07010d8..fde319dd1d 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -122,6 +122,9 @@ pub struct Store { /// The copy-claim set installed on every location by /// set_volume_load_guard, kept here so a later add_location gets it too. load_guard: Option>>>, + /// The data directory of every disk this server loads, set before the + /// first `add_location` so a disk's EC mounts can see its siblings. + pub store_dirs: Vec, } impl Store { @@ -148,6 +151,7 @@ impl Store { disk_stats_of: super::disk_location::get_disk_stats, load_guard: None, preallocate_supported: PREALLOCATE_SUPPORTED, + store_dirs: Vec::new(), } } @@ -170,6 +174,7 @@ impl Store { tags, )?; loc.load_guard = self.load_guard.clone(); + loc.store_dirs = self.store_dirs.clone(); loc.load_existing_volumes(self.needle_map_kind)?; // Check for duplicate volume IDs across existing locations diff --git a/seaweed-volume/src/storage/store_ec_mirror.rs b/seaweed-volume/src/storage/store_ec_mirror.rs index 95ae9df89f..58a1a7c2cc 100644 --- a/seaweed-volume/src/storage/store_ec_mirror.rs +++ b/seaweed-volume/src/storage/store_ec_mirror.rs @@ -8,7 +8,9 @@ use std::path::Path; use tracing::{info, warn}; -use crate::storage::disk_location::{DiskLocation, parse_collection_volume_id_pub}; +use crate::storage::disk_location::{ + DiskLocation, parse_collection_volume_id_pub, vif_is_ec_volume, +}; use crate::storage::store::Store; use crate::storage::types::VolumeId; @@ -207,6 +209,33 @@ fn mirror_sidecars_for_volume( collection: &str, vid: VolumeId, ) -> io::Result { + // A local .vif with no EC config (a source volume's own) would shadow the + // owner's once the index is copied next to it; keep mounting across disks. + let dst_vif = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ".vif"); + if path_is_regular_file(&dst_vif) + && !vif_is_ec_volume(&dst_vif) + && (vif_is_ec_volume(&sidecar_dest_path( + src_data_dir, + src_idx_dir, + collection, + vid, + ".vif", + )) || vif_is_ec_volume(&sidecar_dest_path( + src_idx_dir, + src_data_dir, + collection, + vid, + ".vif", + ))) + { + info!( + volume_id = vid.0, + collection = %collection, + vif = %dst_vif, + "local .vif records no EC config; not mirroring EC sidecars", + ); + return Ok(0); + } let mut copied = 0usize; for ext in EC_MIRRORED_SIDECARS { let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext); @@ -391,6 +420,222 @@ mod tests { assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source"); } + /// Shards on a disk that still holds the source volume's config-free + /// `.vif` must mount with the owner's encode identity: mirroring the index + /// next to that `.vif` would make the disk mount through it. + #[test] + fn mirror_skips_disk_with_config_free_vif() { + let tmp = TempDir::new().unwrap(); + let dir0 = tmp.path().join("data0"); + let dir1 = tmp.path().join("data1"); + fs::create_dir_all(&dir0).unwrap(); + fs::create_dir_all(&dir1).unwrap(); + + let collection = "c"; + let vid = 4122u32; + plant_shard(&dir0, collection, vid, 2); + plant_shard(&dir1, collection, vid, 0); + fs::write( + dir0.join(format!("{}_{}.vif", collection, vid)), + r#"{"version":3,"readOnly":true}"#, + ) + .unwrap(); + plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]); + let vif = VifVolumeInfo { + version: 3, + ec_shard_config: Some(VifEcShardConfig { + data_shards: 10, + parity_shards: 4, + encode_ts_ns: 1_700_000_000_000_000_042, + ..Default::default() + }), + ..Default::default() + }; + fs::write( + dir1.join(format!("{}_{}.vif", collection, vid)), + serde_json::to_string(&vif).unwrap(), + ) + .unwrap(); + + let mut store = Store::new(NeedleMapKind::InMemory); + add_loc(&mut store, &dir0); + add_loc(&mut store, &dir1); + + assert!( + !dir0.join(format!("{}_{}.ecx", collection, vid)).exists(), + ".ecx mirrored next to the config-free .vif" + ); + let ec_vol = store.locations[0] + .find_ec_volume(VolumeId(vid)) + .expect("dir0 ec volume mounted"); + assert_eq!(ec_vol.encode_ts_ns, 1_700_000_000_000_000_042); + } + + /// With a shared `-dir.idx` every disk sees the `.ecx`, and only the disk + /// that received it holds the encode's `.vif`. A disk loaded first that + /// still holds the source volume's config-free `.vif` must take that + /// identity from its sibling. + #[test] + fn shared_index_disk_takes_sibling_vif_on_startup() { + let tmp = TempDir::new().unwrap(); + let dir0 = tmp.path().join("data0"); + let dir1 = tmp.path().join("data1"); + let idx = tmp.path().join("idx"); + for d in [&dir0, &dir1, &idx] { + fs::create_dir_all(d).unwrap(); + } + + let collection = "c"; + let vid = 4123u32; + plant_shard(&dir0, collection, vid, 2); + plant_shard(&dir1, collection, vid, 0); + fs::write( + dir0.join(format!("{}_{}.vif", collection, vid)), + r#"{"version":3,"readOnly":true}"#, + ) + .unwrap(); + plant_ecx(&idx, collection, vid, &[0xA1u8; 20]); + let vif = VifVolumeInfo { + version: 3, + ec_shard_config: Some(VifEcShardConfig { + data_shards: 10, + parity_shards: 4, + encode_ts_ns: 1_700_000_000_000_000_043, + ..Default::default() + }), + ..Default::default() + }; + fs::write( + dir1.join(format!("{}_{}.vif", collection, vid)), + serde_json::to_string(&vif).unwrap(), + ) + .unwrap(); + + let mut store = Store::new(NeedleMapKind::InMemory); + store.store_dirs = vec![ + dir0.to_str().unwrap().to_string(), + dir1.to_str().unwrap().to_string(), + ]; + for dir in [&dir0, &dir1] { + store + .add_location( + dir.to_str().unwrap(), + idx.to_str().unwrap(), + 100, + DiskType::HardDrive, + MinFreeSpace::Percent(0.0), + Vec::new(), + ) + .unwrap(); + } + + for loc in &store.locations { + let ec_vol = loc + .find_ec_volume(VolumeId(vid)) + .unwrap_or_else(|| panic!("{} ec volume mounted", loc.directory)); + assert_eq!( + ec_vol.encode_ts_ns, 1_700_000_000_000_000_043, + "{}", + loc.directory + ); + } + } + + /// Starts a store whose first disk still holds the source `.dat`, its + /// config-free `.vif` and one 12+4 shard, with the `.ecx` in a shared + /// index dir and one sibling encode `.vif` per entry of `sibling_runs`. + /// Asserts the source `.dat`, the shard and the `.ecx` survive startup. + fn start_with_sibling_encode_vifs(sibling_runs: &[i64]) -> Store { + let tmp = TempDir::new().unwrap(); + let dirs: Vec<_> = (0..=sibling_runs.len()) + .map(|i| tmp.path().join(format!("data{}", i))) + .collect(); + let idx = tmp.path().join("idx"); + for d in dirs.iter().chain([&idx]) { + fs::create_dir_all(d).unwrap(); + } + + let collection = "c"; + let vid = 4124u32; + let base0 = dirs[0].join(format!("{}_{}", collection, vid)); + let dat = format!("{}.dat", base0.display()); + let shard = format!("{}.ec02", base0.display()); + fs::File::create(&dat) + .unwrap() + .set_len(11 * 1024 * 1024) + .unwrap(); + fs::File::create(&shard) + .unwrap() + .set_len(1024 * 1024) + .unwrap(); + fs::write( + format!("{}.vif", base0.display()), + r#"{"version":3,"readOnly":true}"#, + ) + .unwrap(); + plant_ecx(&idx, collection, vid, &[0xA1u8; 20]); + for (dir, run) in dirs[1..].iter().zip(sibling_runs) { + let vif = VifVolumeInfo { + version: 3, + ec_shard_config: Some(VifEcShardConfig { + data_shards: 12, + parity_shards: 4, + encode_ts_ns: *run, + ..Default::default() + }), + ..Default::default() + }; + fs::write( + dir.join(format!("{}_{}.vif", collection, vid)), + serde_json::to_string(&vif).unwrap(), + ) + .unwrap(); + } + + let mut store = Store::new(NeedleMapKind::InMemory); + store.store_dirs = dirs + .iter() + .map(|d| d.to_str().unwrap().to_string()) + .collect(); + for dir in &dirs { + store + .add_location( + dir.to_str().unwrap(), + idx.to_str().unwrap(), + 100, + DiskType::HardDrive, + MinFreeSpace::Percent(0.0), + Vec::new(), + ) + .unwrap(); + } + + let ecx = idx.join(format!("{}_{}.ecx", collection, vid)); + for path in [dat.as_str(), shard.as_str(), ecx.to_str().unwrap()] { + assert!(Path::new(path).exists(), "{} removed at startup", path); + } + store + } + + /// A disk still holding the source `.dat` sizes its shards by the + /// sibling's encode `.vif` before the startup check decides whether they + /// are leftovers of an interrupted encode. + #[test] + fn shared_index_startup_validation_uses_sibling_vif_ratio() { + let store = start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044]); + let ec_vol = store.locations[0] + .find_ec_volume(VolumeId(4124)) + .expect("dir0 ec volume mounted"); + assert!(ec_vol.has_shard(2), "shard 2 not mounted on dir0"); + } + + /// Sibling `.vif` files naming different encode runs leave the ratio + /// unknown, so the startup check keeps the shards. + #[test] + fn shared_index_startup_validation_keeps_shards_on_disagreeing_siblings() { + start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044, 1_700_000_000_000_000_045]); + } + #[test] fn mirror_preserves_existing_destination_sidecars() { let tmp = TempDir::new().unwrap(); diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 967632f51d..c80bb8ea2d 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -59,6 +59,8 @@ type DiskLocation struct { ecVolumesDestroying map[needle.VolumeId]chan struct{} ecShardNotifyHandler func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume) + // storeDirs is the data directory of every disk on this server. + storeDirs []string // loadGuard serializes a volume's load-and-register against an in-flight // VolumeCopy of the same id on the server. When set, the hook runs before diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go index 5cf6890a78..f08bf0cc5a 100644 --- a/weed/storage/disk_location_ec.go +++ b/weed/storage/disk_location_ec.go @@ -217,8 +217,10 @@ func (l *DiskLocation) loadEcShardWithIdxDir(collection string, vid needle.Volum ecVolume, found := l.ecVolumes[vid] if !found { var err error - ecVolume, err = erasure_coding.NewEcVolume(l.DiskType, l.Directory, idxDir, collection, vid) + ecVolume, err = erasure_coding.NewEcVolume(l.DiskType, l.Directory, idxDir, collection, vid, l.storeDirs...) if err != nil { + ecVolumeShard.Unmount() + ecVolumeShard.Close() // Wrap with %w so MountEcShards / startup reconcile can use // errors.Is(err, os.ErrNotExist) to decide whether to try the // next local disk vs. surface the failure. @@ -658,6 +660,10 @@ func (l *DiskLocation) validateEcVolume(collection string, vid needle.VolumeId) // Custom ratio comes from the volume's own .vif; the server holds no // cluster EC config in memory. dataShards := l.ecDataShardsFromVif(collection, vid) + if dataShards == 0 { + glog.Warningf("EC volume %d: sibling .vif files disagree on the encode; keeping EC shards", vid) + return true + } // On-disk .dat size, or -1 when absent (an empty <= superblock .dat is a // stub). A transient stat error keeps the shards rather than deleting. @@ -715,7 +721,8 @@ func (l *DiskLocation) validateEcVolume(collection string, vid needle.VolumeId) } // ecDataShardsFromVif resolves the data-shard count for an EC volume from -// its own .vif (EcShardConfig), checking the data dir then the idx dir. The +// its own .vif (EcShardConfig), checking the data dir, the idx dir, then the +// sibling data dir the mount takes the .vif from under a shared idx dir. The // .vif is the source of truth for custom ratios on the volume server, which // never holds the cluster EC config in memory. Falls back to the default // ratio when the .vif carries no EC shard config. @@ -731,6 +738,15 @@ func (l *DiskLocation) ecDataShardsFromVif(collection string, vid needle.VolumeI if ds := ecDataShardsFromVifDir(collection, l.IdxDirectory, vid); ds > 0 { return ds } + vif, ambiguous := erasure_coding.SiblingEcConfigVif(collection, vid, l.Directory, l.IdxDirectory, l.storeDirs) + if ambiguous { + return 0 + } + if vif != "" { + if ds := ecDataShardsFromVifPath(vif); ds > 0 { + return ds + } + } } return erasure_coding.DataShardsCount } @@ -738,7 +754,12 @@ func (l *DiskLocation) ecDataShardsFromVif(collection string, vid needle.VolumeI // ecDataShardsFromVifDir returns the .vif EcShardConfig data-shard count for // (collection, vid) under dir, or 0 when absent / not custom. func ecDataShardsFromVifDir(collection, dir string, vid needle.VolumeId) int { - vifName := erasure_coding.EcShardFileName(collection, dir, int(vid)) + ".vif" + return ecDataShardsFromVifPath(erasure_coding.EcShardFileName(collection, dir, int(vid)) + ".vif") +} + +// ecDataShardsFromVifPath returns the EcShardConfig data-shard count recorded +// in one .vif, or 0 when absent / not custom. +func ecDataShardsFromVifPath(vifName string) int { if vi, _, found, _ := volume_info.MaybeLoadVolumeInfo(vifName); found && vi.EcShardConfig != nil { if ds := int(vi.EcShardConfig.DataShards); ds > 0 { return ds diff --git a/weed/storage/disk_location_ec_test.go b/weed/storage/disk_location_ec_test.go index 335a09e190..a4004ff1c1 100644 --- a/weed/storage/disk_location_ec_test.go +++ b/weed/storage/disk_location_ec_test.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "os" "path/filepath" "strings" @@ -847,6 +848,33 @@ func TestLoadEcShardDuplicateReleasesTheNewShard(t *testing.T) { } } +// TestLoadEcShardReleasesShardWhenVolumeFails: the shard file opens, but the +// EcVolume cannot be built (no .ecx). The opened shard is not registered, so +// the loader must release its fd and mount gauge itself. +func TestLoadEcShardReleasesShardWhenVolumeFails(t *testing.T) { + dir := t.TempDir() + diskLocation := NewDiskLocation(dir, 10, util.MinFreeSpace{}, dir, types.HardDriveType, nil, stats.DefaultDiskIOProbeConfig()) + defer diskLocation.Close() + + if err := os.WriteFile(filepath.Join(dir, "125.ec00"), []byte("shard bytes"), 0o644); err != nil { + t.Fatalf("seed .ec00: %v", err) + } + + gauge := stats.VolumeServerVolumeGauge.WithLabelValues("", "ec_shards") + before := testutil.ToFloat64(gauge) + + _, err := diskLocation.LoadEcShard("", needle.VolumeId(125), erasure_coding.ShardId(0)) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("LoadEcShard without .ecx: got %v, want os.ErrNotExist", err) + } + if after := testutil.ToFloat64(gauge); after != before { + t.Errorf("mount gauge at %v after a failed load; want %v", after, before) + } + if _, found := diskLocation.FindEcVolume(needle.VolumeId(125)); found { + t.Errorf("a failed load must not register an EcVolume") + } +} + // TestLoadEcShardWaitsForPendingDestroy: DestroyEcVolume deletes the map entry // and destroys files outside ecVolumesLock, so a remount could otherwise open // shard files mid-unlink. The ecVolumesDestroying tombstone must hold the mount diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index a77ab21f85..5c780d886f 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -171,14 +171,63 @@ func statEcxSize(path string) (int64, error) { return info.Size(), nil } -func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) { - return newEcVolumeWith(diskType, dir, dirIdx, collection, vid, realEcjFsOps) +func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId, vifDirs ...string) (ev *EcVolume, err error) { + return newEcVolumeWith(diskType, dir, dirIdx, collection, vid, realEcjFsOps, vifDirs...) +} + +// preferEcConfigVif returns alt when local records no EC config but alt does. +// A source volume's own .vif stays on a disk that later receives shards of the +// same volume id, while the encode's .vif lands with the index on a sibling. +func preferEcConfigVif(local, alt string) string { + localInfo, _, found, err := volume_info.MaybeLoadVolumeInfo(local) + if err != nil || (found && localInfo.EcShardConfig != nil) { + return local + } + altInfo, _, altFound, altErr := volume_info.MaybeLoadVolumeInfo(alt) + if altErr != nil || !altFound || altInfo.EcShardConfig == nil { + return local + } + return alt +} + +// SiblingEcConfigVif returns the encode's .vif that a disk mounting against a +// shared -dir.idx takes from another data dir in dirs: dir holds no non-empty +// .ecx, dirIdx is not a data dir, and every sibling .vif recording an EC config +// names the same encode run. It returns "" otherwise, with ambiguous set when +// a sibling .vif cannot be read or the siblings name different encode runs. +func SiblingEcConfigVif(collection string, vid needle.VolumeId, dir, dirIdx string, dirs []string) (vif string, ambiguous bool) { + if dirIdx == dir || slices.Contains(dirs, dirIdx) { + return "", false + } + if size, err := statEcxSize(EcShardFileName(collection, dir, int(vid)) + ".ecx"); err == nil && size > 0 { + return "", false + } + found, foundTsNs := "", int64(0) + for _, d := range dirs { + if d == dir { + continue + } + vif := EcShardFileName(collection, d, int(vid)) + ".vif" + info, _, ok, err := volume_info.MaybeLoadVolumeInfo(vif) + if err != nil { + return "", true + } + if !ok || info.EcShardConfig == nil { + continue + } + if found == "" { + found, foundTsNs = vif, info.EcShardConfig.EncodeTsNs + } else if info.EcShardConfig.EncodeTsNs != foundTsNs { + return "", true + } + } + return found, false } // newEcVolumeWith is NewEcVolume with the journal-compaction filesystem steps // supplied, so tests can drive a failing load or publish through the real // mount. -func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId, ecjOps ecjFsOps) (ev *EcVolume, err error) { +func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId, ecjOps ecjFsOps, vifDirs ...string) (ev *EcVolume, err error) { ev = &EcVolume{dir: dir, dirIdx: dirIdx, Collection: collection, VolumeId: vid, diskType: diskType} dataBaseFileName := EcShardFileName(collection, dir, int(vid)) @@ -312,6 +361,15 @@ func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collect } } } + if dirIdx != dir && vifFileName == dataBaseFileName+".vif" { + vifFileName = preferEcConfigVif(vifFileName, EcShardFileName(collection, dirIdx, int(vid))+".vif") + // A shared -dir.idx holds no .vif: the encode's sits in a sibling data dir. + if vifFileName == dataBaseFileName+".vif" { + if alt, _ := SiblingEcConfigVif(collection, vid, dir, dirIdx, vifDirs); alt != "" { + vifFileName = preferEcConfigVif(vifFileName, alt) + } + } + } ev.Version = needle.Version3 // A present-but-unreadable or malformed .vif FAILS the mount: every new // encode records a positive uniform block size there, and defaulting to diff --git a/weed/storage/erasure_coding/ec_volume_vif_test.go b/weed/storage/erasure_coding/ec_volume_vif_test.go index e054d7612a..6dfef04c22 100644 --- a/weed/storage/erasure_coding/ec_volume_vif_test.go +++ b/weed/storage/erasure_coding/ec_volume_vif_test.go @@ -185,3 +185,131 @@ func TestNewEcVolumeRejectsUnusableSoleSidecar(t *testing.T) { } } } + +// Shards placed on the disk that still holds the source volume's .vif mount +// against an index on a sibling disk; the encode identity and layout come from +// the sibling's .vif, not from the source's config-free one. +func TestNewEcVolumePrefersIndexVifOverConfigFreeLocalVif(t *testing.T) { + dataDir, idxDir := t.TempDir(), t.TempDir() + const encodeTsNs = int64(1_700_000_000_123_456_789) + if err := volume_info.SaveVolumeInfo(filepath.Join(dataDir, "c_1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + ReadOnly: true, + }); err != nil { + t.Fatalf("seed source .vif: %v", err) + } + if err := os.WriteFile(filepath.Join(idxDir, "c_1.ecx"), []byte{}, 0644); err != nil { + t.Fatalf("seed .ecx: %v", err) + } + if err := volume_info.SaveVolumeInfo(filepath.Join(idxDir, "c_1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: 4096, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: 12, ParityShards: 4, BlockSize: 3 * 1024 * 1024, EncodeTsNs: encodeTsNs, + }, + }); err != nil { + t.Fatalf("seed index .vif: %v", err) + } + + ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "c", needle.VolumeId(1)) + if err != nil { + t.Fatalf("mount: %v", err) + } + defer ev.Close() + if ev.EncodeTsNs != encodeTsNs { + t.Errorf("EncodeTsNs = %d, want %d", ev.EncodeTsNs, encodeTsNs) + } + if got := ev.ECContext; got.DataShards != 12 || got.ParityShards != 4 || got.BlockSize != 3*1024*1024 { + t.Errorf("layout = %d+%d block %d, want 12+4 block 3MiB", got.DataShards, got.ParityShards, got.BlockSize) + } + if ev.datFileSize != 4096 { + t.Errorf("datFileSize = %d, want 4096", ev.datFileSize) + } +} + +// A local .vif that records an EC config stays authoritative for its disk. +func TestNewEcVolumeKeepsLocalVifWithEcConfig(t *testing.T) { + dataDir, idxDir := t.TempDir(), t.TempDir() + if err := volume_info.SaveVolumeInfo(filepath.Join(dataDir, "1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 11}, + }); err != nil { + t.Fatalf("seed local .vif: %v", err) + } + if err := os.WriteFile(filepath.Join(idxDir, "1.ecx"), []byte{}, 0644); err != nil { + t.Fatalf("seed .ecx: %v", err) + } + if err := volume_info.SaveVolumeInfo(filepath.Join(idxDir, "1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 22}, + }); err != nil { + t.Fatalf("seed index .vif: %v", err) + } + + ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", needle.VolumeId(1)) + if err != nil { + t.Fatalf("mount: %v", err) + } + defer ev.Close() + if ev.EncodeTsNs != 11 { + t.Errorf("EncodeTsNs = %d, want the local 11", ev.EncodeTsNs) + } +} + +// With a shared -dir.idx the encode's .vif sits in the data dir of the disk +// that received the index, so a disk holding a source volume's config-free +// .vif takes it from that sibling, but only when the siblings agree on one run. +func TestNewEcVolumeTakesSiblingVifWithSharedIndex(t *testing.T) { + const runA, runB = int64(1_700_000_000_000_000_011), int64(1_700_000_000_000_000_022) + cases := []struct { + name string + siblingTs []int64 + localIndex bool + want int64 + }{ + {name: "one sibling", siblingTs: []int64{runA}, want: runA}, + {name: "siblings agree", siblingTs: []int64{runA, runA}, want: runA}, + {name: "siblings disagree", siblingTs: []int64{runA, runB}, want: 0}, + {name: "local index", siblingTs: []int64{runA}, localIndex: true, want: 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dataDir, idxDir := t.TempDir(), t.TempDir() + dirs := []string{dataDir} + if err := volume_info.SaveVolumeInfo(filepath.Join(dataDir, "c_1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + ReadOnly: true, + }); err != nil { + t.Fatalf("seed source .vif: %v", err) + } + if err := os.WriteFile(filepath.Join(idxDir, "c_1.ecx"), make([]byte, 20), 0644); err != nil { + t.Fatalf("seed .ecx: %v", err) + } + if tc.localIndex { + if err := os.WriteFile(filepath.Join(dataDir, "c_1.ecx"), make([]byte, 20), 0644); err != nil { + t.Fatalf("seed local .ecx: %v", err) + } + } + for _, ts := range tc.siblingTs { + sibling := t.TempDir() + dirs = append(dirs, sibling) + if err := volume_info.SaveVolumeInfo(filepath.Join(sibling, "c_1.vif"), &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: 4096, + EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: ts}, + }); err != nil { + t.Fatalf("seed sibling .vif: %v", err) + } + } + + ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "c", needle.VolumeId(1), dirs...) + if err != nil { + t.Fatalf("mount: %v", err) + } + defer ev.Close() + if ev.EncodeTsNs != tc.want { + t.Errorf("EncodeTsNs = %d, want %d", ev.EncodeTsNs, tc.want) + } + }) + } +} diff --git a/weed/storage/store.go b/weed/storage/store.go index 05781ef1e4..407b6d009d 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -156,6 +156,10 @@ func NewStore( DeletedEcShardsChan: make(chan *master_pb.VolumeEcShardInformationMessage, HEARTBEAT_CHAN_SIZE), } + storeDirs := make([]string, len(dirnames)) + for i, dir := range dirnames { + storeDirs[i] = util.ResolvePath(dir) + } var wg sync.WaitGroup for i := 0; i < len(dirnames); i++ { var tags []string @@ -163,6 +167,7 @@ func NewStore( tags = diskTags[i] } location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i], tags, diskProbeConfig) + location.storeDirs = storeDirs s.Locations = append(s.Locations, location) stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i])) diff --git a/weed/storage/store_ec_mirror.go b/weed/storage/store_ec_mirror.go index 865e4fa7c3..01b8cc6cbb 100644 --- a/weed/storage/store_ec_mirror.go +++ b/weed/storage/store_ec_mirror.go @@ -101,6 +101,15 @@ func (l *DiskLocation) mirrorEcSidecarsFrom(owner ecxOwnerInfo, collection strin srcIdxBase := erasure_coding.EcShardFileName(collection, owner.idxDir, int(vid)) srcDataBase := erasure_coding.EcShardFileName(collection, owner.location.Directory, int(vid)) + // A local .vif with no EC config (a source volume's own) would shadow the + // owner's once the index is copied next to it; keep mounting across disks. + if dstVif := l.ecSidecarDestPath(collection, vid, ".vif"); statRegular(dstVif) && !vifIsEcVolume(dstVif) && + (vifIsEcVolume(srcIdxBase+".vif") || vifIsEcVolume(srcDataBase+".vif")) { + glog.V(0).Infof("ec volume %d (collection=%q): %s records no EC config; not mirroring sidecars from %s", + vid, collection, dstVif, owner.location.Directory) + return 0, nil + } + copied := 0 for _, ext := range ecMirroredSidecars { dst := l.ecSidecarDestPath(collection, vid, ext) diff --git a/weed/storage/store_ec_mirror_test.go b/weed/storage/store_ec_mirror_test.go index 96ccbdac71..7dd4a899b6 100644 --- a/weed/storage/store_ec_mirror_test.go +++ b/weed/storage/store_ec_mirror_test.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "fmt" "os" "path/filepath" "testing" @@ -283,3 +284,284 @@ func TestMirrorEcMetadataOnStartup_NoOpWhenAlreadyMirrored(t *testing.T) { t.Errorf("dir0 .ecx resolved at %q, want %q", got, want) } } + +// Shards on a disk that still holds the source volume's config-free .vif must +// mount with the owner's encode identity after a restart: mirroring the index +// next to that .vif would make the disk mount through it. +func TestMirrorEcMetadataOnStartup_KeepsConfigFreeVifDiskCrossDisk(t *testing.T) { + tempDir := t.TempDir() + dir0 := filepath.Join(tempDir, "data1") + dir1 := filepath.Join(tempDir, "data2") + for _, d := range []string{dir0, dir1} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + + collection := "c" + vid := needle.VolumeId(4122) + const dataShards, parityShards = 10, 4 + const datSize int64 = 10 * 1024 * 1024 + const encodeTsNs = int64(1_700_000_000_000_000_042) + expectedShardSize := calculateExpectedShardSize(datSize, dataShards) + base0 := erasure_coding.EcShardFileName(collection, dir0, int(vid)) + base1 := erasure_coding.EcShardFileName(collection, dir1, int(vid)) + for base, shardId := range map[string]int{base0: 2, base1: 0} { + if err := os.WriteFile(base+erasure_coding.ToExt(shardId), nil, 0o644); err != nil { + t.Fatalf("create shard: %v", err) + } + if err := os.Truncate(base+erasure_coding.ToExt(shardId), expectedShardSize); err != nil { + t.Fatalf("size shard: %v", err) + } + } + if err := volume_info.SaveVolumeInfo(base0+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + ReadOnly: true, + }); err != nil { + t.Fatalf("save source .vif: %v", err) + } + if err := os.WriteFile(base1+".ecx", bytes.Repeat([]byte{0xA1}, 20), 0o644); err != nil { + t.Fatalf("write .ecx: %v", err) + } + if err := volume_info.SaveVolumeInfo(base1+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: datSize, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: dataShards, + ParityShards: parityShards, + EncodeTsNs: encodeTsNs, + }, + }); err != nil { + t.Fatalf("save .vif: %v", err) + } + + store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{dir0, dir1}, + []int32{100, 100}, + []util.MinFreeSpace{{}, {}}, + "", + NeedleMapInMemory, + []types.DiskType{types.HardDriveType, types.HardDriveType}, + nil, + 3, + stats.DefaultDiskIOProbeConfig(), + ) + done := make(chan struct{}) + go func() { + for { + select { + case <-store.NewEcShardsChan: + case <-store.NewVolumesChan: + case <-store.DeletedVolumesChan: + case <-store.DeletedEcShardsChan: + case <-store.StateUpdateChan: + case <-done: + return + } + } + }() + t.Cleanup(func() { + store.Close() + close(done) + }) + + if _, err := os.Stat(base0 + ".ecx"); !os.IsNotExist(err) { + t.Errorf(".ecx mirrored next to the config-free .vif: %v", err) + } + ev0, found := store.Locations[0].FindEcVolume(vid) + if !found { + t.Fatalf("dir0 EcVolume %d not loaded", vid) + } + if _, ok := ev0.FindEcVolumeShard(2); !ok { + t.Errorf("shard 2 not registered on dir0") + } + if ev0.EncodeTsNs != encodeTsNs { + t.Errorf("dir0 EncodeTsNs = %d, want %d", ev0.EncodeTsNs, encodeTsNs) + } +} + +// With a shared -dir.idx every disk sees the .ecx, and only the disk that +// received it holds the encode's .vif. A sibling disk that still holds the +// source volume's config-free .vif must load its shards with that identity. +func TestLoadEcShardsOnStartup_SharedIndexTakesSiblingVif(t *testing.T) { + tempDir := t.TempDir() + dir0 := filepath.Join(tempDir, "data1") + dir1 := filepath.Join(tempDir, "data2") + idxDir := filepath.Join(tempDir, "idx") + for _, d := range []string{dir0, dir1, idxDir} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + + collection := "c" + vid := needle.VolumeId(4123) + const dataShards, parityShards = 10, 4 + const datSize int64 = 10 * 1024 * 1024 + const encodeTsNs = int64(1_700_000_000_000_000_043) + expectedShardSize := calculateExpectedShardSize(datSize, dataShards) + base0 := erasure_coding.EcShardFileName(collection, dir0, int(vid)) + base1 := erasure_coding.EcShardFileName(collection, dir1, int(vid)) + for base, shardId := range map[string]int{base0: 2, base1: 0} { + if err := os.WriteFile(base+erasure_coding.ToExt(shardId), nil, 0o644); err != nil { + t.Fatalf("create shard: %v", err) + } + if err := os.Truncate(base+erasure_coding.ToExt(shardId), expectedShardSize); err != nil { + t.Fatalf("size shard: %v", err) + } + } + if err := volume_info.SaveVolumeInfo(base0+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + ReadOnly: true, + }); err != nil { + t.Fatalf("save source .vif: %v", err) + } + if err := os.WriteFile(erasure_coding.EcShardFileName(collection, idxDir, int(vid))+".ecx", bytes.Repeat([]byte{0xA1}, 20), 0o644); err != nil { + t.Fatalf("write .ecx: %v", err) + } + if err := volume_info.SaveVolumeInfo(base1+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: datSize, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: dataShards, + ParityShards: parityShards, + EncodeTsNs: encodeTsNs, + }, + }); err != nil { + t.Fatalf("save .vif: %v", err) + } + + store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{dir0, dir1}, + []int32{100, 100}, + []util.MinFreeSpace{{}, {}}, + idxDir, + NeedleMapInMemory, + []types.DiskType{types.HardDriveType, types.HardDriveType}, + nil, + 3, + stats.DefaultDiskIOProbeConfig(), + ) + done := make(chan struct{}) + go func() { + for { + select { + case <-store.NewEcShardsChan: + case <-store.NewVolumesChan: + case <-store.DeletedVolumesChan: + case <-store.DeletedEcShardsChan: + case <-store.StateUpdateChan: + case <-done: + return + } + } + }() + t.Cleanup(func() { + store.Close() + close(done) + }) + + for i, shardId := range []erasure_coding.ShardId{2, 0} { + ev, found := store.Locations[i].FindEcVolume(vid) + if !found { + t.Fatalf("disk %d EcVolume %d not loaded", i, vid) + } + if _, ok := ev.FindEcVolumeShard(shardId); !ok { + t.Errorf("shard %d not registered on disk %d", shardId, i) + } + if ev.EncodeTsNs != encodeTsNs { + t.Errorf("disk %d EncodeTsNs = %d, want %d", i, ev.EncodeTsNs, encodeTsNs) + } + } +} + +func TestLoadEcShardsOnStartup_SharedIndexSiblingVifSizesValidation(t *testing.T) { + for _, tc := range []struct { + name string + siblingRuns []int64 + }{ + {"one sibling", []int64{1_700_000_000_000_000_044}}, + {"siblings name different runs", []int64{1_700_000_000_000_000_044, 1_700_000_000_000_000_045}}, + } { + t.Run(tc.name, func(t *testing.T) { + tempDir := t.TempDir() + dirs := []string{filepath.Join(tempDir, "data0")} + for i := range tc.siblingRuns { + dirs = append(dirs, filepath.Join(tempDir, fmt.Sprintf("data%d", i+1))) + } + idxDir := filepath.Join(tempDir, "idx") + for _, d := range append([]string{idxDir}, dirs...) { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + + collection := "c" + vid := needle.VolumeId(4124) + const dataShards, parityShards = 12, 4 + const datSize int64 = 11 * 1024 * 1024 + shardSize := calculateExpectedShardSize(datSize, dataShards) + if shardSize >= calculateExpectedShardSize(datSize, erasure_coding.DataShardsCount) { + t.Fatalf("test premise broken: default-ratio shard size must exceed %d", shardSize) + } + base0 := erasure_coding.EcShardFileName(collection, dirs[0], int(vid)) + if err := os.WriteFile(base0+".dat", make([]byte, datSize), 0o644); err != nil { + t.Fatalf("write .dat: %v", err) + } + if err := os.WriteFile(base0+erasure_coding.ToExt(2), nil, 0o644); err != nil { + t.Fatalf("create shard: %v", err) + } + if err := os.Truncate(base0+erasure_coding.ToExt(2), shardSize); err != nil { + t.Fatalf("size shard: %v", err) + } + if err := volume_info.SaveVolumeInfo(base0+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + ReadOnly: true, + }); err != nil { + t.Fatalf("save source .vif: %v", err) + } + ecx := erasure_coding.EcShardFileName(collection, idxDir, int(vid)) + ".ecx" + if err := os.WriteFile(ecx, bytes.Repeat([]byte{0xA1}, 20), 0o644); err != nil { + t.Fatalf("write .ecx: %v", err) + } + for i, run := range tc.siblingRuns { + if err := volume_info.SaveVolumeInfo(erasure_coding.EcShardFileName(collection, dirs[i+1], int(vid))+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: datSize, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: dataShards, + ParityShards: parityShards, + EncodeTsNs: run, + }, + }); err != nil { + t.Fatalf("save .vif: %v", err) + } + } + + maxCounts := make([]int32, len(dirs)) + minFree := make([]util.MinFreeSpace, len(dirs)) + diskTypes := make([]types.DiskType, len(dirs)) + for i := range dirs { + maxCounts[i], diskTypes[i] = 100, types.HardDriveType + } + store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + dirs, maxCounts, minFree, idxDir, NeedleMapInMemory, diskTypes, nil, 3, stats.DefaultDiskIOProbeConfig()) + drainStoreChans(t, store) + + for _, path := range []string{base0 + ".dat", base0 + erasure_coding.ToExt(2), ecx} { + if _, err := os.Stat(path); err != nil { + t.Errorf("%s removed at startup: %v", path, err) + } + } + if len(tc.siblingRuns) == 1 { + ev, found := store.Locations[0].FindEcVolume(vid) + if !found { + t.Fatalf("disk 0 EcVolume %d not loaded", vid) + } + if _, ok := ev.FindEcVolumeShard(2); !ok { + t.Errorf("shard 2 not registered on disk 0") + } + } + }) + } +}