diff --git a/seaweed-volume/src/main.rs b/seaweed-volume/src/main.rs index 33a800a18f..b5387701d1 100644 --- a/seaweed-volume/src/main.rs +++ b/seaweed-volume/src/main.rs @@ -416,6 +416,7 @@ async fn run( .to_string_lossy() .into_owned() }, + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight, ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index e655b43a57..a716115b41 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -3420,6 +3420,12 @@ impl VolumeServer for VolumeGrpcService { let vid = VolumeId(req.volume_id); let collection = &req.collection; + // Lifecycle lock OUTERMOST: rebuild writes canonical .ecNN/.ecx for this vid; + // serialize it against a concurrent copy/generate/teardown for the same vid so + // a teardown wipe cannot clobber rebuilt files (and vice versa). Held across the + // rebuild IO. + let _lifecycle = self.state.ec_lifecycle_lock(vid).lock_owned().await; + // Search ALL locations for shards, pick the best rebuild location // (most shards + has .ecx), collect additional dirs. // Matches Go's multi-location search in VolumeEcShardsRebuild. @@ -3888,290 +3894,32 @@ impl VolumeServer for VolumeGrpcService { // and 270 (would alias 14). let mut shard_ids: Vec = Vec::with_capacity(req.shard_ids.len()); for &sid in &req.shard_ids { - shard_ids.push(shard_id_try_from(sid).map_err(Status::invalid_argument)?); + let sid = shard_id_try_from(sid).map_err(Status::invalid_argument)?; + if !shard_ids.contains(&sid) { + shard_ids.push(sid); + } } - // Select target location: - // When disk_id > 0: use that specific location. - // When disk_id == 0 (unset): auto-select via - // find_ec_shard_target_location, which prefers a disk that - // already owns one of the shards being copied (a retried move - // must overwrite in place, not leave two disks of this server - // claiming the same shard), then a disk that already has the EC - // volume mounted, then a disk that owns the .ecx on disk (volume - // not yet mounted — relevant for ec.rebuild, where only the - // first shard carries .ecx and subsequent shards must land on - // the same disk; see #9212), then any HDD, then any disk. The - // free-slot maths uses the volume's own data-shard count from its - // .vif so a custom-ratio volume is not scored as 10+4. - let (dest_dir, dest_idx_dir) = { - let store = self.state.store.read().unwrap(); - let count = store.locations.len(); - - if req.disk_id > 0 { - // Explicit disk selection - if (req.disk_id as usize) >= count { - return Err(Status::invalid_argument(format!( - "invalid disk_id {}: only have {} disks", - req.disk_id, count - ))); - } - let loc = &store.locations[req.disk_id as usize]; - (loc.directory.clone(), loc.idx_directory.clone()) - } else { - // A batch whose requested shards are already owned by - // different local disks has no single correct destination: - // writing them all to one disk would duplicate the other - // disks' claims. Refuse so the caller splits the batch per - // shard (or chooses explicitly via disk_id). - let owners = store.ec_shard_owner_disks(vid, &shard_ids); - if owners.len() > 1 { - let dirs: Vec<&str> = owners - .iter() - .map(|&i| store.locations[i].directory.as_str()) - .collect(); - return Err(Status::failed_precondition(format!( - "volume {} shards {:?} are already owned by multiple local disks {:?}: no single destination; copy per shard or pass disk_id", - req.volume_id, shard_ids, dirs - ))); - } - let data_shards = store.lookup_ec_data_shards(&req.collection, vid); - match store.find_ec_shard_target_location( - &req.collection, - vid, - data_shards, - &shard_ids, - ) { - Some(i) => { - let loc = &store.locations[i]; - (loc.directory.clone(), loc.idx_directory.clone()) - } - None => { - return Err(Status::internal("no space left".to_string())); - } - } - } + // Overlapping copies of one vid would share the staged index, and a + // failed one's cleanup guards would remove shards another committed. + let lifecycle = self.state.ec_lifecycle_lock(vid).lock_owned().await; + // The copy, and the cleanup of a failed one, run to completion like Go's + // even if the caller goes away: dropping them mid-await would release + // the lock while file operations they started are still landing. + let svc = VolumeGrpcService { + state: self.state.clone(), }; - - // Connect to source and copy shard files via CopyFile - let source = &req.source_data_node; - let grpc_addr = parse_grpc_address(source).map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} invalid source_data_node {}: {}", - vid, source, e - )) - })?; - - let channel = connect_channel_guarded( - &grpc_addr, - source, - self.state.outgoing_grpc_tls.as_ref(), - GrpcDialOptions::stream(), - self.state.allow_untrusted_remote_endpoints, - ) + let request_id = crate::server::request_id::current_request_id(); + tokio::spawn(async move { + let _lifecycle = lifecycle; + let copy = svc.volume_ec_shards_copy_locked(req, vid, shard_ids); + match request_id { + Some(id) => crate::server::request_id::scope_request_id(id, copy).await, + None => copy.await, + } + }) .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} connect to {}: {}", - vid, grpc_addr, e - )) - })?; - - let mut client = volume_server_client(channel); - - // Copy each shard - for &shard_id in &shard_ids { - let ext = format!(".ec{:02}", shard_id); - let copy_req = volume_server_pb::CopyFileRequest { - volume_id: req.volume_id, - collection: req.collection.clone(), - is_ec_volume: true, - ext: ext.clone(), - compaction_revision: u32::MAX, - stop_offset: i64::MAX as u64, - ..Default::default() - }; - let mut stream = client - .copy_file(copy_req) - .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} copy {}: {}", - vid, ext, e - )) - })? - .into_inner(); - - let file_path = { - let base = - crate::storage::volume::volume_file_name(&dest_dir, &req.collection, vid); - format!("{}{}", base, ext) - }; - let file = tokio::fs::File::create(&file_path) - .await - .map_err(|e| Status::internal(format!("create {}: {}", file_path, e)))?; - drain_copy_stream_to_file(&mut stream, file, &file_path, &ext).await?; - } - - // Copy .ecx file if requested - if req.copy_ecx_file { - let copy_req = volume_server_pb::CopyFileRequest { - volume_id: req.volume_id, - collection: req.collection.clone(), - is_ec_volume: true, - ext: ".ecx".to_string(), - compaction_revision: u32::MAX, - stop_offset: i64::MAX as u64, - ..Default::default() - }; - let mut stream = client - .copy_file(copy_req) - .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} copy .ecx: {}", - vid, e - )) - })? - .into_inner(); - - let file_path = { - let base = - crate::storage::volume::volume_file_name(&dest_idx_dir, &req.collection, vid); - format!("{}.ecx", base) - }; - let file = tokio::fs::File::create(&file_path) - .await - .map_err(|e| Status::internal(format!("create {}: {}", file_path, e)))?; - let written = drain_copy_stream_to_file(&mut stream, file, &file_path, ".ecx").await?; - // A source that genuinely holds a 0-byte .ecx would leave a stub - // here that no placement or mount decision accepts. Catch it at - // distribute time, as Go does, so the orchestrator can pick another - // source instead of learning about it at mount. - if written == 0 { - let _ = tokio::fs::remove_file(&file_path).await; - return Err(Status::internal(format!( - "VolumeEcShardsCopy volume {}: source .ecx is 0 bytes", - vid - ))); - } - } - - // Copy .ecj file if requested. The journal is a *set* of ids: merge - // the source's into the local one as a union, never append it whole, - // or every balance round trip doubles it. A source without one - // is not an error. - if req.copy_ecj_file { - let copy_req = volume_server_pb::CopyFileRequest { - volume_id: req.volume_id, - collection: req.collection.clone(), - is_ec_volume: true, - ext: ".ecj".to_string(), - compaction_revision: u32::MAX, - stop_offset: i64::MAX as u64, - ignore_source_file_not_found: true, - ..Default::default() - }; - let mut stream = client - .copy_file(copy_req) - .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} copy .ecj: {}", - vid, e - )) - })? - .into_inner(); - let (ids, found) = receive_ecj_ids(&mut stream).await.map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} copy .ecj: {}", - vid, e - )) - })?; - if found { - let ecj_path = format!( - "{}.ecj", - crate::storage::volume::volume_file_name(&dest_idx_dir, &req.collection, vid) - ); - merge_ecj_ids(&self.state, vid, dest_dir.clone(), ecj_path, ids) - .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} merge .ecj: {}", - vid, e - )) - })?; - } - } - - // Copy .vif file if requested - if req.copy_vif_file { - let copy_req = volume_server_pb::CopyFileRequest { - volume_id: req.volume_id, - collection: req.collection.clone(), - is_ec_volume: true, - ext: ".vif".to_string(), - compaction_revision: u32::MAX, - stop_offset: i64::MAX as u64, - ignore_source_file_not_found: true, - ..Default::default() - }; - let mut stream = client - .copy_file(copy_req) - .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} copy .vif: {}", - vid, e - )) - })? - .into_inner(); - - let file_path = { - let base = - crate::storage::volume::volume_file_name(&dest_dir, &req.collection, vid); - format!("{}.vif", base) - }; - drain_optional_copy_to_file(&mut stream, &file_path, ".vif").await?; - } - - // Copy the generation-0 bitrot checksum sidecar (.ecsum) when requested, so - // protection travels with the shards. Tolerant of a missing source (no-op): - // this non-2PC path has no Prepare backstop, and an unprotected source - // simply leaves the copy unprotected. - if req.copy_ecsum_file { - let copy_req = volume_server_pb::CopyFileRequest { - volume_id: req.volume_id, - collection: req.collection.clone(), - is_ec_volume: true, - ext: ".ecsum".to_string(), - compaction_revision: u32::MAX, - stop_offset: i64::MAX as u64, - ignore_source_file_not_found: true, - ..Default::default() - }; - let mut stream = client - .copy_file(copy_req) - .await - .map_err(|e| { - Status::internal(format!( - "VolumeEcShardsCopy volume {} copy .ecsum: {}", - vid, e - )) - })? - .into_inner(); - - let file_path = { - let base = - crate::storage::volume::volume_file_name(&dest_dir, &req.collection, vid); - format!("{}.ecsum", base) - }; - drain_optional_copy_to_file(&mut stream, &file_path, ".ecsum").await?; - } - - Ok(Response::new( - volume_server_pb::VolumeEcShardsCopyResponse {}, - )) + .map_err(|e| Status::internal(format!("VolumeEcShardsCopy volume {}: {}", vid, e)))? } /// Allowed in maintenance mode: like `volume_delete` it only removes data, and @@ -4194,6 +3942,7 @@ impl VolumeServer for VolumeGrpcService { // // Unload every disk holding the vid (Go's Store.UnloadEcVolume): // each registration returns its descriptors and ec_shards gauge. + let _lifecycle = self.state.ec_lifecycle_lock(vid).lock_owned().await; { let mut store = self.state.store.write().unwrap(); store.unload_ec_volume(vid); @@ -4293,6 +4042,9 @@ impl VolumeServer for VolumeGrpcService { )); } + // A copy's staged shards are invisible to the last-shard check, so wait + // for it rather than remove the index it is about to need. + let _lifecycle = self.state.ec_lifecycle_lock(vid).lock_owned().await; let mut store = self.state.store.write().unwrap(); let mut shard_ids: Vec = Vec::with_capacity(req.shard_ids.len()); for &sid in &req.shard_ids { @@ -4338,14 +4090,28 @@ impl VolumeServer for VolumeGrpcService { // a local index can be mounted (issue #10104). Driven on demand by // ec.rebuild. volume_id 0 recovers every orphan on this server, including // volumes the master never learned about. + // Like Go's, recovery runs to completion if the caller goes away, so + // the lifecycle lock it holds outlives the journal merge it started. if req.recover_missing_index { - crate::server::store_ec::recover_missing_ec_indexes(&self.state, req.volume_id).await; + let state = self.state.clone(); + let filter_vid = req.volume_id; + tokio::spawn(async move { + crate::server::store_ec::recover_missing_ec_indexes(&state, filter_vid).await; + state.volume_state_notify.notify_one(); + }) + .await + .map_err(|e| { + Status::internal(format!("recover missing EC index of {}: {}", filter_vid, e)) + })?; } // Mount one shard at a time, returning error on first failure. // Matches Go: for _, shardId := range req.ShardIds { err = vs.store.MountEcShards(...) } // A failed shard stops the batch, but the shards mounted before it still - // get the sidecar reload and the heartbeat notify below. + // get the sidecar reload and the heartbeat notify below. Like Go's + // MountEcShards, wait out an in-flight copy, whose failure cleanup would + // remove a shard mounted meanwhile. + let _lifecycle = self.state.ec_lifecycle_lock(vid).lock_owned().await; let mut store = self.state.store.write().unwrap(); let mut mount_err = None; for &shard_id in &validated { @@ -6445,6 +6211,359 @@ impl CopyDestination { } impl VolumeGrpcService { + /// The body of `volume_ec_shards_copy`, run under the volume's EC lifecycle lock. + async fn volume_ec_shards_copy_locked( + &self, + req: volume_server_pb::VolumeEcShardsCopyRequest, + vid: VolumeId, + shard_ids: Vec, + ) -> Result, Status> { + // Select target location: + // When disk_id > 0: use that specific location. + // When disk_id == 0 (unset): auto-select via + // find_ec_shard_target_location, which prefers a disk that + // already owns one of the shards being copied (a retried move + // must overwrite in place, not leave two disks of this server + // claiming the same shard), then a disk that already has the EC + // volume mounted, then a disk that owns the .ecx on disk (volume + // not yet mounted — relevant for ec.rebuild, where only the + // first shard carries .ecx and subsequent shards must land on + // the same disk; see #9212), then any HDD, then any disk. The + // free-slot maths uses the volume's own data-shard count from its + // .vif so a custom-ratio volume is not scored as 10+4. + let (dest_dir, dest_idx_dir) = { + let store = self.state.store.read().unwrap(); + let count = store.locations.len(); + + if req.disk_id > 0 { + // Explicit disk selection + if (req.disk_id as usize) >= count { + return Err(Status::invalid_argument(format!( + "invalid disk_id {}: only have {} disks", + req.disk_id, count + ))); + } + let loc = &store.locations[req.disk_id as usize]; + (loc.directory.clone(), loc.idx_directory.clone()) + } else { + // A batch whose requested shards are already owned by + // different local disks has no single correct destination: + // writing them all to one disk would duplicate the other + // disks' claims. Refuse so the caller splits the batch per + // shard (or chooses explicitly via disk_id). + let owners = store.ec_shard_owner_disks(vid, &shard_ids); + if owners.len() > 1 { + let dirs: Vec<&str> = owners + .iter() + .map(|&i| store.locations[i].directory.as_str()) + .collect(); + return Err(Status::failed_precondition(format!( + "volume {} shards {:?} are already owned by multiple local disks {:?}: no single destination; copy per shard or pass disk_id", + req.volume_id, shard_ids, dirs + ))); + } + let data_shards = store.lookup_ec_data_shards(&req.collection, vid); + match store.find_ec_shard_target_location( + &req.collection, + vid, + data_shards, + &shard_ids, + ) { + Some(i) => { + let loc = &store.locations[i]; + (loc.directory.clone(), loc.idx_directory.clone()) + } + None => { + return Err(Status::internal("no space left".to_string())); + } + } + } + }; + + // Connect to source and copy shard files via CopyFile + let source = &req.source_data_node; + let grpc_addr = parse_grpc_address(source).map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} invalid source_data_node {}: {}", + vid, source, e + )) + })?; + + let channel = connect_channel_guarded( + &grpc_addr, + source, + self.state.outgoing_grpc_tls.as_ref(), + GrpcDialOptions::stream(), + self.state.allow_untrusted_remote_endpoints, + ) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} connect to {}: {}", + vid, grpc_addr, e + )) + })?; + + let mut client = volume_server_client(channel); + let data_base = crate::storage::volume::volume_file_name(&dest_dir, &req.collection, vid); + let index_base = + crate::storage::volume::volume_file_name(&dest_idx_dir, &req.collection, vid); + + // A failed copy removes what it added, or every retry against the same + // unusable source strands another unmounted shard on this disk. Files + // already on the disk before the call are never tracked. + let mut created: Vec = Vec::new(); + + // The index is staged first, so a source whose .ecx is unusable fails + // the copy before any shard lands here. It is committed after every + // other file, so a failed copy leaves an existing holder's index as is. + let ecx_path = format!("{}.ecx", index_base); + let mut staged_ecx: Option = None; + if req.copy_ecx_file { + let copy_req = volume_server_pb::CopyFileRequest { + volume_id: req.volume_id, + collection: req.collection.clone(), + is_ec_volume: true, + ext: ".ecx".to_string(), + compaction_revision: u32::MAX, + stop_offset: i64::MAX as u64, + ..Default::default() + }; + let mut stream = client + .copy_file(copy_req) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} copy .ecx: {}", + vid, e + )) + })? + .into_inner(); + + let staging = format!("{}.copying", ecx_path); + let staged = crate::server::store_ec::RemoveIncomplete::new(&staging); + let file = tokio::fs::File::create(&staging) + .await + .map_err(|e| Status::internal(format!("create {}: {}", staging, e)))?; + let written = drain_copy_stream_to_file(&mut stream, file, &staging, ".ecx").await?; + if written == 0 { + return Err(Status::internal(format!( + "VolumeEcShardsCopy volume {}: source .ecx is 0 bytes", + vid + ))); + } + staged_ecx = Some(staged); + } + + // Copy each shard. Each is staged and moved into place only once the + // whole copy succeeds, so a failed transfer leaves a shard already here + // as is and never removes one another writer put there. + let mut staged_shards: Vec<(ShardId, crate::server::store_ec::RemoveIncomplete)> = + Vec::new(); + for &shard_id in &shard_ids { + let ext = format!(".ec{:02}", shard_id); + let file_path = format!("{}{}.copying", data_base, ext); + staged_shards.push(( + shard_id, + crate::server::store_ec::RemoveIncomplete::new(&file_path), + )); + + let copy_req = volume_server_pb::CopyFileRequest { + volume_id: req.volume_id, + collection: req.collection.clone(), + is_ec_volume: true, + ext: ext.clone(), + compaction_revision: u32::MAX, + stop_offset: i64::MAX as u64, + ..Default::default() + }; + let mut stream = client + .copy_file(copy_req) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} copy {}: {}", + vid, ext, e + )) + })? + .into_inner(); + + let file = tokio::fs::File::create(&file_path) + .await + .map_err(|e| Status::internal(format!("create {}: {}", file_path, e)))?; + drain_copy_stream_to_file(&mut stream, file, &file_path, &ext).await?; + } + + // Copy .ecj file if requested. The journal is a *set* of ids: merge + // the source's into the local one as a union, never append it whole, + // or every balance round trip doubles it. A source without one + // is not an error. + if req.copy_ecj_file { + let copy_req = volume_server_pb::CopyFileRequest { + volume_id: req.volume_id, + collection: req.collection.clone(), + is_ec_volume: true, + ext: ".ecj".to_string(), + compaction_revision: u32::MAX, + stop_offset: i64::MAX as u64, + ignore_source_file_not_found: true, + ..Default::default() + }; + let mut stream = client + .copy_file(copy_req) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} copy .ecj: {}", + vid, e + )) + })? + .into_inner(); + let (ids, found) = receive_ecj_ids(&mut stream).await.map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} copy .ecj: {}", + vid, e + )) + })?; + if found { + let ecj_path = format!("{}.ecj", index_base); + track_ec_copy_created(&mut created, &ecj_path); + merge_ecj_ids(&self.state, vid, dest_dir.clone(), ecj_path, ids) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} merge .ecj: {}", + vid, e + )) + })?; + } + } + + // The sidecars are staged too and committed with the index, so a failed + // copy cannot pair a new .vif with the index this disk keeps. + let mut staged_sidecars: Vec<(String, crate::server::store_ec::RemoveIncomplete)> = + Vec::new(); + if req.copy_vif_file { + let copy_req = volume_server_pb::CopyFileRequest { + volume_id: req.volume_id, + collection: req.collection.clone(), + is_ec_volume: true, + ext: ".vif".to_string(), + compaction_revision: u32::MAX, + stop_offset: i64::MAX as u64, + ignore_source_file_not_found: true, + ..Default::default() + }; + let mut stream = client + .copy_file(copy_req) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} copy .vif: {}", + vid, e + )) + })? + .into_inner(); + + let file_path = format!("{}.vif", data_base); + let staging = format!("{}.copying", file_path); + let staged = crate::server::store_ec::RemoveIncomplete::new(&staging); + if drain_optional_copy_to_file(&mut stream, &staging, ".vif").await? > 0 { + staged_sidecars.push((file_path, staged)); + } + } + + // Copy the generation-0 bitrot checksum sidecar (.ecsum) when requested, so + // protection travels with the shards. Tolerant of a missing source (no-op): + // this non-2PC path has no Prepare backstop, and an unprotected source + // simply leaves the copy unprotected. + if req.copy_ecsum_file { + let copy_req = volume_server_pb::CopyFileRequest { + volume_id: req.volume_id, + collection: req.collection.clone(), + is_ec_volume: true, + ext: ".ecsum".to_string(), + compaction_revision: u32::MAX, + stop_offset: i64::MAX as u64, + ignore_source_file_not_found: true, + ..Default::default() + }; + let mut stream = client + .copy_file(copy_req) + .await + .map_err(|e| { + Status::internal(format!( + "VolumeEcShardsCopy volume {} copy .ecsum: {}", + vid, e + )) + })? + .into_inner(); + + let file_path = format!("{}.ecsum", data_base); + let staging = format!("{}.copying", file_path); + let staged = crate::server::store_ec::RemoveIncomplete::new(&staging); + if drain_optional_copy_to_file(&mut stream, &staging, ".ecsum").await? > 0 { + staged_sidecars.push((file_path, staged)); + } + } + + for (file_path, mut staged) in staged_sidecars { + track_ec_copy_created(&mut created, &file_path); + tokio::fs::rename(format!("{}.copying", file_path), &file_path) + .await + .map_err(|e| Status::internal(format!("commit {}: {}", file_path, e)))?; + staged.committed(); + } + + for (shard_id, mut staged) in staged_shards { + let shard_path = format!("{}.ec{:02}", data_base, shard_id); + // A mounted shard reads the file once reopened, so it is not ours to remove. + let mounted = self + .state + .store + .read() + .unwrap() + .locations + .iter() + .find(|l| l.directory == dest_dir) + .and_then(|l| l.find_ec_volume(vid)) + .is_some_and(|v| v.has_shard(shard_id)); + if !mounted { + track_ec_copy_created(&mut created, &shard_path); + } + tokio::fs::rename(format!("{}.copying", shard_path), &shard_path) + .await + .map_err(|e| Status::internal(format!("commit {}: {}", shard_path, e)))?; + staged.committed(); + // A mounted shard keeps reading the replaced inode until reopened. + let mut store = self.state.store.write().unwrap(); + if let Some(loc) = store.locations.iter_mut().find(|l| l.directory == dest_dir) { + if let Err(e) = loc.reopen_ec_shard(vid, shard_id) { + // Only this disk's mount reads the replaced file. + loc.unmount_ec_shards(vid, &[shard_id]); + drop(store); + self.state.volume_state_notify.notify_one(); + return Err(Status::internal(format!( + "VolumeEcShardsCopy volume {}: reopen replaced shard {}: {}", + vid, shard_id, e + ))); + } + } + } + + if let Some(_staged) = staged_ecx { + track_ec_copy_created(&mut created, &ecx_path); + commit_staged_ecx(&format!("{}.copying", ecx_path), &ecx_path) + .await + .map_err(|e| Status::internal(format!("commit {}: {}", ecx_path, e)))?; + } + created.iter_mut().for_each(|file| file.committed()); + + Ok(Response::new( + volume_server_pb::VolumeEcShardsCopyResponse {}, + )) + } + async fn connect_to_copy_source( &self, vid: VolumeId, @@ -7034,6 +7153,37 @@ async fn ping_filer_target( // call site. use super::grpc_client::parse_grpc_address; +/// Records a file volume_ec_shards_copy is about to add, so a failed copy +/// removes it; a file already on the disk is left out. +fn track_ec_copy_created(created: &mut Vec, path: &str) { + if matches!(std::fs::metadata(path), Err(e) if e.kind() == std::io::ErrorKind::NotFound) { + created.push(crate::server::store_ec::RemoveIncomplete::new(path)); + } +} + +/// Installs a staged index at `ecx_path`. An index already there is +/// overwritten in place: mounted EC volumes read it through open handles, +/// which a rename would leave on the replaced inode. +async fn commit_staged_ecx(staged: &str, ecx_path: &str) -> std::io::Result<()> { + use tokio::io::AsyncWriteExt; + let mut dst = match tokio::fs::OpenOptions::new() + .write(true) + .open(ecx_path) + .await + { + Ok(dst) => dst, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return tokio::fs::rename(staged, ecx_path).await; + } + Err(e) => return Err(e), + }; + let mut src = tokio::fs::File::open(staged).await?; + let n = tokio::io::copy(&mut src, &mut dst).await?; + dst.flush().await?; + dst.set_len(n).await?; + dst.sync_all().await +} + /// Set the modification time of a file from nanoseconds since Unix epoch. fn set_file_mtime(path: &str, modified_ts_ns: i64) -> std::io::Result<()> { use std::time::{Duration, SystemTime}; @@ -8526,6 +8676,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), @@ -8705,6 +8856,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), @@ -10056,6 +10208,428 @@ mod tests { (port, shutdown_tx) } + fn sorted_dir_names(dir: &str) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + + async fn ec_copy_destination() -> (VolumeGrpcService, TempDir, String) { + let (service, tmp) = make_local_service_with_volume("", None); + for loc in &service.state.store.read().unwrap().locations { + loc.check_disk_space(); + } + let dir = tmp.path().to_str().unwrap().to_string(); + (service, tmp, dir) + } + + // A source whose .ecx is 0 bytes must fail the copy without stranding a + // shard, index or sidecar on the destination, and without touching what an + // existing holder of the same volume already has there. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_copy_empty_source_ecx_leaves_no_files() { + let (source_service, source_tmp) = make_local_service_with_volume("", None); + let src = source_tmp.path().to_str().unwrap().to_string(); + std::fs::write(format!("{}/57.ecx", src), b"").unwrap(); + std::fs::write(format!("{}/57.ec00", src), vec![1u8; 4096]).unwrap(); + std::fs::write(format!("{}/57.ecj", src), b"").unwrap(); + std::fs::write(format!("{}/57.vif", src), b"{}").unwrap(); + let (port, _shutdown) = serve_source(source_service).await; + let req = volume_server_pb::VolumeEcShardsCopyRequest { + volume_id: 57, + shard_ids: vec![0], + copy_ecx_file: true, + copy_ecj_file: true, + copy_vif_file: true, + copy_ecsum_file: true, + source_data_node: format!("127.0.0.1:1.{}", port), + ..Default::default() + }; + + // Fresh destination. + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + let before = sorted_dir_names(&dir); + let err = dest + .volume_ec_shards_copy(Request::new(req.clone())) + .await + .expect_err("a 0-byte source .ecx must fail the copy"); + assert!(err.message().contains("source .ecx is 0 bytes"), "{}", err); + assert_eq!(sorted_dir_names(&dir), before); + + // Existing holder of other shards and a valid index. + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + let ecx = vec![7u8; NEEDLE_MAP_ENTRY_SIZE as usize]; + let ec05 = vec![5u8; 4096]; + std::fs::write(format!("{}/57.ecx", dir), &ecx).unwrap(); + std::fs::write(format!("{}/57.ec05", dir), &ec05).unwrap(); + let before = sorted_dir_names(&dir); + dest.volume_ec_shards_copy(Request::new(req)) + .await + .expect_err("a 0-byte source .ecx must fail the copy"); + assert_eq!(sorted_dir_names(&dir), before); + assert_eq!(std::fs::read(format!("{}/57.ecx", dir)).unwrap(), ecx); + assert_eq!(std::fs::read(format!("{}/57.ec05", dir)).unwrap(), ec05); + } + + // A copy failing after some shards landed removes exactly those shards and + // keeps the ones the destination held before the call, its index included. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_copy_failed_shard_removes_copied_shards() { + let (source_service, source_tmp) = make_local_service_with_volume("", None); + let src = source_tmp.path().to_str().unwrap().to_string(); + std::fs::write( + format!("{}/58.ecx", src), + vec![9u8; NEEDLE_MAP_ENTRY_SIZE as usize], + ) + .unwrap(); + std::fs::write(format!("{}/58.ec00", src), vec![1u8; 4096]).unwrap(); + std::fs::write(format!("{}/58.ec05", src), vec![2u8; 4096]).unwrap(); + let (port, _shutdown) = serve_source(source_service).await; + + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + let ecx = vec![7u8; NEEDLE_MAP_ENTRY_SIZE as usize]; + std::fs::write(format!("{}/58.ecx", dir), &ecx).unwrap(); + let ec05 = vec![5u8; 4096]; + std::fs::write(format!("{}/58.ec05", dir), &ec05).unwrap(); + let ec01 = vec![6u8; 4096]; + std::fs::write(format!("{}/58.ec01", dir), &ec01).unwrap(); + let before = sorted_dir_names(&dir); + + // Shard 0 copies, shard 5 replaces the existing holder's copy, shard 1 is + // held here but missing on the source and fails the request. + let err = dest + .volume_ec_shards_copy(Request::new(volume_server_pb::VolumeEcShardsCopyRequest { + volume_id: 58, + shard_ids: vec![0, 5, 1], + copy_ecx_file: true, + source_data_node: format!("127.0.0.1:1.{}", port), + ..Default::default() + })) + .await + .expect_err("shard 1 is missing on the source"); + assert!(err.message().contains(".ec01"), "{}", err); + assert_eq!(sorted_dir_names(&dir), before); + assert_eq!(std::fs::read(format!("{}/58.ecx", dir)).unwrap(), ecx); + assert_eq!(std::fs::read(format!("{}/58.ec05", dir)).unwrap(), ec05); + assert_eq!(std::fs::read(format!("{}/58.ec01", dir)).unwrap(), ec01); + } + + // A copied index replaces an existing holder's in place, so a mounted EC + // volume's open handle reads the source's index, tombstones included. + #[cfg(unix)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_copy_existing_ecx_updated_in_place() { + let (source_service, source_tmp) = make_local_service_with_volume("", None); + let src = source_tmp.path().to_str().unwrap().to_string(); + let src_ecx = vec![9u8; 2 * NEEDLE_MAP_ENTRY_SIZE as usize]; + std::fs::write(format!("{}/59.ecx", src), &src_ecx).unwrap(); + let src_ec00 = vec![1u8; 4096]; + std::fs::write(format!("{}/59.ec00", src), &src_ec00).unwrap(); + let src_vif = br#"{"version":3}"#.to_vec(); + std::fs::write(format!("{}/59.vif", src), &src_vif).unwrap(); + let (port, _shutdown) = serve_source(source_service).await; + + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + std::fs::write( + format!("{}/59.ecx", dir), + vec![0u8; 2 * NEEDLE_MAP_ENTRY_SIZE as usize], + ) + .unwrap(); + std::fs::write(format!("{}/59.ec00", dir), vec![3u8; 1024]).unwrap(); + let held = std::fs::File::open(format!("{}/59.ecx", dir)).unwrap(); + + dest.volume_ec_shards_copy(Request::new(volume_server_pb::VolumeEcShardsCopyRequest { + volume_id: 59, + shard_ids: vec![0], + copy_ecx_file: true, + copy_vif_file: true, + copy_ecsum_file: true, + source_data_node: format!("127.0.0.1:1.{}", port), + ..Default::default() + })) + .await + .expect("copy onto an existing holder"); + let mut got = vec![0u8; src_ecx.len()]; + std::os::unix::fs::FileExt::read_exact_at(&held, &mut got, 0).unwrap(); + assert_eq!(got, src_ecx, "open .ecx handle must read the source index"); + assert_eq!(std::fs::read(format!("{}/59.ec00", dir)).unwrap(), src_ec00); + assert_eq!(std::fs::read(format!("{}/59.vif", dir)).unwrap(), src_vif); + assert!(!std::path::Path::new(&format!("{}/59.ecsum", dir)).exists()); + assert!( + sorted_dir_names(&dir) + .iter() + .all(|n| !n.ends_with(".copying")), + "{:?}", + sorted_dir_names(&dir) + ); + } + + // A shard listed twice is copied once, and replacing one already held here + // still succeeds. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_copy_duplicate_shard_id_copied_once() { + let (source_service, source_tmp) = make_local_service_with_volume("", None); + let src = source_tmp.path().to_str().unwrap().to_string(); + std::fs::write( + format!("{}/62.ecx", src), + vec![9u8; NEEDLE_MAP_ENTRY_SIZE as usize], + ) + .unwrap(); + let src_ec05 = vec![2u8; 4096]; + std::fs::write(format!("{}/62.ec05", src), &src_ec05).unwrap(); + let (port, _shutdown) = serve_source(source_service).await; + + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + std::fs::write( + format!("{}/62.ecx", dir), + vec![0u8; NEEDLE_MAP_ENTRY_SIZE as usize], + ) + .unwrap(); + std::fs::write(format!("{}/62.ec05", dir), vec![5u8; 4096]).unwrap(); + + dest.volume_ec_shards_copy(Request::new(volume_server_pb::VolumeEcShardsCopyRequest { + volume_id: 62, + shard_ids: vec![5, 5], + copy_ecx_file: true, + source_data_node: format!("127.0.0.1:1.{}", port), + ..Default::default() + })) + .await + .expect("a duplicated shard id copies once"); + assert_eq!(std::fs::read(format!("{}/62.ec05", dir)).unwrap(), src_ec05); + } + + // A copy failing after the .vif transferred keeps the existing holder's + // .vif, which must stay paired with the index the failed copy left in place. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_copy_failed_sidecar_keeps_existing_vif() { + let (source_service, source_tmp) = make_local_service_with_volume("", None); + let src = source_tmp.path().to_str().unwrap().to_string(); + std::fs::write( + format!("{}/60.ecx", src), + vec![9u8; NEEDLE_MAP_ENTRY_SIZE as usize], + ) + .unwrap(); + std::fs::write(format!("{}/60.ec00", src), vec![1u8; 4096]).unwrap(); + std::fs::write(format!("{}/60.vif", src), br#"{"version":3}"#).unwrap(); + // A directory exists but cannot be read, even by root. + std::fs::create_dir(format!("{}/60.ecsum", src)).unwrap(); + let (port, _shutdown) = serve_source(source_service).await; + + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + let ecx = vec![0u8; NEEDLE_MAP_ENTRY_SIZE as usize]; + std::fs::write(format!("{}/60.ecx", dir), &ecx).unwrap(); + let vif = br#"{"version":2}"#.to_vec(); + std::fs::write(format!("{}/60.vif", dir), &vif).unwrap(); + let before = sorted_dir_names(&dir); + + let err = dest + .volume_ec_shards_copy(Request::new(volume_server_pb::VolumeEcShardsCopyRequest { + volume_id: 60, + shard_ids: vec![0], + copy_ecx_file: true, + copy_vif_file: true, + copy_ecsum_file: true, + source_data_node: format!("127.0.0.1:1.{}", port), + ..Default::default() + })) + .await + .expect_err("the source .ecsum is unreadable"); + assert!(err.message().contains(".ecsum"), "{}", err); + assert_eq!(sorted_dir_names(&dir), before); + assert_eq!(std::fs::read(format!("{}/60.vif", dir)).unwrap(), vif); + assert_eq!(std::fs::read(format!("{}/60.ecx", dir)).unwrap(), ecx); + } + + // A caller that goes away mid-copy must not release the volume's lifecycle + // lock before the copy has finished with its files. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_copy_runs_to_completion_after_cancel() { + let (source_service, source_tmp) = make_local_service_with_volume("", None); + let src = source_tmp.path().to_str().unwrap().to_string(); + std::fs::write( + format!("{}/61.ecx", src), + vec![9u8; NEEDLE_MAP_ENTRY_SIZE as usize], + ) + .unwrap(); + let shard_len = 1u64 << 20; + std::fs::write(format!("{}/61.ec00", src), vec![1u8; shard_len as usize]).unwrap(); + let (port, _shutdown) = serve_source(source_service).await; + + let (dest, _dest_tmp, dir) = ec_copy_destination().await; + let state = dest.state.clone(); + // The copy's first step reads the store, so it stays in flight until + // the handler has been cancelled. + let store_guard = state.store.write().unwrap(); + let copy = tokio::spawn(async move { + dest.volume_ec_shards_copy(Request::new(volume_server_pb::VolumeEcShardsCopyRequest { + volume_id: 61, + shard_ids: vec![0], + copy_ecx_file: true, + source_data_node: format!("127.0.0.1:1.{}", port), + ..Default::default() + })) + .await + }); + let lock = state.ec_lifecycle_lock(VolumeId(61)); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while lock.try_lock().is_ok() { + assert!( + std::time::Instant::now() < deadline, + "copy never took the lifecycle lock" + ); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + copy.abort(); + let joined = tokio::time::timeout(std::time::Duration::from_secs(10), copy) + .await + .expect("the cancelled handler did not return"); + assert!( + joined.as_ref().is_err_and(|e| e.is_cancelled()), + "handler was not cancelled mid-copy: {:?}", + joined.map(|r| r.map(|_| ())) + ); + drop(store_guard); + + let _lifecycle = + tokio::time::timeout(std::time::Duration::from_secs(10), lock.lock_owned()) + .await + .expect("the detached copy did not finish"); + let ec00 = format!("{}/61.ec00", dir); + assert_eq!(std::fs::metadata(&ec00).unwrap().len(), shard_len); + assert!(std::path::Path::new(&format!("{}/61.ecx", dir)).exists()); + } + + // A mount waits out an in-flight copy of the same volume, whose failure + // cleanup would remove a shard mounted meanwhile. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_mount_waits_for_in_flight_copy() { + let (dest, _dest_tmp, _dir) = ec_copy_destination().await; + let lifecycle = dest + .state + .ec_lifecycle_lock(VolumeId(63)) + .lock_owned() + .await; + let handler = dest.volume_ec_shards_mount(Request::new( + volume_server_pb::VolumeEcShardsMountRequest { + volume_id: 63, + collection: "col".to_string(), + shard_ids: vec![0], + ..Default::default() + }, + )); + tokio::pin!(handler); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut handler) + .await + .is_err(), + "mount ran while a copy of the same volume held its lifecycle lock" + ); + drop(lifecycle); + tokio::time::timeout(std::time::Duration::from_secs(10), handler) + .await + .expect("mount did not finish after the copy released the lifecycle lock") + .ok(); + } + + // A rebuild waits out an in-flight copy of the same volume, which would + // otherwise overwrite the index the rebuild folded tombstones into. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_rebuild_waits_for_in_flight_copy() { + let (dest, _dest_tmp, _dir) = ec_copy_destination().await; + let lifecycle = dest + .state + .ec_lifecycle_lock(VolumeId(66)) + .lock_owned() + .await; + let handler = dest.volume_ec_shards_rebuild(Request::new( + volume_server_pb::VolumeEcShardsRebuildRequest { + volume_id: 66, + collection: "col".to_string(), + ..Default::default() + }, + )); + tokio::pin!(handler); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut handler) + .await + .is_err(), + "rebuild ran while a copy of the same volume held its lifecycle lock" + ); + drop(lifecycle); + tokio::time::timeout(std::time::Duration::from_secs(10), handler) + .await + .expect("rebuild did not finish after the copy released the lifecycle lock") + .ok(); + } + + // A full teardown waits out an in-flight copy of the same volume, which + // would otherwise recreate the files after the teardown is acknowledged. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_full_teardown_waits_for_in_flight_copy() { + let (dest, _dest_tmp, _dir) = ec_copy_destination().await; + let lifecycle = dest + .state + .ec_lifecycle_lock(VolumeId(64)) + .lock_owned() + .await; + let handler = dest.volume_ec_shards_delete(Request::new( + volume_server_pb::VolumeEcShardsDeleteRequest { + volume_id: 64, + collection: "col".to_string(), + full_teardown: true, + ..Default::default() + }, + )); + tokio::pin!(handler); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut handler) + .await + .is_err(), + "full teardown ran while a copy of the same volume held its lifecycle lock" + ); + drop(lifecycle); + tokio::time::timeout(std::time::Duration::from_secs(10), handler) + .await + .expect("full teardown did not finish after the copy released the lifecycle lock") + .ok(); + } + + // A shard delete waits out an in-flight copy of the same volume, whose + // staged shards the delete's last-shard check cannot see. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_ec_shards_delete_waits_for_in_flight_copy() { + let (dest, _dest_tmp, _dir) = ec_copy_destination().await; + let lifecycle = dest + .state + .ec_lifecycle_lock(VolumeId(67)) + .lock_owned() + .await; + let handler = dest.volume_ec_shards_delete(Request::new( + volume_server_pb::VolumeEcShardsDeleteRequest { + volume_id: 67, + collection: "col".to_string(), + shard_ids: vec![0], + ..Default::default() + }, + )); + tokio::pin!(handler); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut handler) + .await + .is_err(), + "shard delete ran while a copy of the same volume held its lifecycle lock" + ); + drop(lifecycle); + tokio::time::timeout(std::time::Duration::from_secs(10), handler) + .await + .expect("shard delete did not finish after the copy released the lifecycle lock") + .ok(); + } + // The transfer itself must notice a caller that has gone, with no progress // report to carry the news. The first report is 128MB in, so for anything // smaller -- the ordinary case for a balance move -- the send result says @@ -11927,6 +12501,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), @@ -14153,6 +14728,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 9b1d4b21ed..d63cce3ea7 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -6278,6 +6278,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), diff --git a/seaweed-volume/src/server/heartbeat.rs b/seaweed-volume/src/server/heartbeat.rs index 07c4209ae1..b85fe5a4aa 100644 --- a/seaweed-volume/src/server/heartbeat.rs +++ b/seaweed-volume/src/server/heartbeat.rs @@ -1698,6 +1698,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), diff --git a/seaweed-volume/src/server/store_ec.rs b/seaweed-volume/src/server/store_ec.rs index 8c2a84ca64..fa7dd15831 100644 --- a/seaweed-volume/src/server/store_ec.rs +++ b/seaweed-volume/src/server/store_ec.rs @@ -2384,23 +2384,45 @@ pub(crate) async fn recover_missing_ec_indexes( "cannot look up peers to recover missing .ecx: {}", e ); - continue; + Vec::new() } }; - if peers.is_empty() { - tracing::warn!( - volume_id = m.vid.0, - "shards present locally but .ecx missing and no peer holds it; leaving shards unloaded" - ); - continue; + // Under the volume's lifecycle lock, so an in-flight VolumeEcShardsCopy + // of it, which may unlink the shards it renamed in, never has them + // mounted from under it. + let _lifecycle = state.ec_lifecycle_lock(m.vid).lock_owned().await; + let current = state + .store + .read() + .unwrap() + .collect_ec_volumes_missing_index_only(Some(m.vid)) + .into_iter() + .find(|x| x.vid == m.vid && x.collection == m.collection); + // A copy holding the lock may have installed the index meanwhile: skip + // the fetch, but its shards still need the mount. + if let Some(current) = ¤t { + if peers.is_empty() { + tracing::warn!( + volume_id = m.vid.0, + "shards present locally but .ecx missing and no peer holds it; leaving shards unloaded" + ); + continue; + } + if !fetch_ec_index_from_peers(state, current, &peers).await { + continue; + } } - if fetch_ec_index_from_peers(state, m, &peers).await { + state + .store + .write() + .unwrap() + .mount_recovered_ec_shards_only(Some(m.vid)); + if current.is_some() { recovered += 1; } } if recovered > 0 { - state.store.write().unwrap().mount_recovered_ec_shards(); tracing::info!( "recovered missing EC index for {} volume(s) from peers and mounted their shards", recovered diff --git a/seaweed-volume/src/server/volume_server.rs b/seaweed-volume/src/server/volume_server.rs index b677220c3e..b5b3801263 100644 --- a/seaweed-volume/src/server/volume_server.rs +++ b/seaweed-volume/src/server/volume_server.rs @@ -125,6 +125,11 @@ pub struct VolumeServerState { pub cli_white_list: Vec, /// Path to state.pb file for persisting VolumeServerState across restarts. pub state_file_path: String, + /// Per-vid EC lifecycle lock, serializing EC ops that install or remove a + /// vid's files. A tokio Mutex so the guard survives the copy's `.await` IO. + pub ec_lifecycle_locks: std::sync::Mutex< + std::collections::HashMap>>, + >, /// Volumes with an EC decode in flight. A dropped request leaves the /// blocking job running; this keeps a retry from racing it on the /// same volume files. @@ -151,6 +156,20 @@ pub struct VolumeServerState { } impl VolumeServerState { + /// Return the per-vid EC lifecycle lock, creating it on first use. Entries + /// are never reaped. + pub fn ec_lifecycle_lock( + &self, + vid: crate::storage::types::VolumeId, + ) -> Arc> { + self.ec_lifecycle_locks + .lock() + .unwrap() + .entry(vid) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } + /// The master lookups should go to: the leader the heartbeat confirmed, /// or empty while none is known, in its announced `host:port[.grpcPort]` /// form. Mirrors Go's `vs.getCurrentMaster()`. diff --git a/seaweed-volume/src/server/write_queue.rs b/seaweed-volume/src/server/write_queue.rs index 2398ec75d1..440893cafa 100644 --- a/seaweed-volume/src/server/write_queue.rs +++ b/seaweed-volume/src/server/write_queue.rs @@ -467,6 +467,7 @@ mod tests { security_file: String::new(), cli_white_list: vec![], state_file_path: String::new(), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), diff --git a/seaweed-volume/src/storage/store_ec_mirror.rs b/seaweed-volume/src/storage/store_ec_mirror.rs index 58a1a7c2cc..8cd2cc6810 100644 --- a/seaweed-volume/src/storage/store_ec_mirror.rs +++ b/seaweed-volume/src/storage/store_ec_mirror.rs @@ -36,10 +36,14 @@ impl Store { /// `reconcile_ec_shards_across_disks` so the orphan pass can /// prefer the local idx_directory. pub fn mirror_ec_metadata_to_shard_disks(&mut self) { + self.mirror_ec_metadata_to_shard_disks_only(None) + } + + pub(crate) fn mirror_ec_metadata_to_shard_disks_only(&mut self, only: Option) { if self.locations.len() < 2 { return; } - let owners = self.index_ecx_owners_for_mirror(); + let owners = self.index_ecx_owners_for_mirror(only); if owners.is_empty() { return; } @@ -54,7 +58,7 @@ impl Store { } let mut mirrors: Vec = Vec::new(); for (loc_idx, loc) in self.locations.iter().enumerate() { - let orphans = collect_shard_disk_volumes(loc); + let orphans = collect_shard_disk_volumes(loc, only); for (key, _shards) in orphans { let Some(owner) = owners.get(&key) else { continue; @@ -113,7 +117,7 @@ impl Store { // Records both idx_dir (where .ecx was found) and data_dir, so // the mirror can resolve .vif from data_dir even when .ecx lives // in idx_directory. - fn index_ecx_owners_for_mirror(&self) -> HashMap { + fn index_ecx_owners_for_mirror(&self, only: Option) -> HashMap { let mut owners: HashMap = HashMap::new(); for (loc_idx, loc) in self.locations.iter().enumerate() { let mut seen: Vec<&str> = Vec::with_capacity(2); @@ -133,15 +137,18 @@ impl Store { let Some(base) = name.strip_suffix(".ecx") else { continue; }; + let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { + continue; + }; + if only.is_some_and(|o| o != vid) { + continue; + } // A 0-byte .ecx is a corrupt stub from a failed copy, not a // credible owner — skip it so the scan keeps looking for a // real index on a sibling disk (Go's indexEcxOwners). if !ent.metadata().is_ok_and(|m| m.len() > 0) { continue; } - let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { - continue; - }; owners .entry(EcKey { collection, vid }) .or_insert_with(|| EcxOwner { @@ -297,7 +304,10 @@ fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> { Ok(()) } -fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap> { +fn collect_shard_disk_volumes( + loc: &DiskLocation, + only: Option, +) -> HashMap> { let mut out: HashMap> = HashMap::new(); let Ok(read) = fs::read_dir(&loc.directory) else { return out; @@ -314,13 +324,16 @@ fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap> if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() { continue; } + let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { + continue; + }; + if only.is_some_and(|o| o != vid) { + continue; + } match ent.metadata() { Ok(meta) if meta.len() > 0 => {} _ => continue, } - let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { - continue; - }; out.entry(EcKey { collection, vid }).or_default().push(name); } out @@ -688,7 +701,7 @@ mod tests { add_loc(&mut store, &dir0); add_loc(&mut store, &dir1); - let owners = store.index_ecx_owners_for_mirror(); + let owners = store.index_ecx_owners_for_mirror(None); let owner = owners .get(&EcKey { collection: collection.to_string(), diff --git a/seaweed-volume/src/storage/store_ec_reconcile.rs b/seaweed-volume/src/storage/store_ec_reconcile.rs index 0c81e42443..e1fa463164 100644 --- a/seaweed-volume/src/storage/store_ec_reconcile.rs +++ b/seaweed-volume/src/storage/store_ec_reconcile.rs @@ -92,11 +92,15 @@ impl Store { /// /// Mirrors `Store.reconcileEcShardsAcrossDisks` in Go. pub fn reconcile_ec_shards_across_disks(&mut self) { + self.reconcile_ec_shards_across_disks_only(None) + } + + pub(crate) fn reconcile_ec_shards_across_disks_only(&mut self, only: Option) { if self.locations.len() < 2 { return; } - let owners = self.index_ecx_owners(); + let owners = self.index_ecx_owners(only); if owners.is_empty() { return; } @@ -106,7 +110,7 @@ impl Store { // loc.idx_directory instead of the owner disk. let mut to_load: Vec = Vec::new(); for (loc_idx, loc) in self.locations.iter().enumerate() { - let orphans = collect_orphan_ec_shards(loc, loc_idx); + let orphans = collect_orphan_ec_shards(loc, loc_idx, only); for (key, shards) in orphans { let Some(owner) = owners.get(&key) else { warn!( @@ -402,7 +406,7 @@ impl Store { /// matched so downstream loaders point `EcVolume::new` at the /// directory that really has the file. The first owner found wins; /// duplicates across disks are unusual but tolerated. - fn index_ecx_owners(&self) -> HashMap { + fn index_ecx_owners(&self, only: Option) -> HashMap { let mut owners: HashMap = HashMap::new(); for (loc_idx, loc) in self.locations.iter().enumerate() { let mut seen: Vec<&str> = Vec::with_capacity(2); @@ -422,15 +426,18 @@ impl Store { let Some(base) = name.strip_suffix(".ecx") else { continue; }; + let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { + continue; + }; + if only.is_some_and(|o| o != vid) { + continue; + } // A 0-byte .ecx is a corrupt stub from a failed copy, not a // credible owner — skip it so the scan keeps looking for a // real index on a sibling disk (Go's indexEcxOwners). if !ent.metadata().is_ok_and(|m| m.len() > 0) { continue; } - let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { - continue; - }; let key = EcKey { collection, vid }; owners.entry(key).or_insert(EcxOwnerInfo { location: loc_idx, @@ -450,11 +457,18 @@ impl Store { /// /// Mirrors `Store.CollectEcVolumesMissingIndex` in Go. pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec { - let owners = self.index_ecx_owners(); + self.collect_ec_volumes_missing_index_only(None) + } + + pub(crate) fn collect_ec_volumes_missing_index_only( + &self, + only: Option, + ) -> Vec { + let owners = self.index_ecx_owners(only); let mut seen: HashSet = HashSet::new(); let mut missing = Vec::new(); for (loc_idx, loc) in self.locations.iter().enumerate() { - for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) { + for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx, only) { if owners.contains_key(&key) || !seen.insert(key.clone()) { continue; } @@ -474,18 +488,24 @@ impl Store { /// the disks that now have a local index, then fall back to the cross-disk /// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go. pub(crate) fn mount_recovered_ec_shards(&mut self) { - self.mirror_ec_metadata_to_shard_disks(); - self.load_orphan_ec_shards_with_local_index(); - self.reconcile_ec_shards_across_disks(); + self.mount_recovered_ec_shards_only(None) + } + + /// `mount_recovered_ec_shards` restricted to one volume's shards when + /// `only` names it. + pub(crate) fn mount_recovered_ec_shards_only(&mut self, only: Option) { + self.mirror_ec_metadata_to_shard_disks_only(only); + self.load_orphan_ec_shards_with_local_index(only); + self.reconcile_ec_shards_across_disks_only(only); } /// Mount on-disk EC shards whose `.ecx` index is now present on the same disk. /// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a /// single-disk store recovers once its index has been fetched from a peer. - fn load_orphan_ec_shards_with_local_index(&mut self) { + fn load_orphan_ec_shards_with_local_index(&mut self, only: Option) { let mut work: Vec<(usize, EcKey, Vec)> = Vec::new(); for (loc_idx, loc) in self.locations.iter().enumerate() { - for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) { + for (key, shards) in collect_orphan_ec_shards(loc, loc_idx, only) { if !loc.has_ecx_file_on_disk(&key.collection, key.vid) { continue; } @@ -516,6 +536,7 @@ impl Store { fn collect_orphan_ec_shards( loc: &crate::storage::disk_location::DiskLocation, _loc_idx: usize, + only: Option, ) -> HashMap> { let mut orphans: HashMap> = HashMap::new(); let Ok(read) = fs::read_dir(&loc.directory) else { @@ -533,6 +554,12 @@ fn collect_orphan_ec_shards( let Some(shard_id) = is_ec_shard_extension(ext) else { continue; }; + let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { + continue; + }; + if only.is_some_and(|o| o != vid) { + continue; + } // Ignore zero-byte shards. Use the DirEntry's metadata so we // don't pay a second stat syscall per file beyond what // read_dir already returned. @@ -540,9 +567,6 @@ fn collect_orphan_ec_shards( Ok(meta) if meta.len() > 0 => {} _ => continue, } - let Some((collection, vid)) = parse_collection_volume_id_pub(base) else { - continue; - }; // Skip shards that are already registered to an EcVolume. if let Some(ecv) = loc.find_ec_volume(vid) && ecv.has_shard(shard_id) @@ -655,7 +679,7 @@ mod tests { std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap(); write_index_files(&d1, "pics", 7, 10, 4); - let owners = store.index_ecx_owners(); + let owners = store.index_ecx_owners(None); let owner = owners .get(&EcKey { collection: "pics".to_string(), @@ -667,7 +691,7 @@ mod tests { // A stub with no real index anywhere owns nothing. std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap(); - assert!(!store.index_ecx_owners().contains_key(&EcKey { + assert!(!store.index_ecx_owners(None).contains_key(&EcKey { collection: "pics".to_string(), vid: VolumeId(8), })); @@ -1240,6 +1264,79 @@ mod tests { ); } + #[test] + fn test_mount_recovered_ec_shards_only_named_volume() { + // Recovering one volume mounts only its shards. Another volume's orphan + // shards, such as ones an in-flight copy just renamed in, stay unmounted + // even when an index for them is already on disk. Mirrors Go + // TestMountRecoveredEcShards_OnlyNamedVolume. + let tmp = TempDir::new().unwrap(); + let dir0 = tmp.path().join("data0"); + let dir1 = tmp.path().join("data1"); + std::fs::create_dir_all(&dir0).unwrap(); + std::fs::create_dir_all(&dir1).unwrap(); + + let collection = "col"; + let (recovered_vid, other_vid) = (71u32, 72u32); + let mut store = Store::new(NeedleMapKind::InMemory); + for d in [&dir0, &dir1] { + store + .add_location( + d.to_str().unwrap(), + d.to_str().unwrap(), + 100, + DiskType::HardDrive, + MinFreeSpace::Percent(0.0), + Vec::new(), + ) + .unwrap(); + } + for vid in [recovered_vid, other_vid] { + write_shard(dir0.to_str().unwrap(), collection, vid, 0); + write_shard(dir1.to_str().unwrap(), collection, vid, 1); + } + write_index_files(dir0.to_str().unwrap(), collection, other_vid, 10, 4); + let missing = store.collect_ec_volumes_missing_index(); + let m = missing + .iter() + .find(|m| m.vid == VolumeId(recovered_vid)) + .expect("recovered volume not reported as missing its index"); + write_index_files(&m.idx_dir, collection, recovered_vid, 10, 4); + + store.mount_recovered_ec_shards_only(Some(VolumeId(recovered_vid))); + + for (i, loc) in store.locations.iter().enumerate() { + let ev = loc + .find_ec_volume(VolumeId(recovered_vid)) + .unwrap_or_else(|| panic!("volume {} not mounted on disk{}", recovered_vid, i)); + assert!( + ev.has_shard(i as u8), + "shard {}.{} not mounted", + recovered_vid, + i + ); + assert!( + loc.find_ec_volume(VolumeId(other_vid)).is_none(), + "volume {} mounted on disk{} by the recovery of volume {}", + other_vid, + i, + recovered_vid, + ); + } + let mirrored = format!( + "{}/{}_{}.ecx", + dir1.to_str().unwrap(), + collection, + other_vid + ); + assert!( + !std::path::Path::new(&mirrored).exists(), + "volume {} index mirrored to disk1 by the recovery of volume {}", + other_vid, + recovered_vid, + ); + } + #[test] fn test_collect_missing_index_excludes_cross_disk_orphan() { // A .ecx merely on a sibling disk is the same-server case the reconcile diff --git a/seaweed-volume/tests/http_integration.rs b/seaweed-volume/tests/http_integration.rs index 8b539ab8b4..c1aaa805b0 100644 --- a/seaweed-volume/tests/http_integration.rs +++ b/seaweed-volume/tests/http_integration.rs @@ -110,6 +110,7 @@ fn build_test_state( vacuum_commit_slots: None, vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0), is_heartbeating: std::sync::atomic::AtomicBool::new(true), + ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()), ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()), volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()), diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index f83aa2fa13..a0e4fcbdc1 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "slices" "strconv" "strings" "time" @@ -209,6 +210,14 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s } glog.V(0).Infof("VolumeEcShardsRebuild: %v", req) + + // Serialize against full teardown and the other canonical-EC-file writers for this + // vid so a concurrent teardown wipe cannot clobber the rebuilt .ecNN/.ecx, and vice + // versa. Held across the rebuild IO. + lifecycle := vs.store.EcLifecycleLock(needle.VolumeId(req.VolumeId)) + lifecycle.Lock() + defer lifecycle.Unlock() + baseFileName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId)) var rebuiltShardIds []uint32 @@ -399,6 +408,12 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv glog.V(0).Infof("VolumeEcShardsCopy: %v", req) + // Overlapping copies of one vid would share the staged index, and a failed + // one's cleanup would remove shards another already committed. + lifecycle := vs.store.EcLifecycleLock(needle.VolumeId(req.VolumeId)) + lifecycle.Lock() + defer lifecycle.Unlock() + var location *storage.DiskLocation // Select the target location for storing EC shard files. @@ -467,62 +482,65 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv } throttler := util.NewWriteThrottler(ioBytePerSecond) + // A failed copy removes what it added, or every retry against the same + // unusable source strands another unmounted shard on this disk. + var created ecCopyCreatedFiles err := operation.WithVolumeServerClientOptions(true, pb.ServerAddress(req.SourceDataNode), func(client volume_server_pb.VolumeServerClient) error { - // copy ec data slices - for _, shardId := range req.ShardIds { - if _, err := vs.doCopyFileWithThrottler(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.ToExt(int(shardId)), false, false, nil, throttler); err != nil { + // The index is staged first, so a source whose .ecx is unusable fails + // the copy before any shard lands here. It is committed after every + // other file, so a failed copy leaves an existing holder's index as is. + ecxPath := indexBaseFileName + ".ecx" + var stagedEcx string + if req.CopyEcxFile { + staged, err := vs.stageEcxFile(client, req.Collection, req.VolumeId, ecxPath, throttler) + if err != nil { return err } + stagedEcx = staged + defer os.Remove(stagedEcx) } - if req.CopyEcxFile { - - // copy ecx file - if _, err := vs.doCopyFileWithThrottler(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecx", false, false, nil, throttler); err != nil { + // copy ec data slices. Each shard is staged and moved into place only + // once the whole copy succeeds, so a failed transfer leaves a shard + // already here as is and never removes one another writer put there. + var stagedShards []uint32 + seen := make(map[uint32]bool, len(req.ShardIds)) + for _, shardId := range req.ShardIds { + if seen[shardId] { + continue + } + seen[shardId] = true + ext := erasure_coding.ToExt(int(shardId)) + staged, err := vs.stageEcCopyFile(client, req.Collection, req.VolumeId, ext, dataBaseFileName+ext, false, throttler) + if err != nil { return err } - // Defense in depth: writeToFile now removes partial files on - // stream error, but a source that genuinely held a 0-byte - // .ecx (e.g. a corrupted upstream replica) would otherwise - // leave a 0-byte file here and the mount path would reject - // it later. Catch that at distribute time so the orchestrator - // can pick a different source rather than learning about it - // at mount. - // Stat failure must not silently pass. doCopyFile reported - // success, but if the file is gone, unreadable, or a directory - // somehow, the orchestrator should learn now — at mount time - // the operator only sees "no .ecx found" with no useful context - // about which step actually failed. - ecxPath := indexBaseFileName + ".ecx" - info, statErr := os.Stat(ecxPath) - if statErr != nil { - return fmt.Errorf("VolumeEcShardsCopy volume %d: stat copied .ecx %s: %w", req.VolumeId, ecxPath, statErr) - } - if info.IsDir() { - return fmt.Errorf("VolumeEcShardsCopy volume %d: copied .ecx path %s is a directory", req.VolumeId, ecxPath) - } - if info.Size() == 0 { - if removeErr := os.Remove(ecxPath); removeErr != nil && !os.IsNotExist(removeErr) { - glog.Warningf("VolumeEcShardsCopy volume %d: remove 0-byte .ecx %s: %v", req.VolumeId, ecxPath, removeErr) - } - return fmt.Errorf("VolumeEcShardsCopy volume %d: source .ecx is 0 bytes", req.VolumeId) - } + defer os.Remove(staged) + stagedShards = append(stagedShards, shardId) } if req.CopyEcjFile { // The journal is a *set* of ids: merge the source's into the // local one as a union, never append it whole. + created.track(indexBaseFileName + ".ecj") if err := vs.copyEcjAndMerge(client, req.Collection, req.VolumeId, location.Directory, indexBaseFileName, throttler); err != nil { return err } } + // The sidecars are staged too and committed with the index, so a failed + // copy cannot pair a new .vif with the index this disk keeps. + var stagedSidecars []string if req.CopyVifFile { - // copy vif file - if _, err := vs.doCopyFileWithThrottler(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil, throttler); err != nil { + staged, err := vs.stageEcCopyFile(client, req.Collection, req.VolumeId, ".vif", dataBaseFileName+".vif", true, throttler) + if err != nil { return err } + if staged != "" { + defer os.Remove(staged) + stagedSidecars = append(stagedSidecars, dataBaseFileName+".vif") + } } if req.CopyEcsumFile { @@ -531,19 +549,158 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv // distribution) has no Prepare backstop, and fresh-encode sidecar // writes are best-effort, so a missing source sidecar is a no-op // (ignore-not-found): the holder is simply unprotected. - if _, err := vs.doCopyFileWithThrottler(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.BitrotSidecarExt, false, true, nil, throttler); err != nil { + sidecarPath := dataBaseFileName + erasure_coding.BitrotSidecarExt + staged, err := vs.stageEcCopyFile(client, req.Collection, req.VolumeId, erasure_coding.BitrotSidecarExt, sidecarPath, true, throttler) + if err != nil { return fmt.Errorf("VolumeEcShardsCopy volume %d: copy %s sidecar: %w", req.VolumeId, erasure_coding.BitrotSidecarExt, err) } + if staged != "" { + defer os.Remove(staged) + stagedSidecars = append(stagedSidecars, sidecarPath) + } + } + + for _, path := range stagedSidecars { + created.track(path) + if err := os.Rename(path+".copying", path); err != nil { + return fmt.Errorf("VolumeEcShardsCopy volume %d: commit copied %s: %w", req.VolumeId, path, err) + } + } + + for _, shardId := range stagedShards { + shardPath := dataBaseFileName + erasure_coding.ToExt(int(shardId)) + // A mounted shard reads the file once reopened, so it is not ours to remove. + if _, mounted := location.FindEcShard(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)); !mounted { + created.track(shardPath) + } + if err := os.Rename(shardPath+".copying", shardPath); err != nil { + return fmt.Errorf("VolumeEcShardsCopy volume %d: commit copied %s: %w", req.VolumeId, shardPath, err) + } + // A mounted shard keeps reading the replaced inode until reopened. + if _, err := location.ReopenEcShard(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)); err != nil { + if diskId := slices.Index(vs.store.Locations, location); diskId >= 0 { + vs.store.UnmountEcShardOnDisk(diskId, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)) + } + return fmt.Errorf("VolumeEcShardsCopy volume %d: reopen replaced shard %d: %w", req.VolumeId, shardId, err) + } + } + + if stagedEcx != "" { + created.track(ecxPath) + if err := commitStagedEcx(stagedEcx, ecxPath); err != nil { + return fmt.Errorf("VolumeEcShardsCopy volume %d: commit copied .ecx %s: %w", req.VolumeId, ecxPath, err) + } } return nil }, vs.grpcDialOption, vs.guardedGrpcDialOption(req.SourceDataNode)) if err != nil { + created.remove(req.VolumeId) return nil, fmt.Errorf("VolumeEcShardsCopy volume %d: %v", req.VolumeId, err) } return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil } +// ecCopyCreatedFiles records the files a VolumeEcShardsCopy adds to its disk, +// so a failed copy removes exactly those and never one that was already there. +type ecCopyCreatedFiles []string + +func (c *ecCopyCreatedFiles) track(path string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + *c = append(*c, path) + } +} + +func (c ecCopyCreatedFiles) remove(vid uint32) { + for _, path := range c { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + glog.Warningf("VolumeEcShardsCopy volume %d: remove %s after failed copy: %v", vid, path, err) + } + } +} + +// stageEcxFile copies the source's .ecx beside ecxPath and returns the staged +// path once it proves non-empty; the caller renames it into place. A source +// holding a 0-byte index neither lands a stub nor truncates an .ecx this disk +// already has. +func (vs *VolumeServer) stageEcxFile(client volume_server_pb.VolumeServerClient, collection string, vid uint32, ecxPath string, throttler *util.WriteThrottler) (string, error) { + staging, err := vs.stageEcCopyFile(client, collection, vid, ".ecx", ecxPath, false, throttler) + if err != nil { + return "", err + } + // Stat failure must not silently pass: at mount time the operator would + // only see "no .ecx found" with no context about which step failed. + info, err := os.Stat(staging) + if err != nil { + return "", fmt.Errorf("VolumeEcShardsCopy volume %d: stat copied .ecx %s: %w", vid, staging, err) + } + if info.IsDir() { + return "", fmt.Errorf("VolumeEcShardsCopy volume %d: copied .ecx path %s is a directory", vid, staging) + } + if info.Size() == 0 { + if removeErr := os.Remove(staging); removeErr != nil && !os.IsNotExist(removeErr) { + glog.Warningf("VolumeEcShardsCopy volume %d: remove 0-byte .ecx %s: %v", vid, staging, removeErr) + } + return "", fmt.Errorf("VolumeEcShardsCopy volume %d: source .ecx is 0 bytes", vid) + } + return staging, nil +} + +// stageEcCopyFile copies the source's ext file of vid to path+".copying" and +// returns the staged path; the caller renames it into place. With +// ignoreSourceFileNotFound, a source without the file stages nothing and +// returns "". +func (vs *VolumeServer) stageEcCopyFile(client volume_server_pb.VolumeServerClient, collection string, vid uint32, ext, path string, ignoreSourceFileNotFound bool, throttler *util.WriteThrottler) (string, error) { + stream, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{ + VolumeId: vid, + Ext: ext, + CompactionRevision: math.MaxUint32, + StopOffset: math.MaxInt64, + Collection: collection, + IsEcVolume: true, + IgnoreSourceFileNotFound: ignoreSourceFileNotFound, + }) + if err != nil { + return "", fmt.Errorf("failed to start copying volume %d %s file: %v", vid, ext, err) + } + staging := path + ".copying" + modifiedTsNs, err := writeToFile(stream, staging, throttler, false, ignoreSourceFileNotFound, nil) + if err != nil { + return "", fmt.Errorf("failed to copy %s file: %v", path, err) + } + if ignoreSourceFileNotFound && modifiedTsNs == 0 { + return "", nil + } + return staging, nil +} + +// commitStagedEcx installs a staged index at ecxPath. An index already there +// is overwritten in place: mounted EcVolumes read it through open handles, which +// a rename would leave on the replaced inode. +func commitStagedEcx(staged, ecxPath string) error { + dst, err := os.OpenFile(ecxPath, os.O_WRONLY, 0) + if os.IsNotExist(err) { + return os.Rename(staged, ecxPath) + } + if err != nil { + return err + } + defer dst.Close() + src, err := os.Open(staged) + if err != nil { + return err + } + defer src.Close() + n, err := io.Copy(dst, src) + if err != nil { + return err + } + if err := dst.Truncate(n); err != nil { + return err + } + return dst.Sync() +} + // copyEcjAndMerge folds the source peer's .ecj into the local journal of vid // as a set union: only ids the local journal lacks are appended, so a // shard bounced between servers cannot grow it. The source journal streams @@ -616,6 +773,9 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se // volume and wipe every EC artifact for it on every disk, not just the listed // shards, so a remote node retains no stale generation a fresh copy collides with. glog.V(0).Infof("ec volume %s full teardown", bName) + lifecycle := vs.store.EcLifecycleLock(needle.VolumeId(req.VolumeId)) + lifecycle.Lock() + defer lifecycle.Unlock() vs.store.UnloadEcVolume(needle.VolumeId(req.VolumeId)) for _, location := range vs.store.Locations { dataBase := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId)) @@ -671,6 +831,12 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se glog.V(0).Infof("ec volume %s shard delete %v", bName, req.ShardIds) + // A copy's staged shards are invisible to the last-shard check below, so + // wait for it rather than remove the index it is about to need. + lifecycle := vs.store.EcLifecycleLock(needle.VolumeId(req.VolumeId)) + lifecycle.Lock() + defer lifecycle.Unlock() + // Pass 1: delete the requested shard files (and any now-orphaned per-disk bitrot // sidecars) on every disk. for diskId, location := range vs.store.Locations { diff --git a/weed/server/volume_grpc_erasure_coding_copy_cleanup_test.go b/weed/server/volume_grpc_erasure_coding_copy_cleanup_test.go new file mode 100644 index 0000000000..71933a4488 --- /dev/null +++ b/weed/server/volume_grpc_erasure_coding_copy_cleanup_test.go @@ -0,0 +1,615 @@ +package weed_server + +import ( + "bytes" + "context" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// startEcCopySource serves srcDir's EC files over CopyFile and returns the +// address a VolumeEcShardsCopy names as its source. +func startEcCopySource(t *testing.T, srcDir string) string { + t.Helper() + return serveEcCopySource(t, &VolumeServer{store: newRecoverTestStore(t, srcDir)}) +} + +func serveEcCopySource(t *testing.T, srv volume_server_pb.VolumeServerServer) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + volume_server_pb.RegisterVolumeServerServer(grpcServer, srv) + go grpcServer.Serve(lis) + t.Cleanup(grpcServer.Stop) + port := lis.Addr().(*net.TCPAddr).Port + return string(pb.NewServerAddress("127.0.0.1", port-10000, port)) +} + +// failingExtCopySource counts CopyFile requests per extension and fails the +// one for failExt, running onFail first. +type failingExtCopySource struct { + *VolumeServer + failExt string + onFail func() error + mu sync.Mutex + requested map[string]int +} + +func (s *failingExtCopySource) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error { + s.mu.Lock() + if s.requested == nil { + s.requested = make(map[string]int) + } + s.requested[req.Ext]++ + s.mu.Unlock() + if req.Ext == s.failExt { + if s.onFail != nil { + if err := s.onFail(); err != nil { + return status.Errorf(codes.Internal, "fixture: %v", err) + } + } + return status.Errorf(codes.Unavailable, "injected %s failure", req.Ext) + } + return s.VolumeServer.CopyFile(req, stream) +} + +func (s *failingExtCopySource) requests(ext string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.requested[ext] +} + +func newEcCopyReceiver(t *testing.T, dstDir string) *VolumeServer { + t.Helper() + return &VolumeServer{ + store: newRecoverTestStore(t, dstDir), + grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + AllowUntrustedRemoteEndpoints: true, + } +} + +func listDir(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir %s: %v", dir, err) + } + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + sort.Strings(names) + return names +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// A source whose .ecx is 0 bytes must fail the copy without stranding a shard, +// index or sidecar on the destination, and without touching what an existing +// holder of the same volume already has there. +// waitForLockWaiter waits until a goroutine running handler is blocked on a +// mutex. The test holds the lifecycle lock and nothing else, so that is the +// lock the handler waits for. +func waitForLockWaiter(t *testing.T, handler string) { + t.Helper() + buf := make([]byte, 1<<20) + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + n := runtime.Stack(buf, true) + if n == len(buf) { + buf = make([]byte, 2*len(buf)) + continue + } + for _, g := range strings.Split(string(buf[:n]), "\n\n") { + if strings.Contains(g, "sync.(*Mutex).lockSlow") && strings.Contains(g, handler) { + return + } + } + time.Sleep(time.Millisecond) + } + t.Fatalf("%s never waited for the lifecycle lock", handler) +} + +func TestVolumeEcShardsCopy_EmptySourceEcxLeavesNoFiles(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(57) + + srcDir := filepath.Join(t.TempDir(), "src") + source := startEcCopySource(t, srcDir) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + writeFile(t, srcBase+".ecx", nil) + writeFile(t, srcBase+".ec00", bytes.Repeat([]byte{1}, 4096)) + writeFile(t, srcBase+".ecj", nil) + writeFile(t, srcBase+".vif", []byte("{}")) + + req := &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0}, + CopyEcxFile: true, + CopyEcjFile: true, + CopyVifFile: true, + CopyEcsumFile: true, + SourceDataNode: source, + } + + t.Run("fresh destination", func(t *testing.T) { + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + before := listDir(t, dstDir) + + _, err := receiver.VolumeEcShardsCopy(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "source .ecx is 0 bytes") { + t.Fatalf("VolumeEcShardsCopy err = %v, want source .ecx is 0 bytes", err) + } + if after := listDir(t, dstDir); strings.Join(after, ",") != strings.Join(before, ",") { + t.Fatalf("destination files after failed copy = %v, want %v", after, before) + } + }) + + t.Run("existing holder", func(t *testing.T) { + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + ecx := make([]byte, types.NeedleMapEntrySize) + ecx[0] = 7 + ec05 := bytes.Repeat([]byte{5}, 4096) + writeFile(t, dstBase+".ecx", ecx) + writeFile(t, dstBase+".ec05", ec05) + before := listDir(t, dstDir) + + if _, err := receiver.VolumeEcShardsCopy(context.Background(), req); err == nil { + t.Fatalf("VolumeEcShardsCopy succeeded against a 0-byte source .ecx") + } + if after := listDir(t, dstDir); strings.Join(after, ",") != strings.Join(before, ",") { + t.Fatalf("destination files after failed copy = %v, want %v", after, before) + } + if got, _ := os.ReadFile(dstBase + ".ecx"); !bytes.Equal(got, ecx) { + t.Errorf("pre-existing .ecx changed: %x, want %x", got, ecx) + } + if got, _ := os.ReadFile(dstBase + ".ec05"); !bytes.Equal(got, ec05) { + t.Errorf("pre-existing .ec05 changed") + } + }) +} + +// A copy failing after some shards landed removes exactly those shards and +// keeps the ones the destination held before the call, its index included. +func TestVolumeEcShardsCopy_FailedShardRemovesCopiedShards(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(58) + + srcDir := filepath.Join(t.TempDir(), "src") + source := startEcCopySource(t, srcDir) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + srcEcx := make([]byte, types.NeedleMapEntrySize) + srcEcx[0] = 9 + writeFile(t, srcBase+".ecx", srcEcx) + writeFile(t, srcBase+".ec00", bytes.Repeat([]byte{1}, 4096)) + writeFile(t, srcBase+".ec05", bytes.Repeat([]byte{2}, 4096)) + + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + ecx := make([]byte, types.NeedleMapEntrySize) + ecx[0] = 7 + writeFile(t, dstBase+".ecx", ecx) + ec05 := bytes.Repeat([]byte{5}, 4096) + writeFile(t, dstBase+".ec05", ec05) + ec01 := bytes.Repeat([]byte{6}, 4096) + writeFile(t, dstBase+".ec01", ec01) + before := listDir(t, dstDir) + + // Shard 0 copies, shard 5 replaces the existing holder's copy, shard 1 is + // held here but missing on the source and fails the request. + _, err := receiver.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0, 5, 1}, + CopyEcxFile: true, + SourceDataNode: source, + }) + if err == nil || !strings.Contains(err.Error(), ".ec01") { + t.Fatalf("VolumeEcShardsCopy error = %v, want the .ec01 transfer to fail", err) + } + if after := listDir(t, dstDir); strings.Join(after, ",") != strings.Join(before, ",") { + t.Fatalf("destination files after failed copy = %v, want %v", after, before) + } + if got, _ := os.ReadFile(dstBase + ".ecx"); !bytes.Equal(got, ecx) { + t.Errorf("pre-existing .ecx changed: %x, want %x", got, ecx) + } + if got, _ := os.ReadFile(dstBase + ".ec05"); !bytes.Equal(got, ec05) { + t.Errorf("pre-existing .ec05 changed") + } + if got, _ := os.ReadFile(dstBase + ".ec01"); !bytes.Equal(got, ec01) { + t.Errorf("pre-existing .ec01 changed by its failed transfer") + } +} + +// A copied index replaces an existing holder's in place, so a mounted +// EcVolume's open handle reads the source's index, tombstones included. +func TestVolumeEcShardsCopy_ExistingEcxUpdatedInPlace(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(59) + + srcDir := filepath.Join(t.TempDir(), "src") + source := startEcCopySource(t, srcDir) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + srcEcx := bytes.Repeat([]byte{9}, 2*types.NeedleMapEntrySize) + writeFile(t, srcBase+".ecx", srcEcx) + srcEc00 := bytes.Repeat([]byte{1}, 4096) + writeFile(t, srcBase+".ec00", srcEc00) + srcVif := []byte(`{"version":3}`) + writeFile(t, srcBase+".vif", srcVif) + + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + writeFile(t, dstBase+".ecx", make([]byte, 2*types.NeedleMapEntrySize)) + writeFile(t, dstBase+".ec00", bytes.Repeat([]byte{3}, 1024)) + held, err := os.Open(dstBase + ".ecx") + if err != nil { + t.Fatalf("open .ecx: %v", err) + } + defer held.Close() + + if _, err := receiver.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0}, + CopyEcxFile: true, + CopyVifFile: true, + CopyEcsumFile: true, + SourceDataNode: source, + }); err != nil { + t.Fatalf("VolumeEcShardsCopy: %v", err) + } + got := make([]byte, len(srcEcx)) + if _, err := held.ReadAt(got, 0); err != nil { + t.Fatalf("read open .ecx: %v", err) + } + if !bytes.Equal(got, srcEcx) { + t.Fatalf("open .ecx handle reads %x after the copy, want the source index %x", got, srcEcx) + } + if got, _ := os.ReadFile(dstBase + ".ec00"); !bytes.Equal(got, srcEc00) { + t.Errorf("existing .ec00 not replaced by the source's") + } + if got, _ := os.ReadFile(dstBase + ".vif"); !bytes.Equal(got, srcVif) { + t.Errorf("copied .vif = %s, want %s", got, srcVif) + } + if _, err := os.Stat(dstBase + erasure_coding.BitrotSidecarExt); !os.IsNotExist(err) { + t.Errorf("a source without .ecsum left one here: %v", err) + } + for _, name := range listDir(t, dstDir) { + if strings.HasSuffix(name, ".copying") { + t.Errorf("staged file %s left behind", name) + } + } +} + +// A shard listed twice is copied once, and replacing one already held here +// still succeeds. +func TestVolumeEcShardsCopy_DuplicateShardIdCopiedOnce(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(62) + + srcDir := filepath.Join(t.TempDir(), "src") + src := &failingExtCopySource{VolumeServer: &VolumeServer{store: newRecoverTestStore(t, srcDir)}} + source := serveEcCopySource(t, src) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + writeFile(t, srcBase+".ecx", bytes.Repeat([]byte{9}, types.NeedleMapEntrySize)) + srcEc05 := bytes.Repeat([]byte{2}, 4096) + writeFile(t, srcBase+".ec05", srcEc05) + + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + writeFile(t, dstBase+".ecx", make([]byte, types.NeedleMapEntrySize)) + writeFile(t, dstBase+".ec05", bytes.Repeat([]byte{5}, 4096)) + + if _, err := receiver.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{5, 5}, + CopyEcxFile: true, + SourceDataNode: source, + }); err != nil { + t.Fatalf("VolumeEcShardsCopy: %v", err) + } + if got, _ := os.ReadFile(dstBase + ".ec05"); !bytes.Equal(got, srcEc05) { + t.Errorf("existing .ec05 not replaced by the source's") + } + if n := src.requests(".ec05"); n != 1 { + t.Errorf("source served .ec05 %d times, want once", n) + } +} + +// A copy failing after the .vif transferred keeps the existing holder's .vif, +// which must stay paired with the index the failed copy left in place. +func TestVolumeEcShardsCopy_FailedSidecarKeepsExistingVif(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(60) + + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + + srcVif := []byte(`{"version":3}`) + stagedVif := make(chan []byte, 1) + srcDir := filepath.Join(t.TempDir(), "src") + src := &failingExtCopySource{ + VolumeServer: &VolumeServer{store: newRecoverTestStore(t, srcDir)}, + failExt: erasure_coding.BitrotSidecarExt, + onFail: func() error { + data, err := os.ReadFile(dstBase + ".vif.copying") + stagedVif <- data + return err + }, + } + source := serveEcCopySource(t, src) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + writeFile(t, srcBase+".ecx", bytes.Repeat([]byte{9}, types.NeedleMapEntrySize)) + writeFile(t, srcBase+".ec00", bytes.Repeat([]byte{1}, 4096)) + writeFile(t, srcBase+".vif", srcVif) + writeFile(t, srcBase+erasure_coding.BitrotSidecarExt, []byte("sum")) + + ecx := make([]byte, types.NeedleMapEntrySize) + writeFile(t, dstBase+".ecx", ecx) + vif := []byte(`{"version":2}`) + writeFile(t, dstBase+".vif", vif) + before := listDir(t, dstDir) + + _, err := receiver.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0}, + CopyEcxFile: true, + CopyVifFile: true, + CopyEcsumFile: true, + SourceDataNode: source, + }) + if err == nil || !strings.Contains(err.Error(), "injected "+erasure_coding.BitrotSidecarExt+" failure") { + t.Fatalf("VolumeEcShardsCopy error = %v, want the %s transfer to fail", err, erasure_coding.BitrotSidecarExt) + } + select { + case got := <-stagedVif: + if !bytes.Equal(got, srcVif) { + t.Fatalf("staged .vif at the %s failure = %q, want the source's %q", erasure_coding.BitrotSidecarExt, got, srcVif) + } + default: + t.Fatalf("the %s request never reached the source", erasure_coding.BitrotSidecarExt) + } + if after := listDir(t, dstDir); strings.Join(after, ",") != strings.Join(before, ",") { + t.Fatalf("destination files after failed copy = %v, want %v", after, before) + } + if got, _ := os.ReadFile(dstBase + ".vif"); !bytes.Equal(got, vif) { + t.Errorf("pre-existing .vif changed to %s, want %s", got, vif) + } + if got, _ := os.ReadFile(dstBase + ".ecx"); !bytes.Equal(got, ecx) { + t.Errorf("pre-existing .ecx changed") + } +} + +// A mount waits out an in-flight copy of the same volume, whose failure cleanup +// would remove a shard mounted meanwhile. +func TestVolumeEcShardsMount_WaitsForInFlightCopy(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(63) + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + writeFile(t, dstBase+".ecx", make([]byte, types.NeedleMapEntrySize)) + writeFile(t, dstBase+".ec00", bytes.Repeat([]byte{1}, 4096)) + + lifecycle := receiver.store.EcLifecycleLock(vid) + lifecycle.Lock() + done := make(chan error, 1) + go func() { + _, err := receiver.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0}, + }) + done <- err + }() + waitForLockWaiter(t, "VolumeEcShardsMount") + _, _, mountedEarly := receiver.store.FindEcShard(vid, 0) + lifecycle.Unlock() + if mountedEarly { + t.Fatal("mount ran while a copy of the same volume held its lifecycle lock") + } + select { + case err := <-done: + if err != nil { + t.Fatalf("mount: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("mount did not finish after the copy released the lifecycle lock") + } + if _, _, found := receiver.store.FindEcShard(vid, 0); !found { + t.Fatal("shard 0 not mounted after the copy released the lifecycle lock") + } +} + +// A full teardown waits out an in-flight copy of the same volume, which would +// otherwise recreate the files after the teardown is acknowledged. +func TestVolumeEcShardsDelete_FullTeardownWaitsForInFlightCopy(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(64) + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + writeFile(t, dstBase+".ecx", make([]byte, types.NeedleMapEntrySize)) + writeFile(t, dstBase+".ec00", bytes.Repeat([]byte{1}, 4096)) + + lifecycle := receiver.store.EcLifecycleLock(vid) + lifecycle.Lock() + done := make(chan error, 1) + go func() { + _, err := receiver.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: uint32(vid), + Collection: collection, + FullTeardown: true, + }) + done <- err + }() + waitForLockWaiter(t, "VolumeEcShardsDelete") + _, statErr := os.Stat(dstBase + ".ecx") + lifecycle.Unlock() + if statErr != nil { + t.Fatalf("full teardown ran while a copy of the same volume held its lifecycle lock: %v", statErr) + } + select { + case err := <-done: + if err != nil { + t.Fatalf("full teardown: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("full teardown did not finish after the copy released the lifecycle lock") + } + if _, err := os.Stat(dstBase + ".ecx"); !os.IsNotExist(err) { + t.Fatalf(".ecx after the teardown: %v, want it removed", err) + } +} + +// A shard another writer puts here while a copy is in flight survives the +// copy's failure cleanup. +func TestVolumeEcShardsCopy_FailedCopyKeepsShardWrittenMeanwhile(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(65) + + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + received := bytes.Repeat([]byte{7}, 4096) + sent := bytes.Repeat([]byte{1}, 4096) + + srcDir := filepath.Join(t.TempDir(), "src") + src := &failingExtCopySource{ + VolumeServer: &VolumeServer{store: newRecoverTestStore(t, srcDir)}, + failExt: ".ec01", + onFail: func() error { + if staged, err := os.ReadFile(dstBase + ".ec00.copying"); err != nil || !bytes.Equal(staged, sent) { + return fmt.Errorf(".ec00 not staged before .ec01: %d bytes, %v", len(staged), err) + } + return os.WriteFile(dstBase+".ec00", received, 0o644) + }, + } + source := serveEcCopySource(t, src) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + writeFile(t, srcBase+".ecx", bytes.Repeat([]byte{9}, types.NeedleMapEntrySize)) + writeFile(t, srcBase+".ec00", sent) + writeFile(t, srcBase+".ec01", bytes.Repeat([]byte{2}, 4096)) + + _, err := receiver.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0, 1}, + CopyEcxFile: true, + SourceDataNode: source, + }) + if err == nil || !strings.Contains(err.Error(), "injected .ec01 failure") { + t.Fatalf("VolumeEcShardsCopy error = %v, want the .ec01 transfer to fail", err) + } + if got, err := os.ReadFile(dstBase + ".ec00"); err != nil || !bytes.Equal(got, received) { + t.Fatalf(".ec00 written during the copy = %d bytes, %v; want it kept", len(got), err) + } +} + +// A rebuild waits out an in-flight copy of the same volume, which would +// otherwise overwrite the index the rebuild folded tombstones into. +func TestVolumeEcShardsRebuild_WaitsForInFlightCopy(t *testing.T) { + vid := needle.VolumeId(66) + receiver := newEcCopyReceiver(t, filepath.Join(t.TempDir(), "dst")) + lifecycle := receiver.store.EcLifecycleLock(vid) + lifecycle.Lock() + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = receiver.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{ + VolumeId: uint32(vid), + Collection: "col", + }) + }() + waitForLockWaiter(t, "VolumeEcShardsRebuild") + select { + case <-done: + lifecycle.Unlock() + t.Fatal("rebuild ran while a copy of the same volume held its lifecycle lock") + default: + } + lifecycle.Unlock() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("rebuild did not finish after the copy released the lifecycle lock") + } +} + +// A shard delete waits out an in-flight copy of the same volume, whose staged +// shards the delete's last-shard check cannot see. +func TestVolumeEcShardsDelete_WaitsForInFlightCopy(t *testing.T) { + const collection = "col" + vid := needle.VolumeId(67) + dstDir := filepath.Join(t.TempDir(), "dst") + receiver := newEcCopyReceiver(t, dstDir) + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + writeFile(t, dstBase+".ecx", make([]byte, types.NeedleMapEntrySize)) + writeFile(t, dstBase+".ec00", bytes.Repeat([]byte{1}, 4096)) + + lifecycle := receiver.store.EcLifecycleLock(vid) + lifecycle.Lock() + done := make(chan error, 1) + go func() { + _, err := receiver.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: []uint32{0}, + }) + done <- err + }() + waitForLockWaiter(t, "VolumeEcShardsDelete") + _, statErr := os.Stat(dstBase + ".ec00") + lifecycle.Unlock() + if statErr != nil { + t.Fatalf("shard deleted while a copy of the same volume held its lifecycle lock: %v", statErr) + } + select { + case err := <-done: + if err != nil { + t.Fatalf("shard delete: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("shard delete did not finish after the copy released the lifecycle lock") + } + if _, err := os.Stat(dstBase + ".ec00"); !os.IsNotExist(err) { + t.Fatalf(".ec00 after the delete: %v, want it removed", err) + } +} diff --git a/weed/server/volume_grpc_erasure_coding_recover.go b/weed/server/volume_grpc_erasure_coding_recover.go index 3e28b4ae81..1ab3dee7ed 100644 --- a/weed/server/volume_grpc_erasure_coding_recover.go +++ b/weed/server/volume_grpc_erasure_coding_recover.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "os" + "slices" "time" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -49,34 +50,55 @@ func (vs *VolumeServer) recoverMissingEcIndexes(filterVid uint32) int { master := vs.getCurrentMaster() if master == "" { - glog.Warningf("cannot recover missing EC index without a master connection") - return 0 + glog.Warningf("cannot look up peers to recover missing EC indexes without a master connection") } self := pb.NewServerAddress(vs.store.Ip, vs.store.Port, vs.store.GrpcPort) recovered := 0 for _, m := range orphans { - peers, err := vs.lookupEcVolumePeers(master, uint32(m.VolumeId), self) - if err != nil { - glog.Warningf("ec volume %d: cannot look up peers to recover missing .ecx: %v", m.VolumeId, err) - continue + var peers []pb.ServerAddress + if master != "" { + var err error + if peers, err = vs.lookupEcVolumePeers(master, uint32(m.VolumeId), self); err != nil { + glog.Warningf("ec volume %d: cannot look up peers to recover missing .ecx: %v", m.VolumeId, err) + } } - if len(peers) == 0 { - glog.Warningf("ec volume %d: shards present locally but .ecx missing and no peer holds it; leaving shards unloaded", m.VolumeId) - continue - } - if vs.fetchEcIndexFromPeers(peers, m) { + if vs.recoverEcIndexLocked(peers, m) { recovered++ } } if recovered > 0 { - vs.store.MountRecoveredEcShards() glog.V(0).Infof("recovered missing EC index for %d volume(s) from peers and mounted their shards", recovered) } return recovered } +// recoverEcIndexLocked fetches m's index and mounts that volume's shards under +// its lifecycle lock, so an in-flight VolumeEcShardsCopy of the volume, which +// may unlink the shards it renamed in, never has them mounted from under it. +func (vs *VolumeServer) recoverEcIndexLocked(peers []pb.ServerAddress, m storage.EcVolumeMissingIndex) bool { + lifecycle := vs.store.EcLifecycleLock(m.VolumeId) + lifecycle.Lock() + defer lifecycle.Unlock() + // A copy holding the lock may have installed the index meanwhile: skip + // the fetch, but its shards still need the mount. + fetched := false + current := vs.store.CollectEcVolumesMissingIndex(m.VolumeId) + if i := slices.IndexFunc(current, func(x storage.EcVolumeMissingIndex) bool { return x.VolumeId == m.VolumeId && x.Collection == m.Collection }); i >= 0 { + if len(peers) == 0 { + glog.Warningf("ec volume %d: shards present locally but .ecx missing and no peer holds it; leaving shards unloaded", m.VolumeId) + return false + } + if !vs.fetchEcIndexFromPeers(peers, current[i]) { + return false + } + fetched = true + } + vs.store.MountRecoveredEcShards(m.VolumeId) + return fetched +} + // lookupEcVolumePeers asks the master which servers hold shards for vid and // returns the unique peer addresses, excluding this server itself. func (vs *VolumeServer) lookupEcVolumePeers(master pb.ServerAddress, vid uint32, self pb.ServerAddress) ([]pb.ServerAddress, error) { diff --git a/weed/storage/store.go b/weed/storage/store.go index 407b6d009d..d46b724fb6 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -102,7 +102,10 @@ type Store struct { NewEcShardsChan chan *master_pb.VolumeEcShardInformationMessage DeletedEcShardsChan chan *master_pb.VolumeEcShardInformationMessage isStopping atomic.Bool - volumeReport volumeReportState + // ecLifecycleLocks holds a per-vid *sync.Mutex serializing EC ops that + // install or remove a vid's files. Entries are never reaped. + ecLifecycleLocks sync.Map + volumeReport volumeReportState // One heartbeat at a time: the report state is marked in place as the scan // runs, so two overlapping scans would each forget what the other marked // and name every volume it holds as departed. diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 6dd65bd865..d91671b012 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -253,6 +253,14 @@ func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat { } func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, sourceDiskType string) error { + // Serialize against full teardown and the other canonical-EC-file writers for + // this vid (which all hold the per-vid lifecycle lock) so a fresh generation + // cannot be installed while a teardown is path-wiping this vid's files — otherwise + // the identity-blind wipe could clobber the freshly mounted shards. + lifecycle := s.EcLifecycleLock(vid) + lifecycle.Lock() + defer lifecycle.Unlock() + ownersBefore := s.EcShardOwnerDisks(vid, []erasure_coding.ShardId{shardId}) ecVolume, diskId, err := s.LoadEcShardAcrossDisks(collection, vid, shardId) if err != nil { @@ -426,6 +434,36 @@ func (s *Store) UnmountEcShards(vid needle.VolumeId, shardId erasure_coding.Shar return nil } +// UnmountEcShardOnDisk unloads one shard from the disk at diskId only and +// reports its removal; copies of it on other disks stay mounted. +func (s *Store) UnmountEcShardOnDisk(diskId int, vid needle.VolumeId, shardId erasure_coding.ShardId) bool { + location := s.Locations[diskId] + ecShard, found := location.FindEcShard(vid, shardId) + if !found { + return false + } + var encodeTsNs int64 + if ecVolume, ok := location.FindEcVolume(vid); ok { + encodeTsNs = ecVolume.EncodeTsNs + } + if !location.UnloadEcShard(vid, shardId) { + return false + } + si := erasure_coding.NewShardsInfo() + si.Set(erasure_coding.NewShardInfo(shardId, 0)) + s.DeletedEcShardsChan <- &master_pb.VolumeEcShardInformationMessage{ + Id: uint32(vid), + Collection: ecShard.Collection, + EcIndexBits: si.Bitmap(), + ShardSizes: si.SizesInt64(), + DiskType: string(ecShard.DiskType), + DiskId: uint32(diskId), + EncodeTsNs: encodeTsNs, + } + glog.V(0).Infof("UnmountEcShards %d.%d disk_id:%d", vid, shardId, diskId) + return true +} + func (s *Store) findEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (diskId uint32, shard *erasure_coding.EcVolumeShard, found bool) { for diskId, location := range s.Locations { if v, found := location.FindEcShard(vid, shardId); found { @@ -1538,3 +1576,9 @@ func (s *Store) EcVolumes() (ecVolumes []*erasure_coding.EcVolume) { }) return ecVolumes } + +// EcLifecycleLock returns the per-vid EC lifecycle mutex, creating it on first use. +func (s *Store) EcLifecycleLock(vid needle.VolumeId) *sync.Mutex { + actual, _ := s.ecLifecycleLocks.LoadOrStore(vid, &sync.Mutex{}) + return actual.(*sync.Mutex) +} diff --git a/weed/storage/store_ec_mirror.go b/weed/storage/store_ec_mirror.go index 01b8cc6cbb..49269d54e3 100644 --- a/weed/storage/store_ec_mirror.go +++ b/weed/storage/store_ec_mirror.go @@ -20,18 +20,18 @@ var ecMirroredSidecars = []string{".ecx", ".ecj", ".vif"} // of reaching across to a sibling. Runs before // reconcileEcShardsAcrossDisks; the cross-disk virtual mount stays // as the fallback when mirroring fails. -func (s *Store) mirrorEcMetadataToShardDisks() { +func (s *Store) mirrorEcMetadataToShardDisks(only ...needle.VolumeId) { if len(s.Locations) < 2 { return } - ecxOwners := s.indexEcxOwners() + ecxOwners := s.indexEcxOwners(only...) if len(ecxOwners) == 0 { return } for _, loc := range s.Locations { - orphans := loc.collectOrphanEcShards() + orphans := loc.collectOrphanEcShards(only...) if len(orphans) == 0 { continue } diff --git a/weed/storage/store_ec_reconcile.go b/weed/storage/store_ec_reconcile.go index c85778470c..4f8323ebd2 100644 --- a/weed/storage/store_ec_reconcile.go +++ b/weed/storage/store_ec_reconcile.go @@ -57,18 +57,18 @@ type ecxOwnerInfo struct { // ecVolumes map (so heartbeat reporting carries the right DiskId per // shard), but point the EcVolume at the sibling disk's index files so it // can serve reads and route deletes through a real .ecx / .ecj. -func (s *Store) reconcileEcShardsAcrossDisks() { +func (s *Store) reconcileEcShardsAcrossDisks(only ...needle.VolumeId) { if len(s.Locations) < 2 { return } - ecxOwners := s.indexEcxOwners() + ecxOwners := s.indexEcxOwners(only...) if len(ecxOwners) == 0 { return } for _, loc := range s.Locations { - orphans := loc.collectOrphanEcShards() + orphans := loc.collectOrphanEcShards(only...) if len(orphans) == 0 { continue } @@ -146,7 +146,7 @@ func (s *Store) findEcxIdxDirForVolume(collection string, vid needle.VolumeId) ( // downstream loaders point NewEcVolume at the directory that really has // the file. The first owner found wins; duplicates across disks are // unusual but tolerated. -func (s *Store) indexEcxOwners() map[ecKeyForReconcile]ecxOwnerInfo { +func (s *Store) indexEcxOwners(only ...needle.VolumeId) map[ecKeyForReconcile]ecxOwnerInfo { owners := make(map[ecKeyForReconcile]ecxOwnerInfo) for _, loc := range s.Locations { seen := make(map[string]bool, 2) @@ -163,6 +163,11 @@ func (s *Store) indexEcxOwners() map[ecKeyForReconcile]ecxOwnerInfo { if !strings.HasSuffix(name, ".ecx") { return true } + base := name[:len(name)-len(".ecx")] + collection, vid, err := parseCollectionVolumeId(base) + if err != nil || !ecVolumeSelected(vid, only) { + return true + } // A 0-byte .ecx is a corrupt stub from a failed copy and // not a credible owner — skip it so the scan keeps looking // for a real index on a sibling disk. Without this, an @@ -174,11 +179,6 @@ func (s *Store) indexEcxOwners() map[ecKeyForReconcile]ecxOwnerInfo { if statErr != nil || info.Size() == 0 { return true } - base := name[:len(name)-len(".ecx")] - collection, vid, err := parseCollectionVolumeId(base) - if err != nil { - return true - } key := ecKeyForReconcile{collection: collection, vid: vid} if _, exists := owners[key]; !exists { owners[key] = ecxOwnerInfo{location: loc, idxDir: scan} @@ -399,7 +399,7 @@ func (s *Store) indexDatOwners() map[ecKeyForReconcile]datOwnerInfo { // // Zero-byte shard files are ignored — loadAllEcShards already treats them // as cleanup-worthy noise and we want the same shape here. -func (l *DiskLocation) collectOrphanEcShards() map[ecKeyForReconcile][]string { +func (l *DiskLocation) collectOrphanEcShards(only ...needle.VolumeId) map[ecKeyForReconcile][]string { orphans := make(map[ecKeyForReconcile][]string) if err := eachDirEntry(l.Directory, func(entry os.DirEntry) bool { if entry.IsDir() { @@ -410,17 +410,17 @@ func (l *DiskLocation) collectOrphanEcShards() map[ecKeyForReconcile][]string { if !re.MatchString(ext) { return true } - info, err := entry.Info() - if err != nil || info.Size() == 0 { - return true - } shardId, err := strconv.ParseInt(ext[3:], 10, 64) if err != nil || shardId < 0 || shardId > 255 { return true } base := name[:len(name)-len(ext)] collection, vid, err := parseCollectionVolumeId(base) - if err != nil { + if err != nil || !ecVolumeSelected(vid, only) { + return true + } + info, err := entry.Info() + if err != nil || info.Size() == 0 { return true } if _, loaded := l.FindEcShard(vid, erasure_coding.ShardId(shardId)); loaded { diff --git a/weed/storage/store_ec_recover.go b/weed/storage/store_ec_recover.go index 74de9d78ba..2e8130e007 100644 --- a/weed/storage/store_ec_recover.go +++ b/weed/storage/store_ec_recover.go @@ -1,6 +1,8 @@ package storage import ( + "slices" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) @@ -27,13 +29,13 @@ type EcVolumeMissingIndex struct { // // It scans on-disk shard files directly, so it surfaces volumes the master never // learned about — including those whose every holder is missing its index. -func (s *Store) CollectEcVolumesMissingIndex() []EcVolumeMissingIndex { - ecxOwners := s.indexEcxOwners() +func (s *Store) CollectEcVolumesMissingIndex(only ...needle.VolumeId) []EcVolumeMissingIndex { + ecxOwners := s.indexEcxOwners(only...) seen := make(map[ecKeyForReconcile]bool) var missing []EcVolumeMissingIndex for _, loc := range s.Locations { - for key := range loc.collectOrphanEcShards() { + for key := range loc.collectOrphanEcShards(only...) { if _, hasLocalIndex := ecxOwners[key]; hasLocalIndex { continue } @@ -58,19 +60,24 @@ func (s *Store) CollectEcVolumesMissingIndex() []EcVolumeMissingIndex { // back to the cross-disk virtual mount for any disk the mirror could not reach. // Each shard load announces itself through ecShardNotifyHandler so the master // learns about the now-registered shards. -func (s *Store) MountRecoveredEcShards() { - s.mirrorEcMetadataToShardDisks() - s.loadOrphanEcShardsWithLocalIndex() - s.reconcileEcShardsAcrossDisks() +// Given volume ids, it touches only those volumes' shards. +func (s *Store) MountRecoveredEcShards(only ...needle.VolumeId) { + s.mirrorEcMetadataToShardDisks(only...) + s.loadOrphanEcShardsWithLocalIndex(only...) + s.reconcileEcShardsAcrossDisks(only...) +} + +func ecVolumeSelected(vid needle.VolumeId, only []needle.VolumeId) bool { + return len(only) == 0 || slices.Contains(only, vid) } // loadOrphanEcShardsWithLocalIndex mounts on-disk EC shards whose .ecx index is // now present on the same disk. Unlike reconcileEcShardsAcrossDisks it does not // require a sibling disk, so a single-disk store recovers too once its index has // been fetched from a peer. -func (s *Store) loadOrphanEcShardsWithLocalIndex() { +func (s *Store) loadOrphanEcShardsWithLocalIndex(only ...needle.VolumeId) { for _, loc := range s.Locations { - orphans := loc.collectOrphanEcShards() + orphans := loc.collectOrphanEcShards(only...) for key, shards := range orphans { if !loc.HasEcxFileOnDisk(key.collection, key.vid) { continue diff --git a/weed/storage/store_ec_recover_test.go b/weed/storage/store_ec_recover_test.go index ecc0c7905e..3c514be87a 100644 --- a/weed/storage/store_ec_recover_test.go +++ b/weed/storage/store_ec_recover_test.go @@ -199,3 +199,44 @@ func TestCollectEcVolumesMissingIndex_ExcludesCrossDiskOrphan(t *testing.T) { t.Errorf("cross-disk .ecx must not be reported as a cross-server orphan") } } + +// Recovering one volume mounts only that volume's shards. Another volume's +// orphan shards, such as ones an in-flight copy just renamed in, stay +// unmounted even when an index for them is already on disk. +func TestMountRecoveredEcShards_OnlyNamedVolume(t *testing.T) { + tempDir := t.TempDir() + dir0 := filepath.Join(tempDir, "disk0") + dir1 := filepath.Join(tempDir, "disk1") + const collection = "col" + recoveredVid, otherVid := needle.VolumeId(71), needle.VolumeId(72) + const datSize int64 = 10 * 1024 * 1024 + shardSize := calculateExpectedShardSize(datSize, 10) + + store := newDrainingStore(t, []string{dir0, dir1}) + plantEcShard(t, dir0, collection, recoveredVid, 0, shardSize) + plantEcShard(t, dir1, collection, recoveredVid, 1, shardSize) + plantEcShard(t, dir0, collection, otherVid, 0, shardSize) + plantEcShard(t, dir1, collection, otherVid, 1, shardSize) + plantEcIndex(t, dir0, dir0, collection, otherVid, datSize) + m, ok := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, recoveredVid) + if !ok { + t.Fatalf("volume %d not reported as missing its index", recoveredVid) + } + plantEcIndex(t, m.IdxDir, m.DataDir, collection, recoveredVid, datSize) + + store.MountRecoveredEcShards(recoveredVid) + + for i, loc := range store.Locations { + if ev, found := loc.FindEcVolume(recoveredVid); !found { + t.Errorf("volume %d not mounted on disk%d", recoveredVid, i) + } else if _, ok := ev.FindEcVolumeShard(erasure_coding.ShardId(i)); !ok { + t.Errorf("shard %d.%d not mounted on disk%d", recoveredVid, i, i) + } + if _, found := loc.FindEcVolume(otherVid); found { + t.Errorf("volume %d mounted on disk%d by the recovery of volume %d", otherVid, i, recoveredVid) + } + } + if _, err := os.Stat(erasure_coding.EcShardFileName(collection, dir1, int(otherVid)) + ".ecx"); !os.IsNotExist(err) { + t.Errorf("volume %d index mirrored to disk1 by the recovery of volume %d: %v", otherVid, recoveredVid, err) + } +} diff --git a/weed/worker/tasks/ec_balance/detection.go b/weed/worker/tasks/ec_balance/detection.go index b29c83ee2a..56c4a9cc71 100644 --- a/weed/worker/tasks/ec_balance/detection.go +++ b/weed/worker/tasks/ec_balance/detection.go @@ -172,6 +172,7 @@ func buildBalancerTopology(topoInfo *master_pb.TopologyInfo, config *Config, all vid uint32 } volRatios := make(map[volRatioKey][2]int) + emptyIndex := emptyIndexEcVolumes(topoInfo) // Normalize the disk-type filter: "hdd" (and the default "") map to the // HardDriveType, which the topology reports under the empty-string key. Keep a @@ -288,6 +289,9 @@ func buildBalancerTopology(topoInfo *master_pb.TopologyInfo, config *Config, all if !allowedCollections.Matches(eci.Collection) { continue } + if emptyIndex[ecVolumeKey{eci.Collection, eci.Id}] { + continue + } node.AddShards(eci.Id, eci.Collection, eci.DiskId, erasure_coding.ShardBits(eci.EcIndexBits)) if d, p := ecbalancer.VolumeShardRatio(eci); d > 0 || p > 0 { volRatios[volRatioKey{eci.Collection, eci.Id}] = [2]int{d, p} @@ -307,6 +311,40 @@ func buildBalancerTopology(topoInfo *master_pb.TopologyInfo, config *Config, all return topo, nodeCount, volumeRatio } +// ecVolumeKey names an EC volume: ids repeat across collections. +type ecVolumeKey struct { + collection string + vid uint32 +} + +// emptyIndexEcVolumes are the EC volumes no holder reports a needle for, i.e. +// every .ecx is empty. A shard copy refuses a 0-byte .ecx, so a move of one +// can only fail, every detection cycle; there is no data in it to balance. +// Their shards still count toward disk occupancy. +func emptyIndexEcVolumes(topoInfo *master_pb.TopologyInfo) map[ecVolumeKey]bool { + empty := make(map[ecVolumeKey]bool) + for _, dc := range topoInfo.GetDataCenterInfos() { + for _, rack := range dc.GetRackInfos() { + for _, dn := range rack.GetDataNodeInfos() { + for _, diskInfo := range dn.GetDiskInfos() { + for _, eci := range diskInfo.GetEcShardInfos() { + k := ecVolumeKey{eci.Collection, eci.Id} + if isEmpty, seen := empty[k]; !seen || isEmpty { + empty[k] = eci.FileCount == 0 + } + } + } + } + } + } + for k, isEmpty := range empty { + if !isEmpty { + delete(empty, k) + } + } + return empty +} + // resolveECRatio returns the (dataShards, parityShards) for a collection from the // admin EC config snapshot when present, else the local default. This keeps the // enterprise-only custom-ratio plumbing out of the shared planner. diff --git a/weed/worker/tasks/ec_balance/detection_test.go b/weed/worker/tasks/ec_balance/detection_test.go index a80ec0c3e2..9a559b5ab8 100644 --- a/weed/worker/tasks/ec_balance/detection_test.go +++ b/weed/worker/tasks/ec_balance/detection_test.go @@ -24,7 +24,7 @@ func ecTopo(node1Collection string) *master_pb.TopologyInfo { Id: "node1", DiskInfos: map[string]*master_pb.DiskInfo{ "": {Type: "", MaxVolumeCount: 100, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - {Id: 100, Collection: node1Collection, DiskId: 0, EcIndexBits: 0x3FFF}, // 14 shards + {Id: 100, FileCount: 1, Collection: node1Collection, DiskId: 0, EcIndexBits: 0x3FFF}, // 14 shards }}, }, } @@ -64,7 +64,7 @@ func TestBuildBalancerTopologyGroupsByHost(t *testing.T) { mkNode := func(id string, bits uint32) *master_pb.DataNodeInfo { di := &master_pb.DiskInfo{Type: "", MaxVolumeCount: 100} if bits != 0 { - di.EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{{Id: 100, Collection: "col1", DiskId: 0, EcIndexBits: bits}} + di.EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{{Id: 100, FileCount: 1, Collection: "col1", DiskId: 0, EcIndexBits: bits}} } return &master_pb.DataNodeInfo{Id: id, DiskInfos: map[string]*master_pb.DiskInfo{"": di}} } @@ -209,13 +209,13 @@ func TestDetectionKeepsGlobalMovesUnderTheCap(t *testing.T) { // server (global moves). var loaded []*master_pb.VolumeEcShardInformationMessage for vid := uint32(1); vid <= 20; vid++ { - loaded = append(loaded, &master_pb.VolumeEcShardInformationMessage{Id: vid, Collection: "col1", EcIndexBits: 1}) + loaded = append(loaded, &master_pb.VolumeEcShardInformationMessage{Id: vid, FileCount: 1, Collection: "col1", EcIndexBits: 1}) } topoInfo := &master_pb.TopologyInfo{ DataCenterInfos: []*master_pb.DataCenterInfo{{ Id: "dc1", RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ - {Id: "10.0.0.1:8080", DiskInfos: disk(&master_pb.VolumeEcShardInformationMessage{Id: 100, Collection: "col1", EcIndexBits: 0x3FFF})}, + {Id: "10.0.0.1:8080", DiskInfos: disk(&master_pb.VolumeEcShardInformationMessage{Id: 100, FileCount: 1, Collection: "col1", EcIndexBits: 0x3FFF})}, {Id: "10.0.0.2:8080", DiskInfos: disk(loaded...)}, {Id: "10.0.0.3:8080", DiskInfos: disk()}, {Id: "10.0.0.4:8080", DiskInfos: disk()}, @@ -262,7 +262,7 @@ func TestDetectionUsesConfiguredGlobalCap(t *testing.T) { } var loaded []*master_pb.VolumeEcShardInformationMessage for vid := uint32(1); vid <= 40; vid++ { - loaded = append(loaded, &master_pb.VolumeEcShardInformationMessage{Id: vid, Collection: "col1", EcIndexBits: 1}) + loaded = append(loaded, &master_pb.VolumeEcShardInformationMessage{Id: vid, FileCount: 1, Collection: "col1", EcIndexBits: 1}) } topoInfo := &master_pb.TopologyInfo{ DataCenterInfos: []*master_pb.DataCenterInfo{{ diff --git a/weed/worker/tasks/ec_balance/multidisk_detection_test.go b/weed/worker/tasks/ec_balance/multidisk_detection_test.go index 37b70f6d74..a5736ad149 100644 --- a/weed/worker/tasks/ec_balance/multidisk_detection_test.go +++ b/weed/worker/tasks/ec_balance/multidisk_detection_test.go @@ -50,6 +50,7 @@ func buildMasterTopology(collection string, vid uint32, maxVolPerDisk int, specs ecShards = append(ecShards, &master_pb.VolumeEcShardInformationMessage{ Id: vid, Collection: collection, + FileCount: 1, DiskId: diskID, EcIndexBits: uint32(bits), }) @@ -354,7 +355,7 @@ func TestBuildBalancerTopologyKeepsSlotRemainder(t *testing.T) { MaxVolumeCount: 10, VolumeCount: 1, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - {Id: 100, Collection: "col1", DiskId: 0, EcIndexBits: all}, + {Id: 100, FileCount: 1, Collection: "col1", DiskId: 0, EcIndexBits: all}, }, VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 90000, DiskId: 0}}, }}, @@ -366,8 +367,8 @@ func TestBuildBalancerTopologyKeepsSlotRemainder(t *testing.T) { MaxVolumeCount: 7, VolumeCount: 6, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ - {Id: 200, Collection: "other", DiskId: 0, EcIndexBits: 0x1F}, - {Id: 200, Collection: "other", DiskId: 1, EcIndexBits: 0x1E0}, + {Id: 200, FileCount: 1, Collection: "other", DiskId: 0, EcIndexBits: 0x1F}, + {Id: 200, FileCount: 1, Collection: "other", DiskId: 1, EcIndexBits: 0x1E0}, }, VolumeInfos: []*master_pb.VolumeInformationMessage{ {Id: 90001, DiskId: 0}, {Id: 90002, DiskId: 0}, {Id: 90003, DiskId: 0}, @@ -400,3 +401,42 @@ func TestBuildBalancerTopologyKeepsSlotRemainder(t *testing.T) { t.Errorf("volume 100 moves to n2 = %d, want 1 (the node's one free slot divided across two disks must not floor to zero)", toN2) } } + +// TestEmptyIndexVolumeIsNotMoved: a concentrated volume whose holders all +// report an empty .ecx gets no moves, since every shard copy of it fails; one +// holder with needles is enough to keep it balanced. +func TestEmptyIndexVolumeIsNotMoved(t *testing.T) { + cfg := NewDefaultConfig() + cfg.ImbalanceThreshold = 0.0 + specs := []nodeSpec{ + {id: "n1", rack: "rack1", disks: map[uint32][]int{0: {0, 1, 2}, 1: {3, 4, 5}, 2: {6, 7}, 3: {8, 9}, 4: {10, 11}, 5: {12}}}, + {id: "n2", rack: "rack2", disks: map[uint32][]int{0: {13}}}, + {id: "n3", rack: "rack3", disks: map[uint32][]int{}}, + } + withFileCounts := func(fileCount func(node string) uint64) *master_pb.TopologyInfo { + topo := buildMasterTopology("col1", 28, 50, specs) + for _, rack := range topo.DataCenterInfos[0].RackInfos { + for _, dn := range rack.DataNodeInfos { + for _, info := range dn.DiskInfos[""].EcShardInfos { + info.FileCount = fileCount(dn.Id) + } + } + } + return topo + } + + empty := withFileCounts(func(string) uint64 { return 0 }) + if results := runDetection(t, empty, cfg); len(results) != 0 { + t.Errorf("moved %d shards of a volume whose every .ecx is empty", len(results)) + } + + partly := withFileCounts(func(node string) uint64 { + if node == "n2" { + return 3 + } + return 0 + }) + if len(runDetection(t, partly, cfg)) == 0 { + t.Error("expected moves for a volume one of whose holders reports needles") + } +}