diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index a716115b41..23eb10d2db 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -1276,7 +1276,7 @@ impl VolumeServer for VolumeGrpcService { Some(vol) if vol.is_no_write_or_delete() => { Err(crate::storage::volume::VolumeError::ReadOnly(file_id.volume_id)) } - Some(vol) => vol.delete_needle(&mut n), + Some(vol) => vol.delete_needle(&mut n, false), }; match delete_result { Ok(size) => { @@ -9260,6 +9260,7 @@ mod tests { id: NeedleId(7), ..Needle::default() }, + false, ) .unwrap(); } @@ -9685,7 +9686,9 @@ mod tests { cookie: Cookie(1), ..Needle::default() }; - store.delete_volume_needle(VolumeId(1), &mut n).unwrap(); + store + .delete_volume_needle(VolumeId(1), &mut n, false) + .unwrap(); } fn read_surviving_needle( @@ -14158,7 +14161,9 @@ mod tests { cookie: Cookie(0x3344), ..Needle::default() }; - store.delete_volume_needle(VolumeId(1), &mut n).unwrap(); + store + .delete_volume_needle(VolumeId(1), &mut n, false) + .unwrap(); } let err = needle_status(&service, 11).await.unwrap_err(); assert_eq!(err.code(), tonic::Code::Unknown, "{err}"); @@ -15281,11 +15286,14 @@ mod tests { { let store = service.state.store.read().unwrap(); let (_, vol) = store.find_volume(VolumeId(1)).unwrap(); - vol.delete_needle(&mut Needle { - id: NeedleId(11), - cookie: Cookie(0x3344), - ..Needle::default() - }) + vol.delete_needle( + &mut Needle { + id: NeedleId(11), + cookie: Cookie(0x3344), + ..Needle::default() + }, + false, + ) .unwrap(); vol.sync_to_disk().unwrap(); } @@ -15344,11 +15352,14 @@ mod tests { { let store = service.state.store.read().unwrap(); let (_, vol) = store.find_volume(VolumeId(1)).unwrap(); - vol.delete_needle(&mut Needle { - id: NeedleId(11), - cookie: Cookie(0x3344), - ..Needle::default() - }) + vol.delete_needle( + &mut Needle { + id: NeedleId(11), + cookie: Cookie(0x3344), + ..Needle::default() + }, + false, + ) .unwrap(); vol.sync_to_disk().unwrap(); } diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index fad351c545..73af9c0058 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -3680,11 +3680,16 @@ pub async fn delete_handler( // Read needle first to validate cookie (matching Go behavior) let original_cookie = cookie; - let is_replicate = serde_urlencoded::from_str::>(&del_query) - .unwrap_or_default() - .into_iter() - .find(|(name, _)| name == "type") - .is_some_and(|(_, value)| value == "replicate"); + let del_fields = + serde_urlencoded::from_str::>(&del_query).unwrap_or_default(); + let del_field = |key: &str| { + del_fields + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.as_str()) + }; + let is_replicate = del_field("type") == Some("replicate"); + let fsync = del_field("fsync") == Some("true"); { let vol = { let store = state.store.read().unwrap(); @@ -3786,7 +3791,16 @@ pub async fn delete_handler( Ok(vol) if vol.is_no_write_or_delete() => { Err(crate::storage::volume::VolumeError::ReadOnly(vid)) } - Ok(vol) => vol.delete_needle(&mut n), + Ok(vol) if fsync => { + tokio::task::spawn_blocking(move || vol.delete_needle(&mut n, true)) + .await + .unwrap_or_else(|e| { + Err(crate::storage::volume::VolumeError::Io( + std::io::Error::other(e), + )) + }) + } + Ok(vol) => vol.delete_needle(&mut n, false), } }; @@ -8760,7 +8774,7 @@ mod tests { .store .write() .unwrap() - .delete_volume_needle(VolumeId(1), &mut filler) + .delete_volume_needle(VolumeId(1), &mut filler, false) .unwrap(); let data_offset = || { let mut plan = crate::storage::store::Store::needle_read_plan( diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index fde319dd1d..4f9d32c793 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -1042,13 +1042,18 @@ impl Store { } /// Delete a needle from a volume. - pub fn delete_volume_needle(&self, vid: VolumeId, n: &mut Needle) -> Result { + pub fn delete_volume_needle( + &self, + vid: VolumeId, + n: &mut Needle, + fsync: bool, + ) -> Result { // Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding. let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?; if vol.is_no_write_or_delete() { return Err(VolumeError::ReadOnly(vid)); } - vol.delete_needle(n) + vol.delete_needle(n, fsync) } // ---- Collection operations ---- @@ -2923,7 +2928,9 @@ mod tests { cookie: Cookie(0xaa), ..Needle::default() }; - let deleted = store.delete_volume_needle(VolumeId(1), &mut del_n).unwrap(); + let deleted = store + .delete_volume_needle(VolumeId(1), &mut del_n, false) + .unwrap(); assert!(deleted.0 > 0); } @@ -3400,7 +3407,7 @@ mod tests { cookie: Cookie(2), ..Needle::default() }; - store.delete_volume_needle(vid, &mut del).unwrap(); + store.delete_volume_needle(vid, &mut del, false).unwrap(); let size_before = store.find_volume(vid).unwrap().1.dat_file_size().unwrap(); @@ -3724,7 +3731,7 @@ mod tests { cookie: Cookie(2), ..Needle::default() }; - store.delete_volume_needle(vid, &mut del).unwrap(); + store.delete_volume_needle(vid, &mut del, false).unwrap(); let revision = { let (_, v) = store.find_volume(vid).unwrap(); v.super_block().compaction_revision @@ -3749,7 +3756,7 @@ mod tests { cookie: Cookie(4), ..Needle::default() }; - store.delete_volume_needle(vid, &mut del).unwrap(); + store.delete_volume_needle(vid, &mut del, false).unwrap(); drop(release_tx); copy.join().unwrap().unwrap(); diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index df565c3cfc..0c9b1d1dab 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -1621,6 +1621,8 @@ pub struct Volume { dat_syncs_for_test: std::sync::atomic::AtomicUsize, #[cfg(test)] idx_syncs_for_test: std::sync::atomic::AtomicUsize, + #[cfg(test)] + on_flush_dat_for_test: Mutex>>, needle_map_kind: NeedleMapKind, data_file_access_control: Arc, /// Serializes a data-file mutation with the sync that makes it durable: @@ -1702,6 +1704,12 @@ pub struct PendingWrite { flag: Arc, } +/// A write cancelled before it appends is refused: its delete may already be +/// durable, and an unsynced row could outlive its tombstone. +fn is_cancelled_write(pending: Option<&PendingWrite>) -> bool { + pending.is_some_and(|p| p.flag.cancelled.load(Ordering::Acquire)) +} + impl Drop for PendingWrite { fn drop(&mut self) { let mut by_id = self.writes.0.lock().unwrap(); @@ -1785,6 +1793,8 @@ impl Volume { dat_syncs_for_test: Default::default(), #[cfg(test)] idx_syncs_for_test: Default::default(), + #[cfg(test)] + on_flush_dat_for_test: Default::default(), needle_map_kind, data_file_access_control: Arc::new(DataFileAccessControl::default()), data_file_sync_lock: std::sync::Mutex::new(()), @@ -1844,6 +1854,8 @@ impl Volume { dat_syncs_for_test: Default::default(), #[cfg(test)] idx_syncs_for_test: Default::default(), + #[cfg(test)] + on_flush_dat_for_test: Default::default(), needle_map_kind: NeedleMapKind::InMemory, data_file_access_control: Arc::new(DataFileAccessControl::default()), data_file_sync_lock: std::sync::Mutex::new(()), @@ -3062,6 +3074,9 @@ impl Volume { let _sync_guard = self.data_file_sync_lock.lock().unwrap(); let _guard = self.data_file_access_control.write_lock(); let mut io = self.io.write().unwrap(); + if is_cancelled_write(pending) { + return Err(VolumeError::Deleted); + } if let Err(refused) = self.check_writable() { return Ok(Err(refused)); } @@ -3112,11 +3127,14 @@ impl Volume { let _guard = self.data_file_access_control.write_lock(); let mut io = self.io.write().unwrap(); for (i, (n, _)) in run.iter_mut().enumerate() { - let written = self - .check_writable() - .and_then(|()| self.do_write_request(&mut io, n, true, false)); + let pending = tracked.get(i).and_then(Option::as_ref); + let written = self.check_writable().and_then(|()| { + if is_cancelled_write(pending) { + return Err(VolumeError::Deleted); + } + self.do_write_request(&mut io, n, true, false) + }); results.push(written.and_then(|written| { - let pending = tracked.get(i).and_then(Option::as_ref); self.drop_cancelled_write(&mut io, pending, n, &written) .map(|()| written) })); @@ -3147,8 +3165,12 @@ impl Volume { let _guard = self.data_file_access_control.write_lock(); let mut io = self.io.write().unwrap(); last_append_at_ns = io.last_append_at_ns; - for (n, fsync) in run.iter_mut() { - let r = self.append_unpublished(&mut io, n, &mut last_append_at_ns); + for (i, (n, fsync)) in run.iter_mut().enumerate() { + let r = if is_cancelled_write(pending.get(i).and_then(Option::as_ref)) { + Err(VolumeError::Deleted) + } else { + self.append_unpublished(&mut io, n, &mut last_append_at_ns) + }; marks.push(self.io_errors.mark()); if let Ok(Some(offset)) = r { run_start.get_or_insert(offset); @@ -3295,6 +3317,13 @@ impl Volume { #[cfg(test)] self.dat_syncs_for_test.fetch_add(1, Ordering::Relaxed); #[cfg(test)] + { + let hook = self.on_flush_dat_for_test.lock().unwrap().take(); + if let Some(hook) = hook { + hook(self); + } + } + #[cfg(test)] if io.fail_fsync_for_test { return Err(io::Error::other("injected fsync failure")); } @@ -3805,28 +3834,85 @@ impl Volume { // ---- Delete ---- - /// Delete a needle from the volume. - pub fn delete_needle(&self, n: &mut Needle) -> Result { + /// Delete a needle from the volume. With `fsync` the delete is only + /// acknowledged once its tombstone and .idx row are on disk, in the same + /// order as a durable write; the write guards are dropped for each flush + /// as in write_synced_run, so reads do not wait on the fsync. + pub fn delete_needle(&self, n: &mut Needle, fsync: bool) -> Result { self.cancel_pending_writes(n.id, n.cookie); let _sync_guard = self.data_file_sync_lock.lock().unwrap(); + let staged = { + let _guard = self.data_file_access_control.write_lock(); + let mut io = self.io.write().unwrap(); + if self.is_closed() { + // A detached handle has no index; without this check the tombstone + // would silently no-op as Ok(0) while the needle lives on the + // remounted volume. + return Err(VolumeError::NotFound); + } + if let Some(e) = self.unavailable_error() { + return Err(e); + } + if self.no_write_flags().0 { + return Err(VolumeError::ReadOnly(self.id)); + } + if !fsync { + return self.do_delete_request(&mut io, n); + } + self.append_delete_request(&mut io, n)? + }; + // An already-deleted needle still syncs: an earlier delete may have + // skipped it. + if !self.has_remote_file() { + let io = self.io.read().unwrap(); + if let Err(e) = self.flush_dat(&io) { + self.check_read_write_error(Some(&e)); + if let Some((offset, _)) = staged { + self.undo_unsynced_append(&io, offset); + } + return Err(VolumeError::Io(e)); + } + } + if let Some((offset, _)) = staged { + let _guard = self.data_file_access_control.write_lock(); + let mut io = self.io.write().unwrap(); + self.publish_delete_request(&mut io, n, offset)?; + } + self.flush_idx(&self.io.read().unwrap())?; + let Some((_, size)) = staged else { + return Ok(Size(0)); + }; let _guard = self.data_file_access_control.write_lock(); let mut io = self.io.write().unwrap(); - if self.is_closed() { - // A detached handle has no index; without this check the tombstone - // would silently no-op as Ok(0) while the needle lives on the - // remounted volume. - return Err(VolumeError::NotFound); + if self.maybe_checkpoint_index(&mut io, true) { + self.check_read_write_error(None); } - if let Some(e) = self.unavailable_error() { - return Err(e); - } - if self.no_write_flags().0 { - return Err(VolumeError::ReadOnly(self.id)); - } - self.do_delete_request(&mut io, n) + Ok(size) } fn do_delete_request(&self, io: &mut VolumeIo, n: &mut Needle) -> Result { + let Some((offset, size)) = self.append_delete_request(io, n)? else { + return Ok(Size(0)); + }; + self.publish_delete_request(io, n, offset)?; + let checkpoint_ok = self.maybe_checkpoint_index(io, false); + + // Clear the EIO streak after a successful delete (tombstone append + + // index update + checkpoint), mirroring do_write_request. + if checkpoint_ok { + self.check_read_write_error(None); + } + + Ok(size) + } + + /// do_delete_request through the tombstone append, leaving the index + /// update to publish_delete_request. None when there is nothing to delete. + fn append_delete_request( + &self, + io: &mut VolumeIo, + n: &mut Needle, + ) -> Result, VolumeError> { let (found, size, _stored_offset) = if let Some(nm) = &io.nm { // Propagate: acknowledging the delete as a no-op because the index // could not be read would drop the tombstone on the floor while the @@ -3841,11 +3927,11 @@ impl Volume { (false, Size(0), Offset::default()) } } else { - return Ok(Size(0)); + return Ok(None); }; if !found { - return Ok(Size(0)); + return Ok(None); } // Write tombstone: append needle with empty data @@ -3860,26 +3946,38 @@ impl Volume { // Remote-tiered volume: skip .dat append, use offset 0 0 }; + Ok(Some((offset, size))) + } + + fn publish_delete_request( + &self, + io: &mut VolumeIo, + n: &Needle, + offset: u64, + ) -> Result<(), VolumeError> { io.last_append_at_ns = n.append_at_ns; // Update index - if let Some(nm) = &mut io.nm { - nm.delete(n.id, Offset::from_actual_offset(offset as i64))?; + if let Some(nm) = &mut io.nm + && let Err(e) = nm.delete(n.id, Offset::from_actual_offset(offset as i64)) + { + // The map already dropped the needle, so a retry, durable or not, + // would ack without the missing .idx row. Stop taking writes. + self.set_no_write_flag(NO_WRITE_OR_DELETE, true); + tracing::error!( + "volume {}: failed to index a delete at {}, marking read only: {}", + self.id.0, + offset, + e + ); + return Err(VolumeError::Io(e)); } if n.id == io.last_write_needle_key { io.last_write_append_at_ns = 0; io.last_write_needle_key = NeedleId(0); io.last_write_deleted = true; } - let checkpoint_ok = self.maybe_checkpoint_index(io, false); - - // Clear the EIO streak after a successful delete (tombstone append + - // index update + checkpoint), mirroring do_write_request. - if checkpoint_ok { - self.check_read_write_error(None); - } - - Ok(size) + Ok(()) } // ---- Metrics ---- @@ -7244,11 +7342,14 @@ mod tests { let (_offset, is_last) = v.binary_search_by_append_at_ns(newest_ns).unwrap(); assert!(is_last, "precondition: the caller starts caught up"); - v.delete_needle(&mut Needle { - id: NeedleId(2), - cookie: Cookie(0x12345678), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(2), + cookie: Cookie(0x12345678), + ..Needle::default() + }, + false, + ) .unwrap(); let (offset, is_last) = v.binary_search_by_append_at_ns(newest_ns).unwrap(); @@ -7495,11 +7596,14 @@ mod tests { let written: Vec = (1..=3u64) .map(|id| write_test_needle(&mut v, id, b"scan")) .collect(); - v.delete_needle(&mut Needle { - id: NeedleId(2), - cookie: Cookie(0x12345678), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(2), + cookie: Cookie(0x12345678), + ..Needle::default() + }, + false, + ) .unwrap(); let sb_size = v.super_block().block_size() as u64; @@ -8272,6 +8376,115 @@ mod tests { std::fs::metadata(v.file_name(".dat")).unwrap().len() } + #[test] + fn test_durable_delete_syncs_tombstone() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut writes: Vec<_> = (1..=2u64) + .map(|id| (batch_needle(id, 0xaa, b"payload"), false)) + .collect(); + assert!( + v.write_needles_grouped(&mut writes) + .iter() + .all(|r| r.is_ok()) + ); + assert_eq!(v.sync_counts_for_test(), (0, 0)); + + v.delete_needle(&mut batch_needle(1, 0xaa, b""), false) + .unwrap(); + assert_eq!(v.sync_counts_for_test(), (0, 0)); + + let size = v + .delete_needle(&mut batch_needle(2, 0xaa, b""), true) + .unwrap(); + assert!(size.0 > 0); + assert_eq!(v.sync_counts_for_test(), (1, 1)); + } + + #[test] + fn test_delete_needle_failed_fsync_keeps_needle() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut writes = vec![(batch_needle(1, 0xaa, b"payload"), false)]; + assert!( + v.write_needles_grouped(&mut writes) + .iter() + .all(|r| r.is_ok()) + ); + + v.fail_next_fsync_for_test(true); + assert!( + v.delete_needle(&mut batch_needle(1, 0xaa, b""), true) + .is_err() + ); + v.fail_next_fsync_for_test(false); + + let mut n = Needle { + id: NeedleId(1), + ..Needle::default() + }; + v.read_needle(&mut n).unwrap(); + assert_eq!(n.data, b"payload"); + } + + #[test] + fn test_durable_delete_syncs_after_unsynced_delete() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut writes = vec![(batch_needle(1, 0xaa, b"payload"), false)]; + assert!( + v.write_needles_grouped(&mut writes) + .iter() + .all(|r| r.is_ok()) + ); + v.delete_needle(&mut batch_needle(1, 0xaa, b""), false) + .unwrap(); + assert_eq!(v.sync_counts_for_test(), (0, 0)); + + let size = v + .delete_needle(&mut batch_needle(1, 0xaa, b""), true) + .unwrap(); + assert_eq!(size.0, 0); + assert_eq!(v.sync_counts_for_test(), (1, 1)); + } + + /// A durable delete flushes with the write guards dropped, so a read of + /// the volume does not wait on its fsync. + #[test] + fn test_durable_delete_reads_during_flush() { + let tmp = TempDir::new().unwrap(); + let v = Arc::new(make_test_volume(tmp.path().to_str().unwrap())); + let mut writes: Vec<_> = (1..=2u64) + .map(|id| (batch_needle(id, 0xaa, b"payload"), false)) + .collect(); + assert!( + v.write_needles_grouped(&mut writes) + .iter() + .all(|r| r.is_ok()) + ); + + let read_during_flush = Arc::new(AtomicBool::new(false)); + let (reader, flag) = (v.clone(), read_during_flush.clone()); + *v.on_flush_dat_for_test.lock().unwrap() = Some(Box::new(move |_: &Volume| { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut n = batch_needle(2, 0xaa, b""); + let _ = tx.send(reader.read_needle(&mut n).is_ok()); + }); + let read = rx.recv_timeout(std::time::Duration::from_secs(5)); + flag.store(read == Ok(true), Ordering::Release); + })); + v.delete_needle(&mut batch_needle(1, 0xaa, b""), true) + .unwrap(); + assert!(read_during_flush.load(Ordering::Acquire)); + } + /// A batch of durable writes shares one .dat sync and one .idx sync, /// where writing them one at a time syncs both files per needle. #[test] @@ -8729,6 +8942,77 @@ mod tests { assert_eq!(size, idx_len + 8); } + /// A delete whose .idx row fails has already dropped the needle + /// from the map, so a retry would find nothing to delete and ack. The + /// volume stops taking writes instead: nothing is appended past the + /// tombstone, so a reload re-indexes it from the .dat tail rather than + /// resurrect the needle. + #[test] + fn test_durable_delete_index_failure_refuses_writes() { + for first_fsync in [true, false] { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + v.write_needle(&mut batch_needle(1, 0xaa, b"payload"), true, true) + .unwrap(); + + { + let mut io = v.io.write().unwrap(); + let nm = io.nm.as_mut().unwrap(); + let idx_len = nm.index_file_size(); + let file = OpenOptions::new() + .write(true) + .open(format!("{dir}/1.idx")) + .unwrap(); + nm.set_idx_file( + Box::new(TornIdxWriter { + file, + writes: 0, + tear_at: 1, + fail_truncates: false, + }), + idx_len, + ); + } + + assert!( + v.delete_needle(&mut batch_needle(1, 0xaa, b""), first_fsync) + .is_err() + ); + assert!( + v.write_needle(&mut batch_needle(2, 0xbb, b"later"), true, true) + .is_err(), + "nothing may be appended past the unindexed tombstone" + ); + assert!( + v.delete_needle(&mut batch_needle(1, 0xaa, b""), true) + .is_err(), + "a retry must not ack a delete whose .idx row never landed" + ); + drop(v); + + let reopened = match Volume::new( + dir, + dir, + VolumeId(1), + NeedleMapKind::InMemory, + &VolumeSpec::default(), + ) { + Ok(v) => v, + Err(e) => panic!("the volume does not reload: {e}"), + }; + let mut got = Needle { + id: NeedleId(1), + ..Needle::default() + }; + let read = reopened.read_needle(&mut got); + assert!( + !(read.is_ok() && got.data == b"payload"), + "the deleted needle must not come back on reload" + ); + } + } + /// The I/O error streak after writing four durable needles, the ones in /// `failing` with a media error on their append, first one at a time and /// then as one grouped run. Also returns which grouped writes landed. @@ -8882,11 +9166,14 @@ mod tests { assert_eq!(v.file_count(), 1); let deleted_size = v - .delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(0xbb), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(0xbb), + ..Needle::default() + }, + false, + ) .unwrap(); assert!(deleted_size.0 > 0); // Additive-only: file_count stays at 1 after delete @@ -8934,12 +9221,12 @@ mod tests { let n = pending_needle(1, b"late write"); let pending = v.track_pending_write(&n); - v.delete_needle(&mut pending_needle(1, b"")).unwrap(); + v.delete_needle(&mut pending_needle(1, b""), false).unwrap(); let err = write_pending(&v, n, fsync, pending).unwrap_err(); assert!(matches!(err, VolumeError::Deleted)); let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err(); - assert!(matches!(err, VolumeError::Deleted)); + assert!(matches!(err, VolumeError::NotFound)); assert!(v.pending_writes.0.lock().unwrap().is_empty()); v.close(); @@ -8955,7 +9242,7 @@ mod tests { } } - /// The unsynced write path takes a cancelled write back the same way. + /// The unsynced write path refuses a cancelled write the same way. #[test] fn test_unsynced_write_cancelled_by_delete() { let tmp = TempDir::new().unwrap(); @@ -8969,7 +9256,7 @@ mod tests { .unwrap_err(); assert!(matches!(err, VolumeError::Deleted)); let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err(); - assert!(matches!(err, VolumeError::Deleted)); + assert!(matches!(err, VolumeError::NotFound)); } /// A delete that finds a copy also cancels a retry of the same write that @@ -8984,13 +9271,38 @@ mod tests { .unwrap(); let rewrite = pending_needle(1, b"retry"); let pending = v.track_pending_write(&rewrite); - v.delete_needle(&mut pending_needle(1, b"")).unwrap(); + v.delete_needle(&mut pending_needle(1, b""), false).unwrap(); let err = write_pending(&v, rewrite, false, pending).unwrap_err(); assert!(matches!(err, VolumeError::Deleted)); let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err(); assert!(matches!(err, VolumeError::Deleted)); } + /// A rewrite cancelled before it takes the volume locks appends nothing, + /// so no unsynced row of it can outlive a durable delete after a crash. + #[test] + fn test_cancelled_rewrite_does_not_append() { + let tmp = TempDir::new().unwrap(); + let v = make_test_volume(tmp.path().to_str().unwrap()); + + v.write_needle(&mut pending_needle(1, b"first"), true, false) + .unwrap(); + let before = std::fs::metadata(v.file_name(".dat")).unwrap().len(); + let rewrite = pending_needle(1, b"retry"); + let pending = v.track_pending_write(&rewrite); + v.cancel_pending_writes(rewrite.id, rewrite.cookie); + let err = write_pending(&v, rewrite, false, pending).unwrap_err(); + assert!(matches!(err, VolumeError::Deleted)); + assert_eq!( + std::fs::metadata(v.file_name(".dat")).unwrap().len(), + before + ); + + v.delete_needle(&mut pending_needle(1, b""), true).unwrap(); + let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err(); + assert!(matches!(err, VolumeError::Deleted)); + } + /// Only writes already in flight and carrying the delete's cookie are /// cancelled. #[test] @@ -8999,7 +9311,7 @@ mod tests { let dir = tmp.path().to_str().unwrap(); let v = make_test_volume(dir); - v.delete_needle(&mut pending_needle(1, b"")).unwrap(); + v.delete_needle(&mut pending_needle(1, b""), false).unwrap(); let n = pending_needle(1, b"after the delete"); let pending = v.track_pending_write(&n); write_pending(&v, n, false, pending).unwrap(); @@ -9056,7 +9368,10 @@ mod tests { let n = pending_needle(1, b"late write"); let pending = v.track_pending_write(&n); - v.cancel_pending_writes(n.id, n.cookie); + let (id, cookie) = (n.id, n.cookie); + *v.on_flush_dat_for_test.lock().unwrap() = Some(Box::new(move |v: &Volume| { + v.cancel_pending_writes(id, cookie) + })); let err = write_pending(&v, n, true, pending).unwrap_err(); assert!(matches!(err, VolumeError::Deleted)); assert_eq!(v.sync_counts_for_test(), (2, 1)); @@ -9091,11 +9406,14 @@ mod tests { }; v.write_needle(&mut n, true, false).unwrap(); } - v.delete_needle(&mut Needle { - id: NeedleId(2), - cookie: Cookie(2), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(2), + cookie: Cookie(2), + ..Needle::default() + }, + false, + ) .unwrap(); v.sync_to_disk().unwrap(); } @@ -9149,11 +9467,14 @@ mod tests { // More than one tombstone: the scan has to walk back over the whole // run of them to reach a write. for i in [2u64, 3] { - v.delete_needle(&mut Needle { - id: NeedleId(i), - cookie: Cookie(i as u32), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(i), + cookie: Cookie(i as u32), + ..Needle::default() + }, + false, + ) .unwrap(); } v.sync_to_disk().unwrap(); @@ -9249,11 +9570,14 @@ mod tests { let (offset, _, _) = v.write_needle(&mut n, true, false).unwrap(); written.push((offset, n.size)); } - v.delete_needle(&mut Needle { - id: NeedleId(2), - cookie: Cookie(2), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(2), + cookie: Cookie(2), + ..Needle::default() + }, + false, + ) .unwrap(); v.sync_to_disk().unwrap(); for (offset, size) in written { @@ -9341,11 +9665,14 @@ mod tests { // The delete lands inside the commit window: makeup_diff replays its // tombstone into the new .idx behind the write row the copy carried. v.compact_by_index(0, 0, |_| true).unwrap(); - v.delete_needle(&mut Needle { - id: NeedleId(2), - cookie: Cookie(2), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(2), + cookie: Cookie(2), + ..Needle::default() + }, + false, + ) .unwrap(); assert!( v.io.read().unwrap().last_write_deleted, @@ -9528,11 +9855,14 @@ mod tests { ..Needle::default() }; v.write_needle(&mut n, true, false).unwrap(); - v.delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(1), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(1), + ..Needle::default() + }, + false, + ) .unwrap(); v.sync_to_disk().unwrap(); } @@ -9735,11 +10065,14 @@ mod tests { let mut v = make_test_volume(dir); write_test_needle(&mut v, 1, b"needle data"); - v.delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(0x12345678), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(0x12345678), + ..Needle::default() + }, + false, + ) .unwrap(); // A later record keeps the tombstone mid-file rather than at the tail. write_test_needle(&mut v, 2, b"needle data"); @@ -9836,11 +10169,14 @@ mod tests { let mut v = make_test_volume(dir); write_test_needle(&mut v, 1, b"needle data"); - v.delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(0x12345678), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(0x12345678), + ..Needle::default() + }, + false, + ) .unwrap(); v.sync_to_disk().unwrap(); @@ -10540,7 +10876,7 @@ mod tests { cookie: Cookie(2), ..Needle::default() }; - v.delete_needle(&mut del).unwrap(); + v.delete_needle(&mut del, false).unwrap(); // Additive-only: file_count stays at 3 after delete assert_eq!(v.file_count(), 3); assert_eq!(v.deleted_count(), 1); @@ -10966,11 +11302,14 @@ mod tests { assert!(matches!(err, VolumeError::ReadOnly(_))); let deleted_size = v - .delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(0x1234), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(0x1234), + ..Needle::default() + }, + false, + ) .unwrap(); assert!(deleted_size.0 > 0); assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload); @@ -11799,11 +12138,14 @@ mod tests { // Deletes are still allowed on a tiered volume and land in .idx. let idx_before = std::fs::metadata(&idx_path).unwrap().len(); let deleted = v - .delete_needle(&mut Needle { - id: NeedleId(3), - cookie: Cookie(0x1234), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(3), + cookie: Cookie(0x1234), + ..Needle::default() + }, + false, + ) .unwrap(); assert!(deleted.0 > 0); assert_eq!( @@ -11989,11 +12331,14 @@ mod tests { assert!(matches!(err, VolumeError::ReadOnly(_))); let deleted = v - .delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(1), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(1), + ..Needle::default() + }, + false, + ) .unwrap(); assert!(deleted.0 > 0, "deletes must still land"); v.sync_to_disk().unwrap(); @@ -12030,11 +12375,14 @@ mod tests { assert!(matches!(err, VolumeError::ReadOnly(_))); let deleted = v - .delete_needle(&mut Needle { - id: NeedleId(2), - cookie: Cookie(2), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(2), + cookie: Cookie(2), + ..Needle::default() + }, + false, + ) .unwrap(); assert!(deleted.0 > 0, "deletes must still land after restart"); @@ -12104,11 +12452,14 @@ mod tests { Some(NeedleMap::SortedFile(_)) )); for id in ids.clone() { - v.delete_needle(&mut Needle { - id: NeedleId(id), - cookie: Cookie(id as u32), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(id), + cookie: Cookie(id as u32), + ..Needle::default() + }, + false, + ) .unwrap(); } let after_deletes = counters(&v); @@ -12156,11 +12507,14 @@ mod tests { .unwrap(); assert!(v.no_write_flags().0); let err = v - .delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(1), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(1), + ..Needle::default() + }, + false, + ) .unwrap_err(); assert!( matches!(err, VolumeError::ReadOnly(_)), @@ -12175,11 +12529,14 @@ mod tests { "upgrade to canDelete must attach the .idx writer" ); let deleted = v - .delete_needle(&mut Needle { - id: NeedleId(1), - cookie: Cookie(1), - ..Needle::default() - }) + .delete_needle( + &mut Needle { + id: NeedleId(1), + cookie: Cookie(1), + ..Needle::default() + }, + false, + ) .unwrap(); assert!(deleted.0 > 0, "deletes must land once canDelete is set"); } @@ -12830,7 +13187,7 @@ mod tests { cookie: Cookie(id as u32), ..Needle::default() }; - v.delete_needle(&mut del).unwrap(); + v.delete_needle(&mut del, false).unwrap(); } let expected_live = v.file_count() - v.deleted_count(); diff --git a/seaweed-volume/src/storage/volume_idx_rebuild.rs b/seaweed-volume/src/storage/volume_idx_rebuild.rs index 00533cd5d9..a5810f8643 100644 --- a/seaweed-volume/src/storage/volume_idx_rebuild.rs +++ b/seaweed-volume/src/storage/volume_idx_rebuild.rs @@ -188,7 +188,7 @@ mod tests { for id in 1..=3 { v.write_needle(&mut needle(id), true, false).unwrap(); } - v.delete_needle(&mut needle(2)).unwrap(); + v.delete_needle(&mut needle(2), false).unwrap(); v.sync_to_disk().unwrap(); let (want_count, want_deleted) = (v.file_count(), v.deleted_count()); drop(v); diff --git a/seaweed-volume/src/storage/volume_idx_repair.rs b/seaweed-volume/src/storage/volume_idx_repair.rs index 0fd410db8a..5648fee5ad 100644 --- a/seaweed-volume/src/storage/volume_idx_repair.rs +++ b/seaweed-volume/src/storage/volume_idx_repair.rs @@ -382,10 +382,13 @@ pub(crate) mod tests { write_test_volume(dir, 6); let v = open_volume(dir); - v.delete_needle(&mut Needle { - id: NeedleId(3), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(3), + ..Needle::default() + }, + false, + ) .unwrap(); drop(v); @@ -417,10 +420,13 @@ pub(crate) mod tests { write_test_volume(dir, 8); let v = open_volume(dir); - v.delete_needle(&mut Needle { - id: NeedleId(2), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(2), + ..Needle::default() + }, + false, + ) .unwrap(); drop(v); diff --git a/seaweed-volume/src/storage/volume_tail_repair.rs b/seaweed-volume/src/storage/volume_tail_repair.rs index 178cf1981f..fa9a5fd590 100644 --- a/seaweed-volume/src/storage/volume_tail_repair.rs +++ b/seaweed-volume/src/storage/volume_tail_repair.rs @@ -505,10 +505,13 @@ mod tests { f.written.insert(*id, n.data.clone()); } Op::Del(id) => { - v.delete_needle(&mut Needle { - id: NeedleId(*id), - ..Needle::default() - }) + v.delete_needle( + &mut Needle { + id: NeedleId(*id), + ..Needle::default() + }, + false, + ) .unwrap(); } Op::Empty(id) => { diff --git a/weed/server/volume_grpc_batch_delete.go b/weed/server/volume_grpc_batch_delete.go index 6b4ab1af69..df996b38ab 100644 --- a/weed/server/volume_grpc_batch_delete.go +++ b/weed/server/volume_grpc_batch_delete.go @@ -105,7 +105,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B // the delete cancels in-flight writes carrying this cookie n.Cookie = fidCookie } - if size, err := vs.store.DeleteVolumeNeedle(volumeId, n); err != nil { + if size, err := vs.store.DeleteVolumeNeedle(volumeId, n, false); err != nil { resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{ FileId: fid, Status: http.StatusInternalServerError, diff --git a/weed/server/volume_grpc_copy_incremental_test.go b/weed/server/volume_grpc_copy_incremental_test.go index 7d5b0e65aa..52b5910a25 100644 --- a/weed/server/volume_grpc_copy_incremental_test.go +++ b/weed/server/volume_grpc_copy_incremental_test.go @@ -24,7 +24,7 @@ import ( func TestIncrementalCopyCarriesRemoteTierTombstone(t *testing.T) { store := tierUploadForDeletes(t, "000") const vid = needle.VolumeId(1) - if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 2}); err != nil { + if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 2}, false); err != nil { t.Fatalf("delete on tiered volume: %v", err) } source := store.GetVolume(vid) @@ -101,7 +101,7 @@ func TestIncrementalCopyCarriesRemoteTierTombstone(t *testing.T) { // a delete landing after the copy rides the index stream alone: the run // appends just the tombstone row and the needle stays deleted - if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 3}); err != nil { + if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 3}, false); err != nil { t.Fatalf("second delete on tiered volume: %v", err) } if err := checkVolume.IncrementalBackup(serverAddress, dialOption); err != nil { diff --git a/weed/server/volume_grpc_tier_upload_compact_test.go b/weed/server/volume_grpc_tier_upload_compact_test.go index 9295ed81e6..fe3d6daedb 100644 --- a/weed/server/volume_grpc_tier_upload_compact_test.go +++ b/weed/server/volume_grpc_tier_upload_compact_test.go @@ -57,7 +57,7 @@ func TestTierUploadRefusesADatReplacedByACompactionCommit(t *testing.T) { written[i] = n.Data } for i := uint64(1); i <= 5; i++ { - if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: types.Uint64ToNeedleId(i)}); err != nil { + if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: types.Uint64ToNeedleId(i)}, false); err != nil { t.Fatalf("delete %d: %v", i, err) } } diff --git a/weed/server/volume_grpc_tier_upload_deletes_test.go b/weed/server/volume_grpc_tier_upload_deletes_test.go index ebd73a244a..bb171c96d4 100644 --- a/weed/server/volume_grpc_tier_upload_deletes_test.go +++ b/weed/server/volume_grpc_tier_upload_deletes_test.go @@ -50,7 +50,7 @@ func tierUploadForDeletes(t *testing.T, replication string) *storage.Store { // then, leaving the needles live in the index. func TestTierUploadAcceptsDeletesOnASingleCopyVolume(t *testing.T) { store := tierUploadForDeletes(t, "000") - if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}); err != nil { + if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}, false); err != nil { t.Fatalf("delete on the freshly tiered volume: %v", err) } if _, err := store.ReadVolumeNeedle(1, &needle.Needle{Id: 2}, nil, nil); err == nil { @@ -71,7 +71,7 @@ func TestTierUploadAcceptsDeletesOnASingleCopyVolume(t *testing.T) { // written before that copy would reach one replica only. func TestTierUploadKeepsRefusingDeletesOnAReplicatedVolume(t *testing.T) { store := tierUploadForDeletes(t, "001") - if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}); err == nil { + if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}, false); err == nil { t.Fatalf("a replicated volume accepted a delete right after the tier upload") } } diff --git a/weed/storage/store.go b/weed/storage/store.go index d46b724fb6..864ea925c2 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -998,7 +998,7 @@ func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle, checkCook return } -func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (Size, error) { +func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle, fsync bool) (Size, error) { if v := s.findVolume(i); v != nil { v.noWriteLock.RLock() noWriteOrDelete := v.noWriteOrDelete @@ -1006,7 +1006,7 @@ func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (Size, e if noWriteOrDelete { return 0, fmt.Errorf("volume %d is read only", i) } - return v.deleteNeedle2(n) + return v.deleteNeedle2(n, fsync) } return 0, fmt.Errorf("volume %d not found on %s:%d", i, s.Ip, s.Port) } diff --git a/weed/storage/store_collection_metrics_test.go b/weed/storage/store_collection_metrics_test.go index e4618ca290..00e581676c 100644 --- a/weed/storage/store_collection_metrics_test.go +++ b/weed/storage/store_collection_metrics_test.go @@ -107,7 +107,7 @@ func TestCollectHeartbeatSizesOnlySurvivingVolumes(t *testing.T) { if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil { t.Fatalf("write needle: %v", err) } - if _, err := v.deleteNeedle2(n); err != nil { + if _, err := v.deleteNeedle2(n, false); err != nil { t.Fatalf("delete needle: %v", err) } } diff --git a/weed/storage/store_delete_volume_test.go b/weed/storage/store_delete_volume_test.go index 52d7aa31d5..1dc34b47f5 100644 --- a/weed/storage/store_delete_volume_test.go +++ b/weed/storage/store_delete_volume_test.go @@ -51,7 +51,7 @@ func TestDeleteVolumeOnlyGarbage(t *testing.T) { t.Fatal("refused delete removed the volume") } - if _, err := store.DeleteVolumeNeedle(5, n); err != nil { + if _, err := store.DeleteVolumeNeedle(5, n, false); err != nil { t.Fatal(err) } // The shell sends both flags so a pre-upgrade server still refuses; either diff --git a/weed/storage/store_duplicate_vid_test.go b/weed/storage/store_duplicate_vid_test.go index 740d2bab54..530e53e2b5 100644 --- a/weed/storage/store_duplicate_vid_test.go +++ b/weed/storage/store_duplicate_vid_test.go @@ -73,7 +73,7 @@ func TestDeleteVolumeGuardChecksAllCopiesBeforeDeleting(t *testing.T) { require.NoError(t, err) _, _, _, err = copy1.writeNeedle2(n, false, false, false) require.NoError(t, err) - _, err = copy0.deleteNeedle2(n) + _, err = copy0.deleteNeedle2(n, false) require.NoError(t, err) err = store.DeleteVolume(vid, false, true, false) diff --git a/weed/storage/store_mark_readonly_can_delete_test.go b/weed/storage/store_mark_readonly_can_delete_test.go index 40f536c732..e8da8da9e8 100644 --- a/weed/storage/store_mark_readonly_can_delete_test.go +++ b/weed/storage/store_mark_readonly_can_delete_test.go @@ -62,7 +62,7 @@ func TestMarkVolumeReadonlyCanDelete(t *testing.T) { _, err = store.WriteVolumeNeedle(vid, newRandomNeedle(3), true, false) require.Error(t, err, "writes must be rejected") - _, err = store.DeleteVolumeNeedle(vid, n1) + _, err = store.DeleteVolumeNeedle(vid, n1, false) require.NoError(t, err, "deletes must still land") store.Close() @@ -74,14 +74,14 @@ func TestMarkVolumeReadonlyCanDelete(t *testing.T) { _, err = store2.WriteVolumeNeedle(vid, newRandomNeedle(4), true, false) require.Error(t, err, "writes must stay rejected after restart") - _, err = store2.DeleteVolumeNeedle(vid, n2) + _, err = store2.DeleteVolumeNeedle(vid, n2, false) require.NoError(t, err, "deletes must still land after restart") // Downgrading to plain readonly clears the canDelete flag. require.NoError(t, store2.MarkVolumeReadonly(vid, false, true)) require.True(t, v2.noWriteOrDelete) require.False(t, v2.noWriteCanDelete) - _, err = store2.DeleteVolumeNeedle(vid, n2) + _, err = store2.DeleteVolumeNeedle(vid, n2, false) require.Error(t, err, "plain readonly must reject deletes") // -writable clears the state again. @@ -110,13 +110,13 @@ func TestMarkVolumeReadonlyCanDelete_AfterReadonlyBoot(t *testing.T) { v := store2.GetVolume(vid) require.NotNil(t, v) require.True(t, v.noWriteOrDelete) - _, err = store2.DeleteVolumeNeedle(vid, n) + _, err = store2.DeleteVolumeNeedle(vid, n, false) require.Error(t, err, "plain readonly must reject deletes") require.NoError(t, store2.MarkVolumeReadonly(vid, true, true)) require.False(t, v.noWriteOrDelete) require.True(t, v.noWriteCanDelete) - _, err = store2.DeleteVolumeNeedle(vid, n) + _, err = store2.DeleteVolumeNeedle(vid, n, false) require.NoError(t, err, "deletes must land once canDelete is set") store2.Close() } @@ -175,7 +175,7 @@ func TestMarkVolumeReadonlyCanDelete_CountsDeletesAfterRestart(t *testing.T) { require.NotNil(t, v) require.IsType(t, &SortedFileNeedleMap{}, v.nm) for _, n := range needles { - _, err := store2.DeleteVolumeNeedle(vid, n) + _, err := store2.DeleteVolumeNeedle(vid, n, false) require.NoError(t, err) } counters := func(v *Volume) [4]uint64 { diff --git a/weed/storage/store_vacuum_test.go b/weed/storage/store_vacuum_test.go index 3adc164330..bcdc14e650 100644 --- a/weed/storage/store_vacuum_test.go +++ b/weed/storage/store_vacuum_test.go @@ -28,7 +28,7 @@ func newVolumeWithGarbage(t *testing.T, live, deleted int) *Volume { } } for i := 1; i <= deleted; i++ { - if _, err := v.deleteNeedle2(newEmptyNeedle(uint64(i))); err != nil { + if _, err := v.deleteNeedle2(newEmptyNeedle(uint64(i)), false); err != nil { t.Fatalf("delete needle %d: %v", i, err) } } diff --git a/weed/storage/volume_idx_rebuild_test.go b/weed/storage/volume_idx_rebuild_test.go index a71abee833..a369801d71 100644 --- a/weed/storage/volume_idx_rebuild_test.go +++ b/weed/storage/volume_idx_rebuild_test.go @@ -34,7 +34,7 @@ func TestLoad_MovedIdxDirectory_RebuildsIdx(t *testing.T) { t.Fatalf("seed write %d: %v", id, err) } } - if _, err := v.deleteNeedle2(newRandomNeedle(2)); err != nil { + if _, err := v.deleteNeedle2(newRandomNeedle(2), false); err != nil { t.Fatalf("seed delete: %v", err) } wantCount, wantDeleted := v.nm.FileCount(), v.nm.DeletedCount() diff --git a/weed/storage/volume_pending_write_test.go b/weed/storage/volume_pending_write_test.go index 0ca6a84bf9..8c754f2400 100644 --- a/weed/storage/volume_pending_write_test.go +++ b/weed/storage/volume_pending_write_test.go @@ -60,14 +60,16 @@ func TestDeleteOfMissingNeedleCancelsQueuedWrite(t *testing.T) { v.dataFileSyncLock.Unlock() require.ErrorIs(t, <-written, ErrorDeleted) - requireNeedleDeleted(t, v, n.Id) + _, err = v.readNeedle(&needle.Needle{Id: n.Id}, nil, nil) + require.ErrorIs(t, err, ErrorNotFound) require.Empty(t, v.pendingWrites.byId) v.Close() v, err = NewVolume(dir, dir, "", 1, NeedleMapInMemory, nil, nil, 0, needle.GetCurrentVersion(), 0, 0) require.NoError(t, err) defer v.Close() - requireNeedleDeleted(t, v, n.Id) + _, err = v.readNeedle(&needle.Needle{Id: n.Id}, nil, nil) + require.ErrorIs(t, err, ErrorNotFound) }) } } @@ -91,7 +93,7 @@ func TestDeleteCancelsQueuedRewrite(t *testing.T) { }() waitForPendingWrites(t, v, n.Id, 1) - _, err = v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}) + _, err = v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}, false) require.NoError(t, err) requireNeedleDeleted(t, v, n.Id) v.asyncWorkerLock.Unlock() @@ -101,13 +103,45 @@ func TestDeleteCancelsQueuedRewrite(t *testing.T) { require.Empty(t, v.pendingWrites.byId) } +// A rewrite cancelled before it takes the volume locks appends nothing, so +// no unsynced row of it can outlive a durable delete after a crash. +func TestCancelledRewriteDoesNotAppend(t *testing.T) { + v := newWorkerTestVolume(t) + n := newRandomNeedle(1) + _, _, _, err := v.writeNeedle2(n, true, false, false) + require.NoError(t, err) + before, _, err := v.DataBackend.GetStat() + require.NoError(t, err) + + rewrite := newRandomNeedle(1) + rewrite.Cookie = n.Cookie + v.dataFileSyncLock.Lock() + written := make(chan error, 1) + go func() { + _, _, _, err := v.writeNeedle2(rewrite, true, false, false) + written <- err + }() + waitForPendingWrites(t, v, n.Id, 1) + v.CancelPendingWrites(n.Id, n.Cookie) + v.dataFileSyncLock.Unlock() + + require.ErrorIs(t, <-written, ErrorDeleted) + after, _, err := v.DataBackend.GetStat() + require.NoError(t, err) + require.Equal(t, before, after) + + _, err = v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}, true) + require.NoError(t, err) + requireNeedleDeleted(t, v, n.Id) +} + // Only writes already in flight are cancelled: writing the needle again after // the delete works as before. func TestWriteAfterDeleteIsNotCancelled(t *testing.T) { v := newWorkerTestVolume(t) n := newRandomNeedle(1) v.CancelPendingWrites(n.Id, n.Cookie) - _, err := v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}) + _, err := v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}, false) require.NoError(t, err) _, _, _, err = v.writeNeedle2(n, true, false, false) @@ -189,9 +223,14 @@ func TestWriteAfterCancelledWriteStaysLive(t *testing.T) { type syncedSizeBackend struct { backend.BackendStorageFile synced atomic.Int64 + onSync func() } func (b *syncedSizeBackend) Sync() error { + if f := b.onSync; f != nil { + b.onSync = nil + f() + } size, _, err := b.GetStat() if err != nil { return err @@ -210,17 +249,9 @@ func TestTakenBackTombstoneIsSynced(t *testing.T) { v.DataBackend = synced n := newRandomNeedle(1) - v.dataFileSyncLock.Lock() - written := make(chan error, 1) - go func() { - _, _, _, err := v.writeNeedle2(n, true, true, isStopping) - written <- err - }() - waitForPendingWrites(t, v, n.Id, 1) - v.CancelPendingWrites(n.Id, n.Cookie) - v.dataFileSyncLock.Unlock() - - require.ErrorIs(t, <-written, ErrorDeleted) + synced.onSync = func() { v.CancelPendingWrites(n.Id, n.Cookie) } + _, _, _, err := v.writeNeedle2(n, true, true, isStopping) + require.ErrorIs(t, err, ErrorDeleted) size, _, err := v.DataBackend.GetStat() require.NoError(t, err) require.Equal(t, size, synced.synced.Load(), "the tombstone was not flushed") diff --git a/weed/storage/volume_read_test.go b/weed/storage/volume_read_test.go index 4a149d0c63..cc59637176 100644 --- a/weed/storage/volume_read_test.go +++ b/weed/storage/volume_read_test.go @@ -113,7 +113,7 @@ func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) { t.Fatalf("write needle %d: %v", i, err) } if i < 5 { - size, err := v.deleteNeedle2(n) + size, err := v.deleteNeedle2(n, false) if err != nil { t.Fatalf("delete needle %d: %v", i, err) } @@ -706,7 +706,7 @@ func TestReadNeedleDataIntoFollowsACompaction(t *testing.T) { t.Fatalf("write needle %d: %v", n.Id, err) } } - if _, err := v.deleteNeedle2(newEmptyNeedle(1)); err != nil { + if _, err := v.deleteNeedle2(newEmptyNeedle(1), false); err != nil { t.Fatalf("delete: %v", err) } diff --git a/weed/storage/volume_tier_deletable_test.go b/weed/storage/volume_tier_deletable_test.go index e1d1af974c..c90e817156 100644 --- a/weed/storage/volume_tier_deletable_test.go +++ b/weed/storage/volume_tier_deletable_test.go @@ -112,12 +112,12 @@ func TestDeleteVolumeNeedleRacesMarkTieredDeletable(t *testing.T) { } }() for i := 0; i < 100; i++ { - _, _ = s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1000)}) + _, _ = s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1000)}, false) } <-done // Positive control: once deletable, a delete of a live needle goes through. - size, err := s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1)}) + size, err := s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1)}, false) require.NoError(t, err) require.NotZero(t, size) } diff --git a/weed/storage/volume_vacuum_crash_safe_test.go b/weed/storage/volume_vacuum_crash_safe_test.go index deca5c7c60..acf4373948 100644 --- a/weed/storage/volume_vacuum_crash_safe_test.go +++ b/weed/storage/volume_vacuum_crash_safe_test.go @@ -48,7 +48,7 @@ func TestReconcileRollForwardMarkerOnly(t *testing.T) { } // Delete two so the compacted generation is strictly smaller than the source. for _, id := range []uint64{2, 5} { - if _, err := v.deleteNeedle2(newEmptyNeedle(id)); err != nil { + if _, err := v.deleteNeedle2(newEmptyNeedle(id), false); err != nil { t.Fatalf("delete %d: %v", id, err) } } @@ -122,7 +122,7 @@ func TestReconcileRollForwardPartialRename(t *testing.T) { } } for _, id := range []uint64{2, 5} { - if _, err := v.deleteNeedle2(newEmptyNeedle(id)); err != nil { + if _, err := v.deleteNeedle2(newEmptyNeedle(id), false); err != nil { t.Fatalf("delete %d: %v", id, err) } } diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go index b144d8096b..2030c31074 100644 --- a/weed/storage/volume_vacuum_test.go +++ b/weed/storage/volume_vacuum_test.go @@ -179,7 +179,7 @@ func TestCommitCompactDeletionTailKeepsWritable(t *testing.T) { // The sole change in the commit window is a deletion, so makeupDiff appends // its tombstone last, putting it at the .dat tail. - if _, err := v.deleteNeedle2(newEmptyNeedle(3)); err != nil { + if _, err := v.deleteNeedle2(newEmptyNeedle(3), false); err != nil { t.Fatalf("delete: %v", err) } @@ -301,7 +301,7 @@ func TestCommitCompactMakeupDiffFailureKeepsVolumeLoaded(t *testing.T) { } } // Moves needle 3 in the compacted copy, so reads through its map would miss. - if _, err := v.deleteNeedle2(newEmptyNeedle(1)); err != nil { + if _, err := v.deleteNeedle2(newEmptyNeedle(1), false); err != nil { t.Fatalf("delete: %v", err) } delete(written, 1) @@ -733,7 +733,7 @@ func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) { if rand.Float64() < 0.03 { toBeDeleted := rand.Intn(i) + 1 oldNeedle := newEmptyNeedle(uint64(toBeDeleted)) - v.deleteNeedle2(oldNeedle) + v.deleteNeedle2(oldNeedle, false) // println("deleted file", toBeDeleted) infos[toBeDeleted-1] = &needleInfo{ size: 0, diff --git a/weed/storage/volume_vacuum_tiered_test.go b/weed/storage/volume_vacuum_tiered_test.go index e39de29385..2fcc824a25 100644 --- a/weed/storage/volume_vacuum_tiered_test.go +++ b/weed/storage/volume_vacuum_tiered_test.go @@ -27,7 +27,7 @@ func TestCompactRefusesTieredVolume(t *testing.T) { defer v.Close() require.True(t, v.HasRemoteFile()) for i := 1; i <= 4; i++ { - _, err := v.deleteNeedle2(newRandomNeedle(uint64(i))) + _, err := v.deleteNeedle2(newRandomNeedle(uint64(i)), false) require.NoError(t, err) } garbage := v.garbageLevel() @@ -74,7 +74,7 @@ func testCommitCompactRefusesVolumeTieredDuringCopy(t *testing.T, kind NeedleMap written[i] = n.Data } for i := uint64(1); i <= 3; i++ { - _, err := v.deleteNeedle2(newEmptyNeedle(i)) + _, err := v.deleteNeedle2(newEmptyNeedle(i), false) require.NoError(t, err) } require.NoError(t, v.CompactByIndex(nil)) diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index e9c2bada19..5ffec7ed54 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -163,11 +163,17 @@ func removeVolumeFiles(filename string, keepVif bool) { // asyncRequestAppend queues a request for the batch worker, starting it on the // first one. It reports false for a destroyed volume, so the caller writes // inline rather than wait on a worker that will never answer. -func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) bool { +func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) (queued bool) { requests := v.startWorker() if requests == nil { return false } + // stopWorker may close the channel between startWorker and the send. + defer func() { + if recover() != nil { + queued = false + } + }() requests <- request return true } @@ -344,6 +350,11 @@ func (v *Volume) appendWriteRequest(n *needle.Needle, checkCookie bool, fsync bo if v.IsReadOnly() { return 0, 0, false, fmt.Errorf("volume %d is %w", v.Id, ErrorReadOnly) } + // A write cancelled before it got here must not append: its delete may + // already be durable, and an unsynced row could outlive its tombstone. + if v.isCancelledWrite(n) { + return 0, 0, false, fmt.Errorf("needle %s: %w", needle.NewFileIdFromNeedle(v.Id, n), ErrorDeleted) + } if v.isFileUnchanged(n) { size = Size(n.DataSize) isUnchanged = true @@ -438,7 +449,7 @@ func (v *Volume) publishWriteRequest(n *needle.Needle, offset uint64, fsync bool return } -func (v *Volume) syncDelete(n *needle.Needle) (Size, error) { +func (v *Volume) syncDelete(n *needle.Needle, fsync bool) (Size, error) { // glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String()) v.dataFileSyncLock.Lock() defer v.dataFileSyncLock.Unlock() @@ -456,28 +467,66 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) { return 0, nil } - return v.doDeleteRequest(n) + if !fsync { + return v.doDeleteRequest(n) + } + + end, _, statErr := v.DataBackend.GetStat() + if statErr != nil { + return 0, fmt.Errorf("cannot read current volume position: %v", statErr) + } + lastAppendAtNs := v.lastAppendAtNs + + size, offset, tombstoned, err := v.appendDeleteRequest(n) + if err != nil { + return size, err + } + // Same order as a durable write: the tombstone is down before the map + // drops the needle, and its .idx row before the delete is acknowledged. + // An already-deleted needle still syncs: an earlier delete may have skipped it. + backend := v.DataBackend + v.dataFileAccessLock.Unlock() + syncErr := backend.Sync() + v.dataFileAccessLock.Lock() + if syncErr != nil { + v.checkReadWriteError(syncErr) + v.lastAppendAtNs = lastAppendAtNs + if !v.HasRemoteFile() { + if te := v.DataBackend.Truncate(end); te != nil { + v.markIoUnavailable(fmt.Errorf("truncate %s back to %d after a failed fsync: %w", + v.DataBackend.Name(), end, te)) + } + } + return 0, syncErr + } + if tombstoned { + if err = v.publishDeleteRequest(n, offset); err != nil { + return size, err + } + } + v.dataFileAccessLock.Unlock() + err = v.syncIndex() + v.dataFileAccessLock.Lock() + return size, err } -func (v *Volume) deleteNeedle2(n *needle.Needle) (Size, error) { +// deleteNeedle2 acknowledges a durable delete only once its tombstone and .idx +// row are on disk, through the batch worker like a durable write. +func (v *Volume) deleteNeedle2(n *needle.Needle, fsync bool) (Size, error) { v.CancelPendingWrites(n.Id, n.Cookie) - // todo: delete info is always appended no fsync, it may need fsync in future - fsync := false - if !fsync { - return v.syncDelete(n) - } else { - asyncRequest := needle.NewAsyncRequest(n, false) - asyncRequest.ActualSize = needle.GetActualSize(0, v.Version()) - - if !v.asyncRequestAppend(asyncRequest) { - return v.syncDelete(n) - } - _, size, _, err := asyncRequest.WaitComplete() - - return Size(size), err + return v.syncDelete(n, false) } + asyncRequest := needle.NewAsyncRequest(n, false) + asyncRequest.ActualSize = needle.GetActualSize(0, v.Version()) + + if !v.asyncRequestAppend(asyncRequest) { + return v.syncDelete(n, true) + } + _, size, _, err := asyncRequest.WaitComplete() + + return Size(size), err } func (v *Volume) doDeleteRequest(n *needle.Needle) (Size, error) { @@ -517,6 +566,11 @@ func (v *Volume) appendDeleteRequest(n *needle.Needle) (size Size, offset uint64 func (v *Volume) publishDeleteRequest(n *needle.Needle, offset uint64) error { if err := v.nm.Delete(n.Id, ToOffset(int64(offset))); err != nil { + // The map already dropped the needle, so a retry, durable or not, would + // ack without the missing .idx row. Stop taking writes. + v.noWriteLock.Lock() + v.noWriteOrDelete = true + v.noWriteLock.Unlock() return err } if n.Id == v.lastWriteNeedleKey { @@ -677,6 +731,7 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { } else if stagedDeleteOk[i] { if perr := v.publishDeleteRequest(run[i].N, stagedDeleteOffsets[i]); perr != nil { run[i].UpdateResult(0, 0, false, perr) + refused = true } } } @@ -769,7 +824,7 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { if request == nil { break } - if MaxPossibleVolumeSize < v.ContentSize()+uint64(currentBytesToWrite+request.ActualSize) { + if request.IsWriteRequest && MaxPossibleVolumeSize < v.ContentSize()+uint64(currentBytesToWrite+request.ActualSize) { request.Complete(0, 0, false, fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.ContentSize())) break diff --git a/weed/storage/volume_write_fsync_test.go b/weed/storage/volume_write_fsync_test.go index 9e62baff83..1a45827c64 100644 --- a/weed/storage/volume_write_fsync_test.go +++ b/weed/storage/volume_write_fsync_test.go @@ -2,6 +2,8 @@ package storage import ( "errors" + "os" + "sync/atomic" "testing" "github.com/seaweedfs/seaweedfs/weed/storage/backend" @@ -608,3 +610,135 @@ func TestStoreWriteVolumeNeedleStaysDurableWhileStopping(t *testing.T) { require.False(t, isUnchanged) require.Equal(t, 1, counting.syncCount, "the accepted write must actually be on disk") } + +func TestDurableDeleteSyncsTombstone(t *testing.T) { + v, counting := newCountingVolume(t) + + for id := uint64(1); id <= 2; id++ { + _, _, _, err := v.writeNeedle2(fixedNeedle(id, "payload"), true, false, false) + require.NoError(t, err) + } + require.Equal(t, 0, counting.syncCount) + + _, err := v.deleteNeedle2(fixedNeedle(1, ""), false) + require.NoError(t, err) + require.Equal(t, 0, counting.syncCount, "a delete that did not ask for fsync must not pay for one") + + size, err := v.deleteNeedle2(fixedNeedle(2, ""), true) + require.NoError(t, err) + require.NotZero(t, size) + require.Equal(t, 1, counting.syncCount, "a durable delete must flush its tombstone before the ack") + + reloaded := reopenCountingVolume(t, v) + nv, found := reloaded.nm.Get(types.Uint64ToNeedleId(2)) + require.True(t, !found || nv.Size.IsDeleted(), "the durable delete must survive a reload") +} + +func TestDurableDeleteTakesBackTombstoneWhenFsyncFails(t *testing.T) { + v, counting := newCountingVolume(t) + + _, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, true, true) + require.NoError(t, err) + before, _, err := v.DataBackend.GetStat() + require.NoError(t, err) + syncs := counting.syncCount + + counting.syncErr = errors.New("disk went away") + _, err = v.syncDelete(fixedNeedle(1, ""), true) + require.Error(t, err, "a delete whose fsync failed must not be acked") + require.Equal(t, syncs+1, counting.syncCount) + + after, _, err := v.DataBackend.GetStat() + require.NoError(t, err) + require.Equal(t, before, after, "the unflushed tombstone should be taken back off the .dat") + nv, found := v.nm.Get(types.Uint64ToNeedleId(1)) + require.True(t, found) + require.False(t, nv.Size.IsDeleted(), "the needle must stay live when its delete was not acked") +} + +func TestDurableDeleteOnFullVolume(t *testing.T) { + v, counting := newCountingVolume(t) + + _, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, false, false) + require.NoError(t, err) + atomic.StoreUint64(&v.nm.(*NeedleMap).FileByteCounter, types.MaxPossibleVolumeSize) + + _, err = v.deleteNeedle2(fixedNeedle(1, ""), true) + require.NoError(t, err, "a full volume must still take a durable delete") + require.Equal(t, 1, counting.syncCount) + nv, found := v.nm.Get(types.Uint64ToNeedleId(1)) + require.True(t, !found || nv.Size.IsDeleted()) +} + +func TestDurableDeleteSyncsAfterUnsyncedDelete(t *testing.T) { + v, counting := newCountingVolume(t) + + _, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, false, false) + require.NoError(t, err) + _, err = v.syncDelete(fixedNeedle(1, ""), false) + require.NoError(t, err) + require.Equal(t, 0, counting.syncCount) + + size, err := v.syncDelete(fixedNeedle(1, ""), true) + require.NoError(t, err) + require.Zero(t, size) + require.Equal(t, 1, counting.syncCount, "an earlier unsynced delete must still be flushed before a durable ack") +} + +func TestAsyncRequestAppendRefusesAfterChannelClosed(t *testing.T) { + v, _ := newCountingVolume(t) + + closed := make(chan *needle.AsyncRequest) + close(closed) + v.asyncWorkerLock.Lock() + v.asyncRequestsChan = closed + v.asyncWorkerLock.Unlock() + + require.False(t, v.asyncRequestAppend(needle.NewAsyncRequest(fixedNeedle(1, "payload"), true)), + "a request racing stopWorker must be refused, not panic") +} + +// A durable delete whose .idx row fails has already dropped the needle from +// the map, so a retry would find nothing to delete and ack. The volume stops +// taking writes instead: nothing is appended past the tombstone, so a reload +// re-indexes it from the .dat tail rather than resurrect the needle. +func TestDurableDeleteIndexFailureStopsWrites(t *testing.T) { + durableBatch := func(v *Volume, n *needle.Needle) (types.Size, error) { return v.deleteNeedle2(n, true) } + durableInline := func(v *Volume, n *needle.Needle) (types.Size, error) { return v.syncDelete(n, true) } + for _, tc := range []struct { + name string + first func(*Volume, *needle.Needle) (types.Size, error) + delete func(*Volume, *needle.Needle) (types.Size, error) + }{ + {"batch", durableBatch, durableBatch}, + {"inline", durableInline, durableInline}, + {"non-durable then durable retry", func(v *Volume, n *needle.Needle) (types.Size, error) { return v.deleteNeedle2(n, false) }, durableBatch}, + } { + t.Run(tc.name, func(t *testing.T) { + v, _ := newCountingVolume(t) + _, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, true, true) + require.NoError(t, err) + + nm := v.nm.(*NeedleMap) + indexFile := nm.indexFile + readOnlyIndex, err := os.Open(indexFile.Name()) + require.NoError(t, err) + defer readOnlyIndex.Close() + nm.indexFile = readOnlyIndex + _, err = tc.first(v, fixedNeedle(1, "")) + nm.indexFile = indexFile + require.Error(t, err) + + _, _, _, err = v.writeNeedle2(fixedNeedle(2, "later"), true, true, true) + require.Error(t, err, "nothing may be appended past the unindexed tombstone") + _, err = tc.delete(v, fixedNeedle(1, "")) + require.Error(t, err, "a retry must not ack a delete whose .idx row never landed") + + reloaded := reopenCountingVolume(t, v) + readBack := fixedNeedle(1, "") + _, err = reloaded.readNeedle(readBack, nil, nil) + require.False(t, err == nil && string(readBack.Data) == "payload", + "the deleted needle must not come back on reload") + }) + } +} diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index e8851a76d6..93aa3a5e25 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -267,7 +267,8 @@ func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOp }(time.Now()) } - size, err = store.DeleteVolumeNeedle(volumeId, n) + fsync := r.FormValue("fsync") == "true" + size, err = store.DeleteVolumeNeedle(volumeId, n, fsync) if err != nil { glog.V(0).Infoln("delete error:", err) return @@ -279,7 +280,11 @@ func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOp if replicaCount > 0 { //send to other replica locations // background, not r.Context(): a client disconnect must not orphan replica deletes if err = DistributedOperation(context.Background(), remoteLocations, func(ctx context.Context, location operation.Location) error { - url, normalizeErr := util_http.GetGlobalHttpClient().NormalizeHttpScheme(location.Url + r.URL.Path + "?type=replicate") + query := "?type=replicate" + if fsync { + query += "&fsync=true" + } + url, normalizeErr := util_http.GetGlobalHttpClient().NormalizeHttpScheme(location.Url + r.URL.Path + query) if normalizeErr != nil { return normalizeErr }