mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
volume: fsync a needle delete that asks for it (#12736)
* volume: fsync a needle delete that asks for it A delete now takes fsync like a write: the tombstone is synced before the needle map drops the needle, and the .idx row before the delete is acknowledged. A failed .dat sync takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: honor fsync=true on DELETE and forward it to replicas Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): fsync a needle delete that asks for it Mirrors the Go volume server: DELETE with fsync=true flushes the tombstone before the index drops the needle and the .idx before the ack, and a failed .dat flush takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: let a durable delete through on a full volume A queued delete hit the batch worker's volume size check, which a tombstone append is exempt from inline, so fsync=true deletes failed once the volume reached its hard limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: sync a durable delete of an already-deleted needle An earlier delete without fsync may have dropped the needle with its tombstone still unsynced, so the inline path syncs before acking anyway, as the batch worker already does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a batch request that races the worker stopping stopWorker can close the batch channel between startWorker handing it out and the send, which panicked. The request now falls back to the inline path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): sync a durable delete of an already-deleted needle Mirrors the Go volume server: an earlier delete without fsync may have dropped the needle with its tombstone still unsynced. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): run a durable delete on a blocking thread Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: keep syncDelete's signature for existing callers Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a write a delete cancelled before it appends A write cancelled while still queued behind the volume locks used to append, publish and then tombstone itself without a sync, after its delete may already have been acknowledged as durable. A crash could keep the row and lose the tombstone. Refuse it before the append. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): refuse a write a delete cancelled before it appends Mirrors the Go change: a write cancelled while still queued behind the volume locks no longer appends, publishes and tombstones itself without a sync after its delete may already be durable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): drop the write guards for a durable delete's flushes The durable delete held the write lease and the io write guard across both fsyncs, so reads of the volume waited on them. Append and publish the tombstone under the guards and flush with only the io read side, as write_synced_run and the Go path already do. do_delete_request keeps its signature. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when a durable delete fails to index The needle map drops the needle before it appends the .idx row, so after a failed append a durable retry found nothing to delete and acked, and a later write could bury the unindexed tombstone so a reload brought the needle back. Fence the volume the way a failed durable write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when a durable delete fails to index Rust counterpart of the Go change: the needle map drops the needle before it appends the .idx row, so a failed append left a durable retry nothing to delete, and a later write could bury the unindexed tombstone. publish_delete_request now fences the volume for a durable delete, as publish_write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when any delete fails to index A non-durable delete whose .idx append failed left the volume writable with the needle already dropped from the map, so a later durable retry found nothing to delete and acknowledged without the missing row. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when any delete fails to index Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: take the fsync flag on the existing delete functions DeleteVolumeNeedle, deleteNeedle2 and syncDelete take fsync instead of gaining *WithFsync variants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): take the fsync flag on the existing delete functions delete_volume_needle and delete_needle take fsync instead of gaining *_with_fsync variants; the durable path and the deletable checks fold back into delete_needle, and the cancelled-write check mirrors Go's isCancelledWrite. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
23eef59417
commit
9cd183b04f
27 files changed
+855
-232
No files matched your search
@@ -1276,7 +1276,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
Some(vol) if vol.is_no_write_or_delete() => {
|
||||
Err(crate::storage::volume::VolumeError::ReadOnly(file_id.volume_id))
|
||||
}
|
||||
Some(vol) => vol.delete_needle(&mut n),
|
||||
Some(vol) => vol.delete_needle(&mut n, false),
|
||||
};
|
||||
match delete_result {
|
||||
Ok(size) => {
|
||||
@@ -9260,6 +9260,7 @@ mod tests {
|
||||
id: NeedleId(7),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -9685,7 +9686,9 @@ mod tests {
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(VolumeId(1), &mut n).unwrap();
|
||||
store
|
||||
.delete_volume_needle(VolumeId(1), &mut n, false)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn read_surviving_needle(
|
||||
@@ -14158,7 +14161,9 @@ mod tests {
|
||||
cookie: Cookie(0x3344),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(VolumeId(1), &mut n).unwrap();
|
||||
store
|
||||
.delete_volume_needle(VolumeId(1), &mut n, false)
|
||||
.unwrap();
|
||||
}
|
||||
let err = needle_status(&service, 11).await.unwrap_err();
|
||||
assert_eq!(err.code(), tonic::Code::Unknown, "{err}");
|
||||
@@ -15281,11 +15286,14 @@ mod tests {
|
||||
{
|
||||
let store = service.state.store.read().unwrap();
|
||||
let (_, vol) = store.find_volume(VolumeId(1)).unwrap();
|
||||
vol.delete_needle(&mut Needle {
|
||||
id: NeedleId(11),
|
||||
cookie: Cookie(0x3344),
|
||||
..Needle::default()
|
||||
})
|
||||
vol.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(11),
|
||||
cookie: Cookie(0x3344),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
vol.sync_to_disk().unwrap();
|
||||
}
|
||||
@@ -15344,11 +15352,14 @@ mod tests {
|
||||
{
|
||||
let store = service.state.store.read().unwrap();
|
||||
let (_, vol) = store.find_volume(VolumeId(1)).unwrap();
|
||||
vol.delete_needle(&mut Needle {
|
||||
id: NeedleId(11),
|
||||
cookie: Cookie(0x3344),
|
||||
..Needle::default()
|
||||
})
|
||||
vol.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(11),
|
||||
cookie: Cookie(0x3344),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
vol.sync_to_disk().unwrap();
|
||||
}
|
||||
|
||||
@@ -3680,11 +3680,16 @@ pub async fn delete_handler(
|
||||
|
||||
// Read needle first to validate cookie (matching Go behavior)
|
||||
let original_cookie = cookie;
|
||||
let is_replicate = serde_urlencoded::from_str::<Vec<(String, String)>>(&del_query)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.find(|(name, _)| name == "type")
|
||||
.is_some_and(|(_, value)| value == "replicate");
|
||||
let del_fields =
|
||||
serde_urlencoded::from_str::<Vec<(String, String)>>(&del_query).unwrap_or_default();
|
||||
let del_field = |key: &str| {
|
||||
del_fields
|
||||
.iter()
|
||||
.find(|(name, _)| name == key)
|
||||
.map(|(_, value)| value.as_str())
|
||||
};
|
||||
let is_replicate = del_field("type") == Some("replicate");
|
||||
let fsync = del_field("fsync") == Some("true");
|
||||
{
|
||||
let vol = {
|
||||
let store = state.store.read().unwrap();
|
||||
@@ -3786,7 +3791,16 @@ pub async fn delete_handler(
|
||||
Ok(vol) if vol.is_no_write_or_delete() => {
|
||||
Err(crate::storage::volume::VolumeError::ReadOnly(vid))
|
||||
}
|
||||
Ok(vol) => vol.delete_needle(&mut n),
|
||||
Ok(vol) if fsync => {
|
||||
tokio::task::spawn_blocking(move || vol.delete_needle(&mut n, true))
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
Err(crate::storage::volume::VolumeError::Io(
|
||||
std::io::Error::other(e),
|
||||
))
|
||||
})
|
||||
}
|
||||
Ok(vol) => vol.delete_needle(&mut n, false),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -8760,7 +8774,7 @@ mod tests {
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.delete_volume_needle(VolumeId(1), &mut filler)
|
||||
.delete_volume_needle(VolumeId(1), &mut filler, false)
|
||||
.unwrap();
|
||||
let data_offset = || {
|
||||
let mut plan = crate::storage::store::Store::needle_read_plan(
|
||||
|
||||
@@ -1042,13 +1042,18 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Delete a needle from a volume.
|
||||
pub fn delete_volume_needle(&self, vid: VolumeId, n: &mut Needle) -> Result<Size, VolumeError> {
|
||||
pub fn delete_volume_needle(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
fsync: bool,
|
||||
) -> Result<Size, VolumeError> {
|
||||
// Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding.
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
if vol.is_no_write_or_delete() {
|
||||
return Err(VolumeError::ReadOnly(vid));
|
||||
}
|
||||
vol.delete_needle(n)
|
||||
vol.delete_needle(n, fsync)
|
||||
}
|
||||
|
||||
// ---- Collection operations ----
|
||||
@@ -2923,7 +2928,9 @@ mod tests {
|
||||
cookie: Cookie(0xaa),
|
||||
..Needle::default()
|
||||
};
|
||||
let deleted = store.delete_volume_needle(VolumeId(1), &mut del_n).unwrap();
|
||||
let deleted = store
|
||||
.delete_volume_needle(VolumeId(1), &mut del_n, false)
|
||||
.unwrap();
|
||||
assert!(deleted.0 > 0);
|
||||
}
|
||||
|
||||
@@ -3400,7 +3407,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
|
||||
let size_before = store.find_volume(vid).unwrap().1.dat_file_size().unwrap();
|
||||
|
||||
@@ -3724,7 +3731,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
let revision = {
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
v.super_block().compaction_revision
|
||||
@@ -3749,7 +3756,7 @@ mod tests {
|
||||
cookie: Cookie(4),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
|
||||
drop(release_tx);
|
||||
copy.join().unwrap().unwrap();
|
||||
|
||||
@@ -1621,6 +1621,8 @@ pub struct Volume {
|
||||
dat_syncs_for_test: std::sync::atomic::AtomicUsize,
|
||||
#[cfg(test)]
|
||||
idx_syncs_for_test: std::sync::atomic::AtomicUsize,
|
||||
#[cfg(test)]
|
||||
on_flush_dat_for_test: Mutex<Option<Box<dyn FnOnce(&Volume) + Send>>>,
|
||||
needle_map_kind: NeedleMapKind,
|
||||
data_file_access_control: Arc<DataFileAccessControl>,
|
||||
/// Serializes a data-file mutation with the sync that makes it durable:
|
||||
@@ -1702,6 +1704,12 @@ pub struct PendingWrite {
|
||||
flag: Arc<CancelFlag>,
|
||||
}
|
||||
|
||||
/// A write cancelled before it appends is refused: its delete may already be
|
||||
/// durable, and an unsynced row could outlive its tombstone.
|
||||
fn is_cancelled_write(pending: Option<&PendingWrite>) -> bool {
|
||||
pending.is_some_and(|p| p.flag.cancelled.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
impl Drop for PendingWrite {
|
||||
fn drop(&mut self) {
|
||||
let mut by_id = self.writes.0.lock().unwrap();
|
||||
@@ -1785,6 +1793,8 @@ impl Volume {
|
||||
dat_syncs_for_test: Default::default(),
|
||||
#[cfg(test)]
|
||||
idx_syncs_for_test: Default::default(),
|
||||
#[cfg(test)]
|
||||
on_flush_dat_for_test: Default::default(),
|
||||
needle_map_kind,
|
||||
data_file_access_control: Arc::new(DataFileAccessControl::default()),
|
||||
data_file_sync_lock: std::sync::Mutex::new(()),
|
||||
@@ -1844,6 +1854,8 @@ impl Volume {
|
||||
dat_syncs_for_test: Default::default(),
|
||||
#[cfg(test)]
|
||||
idx_syncs_for_test: Default::default(),
|
||||
#[cfg(test)]
|
||||
on_flush_dat_for_test: Default::default(),
|
||||
needle_map_kind: NeedleMapKind::InMemory,
|
||||
data_file_access_control: Arc::new(DataFileAccessControl::default()),
|
||||
data_file_sync_lock: std::sync::Mutex::new(()),
|
||||
@@ -3062,6 +3074,9 @@ impl Volume {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
if is_cancelled_write(pending) {
|
||||
return Err(VolumeError::Deleted);
|
||||
}
|
||||
if let Err(refused) = self.check_writable() {
|
||||
return Ok(Err(refused));
|
||||
}
|
||||
@@ -3112,11 +3127,14 @@ impl Volume {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
for (i, (n, _)) in run.iter_mut().enumerate() {
|
||||
let written = self
|
||||
.check_writable()
|
||||
.and_then(|()| self.do_write_request(&mut io, n, true, false));
|
||||
let pending = tracked.get(i).and_then(Option::as_ref);
|
||||
let written = self.check_writable().and_then(|()| {
|
||||
if is_cancelled_write(pending) {
|
||||
return Err(VolumeError::Deleted);
|
||||
}
|
||||
self.do_write_request(&mut io, n, true, false)
|
||||
});
|
||||
results.push(written.and_then(|written| {
|
||||
let pending = tracked.get(i).and_then(Option::as_ref);
|
||||
self.drop_cancelled_write(&mut io, pending, n, &written)
|
||||
.map(|()| written)
|
||||
}));
|
||||
@@ -3147,8 +3165,12 @@ impl Volume {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
last_append_at_ns = io.last_append_at_ns;
|
||||
for (n, fsync) in run.iter_mut() {
|
||||
let r = self.append_unpublished(&mut io, n, &mut last_append_at_ns);
|
||||
for (i, (n, fsync)) in run.iter_mut().enumerate() {
|
||||
let r = if is_cancelled_write(pending.get(i).and_then(Option::as_ref)) {
|
||||
Err(VolumeError::Deleted)
|
||||
} else {
|
||||
self.append_unpublished(&mut io, n, &mut last_append_at_ns)
|
||||
};
|
||||
marks.push(self.io_errors.mark());
|
||||
if let Ok(Some(offset)) = r {
|
||||
run_start.get_or_insert(offset);
|
||||
@@ -3295,6 +3317,13 @@ impl Volume {
|
||||
#[cfg(test)]
|
||||
self.dat_syncs_for_test.fetch_add(1, Ordering::Relaxed);
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = self.on_flush_dat_for_test.lock().unwrap().take();
|
||||
if let Some(hook) = hook {
|
||||
hook(self);
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if io.fail_fsync_for_test {
|
||||
return Err(io::Error::other("injected fsync failure"));
|
||||
}
|
||||
@@ -3805,28 +3834,85 @@ impl Volume {
|
||||
|
||||
// ---- Delete ----
|
||||
|
||||
/// Delete a needle from the volume.
|
||||
pub fn delete_needle(&self, n: &mut Needle) -> Result<Size, VolumeError> {
|
||||
/// Delete a needle from the volume. With `fsync` the delete is only
|
||||
/// acknowledged once its tombstone and .idx row are on disk, in the same
|
||||
/// order as a durable write; the write guards are dropped for each flush
|
||||
/// as in write_synced_run, so reads do not wait on the fsync.
|
||||
pub fn delete_needle(&self, n: &mut Needle, fsync: bool) -> Result<Size, VolumeError> {
|
||||
self.cancel_pending_writes(n.id, n.cookie);
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let staged = {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
if self.is_closed() {
|
||||
// A detached handle has no index; without this check the tombstone
|
||||
// would silently no-op as Ok(0) while the needle lives on the
|
||||
// remounted volume.
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
if let Some(e) = self.unavailable_error() {
|
||||
return Err(e);
|
||||
}
|
||||
if self.no_write_flags().0 {
|
||||
return Err(VolumeError::ReadOnly(self.id));
|
||||
}
|
||||
if !fsync {
|
||||
return self.do_delete_request(&mut io, n);
|
||||
}
|
||||
self.append_delete_request(&mut io, n)?
|
||||
};
|
||||
// An already-deleted needle still syncs: an earlier delete may have
|
||||
// skipped it.
|
||||
if !self.has_remote_file() {
|
||||
let io = self.io.read().unwrap();
|
||||
if let Err(e) = self.flush_dat(&io) {
|
||||
self.check_read_write_error(Some(&e));
|
||||
if let Some((offset, _)) = staged {
|
||||
self.undo_unsynced_append(&io, offset);
|
||||
}
|
||||
return Err(VolumeError::Io(e));
|
||||
}
|
||||
}
|
||||
if let Some((offset, _)) = staged {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
self.publish_delete_request(&mut io, n, offset)?;
|
||||
}
|
||||
self.flush_idx(&self.io.read().unwrap())?;
|
||||
let Some((_, size)) = staged else {
|
||||
return Ok(Size(0));
|
||||
};
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
if self.is_closed() {
|
||||
// A detached handle has no index; without this check the tombstone
|
||||
// would silently no-op as Ok(0) while the needle lives on the
|
||||
// remounted volume.
|
||||
return Err(VolumeError::NotFound);
|
||||
if self.maybe_checkpoint_index(&mut io, true) {
|
||||
self.check_read_write_error(None);
|
||||
}
|
||||
if let Some(e) = self.unavailable_error() {
|
||||
return Err(e);
|
||||
}
|
||||
if self.no_write_flags().0 {
|
||||
return Err(VolumeError::ReadOnly(self.id));
|
||||
}
|
||||
self.do_delete_request(&mut io, n)
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
fn do_delete_request(&self, io: &mut VolumeIo, n: &mut Needle) -> Result<Size, VolumeError> {
|
||||
let Some((offset, size)) = self.append_delete_request(io, n)? else {
|
||||
return Ok(Size(0));
|
||||
};
|
||||
self.publish_delete_request(io, n, offset)?;
|
||||
let checkpoint_ok = self.maybe_checkpoint_index(io, false);
|
||||
|
||||
// Clear the EIO streak after a successful delete (tombstone append +
|
||||
// index update + checkpoint), mirroring do_write_request.
|
||||
if checkpoint_ok {
|
||||
self.check_read_write_error(None);
|
||||
}
|
||||
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
/// do_delete_request through the tombstone append, leaving the index
|
||||
/// update to publish_delete_request. None when there is nothing to delete.
|
||||
fn append_delete_request(
|
||||
&self,
|
||||
io: &mut VolumeIo,
|
||||
n: &mut Needle,
|
||||
) -> Result<Option<(u64, Size)>, VolumeError> {
|
||||
let (found, size, _stored_offset) = if let Some(nm) = &io.nm {
|
||||
// Propagate: acknowledging the delete as a no-op because the index
|
||||
// could not be read would drop the tombstone on the floor while the
|
||||
@@ -3841,11 +3927,11 @@ impl Volume {
|
||||
(false, Size(0), Offset::default())
|
||||
}
|
||||
} else {
|
||||
return Ok(Size(0));
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !found {
|
||||
return Ok(Size(0));
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Write tombstone: append needle with empty data
|
||||
@@ -3860,26 +3946,38 @@ impl Volume {
|
||||
// Remote-tiered volume: skip .dat append, use offset 0
|
||||
0
|
||||
};
|
||||
Ok(Some((offset, size)))
|
||||
}
|
||||
|
||||
fn publish_delete_request(
|
||||
&self,
|
||||
io: &mut VolumeIo,
|
||||
n: &Needle,
|
||||
offset: u64,
|
||||
) -> Result<(), VolumeError> {
|
||||
io.last_append_at_ns = n.append_at_ns;
|
||||
|
||||
// Update index
|
||||
if let Some(nm) = &mut io.nm {
|
||||
nm.delete(n.id, Offset::from_actual_offset(offset as i64))?;
|
||||
if let Some(nm) = &mut io.nm
|
||||
&& let Err(e) = nm.delete(n.id, Offset::from_actual_offset(offset as i64))
|
||||
{
|
||||
// The map already dropped the needle, so a retry, durable or not,
|
||||
// would ack without the missing .idx row. Stop taking writes.
|
||||
self.set_no_write_flag(NO_WRITE_OR_DELETE, true);
|
||||
tracing::error!(
|
||||
"volume {}: failed to index a delete at {}, marking read only: {}",
|
||||
self.id.0,
|
||||
offset,
|
||||
e
|
||||
);
|
||||
return Err(VolumeError::Io(e));
|
||||
}
|
||||
if n.id == io.last_write_needle_key {
|
||||
io.last_write_append_at_ns = 0;
|
||||
io.last_write_needle_key = NeedleId(0);
|
||||
io.last_write_deleted = true;
|
||||
}
|
||||
let checkpoint_ok = self.maybe_checkpoint_index(io, false);
|
||||
|
||||
// Clear the EIO streak after a successful delete (tombstone append +
|
||||
// index update + checkpoint), mirroring do_write_request.
|
||||
if checkpoint_ok {
|
||||
self.check_read_write_error(None);
|
||||
}
|
||||
|
||||
Ok(size)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Metrics ----
|
||||
@@ -7244,11 +7342,14 @@ mod tests {
|
||||
let (_offset, is_last) = v.binary_search_by_append_at_ns(newest_ns).unwrap();
|
||||
assert!(is_last, "precondition: the caller starts caught up");
|
||||
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (offset, is_last) = v.binary_search_by_append_at_ns(newest_ns).unwrap();
|
||||
@@ -7495,11 +7596,14 @@ mod tests {
|
||||
let written: Vec<u64> = (1..=3u64)
|
||||
.map(|id| write_test_needle(&mut v, id, b"scan"))
|
||||
.collect();
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let sb_size = v.super_block().block_size() as u64;
|
||||
@@ -8272,6 +8376,115 @@ mod tests {
|
||||
std::fs::metadata(v.file_name(".dat")).unwrap().len()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durable_delete_syncs_tombstone() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut writes: Vec<_> = (1..=2u64)
|
||||
.map(|id| (batch_needle(id, 0xaa, b"payload"), false))
|
||||
.collect();
|
||||
assert!(
|
||||
v.write_needles_grouped(&mut writes)
|
||||
.iter()
|
||||
.all(|r| r.is_ok())
|
||||
);
|
||||
assert_eq!(v.sync_counts_for_test(), (0, 0));
|
||||
|
||||
v.delete_needle(&mut batch_needle(1, 0xaa, b""), false)
|
||||
.unwrap();
|
||||
assert_eq!(v.sync_counts_for_test(), (0, 0));
|
||||
|
||||
let size = v
|
||||
.delete_needle(&mut batch_needle(2, 0xaa, b""), true)
|
||||
.unwrap();
|
||||
assert!(size.0 > 0);
|
||||
assert_eq!(v.sync_counts_for_test(), (1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_needle_failed_fsync_keeps_needle() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut writes = vec![(batch_needle(1, 0xaa, b"payload"), false)];
|
||||
assert!(
|
||||
v.write_needles_grouped(&mut writes)
|
||||
.iter()
|
||||
.all(|r| r.is_ok())
|
||||
);
|
||||
|
||||
v.fail_next_fsync_for_test(true);
|
||||
assert!(
|
||||
v.delete_needle(&mut batch_needle(1, 0xaa, b""), true)
|
||||
.is_err()
|
||||
);
|
||||
v.fail_next_fsync_for_test(false);
|
||||
|
||||
let mut n = Needle {
|
||||
id: NeedleId(1),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut n).unwrap();
|
||||
assert_eq!(n.data, b"payload");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durable_delete_syncs_after_unsynced_delete() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut writes = vec![(batch_needle(1, 0xaa, b"payload"), false)];
|
||||
assert!(
|
||||
v.write_needles_grouped(&mut writes)
|
||||
.iter()
|
||||
.all(|r| r.is_ok())
|
||||
);
|
||||
v.delete_needle(&mut batch_needle(1, 0xaa, b""), false)
|
||||
.unwrap();
|
||||
assert_eq!(v.sync_counts_for_test(), (0, 0));
|
||||
|
||||
let size = v
|
||||
.delete_needle(&mut batch_needle(1, 0xaa, b""), true)
|
||||
.unwrap();
|
||||
assert_eq!(size.0, 0);
|
||||
assert_eq!(v.sync_counts_for_test(), (1, 1));
|
||||
}
|
||||
|
||||
/// A durable delete flushes with the write guards dropped, so a read of
|
||||
/// the volume does not wait on its fsync.
|
||||
#[test]
|
||||
fn test_durable_delete_reads_during_flush() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let v = Arc::new(make_test_volume(tmp.path().to_str().unwrap()));
|
||||
let mut writes: Vec<_> = (1..=2u64)
|
||||
.map(|id| (batch_needle(id, 0xaa, b"payload"), false))
|
||||
.collect();
|
||||
assert!(
|
||||
v.write_needles_grouped(&mut writes)
|
||||
.iter()
|
||||
.all(|r| r.is_ok())
|
||||
);
|
||||
|
||||
let read_during_flush = Arc::new(AtomicBool::new(false));
|
||||
let (reader, flag) = (v.clone(), read_during_flush.clone());
|
||||
*v.on_flush_dat_for_test.lock().unwrap() = Some(Box::new(move |_: &Volume| {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let mut n = batch_needle(2, 0xaa, b"");
|
||||
let _ = tx.send(reader.read_needle(&mut n).is_ok());
|
||||
});
|
||||
let read = rx.recv_timeout(std::time::Duration::from_secs(5));
|
||||
flag.store(read == Ok(true), Ordering::Release);
|
||||
}));
|
||||
v.delete_needle(&mut batch_needle(1, 0xaa, b""), true)
|
||||
.unwrap();
|
||||
assert!(read_during_flush.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
/// A batch of durable writes shares one .dat sync and one .idx sync,
|
||||
/// where writing them one at a time syncs both files per needle.
|
||||
#[test]
|
||||
@@ -8729,6 +8942,77 @@ mod tests {
|
||||
assert_eq!(size, idx_len + 8);
|
||||
}
|
||||
|
||||
/// A delete whose .idx row fails has already dropped the needle
|
||||
/// from the map, so a retry would find nothing to delete and ack. The
|
||||
/// volume stops taking writes instead: nothing is appended past the
|
||||
/// tombstone, so a reload re-indexes it from the .dat tail rather than
|
||||
/// resurrect the needle.
|
||||
#[test]
|
||||
fn test_durable_delete_index_failure_refuses_writes() {
|
||||
for first_fsync in [true, false] {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
v.write_needle(&mut batch_needle(1, 0xaa, b"payload"), true, true)
|
||||
.unwrap();
|
||||
|
||||
{
|
||||
let mut io = v.io.write().unwrap();
|
||||
let nm = io.nm.as_mut().unwrap();
|
||||
let idx_len = nm.index_file_size();
|
||||
let file = OpenOptions::new()
|
||||
.write(true)
|
||||
.open(format!("{dir}/1.idx"))
|
||||
.unwrap();
|
||||
nm.set_idx_file(
|
||||
Box::new(TornIdxWriter {
|
||||
file,
|
||||
writes: 0,
|
||||
tear_at: 1,
|
||||
fail_truncates: false,
|
||||
}),
|
||||
idx_len,
|
||||
);
|
||||
}
|
||||
|
||||
assert!(
|
||||
v.delete_needle(&mut batch_needle(1, 0xaa, b""), first_fsync)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
v.write_needle(&mut batch_needle(2, 0xbb, b"later"), true, true)
|
||||
.is_err(),
|
||||
"nothing may be appended past the unindexed tombstone"
|
||||
);
|
||||
assert!(
|
||||
v.delete_needle(&mut batch_needle(1, 0xaa, b""), true)
|
||||
.is_err(),
|
||||
"a retry must not ack a delete whose .idx row never landed"
|
||||
);
|
||||
drop(v);
|
||||
|
||||
let reopened = match Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
&VolumeSpec::default(),
|
||||
) {
|
||||
Ok(v) => v,
|
||||
Err(e) => panic!("the volume does not reload: {e}"),
|
||||
};
|
||||
let mut got = Needle {
|
||||
id: NeedleId(1),
|
||||
..Needle::default()
|
||||
};
|
||||
let read = reopened.read_needle(&mut got);
|
||||
assert!(
|
||||
!(read.is_ok() && got.data == b"payload"),
|
||||
"the deleted needle must not come back on reload"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The I/O error streak after writing four durable needles, the ones in
|
||||
/// `failing` with a media error on their append, first one at a time and
|
||||
/// then as one grouped run. Also returns which grouped writes landed.
|
||||
@@ -8882,11 +9166,14 @@ mod tests {
|
||||
assert_eq!(v.file_count(), 1);
|
||||
|
||||
let deleted_size = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0xbb),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0xbb),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(deleted_size.0 > 0);
|
||||
// Additive-only: file_count stays at 1 after delete
|
||||
@@ -8934,12 +9221,12 @@ mod tests {
|
||||
|
||||
let n = pending_needle(1, b"late write");
|
||||
let pending = v.track_pending_write(&n);
|
||||
v.delete_needle(&mut pending_needle(1, b"")).unwrap();
|
||||
v.delete_needle(&mut pending_needle(1, b""), false).unwrap();
|
||||
let err = write_pending(&v, n, fsync, pending).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
|
||||
let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
assert!(matches!(err, VolumeError::NotFound));
|
||||
assert!(v.pending_writes.0.lock().unwrap().is_empty());
|
||||
|
||||
v.close();
|
||||
@@ -8955,7 +9242,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The unsynced write path takes a cancelled write back the same way.
|
||||
/// The unsynced write path refuses a cancelled write the same way.
|
||||
#[test]
|
||||
fn test_unsynced_write_cancelled_by_delete() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -8969,7 +9256,7 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
assert!(matches!(err, VolumeError::NotFound));
|
||||
}
|
||||
|
||||
/// A delete that finds a copy also cancels a retry of the same write that
|
||||
@@ -8984,13 +9271,38 @@ mod tests {
|
||||
.unwrap();
|
||||
let rewrite = pending_needle(1, b"retry");
|
||||
let pending = v.track_pending_write(&rewrite);
|
||||
v.delete_needle(&mut pending_needle(1, b"")).unwrap();
|
||||
v.delete_needle(&mut pending_needle(1, b""), false).unwrap();
|
||||
let err = write_pending(&v, rewrite, false, pending).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
}
|
||||
|
||||
/// A rewrite cancelled before it takes the volume locks appends nothing,
|
||||
/// so no unsynced row of it can outlive a durable delete after a crash.
|
||||
#[test]
|
||||
fn test_cancelled_rewrite_does_not_append() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let v = make_test_volume(tmp.path().to_str().unwrap());
|
||||
|
||||
v.write_needle(&mut pending_needle(1, b"first"), true, false)
|
||||
.unwrap();
|
||||
let before = std::fs::metadata(v.file_name(".dat")).unwrap().len();
|
||||
let rewrite = pending_needle(1, b"retry");
|
||||
let pending = v.track_pending_write(&rewrite);
|
||||
v.cancel_pending_writes(rewrite.id, rewrite.cookie);
|
||||
let err = write_pending(&v, rewrite, false, pending).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
assert_eq!(
|
||||
std::fs::metadata(v.file_name(".dat")).unwrap().len(),
|
||||
before
|
||||
);
|
||||
|
||||
v.delete_needle(&mut pending_needle(1, b""), true).unwrap();
|
||||
let err = v.read_needle(&mut pending_needle(1, b"")).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
}
|
||||
|
||||
/// Only writes already in flight and carrying the delete's cookie are
|
||||
/// cancelled.
|
||||
#[test]
|
||||
@@ -8999,7 +9311,7 @@ mod tests {
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
v.delete_needle(&mut pending_needle(1, b"")).unwrap();
|
||||
v.delete_needle(&mut pending_needle(1, b""), false).unwrap();
|
||||
let n = pending_needle(1, b"after the delete");
|
||||
let pending = v.track_pending_write(&n);
|
||||
write_pending(&v, n, false, pending).unwrap();
|
||||
@@ -9056,7 +9368,10 @@ mod tests {
|
||||
|
||||
let n = pending_needle(1, b"late write");
|
||||
let pending = v.track_pending_write(&n);
|
||||
v.cancel_pending_writes(n.id, n.cookie);
|
||||
let (id, cookie) = (n.id, n.cookie);
|
||||
*v.on_flush_dat_for_test.lock().unwrap() = Some(Box::new(move |v: &Volume| {
|
||||
v.cancel_pending_writes(id, cookie)
|
||||
}));
|
||||
let err = write_pending(&v, n, true, pending).unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
assert_eq!(v.sync_counts_for_test(), (2, 1));
|
||||
@@ -9091,11 +9406,14 @@ mod tests {
|
||||
};
|
||||
v.write_needle(&mut n, true, false).unwrap();
|
||||
}
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
}
|
||||
@@ -9149,11 +9467,14 @@ mod tests {
|
||||
// More than one tombstone: the scan has to walk back over the whole
|
||||
// run of them to reach a write.
|
||||
for i in [2u64, 3] {
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
@@ -9249,11 +9570,14 @@ mod tests {
|
||||
let (offset, _, _) = v.write_needle(&mut n, true, false).unwrap();
|
||||
written.push((offset, n.size));
|
||||
}
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
for (offset, size) in written {
|
||||
@@ -9341,11 +9665,14 @@ mod tests {
|
||||
// The delete lands inside the commit window: makeup_diff replays its
|
||||
// tombstone into the new .idx behind the write row the copy carried.
|
||||
v.compact_by_index(0, 0, |_| true).unwrap();
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
v.io.read().unwrap().last_write_deleted,
|
||||
@@ -9528,11 +9855,14 @@ mod tests {
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true, false).unwrap();
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
}
|
||||
@@ -9735,11 +10065,14 @@ mod tests {
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
write_test_needle(&mut v, 1, b"needle data");
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
// A later record keeps the tombstone mid-file rather than at the tail.
|
||||
write_test_needle(&mut v, 2, b"needle data");
|
||||
@@ -9836,11 +10169,14 @@ mod tests {
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
write_test_needle(&mut v, 1, b"needle data");
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x12345678),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
@@ -10540,7 +10876,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
v.delete_needle(&mut del).unwrap();
|
||||
v.delete_needle(&mut del, false).unwrap();
|
||||
// Additive-only: file_count stays at 3 after delete
|
||||
assert_eq!(v.file_count(), 3);
|
||||
assert_eq!(v.deleted_count(), 1);
|
||||
@@ -10966,11 +11302,14 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::ReadOnly(_)));
|
||||
|
||||
let deleted_size = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x1234),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x1234),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(deleted_size.0 > 0);
|
||||
assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
|
||||
@@ -11799,11 +12138,14 @@ mod tests {
|
||||
// Deletes are still allowed on a tiered volume and land in .idx.
|
||||
let idx_before = std::fs::metadata(&idx_path).unwrap().len();
|
||||
let deleted = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(3),
|
||||
cookie: Cookie(0x1234),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(3),
|
||||
cookie: Cookie(0x1234),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(deleted.0 > 0);
|
||||
assert_eq!(
|
||||
@@ -11989,11 +12331,14 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::ReadOnly(_)));
|
||||
|
||||
let deleted = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(deleted.0 > 0, "deletes must still land");
|
||||
v.sync_to_disk().unwrap();
|
||||
@@ -12030,11 +12375,14 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::ReadOnly(_)));
|
||||
|
||||
let deleted = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(deleted.0 > 0, "deletes must still land after restart");
|
||||
|
||||
@@ -12104,11 +12452,14 @@ mod tests {
|
||||
Some(NeedleMap::SortedFile(_))
|
||||
));
|
||||
for id in ids.clone() {
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: Cookie(id as u32),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: Cookie(id as u32),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let after_deletes = counters(&v);
|
||||
@@ -12156,11 +12507,14 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(v.no_write_flags().0);
|
||||
let err = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(err, VolumeError::ReadOnly(_)),
|
||||
@@ -12175,11 +12529,14 @@ mod tests {
|
||||
"upgrade to canDelete must attach the .idx writer"
|
||||
);
|
||||
let deleted = v
|
||||
.delete_needle(&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
})
|
||||
.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(deleted.0 > 0, "deletes must land once canDelete is set");
|
||||
}
|
||||
@@ -12830,7 +13187,7 @@ mod tests {
|
||||
cookie: Cookie(id as u32),
|
||||
..Needle::default()
|
||||
};
|
||||
v.delete_needle(&mut del).unwrap();
|
||||
v.delete_needle(&mut del, false).unwrap();
|
||||
}
|
||||
let expected_live = v.file_count() - v.deleted_count();
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ mod tests {
|
||||
for id in 1..=3 {
|
||||
v.write_needle(&mut needle(id), true, false).unwrap();
|
||||
}
|
||||
v.delete_needle(&mut needle(2)).unwrap();
|
||||
v.delete_needle(&mut needle(2), false).unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
let (want_count, want_deleted) = (v.file_count(), v.deleted_count());
|
||||
drop(v);
|
||||
|
||||
@@ -382,10 +382,13 @@ pub(crate) mod tests {
|
||||
|
||||
write_test_volume(dir, 6);
|
||||
let v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
@@ -417,10 +420,13 @@ pub(crate) mod tests {
|
||||
|
||||
write_test_volume(dir, 8);
|
||||
let v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
|
||||
@@ -505,10 +505,13 @@ mod tests {
|
||||
f.written.insert(*id, n.data.clone());
|
||||
}
|
||||
Op::Del(id) => {
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(*id),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(*id),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
Op::Empty(id) => {
|
||||
|
||||
@@ -105,7 +105,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
|
||||
// the delete cancels in-flight writes carrying this cookie
|
||||
n.Cookie = fidCookie
|
||||
}
|
||||
if size, err := vs.store.DeleteVolumeNeedle(volumeId, n); err != nil {
|
||||
if size, err := vs.store.DeleteVolumeNeedle(volumeId, n, false); err != nil {
|
||||
resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
|
||||
FileId: fid,
|
||||
Status: http.StatusInternalServerError,
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
func TestIncrementalCopyCarriesRemoteTierTombstone(t *testing.T) {
|
||||
store := tierUploadForDeletes(t, "000")
|
||||
const vid = needle.VolumeId(1)
|
||||
if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 2}); err != nil {
|
||||
if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 2}, false); err != nil {
|
||||
t.Fatalf("delete on tiered volume: %v", err)
|
||||
}
|
||||
source := store.GetVolume(vid)
|
||||
@@ -101,7 +101,7 @@ func TestIncrementalCopyCarriesRemoteTierTombstone(t *testing.T) {
|
||||
|
||||
// a delete landing after the copy rides the index stream alone: the run
|
||||
// appends just the tombstone row and the needle stays deleted
|
||||
if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 3}); err != nil {
|
||||
if _, err := store.DeleteVolumeNeedle(vid, &needle.Needle{Id: 3}, false); err != nil {
|
||||
t.Fatalf("second delete on tiered volume: %v", err)
|
||||
}
|
||||
if err := checkVolume.IncrementalBackup(serverAddress, dialOption); err != nil {
|
||||
|
||||
@@ -57,7 +57,7 @@ func TestTierUploadRefusesADatReplacedByACompactionCommit(t *testing.T) {
|
||||
written[i] = n.Data
|
||||
}
|
||||
for i := uint64(1); i <= 5; i++ {
|
||||
if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: types.Uint64ToNeedleId(i)}); err != nil {
|
||||
if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: types.Uint64ToNeedleId(i)}, false); err != nil {
|
||||
t.Fatalf("delete %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ func tierUploadForDeletes(t *testing.T, replication string) *storage.Store {
|
||||
// then, leaving the needles live in the index.
|
||||
func TestTierUploadAcceptsDeletesOnASingleCopyVolume(t *testing.T) {
|
||||
store := tierUploadForDeletes(t, "000")
|
||||
if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}); err != nil {
|
||||
if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}, false); err != nil {
|
||||
t.Fatalf("delete on the freshly tiered volume: %v", err)
|
||||
}
|
||||
if _, err := store.ReadVolumeNeedle(1, &needle.Needle{Id: 2}, nil, nil); err == nil {
|
||||
@@ -71,7 +71,7 @@ func TestTierUploadAcceptsDeletesOnASingleCopyVolume(t *testing.T) {
|
||||
// written before that copy would reach one replica only.
|
||||
func TestTierUploadKeepsRefusingDeletesOnAReplicatedVolume(t *testing.T) {
|
||||
store := tierUploadForDeletes(t, "001")
|
||||
if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}); err == nil {
|
||||
if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: 2}, false); err == nil {
|
||||
t.Fatalf("a replicated volume accepted a delete right after the tier upload")
|
||||
}
|
||||
}
|
||||
@@ -998,7 +998,7 @@ func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle, checkCook
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (Size, error) {
|
||||
func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle, fsync bool) (Size, error) {
|
||||
if v := s.findVolume(i); v != nil {
|
||||
v.noWriteLock.RLock()
|
||||
noWriteOrDelete := v.noWriteOrDelete
|
||||
@@ -1006,7 +1006,7 @@ func (s *Store) DeleteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (Size, e
|
||||
if noWriteOrDelete {
|
||||
return 0, fmt.Errorf("volume %d is read only", i)
|
||||
}
|
||||
return v.deleteNeedle2(n)
|
||||
return v.deleteNeedle2(n, fsync)
|
||||
}
|
||||
return 0, fmt.Errorf("volume %d not found on %s:%d", i, s.Ip, s.Port)
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ func TestCollectHeartbeatSizesOnlySurvivingVolumes(t *testing.T) {
|
||||
if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
|
||||
t.Fatalf("write needle: %v", err)
|
||||
}
|
||||
if _, err := v.deleteNeedle2(n); err != nil {
|
||||
if _, err := v.deleteNeedle2(n, false); err != nil {
|
||||
t.Fatalf("delete needle: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ func TestDeleteVolumeOnlyGarbage(t *testing.T) {
|
||||
t.Fatal("refused delete removed the volume")
|
||||
}
|
||||
|
||||
if _, err := store.DeleteVolumeNeedle(5, n); err != nil {
|
||||
if _, err := store.DeleteVolumeNeedle(5, n, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The shell sends both flags so a pre-upgrade server still refuses; either
|
||||
|
||||
@@ -73,7 +73,7 @@ func TestDeleteVolumeGuardChecksAllCopiesBeforeDeleting(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
_, _, _, err = copy1.writeNeedle2(n, false, false, false)
|
||||
require.NoError(t, err)
|
||||
_, err = copy0.deleteNeedle2(n)
|
||||
_, err = copy0.deleteNeedle2(n, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = store.DeleteVolume(vid, false, true, false)
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestMarkVolumeReadonlyCanDelete(t *testing.T) {
|
||||
|
||||
_, err = store.WriteVolumeNeedle(vid, newRandomNeedle(3), true, false)
|
||||
require.Error(t, err, "writes must be rejected")
|
||||
_, err = store.DeleteVolumeNeedle(vid, n1)
|
||||
_, err = store.DeleteVolumeNeedle(vid, n1, false)
|
||||
require.NoError(t, err, "deletes must still land")
|
||||
|
||||
store.Close()
|
||||
@@ -74,14 +74,14 @@ func TestMarkVolumeReadonlyCanDelete(t *testing.T) {
|
||||
|
||||
_, err = store2.WriteVolumeNeedle(vid, newRandomNeedle(4), true, false)
|
||||
require.Error(t, err, "writes must stay rejected after restart")
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n2)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n2, false)
|
||||
require.NoError(t, err, "deletes must still land after restart")
|
||||
|
||||
// Downgrading to plain readonly clears the canDelete flag.
|
||||
require.NoError(t, store2.MarkVolumeReadonly(vid, false, true))
|
||||
require.True(t, v2.noWriteOrDelete)
|
||||
require.False(t, v2.noWriteCanDelete)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n2)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n2, false)
|
||||
require.Error(t, err, "plain readonly must reject deletes")
|
||||
|
||||
// -writable clears the state again.
|
||||
@@ -110,13 +110,13 @@ func TestMarkVolumeReadonlyCanDelete_AfterReadonlyBoot(t *testing.T) {
|
||||
v := store2.GetVolume(vid)
|
||||
require.NotNil(t, v)
|
||||
require.True(t, v.noWriteOrDelete)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n, false)
|
||||
require.Error(t, err, "plain readonly must reject deletes")
|
||||
|
||||
require.NoError(t, store2.MarkVolumeReadonly(vid, true, true))
|
||||
require.False(t, v.noWriteOrDelete)
|
||||
require.True(t, v.noWriteCanDelete)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n)
|
||||
_, err = store2.DeleteVolumeNeedle(vid, n, false)
|
||||
require.NoError(t, err, "deletes must land once canDelete is set")
|
||||
store2.Close()
|
||||
}
|
||||
@@ -175,7 +175,7 @@ func TestMarkVolumeReadonlyCanDelete_CountsDeletesAfterRestart(t *testing.T) {
|
||||
require.NotNil(t, v)
|
||||
require.IsType(t, &SortedFileNeedleMap{}, v.nm)
|
||||
for _, n := range needles {
|
||||
_, err := store2.DeleteVolumeNeedle(vid, n)
|
||||
_, err := store2.DeleteVolumeNeedle(vid, n, false)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
counters := func(v *Volume) [4]uint64 {
|
||||
|
||||
@@ -28,7 +28,7 @@ func newVolumeWithGarbage(t *testing.T, live, deleted int) *Volume {
|
||||
}
|
||||
}
|
||||
for i := 1; i <= deleted; i++ {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(uint64(i))); err != nil {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(uint64(i)), false); err != nil {
|
||||
t.Fatalf("delete needle %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestLoad_MovedIdxDirectory_RebuildsIdx(t *testing.T) {
|
||||
t.Fatalf("seed write %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
if _, err := v.deleteNeedle2(newRandomNeedle(2)); err != nil {
|
||||
if _, err := v.deleteNeedle2(newRandomNeedle(2), false); err != nil {
|
||||
t.Fatalf("seed delete: %v", err)
|
||||
}
|
||||
wantCount, wantDeleted := v.nm.FileCount(), v.nm.DeletedCount()
|
||||
|
||||
@@ -60,14 +60,16 @@ func TestDeleteOfMissingNeedleCancelsQueuedWrite(t *testing.T) {
|
||||
v.dataFileSyncLock.Unlock()
|
||||
|
||||
require.ErrorIs(t, <-written, ErrorDeleted)
|
||||
requireNeedleDeleted(t, v, n.Id)
|
||||
_, err = v.readNeedle(&needle.Needle{Id: n.Id}, nil, nil)
|
||||
require.ErrorIs(t, err, ErrorNotFound)
|
||||
require.Empty(t, v.pendingWrites.byId)
|
||||
|
||||
v.Close()
|
||||
v, err = NewVolume(dir, dir, "", 1, NeedleMapInMemory, nil, nil, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
require.NoError(t, err)
|
||||
defer v.Close()
|
||||
requireNeedleDeleted(t, v, n.Id)
|
||||
_, err = v.readNeedle(&needle.Needle{Id: n.Id}, nil, nil)
|
||||
require.ErrorIs(t, err, ErrorNotFound)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -91,7 +93,7 @@ func TestDeleteCancelsQueuedRewrite(t *testing.T) {
|
||||
}()
|
||||
waitForPendingWrites(t, v, n.Id, 1)
|
||||
|
||||
_, err = v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie})
|
||||
_, err = v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}, false)
|
||||
require.NoError(t, err)
|
||||
requireNeedleDeleted(t, v, n.Id)
|
||||
v.asyncWorkerLock.Unlock()
|
||||
@@ -101,13 +103,45 @@ func TestDeleteCancelsQueuedRewrite(t *testing.T) {
|
||||
require.Empty(t, v.pendingWrites.byId)
|
||||
}
|
||||
|
||||
// A rewrite cancelled before it takes the volume locks appends nothing, so
|
||||
// no unsynced row of it can outlive a durable delete after a crash.
|
||||
func TestCancelledRewriteDoesNotAppend(t *testing.T) {
|
||||
v := newWorkerTestVolume(t)
|
||||
n := newRandomNeedle(1)
|
||||
_, _, _, err := v.writeNeedle2(n, true, false, false)
|
||||
require.NoError(t, err)
|
||||
before, _, err := v.DataBackend.GetStat()
|
||||
require.NoError(t, err)
|
||||
|
||||
rewrite := newRandomNeedle(1)
|
||||
rewrite.Cookie = n.Cookie
|
||||
v.dataFileSyncLock.Lock()
|
||||
written := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := v.writeNeedle2(rewrite, true, false, false)
|
||||
written <- err
|
||||
}()
|
||||
waitForPendingWrites(t, v, n.Id, 1)
|
||||
v.CancelPendingWrites(n.Id, n.Cookie)
|
||||
v.dataFileSyncLock.Unlock()
|
||||
|
||||
require.ErrorIs(t, <-written, ErrorDeleted)
|
||||
after, _, err := v.DataBackend.GetStat()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, before, after)
|
||||
|
||||
_, err = v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}, true)
|
||||
require.NoError(t, err)
|
||||
requireNeedleDeleted(t, v, n.Id)
|
||||
}
|
||||
|
||||
// Only writes already in flight are cancelled: writing the needle again after
|
||||
// the delete works as before.
|
||||
func TestWriteAfterDeleteIsNotCancelled(t *testing.T) {
|
||||
v := newWorkerTestVolume(t)
|
||||
n := newRandomNeedle(1)
|
||||
v.CancelPendingWrites(n.Id, n.Cookie)
|
||||
_, err := v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie})
|
||||
_, err := v.deleteNeedle2(&needle.Needle{Id: n.Id, Cookie: n.Cookie}, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, _, err = v.writeNeedle2(n, true, false, false)
|
||||
@@ -189,9 +223,14 @@ func TestWriteAfterCancelledWriteStaysLive(t *testing.T) {
|
||||
type syncedSizeBackend struct {
|
||||
backend.BackendStorageFile
|
||||
synced atomic.Int64
|
||||
onSync func()
|
||||
}
|
||||
|
||||
func (b *syncedSizeBackend) Sync() error {
|
||||
if f := b.onSync; f != nil {
|
||||
b.onSync = nil
|
||||
f()
|
||||
}
|
||||
size, _, err := b.GetStat()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -210,17 +249,9 @@ func TestTakenBackTombstoneIsSynced(t *testing.T) {
|
||||
v.DataBackend = synced
|
||||
|
||||
n := newRandomNeedle(1)
|
||||
v.dataFileSyncLock.Lock()
|
||||
written := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := v.writeNeedle2(n, true, true, isStopping)
|
||||
written <- err
|
||||
}()
|
||||
waitForPendingWrites(t, v, n.Id, 1)
|
||||
v.CancelPendingWrites(n.Id, n.Cookie)
|
||||
v.dataFileSyncLock.Unlock()
|
||||
|
||||
require.ErrorIs(t, <-written, ErrorDeleted)
|
||||
synced.onSync = func() { v.CancelPendingWrites(n.Id, n.Cookie) }
|
||||
_, _, _, err := v.writeNeedle2(n, true, true, isStopping)
|
||||
require.ErrorIs(t, err, ErrorDeleted)
|
||||
size, _, err := v.DataBackend.GetStat()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, size, synced.synced.Load(), "the tombstone was not flushed")
|
||||
|
||||
@@ -113,7 +113,7 @@ func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
|
||||
t.Fatalf("write needle %d: %v", i, err)
|
||||
}
|
||||
if i < 5 {
|
||||
size, err := v.deleteNeedle2(n)
|
||||
size, err := v.deleteNeedle2(n, false)
|
||||
if err != nil {
|
||||
t.Fatalf("delete needle %d: %v", i, err)
|
||||
}
|
||||
@@ -706,7 +706,7 @@ func TestReadNeedleDataIntoFollowsACompaction(t *testing.T) {
|
||||
t.Fatalf("write needle %d: %v", n.Id, err)
|
||||
}
|
||||
}
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(1)); err != nil {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(1), false); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -112,12 +112,12 @@ func TestDeleteVolumeNeedleRacesMarkTieredDeletable(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 100; i++ {
|
||||
_, _ = s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1000)})
|
||||
_, _ = s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1000)}, false)
|
||||
}
|
||||
<-done
|
||||
|
||||
// Positive control: once deletable, a delete of a live needle goes through.
|
||||
size, err := s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1)})
|
||||
size, err := s.DeleteVolumeNeedle(85, &needle.Needle{Id: types.Uint64ToNeedleId(1)}, false)
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, size)
|
||||
}
|
||||
@@ -48,7 +48,7 @@ func TestReconcileRollForwardMarkerOnly(t *testing.T) {
|
||||
}
|
||||
// Delete two so the compacted generation is strictly smaller than the source.
|
||||
for _, id := range []uint64{2, 5} {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(id)); err != nil {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(id), false); err != nil {
|
||||
t.Fatalf("delete %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
@@ -122,7 +122,7 @@ func TestReconcileRollForwardPartialRename(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, id := range []uint64{2, 5} {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(id)); err != nil {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(id), false); err != nil {
|
||||
t.Fatalf("delete %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +179,7 @@ func TestCommitCompactDeletionTailKeepsWritable(t *testing.T) {
|
||||
|
||||
// The sole change in the commit window is a deletion, so makeupDiff appends
|
||||
// its tombstone last, putting it at the .dat tail.
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(3)); err != nil {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(3), false); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
|
||||
@@ -301,7 +301,7 @@ func TestCommitCompactMakeupDiffFailureKeepsVolumeLoaded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
// Moves needle 3 in the compacted copy, so reads through its map would miss.
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(1)); err != nil {
|
||||
if _, err := v.deleteNeedle2(newEmptyNeedle(1), false); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
delete(written, 1)
|
||||
@@ -733,7 +733,7 @@ func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) {
|
||||
if rand.Float64() < 0.03 {
|
||||
toBeDeleted := rand.Intn(i) + 1
|
||||
oldNeedle := newEmptyNeedle(uint64(toBeDeleted))
|
||||
v.deleteNeedle2(oldNeedle)
|
||||
v.deleteNeedle2(oldNeedle, false)
|
||||
// println("deleted file", toBeDeleted)
|
||||
infos[toBeDeleted-1] = &needleInfo{
|
||||
size: 0,
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestCompactRefusesTieredVolume(t *testing.T) {
|
||||
defer v.Close()
|
||||
require.True(t, v.HasRemoteFile())
|
||||
for i := 1; i <= 4; i++ {
|
||||
_, err := v.deleteNeedle2(newRandomNeedle(uint64(i)))
|
||||
_, err := v.deleteNeedle2(newRandomNeedle(uint64(i)), false)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
garbage := v.garbageLevel()
|
||||
@@ -74,7 +74,7 @@ func testCommitCompactRefusesVolumeTieredDuringCopy(t *testing.T, kind NeedleMap
|
||||
written[i] = n.Data
|
||||
}
|
||||
for i := uint64(1); i <= 3; i++ {
|
||||
_, err := v.deleteNeedle2(newEmptyNeedle(i))
|
||||
_, err := v.deleteNeedle2(newEmptyNeedle(i), false)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, v.CompactByIndex(nil))
|
||||
|
||||
@@ -163,11 +163,17 @@ func removeVolumeFiles(filename string, keepVif bool) {
|
||||
// asyncRequestAppend queues a request for the batch worker, starting it on the
|
||||
// first one. It reports false for a destroyed volume, so the caller writes
|
||||
// inline rather than wait on a worker that will never answer.
|
||||
func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) bool {
|
||||
func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) (queued bool) {
|
||||
requests := v.startWorker()
|
||||
if requests == nil {
|
||||
return false
|
||||
}
|
||||
// stopWorker may close the channel between startWorker and the send.
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
queued = false
|
||||
}
|
||||
}()
|
||||
requests <- request
|
||||
return true
|
||||
}
|
||||
@@ -344,6 +350,11 @@ func (v *Volume) appendWriteRequest(n *needle.Needle, checkCookie bool, fsync bo
|
||||
if v.IsReadOnly() {
|
||||
return 0, 0, false, fmt.Errorf("volume %d is %w", v.Id, ErrorReadOnly)
|
||||
}
|
||||
// A write cancelled before it got here must not append: its delete may
|
||||
// already be durable, and an unsynced row could outlive its tombstone.
|
||||
if v.isCancelledWrite(n) {
|
||||
return 0, 0, false, fmt.Errorf("needle %s: %w", needle.NewFileIdFromNeedle(v.Id, n), ErrorDeleted)
|
||||
}
|
||||
if v.isFileUnchanged(n) {
|
||||
size = Size(n.DataSize)
|
||||
isUnchanged = true
|
||||
@@ -438,7 +449,7 @@ func (v *Volume) publishWriteRequest(n *needle.Needle, offset uint64, fsync bool
|
||||
return
|
||||
}
|
||||
|
||||
func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
|
||||
func (v *Volume) syncDelete(n *needle.Needle, fsync bool) (Size, error) {
|
||||
// glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
|
||||
v.dataFileSyncLock.Lock()
|
||||
defer v.dataFileSyncLock.Unlock()
|
||||
@@ -456,28 +467,66 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return v.doDeleteRequest(n)
|
||||
if !fsync {
|
||||
return v.doDeleteRequest(n)
|
||||
}
|
||||
|
||||
end, _, statErr := v.DataBackend.GetStat()
|
||||
if statErr != nil {
|
||||
return 0, fmt.Errorf("cannot read current volume position: %v", statErr)
|
||||
}
|
||||
lastAppendAtNs := v.lastAppendAtNs
|
||||
|
||||
size, offset, tombstoned, err := v.appendDeleteRequest(n)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
// Same order as a durable write: the tombstone is down before the map
|
||||
// drops the needle, and its .idx row before the delete is acknowledged.
|
||||
// An already-deleted needle still syncs: an earlier delete may have skipped it.
|
||||
backend := v.DataBackend
|
||||
v.dataFileAccessLock.Unlock()
|
||||
syncErr := backend.Sync()
|
||||
v.dataFileAccessLock.Lock()
|
||||
if syncErr != nil {
|
||||
v.checkReadWriteError(syncErr)
|
||||
v.lastAppendAtNs = lastAppendAtNs
|
||||
if !v.HasRemoteFile() {
|
||||
if te := v.DataBackend.Truncate(end); te != nil {
|
||||
v.markIoUnavailable(fmt.Errorf("truncate %s back to %d after a failed fsync: %w",
|
||||
v.DataBackend.Name(), end, te))
|
||||
}
|
||||
}
|
||||
return 0, syncErr
|
||||
}
|
||||
if tombstoned {
|
||||
if err = v.publishDeleteRequest(n, offset); err != nil {
|
||||
return size, err
|
||||
}
|
||||
}
|
||||
v.dataFileAccessLock.Unlock()
|
||||
err = v.syncIndex()
|
||||
v.dataFileAccessLock.Lock()
|
||||
return size, err
|
||||
}
|
||||
|
||||
func (v *Volume) deleteNeedle2(n *needle.Needle) (Size, error) {
|
||||
// deleteNeedle2 acknowledges a durable delete only once its tombstone and .idx
|
||||
// row are on disk, through the batch worker like a durable write.
|
||||
func (v *Volume) deleteNeedle2(n *needle.Needle, fsync bool) (Size, error) {
|
||||
v.CancelPendingWrites(n.Id, n.Cookie)
|
||||
|
||||
// todo: delete info is always appended no fsync, it may need fsync in future
|
||||
fsync := false
|
||||
|
||||
if !fsync {
|
||||
return v.syncDelete(n)
|
||||
} else {
|
||||
asyncRequest := needle.NewAsyncRequest(n, false)
|
||||
asyncRequest.ActualSize = needle.GetActualSize(0, v.Version())
|
||||
|
||||
if !v.asyncRequestAppend(asyncRequest) {
|
||||
return v.syncDelete(n)
|
||||
}
|
||||
_, size, _, err := asyncRequest.WaitComplete()
|
||||
|
||||
return Size(size), err
|
||||
return v.syncDelete(n, false)
|
||||
}
|
||||
asyncRequest := needle.NewAsyncRequest(n, false)
|
||||
asyncRequest.ActualSize = needle.GetActualSize(0, v.Version())
|
||||
|
||||
if !v.asyncRequestAppend(asyncRequest) {
|
||||
return v.syncDelete(n, true)
|
||||
}
|
||||
_, size, _, err := asyncRequest.WaitComplete()
|
||||
|
||||
return Size(size), err
|
||||
}
|
||||
|
||||
func (v *Volume) doDeleteRequest(n *needle.Needle) (Size, error) {
|
||||
@@ -517,6 +566,11 @@ func (v *Volume) appendDeleteRequest(n *needle.Needle) (size Size, offset uint64
|
||||
|
||||
func (v *Volume) publishDeleteRequest(n *needle.Needle, offset uint64) error {
|
||||
if err := v.nm.Delete(n.Id, ToOffset(int64(offset))); err != nil {
|
||||
// The map already dropped the needle, so a retry, durable or not, would
|
||||
// ack without the missing .idx row. Stop taking writes.
|
||||
v.noWriteLock.Lock()
|
||||
v.noWriteOrDelete = true
|
||||
v.noWriteLock.Unlock()
|
||||
return err
|
||||
}
|
||||
if n.Id == v.lastWriteNeedleKey {
|
||||
@@ -677,6 +731,7 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) {
|
||||
} else if stagedDeleteOk[i] {
|
||||
if perr := v.publishDeleteRequest(run[i].N, stagedDeleteOffsets[i]); perr != nil {
|
||||
run[i].UpdateResult(0, 0, false, perr)
|
||||
refused = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -769,7 +824,7 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest {
|
||||
if request == nil {
|
||||
break
|
||||
}
|
||||
if MaxPossibleVolumeSize < v.ContentSize()+uint64(currentBytesToWrite+request.ActualSize) {
|
||||
if request.IsWriteRequest && MaxPossibleVolumeSize < v.ContentSize()+uint64(currentBytesToWrite+request.ActualSize) {
|
||||
request.Complete(0, 0, false,
|
||||
fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.ContentSize()))
|
||||
break
|
||||
|
||||
@@ -2,6 +2,8 @@ package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
@@ -608,3 +610,135 @@ func TestStoreWriteVolumeNeedleStaysDurableWhileStopping(t *testing.T) {
|
||||
require.False(t, isUnchanged)
|
||||
require.Equal(t, 1, counting.syncCount, "the accepted write must actually be on disk")
|
||||
}
|
||||
|
||||
func TestDurableDeleteSyncsTombstone(t *testing.T) {
|
||||
v, counting := newCountingVolume(t)
|
||||
|
||||
for id := uint64(1); id <= 2; id++ {
|
||||
_, _, _, err := v.writeNeedle2(fixedNeedle(id, "payload"), true, false, false)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Equal(t, 0, counting.syncCount)
|
||||
|
||||
_, err := v.deleteNeedle2(fixedNeedle(1, ""), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, counting.syncCount, "a delete that did not ask for fsync must not pay for one")
|
||||
|
||||
size, err := v.deleteNeedle2(fixedNeedle(2, ""), true)
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, size)
|
||||
require.Equal(t, 1, counting.syncCount, "a durable delete must flush its tombstone before the ack")
|
||||
|
||||
reloaded := reopenCountingVolume(t, v)
|
||||
nv, found := reloaded.nm.Get(types.Uint64ToNeedleId(2))
|
||||
require.True(t, !found || nv.Size.IsDeleted(), "the durable delete must survive a reload")
|
||||
}
|
||||
|
||||
func TestDurableDeleteTakesBackTombstoneWhenFsyncFails(t *testing.T) {
|
||||
v, counting := newCountingVolume(t)
|
||||
|
||||
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, true, true)
|
||||
require.NoError(t, err)
|
||||
before, _, err := v.DataBackend.GetStat()
|
||||
require.NoError(t, err)
|
||||
syncs := counting.syncCount
|
||||
|
||||
counting.syncErr = errors.New("disk went away")
|
||||
_, err = v.syncDelete(fixedNeedle(1, ""), true)
|
||||
require.Error(t, err, "a delete whose fsync failed must not be acked")
|
||||
require.Equal(t, syncs+1, counting.syncCount)
|
||||
|
||||
after, _, err := v.DataBackend.GetStat()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, before, after, "the unflushed tombstone should be taken back off the .dat")
|
||||
nv, found := v.nm.Get(types.Uint64ToNeedleId(1))
|
||||
require.True(t, found)
|
||||
require.False(t, nv.Size.IsDeleted(), "the needle must stay live when its delete was not acked")
|
||||
}
|
||||
|
||||
func TestDurableDeleteOnFullVolume(t *testing.T) {
|
||||
v, counting := newCountingVolume(t)
|
||||
|
||||
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, false, false)
|
||||
require.NoError(t, err)
|
||||
atomic.StoreUint64(&v.nm.(*NeedleMap).FileByteCounter, types.MaxPossibleVolumeSize)
|
||||
|
||||
_, err = v.deleteNeedle2(fixedNeedle(1, ""), true)
|
||||
require.NoError(t, err, "a full volume must still take a durable delete")
|
||||
require.Equal(t, 1, counting.syncCount)
|
||||
nv, found := v.nm.Get(types.Uint64ToNeedleId(1))
|
||||
require.True(t, !found || nv.Size.IsDeleted())
|
||||
}
|
||||
|
||||
func TestDurableDeleteSyncsAfterUnsyncedDelete(t *testing.T) {
|
||||
v, counting := newCountingVolume(t)
|
||||
|
||||
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, false, false)
|
||||
require.NoError(t, err)
|
||||
_, err = v.syncDelete(fixedNeedle(1, ""), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, counting.syncCount)
|
||||
|
||||
size, err := v.syncDelete(fixedNeedle(1, ""), true)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, size)
|
||||
require.Equal(t, 1, counting.syncCount, "an earlier unsynced delete must still be flushed before a durable ack")
|
||||
}
|
||||
|
||||
func TestAsyncRequestAppendRefusesAfterChannelClosed(t *testing.T) {
|
||||
v, _ := newCountingVolume(t)
|
||||
|
||||
closed := make(chan *needle.AsyncRequest)
|
||||
close(closed)
|
||||
v.asyncWorkerLock.Lock()
|
||||
v.asyncRequestsChan = closed
|
||||
v.asyncWorkerLock.Unlock()
|
||||
|
||||
require.False(t, v.asyncRequestAppend(needle.NewAsyncRequest(fixedNeedle(1, "payload"), true)),
|
||||
"a request racing stopWorker must be refused, not panic")
|
||||
}
|
||||
|
||||
// A durable delete whose .idx row fails has already dropped the needle from
|
||||
// the map, so a retry would find nothing to delete and ack. The volume stops
|
||||
// taking writes instead: nothing is appended past the tombstone, so a reload
|
||||
// re-indexes it from the .dat tail rather than resurrect the needle.
|
||||
func TestDurableDeleteIndexFailureStopsWrites(t *testing.T) {
|
||||
durableBatch := func(v *Volume, n *needle.Needle) (types.Size, error) { return v.deleteNeedle2(n, true) }
|
||||
durableInline := func(v *Volume, n *needle.Needle) (types.Size, error) { return v.syncDelete(n, true) }
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
first func(*Volume, *needle.Needle) (types.Size, error)
|
||||
delete func(*Volume, *needle.Needle) (types.Size, error)
|
||||
}{
|
||||
{"batch", durableBatch, durableBatch},
|
||||
{"inline", durableInline, durableInline},
|
||||
{"non-durable then durable retry", func(v *Volume, n *needle.Needle) (types.Size, error) { return v.deleteNeedle2(n, false) }, durableBatch},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
v, _ := newCountingVolume(t)
|
||||
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, true, true)
|
||||
require.NoError(t, err)
|
||||
|
||||
nm := v.nm.(*NeedleMap)
|
||||
indexFile := nm.indexFile
|
||||
readOnlyIndex, err := os.Open(indexFile.Name())
|
||||
require.NoError(t, err)
|
||||
defer readOnlyIndex.Close()
|
||||
nm.indexFile = readOnlyIndex
|
||||
_, err = tc.first(v, fixedNeedle(1, ""))
|
||||
nm.indexFile = indexFile
|
||||
require.Error(t, err)
|
||||
|
||||
_, _, _, err = v.writeNeedle2(fixedNeedle(2, "later"), true, true, true)
|
||||
require.Error(t, err, "nothing may be appended past the unindexed tombstone")
|
||||
_, err = tc.delete(v, fixedNeedle(1, ""))
|
||||
require.Error(t, err, "a retry must not ack a delete whose .idx row never landed")
|
||||
|
||||
reloaded := reopenCountingVolume(t, v)
|
||||
readBack := fixedNeedle(1, "")
|
||||
_, err = reloaded.readNeedle(readBack, nil, nil)
|
||||
require.False(t, err == nil && string(readBack.Data) == "payload",
|
||||
"the deleted needle must not come back on reload")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -267,7 +267,8 @@ func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOp
|
||||
}(time.Now())
|
||||
}
|
||||
|
||||
size, err = store.DeleteVolumeNeedle(volumeId, n)
|
||||
fsync := r.FormValue("fsync") == "true"
|
||||
size, err = store.DeleteVolumeNeedle(volumeId, n, fsync)
|
||||
if err != nil {
|
||||
glog.V(0).Infoln("delete error:", err)
|
||||
return
|
||||
@@ -279,7 +280,11 @@ func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOp
|
||||
if replicaCount > 0 { //send to other replica locations
|
||||
// background, not r.Context(): a client disconnect must not orphan replica deletes
|
||||
if err = DistributedOperation(context.Background(), remoteLocations, func(ctx context.Context, location operation.Location) error {
|
||||
url, normalizeErr := util_http.GetGlobalHttpClient().NormalizeHttpScheme(location.Url + r.URL.Path + "?type=replicate")
|
||||
query := "?type=replicate"
|
||||
if fsync {
|
||||
query += "&fsync=true"
|
||||
}
|
||||
url, normalizeErr := util_http.GetGlobalHttpClient().NormalizeHttpScheme(location.Url + r.URL.Path + query)
|
||||
if normalizeErr != nil {
|
||||
return normalizeErr
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user