storage: copy an empty needle written during compaction in makeupDiff (#12796)

* storage: copy an empty needle written during compaction in makeupDiff

A size-0 index row is a live empty needle, not a delete. makeupDiff
grouped it with tombstones and appended a made-up delete marker, so
the compacted .dat lost the record's cookie and held a delete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): copy an empty needle written during compaction in makeup_diff

A size-0 index row is a live empty needle. makeup_diff wrote a made-up
delete marker and a tombstone row for it, so the needle read as not
found after the commit. Copy the record like any other live row.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: fail makeupDiff when the replayed needle write fails

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): check the compacted empty needle's id and cookie in the test

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-19 15:26:20 -08:00
1 parent ba1bfa816c
commit 4a5dbe4c6f
3 files changed
+97 -3

No files matched your search

+42 -1
View File
@@ -6372,7 +6372,7 @@ impl Volume {
let old_dat_file = File::open(&old_dat_path)?;
for (key, (needle_offset, size)) in incremented_entries {
if !needle_offset.is_zero() && !size.is_deleted() && size.0 > 0 {
if !needle_offset.is_zero() && !size.is_deleted() {
let actual_size = crate::storage::needle::needle::get_actual_size(size, version);
let mut blob = vec![0u8; actual_size as usize];
@@ -10992,6 +10992,47 @@ mod tests {
assert_eq!(got.data, b"late-write");
}
#[test]
fn test_commit_compact_keeps_empty_needle_written_during_compaction() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let v = make_test_volume(dir);
for i in 1..=3u64 {
let mut n = Needle {
id: NeedleId(i),
cookie: Cookie(i as u32),
data: format!("data-{}", i).into_bytes(),
data_size: format!("data-{}", i).len() as u32,
..Needle::default()
};
v.write_needle(&mut n, true, false).unwrap();
}
v.compact_by_index(0, 0, |_| true).unwrap();
let mut empty = Needle {
id: NeedleId(10),
cookie: Cookie(0x0abc_def0),
..Needle::default()
};
v.write_needle(&mut empty, true, false).unwrap();
v.commit_compact().unwrap();
let nv = {
let io = v.io.read().unwrap();
io.nm.as_ref().unwrap().get(NeedleId(10)).unwrap().unwrap()
};
assert_eq!(nv.size, Size(0));
let mut record = Needle::default();
v.read_needle_data_at(&mut record, nv.offset.to_actual_offset(), nv.size)
.unwrap();
assert_eq!(record.id, NeedleId(10));
assert_eq!(record.cookie, Cookie(0x0abc_def0));
assert!(record.data.is_empty());
}
/// Mirrors Go's TestConcurrentWriteCrossesOffsetBoundary (issue #11410):
/// a replayed write's index entry must encode all offset bytes even when
/// old and new offsets sit in different 32 GiB ranges. Unix-only: Windows
+4 -2
View File
@@ -593,7 +593,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
}
}
//updated needle
if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size.IsValid() {
if !increIdxEntry.offset.IsZero() && !increIdxEntry.size.IsDeleted() {
//even the needle cache in memory is hit, the need_bytes is correct
glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size)
var needleBytes []byte
@@ -602,7 +602,9 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
v.checkReadWriteError(err)
return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %w", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err)
}
dstDatBackend.Write(needleBytes)
if _, err = dstDatBackend.Write(needleBytes); err != nil {
return fmt.Errorf("write %s key %d failed: %w", dst.Name(), key, err)
}
idxEntryBytes = needle_map.ToBytes(key, ToOffset(offset), increIdxEntry.size)
} else { //deleted needle
//fakeDelNeedle's default Data field is nil
+51
View File
@@ -204,6 +204,57 @@ func TestCommitCompactDeletionTailKeepsWritable(t *testing.T) {
}
}
func TestCommitCompactKeepsEmptyNeedleWrittenDuringCompaction(t *testing.T) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
defer v.Close()
for i := uint64(1); i <= 3; i++ {
if _, _, _, err := v.writeNeedle2(newRandomNeedle(i), true, false, false); err != nil {
t.Fatalf("write %d: %v", i, err)
}
}
if err := v.CompactByIndex(nil); err != nil {
t.Fatalf("compact: %v", err)
}
empty := newEmptyNeedle(10)
empty.Cookie = 0x0abcdef0
if _, _, _, err := v.writeNeedle2(empty, true, false, false); err != nil {
t.Fatalf("write empty needle: %v", err)
}
if _, _, _, err := v.writeNeedle2(newRandomNeedle(11), true, false, false); err != nil {
t.Fatalf("write needle 11: %v", err)
}
if err := v.CommitCompact(); err != nil {
t.Fatalf("commit compact: %v", err)
}
nv, ok := v.nm.Get(empty.Id)
if !ok || nv.Size != 0 {
t.Fatalf("empty needle index entry after commit: %+v found=%v, want size 0", nv, ok)
}
record, _, _, err := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToActualOffset())
if err != nil {
t.Fatalf("read compacted record header: %v", err)
}
if record.Id != empty.Id || record.Cookie != empty.Cookie {
t.Fatalf("compacted record is id %d cookie %x, want the written id %d cookie %x", record.Id, record.Cookie, empty.Id, empty.Cookie)
}
read := newEmptyNeedle(10)
read.Cookie = empty.Cookie
if _, err := v.readNeedle(read, nil, nil); err != nil {
t.Fatalf("read empty needle after commit: %v", err)
}
}
// A second vacuum must not take a running compaction as its own: reporting
// success would let it commit or delete the other run's .cpd/.cpx.
func TestCompactionInProgressRefusesOtherVacuumSteps(t *testing.T) {