mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
storage: copy an empty needle written during compaction in makeupDiff (#12796)
* storage: copy an empty needle written during compaction in makeupDiff A size-0 index row is a live empty needle, not a delete. makeupDiff grouped it with tombstones and appended a made-up delete marker, so the compacted .dat lost the record's cookie and held a delete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): copy an empty needle written during compaction in makeup_diff A size-0 index row is a live empty needle. makeup_diff wrote a made-up delete marker and a tombstone row for it, so the needle read as not found after the commit. Copy the record like any other live row. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: fail makeupDiff when the replayed needle write fails Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): check the compacted empty needle's id and cookie in the test Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ba1bfa816c
commit
4a5dbe4c6f
3 files changed
+97
-3
No files matched your search
@@ -6372,7 +6372,7 @@ impl Volume {
|
||||
let old_dat_file = File::open(&old_dat_path)?;
|
||||
|
||||
for (key, (needle_offset, size)) in incremented_entries {
|
||||
if !needle_offset.is_zero() && !size.is_deleted() && size.0 > 0 {
|
||||
if !needle_offset.is_zero() && !size.is_deleted() {
|
||||
let actual_size = crate::storage::needle::needle::get_actual_size(size, version);
|
||||
let mut blob = vec![0u8; actual_size as usize];
|
||||
|
||||
@@ -10992,6 +10992,47 @@ mod tests {
|
||||
assert_eq!(got.data, b"late-write");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_commit_compact_keeps_empty_needle_written_during_compaction() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
for i in 1..=3u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("data-{}", i).into_bytes(),
|
||||
data_size: format!("data-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true, false).unwrap();
|
||||
}
|
||||
|
||||
v.compact_by_index(0, 0, |_| true).unwrap();
|
||||
|
||||
let mut empty = Needle {
|
||||
id: NeedleId(10),
|
||||
cookie: Cookie(0x0abc_def0),
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut empty, true, false).unwrap();
|
||||
|
||||
v.commit_compact().unwrap();
|
||||
|
||||
let nv = {
|
||||
let io = v.io.read().unwrap();
|
||||
io.nm.as_ref().unwrap().get(NeedleId(10)).unwrap().unwrap()
|
||||
};
|
||||
assert_eq!(nv.size, Size(0));
|
||||
let mut record = Needle::default();
|
||||
v.read_needle_data_at(&mut record, nv.offset.to_actual_offset(), nv.size)
|
||||
.unwrap();
|
||||
assert_eq!(record.id, NeedleId(10));
|
||||
assert_eq!(record.cookie, Cookie(0x0abc_def0));
|
||||
assert!(record.data.is_empty());
|
||||
}
|
||||
|
||||
/// Mirrors Go's TestConcurrentWriteCrossesOffsetBoundary (issue #11410):
|
||||
/// a replayed write's index entry must encode all offset bytes even when
|
||||
/// old and new offsets sit in different 32 GiB ranges. Unix-only: Windows
|
||||
|
||||
@@ -593,7 +593,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
|
||||
}
|
||||
}
|
||||
//updated needle
|
||||
if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size.IsValid() {
|
||||
if !increIdxEntry.offset.IsZero() && !increIdxEntry.size.IsDeleted() {
|
||||
//even the needle cache in memory is hit, the need_bytes is correct
|
||||
glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size)
|
||||
var needleBytes []byte
|
||||
@@ -602,7 +602,9 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
|
||||
v.checkReadWriteError(err)
|
||||
return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %w", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err)
|
||||
}
|
||||
dstDatBackend.Write(needleBytes)
|
||||
if _, err = dstDatBackend.Write(needleBytes); err != nil {
|
||||
return fmt.Errorf("write %s key %d failed: %w", dst.Name(), key, err)
|
||||
}
|
||||
idxEntryBytes = needle_map.ToBytes(key, ToOffset(offset), increIdxEntry.size)
|
||||
} else { //deleted needle
|
||||
//fakeDelNeedle's default Data field is nil
|
||||
|
||||
@@ -204,6 +204,57 @@ func TestCommitCompactDeletionTailKeepsWritable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommitCompactKeepsEmptyNeedleWrittenDuringCompaction(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("volume creation: %v", err)
|
||||
}
|
||||
defer v.Close()
|
||||
|
||||
for i := uint64(1); i <= 3; i++ {
|
||||
if _, _, _, err := v.writeNeedle2(newRandomNeedle(i), true, false, false); err != nil {
|
||||
t.Fatalf("write %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := v.CompactByIndex(nil); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
empty := newEmptyNeedle(10)
|
||||
empty.Cookie = 0x0abcdef0
|
||||
if _, _, _, err := v.writeNeedle2(empty, true, false, false); err != nil {
|
||||
t.Fatalf("write empty needle: %v", err)
|
||||
}
|
||||
if _, _, _, err := v.writeNeedle2(newRandomNeedle(11), true, false, false); err != nil {
|
||||
t.Fatalf("write needle 11: %v", err)
|
||||
}
|
||||
|
||||
if err := v.CommitCompact(); err != nil {
|
||||
t.Fatalf("commit compact: %v", err)
|
||||
}
|
||||
|
||||
nv, ok := v.nm.Get(empty.Id)
|
||||
if !ok || nv.Size != 0 {
|
||||
t.Fatalf("empty needle index entry after commit: %+v found=%v, want size 0", nv, ok)
|
||||
}
|
||||
record, _, _, err := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToActualOffset())
|
||||
if err != nil {
|
||||
t.Fatalf("read compacted record header: %v", err)
|
||||
}
|
||||
if record.Id != empty.Id || record.Cookie != empty.Cookie {
|
||||
t.Fatalf("compacted record is id %d cookie %x, want the written id %d cookie %x", record.Id, record.Cookie, empty.Id, empty.Cookie)
|
||||
}
|
||||
|
||||
read := newEmptyNeedle(10)
|
||||
read.Cookie = empty.Cookie
|
||||
if _, err := v.readNeedle(read, nil, nil); err != nil {
|
||||
t.Fatalf("read empty needle after commit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A second vacuum must not take a running compaction as its own: reporting
|
||||
// success would let it commit or delete the other run's .cpd/.cpx.
|
||||
func TestCompactionInProgressRefusesOtherVacuumSteps(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user