mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
* volume: fsync a needle delete that asks for it A delete now takes fsync like a write: the tombstone is synced before the needle map drops the needle, and the .idx row before the delete is acknowledged. A failed .dat sync takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: honor fsync=true on DELETE and forward it to replicas Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): fsync a needle delete that asks for it Mirrors the Go volume server: DELETE with fsync=true flushes the tombstone before the index drops the needle and the .idx before the ack, and a failed .dat flush takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: let a durable delete through on a full volume A queued delete hit the batch worker's volume size check, which a tombstone append is exempt from inline, so fsync=true deletes failed once the volume reached its hard limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: sync a durable delete of an already-deleted needle An earlier delete without fsync may have dropped the needle with its tombstone still unsynced, so the inline path syncs before acking anyway, as the batch worker already does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a batch request that races the worker stopping stopWorker can close the batch channel between startWorker handing it out and the send, which panicked. The request now falls back to the inline path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): sync a durable delete of an already-deleted needle Mirrors the Go volume server: an earlier delete without fsync may have dropped the needle with its tombstone still unsynced. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): run a durable delete on a blocking thread Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: keep syncDelete's signature for existing callers Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a write a delete cancelled before it appends A write cancelled while still queued behind the volume locks used to append, publish and then tombstone itself without a sync, after its delete may already have been acknowledged as durable. A crash could keep the row and lose the tombstone. Refuse it before the append. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): refuse a write a delete cancelled before it appends Mirrors the Go change: a write cancelled while still queued behind the volume locks no longer appends, publishes and tombstones itself without a sync after its delete may already be durable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): drop the write guards for a durable delete's flushes The durable delete held the write lease and the io write guard across both fsyncs, so reads of the volume waited on them. Append and publish the tombstone under the guards and flush with only the io read side, as write_synced_run and the Go path already do. do_delete_request keeps its signature. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when a durable delete fails to index The needle map drops the needle before it appends the .idx row, so after a failed append a durable retry found nothing to delete and acked, and a later write could bury the unindexed tombstone so a reload brought the needle back. Fence the volume the way a failed durable write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when a durable delete fails to index Rust counterpart of the Go change: the needle map drops the needle before it appends the .idx row, so a failed append left a durable retry nothing to delete, and a later write could bury the unindexed tombstone. publish_delete_request now fences the volume for a durable delete, as publish_write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when any delete fails to index A non-durable delete whose .idx append failed left the volume writable with the needle already dropped from the map, so a later durable retry found nothing to delete and acknowledged without the missing row. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when any delete fails to index Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: take the fsync flag on the existing delete functions DeleteVolumeNeedle, deleteNeedle2 and syncDelete take fsync instead of gaining *WithFsync variants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): take the fsync flag on the existing delete functions delete_volume_needle and delete_needle take fsync instead of gaining *_with_fsync variants; the durable path and the deletable checks fold back into delete_needle, and the cancelled-write check mirrors Go's isCancelledWrite. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
232 lines
8.6 KiB
Go
232 lines
8.6 KiB
Go
package storage
|
|
|
|
import (
|
|
"errors"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
// newVolumeWithGarbage writes live+deleted needles and deletes the first
|
|
// deleted ones, so the volume carries that much garbage on disk.
|
|
func newVolumeWithGarbage(t *testing.T, live, deleted int) *Volume {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("NewVolume: %v", err)
|
|
}
|
|
t.Cleanup(v.Close)
|
|
for i := 1; i <= live+deleted; i++ {
|
|
if _, _, _, err := v.writeNeedle2(newRandomNeedle(uint64(i)), true, false, false); err != nil {
|
|
t.Fatalf("write needle %d: %v", i, err)
|
|
}
|
|
}
|
|
for i := 1; i <= deleted; i++ {
|
|
if _, err := v.deleteNeedle2(newEmptyNeedle(uint64(i)), false); err != nil {
|
|
t.Fatalf("delete needle %d: %v", i, err)
|
|
}
|
|
}
|
|
return v
|
|
}
|
|
|
|
func stubCompactionDiskFree(t *testing.T, free uint64) {
|
|
t.Helper()
|
|
prev := compactionDiskFree
|
|
compactionDiskFree = func(string) uint64 { return free }
|
|
t.Cleanup(func() { compactionDiskFree = prev })
|
|
}
|
|
|
|
func TestCompactionSpaceNeeded_CountsLiveBytesNotVolumeSize(t *testing.T) {
|
|
v := newVolumeWithGarbage(t, 20, 2000)
|
|
datSize, idxSize, _ := v.FileStat()
|
|
liveContent := int64(v.ContentSize() - v.DeletedSize())
|
|
|
|
dataBytes, indexBytes := compactionSpaceNeeded(v, 0)
|
|
|
|
if dataBytes < liveContent+super_block.SuperBlockSize {
|
|
t.Fatalf("data estimate %d does not cover the %d live content bytes plus the superblock", dataBytes, liveContent)
|
|
}
|
|
if indexBytes < 20*types.NeedleMapEntrySize {
|
|
t.Fatalf("index estimate %d does not cover 20 live entries", indexBytes)
|
|
}
|
|
if dataBytes+indexBytes >= int64(datSize+idxSize) {
|
|
t.Fatalf("space needed %d is not below the current volume size %d: a mostly-garbage volume must not require its own size to compact", dataBytes+indexBytes, datSize+idxSize)
|
|
}
|
|
}
|
|
|
|
func TestCompactionSpaceNeeded_AllGarbageNeedsAlmostNothing(t *testing.T) {
|
|
v := newVolumeWithGarbage(t, 0, 2000)
|
|
datSize, _, _ := v.FileStat()
|
|
|
|
dataBytes, indexBytes := compactionSpaceNeeded(v, 0)
|
|
|
|
if needed := dataBytes + indexBytes; needed > int64(datSize)/10 {
|
|
t.Fatalf("an all-garbage volume of %d bytes still asks for %d bytes", datSize, needed)
|
|
}
|
|
}
|
|
|
|
func TestCompactionSpaceNeeded_NeverAboveCurrentVolume(t *testing.T) {
|
|
// Nothing deleted: the estimate may not exceed what is already on disk.
|
|
v := newVolumeWithGarbage(t, 200, 0)
|
|
datSize, idxSize, _ := v.FileStat()
|
|
|
|
dataBytes, indexBytes := compactionSpaceNeeded(v, 0)
|
|
|
|
if dataBytes > int64(datSize) || indexBytes > int64(idxSize) {
|
|
t.Fatalf("estimate data=%d index=%d exceeds the current files data=%d index=%d", dataBytes, indexBytes, datSize, idxSize)
|
|
}
|
|
}
|
|
|
|
func TestCompactionSpaceNeeded_PreallocateWinsForDataOnly(t *testing.T) {
|
|
// The new .dat is preallocated to this size; the rebuilt index is a
|
|
// separate file and still needs its own room.
|
|
v := newVolumeWithGarbage(t, 5, 5)
|
|
const preallocate = int64(1) << 30
|
|
|
|
dataBytes, indexBytes := compactionSpaceNeeded(v, preallocate)
|
|
|
|
if dataBytes != preallocate {
|
|
t.Fatalf("data estimate %d, want the preallocate size %d", dataBytes, preallocate)
|
|
}
|
|
if indexBytes < 5*types.NeedleMapEntrySize {
|
|
t.Fatalf("index estimate %d does not cover 5 live entries", indexBytes)
|
|
}
|
|
}
|
|
|
|
func TestEnsureCompactVolumeSpace_FullDiskWithGarbage(t *testing.T) {
|
|
// The disk-full case from #11516: free space is far below the volume's
|
|
// size, but well above what compacting its live needles will write.
|
|
v := newVolumeWithGarbage(t, 20, 2000)
|
|
datSize, idxSize, _ := v.FileStat()
|
|
dataBytes, indexBytes := compactionSpaceNeeded(v, 0)
|
|
needed := dataBytes + indexBytes
|
|
if uint64(needed) >= datSize+idxSize {
|
|
t.Fatalf("test setup: estimate %d is not below volume size %d", needed, datSize+idxSize)
|
|
}
|
|
|
|
stubCompactionDiskFree(t, uint64(needed))
|
|
if err := ensureCompactVolumeSpace(v, 0); err != nil {
|
|
t.Fatalf("free %d covers the estimate %d, volume is %d: unexpected %v", needed, needed, datSize+idxSize, err)
|
|
}
|
|
|
|
stubCompactionDiskFree(t, uint64(needed)-1)
|
|
err := ensureCompactVolumeSpace(v, 0)
|
|
if !errors.Is(err, ErrInsufficientSpace) {
|
|
t.Fatalf("free %d below the estimate %d: got %v, want ErrInsufficientSpace", needed-1, needed, err)
|
|
}
|
|
}
|
|
|
|
func TestEnsureCompactVolumeSpace_SeparateIndexDisk(t *testing.T) {
|
|
dataDir, idxDir := t.TempDir(), t.TempDir()
|
|
v, err := NewVolume(dataDir, idxDir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("NewVolume: %v", err)
|
|
}
|
|
t.Cleanup(v.Close)
|
|
for i := 1; i <= 50; i++ {
|
|
if _, _, _, err := v.writeNeedle2(newRandomNeedle(uint64(i)), true, false, false); err != nil {
|
|
t.Fatalf("write needle %d: %v", i, err)
|
|
}
|
|
}
|
|
dataBytes, indexBytes := compactionSpaceNeeded(v, 0)
|
|
|
|
free := map[string]uint64{dataDir: uint64(dataBytes), idxDir: uint64(indexBytes)}
|
|
prevFree, prevSame := compactionDiskFree, compactionSameFilesystem
|
|
compactionDiskFree = func(dir string) uint64 { return free[dir] }
|
|
compactionSameFilesystem = func(string, string) bool { return false }
|
|
t.Cleanup(func() { compactionDiskFree, compactionSameFilesystem = prevFree, prevSame })
|
|
|
|
if err := ensureCompactVolumeSpace(v, 0); err != nil {
|
|
t.Fatalf("each disk covers its own share: unexpected %v", err)
|
|
}
|
|
// Plenty of room on the data disk cannot make up for a full index disk.
|
|
free[dataDir] = uint64(dataBytes) * 10
|
|
free[idxDir] = uint64(indexBytes) - 1
|
|
if err := ensureCompactVolumeSpace(v, 0); !errors.Is(err, ErrInsufficientSpace) {
|
|
t.Fatalf("full index disk: got %v, want ErrInsufficientSpace", err)
|
|
}
|
|
|
|
// Two directories on one filesystem draw on the same free space, so the
|
|
// data and the index estimates must be covered together.
|
|
compactionSameFilesystem = func(string, string) bool { return true }
|
|
free[dataDir] = uint64(dataBytes+indexBytes) - 1
|
|
free[idxDir] = uint64(dataBytes+indexBytes) - 1
|
|
if err := ensureCompactVolumeSpace(v, 0); !errors.Is(err, ErrInsufficientSpace) {
|
|
t.Fatalf("shared filesystem short of the sum: got %v, want ErrInsufficientSpace", err)
|
|
}
|
|
free[dataDir] = uint64(dataBytes + indexBytes)
|
|
free[idxDir] = uint64(dataBytes + indexBytes)
|
|
if err := ensureCompactVolumeSpace(v, 0); err != nil {
|
|
t.Fatalf("shared filesystem covering the sum: unexpected %v", err)
|
|
}
|
|
// A mount point the identity check cannot see: the index directory
|
|
// reports its own, smaller pool and must still be checked.
|
|
free[idxDir] = uint64(indexBytes) - 1
|
|
if err := ensureCompactVolumeSpace(v, 0); !errors.Is(err, ErrInsufficientSpace) {
|
|
t.Fatalf("index mount point short of the index: got %v, want ErrInsufficientSpace", err)
|
|
}
|
|
}
|
|
|
|
func TestSameFilesystem(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if !sameFilesystem(dir, dir) {
|
|
t.Fatal("a directory is on its own filesystem")
|
|
}
|
|
if !sameFilesystem(dir, filepath.Join(dir, "missing")) {
|
|
t.Fatal("an unreadable path must be treated as shared, so the check asks for the sum")
|
|
}
|
|
}
|
|
|
|
// The estimate may not fall below the current .dat size: an all-live
|
|
// volume's compacted copy is byte-for-byte its current one.
|
|
func TestCompactionSpaceNeededCoversNeedleFraming(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
for i := 1; i <= 100; i++ {
|
|
if _, _, _, err := v.writeNeedle2(newRandomNeedle(uint64(i)), true, false, false); err != nil {
|
|
t.Fatalf("write needle %d: %v", i, err)
|
|
}
|
|
}
|
|
datSize, _, _ := v.FileStat()
|
|
|
|
dataBytes, _ := compactionSpaceNeeded(v, 0)
|
|
if dataBytes < int64(datSize) {
|
|
t.Fatalf("estimate %d below .dat size %d for an all-live volume: missing per-needle framing", dataBytes, datSize)
|
|
}
|
|
}
|
|
|
|
// disk_space_low is only reported when low space is the sole read-only cause.
|
|
func TestCheckCompactVolumeDiskLowSoleCauseOnly(t *testing.T) {
|
|
dir := t.TempDir()
|
|
store := newSingleDirStore(t, dir)
|
|
defer store.Close()
|
|
const vid = needle.VolumeId(7)
|
|
require.NoError(t, store.AddVolume(vid, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
|
|
|
|
_, low, err := store.CheckCompactVolume(vid)
|
|
require.NoError(t, err)
|
|
require.False(t, low)
|
|
|
|
store.Locations[0].isDiskSpaceLow.Store(true)
|
|
_, low, err = store.CheckCompactVolume(vid)
|
|
require.NoError(t, err)
|
|
require.True(t, low)
|
|
|
|
require.NoError(t, store.MarkVolumeReadonly(vid, false, false))
|
|
_, low, err = store.CheckCompactVolume(vid)
|
|
require.NoError(t, err)
|
|
require.False(t, low)
|
|
}
|