mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* volume: fsync a needle delete that asks for it A delete now takes fsync like a write: the tombstone is synced before the needle map drops the needle, and the .idx row before the delete is acknowledged. A failed .dat sync takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: honor fsync=true on DELETE and forward it to replicas Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): fsync a needle delete that asks for it Mirrors the Go volume server: DELETE with fsync=true flushes the tombstone before the index drops the needle and the .idx before the ack, and a failed .dat flush takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: let a durable delete through on a full volume A queued delete hit the batch worker's volume size check, which a tombstone append is exempt from inline, so fsync=true deletes failed once the volume reached its hard limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: sync a durable delete of an already-deleted needle An earlier delete without fsync may have dropped the needle with its tombstone still unsynced, so the inline path syncs before acking anyway, as the batch worker already does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a batch request that races the worker stopping stopWorker can close the batch channel between startWorker handing it out and the send, which panicked. The request now falls back to the inline path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): sync a durable delete of an already-deleted needle Mirrors the Go volume server: an earlier delete without fsync may have dropped the needle with its tombstone still unsynced. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): run a durable delete on a blocking thread Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: keep syncDelete's signature for existing callers Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a write a delete cancelled before it appends A write cancelled while still queued behind the volume locks used to append, publish and then tombstone itself without a sync, after its delete may already have been acknowledged as durable. A crash could keep the row and lose the tombstone. Refuse it before the append. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): refuse a write a delete cancelled before it appends Mirrors the Go change: a write cancelled while still queued behind the volume locks no longer appends, publishes and tombstones itself without a sync after its delete may already be durable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): drop the write guards for a durable delete's flushes The durable delete held the write lease and the io write guard across both fsyncs, so reads of the volume waited on them. Append and publish the tombstone under the guards and flush with only the io read side, as write_synced_run and the Go path already do. do_delete_request keeps its signature. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when a durable delete fails to index The needle map drops the needle before it appends the .idx row, so after a failed append a durable retry found nothing to delete and acked, and a later write could bury the unindexed tombstone so a reload brought the needle back. Fence the volume the way a failed durable write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when a durable delete fails to index Rust counterpart of the Go change: the needle map drops the needle before it appends the .idx row, so a failed append left a durable retry nothing to delete, and a later write could bury the unindexed tombstone. publish_delete_request now fences the volume for a durable delete, as publish_write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when any delete fails to index A non-durable delete whose .idx append failed left the volume writable with the needle already dropped from the map, so a later durable retry found nothing to delete and acknowledged without the missing row. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when any delete fails to index Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: take the fsync flag on the existing delete functions DeleteVolumeNeedle, deleteNeedle2 and syncDelete take fsync instead of gaining *WithFsync variants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): take the fsync flag on the existing delete functions delete_volume_needle and delete_needle take fsync instead of gaining *_with_fsync variants; the durable path and the deletable checks fold back into delete_needle, and the cancelled-write check mirrors Go's isCancelledWrite. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
745 lines
26 KiB
Go
745 lines
26 KiB
Go
package storage
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// countingBackend counts Sync calls and can be made to fail them. It also
|
|
// counts random reads, so recovery paths can prove they declined before
|
|
// touching the .dat.
|
|
type countingBackend struct {
|
|
backend.BackendStorageFile
|
|
syncCount int
|
|
syncErr error
|
|
syncErrOnce bool
|
|
truncateErr error
|
|
truncateCount int
|
|
readCount int
|
|
}
|
|
|
|
func (b *countingBackend) ReadAt(p []byte, off int64) (int, error) {
|
|
b.readCount++
|
|
return b.BackendStorageFile.ReadAt(p, off)
|
|
}
|
|
|
|
func (b *countingBackend) Sync() error {
|
|
b.syncCount++
|
|
if b.syncErr != nil {
|
|
err := b.syncErr
|
|
if b.syncErrOnce {
|
|
b.syncErr = nil
|
|
b.syncErrOnce = false
|
|
}
|
|
return err
|
|
}
|
|
return b.BackendStorageFile.Sync()
|
|
}
|
|
|
|
func (b *countingBackend) Truncate(off int64) error {
|
|
b.truncateCount++
|
|
if b.truncateErr != nil {
|
|
return b.truncateErr
|
|
}
|
|
return b.BackendStorageFile.Truncate(off)
|
|
}
|
|
|
|
func newCountingVolume(t *testing.T) (*Volume, *countingBackend) {
|
|
return newCountingVolumeWithKind(t, NeedleMapInMemory)
|
|
}
|
|
|
|
func newCountingVolumeWithKind(t *testing.T, needleMapKind NeedleMapKind) (*Volume, *countingBackend) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
v, err := NewVolume(dir, dir, "", 1, needleMapKind, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
require.NoError(t, err)
|
|
t.Cleanup(v.Close)
|
|
counting := &countingBackend{BackendStorageFile: v.DataBackend}
|
|
v.DataBackend = counting
|
|
return v, counting
|
|
}
|
|
|
|
func reopenCountingVolume(t *testing.T, v *Volume) *Volume {
|
|
t.Helper()
|
|
dir, dirIdx, id, needleMapKind := v.dir, v.dirIdx, v.Id, v.needleMapKind
|
|
v.Close()
|
|
reloaded, err := NewVolume(dir, dirIdx, "", id, needleMapKind,
|
|
&super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
require.NoError(t, err)
|
|
t.Cleanup(reloaded.Close)
|
|
return reloaded
|
|
}
|
|
|
|
// A durable write reaching a stopping server used to silently drop its fsync,
|
|
// so the ack promised durability the .dat did not have. It now flushes inline
|
|
// instead of queueing on the batch worker that is winding down.
|
|
func TestWriteNeedle2FsyncsInlineWhileStopping(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
_, _, _, err := v.writeNeedle2(newRandomNeedle(1), true, true, true)
|
|
require.NoError(t, err, "a durable write must still be accepted while stopping")
|
|
require.Equal(t, 1, counting.syncCount, "the write should have been flushed inline")
|
|
|
|
// A non-durable write keeps the drain cheap: still accepted, still no fsync.
|
|
_, _, _, err = v.writeNeedle2(newRandomNeedle(2), true, false, true)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, counting.syncCount, "a write that did not ask for fsync must not pay for one")
|
|
}
|
|
|
|
func fixedNeedle(id uint64, data string) *needle.Needle {
|
|
n := new(needle.Needle)
|
|
n.Data = []byte(data)
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
n.Id = types.Uint64ToNeedleId(id)
|
|
return n
|
|
}
|
|
|
|
// An append we could not flush is not data to vouch for: the inline path takes
|
|
// it back off the .dat and fails the write. The needle map has to come back
|
|
// with it, or it would resolve to an offset past the truncated end.
|
|
func TestWriteNeedle2TruncatesWhenInlineFsyncFails(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
kept := fixedNeedle(1, "first-copy")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
keptEntry, found := v.nm.Get(kept.Id)
|
|
require.True(t, found)
|
|
keptOffset, keptSize := keptEntry.Offset, keptEntry.Size
|
|
before, _, err := v.DataBackend.GetStat()
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("disk went away")
|
|
_, _, _, err = v.writeNeedle2(fixedNeedle(1, "second-copy"), true, true, true)
|
|
require.Error(t, err, "a write whose fsync failed must not be acked")
|
|
|
|
after, _, err := v.DataBackend.GetStat()
|
|
require.NoError(t, err)
|
|
require.Equal(t, before, after, "the unflushed append should have been truncated away")
|
|
|
|
now, found := v.nm.Get(kept.Id)
|
|
require.True(t, found, "the mapping the failed write replaced should be back")
|
|
require.Equal(t, keptOffset, now.Offset, "the index must not point past the truncated end")
|
|
require.Equal(t, keptSize, now.Size)
|
|
|
|
counting.syncErr = nil
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err, "the surviving needle must still be readable")
|
|
require.Equal(t, []byte("first-copy"), readBack.Data)
|
|
}
|
|
|
|
// Same rollback for a needle the failed write introduced: with nothing to go
|
|
// back to, the mapping goes away rather than pointing past the end.
|
|
func TestWriteNeedle2DropsIndexOfUnflushedNewNeedle(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
counting.syncErr = errors.New("disk went away")
|
|
fresh := fixedNeedle(7, "never-landed")
|
|
_, _, _, err := v.writeNeedle2(fresh, true, true, true)
|
|
require.Error(t, err)
|
|
|
|
if entry, found := v.nm.Get(fresh.Id); found {
|
|
require.True(t, entry.Size.IsDeleted(), "a needle that never reached the disk must not resolve")
|
|
}
|
|
|
|
counting.syncErr = nil
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = fresh.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err, "reading the rolled-back needle should fail cleanly, not read past the end")
|
|
}
|
|
|
|
// The async durable path must roll back the mapper as well as the data tail
|
|
// when its batch fsync fails. The current worker only truncates the data file,
|
|
// so this test exposes the dangling mapping.
|
|
func TestWriteNeedle2RollsBackBatchFsyncFailure(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
initialFileCount := v.nm.FileCount()
|
|
initialDeletedCount := v.nm.DeletedCount()
|
|
initialContentSize := v.nm.ContentSize()
|
|
initialDeletedSize := v.nm.DeletedSize()
|
|
initialMaxFileKey := v.nm.MaxFileKey()
|
|
|
|
before, _, err := v.DataBackend.GetStat()
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
fresh := fixedNeedle(8, "batch-never-landed")
|
|
_, _, _, err = v.writeNeedle2(fresh, true, true, false)
|
|
require.Error(t, err, "a batch whose fsync failed must not be acknowledged")
|
|
|
|
after, _, err := v.DataBackend.GetStat()
|
|
require.NoError(t, err)
|
|
require.Equal(t, before, after, "the failed batch append should be truncated")
|
|
|
|
if entry, found := v.nm.Get(fresh.Id); found {
|
|
require.True(t, entry.Size.IsDeleted(), "the failed batch must not leave a live mapping")
|
|
}
|
|
require.Equal(t, initialFileCount, v.nm.FileCount())
|
|
require.Equal(t, initialDeletedCount, v.nm.DeletedCount())
|
|
require.Equal(t, initialContentSize, v.nm.ContentSize())
|
|
require.Equal(t, initialDeletedSize, v.nm.DeletedSize())
|
|
require.Equal(t, initialMaxFileKey, v.nm.MaxFileKey())
|
|
}
|
|
|
|
func TestWriteNeedle2RollsBackBatchFsyncFailureAndReloadsNewNeedle(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
|
|
fresh := fixedNeedle(9, "batch-never-landed")
|
|
_, _, _, err := v.writeNeedle2(fresh, true, true, false)
|
|
require.Error(t, err)
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
if entry, found := reloaded.nm.Get(fresh.Id); found {
|
|
require.True(t, entry.Size.IsDeleted(), "the failed batch must not reload a live mapping")
|
|
}
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = fresh.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err, "the failed batch must not be readable after reload")
|
|
require.False(t, reloaded.IsReadOnly(), "a completed rollback should keep the volume healthy")
|
|
}
|
|
|
|
func TestWriteNeedle2RollsBackBatchFsyncFailureAndReloadsOverwrite(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
kept := fixedNeedle(10, "first-copy")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
keptEntry, found := v.nm.Get(kept.Id)
|
|
require.True(t, found)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
_, _, _, err = v.writeNeedle2(fixedNeedle(10, "second-copy"), true, true, false)
|
|
require.Error(t, err)
|
|
|
|
now, found := v.nm.Get(kept.Id)
|
|
require.True(t, found)
|
|
require.Equal(t, keptEntry.Offset, now.Offset)
|
|
require.Equal(t, keptEntry.Size, now.Size)
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("first-copy"), readBack.Data)
|
|
}
|
|
|
|
func TestBatchDeleteRollsBackOnFsyncFailure(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
kept := fixedNeedle(11, "delete-me")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
deleteRequest := needle.NewAsyncRequest(&needle.Needle{Id: kept.Id}, false)
|
|
deleteRequest.ActualSize = needle.GetActualSize(0, v.Version())
|
|
require.True(t, v.asyncRequestAppend(deleteRequest))
|
|
_, _, _, err = deleteRequest.WaitComplete()
|
|
require.Error(t, err)
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err, "a failed batch delete must restore the live mapping")
|
|
require.Equal(t, []byte("delete-me"), readBack.Data)
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("delete-me"), readBack.Data)
|
|
}
|
|
|
|
func TestBatchSyncFailureEntersFailClosedWhenTruncateFails(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
counting.truncateErr = errors.New("truncate failed")
|
|
|
|
fresh := fixedNeedle(12, "cannot-be-recovered")
|
|
_, _, _, err := v.writeNeedle2(fresh, true, true, false)
|
|
require.Error(t, err)
|
|
require.True(t, v.IsReadOnly(), "a failed recovery must make the volume unavailable")
|
|
require.NotNil(t, v.UnavailableError())
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = fresh.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err, "an unavailable volume must reject reads")
|
|
|
|
_, _, _, err = v.writeNeedle2(fixedNeedle(13, "after-failure"), true, true, false)
|
|
require.Error(t, err, "an unavailable volume must reject later writes")
|
|
require.Equal(t, 1, counting.truncateCount)
|
|
}
|
|
|
|
func TestUnavailableVolumeCannotBeMarkedWritable(t *testing.T) {
|
|
dir := t.TempDir()
|
|
store := newIdxSplitStore(t, dir, dir)
|
|
const vid = needle.VolumeId(19)
|
|
require.NoError(t, store.AddVolume(vid, "", NeedleMapInMemory, "000", "", 0,
|
|
needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
|
|
|
|
v := store.findVolume(vid)
|
|
require.NotNil(t, v)
|
|
v.markIoUnavailable(errors.New("batch recovery failed"))
|
|
|
|
require.Error(t, store.MarkVolumeWritable(vid), "manual writable transition must not bypass quarantine")
|
|
require.NotNil(t, v.UnavailableError())
|
|
}
|
|
|
|
func TestMixedBatchSyncFailureRollsBackAsOneUnit(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
overwritten := fixedNeedle(14, "keep-overwrite")
|
|
deleted := fixedNeedle(15, "keep-delete")
|
|
_, _, _, err := v.writeNeedle2(overwritten, true, true, true)
|
|
require.NoError(t, err)
|
|
_, _, _, err = v.writeNeedle2(deleted, true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
replacement := fixedNeedle(14, "failed-overwrite")
|
|
fresh := fixedNeedle(16, "failed-new")
|
|
deleteRequest := needle.NewAsyncRequest(&needle.Needle{Id: deleted.Id}, false)
|
|
requests := []*needle.AsyncRequest{
|
|
needle.NewAsyncRequest(replacement, true),
|
|
deleteRequest,
|
|
needle.NewAsyncRequest(fresh, true),
|
|
}
|
|
v.processBatch(requests)
|
|
|
|
for _, request := range requests {
|
|
_, _, _, err = request.WaitComplete()
|
|
require.Error(t, err, "every request in a failed batch must fail")
|
|
}
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = overwritten.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("keep-overwrite"), readBack.Data)
|
|
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = deleted.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("keep-delete"), readBack.Data)
|
|
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = fresh.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err)
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = overwritten.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("keep-overwrite"), readBack.Data)
|
|
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = deleted.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("keep-delete"), readBack.Data)
|
|
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = fresh.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestWriteNeedle2EntersFailClosedWhenInlineRollbackFails(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
counting.syncErr = errors.New("inline fsync failed")
|
|
counting.syncErrOnce = true
|
|
counting.truncateErr = errors.New("truncate failed")
|
|
|
|
_, _, _, err := v.writeNeedle2(fixedNeedle(20, "cannot-be-recovered"), true, true, true)
|
|
require.Error(t, err)
|
|
require.NotNil(t, v.UnavailableError())
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = 20
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestUnavailableVolumeStaysUnavailableAfterReload(t *testing.T) {
|
|
v, _ := newCountingVolume(t)
|
|
v.markIoUnavailable(errors.New("batch recovery failed"))
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
require.NotNil(t, reloaded.UnavailableError(), "the unavailable state must survive a reload")
|
|
require.True(t, reloaded.IsReadOnly())
|
|
|
|
_, _, quarantined := reloaded.getIoErrorState()
|
|
require.True(t, quarantined, "a reloaded unavailable volume must stay out of heartbeats")
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = 1
|
|
_, err := reloaded.readNeedle(readBack, nil, nil)
|
|
require.Error(t, err)
|
|
_, _, _, err = reloaded.writeNeedle2(fixedNeedle(1, "after-reload"), true, true, false)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestBatchFsyncRollbackWithLevelDbMapper(t *testing.T) {
|
|
v, counting := newCountingVolumeWithKind(t, NeedleMapLevelDb)
|
|
|
|
kept := fixedNeedle(17, "leveldb-old")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
_, _, _, err = v.writeNeedle2(fixedNeedle(17, "leveldb-new"), true, true, false)
|
|
require.Error(t, err)
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("leveldb-old"), readBack.Data)
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("leveldb-old"), readBack.Data)
|
|
}
|
|
|
|
func TestBatchFsyncRollbackRestoresOriginalValueAfterRepeatedNeedleId(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
kept := fixedNeedle(18, "original-value")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
firstReplacement := fixedNeedle(18, "first-failed-value")
|
|
secondReplacement := fixedNeedle(18, "second-failed-value")
|
|
requests := []*needle.AsyncRequest{
|
|
needle.NewAsyncRequest(firstReplacement, true),
|
|
needle.NewAsyncRequest(secondReplacement, true),
|
|
}
|
|
v.processBatch(requests)
|
|
|
|
// A repeated needle id starts a new batch run, so the second write syncs
|
|
// on its own: the first run rolls back to "original-value" and the second
|
|
// write lands as a normal sequential overwrite.
|
|
_, _, _, err = requests[0].WaitComplete()
|
|
require.Error(t, err)
|
|
_, _, _, err = requests[1].WaitComplete()
|
|
require.NoError(t, err)
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("second-failed-value"), readBack.Data)
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
readBack = new(needle.Needle)
|
|
readBack.Id = kept.Id
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("second-failed-value"), readBack.Data)
|
|
}
|
|
|
|
// Rolling back a first-time write must not leave a tombstoned map entry: the
|
|
// stale offset would make the next write to that needle fail reading a header
|
|
// that no longer exists.
|
|
func TestBatchRollbackLeavesNoPhantomMapping(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
fresh := fixedNeedle(24, "batch-never-landed")
|
|
_, _, _, err := v.writeNeedle2(fresh, true, true, false)
|
|
require.Error(t, err)
|
|
|
|
_, _, _, err = v.writeNeedle2(fresh, true, true, false)
|
|
require.NoError(t, err, "rewriting a rolled-back needle must succeed")
|
|
|
|
readBack := new(needle.Needle)
|
|
readBack.Id = fresh.Id
|
|
_, err = v.readNeedle(readBack, nil, nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []byte("batch-never-landed"), readBack.Data)
|
|
}
|
|
|
|
// A request that already failed on its own must still surface the batch
|
|
// failure: keeping its earlier error would hide that nothing was persisted,
|
|
// or that recovery itself failed.
|
|
func TestFailedBatchMarksEveryRequestFailed(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
kept := fixedNeedle(22, "original")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
counting.syncErr = errors.New("batch fsync failed")
|
|
counting.syncErrOnce = true
|
|
|
|
badCookie := fixedNeedle(22, "wrong-cookie")
|
|
badCookie.Cookie = kept.Cookie + 1
|
|
requests := []*needle.AsyncRequest{
|
|
needle.NewAsyncRequest(badCookie, true),
|
|
needle.NewAsyncRequest(fixedNeedle(23, "fresh"), true),
|
|
}
|
|
v.processBatch(requests)
|
|
|
|
for _, request := range requests {
|
|
_, _, _, err = request.WaitComplete()
|
|
require.ErrorContains(t, err, "batch fsync failed")
|
|
}
|
|
}
|
|
|
|
// failingPutMapper fails the failAt-th Put.
|
|
type failingPutMapper struct {
|
|
NeedleMapper
|
|
puts int
|
|
failAt int
|
|
err error
|
|
}
|
|
|
|
func (m *failingPutMapper) Put(key types.NeedleId, offset types.Offset, size types.Size) error {
|
|
m.puts++
|
|
if m.puts == m.failAt {
|
|
return m.err
|
|
}
|
|
return m.NeedleMapper.Put(key, offset, size)
|
|
}
|
|
|
|
// A durable write whose index update fails stops the volume taking writes:
|
|
// the requests after it in the same batch are refused, as a write sent on
|
|
// its own would be, instead of indexed behind it. Entries before it stay
|
|
// acked.
|
|
func TestProcessBatchRefusesWritesAfterDurableIndexFailure(t *testing.T) {
|
|
v, _ := newCountingVolume(t)
|
|
|
|
kept := fixedNeedle(4, "kept")
|
|
_, _, _, err := v.writeNeedle2(kept, true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
v.nm = &failingPutMapper{
|
|
NeedleMapper: v.nm,
|
|
failAt: 2,
|
|
err: errors.New("index write failed"),
|
|
}
|
|
|
|
wrongCookie := fixedNeedle(4, "kept")
|
|
wrongCookie.Cookie = kept.Cookie + 1
|
|
requests := []*needle.AsyncRequest{
|
|
needle.NewAsyncRequest(fixedNeedle(1, "before"), true),
|
|
needle.NewAsyncRequest(fixedNeedle(2, "torn"), true),
|
|
needle.NewAsyncRequest(fixedNeedle(3, "after"), true),
|
|
needle.NewAsyncRequest(fixedNeedle(4, "kept"), true),
|
|
needle.NewAsyncRequest(wrongCookie, true),
|
|
}
|
|
v.processBatch(requests)
|
|
|
|
_, _, _, err = requests[0].WaitComplete()
|
|
require.NoError(t, err)
|
|
_, _, _, err = requests[1].WaitComplete()
|
|
require.Error(t, err)
|
|
for _, request := range requests[2:] {
|
|
_, _, _, err = request.WaitComplete()
|
|
require.ErrorContains(t, err, "read only", "refused as it would be sent on its own")
|
|
}
|
|
require.True(t, v.IsReadOnly())
|
|
}
|
|
|
|
// The quarantine is what CollectHeartbeat keys on: an unavailable volume must
|
|
// not be announced to the master at all.
|
|
func TestUnavailableVolumeIsSkippedInHeartbeat(t *testing.T) {
|
|
store := newTestStore(t, 1)
|
|
v := mountTestVolume(t, store.Locations[0], 1, "pics")
|
|
fillTestVolume(t, v)
|
|
v.markIoUnavailable(errors.New("batch recovery failed"))
|
|
|
|
heartbeat := store.CollectHeartbeat()
|
|
for _, m := range heartbeat.Volumes {
|
|
require.NotEqual(t, uint32(1), m.Id, "a failed-recovery volume must not be announced")
|
|
}
|
|
}
|
|
|
|
// The pre-stop drain exists so writes already assigned to this server land.
|
|
// Refusing them once stopping would turn every rolling restart into client
|
|
// write failures for the length of the drain.
|
|
func TestStoreWriteVolumeNeedleStaysDurableWhileStopping(t *testing.T) {
|
|
dir := t.TempDir()
|
|
store := newIdxSplitStore(t, dir, dir)
|
|
const vid = needle.VolumeId(1)
|
|
require.NoError(t, store.AddVolume(vid, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
|
|
|
|
v := store.findVolume(vid)
|
|
require.NotNil(t, v)
|
|
counting := &countingBackend{BackendStorageFile: v.DataBackend}
|
|
v.DataBackend = counting
|
|
|
|
store.SetStopping()
|
|
counting.syncCount = 0
|
|
|
|
isUnchanged, err := store.WriteVolumeNeedle(vid, newRandomNeedle(1), true, true)
|
|
require.NoError(t, err, "the drain must keep accepting durable writes")
|
|
require.False(t, isUnchanged)
|
|
require.Equal(t, 1, counting.syncCount, "the accepted write must actually be on disk")
|
|
}
|
|
|
|
func TestDurableDeleteSyncsTombstone(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
for id := uint64(1); id <= 2; id++ {
|
|
_, _, _, err := v.writeNeedle2(fixedNeedle(id, "payload"), true, false, false)
|
|
require.NoError(t, err)
|
|
}
|
|
require.Equal(t, 0, counting.syncCount)
|
|
|
|
_, err := v.deleteNeedle2(fixedNeedle(1, ""), false)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 0, counting.syncCount, "a delete that did not ask for fsync must not pay for one")
|
|
|
|
size, err := v.deleteNeedle2(fixedNeedle(2, ""), true)
|
|
require.NoError(t, err)
|
|
require.NotZero(t, size)
|
|
require.Equal(t, 1, counting.syncCount, "a durable delete must flush its tombstone before the ack")
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
nv, found := reloaded.nm.Get(types.Uint64ToNeedleId(2))
|
|
require.True(t, !found || nv.Size.IsDeleted(), "the durable delete must survive a reload")
|
|
}
|
|
|
|
func TestDurableDeleteTakesBackTombstoneWhenFsyncFails(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, true, true)
|
|
require.NoError(t, err)
|
|
before, _, err := v.DataBackend.GetStat()
|
|
require.NoError(t, err)
|
|
syncs := counting.syncCount
|
|
|
|
counting.syncErr = errors.New("disk went away")
|
|
_, err = v.syncDelete(fixedNeedle(1, ""), true)
|
|
require.Error(t, err, "a delete whose fsync failed must not be acked")
|
|
require.Equal(t, syncs+1, counting.syncCount)
|
|
|
|
after, _, err := v.DataBackend.GetStat()
|
|
require.NoError(t, err)
|
|
require.Equal(t, before, after, "the unflushed tombstone should be taken back off the .dat")
|
|
nv, found := v.nm.Get(types.Uint64ToNeedleId(1))
|
|
require.True(t, found)
|
|
require.False(t, nv.Size.IsDeleted(), "the needle must stay live when its delete was not acked")
|
|
}
|
|
|
|
func TestDurableDeleteOnFullVolume(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, false, false)
|
|
require.NoError(t, err)
|
|
atomic.StoreUint64(&v.nm.(*NeedleMap).FileByteCounter, types.MaxPossibleVolumeSize)
|
|
|
|
_, err = v.deleteNeedle2(fixedNeedle(1, ""), true)
|
|
require.NoError(t, err, "a full volume must still take a durable delete")
|
|
require.Equal(t, 1, counting.syncCount)
|
|
nv, found := v.nm.Get(types.Uint64ToNeedleId(1))
|
|
require.True(t, !found || nv.Size.IsDeleted())
|
|
}
|
|
|
|
func TestDurableDeleteSyncsAfterUnsyncedDelete(t *testing.T) {
|
|
v, counting := newCountingVolume(t)
|
|
|
|
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, false, false)
|
|
require.NoError(t, err)
|
|
_, err = v.syncDelete(fixedNeedle(1, ""), false)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 0, counting.syncCount)
|
|
|
|
size, err := v.syncDelete(fixedNeedle(1, ""), true)
|
|
require.NoError(t, err)
|
|
require.Zero(t, size)
|
|
require.Equal(t, 1, counting.syncCount, "an earlier unsynced delete must still be flushed before a durable ack")
|
|
}
|
|
|
|
func TestAsyncRequestAppendRefusesAfterChannelClosed(t *testing.T) {
|
|
v, _ := newCountingVolume(t)
|
|
|
|
closed := make(chan *needle.AsyncRequest)
|
|
close(closed)
|
|
v.asyncWorkerLock.Lock()
|
|
v.asyncRequestsChan = closed
|
|
v.asyncWorkerLock.Unlock()
|
|
|
|
require.False(t, v.asyncRequestAppend(needle.NewAsyncRequest(fixedNeedle(1, "payload"), true)),
|
|
"a request racing stopWorker must be refused, not panic")
|
|
}
|
|
|
|
// A durable delete whose .idx row fails has already dropped the needle from
|
|
// the map, so a retry would find nothing to delete and ack. The volume stops
|
|
// taking writes instead: nothing is appended past the tombstone, so a reload
|
|
// re-indexes it from the .dat tail rather than resurrect the needle.
|
|
func TestDurableDeleteIndexFailureStopsWrites(t *testing.T) {
|
|
durableBatch := func(v *Volume, n *needle.Needle) (types.Size, error) { return v.deleteNeedle2(n, true) }
|
|
durableInline := func(v *Volume, n *needle.Needle) (types.Size, error) { return v.syncDelete(n, true) }
|
|
for _, tc := range []struct {
|
|
name string
|
|
first func(*Volume, *needle.Needle) (types.Size, error)
|
|
delete func(*Volume, *needle.Needle) (types.Size, error)
|
|
}{
|
|
{"batch", durableBatch, durableBatch},
|
|
{"inline", durableInline, durableInline},
|
|
{"non-durable then durable retry", func(v *Volume, n *needle.Needle) (types.Size, error) { return v.deleteNeedle2(n, false) }, durableBatch},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
v, _ := newCountingVolume(t)
|
|
_, _, _, err := v.writeNeedle2(fixedNeedle(1, "payload"), true, true, true)
|
|
require.NoError(t, err)
|
|
|
|
nm := v.nm.(*NeedleMap)
|
|
indexFile := nm.indexFile
|
|
readOnlyIndex, err := os.Open(indexFile.Name())
|
|
require.NoError(t, err)
|
|
defer readOnlyIndex.Close()
|
|
nm.indexFile = readOnlyIndex
|
|
_, err = tc.first(v, fixedNeedle(1, ""))
|
|
nm.indexFile = indexFile
|
|
require.Error(t, err)
|
|
|
|
_, _, _, err = v.writeNeedle2(fixedNeedle(2, "later"), true, true, true)
|
|
require.Error(t, err, "nothing may be appended past the unindexed tombstone")
|
|
_, err = tc.delete(v, fixedNeedle(1, ""))
|
|
require.Error(t, err, "a retry must not ack a delete whose .idx row never landed")
|
|
|
|
reloaded := reopenCountingVolume(t, v)
|
|
readBack := fixedNeedle(1, "")
|
|
_, err = reloaded.readNeedle(readBack, nil, nil)
|
|
require.False(t, err == nil && string(readBack.Data) == "payload",
|
|
"the deleted needle must not come back on reload")
|
|
})
|
|
}
|
|
}
|