mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* volume: fsync a needle delete that asks for it A delete now takes fsync like a write: the tombstone is synced before the needle map drops the needle, and the .idx row before the delete is acknowledged. A failed .dat sync takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: honor fsync=true on DELETE and forward it to replicas Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): fsync a needle delete that asks for it Mirrors the Go volume server: DELETE with fsync=true flushes the tombstone before the index drops the needle and the .idx before the ack, and a failed .dat flush takes the tombstone back off the .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: let a durable delete through on a full volume A queued delete hit the batch worker's volume size check, which a tombstone append is exempt from inline, so fsync=true deletes failed once the volume reached its hard limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: sync a durable delete of an already-deleted needle An earlier delete without fsync may have dropped the needle with its tombstone still unsynced, so the inline path syncs before acking anyway, as the batch worker already does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a batch request that races the worker stopping stopWorker can close the batch channel between startWorker handing it out and the send, which panicked. The request now falls back to the inline path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): sync a durable delete of an already-deleted needle Mirrors the Go volume server: an earlier delete without fsync may have dropped the needle with its tombstone still unsynced. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): run a durable delete on a blocking thread Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: keep syncDelete's signature for existing callers Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: refuse a write a delete cancelled before it appends A write cancelled while still queued behind the volume locks used to append, publish and then tombstone itself without a sync, after its delete may already have been acknowledged as durable. A crash could keep the row and lose the tombstone. Refuse it before the append. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): refuse a write a delete cancelled before it appends Mirrors the Go change: a write cancelled while still queued behind the volume locks no longer appends, publishes and tombstones itself without a sync after its delete may already be durable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): drop the write guards for a durable delete's flushes The durable delete held the write lease and the io write guard across both fsyncs, so reads of the volume waited on them. Append and publish the tombstone under the guards and flush with only the io read side, as write_synced_run and the Go path already do. do_delete_request keeps its signature. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when a durable delete fails to index The needle map drops the needle before it appends the .idx row, so after a failed append a durable retry found nothing to delete and acked, and a later write could bury the unindexed tombstone so a reload brought the needle back. Fence the volume the way a failed durable write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when a durable delete fails to index Rust counterpart of the Go change: the needle map drops the needle before it appends the .idx row, so a failed append left a durable retry nothing to delete, and a later write could bury the unindexed tombstone. publish_delete_request now fences the volume for a durable delete, as publish_write does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: stop taking writes when any delete fails to index A non-durable delete whose .idx append failed left the volume writable with the needle already dropped from the map, so a later durable retry found nothing to delete and acknowledged without the missing row. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): stop taking writes when any delete fails to index Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: take the fsync flag on the existing delete functions DeleteVolumeNeedle, deleteNeedle2 and syncDelete take fsync instead of gaining *WithFsync variants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): take the fsync flag on the existing delete functions delete_volume_needle and delete_needle take fsync instead of gaining *_with_fsync variants; the durable path and the deletable checks fold back into delete_needle, and the cancelled-write check mirrors Go's isCancelledWrite. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
828 lines
30 KiB
Go
828 lines
30 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestReadNeedleNilNeedleMap(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
v.dataFileAccessLock.Lock()
|
|
if v.nm != nil {
|
|
v.nm.Close()
|
|
v.nm = nil
|
|
}
|
|
v.dataFileAccessLock.Unlock()
|
|
|
|
n := new(needle.Needle)
|
|
n.Id = types.Uint64ToNeedleId(1)
|
|
|
|
if _, err := v.readNeedle(n, &ReadOption{}, nil); err != ErrorNotFound {
|
|
t.Fatalf("readNeedle: want ErrorNotFound, got %v", err)
|
|
}
|
|
|
|
err = v.readNeedleDataInto(n, &ReadOption{ReadBufferSize: 1024}, &bytes.Buffer{}, 0, 0)
|
|
if err != ErrorNotFound {
|
|
t.Fatalf("readNeedleDataInto: want ErrorNotFound, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
type WriteInfo struct {
|
|
offset int64
|
|
size int32
|
|
}
|
|
writeInfos := make([]WriteInfo, 30)
|
|
mockLastUpdateTime := uint64(1000000000000)
|
|
// initialize 20 needles then update first 10 needles
|
|
for i := 1; i <= 30; i++ {
|
|
n := newRandomNeedle(uint64(i % 20))
|
|
n.Flags = 0x08
|
|
n.LastModified = mockLastUpdateTime
|
|
mockLastUpdateTime += 2000
|
|
offset, _, _, err := v.writeNeedle2(n, true, false, false)
|
|
if err != nil {
|
|
t.Fatalf("write needle %d: %v", i, err)
|
|
}
|
|
writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(n.Size)}
|
|
}
|
|
expectedLastUpdateTime := uint64(1000000000000)
|
|
for i := 0; i < 30; i++ {
|
|
testNeedle := new(needle.Needle)
|
|
testNeedle.Id = types.Uint64ToNeedleId(uint64(i + 1%20))
|
|
testNeedle.Flags = 0x08
|
|
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
|
|
actualLastModifiedTime := testNeedle.LastModified
|
|
if writeInfos[i].size != 0 {
|
|
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
|
|
}
|
|
expectedLastUpdateTime += 2000
|
|
}
|
|
}
|
|
|
|
func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
type WriteInfo struct {
|
|
offset int64
|
|
size int32
|
|
}
|
|
writeInfos := make([]WriteInfo, 10)
|
|
mockLastUpdateTime := uint64(1000000000000)
|
|
for i := 1; i <= 10; i++ {
|
|
n := newRandomNeedle(uint64(i % 5))
|
|
n.Flags = 0x08
|
|
n.LastModified = mockLastUpdateTime
|
|
mockLastUpdateTime += 2000
|
|
offset, _, _, err := v.writeNeedle2(n, true, false, false)
|
|
if err != nil {
|
|
t.Fatalf("write needle %d: %v", i, err)
|
|
}
|
|
if i < 5 {
|
|
size, err := v.deleteNeedle2(n, false)
|
|
if err != nil {
|
|
t.Fatalf("delete needle %d: %v", i, err)
|
|
}
|
|
writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(size)}
|
|
} else {
|
|
writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(n.Size)}
|
|
}
|
|
}
|
|
|
|
expectedLastUpdateTime := uint64(1000000000000)
|
|
for i := 0; i < 10; i++ {
|
|
testNeedle := new(needle.Needle)
|
|
testNeedle.Id = types.Uint64ToNeedleId(uint64(i + 1%5))
|
|
testNeedle.Flags = 0x08
|
|
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
|
|
actualLastModifiedTime := testNeedle.LastModified
|
|
if writeInfos[i].size != 0 {
|
|
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
|
|
}
|
|
expectedLastUpdateTime += 2000
|
|
}
|
|
}
|
|
|
|
// scanRecorder records visited offsets and fails once a scan visits more
|
|
// records than the file holds.
|
|
type scanRecorder struct {
|
|
readBody bool
|
|
maxVisits int
|
|
offsets []int64
|
|
}
|
|
|
|
func (s *scanRecorder) VisitSuperBlock(super_block.SuperBlock) error { return nil }
|
|
|
|
func (s *scanRecorder) ReadNeedleBody() bool { return s.readBody }
|
|
|
|
func (s *scanRecorder) VisitNeedle(_ *needle.Needle, offset int64, _, _ []byte) error {
|
|
s.offsets = append(s.offsets, offset)
|
|
if len(s.offsets) > s.maxVisits {
|
|
return fmt.Errorf("visited %d records in a file of %d, offsets %v", len(s.offsets), s.maxVisits, s.offsets)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var errDidNotReturn = errors.New("did not return in time")
|
|
|
|
// runWithTimeout returns errDidNotReturn if fn does not finish within d.
|
|
func runWithTimeout(d time.Duration, fn func() error) error {
|
|
done := make(chan error, 1)
|
|
go func() { done <- fn() }()
|
|
select {
|
|
case err := <-done:
|
|
return err
|
|
case <-time.After(d):
|
|
return errDidNotReturn
|
|
}
|
|
}
|
|
|
|
// A corrupt .dat header can make a record's length zero or negative; the scan
|
|
// must stop there with ErrorCorrupted rather than re-read the header or step
|
|
// back into the previous record. A negative size whose record length stays
|
|
// positive is stepped over as before.
|
|
func TestScanVolumeFileFrom_StopsAtRecordThatCannotAdvance(t *testing.T) {
|
|
cases := []struct {
|
|
version needle.Version
|
|
size types.Size
|
|
stops bool
|
|
}{
|
|
{needle.Version3, -1, false}, // record length 32
|
|
{needle.Version3, -36, true}, // record length 0
|
|
{needle.Version3, -43, true}, // record length 0
|
|
{needle.Version3, -44, true}, // record length -8
|
|
{needle.Version3, -4096, true}, // record length -4056
|
|
{needle.Version2, -1, false}, // record length 24
|
|
{needle.Version2, -28, true}, // record length 0
|
|
{needle.Version2, -35, true}, // record length 0
|
|
{needle.Version2, -36, true}, // record length -8
|
|
}
|
|
for _, tc := range cases {
|
|
recordLen := needle.GetActualSize(tc.size, tc.version)
|
|
if (recordLen <= 0) != tc.stops {
|
|
t.Fatalf("v%d size %d: record length %d, case expects stops=%v", tc.version, tc.size, recordLen, tc.stops)
|
|
}
|
|
for _, readBody := range []bool{false, true} {
|
|
t.Run(fmt.Sprintf("v%d/size%d/readBody=%v", tc.version, tc.size, readBody), func(t *testing.T) {
|
|
f, err := os.Create(filepath.Join(t.TempDir(), "1.dat"))
|
|
if err != nil {
|
|
t.Fatalf("create dat: %v", err)
|
|
}
|
|
dat := backend.NewDiskFile(f)
|
|
defer dat.Close()
|
|
|
|
first, _, _, err := newRandomNeedle(1).Append(dat, tc.version)
|
|
if err != nil {
|
|
t.Fatalf("append needle 1: %v", err)
|
|
}
|
|
corruptAt, _, err := dat.GetStat()
|
|
if err != nil {
|
|
t.Fatalf("stat dat: %v", err)
|
|
}
|
|
raw := make([]byte, max(recordLen, types.NeedleHeaderSize))
|
|
types.NeedleIdToBytes(raw[types.CookieSize:types.CookieSize+types.NeedleIdSize], types.Uint64ToNeedleId(99))
|
|
types.SizeToBytes(raw[types.CookieSize+types.NeedleIdSize:types.NeedleHeaderSize], tc.size)
|
|
if _, err := dat.WriteAt(raw, corruptAt); err != nil {
|
|
t.Fatalf("append corrupt record: %v", err)
|
|
}
|
|
second, _, _, err := newRandomNeedle(2).Append(dat, tc.version)
|
|
if err != nil {
|
|
t.Fatalf("append needle 2: %v", err)
|
|
}
|
|
|
|
scanner := &scanRecorder{readBody: readBody, maxVisits: 3}
|
|
err = runWithTimeout(10*time.Second, func() error {
|
|
return ScanVolumeFileFrom(tc.version, dat, 0, scanner)
|
|
})
|
|
|
|
want := []int64{int64(first), corruptAt, int64(second)}
|
|
if tc.stops {
|
|
want = want[:2]
|
|
if !errors.Is(err, needle.ErrorCorrupted) {
|
|
t.Errorf("scan error = %v, want one wrapping ErrorCorrupted", err)
|
|
}
|
|
} else if err != nil {
|
|
t.Errorf("scan error = %v, want nil", err)
|
|
}
|
|
if !reflect.DeepEqual(scanner.offsets, want) {
|
|
t.Errorf("visited offsets %v, want %v", scanner.offsets, want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
// The CRC is known only after the last byte, so a full read must keep that
|
|
// page unwritten when the checksum mismatches; otherwise the GET has already
|
|
// committed a corrupt body.
|
|
func TestReadNeedleDataIntoChecksumMismatchHoldsLastPage(t *testing.T) {
|
|
dir := t.TempDir()
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
// 2048 is two pages (one buffer swap), 3000 is three, and 2500 with a
|
|
// buffer larger than the needle is the single Write that used to commit
|
|
// the whole body before the CRC check.
|
|
cases := []struct {
|
|
name string
|
|
size int
|
|
page int
|
|
}{
|
|
{"two pages", 2048, 1024},
|
|
{"three pages", 3000, 1024},
|
|
{"one buffer", 2500, 4096},
|
|
}
|
|
for i, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
data := bytes.Repeat([]byte("abcdefghij"), (tc.size+9)/10)[:tc.size]
|
|
n := new(needle.Needle)
|
|
n.Data = append([]byte(nil), data...)
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
n.Id = types.Uint64ToNeedleId(uint64(i + 1))
|
|
offset, _, _, err := v.writeNeedle2(n, true, false, false)
|
|
if err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
nv, ok := v.nm.Get(n.Id)
|
|
if !ok {
|
|
t.Fatal("needle missing from index")
|
|
}
|
|
actual := nv.Offset.ToActualOffset()
|
|
|
|
read := func() (bytes.Buffer, error) {
|
|
t.Helper()
|
|
meta := new(needle.Needle)
|
|
meta.Id = n.Id
|
|
if err := v.readNeedleMetaAt(meta, actual, int32(nv.Size)); err != nil {
|
|
t.Fatalf("read meta at %d size %d: %v", actual, nv.Size, err)
|
|
}
|
|
var buf bytes.Buffer
|
|
err := v.readNeedleDataInto(meta, &ReadOption{ReadBufferSize: tc.page}, &buf, 0, int64(meta.DataSize))
|
|
return buf, err
|
|
}
|
|
|
|
intact, err := read()
|
|
if err != nil {
|
|
t.Fatalf("intact read: %v", err)
|
|
}
|
|
if !bytes.Equal(intact.Bytes(), data) {
|
|
t.Fatalf("intact read len %d, want %d", intact.Len(), len(data))
|
|
}
|
|
|
|
dataOff := int64(offset) + types.NeedleHeaderSize + types.DataSizeSize
|
|
if _, err := v.DataBackend.WriteAt([]byte{0xff}, dataOff); err != nil {
|
|
t.Fatalf("damage needle: %v", err)
|
|
}
|
|
damaged, err := read()
|
|
if err == nil || !strings.Contains(err.Error(), "checksum") {
|
|
t.Fatalf("damaged read: got %v, want a checksum error", err)
|
|
}
|
|
held := len(data) % tc.page
|
|
if held == 0 {
|
|
held = tc.page
|
|
}
|
|
if damaged.Len() != len(data)-held {
|
|
t.Fatalf("damaged read wrote %d bytes, want %d with the last page held back", damaged.Len(), len(data)-held)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A range carries no checksum of its own, so the first ranged read of a
|
|
// needle checks the whole needle's CRC and writes nothing on a mismatch. The
|
|
// reader can then fall back to another replica.
|
|
func TestReadNeedleDataIntoRangeChecksumMismatch(t *testing.T) {
|
|
dir := t.TempDir()
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
data := bytes.Repeat([]byte("0123456789"), 300)
|
|
// One buffer holds the whole needle, or the check streams it in pages.
|
|
for i, page := range []int{4096, 1024} {
|
|
write := func(id uint64) (types.Offset, int64) {
|
|
n := &needle.Needle{Id: types.Uint64ToNeedleId(id), Cookie: 0x1234, Data: append([]byte(nil), data...)}
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
offset, _, _, err := v.writeNeedle2(n, true, false, false)
|
|
if err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
nv, _ := v.nm.Get(n.Id)
|
|
return nv.Offset, int64(offset)
|
|
}
|
|
read := func(id uint64, at types.Offset) (bytes.Buffer, error) {
|
|
meta := &needle.Needle{Id: types.Uint64ToNeedleId(id)}
|
|
nv, _ := v.nm.Get(meta.Id)
|
|
if err := v.readNeedleMetaAt(meta, at.ToActualOffset(), int32(nv.Size)); err != nil {
|
|
t.Fatalf("read meta: %v", err)
|
|
}
|
|
var buf bytes.Buffer
|
|
err := v.readNeedleDataInto(meta, &ReadOption{ReadBufferSize: page}, &buf, 1000, 500)
|
|
return buf, err
|
|
}
|
|
|
|
intact := uint64(10*i + 1)
|
|
at, _ := write(intact)
|
|
if got, err := read(intact, at); err != nil || !bytes.Equal(got.Bytes(), data[1000:1500]) {
|
|
t.Fatalf("page %d: intact range read: %d bytes, %v", page, got.Len(), err)
|
|
}
|
|
|
|
// Damage a byte outside the range: the range itself is intact, but
|
|
// the needle is not, so this replica must not serve it.
|
|
damaged := uint64(10*i + 2)
|
|
at, raw := write(damaged)
|
|
if _, err := v.DataBackend.WriteAt([]byte{0xff}, raw+types.NeedleHeaderSize+types.DataSizeSize+10); err != nil {
|
|
t.Fatalf("damage needle: %v", err)
|
|
}
|
|
got, err := read(damaged, at)
|
|
if err == nil || !strings.Contains(err.Error(), "checksum") || got.Len() != 0 {
|
|
t.Fatalf("page %d: damaged range read wrote %d bytes, err %v; want a checksum error and nothing written", page, got.Len(), err)
|
|
}
|
|
|
|
// Concurrent first ranges of a needle all see the outcome of the check.
|
|
for _, tc := range []struct {
|
|
id uint64
|
|
damage bool
|
|
}{{uint64(10*i + 3), false}, {uint64(10*i + 4), true}} {
|
|
at, raw := write(tc.id)
|
|
if tc.damage {
|
|
if _, err := v.DataBackend.WriteAt([]byte{0xff}, raw+types.NeedleHeaderSize+types.DataSizeSize+10); err != nil {
|
|
t.Fatalf("damage needle: %v", err)
|
|
}
|
|
}
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, 8)
|
|
for g := range errs {
|
|
wg.Add(1)
|
|
go func(g int) {
|
|
defer wg.Done()
|
|
_, errs[g] = read(tc.id, at)
|
|
}(g)
|
|
}
|
|
wg.Wait()
|
|
for _, err := range errs {
|
|
if (err != nil) != tc.damage {
|
|
t.Fatalf("page %d: concurrent range read of needle %d (damaged %v): %v", page, tc.id, tc.damage, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A verified needle is checked again after a reload, as a remounted
|
|
// copy or a compaction commit gets.
|
|
reloaded := uint64(10*i + 5)
|
|
at, raw = write(reloaded)
|
|
if _, err := read(reloaded, at); err != nil {
|
|
t.Fatalf("page %d: first range read: %v", page, err)
|
|
}
|
|
if _, err := v.DataBackend.WriteAt([]byte{0xff}, raw+types.NeedleHeaderSize+types.DataSizeSize+10); err != nil {
|
|
t.Fatalf("damage needle: %v", err)
|
|
}
|
|
v.loadGeneration.Store(volumeLoads.Add(1))
|
|
if _, err := read(reloaded, at); err == nil || !strings.Contains(err.Error(), "checksum") {
|
|
t.Fatalf("page %d: range read after reload: %v; want a checksum error", page, err)
|
|
}
|
|
}
|
|
|
|
// The verification pass leaves the io error streak to the range read.
|
|
v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
|
|
meta := &needle.Needle{Id: types.Uint64ToNeedleId(1)}
|
|
nv, _ := v.nm.Get(meta.Id)
|
|
if err := v.readNeedleMetaAt(meta, nv.Offset.ToActualOffset(), int32(nv.Size)); err != nil {
|
|
t.Fatalf("read meta: %v", err)
|
|
}
|
|
v.loadGeneration.Store(volumeLoads.Add(1))
|
|
if _, err := v.verifiedNeedleBlocks(meta, &ReadOption{ReadBufferSize: 4096}); err != nil {
|
|
t.Fatalf("verification pass: %v", err)
|
|
}
|
|
if _, count, _ := v.getIoErrorState(); count != 1 {
|
|
t.Errorf("io error streak %d after a verification pass, want 1", count)
|
|
}
|
|
}
|
|
|
|
// Damage that appears after a needle passed its check is caught too: a later
|
|
// range checks the blocks it covers against the CRCs the check recorded, so it
|
|
// never serves damaged bytes, while ranges over intact blocks still read.
|
|
func TestReadNeedleDataIntoRangeDamagedAfterCheck(t *testing.T) {
|
|
dir := t.TempDir()
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
const block = minRangeCheckBlockSize
|
|
data := make([]byte, 5*block-100)
|
|
for i := range data {
|
|
data[i] = byte(i*7 + i/251)
|
|
}
|
|
for i, page := range []int{4096, PagedReadLimit} {
|
|
n := &needle.Needle{Id: types.Uint64ToNeedleId(uint64(i + 1)), Cookie: 0x1234, Data: append([]byte(nil), data...)}
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
raw, _, _, err := v.writeNeedle2(n, true, false, false)
|
|
if err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
read := func(offset, size int64) ([]byte, error) {
|
|
meta := &needle.Needle{Id: n.Id}
|
|
nv, _ := v.nm.Get(meta.Id)
|
|
if err := v.readNeedleMetaAt(meta, nv.Offset.ToActualOffset(), int32(nv.Size)); err != nil {
|
|
t.Fatalf("read meta: %v", err)
|
|
}
|
|
var buf bytes.Buffer
|
|
err := v.readNeedleDataInto(meta, &ReadOption{ReadBufferSize: page}, &buf, offset, size)
|
|
return buf.Bytes(), err
|
|
}
|
|
|
|
ranges := [][2]int64{{3*block + 10, 100}, {block - 50, 3 * block}, {4*block + 5, int64(len(data)) - 4*block - 5}}
|
|
for _, r := range ranges {
|
|
if got, err := read(r[0], r[1]); err != nil || !bytes.Equal(got, data[r[0]:r[0]+r[1]]) {
|
|
t.Fatalf("page %d: range %v before damage: %d bytes, %v", page, r, len(got), err)
|
|
}
|
|
}
|
|
|
|
damagedAt := int64(3*block + 500)
|
|
if _, err := v.DataBackend.WriteAt([]byte{^data[damagedAt]}, int64(raw)+types.NeedleHeaderSize+types.DataSizeSize+damagedAt); err != nil {
|
|
t.Fatalf("damage needle: %v", err)
|
|
}
|
|
for _, r := range ranges[:2] {
|
|
got, err := read(r[0], r[1])
|
|
if err == nil || !strings.Contains(err.Error(), "checksum") {
|
|
t.Fatalf("page %d: range %v over a block damaged after the check: %v; want a checksum error", page, r, err)
|
|
}
|
|
// Only bytes of blocks that passed go out, and they are intact.
|
|
if int64(len(got)) > max(0, 3*block-r[0]) || !bytes.Equal(got, data[r[0]:r[0]+int64(len(got))]) {
|
|
t.Fatalf("page %d: range %v wrote %d bytes reaching into the damaged block", page, r, len(got))
|
|
}
|
|
}
|
|
for _, r := range [][2]int64{ranges[2], {10, 100}, {2*block + 1, block - 1}} {
|
|
if got, err := read(r[0], r[1]); err != nil || !bytes.Equal(got, data[r[0]:r[0]+r[1]]) {
|
|
t.Fatalf("page %d: range %v over intact blocks: %d bytes, %v", page, r, len(got), err)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, tc := range []struct{ size, block int64 }{
|
|
{0, block}, {64 * block, block}, {64*block + 1, 2 * block}, {64 * PagedReadLimit, PagedReadLimit}, {1 << 34, PagedReadLimit},
|
|
} {
|
|
if got := rangeCheckBlockSize(tc.size); got != tc.block {
|
|
t.Errorf("rangeCheckBlockSize(%d) = %d, want %d", tc.size, got, tc.block)
|
|
}
|
|
}
|
|
}
|
|
|
|
// With MustMetaOnly set, the payload is skipped whatever the needle's size.
|
|
// This is the option a delete uses. A chunk manifest is the exception: its
|
|
// data is still read, because a delete needs it to find the chunks.
|
|
func TestReadNeedleMustMetaOnly(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
plain := newRandomNeedle(1)
|
|
plain.Cookie = 0x1234
|
|
plain.Name = []byte("a.jpg")
|
|
plain.NameSize = uint8(len(plain.Name))
|
|
plain.SetHasName()
|
|
plainOffset, _, _, err := v.writeNeedle2(plain, true, false, false)
|
|
if err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
manifest := newRandomNeedle(2)
|
|
manifest.SetIsChunkManifest()
|
|
if _, _, _, err := v.writeNeedle2(manifest, true, false, false); err != nil {
|
|
t.Fatalf("write chunk manifest: %v", err)
|
|
}
|
|
|
|
read := func(id uint64, option *ReadOption) (*needle.Needle, error) {
|
|
n := &needle.Needle{Id: types.Uint64ToNeedleId(id)}
|
|
_, err := v.readNeedle(n, option, nil)
|
|
return n, err
|
|
}
|
|
metaOnly := func() *ReadOption { return &ReadOption{AttemptMetaOnly: true, MustMetaOnly: true} }
|
|
|
|
option := metaOnly()
|
|
got, err := read(1, option)
|
|
if err != nil {
|
|
t.Fatalf("metadata read: %v", err)
|
|
}
|
|
if !option.IsMetaOnly || len(got.Data) != 0 {
|
|
t.Errorf("metadata read: IsMetaOnly %v, %d bytes of data", option.IsMetaOnly, len(got.Data))
|
|
}
|
|
if got.Cookie != plain.Cookie || got.Size != plain.Size || got.DataSize != plain.DataSize || string(got.Name) != "a.jpg" {
|
|
t.Errorf("metadata read: cookie %x size %d data size %d name %q", got.Cookie, got.Size, got.DataSize, got.Name)
|
|
}
|
|
|
|
// With AttemptMetaOnly alone, a small needle is still read in full.
|
|
option = &ReadOption{AttemptMetaOnly: true}
|
|
if got, err = read(1, option); err != nil || option.IsMetaOnly || !bytes.Equal(got.Data, plain.Data) {
|
|
t.Errorf("attempted metadata read of a small needle: IsMetaOnly %v, %d bytes, %v", option.IsMetaOnly, len(got.Data), err)
|
|
}
|
|
|
|
option = metaOnly()
|
|
if got, err = read(2, option); err != nil || option.IsMetaOnly || !bytes.Equal(got.Data, manifest.Data) {
|
|
t.Errorf("metadata read of a chunk manifest: IsMetaOnly %v, %d bytes, %v", option.IsMetaOnly, len(got.Data), err)
|
|
}
|
|
|
|
// A meta-only read never looks at the payload. Corrupt the payload and
|
|
// check that only a full read reports the damage.
|
|
damaged := []byte{^plain.Data[0]}
|
|
if _, err := v.DataBackend.WriteAt(damaged, int64(plainOffset)+types.NeedleHeaderSize+types.DataSizeSize); err != nil {
|
|
t.Fatalf("damage needle: %v", err)
|
|
}
|
|
if _, err := read(1, nil); !errors.Is(err, needle.ErrorCorrupted) {
|
|
t.Errorf("full read of a damaged needle: %v, want %v", err, needle.ErrorCorrupted)
|
|
}
|
|
if _, err := read(1, metaOnly()); err != nil {
|
|
t.Errorf("metadata read of a damaged needle: %v", err)
|
|
}
|
|
}
|
|
|
|
// HasRange skips the payload at any size, like MustMetaOnly, but the range
|
|
// read that follows reports the outcome, so the metadata read leaves the io
|
|
// error streak alone.
|
|
func TestReadNeedleHasRange(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
plain := &needle.Needle{Id: 1, Cookie: 0x1234, Data: bytes.Repeat([]byte("0123456789"), 400)}
|
|
plain.Checksum = needle.NewCRC(plain.Data)
|
|
if _, _, _, err := v.writeNeedle2(plain, true, false, false); err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
manifest := newRandomNeedle(2)
|
|
manifest.SetIsChunkManifest()
|
|
if _, _, _, err := v.writeNeedle2(manifest, true, false, false); err != nil {
|
|
t.Fatalf("write chunk manifest: %v", err)
|
|
}
|
|
compressed := newRandomNeedle(3)
|
|
compressed.SetIsCompressed()
|
|
if _, _, _, err := v.writeNeedle2(compressed, true, false, false); err != nil {
|
|
t.Fatalf("write compressed needle: %v", err)
|
|
}
|
|
|
|
v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
|
|
option := &ReadOption{AttemptMetaOnly: true, HasRange: true, ReadBufferSize: 1024}
|
|
got := &needle.Needle{Id: plain.Id}
|
|
if _, err := v.readNeedle(got, option, nil); err != nil {
|
|
t.Fatalf("metadata read: %v", err)
|
|
}
|
|
if !option.IsMetaOnly || len(got.Data) != 0 || got.Cookie != plain.Cookie || got.DataSize != plain.DataSize {
|
|
t.Errorf("metadata read: IsMetaOnly %v, %d bytes of data, cookie %x, data size %d", option.IsMetaOnly, len(got.Data), got.Cookie, got.DataSize)
|
|
}
|
|
if _, count, _ := v.getIoErrorState(); count != 1 {
|
|
t.Errorf("io error streak %d after the metadata read, want 1", count)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := v.readNeedleDataInto(got, option, &buf, 1500, 2000); err != nil {
|
|
t.Fatalf("range read: %v", err)
|
|
}
|
|
if !bytes.Equal(buf.Bytes(), plain.Data[1500:3500]) {
|
|
t.Errorf("range read returned %d bytes that differ from the needle", buf.Len())
|
|
}
|
|
if _, count, _ := v.getIoErrorState(); count != 0 {
|
|
t.Errorf("io error streak %d after the range read, want 0", count)
|
|
}
|
|
|
|
// A chunk manifest or a compressed needle is still read whole.
|
|
for _, want := range []*needle.Needle{manifest, compressed} {
|
|
option = &ReadOption{AttemptMetaOnly: true, HasRange: true}
|
|
got = &needle.Needle{Id: want.Id}
|
|
if _, err := v.readNeedle(got, option, nil); err != nil || option.IsMetaOnly || !bytes.Equal(got.Data, want.Data) {
|
|
t.Errorf("ranged read of needle %d: IsMetaOnly %v, %d bytes, %v", want.Id, option.IsMetaOnly, len(got.Data), err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A range streams from the record its metadata read resolved, not from a
|
|
// replacement written in between, whatever the replacement's size.
|
|
func TestReadNeedleDataIntoKeepsTheMetaRecord(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
write := func(data []byte) {
|
|
t.Helper()
|
|
n := &needle.Needle{Id: 1, Cookie: 0x1234, Data: data}
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
}
|
|
old := bytes.Repeat([]byte("old-"), 1000)
|
|
for _, replacement := range [][]byte{bytes.Repeat([]byte("new-"), 1000), bytes.Repeat([]byte("short"), 10)} {
|
|
write(old)
|
|
option := &ReadOption{AttemptMetaOnly: true, HasRange: true, ReadBufferSize: 1024}
|
|
got := &needle.Needle{Id: 1}
|
|
if _, err := v.readNeedle(got, option, nil); err != nil || !option.IsMetaOnly {
|
|
t.Fatalf("metadata read: IsMetaOnly %v, %v", option.IsMetaOnly, err)
|
|
}
|
|
write(replacement)
|
|
var buf bytes.Buffer
|
|
if err := v.readNeedleDataInto(got, option, &buf, 100, 2000); err != nil {
|
|
t.Fatalf("range read after a %d-byte replacement: %v", len(replacement), err)
|
|
}
|
|
if !bytes.Equal(buf.Bytes(), old[100:2100]) {
|
|
t.Errorf("range read after a %d-byte replacement returned bytes of another record", len(replacement))
|
|
}
|
|
}
|
|
}
|
|
|
|
// A compaction moves the record between the parts of a multi-range reply;
|
|
// every later part follows it, not only the first. A pre-3.09 checksum,
|
|
// which compaction rewrites in the current encoding, still matches.
|
|
func TestReadNeedleDataIntoFollowsACompaction(t *testing.T) {
|
|
for _, legacy := range []bool{false, true} {
|
|
t.Run(fmt.Sprintf("legacy checksum %v", legacy), func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
gone := &needle.Needle{Id: 1, Cookie: 0x1234, Data: bytes.Repeat([]byte("gone"), 1000)}
|
|
gone.Checksum = needle.NewCRC(gone.Data)
|
|
data := bytes.Repeat([]byte("0123456789"), 400)
|
|
kept := &needle.Needle{Id: 2, Cookie: 0x1234, Data: data}
|
|
kept.Checksum = needle.NewCRC(kept.Data)
|
|
if legacy {
|
|
kept.Checksum = needle.CRC(kept.Checksum.Value())
|
|
}
|
|
for _, n := range []*needle.Needle{gone, kept} {
|
|
if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
|
|
t.Fatalf("write needle %d: %v", n.Id, err)
|
|
}
|
|
}
|
|
if _, err := v.deleteNeedle2(newEmptyNeedle(1), false); err != nil {
|
|
t.Fatalf("delete: %v", err)
|
|
}
|
|
|
|
option := &ReadOption{AttemptMetaOnly: true, HasRange: true, ReadBufferSize: 1024}
|
|
got := &needle.Needle{Id: kept.Id}
|
|
if _, err := v.readNeedle(got, option, nil); err != nil || !option.IsMetaOnly {
|
|
t.Fatalf("metadata read: IsMetaOnly %v, %v", option.IsMetaOnly, err)
|
|
}
|
|
if err := v.CompactByIndex(nil); err != nil {
|
|
t.Fatalf("compact: %v", err)
|
|
}
|
|
if err := v.CommitCompact(); err != nil {
|
|
t.Fatalf("commit compact: %v", err)
|
|
}
|
|
|
|
for _, part := range [][2]int64{{0, 100}, {2000, 500}} {
|
|
var buf bytes.Buffer
|
|
if err := v.readNeedleDataInto(got, option, &buf, part[0], part[1]); err != nil {
|
|
t.Fatalf("part at %d: %v", part[0], err)
|
|
}
|
|
if !bytes.Equal(buf.Bytes(), data[part[0]:part[0]+part[1]]) {
|
|
t.Errorf("part at %d returned bytes of another record", part[0])
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A compaction that drops the record the metadata came from leaves only its
|
|
// replacement; a range then fails rather than send the replacement's bytes,
|
|
// even when the sizes match.
|
|
func TestReadNeedleDataIntoRefusesAReplacementAfterCompaction(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
|
|
write := func(data []byte) {
|
|
t.Helper()
|
|
n := &needle.Needle{Id: 1, Cookie: 0x1234, Data: data}
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
|
|
t.Fatalf("write needle: %v", err)
|
|
}
|
|
}
|
|
write(bytes.Repeat([]byte("old-"), 1000))
|
|
option := &ReadOption{AttemptMetaOnly: true, HasRange: true, ReadBufferSize: 1024}
|
|
got := &needle.Needle{Id: 1}
|
|
if _, err := v.readNeedle(got, option, nil); err != nil || !option.IsMetaOnly {
|
|
t.Fatalf("metadata read: IsMetaOnly %v, %v", option.IsMetaOnly, err)
|
|
}
|
|
write(bytes.Repeat([]byte("new-"), 1000))
|
|
if err := v.CompactByIndex(nil); err != nil {
|
|
t.Fatalf("compact: %v", err)
|
|
}
|
|
if err := v.CommitCompact(); err != nil {
|
|
t.Fatalf("commit compact: %v", err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := v.readNeedleDataInto(got, option, &buf, 100, 2000); err != ErrorNotFound || buf.Len() != 0 {
|
|
t.Errorf("range read after the record was replaced and compacted: %d bytes, %v; want %v", buf.Len(), err, ErrorNotFound)
|
|
}
|
|
}
|
|
|
|
// A version 1 needle has no data size field and no metadata after its data,
|
|
// so the metadata-only reader cannot parse it. A metadata-only read of a
|
|
// version 1 volume must fall back to a full read instead of failing.
|
|
func TestReadNeedleMetaOnlyVersion1(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("volume creation: %v", err)
|
|
}
|
|
defer v.Close()
|
|
// New volumes are always created with the current version, so switch this
|
|
// one to version 1 before anything is written to it.
|
|
v.volumeInfo.Version = uint32(needle.Version1)
|
|
if v.Version() != needle.Version1 {
|
|
t.Fatalf("volume version %d, want %d", v.Version(), needle.Version1)
|
|
}
|
|
|
|
// The payload starts with bytes that would be a huge data size if the
|
|
// metadata-only reader took them for the version 2 data size field.
|
|
small := &needle.Needle{Id: 1, Cookie: 0x1234, Data: []byte("hello world")}
|
|
small.Checksum = needle.NewCRC(small.Data)
|
|
if _, _, _, err := v.writeNeedle2(small, true, false, false); err != nil {
|
|
t.Fatalf("write small needle: %v", err)
|
|
}
|
|
large := &needle.Needle{Id: 2, Cookie: 0x5678, Data: bytes.Repeat([]byte("hello world "), PagedReadLimit/12+1)}
|
|
large.Checksum = needle.NewCRC(large.Data)
|
|
if _, _, _, err := v.writeNeedle2(large, true, false, false); err != nil {
|
|
t.Fatalf("write large needle: %v", err)
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
want *needle.Needle
|
|
option *ReadOption
|
|
}{
|
|
{"must be metadata only, small needle", small, &ReadOption{AttemptMetaOnly: true, MustMetaOnly: true}},
|
|
{"must be metadata only, large needle", large, &ReadOption{AttemptMetaOnly: true, MustMetaOnly: true}},
|
|
{"attempt metadata only, large needle", large, &ReadOption{AttemptMetaOnly: true}},
|
|
} {
|
|
got := &needle.Needle{Id: tc.want.Id}
|
|
if _, err := v.readNeedle(got, tc.option, nil); err != nil {
|
|
t.Errorf("%s: %v", tc.name, err)
|
|
continue
|
|
}
|
|
if tc.option.IsMetaOnly || got.Cookie != tc.want.Cookie || !bytes.Equal(got.Data, tc.want.Data) {
|
|
t.Errorf("%s: IsMetaOnly %v, cookie %x, %d bytes of data", tc.name, tc.option.IsMetaOnly, got.Cookie, len(got.Data))
|
|
}
|
|
}
|
|
}
|