seaweedfs/weed/storage/blockvol/recovery_test.go

package blockvol

import (
	"bytes"
	"path/filepath"
	"testing"
	"time"
)

func TestRecovery(t *testing.T) {
	tests := []struct {
		name string
		run  func(t *testing.T)
	}{
		{name: "recover_empty_wal", run: testRecoverEmptyWAL},
		{name: "recover_one_entry", run: testRecoverOneEntry},
		{name: "recover_many_entries", run: testRecoverManyEntries},
		{name: "recover_torn_write", run: testRecoverTornWrite},
		{name: "recover_after_checkpoint", run: testRecoverAfterCheckpoint},
		{name: "recover_idempotent", run: testRecoverIdempotent},
		{name: "recover_wal_full", run: testRecoverWALFull},
		{name: "recover_barrier_only", run: testRecoverBarrierOnly},
		{name: "recover_defensive_scan_finds_orphaned_entries", run: testRecoverDefensiveScan},
		{name: "recover_defensive_scan_empty_wal_noop", run: testRecoverDefensiveScanEmpty},
		{name: "recover_extended_scan_past_stale_head", run: testRecoverExtendedScanPastStaleHead},
		{name: "recover_extended_scan_no_superblock_persist", run: testRecoverNoSuperblockPersist},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			tt.run(t)
		})
	}
}

// simulateCrash closes the volume without syncing and returns the path.
// simulateCrash stops background goroutines and closes the fd without a clean
// shutdown. The caller is responsible for persisting the superblock AFTER
// stopping the flusher (to avoid concurrent superblock writes).
// For the common case, use simulateCrashWithSuper instead.
func simulateCrash(v *BlockVol) string {
	path := v.Path()
	v.groupCommit.Stop()
	v.flusher.Stop()
	v.fd.Close()
	return path
}

// simulateCrashWithSuper is the safe default: stops background goroutines,
// writes the superblock with current WAL positions, then closes the fd.
func simulateCrashWithSuper(v *BlockVol) string {
	path := v.Path()
	v.groupCommit.Stop()
	v.flusher.Stop()
	v.super.WALHead = v.wal.LogicalHead()
	v.super.WALTail = v.wal.LogicalTail()
	v.fd.Seek(0, 0)
	v.super.WriteTo(v.fd)
	v.fd.Sync()
	v.fd.Close()
	return path
}

func testRecoverEmptyWAL(t *testing.T) {
	v := createTestVol(t)
	// Sync to make superblock durable.
	v.fd.Sync()
	path := simulateCrash(v)

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	// Empty volume: read should return zeros.
	got, err := v2.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA: %v", err)
	}
	if !bytes.Equal(got, make([]byte, 4096)) {
		t.Error("expected zeros from empty volume")
	}
}

func testRecoverOneEntry(t *testing.T) {
	v := createTestVol(t)
	data := makeBlock('A')
	if err := v.WriteLBA(0, data); err != nil {
		t.Fatalf("WriteLBA: %v", err)
	}

	// Sync WAL to make the entry durable.
	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	path := simulateCrashWithSuper(v)

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	got, err := v2.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA after recovery: %v", err)
	}
	if !bytes.Equal(got, data) {
		t.Error("data not recovered")
	}
}

func testRecoverManyEntries(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "many.blockvol")
	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 4 * 1024 * 1024, // 4MB
		BlockSize:  4096,
		WALSize:    2 * 1024 * 1024, // 2MB WAL -- enough for ~500 entries
	})
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}

	const numWrites = 400
	for i := uint64(0); i < numWrites; i++ {
		if err := v.WriteLBA(i, makeBlock(byte(i%26+'A'))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	path = simulateCrashWithSuper(v)

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	for i := uint64(0); i < numWrites; i++ {
		got, err := v2.ReadLBA(i, 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d) after recovery: %v", i, err)
		}
		expected := makeBlock(byte(i%26 + 'A'))
		if !bytes.Equal(got, expected) {
			t.Errorf("block %d: data mismatch after recovery", i)
		}
	}
}

func testRecoverTornWrite(t *testing.T) {
	v := createTestVol(t)

	// Write 2 entries.
	if err := v.WriteLBA(0, makeBlock('A')); err != nil {
		t.Fatalf("WriteLBA(0): %v", err)
	}
	if err := v.WriteLBA(1, makeBlock('B')); err != nil {
		t.Fatalf("WriteLBA(1): %v", err)
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	// Stop background goroutines before manual superblock write.
	v.groupCommit.Stop()
	v.flusher.Stop()

	// Save superblock with correct WAL state.
	v.super.WALHead = v.wal.LogicalHead()
	v.super.WALTail = v.wal.LogicalTail()
	v.fd.Seek(0, 0)
	v.super.WriteTo(v.fd)
	v.fd.Sync()

	// Corrupt the last 2 bytes of the second entry to simulate torn write.
	entrySize := uint64(walEntryHeaderSize + 4096)
	secondEntryEnd := v.super.WALOffset + entrySize*2
	corruptOff := int64(secondEntryEnd - 2)
	v.fd.WriteAt([]byte{0xFF, 0xFF}, corruptOff)
	v.fd.Sync()

	v.fd.Close()
	path := v.Path()

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	// First entry should be recovered.
	got, err := v2.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA(0) after torn recovery: %v", err)
	}
	if !bytes.Equal(got, makeBlock('A')) {
		t.Error("block 0 should be recovered")
	}

	// Second entry was torn -- should NOT be in dirty map.
	// Read returns zeros (from extent).
	got, err = v2.ReadLBA(1, 4096)
	if err != nil {
		t.Fatalf("ReadLBA(1) after torn recovery: %v", err)
	}
	if !bytes.Equal(got, make([]byte, 4096)) {
		t.Error("block 1 (torn) should return zeros")
	}
}

func testRecoverAfterCheckpoint(t *testing.T) {
	v := createTestVol(t)

	// Write blocks 0-9.
	for i := uint64(0); i < 10; i++ {
		if err := v.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	// Flush first 5 blocks (flusher moves them to extent, advances checkpoint).
	f := NewFlusher(FlusherConfig{
		FD:       v.fd,
		Super:    &v.super,
		SuperMu:  &v.superMu,
		WAL:      v.wal,
		DirtyMap: v.dirtyMap,
		Interval: 1 * time.Hour,
	})

	// Flush all (flusher takes all dirty entries).
	// To simulate partial flush, we'll manually set checkpoint to midpoint.
	if err := f.FlushOnce(); err != nil {
		t.Fatalf("FlushOnce: %v", err)
	}

	// Now write 5 more blocks (these will need replay after crash).
	for i := uint64(10); i < 15; i++ {
		if err := v.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	path := simulateCrashWithSuper(v)

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	// Blocks 0-9 should be in extent (flushed). Blocks 10-14 replayed from WAL.
	for i := uint64(0); i < 15; i++ {
		got, err := v2.ReadLBA(i, 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d): %v", i, err)
		}
		expected := makeBlock(byte('A' + i))
		if !bytes.Equal(got, expected) {
			t.Errorf("block %d: data mismatch after checkpoint recovery", i)
		}
	}
}

func testRecoverIdempotent(t *testing.T) {
	v := createTestVol(t)
	data := makeBlock('X')
	if err := v.WriteLBA(0, data); err != nil {
		t.Fatalf("WriteLBA: %v", err)
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	path := simulateCrashWithSuper(v)

	// First recovery.
	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol 1: %v", err)
	}

	// Verify data.
	got, err := v2.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA 1: %v", err)
	}
	if !bytes.Equal(got, data) {
		t.Error("first recovery: data mismatch")
	}

	// Close and recover again (should be idempotent).
	path2 := simulateCrash(v2)

	v3, err := OpenBlockVol(path2)
	if err != nil {
		t.Fatalf("OpenBlockVol 2: %v", err)
	}
	defer v3.Close()

	got, err = v3.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA 2: %v", err)
	}
	if !bytes.Equal(got, data) {
		t.Error("second recovery: data mismatch")
	}
}

func testRecoverWALFull(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "full.blockvol")

	entrySize := uint64(walEntryHeaderSize + 4096)
	walSize := entrySize * 5 // room for exactly 5 entries

	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 * 1024 * 1024,
		BlockSize:  4096,
		WALSize:    walSize,
	})
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}

	// Write exactly 5 entries to fill WAL.
	for i := uint64(0); i < 5; i++ {
		if err := v.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	path = simulateCrashWithSuper(v)

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	for i := uint64(0); i < 5; i++ {
		got, err := v2.ReadLBA(i, 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d): %v", i, err)
		}
		expected := makeBlock(byte('A' + i))
		if !bytes.Equal(got, expected) {
			t.Errorf("block %d: data mismatch after full WAL recovery", i)
		}
	}
}

func testRecoverBarrierOnly(t *testing.T) {
	v := createTestVol(t)

	// Write a barrier entry.
	lsn := v.nextLSN.Add(1) - 1
	entry := &WALEntry{
		LSN:  lsn,
		Type: EntryTypeBarrier,
		LBA:  0,
	}
	if _, err := v.wal.Append(entry); err != nil {
		t.Fatalf("Append barrier: %v", err)
	}

	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	path := simulateCrashWithSuper(v)

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	// No data changes from barrier.
	got, err := v2.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA: %v", err)
	}
	if !bytes.Equal(got, make([]byte, 4096)) {
		t.Error("barrier-only WAL should leave data as zeros")
	}
}

// testRecoverDefensiveScan verifies Fix A: when superblock has WALHead=0
// WALTail=0 CheckpointLSN=0 but valid entries exist in the WAL region,
// the defensive scan finds and replays them.
func testRecoverDefensiveScan(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "test.blockvol")

	// Create volume and write data.
	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 << 20,
		WALSize:    64 << 20,
	})
	if err != nil {
		t.Fatal(err)
	}
	v.SetRole(RolePrimary)
	v.SetEpoch(1)
	v.SetMasterEpoch(1)
	v.lease.Grant(30 * time.Second)

	data := make([]byte, 4096)
	for i := range data {
		data[i] = 'D'
	}
	if err := v.WriteLBA(0, data); err != nil {
		t.Fatalf("WriteLBA: %v", err)
	}
	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	// With the optimized group commit (plain fd.Sync, no superblock persist),
	// WALHead stays 0 after write+sync. The extended recovery scan handles this.
	// Crash without updating superblock.
	path = simulateCrash(v)

	// Reopen — should trigger defensive scan and recover the entry.
	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol after corrupted superblock: %v", err)
	}
	defer v2.Close()

	v2.SetRole(RolePrimary)
	v2.SetEpoch(1)
	v2.SetMasterEpoch(1)
	v2.lease.Grant(10 * time.Second)

	// Read back — should get 'D', not zeros.
	got, err := v2.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA after defensive scan: %v", err)
	}
	if got[0] != 'D' {
		t.Fatalf("LBA 0: got %c, want D — defensive scan failed to recover", got[0])
	}
}

// testRecoverDefensiveScanEmpty verifies that on a genuinely empty WAL
// (fresh volume, no writes), the defensive scan triggers but finds nothing.
// No false positives — zero entries replayed.
func testRecoverDefensiveScanEmpty(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "test.blockvol")

	// Create volume with no writes.
	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 << 20,
		WALSize:    64 << 20,
	})
	if err != nil {
		t.Fatal(err)
	}
	v.Close()

	// Reset superblock to zeros (simulates fresh state).
	// On a genuinely fresh volume, WALHead=0 WALTail=0 is correct.
	// The defensive scan should find zero valid entries.
	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	// If we get here without error, the scan didn't crash on empty WAL. PASS.
}

// testRecoverExtendedScanPastStaleHead verifies that recovery finds entries
// written after the last superblock persist. Simulates: write 5 entries with
// WALHead at entry 3 (stale), crash, recovery should find all 5.
func testRecoverExtendedScanPastStaleHead(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "test.blockvol")

	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 << 20,
		WALSize:    64 << 20,
	})
	if err != nil {
		t.Fatal(err)
	}
	v.SetRole(RolePrimary)
	v.SetEpoch(1)
	v.SetMasterEpoch(1)
	v.lease.Grant(30 * time.Second)

	// Write 3 entries and persist superblock (WALHead covers them).
	for i := uint64(0); i < 3; i++ {
		if err := v.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}
	if err := v.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Save superblock with current WALHead (covers entries 0-2).
	v.groupCommit.Stop()
	v.flusher.Stop()
	staleHead := v.wal.LogicalHead()
	v.super.WALHead = staleHead
	v.super.WALTail = v.wal.LogicalTail()
	v.fd.Seek(0, 0)
	v.super.WriteTo(v.fd)
	v.fd.Sync()

	// Restart group commit for more writes.
	v.groupCommit = NewGroupCommitter(GroupCommitterConfig{
		SyncFunc: v.fd.Sync,
	})
	go v.groupCommit.Run()

	// Write 2 more entries WITHOUT updating superblock.
	for i := uint64(3); i < 5; i++ {
		if err := v.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}
	if err := v.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Crash without updating superblock — WALHead is stale at entry 3.
	v.groupCommit.Stop()
	v.fd.Close()

	// Recovery should find ALL 5 entries via extended scan past head.
	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	v2.SetRole(RolePrimary)
	v2.SetEpoch(1)
	v2.SetMasterEpoch(1)
	v2.lease.Grant(10 * time.Second)

	for i := uint64(0); i < 5; i++ {
		got, err := v2.ReadLBA(i, 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d): %v", i, err)
		}
		expected := makeBlock(byte('A' + i))
		if !bytes.Equal(got, expected) {
			t.Errorf("block %d: expected %c, got %c — extended scan missed entry past stale WALHead",
				i, 'A'+i, got[0])
		}
	}
}

// testRecoverNoSuperblockPersist verifies the fast-path optimization:
// group commit uses plain fd.Sync (no superblock write), and recovery
// still finds all entries via extended scan. This is the exact production
// scenario after removing syncWithWALProgress from the group commit path.
func testRecoverNoSuperblockPersist(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "test.blockvol")

	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 << 20,
		WALSize:    64 << 20,
	})
	if err != nil {
		t.Fatal(err)
	}
	v.SetRole(RolePrimary)
	v.SetEpoch(1)
	v.SetMasterEpoch(1)
	v.lease.Grant(30 * time.Second)

	// Write 10 entries. Group commit uses fd.Sync (no superblock persist).
	// Superblock WALHead stays at 0 (initial value from CreateBlockVol).
	for i := uint64(0); i < 10; i++ {
		if err := v.WriteLBA(i, makeBlock(byte('0'+i))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}
	if err := v.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Crash — superblock WALHead is still at initial value.
	path = simulateCrash(v)

	// Recovery must find all 10 entries via extended/defensive scan.
	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("OpenBlockVol: %v", err)
	}
	defer v2.Close()

	v2.SetRole(RolePrimary)
	v2.SetEpoch(1)
	v2.SetMasterEpoch(1)
	v2.lease.Grant(10 * time.Second)

	for i := uint64(0); i < 10; i++ {
		got, err := v2.ReadLBA(i, 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d): %v", i, err)
		}
		expected := makeBlock(byte('0' + i))
		if !bytes.Equal(got, expected) {
			t.Errorf("block %d: expected %c, got %c — recovery without superblock persist failed",
				i, '0'+i, got[0])
		}
	}
}