seaweedfs/weed/storage/blockvol/qa_phase4a_cp1_test.go

package blockvol

import (
	"errors"
	"math"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

// TestQAPhase4ACP1 tests Phase 4A CP1 fencing primitives adversarially:
// epoch, lease, role state machine, write gate, and full integration.
func TestQAPhase4ACP1(t *testing.T) {
	tests := []struct {
		name string
		run  func(t *testing.T)
	}{
		// QA-4A-1: Epoch Adversarial
		{name: "epoch_concurrent_set_epoch", run: testQAEpochConcurrentSetEpoch},
		{name: "epoch_set_during_write", run: testQAEpochSetDuringWrite},
		{name: "epoch_max_uint64", run: testQAEpochMaxUint64},
		{name: "epoch_reopen_after_concurrent_sets", run: testQAEpochReopenAfterConcurrentSets},

		// QA-4A-2: Lease Adversarial
		{name: "lease_grant_zero_ttl", run: testQALeaseGrantZeroTTL},
		{name: "lease_grant_negative_ttl", run: testQALeaseGrantNegativeTTL},
		{name: "lease_concurrent_grant_revoke", run: testQALeaseConcurrentGrantRevoke},
		{name: "lease_expiry_boundary_1ms", run: testQALeaseExpiryBoundary1ms},
		{name: "lease_revoke_during_write", run: testQALeaseRevokeDuringWrite},

		// QA-4A-3: Role State Machine Adversarial
		{name: "role_concurrent_transitions", run: testQARoleConcurrentTransitions},
		{name: "role_rapid_cycle_100x", run: testQARoleRapidCycle100x},
		{name: "role_callback_panic_recovery", run: testQARoleCallbackPanicRecovery},
		{name: "role_callback_slow_blocks_transition", run: testQARoleCallbackSlowBlocksTransition},
		{name: "role_unknown_value", run: testQARoleUnknownValue},

		// QA-4A-4: Write Gate Adversarial
		{name: "gate_role_change_during_write", run: testQAGateRoleChangeDuringWrite},
		{name: "gate_epoch_bump_during_write", run: testQAGateEpochBumpDuringWrite},
		{name: "gate_lease_expire_during_write", run: testQAGateLeaseExpireDuringWrite},
		{name: "gate_concurrent_100_writers_role_flip", run: testQAGateConcurrent100WritersRoleFlip},
		{name: "gate_gotcha_a_lease_expires_after_wal_before_sync", run: testQAGateGotchaA},

		// QA-4A-5: Integration Adversarial
		{name: "fencing_full_cycle_primary_to_stale", run: testQAFencingFullCycle},
		{name: "fencing_writes_rejected_after_demotion", run: testQAFencingWritesRejectedAfterDemotion},
		{name: "fencing_read_always_works", run: testQAFencingReadAlwaysWorks},
		{name: "fencing_close_during_role_transition", run: testQAFencingCloseDuringRoleTransition},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			tt.run(t)
		})
	}
}

// --- helpers ---

// createFencedVol creates a blockvol configured as Primary with matching epochs and a valid lease.
func createFencedVol(t *testing.T, opts ...CreateOptions) *BlockVol {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "fenced.blockvol")

	o := CreateOptions{
		VolumeSize: 1 * 1024 * 1024, // 1MB
		BlockSize:  4096,
		WALSize:    256 * 1024, // 256KB
	}
	if len(opts) > 0 {
		o = opts[0]
	}

	cfg := DefaultConfig()
	cfg.FlushInterval = 5 * time.Millisecond

	v, err := CreateBlockVol(path, o, cfg)
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}

	// Set up fencing: epoch=1, masterEpoch=1, role=Primary, lease=10s.
	if err := v.SetEpoch(1); err != nil {
		t.Fatalf("SetEpoch: %v", err)
	}
	v.SetMasterEpoch(1)
	if err := v.SetRole(RolePrimary); err != nil {
		t.Fatalf("SetRole(Primary): %v", err)
	}
	v.lease.Grant(10 * time.Second)

	return v
}

// --- QA-4A-1: Epoch Adversarial ---

func testQAEpochConcurrentSetEpoch(t *testing.T) {
	// BUG-4A-1 + BUG-4A-3: concurrent SetEpoch + SetMasterEpoch + WriteLBA.
	// Under -race, expects data race on v.epoch / v.masterEpoch (plain uint64).
	v := createFencedVol(t)
	defer v.Close()

	const goroutines = 8
	const iterations = 50

	var wg sync.WaitGroup

	// Epoch/masterEpoch writers.
	wg.Add(goroutines)
	for g := 0; g < goroutines; g++ {
		go func(id int) {
			defer wg.Done()
			for i := 0; i < iterations; i++ {
				epoch := uint64(id*iterations + i + 1)
				_ = v.SetEpoch(epoch)
				v.SetMasterEpoch(epoch)
			}
		}(g)
	}

	// Concurrent writer.
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < iterations*2; i++ {
			_ = v.WriteLBA(uint64(i%256), makeBlock(byte('A'+i%26)))
		}
	}()

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatal("concurrent SetEpoch + WriteLBA hung for 10s")
	}

	// Verify: superblock is readable and epoch is a valid value (not corrupted).
	finalEpoch := v.Epoch()
	if finalEpoch == 0 || finalEpoch > uint64(goroutines*iterations+1) {
		t.Errorf("epoch = %d, want [1, %d]", finalEpoch, goroutines*iterations+1)
	}

	f, err := os.Open(v.Path())
	if err != nil {
		t.Fatalf("open for verify: %v", err)
	}
	defer f.Close()
	sb, err := ReadSuperblock(f)
	if err != nil {
		t.Fatalf("ReadSuperblock after concurrent sets: %v (superblock corrupted)", err)
	}
	if err := sb.Validate(); err != nil {
		t.Fatalf("superblock validation failed: %v", err)
	}
	t.Logf("final superblock epoch: %d", sb.Epoch)
}

func testQAEpochSetDuringWrite(t *testing.T) {
	// BUG-4A-1: WriteLBA reads v.epoch (plain uint64) while SetEpoch writes it.
	v := createFencedVol(t)
	defer v.Close()

	const writeIter = 200
	const epochBumps = 100

	var wg sync.WaitGroup

	// Writer loop.
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < writeIter; i++ {
			_ = v.WriteLBA(uint64(i%256), makeBlock(byte('W')))
		}
	}()

	// Epoch bumper.
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < epochBumps; i++ {
			epoch := uint64(i + 2)
			_ = v.SetEpoch(epoch)
			v.SetMasterEpoch(epoch)
			time.Sleep(100 * time.Microsecond)
		}
	}()

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatal("epoch_set_during_write hung for 10s")
	}

	// Close and reopen -- verify no corruption and epoch in expected range.
	path := v.Path()
	v.Close()

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("reopen: %v (superblock may be corrupted)", err)
	}
	defer v2.Close()
	e := v2.Epoch()
	if e < 2 || e > uint64(epochBumps+1) {
		t.Errorf("reopened epoch = %d, want [2, %d]", e, epochBumps+1)
	}
	t.Logf("reopened epoch: %d", e)
}

func testQAEpochMaxUint64(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Set both epochs to MaxUint64.
	if err := v.SetEpoch(math.MaxUint64); err != nil {
		t.Fatalf("SetEpoch(MaxUint64): %v", err)
	}
	v.SetMasterEpoch(math.MaxUint64)

	if v.Epoch() != math.MaxUint64 {
		t.Errorf("Epoch() = %d, want MaxUint64", v.Epoch())
	}

	// WriteLBA should succeed (epochs match, role=Primary, lease valid).
	if err := v.WriteLBA(0, makeBlock('M')); err != nil {
		t.Fatalf("WriteLBA at MaxUint64 epoch: %v", err)
	}

	// Verify read.
	data, err := v.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA: %v", err)
	}
	if data[0] != 'M' {
		t.Errorf("data[0] = %c, want M", data[0])
	}

	// Close, reopen, verify epoch persisted.
	path := v.Path()
	v.Close()

	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("reopen: %v", err)
	}
	defer v2.Close()
	if v2.Epoch() != math.MaxUint64 {
		t.Errorf("reopened Epoch() = %d, want MaxUint64", v2.Epoch())
	}
}

func testQAEpochReopenAfterConcurrentSets(t *testing.T) {
	// BUG-4A-3: concurrent SetEpoch -> close -> reopen -> verify no corruption.
	dir := t.TempDir()
	path := filepath.Join(dir, "epoch_reopen.blockvol")

	cfg := DefaultConfig()
	cfg.FlushInterval = 5 * time.Millisecond

	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 * 1024 * 1024,
		BlockSize:  4096,
		WALSize:    256 * 1024,
	}, cfg)
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}

	const goroutines = 4
	const iterations = 50
	validEpochs := make(map[uint64]bool)
	for g := 0; g < goroutines; g++ {
		for i := 0; i < iterations; i++ {
			validEpochs[uint64(g*iterations+i+1)] = true
		}
	}

	var wg sync.WaitGroup
	wg.Add(goroutines)
	for g := 0; g < goroutines; g++ {
		go func(id int) {
			defer wg.Done()
			for i := 0; i < iterations; i++ {
				_ = v.SetEpoch(uint64(id*iterations + i + 1))
			}
		}(g)
	}
	wg.Wait()

	v.Close()

	// Reopen and verify epoch is one of the valid values.
	v2, err := OpenBlockVol(path)
	if err != nil {
		t.Fatalf("reopen after concurrent sets: %v", err)
	}
	defer v2.Close()

	e := v2.Epoch()
	if !validEpochs[e] && e != 0 {
		t.Errorf("reopened epoch %d is not a valid epoch from any goroutine", e)
	}

	// Also verify superblock raw decode works.
	f, err := os.Open(path)
	if err != nil {
		t.Fatalf("open raw: %v", err)
	}
	defer f.Close()
	sb, err := ReadSuperblock(f)
	if err != nil {
		t.Fatalf("raw ReadSuperblock: %v", err)
	}
	if err := sb.Validate(); err != nil {
		t.Fatalf("superblock validation: %v", err)
	}
	t.Logf("reopened epoch: %d, superblock valid", e)
}

// --- QA-4A-2: Lease Adversarial ---

func testQALeaseGrantZeroTTL(t *testing.T) {
	// BUG-4A-5: Grant(0) stores time.Now() -- immediately expired.
	var l Lease
	l.Grant(0)
	if l.IsValid() {
		t.Error("Grant(0): IsValid() = true, want false (immediately expired)")
	}

	// Grant(1ns) -- may or may not be valid, but must not panic.
	l.Grant(time.Nanosecond)
	_ = l.IsValid() // just verify no panic
}

func testQALeaseGrantNegativeTTL(t *testing.T) {
	// BUG-4A-5: Grant(-1s) stores past time -- always expired.
	var l Lease
	l.Grant(-1 * time.Second)
	if l.IsValid() {
		t.Error("Grant(-1s): IsValid() = true, want false")
	}

	// Verify WriteLBA returns ErrLeaseExpired with a negative-TTL lease.
	v := createFencedVol(t)
	defer v.Close()
	v.lease.Grant(-1 * time.Second)

	err := v.WriteLBA(0, makeBlock('N'))
	if !errors.Is(err, ErrLeaseExpired) {
		t.Errorf("WriteLBA with negative lease: got %v, want ErrLeaseExpired", err)
	}
}

func testQALeaseConcurrentGrantRevoke(t *testing.T) {
	// Concurrent Grant + Revoke + IsValid: no panic, no data race.
	var l Lease

	const granters = 4
	const revokers = 4
	const duration = 500 * time.Millisecond

	var wg sync.WaitGroup
	stop := make(chan struct{})

	// Granters.
	wg.Add(granters)
	for i := 0; i < granters; i++ {
		go func() {
			defer wg.Done()
			for {
				select {
				case <-stop:
					return
				default:
					l.Grant(1 * time.Second)
				}
			}
		}()
	}

	// Revokers.
	wg.Add(revokers)
	for i := 0; i < revokers; i++ {
		go func() {
			defer wg.Done()
			for {
				select {
				case <-stop:
					return
				default:
					l.Revoke()
				}
			}
		}()
	}

	// IsValid checker.
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
				_ = l.IsValid()
			}
		}
	}()

	time.Sleep(duration)
	close(stop)

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("concurrent grant/revoke hung for 5s")
	}
}

func testQALeaseExpiryBoundary1ms(t *testing.T) {
	var l Lease
	l.Grant(1 * time.Millisecond)

	// Busy-poll until expired.
	deadline := time.After(50 * time.Millisecond)
	for l.IsValid() {
		select {
		case <-deadline:
			t.Fatal("lease still valid after 50ms (granted 1ms)")
		default:
			// spin
		}
	}

	// Now verify WriteLBA fails.
	v := createFencedVol(t)
	defer v.Close()
	v.lease.Grant(1 * time.Millisecond)
	time.Sleep(5 * time.Millisecond) // ensure expired

	err := v.WriteLBA(0, makeBlock('E'))
	if !errors.Is(err, ErrLeaseExpired) {
		t.Errorf("WriteLBA after 1ms lease expiry: got %v, want ErrLeaseExpired", err)
	}
}

func testQALeaseRevokeDuringWrite(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Write some data first.
	for i := 0; i < 50; i++ {
		if err := v.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))); err != nil {
			t.Fatalf("pre-revoke WriteLBA(%d): %v", i, err)
		}
	}

	// Revoke lease.
	v.lease.Revoke()

	// Subsequent writes must fail.
	err := v.WriteLBA(50, makeBlock('Z'))
	if !errors.Is(err, ErrLeaseExpired) {
		t.Errorf("post-revoke WriteLBA: got %v, want ErrLeaseExpired", err)
	}

	// Reads should still work (ReadLBA doesn't check writeGate).
	for i := 0; i < 50; i++ {
		data, err := v.ReadLBA(uint64(i), 4096)
		if err != nil {
			t.Fatalf("post-revoke ReadLBA(%d): %v", i, err)
		}
		if data[0] != byte('A'+i%26) {
			t.Errorf("LBA %d: data[0] = %c, want %c", i, data[0], byte('A'+i%26))
		}
	}
}

// --- QA-4A-3: Role State Machine Adversarial ---

func testQARoleConcurrentTransitions(t *testing.T) {
	// BUG-4A-2: TOCTOU in SetRole. Two goroutines both see old=Stale,
	// both pass validation, both Store. One should fail (ideally).
	v := createFencedVol(t)
	defer v.Close()

	// Get to Stale: Primary -> Draining -> Stale.
	if err := v.SetRole(RoleDraining); err != nil {
		t.Fatalf("SetRole(Draining): %v", err)
	}
	if err := v.SetRole(RoleStale); err != nil {
		t.Fatalf("SetRole(Stale): %v", err)
	}

	// Now from Stale, both Rebuilding and Replica are valid transitions.
	var wg sync.WaitGroup
	var errA, errB error
	wg.Add(2)
	go func() {
		defer wg.Done()
		errA = v.SetRole(RoleRebuilding)
	}()
	go func() {
		defer wg.Done()
		errB = v.SetRole(RoleReplica)
	}()
	wg.Wait()

	// With CAS fix: exactly one must succeed, one must get ErrInvalidRoleTransition.
	// If both succeed, CAS was reverted to Load+Store (BUG-4A-2 regression).
	finalRole := v.Role()
	t.Logf("concurrent transitions from Stale: errA=%v errB=%v finalRole=%s",
		errA, errB, finalRole)

	if errA == nil && errB == nil {
		t.Error("BUG-4A-2 REGRESSION: both transitions succeeded -- CAS not working")
	}
	if errA != nil && errB != nil {
		t.Error("both transitions failed -- unexpected, at least one should succeed")
	}
	if finalRole != RoleRebuilding && finalRole != RoleReplica {
		t.Errorf("final role = %s, want Rebuilding or Replica", finalRole)
	}
}

func testQARoleRapidCycle100x(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	var callbackCount atomic.Int64
	v.SetRoleCallback(func(old, new Role) {
		callbackCount.Add(1)
	})

	// Full cycle: Primary->Draining->Stale->Rebuilding->Replica->Primary
	cycle := []Role{RoleDraining, RoleStale, RoleRebuilding, RoleReplica, RolePrimary}

	const rounds = 100
	expectedCallbacks := int64(0)
	for r := 0; r < rounds; r++ {
		for _, target := range cycle {
			if err := v.SetRole(target); err != nil {
				t.Fatalf("round %d: SetRole(%s): %v", r, target, err)
			}
			expectedCallbacks++
		}
	}

	if v.Role() != RolePrimary {
		t.Errorf("final role = %s, want Primary", v.Role())
	}

	got := callbackCount.Load()
	if got != expectedCallbacks {
		t.Errorf("callback count = %d, want %d", got, expectedCallbacks)
	}
	t.Logf("rapid cycle: %d rounds, %d callbacks", rounds, got)
}

func testQARoleCallbackPanicRecovery(t *testing.T) {
	// BUG-4A-6 FIXED: callback panic is now recovered by safeCallback.
	v := createFencedVol(t)
	defer v.Close()

	v.SetRoleCallback(func(old, new Role) {
		panic("callback panic!")
	})

	// SetRole(Draining) -- callback panics but safeCallback recovers it.
	// SetRole should return nil (CAS succeeded), not propagate the panic.
	err := v.SetRole(RoleDraining)
	if err != nil {
		t.Fatalf("SetRole(Draining) returned error: %v (expected nil after panic recovery)", err)
	}

	// Role should be Draining (CAS happened before callback).
	if v.Role() != RoleDraining {
		t.Errorf("role after recovered panic = %s, want Draining", v.Role())
	}
}

func testQARoleCallbackSlowBlocksTransition(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	callbackDone := make(chan struct{})
	v.SetRoleCallback(func(old, new Role) {
		time.Sleep(100 * time.Millisecond)
		close(callbackDone)
	})

	// SetRole in goroutine (will be slow due to callback).
	transitionDone := make(chan error, 1)
	go func() {
		transitionDone <- v.SetRole(RoleDraining)
	}()

	// WriteLBA checks role atomically. Since CAS stores the new role before
	// callback runs, writes must see Draining after a short wait.
	time.Sleep(10 * time.Millisecond) // let CAS+Store happen

	err := v.WriteLBA(0, makeBlock('S'))
	if !errors.Is(err, ErrNotPrimary) {
		t.Errorf("write during slow callback: got %v, want ErrNotPrimary (role should already be Draining)", err)
	}

	// Wait for callback to complete.
	select {
	case <-callbackDone:
	case <-time.After(5 * time.Second):
		t.Fatal("slow callback hung for 5s")
	}

	select {
	case err := <-transitionDone:
		if err != nil {
			t.Errorf("SetRole(Draining): %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("SetRole hung for 5s")
	}
}

func testQARoleUnknownValue(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Force an unknown role value (255) via atomic store.
	v.role.Store(255)

	// writeGate should reject (255 != RolePrimary).
	err := v.writeGate()
	if !errors.Is(err, ErrNotPrimary) {
		t.Errorf("writeGate with role=255: got %v, want ErrNotPrimary", err)
	}

	// SetRole(Primary) from unknown role -- not in transition table.
	err = v.SetRole(RolePrimary)
	if !errors.Is(err, ErrInvalidRoleTransition) {
		t.Errorf("SetRole(Primary) from role=255: got %v, want ErrInvalidRoleTransition", err)
	}

	// WriteLBA should also fail.
	err = v.WriteLBA(0, makeBlock('U'))
	if !errors.Is(err, ErrNotPrimary) {
		t.Errorf("WriteLBA with role=255: got %v, want ErrNotPrimary", err)
	}
}

// --- QA-4A-4: Write Gate Adversarial ---

func testQAGateRoleChangeDuringWrite(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	const writers = 16
	const writesPerGoroutine = 50

	var wg sync.WaitGroup
	var succeeded, rejected atomic.Int64

	// Many concurrent writers.
	wg.Add(writers)
	for g := 0; g < writers; g++ {
		go func(id int) {
			defer wg.Done()
			for i := 0; i < writesPerGoroutine; i++ {
				lba := uint64((id*writesPerGoroutine + i) % 256)
				err := v.WriteLBA(lba, makeBlock(byte('A'+id%26)))
				if err == nil {
					succeeded.Add(1)
				} else if errors.Is(err, ErrNotPrimary) {
					rejected.Add(1)
				}
				// other errors (WAL full, etc.) are ok
			}
		}(g)
	}

	// After 10ms, flip role.
	time.Sleep(10 * time.Millisecond)
	if err := v.SetRole(RoleDraining); err != nil {
		t.Logf("SetRole(Draining): %v", err)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatal("gate_role_change_during_write hung for 10s")
	}

	t.Logf("role flip: %d succeeded, %d rejected (ErrNotPrimary)",
		succeeded.Load(), rejected.Load())

	// After transition, ALL new writes must fail.
	err := v.WriteLBA(0, makeBlock('X'))
	if !errors.Is(err, ErrNotPrimary) {
		t.Errorf("write after Draining: got %v, want ErrNotPrimary", err)
	}
}

func testQAGateEpochBumpDuringWrite(t *testing.T) {
	// BUG-4A-1: data race on v.epoch reads in writeGate/WriteLBA vs SetEpoch writes.
	// Use large WAL + short WALFullTimeout to avoid writers blocking on WAL-full.
	dir := t.TempDir()
	path := filepath.Join(dir, "epoch_bump.blockvol")
	cfg := DefaultConfig()
	cfg.FlushInterval = 5 * time.Millisecond
	cfg.WALFullTimeout = 200 * time.Millisecond

	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 * 1024 * 1024,
		BlockSize:  4096,
		WALSize:    1 * 1024 * 1024, // 1MB WAL -- plenty of room
	}, cfg)
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}
	defer v.Close()

	// Set up fencing.
	if err := v.SetEpoch(1); err != nil {
		t.Fatalf("SetEpoch: %v", err)
	}
	v.SetMasterEpoch(1)
	if err := v.SetRole(RolePrimary); err != nil {
		t.Fatalf("SetRole: %v", err)
	}
	v.lease.Grant(10 * time.Second)

	const writers = 8
	const iterations = 30

	var wg sync.WaitGroup

	// Writers.
	wg.Add(writers)
	for g := 0; g < writers; g++ {
		go func(id int) {
			defer wg.Done()
			for i := 0; i < iterations; i++ {
				_ = v.WriteLBA(uint64((id*iterations+i)%256), makeBlock(byte('A'+id%26)))
			}
		}(g)
	}

	// Epoch bumper: set epoch=2, masterEpoch=2.
	wg.Add(1)
	go func() {
		defer wg.Done()
		time.Sleep(5 * time.Millisecond)
		_ = v.SetEpoch(2)
		v.SetMasterEpoch(2)
	}()

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatal("gate_epoch_bump_during_write hung for 10s")
	}

	// After bump, both epochs are 2 -- writes must succeed.
	if v.Epoch() != 2 {
		t.Errorf("Epoch() = %d after bump, want 2", v.Epoch())
	}
	if writeErr := v.WriteLBA(0, makeBlock('Z')); writeErr != nil {
		t.Errorf("write after epoch bump: %v (epochs should match)", writeErr)
	}
}

func testQAGateLeaseExpireDuringWrite(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Short lease -- write in a loop that spans the expiry window.
	v.lease.Grant(10 * time.Millisecond)

	var succeeded, expired int
	deadline := time.After(200 * time.Millisecond)
	i := 0
loop:
	for {
		select {
		case <-deadline:
			break loop
		default:
		}
		writeErr := v.WriteLBA(uint64(i%256), makeBlock(byte('W')))
		if writeErr == nil {
			succeeded++
		} else if errors.Is(writeErr, ErrLeaseExpired) {
			expired++
		}
		i++
		time.Sleep(500 * time.Microsecond)
	}

	t.Logf("lease expire: %d succeeded, %d expired (out of %d)", succeeded, expired, i)

	if expired == 0 {
		t.Error("no writes expired -- lease didn't expire in time?")
	}
	if succeeded == 0 {
		t.Error("no writes succeeded -- lease expired immediately?")
	}

	// Verify monotonicity: once expired, no more successes.
	v.lease.Grant(0) // force expired
	for j := 0; j < 10; j++ {
		writeErr := v.WriteLBA(uint64(j), makeBlock('X'))
		if !errors.Is(writeErr, ErrLeaseExpired) {
			t.Errorf("write %d after forced expiry: got %v, want ErrLeaseExpired", j, writeErr)
		}
	}
}

func testQAGateConcurrent100WritersRoleFlip(t *testing.T) {
	// Use large WAL + short timeout to avoid writers blocking on WAL-full.
	dir := t.TempDir()
	path := filepath.Join(dir, "100writers.blockvol")
	cfg := DefaultConfig()
	cfg.FlushInterval = 5 * time.Millisecond
	cfg.WALFullTimeout = 200 * time.Millisecond

	vol, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 * 1024 * 1024,
		BlockSize:  4096,
		WALSize:    2 * 1024 * 1024, // 2MB WAL
	}, cfg)
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}
	if err := vol.SetEpoch(1); err != nil {
		t.Fatalf("SetEpoch: %v", err)
	}
	vol.SetMasterEpoch(1)
	if err := vol.SetRole(RolePrimary); err != nil {
		t.Fatalf("SetRole: %v", err)
	}
	vol.lease.Grant(10 * time.Second)
	v := vol
	defer v.Close()

	const writers = 100
	const blocksPerWriter = 10

	var wg sync.WaitGroup
	var succeeded, notPrimary atomic.Int64

	// 100 writers.
	wg.Add(writers)
	for g := 0; g < writers; g++ {
		go func(id int) {
			defer wg.Done()
			for i := 0; i < blocksPerWriter; i++ {
				lba := uint64((id*blocksPerWriter + i) % 256)
				err := v.WriteLBA(lba, makeBlock(byte('A'+id%26)))
				if err == nil {
					succeeded.Add(1)
				} else if errors.Is(err, ErrNotPrimary) {
					notPrimary.Add(1)
				}
			}
		}(g)
	}

	// Flip role mid-flight.
	time.Sleep(50 * time.Millisecond)
	_ = v.SetRole(RoleDraining)
	time.Sleep(50 * time.Millisecond)
	_ = v.SetRole(RoleStale)

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatal("100 writers + role flip hung for 10s")
	}

	s, np := succeeded.Load(), notPrimary.Load()
	t.Logf("100 writers: %d succeeded, %d ErrNotPrimary", s, np)

	// Verify data integrity for successful writes: each block is readable.
	// We can't know which exact writes succeeded, but ReadLBA should not error.
	for i := 0; i < 256; i++ {
		_, err := v.ReadLBA(uint64(i), 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d) after stress: %v", i, err)
		}
	}
}

func testQAGateGotchaA(t *testing.T) {
	// Gotcha A: lease expires between WAL append and SyncCache.
	// Data IS in WAL, but SyncCache returns ErrLeaseExpired.
	v := createFencedVol(t)
	defer v.Close()

	// Grant a short lease.
	v.lease.Grant(50 * time.Millisecond)

	// Write data (succeeds -- lease still valid).
	if err := v.WriteLBA(0, makeBlock('G')); err != nil {
		t.Fatalf("WriteLBA: %v", err)
	}

	// Wait for lease to expire.
	time.Sleep(100 * time.Millisecond)

	// SyncCache must fail -- PostSyncCheck calls writeGate, which checks lease.
	err := v.SyncCache()
	if !errors.Is(err, ErrLeaseExpired) {
		t.Errorf("SyncCache after lease expiry: got %v, want ErrLeaseExpired (Gotcha A)", err)
	}

	// Key verification: data IS readable from dirty map / WAL.
	data, err := v.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA after Gotcha A: %v", err)
	}
	if data[0] != 'G' {
		t.Errorf("data[0] = %c, want G (data should be in WAL/dirty map)", data[0])
	}
	t.Log("Gotcha A: data in WAL, SyncCache fenced, ReadLBA returns written data")
}

// --- QA-4A-5: Integration Adversarial ---

func testQAFencingFullCycle(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Write 100 blocks as Primary.
	for i := 0; i < 100; i++ {
		if err := v.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}
	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	// Verify all 100 blocks.
	for i := 0; i < 100; i++ {
		data, err := v.ReadLBA(uint64(i), 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d): %v", i, err)
		}
		if data[0] != byte('A'+i%26) {
			t.Errorf("LBA %d: data[0] = %c, want %c", i, data[0], byte('A'+i%26))
		}
	}

	// Primary -> Draining -- writes should fail.
	if err := v.SetRole(RoleDraining); err != nil {
		t.Fatalf("SetRole(Draining): %v", err)
	}
	if err := v.WriteLBA(0, makeBlock('X')); !errors.Is(err, ErrNotPrimary) {
		t.Errorf("write as Draining: got %v, want ErrNotPrimary", err)
	}

	// Draining -> Stale -- writes rejected.
	if err := v.SetRole(RoleStale); err != nil {
		t.Fatalf("SetRole(Stale): %v", err)
	}
	if err := v.WriteLBA(0, makeBlock('X')); !errors.Is(err, ErrNotPrimary) {
		t.Errorf("write as Stale: got %v, want ErrNotPrimary", err)
	}

	// Bump epoch, transition through Rebuilding -> Replica.
	if err := v.SetEpoch(2); err != nil {
		t.Fatalf("SetEpoch(2): %v", err)
	}
	v.SetMasterEpoch(2)

	if err := v.SetRole(RoleRebuilding); err != nil {
		t.Fatalf("SetRole(Rebuilding): %v", err)
	}
	if err := v.WriteLBA(0, makeBlock('X')); !errors.Is(err, ErrNotPrimary) {
		t.Errorf("write as Rebuilding: got %v, want ErrNotPrimary", err)
	}

	if err := v.SetRole(RoleReplica); err != nil {
		t.Fatalf("SetRole(Replica): %v", err)
	}
	if err := v.WriteLBA(0, makeBlock('X')); !errors.Is(err, ErrNotPrimary) {
		t.Errorf("write as Replica: got %v, want ErrNotPrimary", err)
	}

	// Replica -> Primary with fresh lease -- writes succeed again.
	if err := v.SetRole(RolePrimary); err != nil {
		t.Fatalf("SetRole(Primary): %v", err)
	}
	v.lease.Grant(10 * time.Second)

	if err := v.WriteLBA(200, makeBlock('N')); err != nil {
		t.Fatalf("write as new Primary: %v", err)
	}

	// Original 100 blocks still readable.
	for i := 0; i < 100; i++ {
		data, err := v.ReadLBA(uint64(i), 4096)
		if err != nil {
			t.Fatalf("final ReadLBA(%d): %v", i, err)
		}
		if data[0] != byte('A'+i%26) {
			t.Errorf("final LBA %d: data[0] = %c, want %c", i, data[0], byte('A'+i%26))
		}
	}
	t.Log("full cycle: Primary->Draining->Stale->Rebuilding->Replica->Primary complete")
}

func testQAFencingWritesRejectedAfterDemotion(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Write initial data.
	if err := v.WriteLBA(0, makeBlock('D')); err != nil {
		t.Fatalf("WriteLBA: %v", err)
	}
	if err := v.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	// Revoke lease -> ErrLeaseExpired.
	v.lease.Revoke()
	err := v.WriteLBA(1, makeBlock('X'))
	if !errors.Is(err, ErrLeaseExpired) {
		t.Errorf("after Revoke: got %v, want ErrLeaseExpired", err)
	}

	// Re-grant lease for role checks, then demote.
	v.lease.Grant(10 * time.Second)

	// Primary -> Draining -> ErrNotPrimary.
	if err := v.SetRole(RoleDraining); err != nil {
		t.Fatalf("SetRole(Draining): %v", err)
	}
	err = v.WriteLBA(1, makeBlock('X'))
	if !errors.Is(err, ErrNotPrimary) {
		t.Errorf("after Draining: got %v, want ErrNotPrimary", err)
	}

	// Draining -> Stale -> ErrNotPrimary.
	if err := v.SetRole(RoleStale); err != nil {
		t.Fatalf("SetRole(Stale): %v", err)
	}
	err = v.WriteLBA(1, makeBlock('X'))
	if !errors.Is(err, ErrNotPrimary) {
		t.Errorf("after Stale: got %v, want ErrNotPrimary", err)
	}

	// Verify: LBA 0 has original data, LBA 1 was never written.
	data, readErr := v.ReadLBA(0, 4096)
	if readErr != nil {
		t.Fatalf("ReadLBA(0): %v", readErr)
	}
	if data[0] != 'D' {
		t.Errorf("LBA 0: data[0] = %c, want D", data[0])
	}
}

func testQAFencingReadAlwaysWorks(t *testing.T) {
	v := createFencedVol(t)
	defer v.Close()

	// Write data as Primary.
	for i := 0; i < 10; i++ {
		if err := v.WriteLBA(uint64(i), makeBlock(byte('R'+i%6))); err != nil {
			t.Fatalf("WriteLBA(%d): %v", i, err)
		}
	}

	verifyReads := func(label string) {
		for i := 0; i < 10; i++ {
			data, err := v.ReadLBA(uint64(i), 4096)
			if err != nil {
				t.Fatalf("[%s] ReadLBA(%d): %v", label, i, err)
			}
			if data[0] != byte('R'+i%6) {
				t.Errorf("[%s] LBA %d: data[0] = %c, want %c", label, i, data[0], byte('R'+i%6))
			}
		}
	}

	// Reads work as Primary.
	verifyReads("Primary")

	// Draining.
	if err := v.SetRole(RoleDraining); err != nil {
		t.Fatalf("SetRole(Draining): %v", err)
	}
	verifyReads("Draining")

	// Stale.
	if err := v.SetRole(RoleStale); err != nil {
		t.Fatalf("SetRole(Stale): %v", err)
	}
	verifyReads("Stale")

	// Rebuilding.
	if err := v.SetRole(RoleRebuilding); err != nil {
		t.Fatalf("SetRole(Rebuilding): %v", err)
	}
	verifyReads("Rebuilding")

	// Replica.
	if err := v.SetRole(RoleReplica); err != nil {
		t.Fatalf("SetRole(Replica): %v", err)
	}
	verifyReads("Replica")

	// Expired lease -- reads still work.
	v.lease.Revoke()
	verifyReads("expired_lease")

	// Stale epoch -- reads still work.
	v.SetMasterEpoch(999)
	verifyReads("stale_epoch")

	t.Log("reads work in all roles, with expired lease and stale epoch")
}

func testQAFencingCloseDuringRoleTransition(t *testing.T) {
	v := createFencedVol(t)

	// Start some writes.
	for i := 0; i < 20; i++ {
		_ = v.WriteLBA(uint64(i), makeBlock(byte('C')))
	}

	// Concurrent Close + SetRole.
	var wg sync.WaitGroup
	wg.Add(2)

	go func() {
		defer wg.Done()
		_ = v.Close()
	}()

	go func() {
		defer wg.Done()
		_ = v.SetRole(RoleDraining)
	}()

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
		t.Log("Close + SetRole concurrent: no deadlock, no panic")
	case <-time.After(5 * time.Second):
		t.Fatal("Close + SetRole deadlocked for 5s")
	}
}