seaweedfs/weed/storage/blockvol/qa_phase4a_cp4a_test.go

package blockvol

import (
	"errors"
	"fmt"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

// TestQAPhase4ACP4a runs adversarial tests for Phase 4A CP4a:
// SimulatedMaster, HandleAssignment sequences, failover, and Status().
func TestQAPhase4ACP4a(t *testing.T) {
	tests := []struct {
		name string
		run  func(t *testing.T)
	}{
		// --- SimulatedMaster correctness ---
		{name: "master_epoch_monotonic_across_operations", run: testMasterEpochMonotonic},
		{name: "master_promote_replica_both_fenced", run: testMasterPromoteReplicaBothFenced},
		{name: "master_demote_then_grant_reuses_vol", run: testMasterDemoteThenGrantReusesVol},

		// --- HandleAssignment edge cases ---
		{name: "assign_same_role_equal_epoch_no_persist", run: testAssignSameRoleEqualEpochNoPersist},
		{name: "assign_none_to_replica_epoch_not_persisted", run: testAssignNoneToReplicaEpochNotPersisted},
		{name: "assign_demote_epoch_less_than_current", run: testAssignDemoteEpochLessThanCurrent},
		{name: "assign_closed_volume", run: testAssignClosedVolume},
		{name: "assign_concurrent_demote_and_refresh", run: testAssignConcurrentDemoteAndRefresh},
		{name: "assign_rapid_promote_demote_10x", run: testAssignRapidPromoteDemote10x},

		// --- Status() edge cases ---
		{name: "status_fresh_primary_no_writes", run: testStatusFreshPrimaryNoWrites},
		{name: "status_during_writes", run: testStatusDuringWrites},
		{name: "status_all_roles", run: testStatusAllRoles},
		{name: "status_after_close", run: testStatusAfterClose},
		{name: "status_checkpoint_advances_after_flush", run: testStatusCheckpointAdvances},

		// --- Failover adversarial ---
		{name: "failover_triple_A_B_A_B", run: testFailoverTriple},
		{name: "failover_concurrent_promote_two_vols", run: testFailoverConcurrentPromoteTwoVols},
		{name: "failover_demote_during_active_writes", run: testFailoverDemoteDuringActiveWrites},
		{name: "failover_epoch_always_increases", run: testFailoverEpochAlwaysIncreases},
		{name: "failover_rebuild_then_immediate_failover", run: testFailoverRebuildThenImmediateFailover},
		{name: "failover_dead_zone_no_writes_anywhere", run: testFailoverDeadZoneNoWritesAnywhere},

		// --- beginOp/endOp adversarial ---
		{name: "ops_begin_after_close", run: testOpsBeginAfterClose},
		{name: "ops_drain_blocks_demote", run: testOpsDrainBlocksDemote},
		{name: "ops_concurrent_begin_end_100", run: testOpsConcurrentBeginEnd100},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			tt.run(t)
		})
	}
}

// cp4aVol creates a minimal test volume.
func cp4aVol(t *testing.T, cfgs ...BlockVolConfig) *BlockVol {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "cp4a.blockvol")
	var cfg BlockVolConfig
	if len(cfgs) > 0 {
		cfg = cfgs[0]
	}
	v, err := CreateBlockVol(path, CreateOptions{
		VolumeSize: 1 * 1024 * 1024,
		BlockSize:  4096,
	}, cfg)
	if err != nil {
		t.Fatalf("CreateBlockVol: %v", err)
	}
	return v
}

func cp4aBlock(b byte) []byte {
	buf := make([]byte, 4096)
	for i := range buf {
		buf[i] = b
	}
	return buf
}

// ---------------------------------------------------------------------------
// SimulatedMaster correctness
// ---------------------------------------------------------------------------

func testMasterEpochMonotonic(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()
	m := NewSimulatedMaster(30 * time.Second)

	// GrantPrimary -> epoch 1
	m.GrantPrimary(v)
	e1 := v.Epoch()

	// BumpEpoch -> epoch 2
	m.BumpEpoch(v)
	e2 := v.Epoch()

	// Demote -> epoch 3
	m.Demote(v)
	e3 := v.Epoch()

	if e1 >= e2 || e2 >= e3 {
		t.Errorf("epoch not monotonic: %d -> %d -> %d", e1, e2, e3)
	}

	// Rebuild back to Primary -> epoch 4 then 5
	m.InitiateRebuild(v)
	v.SetRole(RoleReplica)
	m.GrantPrimary(v)
	e4 := v.Epoch()
	if e4 <= e3 {
		t.Errorf("epoch after re-promote (%d) should be > demote epoch (%d)", e4, e3)
	}
}

func testMasterPromoteReplicaBothFenced(t *testing.T) {
	a := cp4aVol(t)
	defer a.Close()
	b := cp4aVol(t)
	defer b.Close()

	m := NewSimulatedMaster(30 * time.Second)
	m.GrantPrimary(a)

	// Write to A.
	if err := a.WriteLBA(0, cp4aBlock('A')); err != nil {
		t.Fatalf("A write: %v", err)
	}

	// Assign B as replica.
	m.AssignReplica(b)

	// PromoteReplica: demotes A, promotes B.
	_, err := m.PromoteReplica(a, b)
	if err != nil {
		t.Fatalf("PromoteReplica: %v", err)
	}

	// A must be fenced (Stale).
	if err := a.WriteLBA(0, cp4aBlock('X')); err == nil {
		t.Error("old primary should be fenced after PromoteReplica")
	}

	// B must be writable (Primary).
	if err := b.WriteLBA(0, cp4aBlock('B')); err != nil {
		t.Errorf("new primary should accept writes: %v", err)
	}

	// Both should have the same epoch.
	if a.Epoch() != b.Epoch() {
		t.Errorf("epochs should match: A=%d B=%d", a.Epoch(), b.Epoch())
	}
}

func testMasterDemoteThenGrantReusesVol(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()
	m := NewSimulatedMaster(30 * time.Second)

	// Primary -> Stale -> Rebuilding -> Replica -> Primary again.
	m.GrantPrimary(v)
	if err := v.WriteLBA(0, cp4aBlock('1')); err != nil {
		t.Fatalf("first write: %v", err)
	}
	m.Demote(v)
	m.InitiateRebuild(v)
	v.SetRole(RoleReplica)
	m.GrantPrimary(v)

	// Write again with new epoch.
	if err := v.WriteLBA(1, cp4aBlock('2')); err != nil {
		t.Fatalf("second write after re-promotion: %v", err)
	}

	// Read both blocks.
	d0, _ := v.ReadLBA(0, 4096)
	d1, _ := v.ReadLBA(1, 4096)
	if d0[0] != '1' {
		t.Errorf("block 0: got %c, want '1'", d0[0])
	}
	if d1[0] != '2' {
		t.Errorf("block 1: got %c, want '2'", d1[0])
	}
}

// ---------------------------------------------------------------------------
// HandleAssignment edge cases
// ---------------------------------------------------------------------------

func testAssignSameRoleEqualEpochNoPersist(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	// Promote to Primary at epoch 1.
	v.HandleAssignment(1, RolePrimary, 30*time.Second)

	// Same role, same epoch -- should be a no-op on epoch, but refreshes lease.
	v.lease.Revoke()
	if v.lease.IsValid() {
		t.Fatal("lease should be revoked")
	}

	err := v.HandleAssignment(1, RolePrimary, 30*time.Second)
	if err != nil {
		t.Fatalf("same role/epoch assignment: %v", err)
	}
	if !v.lease.IsValid() {
		t.Error("lease should be refreshed on same-role assignment")
	}
	if v.Epoch() != 1 {
		t.Errorf("epoch should remain 1, got %d", v.Epoch())
	}
}

func testAssignNoneToReplicaEpochNotPersisted(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	// None -> Replica at epoch 5.
	err := v.HandleAssignment(5, RoleReplica, 0)
	if err != nil {
		t.Fatalf("None->Replica: %v", err)
	}
	if v.Role() != RoleReplica {
		t.Errorf("role: got %s, want Replica", v.Role())
	}
	// masterEpoch is set, but Epoch() (persisted) may or may not be 5.
	// This is by design: replicas don't need persisted epoch until promoted.
	// Key invariant: promote() will call SetEpoch, so the epoch gets
	// persisted when it matters.
	masterEpoch := v.masterEpoch.Load()
	if masterEpoch != 5 {
		t.Errorf("masterEpoch: got %d, want 5", masterEpoch)
	}
}

func testAssignDemoteEpochLessThanCurrent(t *testing.T) {
	// BUG-CP4A-1 (fixed): demote with epoch < current must be rejected
	// to preserve monotonicity.
	v := cp4aVol(t)
	defer v.Close()

	// Promote at epoch 5.
	v.HandleAssignment(5, RolePrimary, 30*time.Second)

	// Demote with epoch 3 (stale) -- must fail with ErrEpochRegression.
	err := v.HandleAssignment(3, RoleStale, 0)
	if err == nil {
		t.Fatal("expected error for epoch regression, got nil")
	}
	if !errors.Is(err, ErrEpochRegression) {
		t.Fatalf("expected ErrEpochRegression, got: %v", err)
	}

	// Volume should still be Primary at epoch 5 (demote was rejected).
	if v.Role() != RolePrimary {
		t.Errorf("role should still be Primary, got %s", v.Role())
	}
	if v.Epoch() != 5 {
		t.Errorf("epoch should still be 5, got %d", v.Epoch())
	}
}

func testAssignClosedVolume(t *testing.T) {
	v := cp4aVol(t)
	v.HandleAssignment(1, RolePrimary, 30*time.Second)
	v.Close()

	// HandleAssignment on closed volume -- should it error?
	// Currently it acquires assignMu and proceeds. SetEpoch writes to fd,
	// which is closed. This should fail with an I/O error.
	err := v.HandleAssignment(2, RolePrimary, 30*time.Second)
	t.Logf("HandleAssignment on closed vol: %v", err)
	// We don't prescribe the exact error, but it shouldn't panic.
}

func testAssignConcurrentDemoteAndRefresh(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)
	v.drainTimeout = 2 * time.Second

	var wg sync.WaitGroup
	results := make(chan string, 2)

	// Goroutine 1: demote.
	wg.Add(1)
	go func() {
		defer wg.Done()
		err := v.HandleAssignment(2, RoleStale, 0)
		if err != nil {
			results <- fmt.Sprintf("demote: %v", err)
		} else {
			results <- "demote: ok"
		}
	}()

	// Goroutine 2: same-role refresh (races with demote for assignMu).
	wg.Add(1)
	go func() {
		defer wg.Done()
		err := v.HandleAssignment(1, RolePrimary, 30*time.Second)
		if err != nil {
			results <- fmt.Sprintf("refresh: %v", err)
		} else {
			results <- "refresh: ok"
		}
	}()

	wg.Wait()
	close(results)

	// One should win the assignMu. If demote wins first, the refresh
	// will see role=Stale and try Primary->Primary (same-role) or
	// Stale->Primary (invalid). Either way, no panic.
	for r := range results {
		t.Logf("  %s", r)
	}

	// Final role should be determinate.
	role := v.Role()
	t.Logf("final role: %s", role)
}

func testAssignRapidPromoteDemote10x(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()
	v.drainTimeout = 100 * time.Millisecond

	for i := 0; i < 10; i++ {
		epoch := uint64(i*2 + 1)
		// Promote.
		if err := v.HandleAssignment(epoch, RolePrimary, 30*time.Second); err != nil {
			// After first demote, role is Stale. Can't go directly to Primary.
			// Need to go through Rebuilding -> Replica first.
			if errors.Is(err, ErrInvalidAssignment) && i > 0 {
				v.HandleAssignment(epoch, RoleRebuilding, 0)
				v.SetRole(RoleReplica)
				if err := v.HandleAssignment(epoch, RolePrimary, 30*time.Second); err != nil {
					t.Fatalf("promote iter %d after rebuild: %v", i, err)
				}
			} else {
				t.Fatalf("promote iter %d: %v", i, err)
			}
		}
		// Write to verify writable.
		if err := v.WriteLBA(0, cp4aBlock(byte('0'+i%10))); err != nil {
			t.Fatalf("write iter %d: %v", i, err)
		}
		// Demote.
		demoteEpoch := epoch + 1
		if err := v.HandleAssignment(demoteEpoch, RoleStale, 0); err != nil {
			t.Fatalf("demote iter %d: %v", i, err)
		}
	}

	// Final: role should be Stale, epoch should be 20.
	if v.Role() != RoleStale {
		t.Errorf("final role: got %s, want Stale", v.Role())
	}
	if v.Epoch() != 20 {
		t.Errorf("final epoch: got %d, want 20", v.Epoch())
	}
}

// ---------------------------------------------------------------------------
// Status() edge cases
// ---------------------------------------------------------------------------

func testStatusFreshPrimaryNoWrites(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)

	st := v.Status()
	if st.Epoch != 1 {
		t.Errorf("Epoch: got %d, want 1", st.Epoch)
	}
	if st.Role != RolePrimary {
		t.Errorf("Role: got %s, want Primary", st.Role)
	}
	if !st.HasLease {
		t.Error("HasLease: got false, want true")
	}
	// WALHeadLSN = nextLSN - 1 = 1 - 1 = 0 (no writes yet).
	if st.WALHeadLSN != 0 {
		t.Errorf("WALHeadLSN: got %d, want 0 (no writes)", st.WALHeadLSN)
	}
}

func testStatusDuringWrites(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)

	// Write some blocks, check status advances.
	for i := 0; i < 5; i++ {
		v.WriteLBA(uint64(i), cp4aBlock(byte('A'+i)))
	}

	st := v.Status()
	if st.WALHeadLSN < 4 {
		t.Errorf("WALHeadLSN: got %d, want >= 4 after 5 writes", st.WALHeadLSN)
	}
}

func testStatusAllRoles(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()
	v.drainTimeout = 100 * time.Millisecond

	roleChecks := []struct {
		setup func()
		role  Role
		lease bool
	}{
		{
			setup: func() { v.HandleAssignment(1, RolePrimary, 30*time.Second) },
			role:  RolePrimary, lease: true,
		},
		{
			setup: func() { v.HandleAssignment(2, RoleStale, 0) },
			role:  RoleStale, lease: false,
		},
		{
			setup: func() { v.HandleAssignment(2, RoleRebuilding, 0) },
			role:  RoleRebuilding, lease: false,
		},
		{
			setup: func() { v.SetRole(RoleReplica) },
			role:  RoleReplica, lease: false,
		},
		{
			setup: func() { v.HandleAssignment(3, RolePrimary, 30*time.Second) },
			role:  RolePrimary, lease: true,
		},
	}

	for _, rc := range roleChecks {
		rc.setup()
		st := v.Status()
		if st.Role != rc.role {
			t.Errorf("after transition: Role got %s, want %s", st.Role, rc.role)
		}
		if st.HasLease != rc.lease {
			t.Errorf("role %s: HasLease got %v, want %v", rc.role, st.HasLease, rc.lease)
		}
	}
}

func testStatusAfterClose(t *testing.T) {
	v := cp4aVol(t)
	v.HandleAssignment(1, RolePrimary, 30*time.Second)
	v.WriteLBA(0, cp4aBlock('X'))
	v.Close()

	// Status() on closed volume should not panic.
	st := v.Status()
	t.Logf("Status after close: epoch=%d role=%s walHead=%d lease=%v cp=%d",
		st.Epoch, st.Role, st.WALHeadLSN, st.HasLease, st.CheckpointLSN)
}

func testStatusCheckpointAdvances(t *testing.T) {
	v := cp4aVol(t, BlockVolConfig{
		FlushInterval: 50 * time.Millisecond,
	})
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)

	// Write a block.
	v.WriteLBA(0, cp4aBlock('C'))

	st1 := v.Status()
	initialCP := st1.CheckpointLSN

	// Force flush by syncing and waiting for flusher.
	v.SyncCache()
	time.Sleep(200 * time.Millisecond) // wait for flusher interval
	v.flusher.NotifyUrgent()
	time.Sleep(200 * time.Millisecond)

	st2 := v.Status()
	t.Logf("checkpoint: before=%d after=%d", initialCP, st2.CheckpointLSN)
	// Checkpoint may or may not have advanced depending on flusher timing.
	// The key thing is no panic and the value is plausible.
}

// ---------------------------------------------------------------------------
// Failover adversarial
// ---------------------------------------------------------------------------

func testFailoverTriple(t *testing.T) {
	a := cp4aVol(t)
	defer a.Close()
	b := cp4aVol(t)
	defer b.Close()

	m := NewSimulatedMaster(30 * time.Second)

	// Round 1: A primary.
	m.GrantPrimary(a)
	a.WriteLBA(0, cp4aBlock('1'))
	m.AssignReplica(b)

	// Failover 1: A -> B.
	m.PromoteReplica(a, b)
	b.WriteLBA(1, cp4aBlock('2'))

	// Rebuild A from B (simulate).
	m.InitiateRebuild(a)
	a.SetRole(RoleReplica)

	// Failover 2: B -> A.
	m.PromoteReplica(b, a)
	a.WriteLBA(2, cp4aBlock('3'))

	// Rebuild B from A (simulate).
	m.InitiateRebuild(b)
	b.SetRole(RoleReplica)

	// Failover 3: A -> B.
	m.PromoteReplica(a, b)
	b.WriteLBA(3, cp4aBlock('4'))

	// Verify B is primary, A is stale.
	if b.Role() != RolePrimary {
		t.Errorf("B role: %s, want Primary", b.Role())
	}
	if a.Role() != RoleStale {
		t.Errorf("A role: %s, want Stale", a.Role())
	}
	// After PromoteReplica, both volumes share the same epoch.
	if a.Epoch() != b.Epoch() {
		t.Errorf("epochs should match after PromoteReplica: A=%d B=%d", a.Epoch(), b.Epoch())
	}
	// Epoch should be > 1 after 3 failovers.
	if b.Epoch() < 3 {
		t.Errorf("epoch after 3 failovers: got %d, want >= 3", b.Epoch())
	}
}

func testFailoverConcurrentPromoteTwoVols(t *testing.T) {
	// Adversarial: two volumes both try to become Primary at same epoch.
	// This simulates a split-brain scenario where master sends conflicting assignments.
	a := cp4aVol(t)
	defer a.Close()
	b := cp4aVol(t)
	defer b.Close()

	var wg sync.WaitGroup
	errA := make(chan error, 1)
	errB := make(chan error, 1)

	// Both try None -> Primary at epoch 1 concurrently.
	wg.Add(2)
	go func() {
		defer wg.Done()
		errA <- a.HandleAssignment(1, RolePrimary, 30*time.Second)
	}()
	go func() {
		defer wg.Done()
		errB <- b.HandleAssignment(1, RolePrimary, 30*time.Second)
	}()
	wg.Wait()

	// Both should succeed (they're independent volumes).
	// The split-brain prevention is at the master level, not per-volume.
	eA := <-errA
	eB := <-errB
	if eA != nil {
		t.Errorf("A promote: %v", eA)
	}
	if eB != nil {
		t.Errorf("B promote: %v", eB)
	}

	// Both are Primary -- this is the split-brain that master must prevent.
	if a.Role() != RolePrimary || b.Role() != RolePrimary {
		t.Errorf("both should be Primary: A=%s B=%s", a.Role(), b.Role())
	}
	t.Log("split-brain: both volumes Primary at epoch 1 -- master must prevent this")
}

func testFailoverDemoteDuringActiveWrites(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)
	v.drainTimeout = 2 * time.Second

	var wg sync.WaitGroup
	var writeSucceeded, writeFailed atomic.Int64

	// Start 50 concurrent writers.
	for i := 0; i < 50; i++ {
		wg.Add(1)
		go func(lba int) {
			defer wg.Done()
			for j := 0; j < 10; j++ {
				if err := v.WriteLBA(uint64(lba%256), cp4aBlock(byte('A'+j%26))); err != nil {
					writeFailed.Add(1)
				} else {
					writeSucceeded.Add(1)
				}
			}
		}(i)
	}

	// Let writers get started.
	time.Sleep(5 * time.Millisecond)

	// Demote.
	demoteErr := v.HandleAssignment(2, RoleStale, 0)
	wg.Wait()

	t.Logf("writes: %d succeeded, %d failed; demote err: %v",
		writeSucceeded.Load(), writeFailed.Load(), demoteErr)

	// After demote, no more writes should succeed.
	if err := v.WriteLBA(0, cp4aBlock('Z')); err == nil {
		t.Error("write should fail after demote")
	}
}

func testFailoverEpochAlwaysIncreases(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()
	v.drainTimeout = 100 * time.Millisecond

	var lastEpoch uint64
	for i := 0; i < 5; i++ {
		promoteEpoch := uint64(i*2 + 1)
		demoteEpoch := uint64(i*2 + 2)

		// Route through rebuild path after first demote.
		if i > 0 {
			v.HandleAssignment(promoteEpoch, RoleRebuilding, 0)
			v.SetRole(RoleReplica)
		}

		v.HandleAssignment(promoteEpoch, RolePrimary, 30*time.Second)
		if v.Epoch() <= lastEpoch && i > 0 {
			t.Errorf("epoch regression at promote iter %d: %d <= %d", i, v.Epoch(), lastEpoch)
		}
		lastEpoch = v.Epoch()

		v.HandleAssignment(demoteEpoch, RoleStale, 0)
		if v.Epoch() <= lastEpoch {
			// BUG-CP4A-1 would cause epoch to regress if demoteEpoch < promoteEpoch.
			// Here we always use increasing epochs, so this should pass.
		}
		lastEpoch = v.Epoch()
	}
	t.Logf("final epoch after 5 failovers: %d", v.Epoch())
}

func testFailoverRebuildThenImmediateFailover(t *testing.T) {
	a := cp4aVol(t)
	defer a.Close()
	b := cp4aVol(t)
	defer b.Close()

	m := NewSimulatedMaster(30 * time.Second)

	// A primary, writes data.
	m.GrantPrimary(a)
	a.WriteLBA(0, cp4aBlock('R'))

	// Start rebuild server on A.
	a.StartRebuildServer("127.0.0.1:0")
	defer a.StopRebuildServer()

	// B: None -> Primary -> Stale -> Rebuilding.
	bEpoch := m.epoch + 1
	b.HandleAssignment(bEpoch, RolePrimary, 30*time.Second)
	b.HandleAssignment(bEpoch, RoleStale, 0)
	b.HandleAssignment(bEpoch, RoleRebuilding, 0)

	// Rebuild B from A.
	StartRebuild(b, a.rebuildServer.Addr(), 1, a.Epoch())

	// B is now Replica. Immediately failover: demote A, promote B.
	m.epoch = bEpoch + 1
	a.HandleAssignment(m.epoch, RoleStale, 0)
	b.HandleAssignment(m.epoch, RolePrimary, 30*time.Second)

	// Verify B can write.
	if err := b.WriteLBA(1, cp4aBlock('S')); err != nil {
		t.Fatalf("B write after immediate failover: %v", err)
	}

	// Verify A is fenced.
	if err := a.WriteLBA(0, cp4aBlock('X')); err == nil {
		t.Error("A should be fenced after demote")
	}
}

func testFailoverDeadZoneNoWritesAnywhere(t *testing.T) {
	// After demote of A but before promote of B, NO volume should accept writes.
	// This is the "dead zone" -- both A and B must reject.
	a := cp4aVol(t)
	defer a.Close()
	b := cp4aVol(t)
	defer b.Close()
	c := cp4aVol(t)
	defer c.Close()

	m := NewSimulatedMaster(30 * time.Second)
	m.GrantPrimary(a)
	m.AssignReplica(b)
	// C is a third volume that also has a stale view.
	c.HandleAssignment(m.epoch, RoleReplica, 0)

	// Demote A.
	m.Demote(a)

	// Dead zone: nobody should accept writes.
	vols := []*BlockVol{a, b, c}
	for i, v := range vols {
		if err := v.WriteLBA(0, cp4aBlock('X')); err == nil {
			t.Errorf("volume %d (role=%s) accepted write in dead zone", i, v.Role())
		}
	}

	// Now promote B.
	b.HandleAssignment(m.epoch, RolePrimary, 30*time.Second)
	if err := b.WriteLBA(0, cp4aBlock('Y')); err != nil {
		t.Errorf("B should accept writes after promotion: %v", err)
	}
}

// ---------------------------------------------------------------------------
// beginOp/endOp adversarial
// ---------------------------------------------------------------------------

func testOpsBeginAfterClose(t *testing.T) {
	v := cp4aVol(t)
	v.HandleAssignment(1, RolePrimary, 30*time.Second)
	v.Close()

	// beginOp after close should return ErrVolumeClosed.
	err := v.beginOp()
	if !errors.Is(err, ErrVolumeClosed) {
		t.Errorf("beginOp after close: got %v, want ErrVolumeClosed", err)
	}

	// opsOutstanding should be 0 (beginOp increments then decrements on closed).
	if ops := v.opsOutstanding.Load(); ops != 0 {
		t.Errorf("opsOutstanding: got %d, want 0", ops)
	}
}

func testOpsDrainBlocksDemote(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)
	v.drainTimeout = 2 * time.Second

	// Simulate 3 in-flight ops.
	v.beginOp()
	v.beginOp()
	v.beginOp()

	demoteDone := make(chan error, 1)
	go func() {
		demoteDone <- v.HandleAssignment(2, RoleStale, 0)
	}()

	// Demote should be blocked.
	select {
	case <-demoteDone:
		t.Fatal("demote should be blocked by outstanding ops")
	case <-time.After(50 * time.Millisecond):
		// Good, still blocked.
	}

	// Release 2 ops -- still blocked.
	v.endOp()
	v.endOp()
	select {
	case <-demoteDone:
		t.Fatal("demote should still be blocked with 1 op outstanding")
	case <-time.After(50 * time.Millisecond):
	}

	// Release last op.
	v.endOp()

	select {
	case err := <-demoteDone:
		if err != nil {
			t.Fatalf("demote failed: %v", err)
		}
	case <-time.After(2 * time.Second):
		t.Fatal("demote should have completed after all ops drained")
	}

	if v.Role() != RoleStale {
		t.Errorf("role: got %s, want Stale", v.Role())
	}
}

func testOpsConcurrentBeginEnd100(t *testing.T) {
	v := cp4aVol(t)
	defer v.Close()

	v.HandleAssignment(1, RolePrimary, 30*time.Second)

	var wg sync.WaitGroup
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 100; j++ {
				if err := v.beginOp(); err != nil {
					return
				}
				// Brief work.
				v.endOp()
			}
		}()
	}
	wg.Wait()

	ops := v.opsOutstanding.Load()
	if ops != 0 {
		t.Errorf("opsOutstanding: got %d, want 0 after all goroutines done", ops)
	}
}