package blockvol import ( "fmt" "net" "path/filepath" "sync" "testing" "time" ) // TestQAPhase4ACP4b4 tests Phase 4A CP4b-4 adversarial scenarios: // partial channel failures, protocol edge cases, rebuild scenarios, // edge interactions, and error injection. func TestQAPhase4ACP4b4(t *testing.T) { tests := []struct { name string run func(t *testing.T) }{ // Group A: Partial Channel Failures {name: "barrier_stale_lsn_after_gap", run: testQA4b4BarrierStaleLSNAfterGap}, {name: "ship_data_ok_ctrl_down", run: testQA4b4ShipDataOkCtrlDown}, {name: "ship_ctrl_ok_data_down", run: testQA4b4ShipCtrlOkDataDown}, // Group B: Protocol Edge Cases {name: "ship_large_entry_near_max", run: testQA4b4ShipLargeEntryNearMax}, {name: "ship_entries_across_wal_wrap", run: testQA4b4ShipEntriesAcrossWALWrap}, // Group C: Rebuild Scenarios {name: "rebuild_catchup_concurrent_writes", run: testQA4b4RebuildCatchupConcurrentWrites}, {name: "rebuild_full_extent_midcopy_writes", run: testQA4b4RebuildFullExtentMidcopyWrites}, {name: "rebuild_interrupted_retry", run: testQA4b4RebuildInterruptedRetry}, // Group D: Edge Interactions {name: "read_during_wal_reclaim", run: testQA4b4ReadDuringWALReclaim}, {name: "trim_blocks_survive_rebuild", run: testQA4b4TrimBlocksSurviveRebuild}, {name: "demote_while_wal_full", run: testQA4b4DemoteWhileWALFull}, // Group E: Error Injection {name: "rebuild_server_epoch_mismatch", run: testQA4b4RebuildServerEpochMismatch}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.run(t) }) } } // ========== Group A: Partial Channel Failures ========== // A1: Ship LSN 1,2,4 (skip 3) to replica receiver, then barrier(4). // Barrier must time out because receiver blocks waiting for contiguous LSN 3. func testQA4b4BarrierStaleLSNAfterGap(t *testing.T) { dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.barrierTimeout = 200 * time.Millisecond // fast timeout for test recv.Serve() defer recv.Stop() dataConn, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial data: %v", err) } defer dataConn.Close() // Send LSN 1, 2, 4 (skip 3). for _, lsn := range []uint64{1, 2, 4} { entry := &WALEntry{LSN: lsn, Epoch: 1, Type: EntryTypeWrite, LBA: lsn - 1, Length: 4096, Data: makeBlock(byte('A' + lsn))} encoded, _ := entry.Encode() if err := WriteFrame(dataConn, MsgWALEntry, encoded); err != nil { t.Fatalf("send LSN=%d: %v", lsn, err) } } time.Sleep(50 * time.Millisecond) // receivedLSN should be 2 (gap at 3 blocks further progress). if got := recv.ReceivedLSN(); got != 2 { t.Fatalf("ReceivedLSN = %d after gap, want 2", got) } // Barrier(4) must time out since LSN 3 is missing. ctrlConn, err := net.Dial("tcp", recv.CtrlAddr()) if err != nil { t.Fatalf("dial ctrl: %v", err) } defer ctrlConn.Close() req := EncodeBarrierRequest(BarrierRequest{LSN: 4, Epoch: 1}) if err := WriteFrame(ctrlConn, MsgBarrierReq, req); err != nil { t.Fatalf("send barrier: %v", err) } ctrlConn.SetReadDeadline(time.Now().Add(3 * time.Second)) _, payload, err := ReadFrame(ctrlConn) if err != nil { t.Fatalf("read barrier response: %v", err) } if payload[0] == BarrierOK { t.Error("barrier returned OK despite gap at LSN 3, expected timeout") } // receivedLSN should still be 2. if got := recv.ReceivedLSN(); got != 2 { t.Errorf("ReceivedLSN = %d after barrier timeout, want 2", got) } } // A2: Data channel works, ctrl listener closed before Barrier. // Barrier must return error. Data entries still applied. func testQA4b4ShipDataOkCtrlDown(t *testing.T) { primary, replica := createReplicaPair(t) defer primary.Close() defer replica.Close() // Write data through primary -> shipped to replica via data channel. for i := 0; i < 5; i++ { if err := primary.WriteLBA(uint64(i), makeBlock(byte('A'+i))); err != nil { t.Fatalf("WriteLBA(%d): %v", i, err) } } time.Sleep(50 * time.Millisecond) // Verify data entries were applied to replica. replicaLSN := replica.replRecv.ReceivedLSN() if replicaLSN < 5 { t.Fatalf("replica receivedLSN = %d, want >= 5", replicaLSN) } // Kill ctrl listener to make barrier fail. replica.replRecv.ctrlListener.Close() // Force primary shipper to reconnect ctrl. shipper := primary.shipperGroup.Shipper(0) shipper.ctrlMu.Lock() if shipper.ctrlConn != nil { shipper.ctrlConn.Close() shipper.ctrlConn = nil } shipper.ctrlMu.Unlock() // Barrier must fail (ctrl down). err := shipper.Barrier(replicaLSN) if err == nil { t.Error("Barrier should fail when ctrl listener is closed") } // Data entries should still be applied (receivedLSN unchanged or higher). if got := replica.replRecv.ReceivedLSN(); got < replicaLSN { t.Errorf("receivedLSN regressed from %d to %d after ctrl failure", replicaLSN, got) } } // A3: Ctrl is up, data listener closed. Ship must fail, barrier may timeout. func testQA4b4ShipCtrlOkDataDown(t *testing.T) { dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.barrierTimeout = 200 * time.Millisecond recv.Serve() defer recv.Stop() ctrlAddr := recv.CtrlAddr() // Close data listener -- data conn will fail. recv.dataListener.Close() // Create shipper pointed at closed data port, working ctrl port. shipper := NewWALShipper(recv.DataAddr(), ctrlAddr, func() uint64 { return 1 }, nil) defer shipper.Stop() // Ship should fail (can't connect data), shipper degrades. entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('X')} _ = shipper.Ship(entry) // receivedLSN should stay 0. if got := recv.ReceivedLSN(); got != 0 { t.Errorf("ReceivedLSN = %d, want 0 (data channel was down)", got) } } // ========== Group B: Protocol Edge Cases ========== // B1: Ship a large WAL entry through the frame codec and verify it round-trips. // Uses WriteFrame/ReadFrame directly to test near-max frame payload. // Then ships a multi-block entry through a real replica receiver. func testQA4b4ShipLargeEntryNearMax(t *testing.T) { // Part 1: Frame codec round-trip with a large payload (1MB). const largeSize = 1 * 1024 * 1024 largePayload := make([]byte, largeSize) for i := range largePayload { largePayload[i] = byte('L') } server, client := net.Pipe() defer server.Close() defer client.Close() writeDone := make(chan error, 1) go func() { writeDone <- WriteFrame(client, MsgWALEntry, largePayload) }() msgType, payload, err := ReadFrame(server) if err != nil { t.Fatalf("ReadFrame large: %v", err) } if msgType != MsgWALEntry { t.Errorf("msgType = 0x%02x, want MsgWALEntry", msgType) } if len(payload) != largeSize { t.Errorf("payload len = %d, want %d", len(payload), largeSize) } if payload[0] != 'L' || payload[largeSize-1] != 'L' { t.Error("payload data corrupted") } if err := <-writeDone; err != nil { t.Fatalf("WriteFrame: %v", err) } // Part 2: Ship a multi-block entry (8 blocks = 32KB) through a real receiver. dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 512 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() defer recv.Stop() dataConn, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial data: %v", err) } defer dataConn.Close() // 8 blocks = 32KB data. const blockCount = 8 bigData := make([]byte, blockCount*4096) for i := range bigData { bigData[i] = byte('B') } entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite, LBA: 0, Length: uint32(len(bigData)), Data: bigData} encoded, err := entry.Encode() if err != nil { t.Fatalf("Encode: %v", err) } if err := WriteFrame(dataConn, MsgWALEntry, encoded); err != nil { t.Fatalf("WriteFrame: %v", err) } time.Sleep(100 * time.Millisecond) if got := recv.ReceivedLSN(); got != 1 { t.Errorf("ReceivedLSN = %d, want 1 (multi-block entry should be applied)", got) } } // B2: Write enough entries to force WAL wrap, ship all, verify replica has data. func testQA4b4ShipEntriesAcrossWALWrap(t *testing.T) { primary, replica := createReplicaPair(t) defer primary.Close() defer replica.Close() // createReplicaPair uses WALSize=512KB, VolumeSize=1MB. // Each write is 4KB WAL entry + header. Fill enough to wrap. // ~512KB WAL / ~4KB per entry ≈ 128 entries before wrap, but flusher // reclaims space so we can write more. const numWrites = 50 for i := 0; i < numWrites; i++ { if err := primary.WriteLBA(uint64(i%256), makeBlock(byte('W'+i%26))); err != nil { // WAL full is acceptable if flusher can't keep up. t.Logf("WriteLBA(%d): %v (expected if WAL fills)", i, err) break } } // SyncCache to push barrier. if err := primary.SyncCache(); err != nil { // Degraded replica is ok -- we test that entries shipped correctly. t.Logf("SyncCache: %v (may degrade)", err) } time.Sleep(100 * time.Millisecond) // Replica should have received entries. replicaLSN := replica.replRecv.ReceivedLSN() if replicaLSN == 0 { t.Error("replica receivedLSN = 0, expected > 0 after writes across WAL") } // Verify first few LBAs on replica. for i := 0; i < 5 && uint64(i) <= replicaLSN; i++ { data, err := replica.ReadLBA(uint64(i), 4096) if err != nil { t.Fatalf("replica ReadLBA(%d): %v", i, err) } if data[0] == 0 { t.Errorf("replica LBA %d is zero, expected non-zero data", i) } } } // ========== Group C: Rebuild Scenarios ========== // C1: Start rebuild while primary receives concurrent writes. func testQA4b4RebuildCatchupConcurrentWrites(t *testing.T) { primary := cp3Primary(t, "rb_cc_pri.bv", 1) defer primary.Close() // Write initial data. for i := 0; i < 5; i++ { primary.WriteLBA(uint64(i), makeBlock(byte('A'+i))) } primary.StartRebuildServer("127.0.0.1:0") defer primary.StopRebuildServer() // Start concurrent writes on primary while rebuild happens. var wg sync.WaitGroup stopWriting := make(chan struct{}) wg.Add(1) go func() { defer wg.Done() lba := uint64(10) for { select { case <-stopWriting: return default: } primary.WriteLBA(lba, makeBlock(byte('C'))) lba++ if lba > 14 { lba = 10 } time.Sleep(1 * time.Millisecond) } }() // Start rebuild client. replica := cp3Rebuilding(t, "rb_cc_rep.bv", 1) defer replica.Close() err := StartRebuild(replica, primary.rebuildServer.Addr(), 1, 1) close(stopWriting) wg.Wait() if err != nil { t.Fatalf("StartRebuild: %v", err) } if replica.Role() != RoleReplica { t.Errorf("role = %s, want Replica", replica.Role()) } // Verify initial data on replica. for i := 0; i < 5; i++ { data, err := replica.ReadLBA(uint64(i), 4096) if err != nil { t.Fatalf("ReadLBA(%d): %v", i, err) } if data[0] != byte('A'+i) { t.Errorf("LBA %d: got %c, want %c", i, data[0], byte('A'+i)) } } } // C2: Write initial data, start full extent rebuild, concurrent writes during copy. func testQA4b4RebuildFullExtentMidcopyWrites(t *testing.T) { primary := cp3Primary(t, "rb_fe_pri.bv", 1) defer primary.Close() // Write and flush to make data in extent. for i := 0; i < 5; i++ { primary.WriteLBA(uint64(i), makeBlock(byte('D'+i))) } primary.SyncCache() time.Sleep(30 * time.Millisecond) // let flusher move to extent primary.StartRebuildServer("127.0.0.1:0") defer primary.StopRebuildServer() // Start concurrent writes during rebuild. var wg sync.WaitGroup stopWriting := make(chan struct{}) wg.Add(1) go func() { defer wg.Done() lba := uint64(10) for { select { case <-stopWriting: return default: } primary.WriteLBA(lba, makeBlock(byte('M'))) lba++ if lba > 14 { lba = 10 } time.Sleep(1 * time.Millisecond) } }() // Force full extent rebuild (fromLSN=0 below checkpoint). replica := cp3Rebuilding(t, "rb_fe_rep.bv", 1) defer replica.Close() err := StartRebuild(replica, primary.rebuildServer.Addr(), 0, 1) close(stopWriting) wg.Wait() if err != nil { t.Fatalf("StartRebuild: %v", err) } if replica.Role() != RoleReplica { t.Errorf("role = %s, want Replica", replica.Role()) } // Verify flushed data on replica (from extent). for i := 0; i < 5; i++ { data, err := replica.ReadLBA(uint64(i), 4096) if err != nil { t.Fatalf("ReadLBA(%d): %v", i, err) } if data[0] != byte('D'+i) { t.Errorf("LBA %d: got %c, want %c", i, data[0], byte('D'+i)) } } } // C3: Start rebuild, read half the entries, close connection, retry from start. func testQA4b4RebuildInterruptedRetry(t *testing.T) { primary := cp3Primary(t, "rb_int_pri.bv", 1) defer primary.Close() for i := 0; i < 10; i++ { primary.WriteLBA(uint64(i), makeBlock(byte('R'+i%26))) } primary.StartRebuildServer("127.0.0.1:0") defer primary.StopRebuildServer() // First attempt: connect, read a few entries, then close. conn1, err := net.Dial("tcp", primary.rebuildServer.Addr()) if err != nil { t.Fatalf("dial 1: %v", err) } req := EncodeRebuildRequest(RebuildRequest{Type: RebuildWALCatchUp, FromLSN: 1, Epoch: 1}) WriteFrame(conn1, MsgRebuildReq, req) // Read a few entries then abort. for i := 0; i < 3; i++ { _, _, err := ReadFrame(conn1) if err != nil { t.Fatalf("read entry %d: %v", i, err) } } conn1.Close() // simulate crash // Second attempt: full rebuild should succeed. replica := cp3Rebuilding(t, "rb_int_rep.bv", 1) defer replica.Close() if err := StartRebuild(replica, primary.rebuildServer.Addr(), 1, 1); err != nil { t.Fatalf("StartRebuild (retry): %v", err) } if replica.Role() != RoleReplica { t.Errorf("role = %s, want Replica", replica.Role()) } // Verify all data. for i := 0; i < 10; i++ { data, err := replica.ReadLBA(uint64(i), 4096) if err != nil { t.Fatalf("ReadLBA(%d): %v", i, err) } expected := byte('R' + i%26) if data[0] != expected { t.Errorf("LBA %d: got %c, want %c", i, data[0], expected) } } } // ========== Group D: Edge Interactions ========== // D1: Write blocks, flush to reclaim WAL, concurrent ReadLBA on flushed blocks. func testQA4b4ReadDuringWALReclaim(t *testing.T) { vol := cp3Primary(t, "read_reclaim.bv", 1) defer vol.Close() // Write data. for i := 0; i < 10; i++ { if err := vol.WriteLBA(uint64(i), makeBlock(byte('R'+i%26))); err != nil { t.Fatalf("WriteLBA(%d): %v", i, err) } } // SyncCache + wait for flusher to reclaim WAL. vol.SyncCache() time.Sleep(50 * time.Millisecond) // Concurrent reads on recently-flushed blocks. var wg sync.WaitGroup errs := make([]error, 10) for i := 0; i < 10; i++ { wg.Add(1) go func(lba int) { defer wg.Done() data, err := vol.ReadLBA(uint64(lba), 4096) if err != nil { errs[lba] = err return } expected := byte('R' + lba%26) if data[0] != expected { errs[lba] = fmt.Errorf("LBA %d: got %c, want %c", lba, data[0], expected) } }(i) } wg.Wait() for i, err := range errs { if err != nil { t.Errorf("read LBA %d: %v", i, err) } } } // D2: Write blocks, TRIM some, rebuild replica, verify trimmed blocks read as zeros. func testQA4b4TrimBlocksSurviveRebuild(t *testing.T) { primary := cp3Primary(t, "trim_rb_pri.bv", 1) defer primary.Close() // Write 10 blocks. for i := 0; i < 10; i++ { primary.WriteLBA(uint64(i), makeBlock(byte('T'+i%26))) } // TRIM blocks 3-5. for i := 3; i <= 5; i++ { if err := primary.Trim(uint64(i), 4096); err != nil { t.Fatalf("Trim(%d): %v", i, err) } } primary.StartRebuildServer("127.0.0.1:0") defer primary.StopRebuildServer() replica := cp3Rebuilding(t, "trim_rb_rep.bv", 1) defer replica.Close() if err := StartRebuild(replica, primary.rebuildServer.Addr(), 1, 1); err != nil { t.Fatalf("StartRebuild: %v", err) } // Trimmed blocks should read as zeros. for i := 3; i <= 5; i++ { data, err := replica.ReadLBA(uint64(i), 4096) if err != nil { t.Fatalf("ReadLBA(%d): %v", i, err) } nonZero := false for _, b := range data { if b != 0 { nonZero = true break } } if nonZero { t.Errorf("LBA %d should be zero after TRIM+rebuild, got non-zero data", i) } } // Non-trimmed blocks should still have data. data0, _ := replica.ReadLBA(0, 4096) if data0[0] != 'T' { t.Errorf("LBA 0: got %c, want 'T'", data0[0]) } } // D3: Fill WAL, start WriteLBA (blocks in WAL-full retry), concurrently demote. func testQA4b4DemoteWhileWALFull(t *testing.T) { // Create volume with tiny WAL. dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 100 * time.Millisecond // slow flusher cfg.WALFullTimeout = 2 * time.Second vol, err := CreateBlockVol(filepath.Join(dir, "demote_walful.bv"), CreateOptions{ VolumeSize: 64 * 1024, BlockSize: 4096, WALSize: 16 * 1024, // tiny WAL: ~4 entries }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.HandleAssignment(1, RolePrimary, 30*time.Second); err != nil { t.Fatalf("assign primary: %v", err) } vol.drainTimeout = 3 * time.Second // Fill WAL. writeCount := 0 for i := 0; i < 20; i++ { if err := vol.WriteLBA(uint64(i%16), makeBlock(byte('F'))); err != nil { break } writeCount++ } if writeCount < 2 { t.Fatalf("only wrote %d entries before WAL full", writeCount) } // Start a WriteLBA that will likely block in WAL-full retry. writeDone := make(chan error, 1) go func() { writeDone <- vol.WriteLBA(0, makeBlock('Z')) }() // Give the write a moment to start retrying. time.Sleep(10 * time.Millisecond) // Concurrently demote. demoteDone := make(chan error, 1) go func() { demoteDone <- vol.HandleAssignment(2, RoleStale, 0) }() // Both should complete within timeout (no deadlock). timer := time.NewTimer(5 * time.Second) defer timer.Stop() select { case err := <-demoteDone: // Demote may succeed or fail with drain timeout -- both OK. t.Logf("demote result: %v", err) case <-timer.C: t.Fatal("demote hung for 5s -- deadlock between WAL-full retry and demote drain") } select { case err := <-writeDone: // Write may fail (demoted, closed, WAL full timeout) -- all acceptable. t.Logf("write result: %v", err) case <-timer.C: t.Fatal("write hung for 5s after demote") } } // ========== Group E: Error Injection ========== // E1: Start rebuild server on primary (epoch=1), client sends epoch=2. // Server should return EPOCH_MISMATCH. func testQA4b4RebuildServerEpochMismatch(t *testing.T) { vol := cp3Primary(t, "rbs_epm.bv", 1) defer vol.Close() vol.StartRebuildServer("127.0.0.1:0") defer vol.StopRebuildServer() // Client sends mismatched epoch=2. conn, err := net.Dial("tcp", vol.rebuildServer.Addr()) if err != nil { t.Fatalf("dial: %v", err) } defer conn.Close() req := EncodeRebuildRequest(RebuildRequest{Type: RebuildWALCatchUp, FromLSN: 1, Epoch: 2}) if err := WriteFrame(conn, MsgRebuildReq, req); err != nil { t.Fatalf("send request: %v", err) } conn.SetReadDeadline(time.Now().Add(3 * time.Second)) msgType, payload, err := ReadFrame(conn) if err != nil { t.Fatalf("read response: %v", err) } if msgType != MsgRebuildError { t.Errorf("msgType = 0x%02x, want MsgRebuildError(0x%02x)", msgType, MsgRebuildError) } if string(payload) != "EPOCH_MISMATCH" { t.Errorf("payload = %q, want EPOCH_MISMATCH", string(payload)) } // Also test epoch=0 (lower than server's epoch=1). conn2, err := net.Dial("tcp", vol.rebuildServer.Addr()) if err != nil { t.Fatalf("dial 2: %v", err) } defer conn2.Close() req2 := EncodeRebuildRequest(RebuildRequest{Type: RebuildFullExtent, Epoch: 0}) if err := WriteFrame(conn2, MsgRebuildReq, req2); err != nil { t.Fatalf("send request 2: %v", err) } conn2.SetReadDeadline(time.Now().Add(3 * time.Second)) msgType2, payload2, err := ReadFrame(conn2) if err != nil { t.Fatalf("read response 2: %v", err) } if msgType2 != MsgRebuildError || string(payload2) != "EPOCH_MISMATCH" { t.Errorf("epoch=0: type=0x%02x payload=%q, want EPOCH_MISMATCH", msgType2, payload2) } }