package blockvol import ( "bytes" "errors" "io" "math" "net" "path/filepath" "sync" "sync/atomic" "testing" "time" ) // TestQAPhase4ACP2 tests Phase 4A CP2 replication primitives adversarially: // frame protocol, WAL shipper, replica receiver, barrier, distributed sync. func TestQAPhase4ACP2(t *testing.T) { tests := []struct { name string run func(t *testing.T) }{ // QA-4A-CP2-1: Frame Protocol Adversarial {name: "frame_max_payload_boundary", run: testQAFrameMaxPayloadBoundary}, {name: "frame_truncated_header", run: testQAFrameTruncatedHeader}, {name: "frame_truncated_payload", run: testQAFrameTruncatedPayload}, {name: "frame_zero_payload", run: testQAFrameZeroPayload}, {name: "frame_concurrent_writes_no_interleave", run: testQAFrameConcurrentWrites}, // QA-4A-CP2-2: WALShipper Adversarial {name: "shipper_ship_after_stop", run: testQAShipperShipAfterStop}, {name: "shipper_barrier_after_degraded", run: testQAShipperBarrierAfterDegraded}, {name: "shipper_stale_epoch_no_shippedlsn", run: testQAShipperStaleEpochNoShippedLSN}, {name: "shipper_degraded_is_permanent", run: testQAShipperDegradedPermanent}, {name: "shipper_concurrent_ship_stop", run: testQAShipperConcurrentShipStop}, // QA-4A-CP2-3: ReplicaReceiver Adversarial {name: "receiver_out_of_order_lsn_skipped", run: testQAReceiverOutOfOrderLSN}, {name: "receiver_concurrent_data_conns", run: testQAReceiverConcurrentDataConns}, {name: "receiver_future_epoch_rejected", run: testQAReceiverFutureEpochRejected}, {name: "receiver_barrier_before_entries_waits", run: testQAReceiverBarrierBeforeEntries}, {name: "receiver_barrier_timeout_no_entries", run: testQAReceiverBarrierTimeoutNoEntries}, {name: "receiver_barrier_epoch_mismatch", run: testQAReceiverBarrierEpochMismatch}, {name: "receiver_stop_unblocks_barrier", run: testQAReceiverStopUnblocksBarrier}, // QA-4A-CP2-4: DistributedSync Adversarial {name: "dsync_local_fail_returns_error", run: testQADSyncLocalFailReturnsError}, {name: "dsync_remote_fail_degrades_not_errors", run: testQADSyncRemoteFailDegrades}, {name: "dsync_both_fail_returns_local", run: testQADSyncBothFail}, {name: "dsync_parallel_execution", run: testQADSyncParallelExecution}, // QA-4A-CP2-5: End-to-end Adversarial {name: "e2e_replica_data_matches_primary", run: testQAE2EReplicaDataMatchesPrimary}, {name: "e2e_close_primary_during_ship", run: testQAE2EClosePrimaryDuringShip}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.run(t) }) } } // --- helpers --- // createReplicaPair creates a fenced primary + replica vol connected via loopback. // Returns primary, replica, and a cleanup function. func createReplicaPair(t *testing.T) (primary, replica *BlockVol) { t.Helper() dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond // Create primary. p, err := CreateBlockVol(filepath.Join(dir, "primary.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 512 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol(primary): %v", err) } if err := p.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } p.SetMasterEpoch(1) if err := p.SetRole(RolePrimary); err != nil { t.Fatalf("SetRole: %v", err) } p.lease.Grant(30 * time.Second) // Create replica. r, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 512 * 1024, }, cfg) if err != nil { p.Close() t.Fatalf("CreateBlockVol(replica): %v", err) } if err := r.SetEpoch(1); err != nil { p.Close() r.Close() t.Fatalf("SetEpoch(replica): %v", err) } // Start replica receiver on ephemeral ports. if err := r.StartReplicaReceiver("127.0.0.1:0", "127.0.0.1:0"); err != nil { p.Close() r.Close() t.Fatalf("StartReplicaReceiver: %v", err) } // Connect primary's shipper to replica's addresses. p.SetReplicaAddr(r.replRecv.DataAddr(), r.replRecv.CtrlAddr()) return p, r } // --- QA-4A-CP2-1: Frame Protocol Adversarial --- func testQAFrameMaxPayloadBoundary(t *testing.T) { // Payload exactly at maxFramePayload -> should succeed. // Payload at maxFramePayload+1 -> ReadFrame must return ErrFrameTooLarge. // Test at boundary: we can't allocate 256MB in a test, but we can test // the ReadFrame parser with a crafted header. var buf bytes.Buffer // Write a frame header claiming payload = maxFramePayload + 1 hdr := make([]byte, frameHeaderSize) hdr[0] = MsgWALEntry bigEndianPut32(hdr[1:5], uint32(maxFramePayload)+1) buf.Write(hdr) _, _, err := ReadFrame(&buf) if !errors.Is(err, ErrFrameTooLarge) { t.Errorf("ReadFrame with oversized payload: got %v, want ErrFrameTooLarge", err) } // Exactly at max: header says maxFramePayload but we won't supply the data. // ReadFrame should try to read and get EOF (not ErrFrameTooLarge). buf.Reset() hdr[0] = MsgWALEntry bigEndianPut32(hdr[1:5], uint32(maxFramePayload)) buf.Write(hdr) // No payload data -> ReadFrame should return io.ErrUnexpectedEOF (not ErrFrameTooLarge). _, _, err = ReadFrame(&buf) if errors.Is(err, ErrFrameTooLarge) { t.Error("payload exactly at maxFramePayload should not be rejected as too large") } // Should be some read error (EOF/UnexpectedEOF) since we didn't write payload bytes. if err == nil { t.Error("ReadFrame with missing payload data should return error") } } func testQAFrameTruncatedHeader(t *testing.T) { // Only 3 bytes (< 5 byte header) -> ReadFrame must return error. buf := bytes.NewReader([]byte{0x01, 0x00, 0x00}) _, _, err := ReadFrame(buf) if err == nil { t.Error("ReadFrame with 3-byte truncated header should return error") } } func testQAFrameTruncatedPayload(t *testing.T) { // Header says 100 bytes payload, but only 10 bytes follow. var buf bytes.Buffer hdr := make([]byte, frameHeaderSize) hdr[0] = MsgWALEntry bigEndianPut32(hdr[1:5], 100) buf.Write(hdr) buf.Write(make([]byte, 10)) // only 10 of 100 bytes _, _, err := ReadFrame(&buf) if err == nil { t.Error("ReadFrame with truncated payload should return error") } } func testQAFrameZeroPayload(t *testing.T) { // Zero-length payload must roundtrip correctly. var buf bytes.Buffer if err := WriteFrame(&buf, 0x42, nil); err != nil { t.Fatalf("WriteFrame(nil payload): %v", err) } if err := WriteFrame(&buf, 0x43, []byte{}); err != nil { t.Fatalf("WriteFrame(empty payload): %v", err) } msgType, payload, err := ReadFrame(&buf) if err != nil { t.Fatalf("ReadFrame(nil payload): %v", err) } if msgType != 0x42 || len(payload) != 0 { t.Errorf("frame 1: type=0x%02x len=%d, want 0x42 len=0", msgType, len(payload)) } msgType, payload, err = ReadFrame(&buf) if err != nil { t.Fatalf("ReadFrame(empty payload): %v", err) } if msgType != 0x43 || len(payload) != 0 { t.Errorf("frame 2: type=0x%02x len=%d, want 0x43 len=0", msgType, len(payload)) } } func testQAFrameConcurrentWrites(t *testing.T) { // Multiple goroutines writing frames to the same connection. // Frames must not interleave (each frame readable as a complete unit). // This relies on the WALShipper's mutex -- test at the connection level. server, client := net.Pipe() defer server.Close() defer client.Close() const writers = 8 const framesPerWriter = 20 var mu sync.Mutex // simulate WALShipper's mutex var wg sync.WaitGroup wg.Add(writers) for g := 0; g < writers; g++ { go func(id int) { defer wg.Done() payload := []byte{byte(id)} for i := 0; i < framesPerWriter; i++ { mu.Lock() _ = WriteFrame(client, MsgWALEntry, payload) mu.Unlock() } }(g) } // Reader: read all frames and verify integrity. totalFrames := writers * framesPerWriter readDone := make(chan error, 1) var frameCount int go func() { for i := 0; i < totalFrames; i++ { msgType, payload, err := ReadFrame(server) if err != nil { readDone <- err return } if msgType != MsgWALEntry { readDone <- errors.New("wrong message type") return } if len(payload) != 1 { readDone <- errors.New("wrong payload length") return } if payload[0] >= byte(writers) { readDone <- errors.New("invalid writer ID in payload") return } frameCount++ } readDone <- nil }() wg.Wait() select { case err := <-readDone: if err != nil { t.Fatalf("frame read error: %v (after %d frames)", err, frameCount) } case <-time.After(5 * time.Second): t.Fatal("frame reader hung for 5s") } if frameCount != totalFrames { t.Errorf("read %d frames, want %d", frameCount, totalFrames) } } // --- QA-4A-CP2-2: WALShipper Adversarial --- func testQAShipperShipAfterStop(t *testing.T) { s := NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 { return 1 }, nil) s.Stop() // Ship after Stop must not panic, must return nil (silently drop). entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('S')} err := s.Ship(entry) if err != nil { t.Errorf("Ship after Stop: got %v, want nil", err) } if s.ShippedLSN() != 0 { t.Errorf("ShippedLSN after stopped Ship: got %d, want 0", s.ShippedLSN()) } } func testQAShipperBarrierAfterDegraded(t *testing.T) { s := NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 { return 1 }, nil) s.state.Store(uint32(ReplicaDegraded)) err := s.Barrier(1) if !errors.Is(err, ErrReplicaDegraded) { t.Errorf("Barrier when degraded: got %v, want ErrReplicaDegraded", err) } } func testQAShipperStaleEpochNoShippedLSN(t *testing.T) { // Ship with epoch != current -> entry silently dropped, shippedLSN unchanged. currentEpoch := uint64(5) s := NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 { return currentEpoch }, nil) entry := &WALEntry{LSN: 10, Epoch: 3, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('X')} err := s.Ship(entry) if err != nil { t.Errorf("Ship with stale epoch: got %v, want nil", err) } if s.ShippedLSN() != 0 { t.Errorf("ShippedLSN after stale epoch Ship: got %d, want 0 (entry should be dropped)", s.ShippedLSN()) } } func testQAShipperDegradedPermanent(t *testing.T) { // After degradation, Ship and Barrier must fail immediately forever. // Create a listener that accepts but immediately closes (triggers write error). ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } defer ln.Close() go func() { for { conn, err := ln.Accept() if err != nil { return } conn.Close() // immediately close -> Ship will get write error } }() s := NewWALShipper(ln.Addr().String(), ln.Addr().String(), func() uint64 { return 1 }, nil) defer s.Stop() // Ship triggers connection -> immediate close -> write error -> degraded. entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('D')} _ = s.Ship(entry) // first attempt may or may not degrade _ = s.Ship(entry) // second attempt more likely to hit closed conn _ = s.Ship(entry) // third attempt for good measure // Force degraded if the connection race didn't trigger it. if !s.IsDegraded() { // The connection might have worked briefly. Manually degrade for the rest of the test. s.state.Store(uint32(ReplicaDegraded)) } // Once degraded, subsequent Ship must not attempt connection. lsnBefore := s.ShippedLSN() for i := 0; i < 100; i++ { _ = s.Ship(entry) } lsnAfter := s.ShippedLSN() if lsnAfter > lsnBefore { t.Errorf("ShippedLSN advanced from %d to %d after degradation -- should not ship when degraded", lsnBefore, lsnAfter) } // Barrier must immediately return ErrReplicaDegraded. err = s.Barrier(1) if !errors.Is(err, ErrReplicaDegraded) { t.Errorf("Barrier after degraded: got %v, want ErrReplicaDegraded", err) } } func testQAShipperConcurrentShipStop(t *testing.T) { // Ship and Stop from concurrent goroutines -> no deadlock, no panic. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } go func() { for { conn, err := ln.Accept() if err != nil { return } // Just consume data. go io.Copy(io.Discard, conn) } }() defer ln.Close() s := NewWALShipper(ln.Addr().String(), ln.Addr().String(), func() uint64 { return 1 }, nil) var wg sync.WaitGroup // Concurrent shippers. wg.Add(4) for g := 0; g < 4; g++ { go func() { defer wg.Done() entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('C')} for i := 0; i < 50; i++ { _ = s.Ship(entry) } }() } // Stop mid-flight. time.Sleep(5 * time.Millisecond) s.Stop() done := make(chan struct{}) go func() { wg.Wait() close(done) }() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("concurrent Ship + Stop deadlocked for 5s") } } // --- QA-4A-CP2-3: ReplicaReceiver Adversarial --- func testQAReceiverOutOfOrderLSN(t *testing.T) { // Tests contiguity enforcement: // 1. Contiguous LSN=1..5 -> all applied // 2. Duplicate LSN=3 -> skipped // 3. Gap LSN=7 (skipping 6) -> rejected // 4. Correct next LSN=6 -> accepted dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() defer recv.Stop() conn, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial data: %v", err) } defer conn.Close() // Case 1: Contiguous LSN=1..5 -> all applied. for lsn := uint64(1); lsn <= 5; lsn++ { entry := &WALEntry{LSN: lsn, Epoch: 1, Type: EntryTypeWrite, LBA: lsn - 1, Length: 4096, Data: makeBlock(byte('A' + lsn))} encoded, _ := entry.Encode() if err := WriteFrame(conn, MsgWALEntry, encoded); err != nil { t.Fatalf("send LSN=%d: %v", lsn, err) } } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 5 { t.Fatalf("ReceivedLSN = %d after LSN 1-5, want 5", recv.ReceivedLSN()) } // Case 2: Duplicate LSN=3 -> skipped. entry3 := &WALEntry{LSN: 3, Epoch: 1, Type: EntryTypeWrite, LBA: 2, Length: 4096, Data: makeBlock('Z')} encoded3, _ := entry3.Encode() if err := WriteFrame(conn, MsgWALEntry, encoded3); err != nil { t.Fatalf("send duplicate LSN=3: %v", err) } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 5 { t.Errorf("ReceivedLSN = %d after duplicate LSN=3, want 5", recv.ReceivedLSN()) } // Case 3: Gap LSN=7 (skips 6) -> rejected by contiguity check. entry7 := &WALEntry{LSN: 7, Epoch: 1, Type: EntryTypeWrite, LBA: 6, Length: 4096, Data: makeBlock('G')} encoded7, _ := entry7.Encode() if err := WriteFrame(conn, MsgWALEntry, encoded7); err != nil { t.Fatalf("send gap LSN=7: %v", err) } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 5 { t.Errorf("ReceivedLSN = %d after gap LSN=7, want 5 (gap must be rejected)", recv.ReceivedLSN()) } // Case 4: Correct next LSN=6 -> accepted. entry6 := &WALEntry{LSN: 6, Epoch: 1, Type: EntryTypeWrite, LBA: 5, Length: 4096, Data: makeBlock('F')} encoded6, _ := entry6.Encode() if err := WriteFrame(conn, MsgWALEntry, encoded6); err != nil { t.Fatalf("send LSN=6: %v", err) } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 6 { t.Errorf("ReceivedLSN = %d after correct LSN=6, want 6", recv.ReceivedLSN()) } } func testQAReceiverConcurrentDataConns(t *testing.T) { // With contiguous LSN enforcement, entries must arrive in order on a single // connection. Test that a stream of 40 contiguous entries from one connection // is fully applied, then a second connection sending a duplicate is rejected. dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 512 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() defer recv.Stop() // Send 40 contiguous entries from a single connection. conn1, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial: %v", err) } defer conn1.Close() for lsn := uint64(1); lsn <= 40; lsn++ { entry := &WALEntry{LSN: lsn, Epoch: 1, Type: EntryTypeWrite, LBA: lsn, Length: 4096, Data: makeBlock(byte('A' + lsn%26))} encoded, _ := entry.Encode() if err := WriteFrame(conn1, MsgWALEntry, encoded); err != nil { t.Fatalf("send LSN=%d: %v", lsn, err) } } time.Sleep(100 * time.Millisecond) got := recv.ReceivedLSN() if got != 40 { t.Errorf("ReceivedLSN = %d after 40 contiguous entries, want 40", got) } // Second connection sending a duplicate LSN=5 -> must be skipped. conn2, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial conn2: %v", err) } defer conn2.Close() dup := &WALEntry{LSN: 5, Epoch: 1, Type: EntryTypeWrite, LBA: 5, Length: 4096, Data: makeBlock('Z')} encodedDup, _ := dup.Encode() WriteFrame(conn2, MsgWALEntry, encodedDup) time.Sleep(20 * time.Millisecond) // receivedLSN must still be 40. if recv.ReceivedLSN() != 40 { t.Errorf("ReceivedLSN = %d after duplicate on conn2, want 40", recv.ReceivedLSN()) } } func testQAReceiverFutureEpochRejected(t *testing.T) { // R1 fix: replica must reject entries with epoch > local (not just <). // A future epoch in the WAL stream is a protocol violation -- only master // can bump epochs via SetEpoch. dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(5); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() defer recv.Stop() conn, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial data: %v", err) } defer conn.Close() // Send entry with stale epoch=3 -> rejected. stale := &WALEntry{LSN: 1, Epoch: 3, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('S')} encodedStale, _ := stale.Encode() if err := WriteFrame(conn, MsgWALEntry, encodedStale); err != nil { t.Fatalf("send stale epoch: %v", err) } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 0 { t.Errorf("ReceivedLSN = %d after stale epoch entry, want 0", recv.ReceivedLSN()) } // Send entry with future epoch=10 -> also rejected (R1 fix). future := &WALEntry{LSN: 1, Epoch: 10, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('F')} encodedFuture, _ := future.Encode() if err := WriteFrame(conn, MsgWALEntry, encodedFuture); err != nil { t.Fatalf("send future epoch: %v", err) } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 0 { t.Errorf("ReceivedLSN = %d after future epoch entry, want 0 (future epoch must be rejected)", recv.ReceivedLSN()) } // Send entry with correct epoch=5 -> accepted. correct := &WALEntry{LSN: 1, Epoch: 5, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('C')} encodedCorrect, _ := correct.Encode() if err := WriteFrame(conn, MsgWALEntry, encodedCorrect); err != nil { t.Fatalf("send correct epoch: %v", err) } time.Sleep(20 * time.Millisecond) if recv.ReceivedLSN() != 1 { t.Errorf("ReceivedLSN = %d after correct epoch entry, want 1", recv.ReceivedLSN()) } } func testQAReceiverBarrierEpochMismatch(t *testing.T) { // Barrier with wrong epoch -> immediate BarrierEpochMismatch (no wait). dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 100 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(5); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() defer recv.Stop() ctrlConn, err := net.Dial("tcp", recv.CtrlAddr()) if err != nil { t.Fatalf("dial ctrl: %v", err) } defer ctrlConn.Close() // Barrier with stale epoch=3 -> immediate EpochMismatch. req := EncodeBarrierRequest(BarrierRequest{LSN: 1, Epoch: 3}) start := time.Now() if err := WriteFrame(ctrlConn, MsgBarrierReq, req); err != nil { t.Fatalf("send barrier: %v", err) } ctrlConn.SetReadDeadline(time.Now().Add(3 * time.Second)) _, payload, err := ReadFrame(ctrlConn) elapsed := time.Since(start) if err != nil { t.Fatalf("read barrier response: %v", err) } if payload[0] != BarrierEpochMismatch { t.Errorf("barrier status = %d, want BarrierEpochMismatch(%d)", payload[0], BarrierEpochMismatch) } // Must be fast (no waiting), well under 100ms. if elapsed > 500*time.Millisecond { t.Errorf("epoch mismatch barrier took %v, expected immediate response", elapsed) } // Barrier with future epoch=99 -> also immediate EpochMismatch. req2 := EncodeBarrierRequest(BarrierRequest{LSN: 1, Epoch: 99}) if err := WriteFrame(ctrlConn, MsgBarrierReq, req2); err != nil { t.Fatalf("send barrier future: %v", err) } ctrlConn.SetReadDeadline(time.Now().Add(3 * time.Second)) _, payload2, err := ReadFrame(ctrlConn) if err != nil { t.Fatalf("read barrier response (future): %v", err) } if payload2[0] != BarrierEpochMismatch { t.Errorf("future epoch barrier status = %d, want BarrierEpochMismatch(%d)", payload2[0], BarrierEpochMismatch) } } func testQAReceiverBarrierBeforeEntries(t *testing.T) { // Barrier arrives BEFORE data entries -> must wait, then succeed when entries arrive. dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 5 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() defer recv.Stop() // Send barrier for LSN=3 BEFORE any data. ctrlConn, err := net.Dial("tcp", recv.CtrlAddr()) if err != nil { t.Fatalf("dial ctrl: %v", err) } defer ctrlConn.Close() barrierDone := make(chan BarrierResponse, 1) go func() { req := EncodeBarrierRequest(BarrierRequest{LSN: 3, Epoch: 1}) if err := WriteFrame(ctrlConn, MsgBarrierReq, req); err != nil { return } _, payload, err := ReadFrame(ctrlConn) if err != nil { return } barrierDone <- BarrierResponse{Status: payload[0]} }() // Barrier should be waiting (not returned yet). select { case resp := <-barrierDone: t.Fatalf("barrier returned immediately with status %d, expected it to wait", resp.Status) case <-time.After(50 * time.Millisecond): // Good -- barrier is waiting. } // Now send entries LSN=1,2,3. dataConn, err := net.Dial("tcp", recv.DataAddr()) if err != nil { t.Fatalf("dial data: %v", err) } defer dataConn.Close() for lsn := uint64(1); lsn <= 3; lsn++ { entry := &WALEntry{LSN: lsn, Epoch: 1, Type: EntryTypeWrite, LBA: lsn, Length: 4096, Data: makeBlock(byte('0' + lsn))} encoded, _ := entry.Encode() if err := WriteFrame(dataConn, MsgWALEntry, encoded); err != nil { t.Fatalf("send LSN=%d: %v", lsn, err) } } // Barrier should now complete with OK. select { case resp := <-barrierDone: if resp.Status != BarrierOK { t.Errorf("barrier status = %d, want BarrierOK(0)", resp.Status) } case <-time.After(5 * time.Second): t.Fatal("barrier hung for 5s after entries were sent") } } func testQAReceiverBarrierTimeoutNoEntries(t *testing.T) { // Barrier for LSN=999 with no entries -> must timeout (not hang forever). // Uses short configurable barrierTimeout for fast test. dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 100 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.barrierTimeout = 100 * time.Millisecond // fast timeout for test recv.Serve() defer recv.Stop() ctrlConn, err := net.Dial("tcp", recv.CtrlAddr()) if err != nil { t.Fatalf("dial ctrl: %v", err) } defer ctrlConn.Close() req := EncodeBarrierRequest(BarrierRequest{LSN: 999, Epoch: 1}) start := time.Now() if err := WriteFrame(ctrlConn, MsgBarrierReq, req); err != nil { t.Fatalf("send barrier: %v", err) } ctrlConn.SetReadDeadline(time.Now().Add(3 * time.Second)) _, payload, err := ReadFrame(ctrlConn) elapsed := time.Since(start) if err != nil { t.Fatalf("read barrier response: %v", err) } if payload[0] != BarrierTimeout { t.Errorf("barrier status = %d, want BarrierTimeout(%d)", payload[0], BarrierTimeout) } // Verify timeout was fast (~100ms, not 5s). if elapsed > 1*time.Second { t.Errorf("barrier timeout took %v, expected ~100ms (configurable timeout)", elapsed) } } func testQAReceiverStopUnblocksBarrier(t *testing.T) { // Barrier waiting for entries -> Stop called -> barrier must return (not hang). dir := t.TempDir() cfg := DefaultConfig() cfg.FlushInterval = 100 * time.Millisecond vol, err := CreateBlockVol(filepath.Join(dir, "replica.blockvol"), CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }, cfg) if err != nil { t.Fatalf("CreateBlockVol: %v", err) } defer vol.Close() if err := vol.SetEpoch(1); err != nil { t.Fatalf("SetEpoch: %v", err) } recv, err := NewReplicaReceiver(vol, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatalf("NewReplicaReceiver: %v", err) } recv.Serve() ctrlConn, err := net.Dial("tcp", recv.CtrlAddr()) if err != nil { t.Fatalf("dial ctrl: %v", err) } defer ctrlConn.Close() // Send barrier for high LSN (will wait). req := EncodeBarrierRequest(BarrierRequest{LSN: math.MaxUint64, Epoch: 1}) if err := WriteFrame(ctrlConn, MsgBarrierReq, req); err != nil { t.Fatalf("send barrier: %v", err) } // Let barrier start waiting. time.Sleep(50 * time.Millisecond) // Stop should unblock the barrier. recv.Stop() ctrlConn.SetReadDeadline(time.Now().Add(3 * time.Second)) _, payload, err := ReadFrame(ctrlConn) if err != nil { // Connection closed by Stop -> also acceptable. t.Logf("barrier read after Stop: %v (connection closed -- acceptable)", err) return } if payload[0] != BarrierTimeout { t.Errorf("barrier after Stop: status = %d, want BarrierTimeout(%d)", payload[0], BarrierTimeout) } } // --- QA-4A-CP2-4: DistributedSync Adversarial --- func testQADSyncLocalFailReturnsError(t *testing.T) { // Local fsync fails -> must return error regardless of remote result. localErr := errors.New("disk on fire") vol := &BlockVol{} vol.nextLSN.Store(10) syncFn := MakeDistributedSync( func() error { return localErr }, nil, // no shipper -> local only vol, ) err := syncFn() if !errors.Is(err, localErr) { t.Errorf("dsync with local fail: got %v, want %v", err, localErr) } } func testQADSyncRemoteFailDegrades(t *testing.T) { // Remote barrier fails -> local succeeded -> must return nil + degrade shipper. primary, replica := createReplicaPair(t) defer primary.Close() defer replica.Close() // Write some data so there's something to sync. if err := primary.WriteLBA(0, makeBlock('D')); err != nil { t.Fatalf("WriteLBA: %v", err) } // Stop the replica receiver to make barrier fail. replica.replRecv.Stop() // Force shipper to reconnect (old ctrl conn may still be open). primary.shipperGroup.Shipper(0).ctrlMu.Lock() if primary.shipperGroup.Shipper(0).ctrlConn != nil { primary.shipperGroup.Shipper(0).ctrlConn.Close() primary.shipperGroup.Shipper(0).ctrlConn = nil } primary.shipperGroup.Shipper(0).ctrlMu.Unlock() // SyncCache triggers distributed sync -> barrier fails -> degrade. err := primary.SyncCache() // Should succeed (local fsync succeeded, remote degraded -- returned nil). if err != nil { t.Errorf("SyncCache after replica stop: got %v, want nil (local succeeded, remote should degrade silently)", err) } // Shipper should be degraded. if !primary.shipperGroup.Shipper(0).IsDegraded() { t.Error("shipper should be degraded after replica receiver stopped") } } func testQADSyncBothFail(t *testing.T) { // Both local and remote fail -> must return local error. localErr := errors.New("local disk fail") vol := &BlockVol{} vol.nextLSN.Store(10) shipper := NewWALShipper("127.0.0.1:1", "127.0.0.1:1", func() uint64 { return 1 }, nil) syncFn := MakeDistributedSync( func() error { return localErr }, NewShipperGroup([]*WALShipper{shipper}), vol, ) err := syncFn() if !errors.Is(err, localErr) { t.Errorf("dsync with both fail: got %v, want %v", err, localErr) } } func testQADSyncParallelExecution(t *testing.T) { // Verify local and remote execute in parallel, not sequentially. // If sequential, total time ≈ 200ms. If parallel, ≈ 100ms. // // Strategy: create a real replica pair where barrier takes ~80ms // (because data arrives with delay) and local fsync also takes ~80ms. // If parallel: ~80ms total. If sequential: ~160ms. primary, replica := createReplicaPair(t) defer primary.Close() defer replica.Close() // Write some data to primary so there's a WAL entry to sync. if err := primary.WriteLBA(0, makeBlock('P')); err != nil { t.Fatalf("WriteLBA: %v", err) } // Give replica time to receive the shipped entry. time.Sleep(30 * time.Millisecond) // Replace the distributed sync with one that has a slow local fsync. // The barrier should be near-instant since replica already has the data. var localStart, localEnd atomic.Int64 slowLocalSync := func() error { localStart.Store(time.Now().UnixNano()) time.Sleep(80 * time.Millisecond) localEnd.Store(time.Now().UnixNano()) return nil } syncFn := MakeDistributedSync(slowLocalSync, primary.shipperGroup, primary) start := time.Now() err := syncFn() elapsed := time.Since(start) if err != nil { t.Fatalf("dsync: %v", err) } // Parallel: total ≈ max(local=80ms, barrier=fast) ≈ 80ms // Sequential: total ≈ local + barrier ≈ 80ms + barrier_overhead // Key check: elapsed should be < 150ms (well under sequential 160ms+) if elapsed > 150*time.Millisecond { t.Errorf("dsync took %v, suggesting sequential execution (expected <150ms for parallel)", elapsed) } // Verify local fsync actually ran (not skipped). if localStart.Load() == 0 { t.Error("local fsync was never called -- distributed sync may have fallen back to local-only") } } // --- QA-4A-CP2-5: End-to-end Adversarial --- func testQAE2EReplicaDataMatchesPrimary(t *testing.T) { // Write N blocks on primary -> verify replica has identical data via ReadLBA. primary, replica := createReplicaPair(t) defer primary.Close() defer replica.Close() const numBlocks = 20 for i := 0; i < numBlocks; i++ { if err := primary.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))); err != nil { t.Fatalf("WriteLBA(%d): %v", i, err) } } // SyncCache ensures WAL entries are durable on both nodes. if err := primary.SyncCache(); err != nil { t.Fatalf("SyncCache: %v", err) } // Give replica time to process entries. time.Sleep(50 * time.Millisecond) // Verify replica has the data. for i := 0; i < numBlocks; i++ { data, err := replica.ReadLBA(uint64(i), 4096) if err != nil { t.Fatalf("replica ReadLBA(%d): %v", i, err) } expected := byte('A' + i%26) if data[0] != expected { t.Errorf("replica LBA %d: data[0] = %c, want %c", i, data[0], expected) } } // Verify receivedLSN on replica matches what primary shipped. primaryLSN := primary.shipperGroup.Shipper(0).ShippedLSN() replicaLSN := replica.replRecv.ReceivedLSN() if replicaLSN < primaryLSN { t.Errorf("replica receivedLSN = %d < primary shippedLSN = %d", replicaLSN, primaryLSN) } } func testQAE2EClosePrimaryDuringShip(t *testing.T) { // Close primary while writes + shipping in progress -> no hang, no panic. primary, replica := createReplicaPair(t) defer replica.Close() var wg sync.WaitGroup // Writer goroutine. wg.Add(1) go func() { defer wg.Done() for i := 0; i < 100; i++ { err := primary.WriteLBA(uint64(i%256), makeBlock(byte('W'))) if err != nil { return // closed or WAL full -- expected } } }() // Close after a few writes. time.Sleep(10 * time.Millisecond) primary.Close() done := make(chan struct{}) go func() { wg.Wait() close(done) }() select { case <-done: case <-time.After(10 * time.Second): t.Fatal("Close primary during ship hung for 10s") } } // --- helper --- func bigEndianPut32(b []byte, v uint32) { b[0] = byte(v >> 24) b[1] = byte(v >> 16) b[2] = byte(v >> 8) b[3] = byte(v) }